Practical, Provably-Correct Interactive Learning in the Realizable Setting: The Power of True Believers
Extracted source text · arXiv 2111.04915
\documentclass{article}
% if you need to pass options to natbib, use, e.g.:
\PassOptionsToPackage{numbers}{natbib}
% before loading neurips_2021
% ready for submission
%\usepackage{neurips_2021}
% to compile a preprint version, e.g., for submission to arXiv, add add the
% [preprint] option:
% \usepackage[preprint]{neurips_2021}
% to compile a camera-ready version, add the [final] option, e.g.:
\usepackage[final]{neurips_2021}
% to avoid loading the natbib package, add option nonatbib:
% \usepackage[nonatbib]{neurips_2021}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{hyperref} % hyperlinks
\usepackage{url} % simple URL typesetting
\usepackage{booktabs} % professional-quality tables
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{xcolor} % colors
%\usepackage[nonatbib]{neurips_2021}
\usepackage{pifont}
\usepackage{wrapfig}
\usepackage{float}
\newcommand{\cmark}{\text{\ding{51}}}
\newcommand{\xmark}{\text{\ding{55}}}
%\usepackage{algorithm2e}
\usepackage{enumitem}
\usepackage{filecontents}
\usepackage{todonotes}
%\usepackage{graphicx}
\usepackage{subcaption}
%\usepackage{algorithm}
%\usepackage[noend]{algorithmic}
\usepackage{mathtools, amsmath,amssymb, amsthm}
\usepackage[noline,boxed, noend]{algorithm2e}
\usepackage{bbm}
%\usepackage{amsmath,amssymb,amsthm,amsfonts,bm,url,dsfont,amsthm,hyperref,color}
\usepackage{amsfonts,bm,dsfont,color}
\usepackage{mathtools}
\newtheorem{theorem}{Theorem}
\newtheorem{proposition}{Proposition}
\newtheorem{claim}{Claim}
\newtheorem{lemma}{Lemma}
\newtheorem{definition}{Definition}
\newtheorem{remark}{Remark}
\newtheorem{assumption}{Assumption}
\newtheorem{corollary}{Corollary}
\newtheorem{example}{Example}
\newtheorem{conjecture}{Conjecture}
\def\calD{\mathcal{D}}
\def\calE{\mathcal{E}}
\def\H{\mathcal{H}}
\def\calS{\mathcal{S}}
\def\M{\mathcal{M}}
\def\calR{\mathcal{R}}
\def\calP{\mathcal{P}}
\def\calI{\mathcal{I}}
\def\I{\mathcal{I}}
\def\Q{\mathcal{Q}}
\def\G{\mathcal{G}}
\def\S{\mathbb{S}}
\def\A{\mathcal{A}}
\def\X{\mathcal{X}}
\def\T{\mathcal{T}}
\def\Z{\mathcal{Z}}
\def\Y{\mathcal{Y}}
\def\B{\mathcal{B}}
\def\C{\mathcal{C}}
\def\U{\mathcal{U}}
\def\L{\mathcal{L}}
\def\calN{\mathcal{N}}
\def\E{\mathbb{E}}
\def\V{\mathbb{V}}
\def\Z{\mathcal{Z}}
\def\1{\mathbf{1}}
\def\P{\mathbb{P}}
\def\R{\mathbb{R}}
\def\N{\mathbb{N}}
\def\mualpha{\mu^{(\alpha)}}
\def\si{\mathbb{S}}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\mb}[1]{\mathbf{#1}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\diam}{diam}
\DeclareMathOperator*{\conv}{conv}
\DeclareMathOperator*{\unif}{uniform}
\DeclareMathOperator*{\dist}{dist}
\DeclareMathOperator*{\deff}{d_{\text{eff}}}
\DeclareMathOperator*{\simp}{\mathbf{\Delta}}
\def\t{\top}
\def\one{\mathbbm{1}}
\def\zero{\bm{0}}
\def\argmin{\text{argmin}}
\def\max{\text{max}}
\def\diag{\text{diag}}
\def\sp{\text{span}}
\def\aff{\text{aff}}
\def\supp{\text{support}}
%macros from fixed budget doc
\def\half{ \frac{1}{2}}
\newcommand{\F}{\mathcal{F}}
\newcommand{\SSS}{\mathbb{S}}
\newcommand{\Spsk}{\Sigma\Pi\Sigma(k)}
\newcommand{\sps}{\Sigma\Pi\Sigma}
\newcommand{\Sps}{\Sigma\Pi\Sigma}
\newcommand{\Spskdr}{\Sigma\Pi\Sigma(k,d,\rho)}
\newcommand{\eqdef}{\stackrel{\Delta}{=}}
%\newcommand{\simp}{\mathrm{sim}}
\newcommand{\im}{\mathrm{Im}}
\newcommand{\rank}{\mathrm{rank}}
\newcommand{\sign}{\mathrm{sign}}
\newcommand{\lin}{\mathrm{Lin}}
\newcommand{\size}{\mathrm{size}}
\newcommand{\Spskd}{\Sigma \Pi \Sigma (k,d)}
\newcommand{\ignore}[1]{}
\newcommand{\floor}[1]{\left\lfloor #1 \right\rfloor}
\newcommand{\ceil}[1]{\left\lceil #1 \right\rceil}
\newcommand{\spa}{\mathrm{span_1}}
\newcommand{\dimone}{\mathrm{dim_1}}
\newcommand{\spareg}{\mathrm{span}}
\newcommand{\regdim}{\mathrm{dim}}
\newcommand{\polylog}{\mathrm{polylog}}
\newcommand{\poly}{\mathrm{poly}}
\newcommand{\eps}{\epsilon}
\newcommand{\ip}[1]{\left\langle #1 \right\rangle}
\newcommand{\abs}[1]{\left| #1 \right|}
\newcommand{\anote}[1]{\footnote{{\bf Amir:} #1}}
\newcommand{\vabs}[1]{\left\| #1 \right\|}
\newcommand{\ve}[1]{\mathbf{vec}( #1 )}
\newcommand{\zk}[1]{{\color{blue}ZK:#1}}
\newcommand{\kevin}[1]{{\color{magenta}KJ:#1}}
\newcommand{\blake}[1]{{\color{red}BM:#1}}
\newcommand{\jks}[1]{{\color{purple}JKS:#1}}
%paper symbols
\DeclareMathOperator*{\simpm}{\mathbf{\tilde{\Delta}}}
\DeclareMathOperator*{\width}{\text{Width}}
\DeclareMathOperator*{\kl}{\mathbf{KL}}
\DeclareMathOperator*{\Fceil}{\mathcal{F}^{\text{ceil}}_x}
%\DeclareMathOperator*{\round}{\calR}
\newcommand{\round}[1]{\calD [ #1 ]}
\def\reg{\calR}
\def\ver{\calR}
\DeclareMathOperator*{\dis}{\chi}
\newcommand{\Smin}{S_{\text{min}}}
\newcommand{\Sminb}{\bar{S}_{\text{min}}}
\def\Delmax{\Delta_{\max}}
\def\Delmin{\Delta_{\min}}
\def\hd{\textrm{HD}}
\def\margin{\gamma^*}
\renewcommand{\cite}{\citep}
\def\bestopt{\Lambda_{\textit{best}}}
\def\bestoptep{\Lambda_{\textit{best},\epsilon}}
\def\classopt{\Lambda_{\textit{class}}}
\def\lossopt{\Lambda_{\textit{loss}}}
\def\bestext{\upsilon^*_{\textit{best}}}
\def\bestextep{\upsilon^*_{\textit{best},\epsilon}}
\def\lossext{\upsilon^*_{\textit{loss}}}
\def\classext{\upsilon^*_{\textit{class}}}
\title{Practical, Provably-Correct Interactive Learning in the Realizable Setting: The Power of True Believers}
% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to LaTeX to determine where to break the
% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
% authors names on the first line, and the last on the second line, try using
% \AND instead of \And before the third author name.
\author{%
Julian Katz-Samuels \\
University of Wisconsin, Madison\\
\texttt{katzsamuels@wisc.edu} \\
% examples of more authors
\And
Blake Mason \\
Rice University \\
\texttt{bm63@rice.edu} \\
\And
Kevin Jamieson \\
University of Washington \\
\texttt{jamieson@cs.washington.edu} \\
\AND
Robert Nowak \\
University of Wisconsin, Madison \\
\texttt{rdnowak@wisc.edu} \\
% \And
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email} \\
}
\begin{document}
\maketitle
\begin{abstract}
% We consider interactive learning in the realizable setting with a focus on best arm identification, cumulative loss minimization, and active classification. The motivation of our work is the observation that agnostic algorithms, a popular algorithmic paradigm, are minimax-\emph{suboptimal} in the realizable setting. In light of this observation, we design new algorithms that are nearly minimax optimal, computationally efficient, and flexible accommodating a wide range of function classes such as kernel methods and convex functions. Our algorithms achieve computational efficiency by sampling from the version space using MCMC algorithms like hit-and-run instead of maintaining it explicitly, an algorithmic technique that may be of use more broadly in interactive learning. A key strength of our approach is its ability to seamlessly incorporate additional prior knowledge to guide learning. Through experiments, we demonstrate that our algorithms are competitive with GP-UCB.
% We consider interactive learning in the realizable setting with a focus on best arm identification, cumulative loss minimization, and active classification.
% We make the surprising observation that agnostic algorithms \emph{cannot} be minimax-optimal in the realizable setting. Hence, we design novel algorithms that are nearly minimax optimal, computationally efficient, and general-purpose accommodating a wide range of function classes such as kernel methods and convex functions. Our algorithms achieve computational efficiency by sampling from the version space using MCMC algorithms like hit-and-run instead of maintaining it explicitly.
% %This algorithmic technique may be of use more broadly in interactive learning.
% A key strength of our approach is its ability to seamlessly leverage prior knowledge to guide learning as is commonly useful in practice. In addition to new theory, we demonstrate empirically that our algorithms are competitive with GP-UCB.
We consider interactive learning in the realizable setting and develop a general framework to handle problems ranging from best arm identification to active classification.
We begin our investigation with the observation that agnostic algorithms \emph{cannot} be minimax-optimal in the realizable setting.
Hence, we design novel computationally efficient algorithms for the realizable setting that match the minimax lower bound up to logarithmic factors and are general-purpose, accommodating a wide variety of function classes including kernel methods, H{\"o}lder smooth functions, and convex functions. The sample complexities of our algorithms can be quantified in terms of well-known quantities like the extended teaching dimension and haystack dimension. However, unlike algorithms based directly on those combinatorial quantities, our algorithms are computationally efficient.
To achieve computational efficiency, our algorithms sample from the version space using Monte Carlo ``hit-and-run'' algorithms instead of maintaining the version space explicitly.
Our approach has two key strengths. First, it is simple, consisting of two unifying, greedy algorithms. Second, our algorithms have the capability to seamlessly leverage prior knowledge that is often available and useful in practice. In addition to our new theoretical results, we demonstrate empirically that our algorithms are competitive with Gaussian process UCB methods.
% We consider interactive learning in the realizable setting with a focus on best arm identification, cumulative loss minimization, and active classification.
% We make the surprising observation that agnostic algorithms \emph{cannot} be minimax-optimal in the realizable setting. Hence, we design novel algorithms for this setting that are nearly minimax optimal, computationally efficient, and general-purpose accommodating a wide range of function classes such as kernel methods and convex functions. Our algorithms achieve computational efficiency by sampling from the version space using MCMC algorithms like hit-and-run instead of maintaining it explicitly.
% Our approach has two key strengths. First, it is simple, consisting of two unifying, greedy algorithms. Second, our algorithms have the capability to seamlessly leverage prior knowledge that is often available and useful in practice. In addition to our new theoretical results, we demonstrate empirically that our algorithms are competitive with GP-UCB.
% \textbf{Blake question: can we work in the fact that this is one alg to do everything not a whole bunch of algs? I think that that is a strength}
% \textbf{Blake comment: maybe we should get rid of the comment about MCMC being of interest to other algs if we need the space? Commenting on the generality of the function may be better}
% We consider interactive learning problems with general function classes, including best arm identification and cumulative loss minimization. We design new realizability-based algorithms that are nearly minimax optimal, computationally efficient and flexible accommodating function classes such as kernel methods and convex functions. Our methods have the capability of easily incorporating prior information about the problem structure that can greatly accelerate learning.
%We complement our algorithmic contributions by showing that when the realizability assumption holds, realizability-based algorithms are necessarily better than agnostic algorithms, which are designed to be robust to model mispecification. \jks{I don't like the last sentence}
\end{abstract}
%\section{TODO}
%\begin{itemize}
% \item make extended teaching dimension lower bounds apply to randomized algorithms (not just deterministic algorithms)
% \begin{itemize}
% \item Note that the extended teaching dimension is no more than a $\log(|\F|) $ larger than the haystack dimension, so maybe this is not high priority.
% \end{itemize}
% \item result showing that agnostic algorithms are bad for regret/loss minimization
% \item make sure examples of sample complexities are strong enough?? \jks{high priority}
% \item example for loss minimization setting \jks{high priority}
% \item a sampling oracle for locally adaptive regression splines?
% \item experiment comparing to GP-UCB (do we want anything else?) \jks{high priority}
% \item Give extensions to other problems like finding an arm below a threshold, $\epsilon$-good arm identification
% \item Should I actually add the real sampling to the algorithms and proofs? \jks{low priority}
% \item mention somewhere that results can be extended to many kinds of feedback models (e.g., semibandit, graph bandit).
%\end{itemize}
\section{Introduction}
% In this paper, we study interactive learning where an agent makes a decision and observes feedback that she then uses to update her behavior. Interactive learning problems are becoming increasingly widespread in the information era.
In this paper, we study interactive learning where an algorithm makes a decision and observes feedback that it then uses to update its behavior. Interactive learning problems are becoming increasingly widespread in the information era. Examples include A/B/n testing where technology companies perform large-scale experiments to adaptively collect data to optimize their products on platforms like websites or smart phone applications \cite{jamieson2018bandit}; active classification where learning algorithms adaptively collect data with the hope of learning high-quality predictive models using a much smaller number of labels than is typically required in supervised learning \cite{settles2009active}; and environmental monitoring using sensor networks \cite{shahriari2015taking}.
% At a high-level, there are two main algorithmic paradigms for interactive learning: agnostic algorithms and realizability-based algorithms. Agnostic algorithms use a model class $\F$ to guide learning, but do not assume that the true data-generating process is well-modeled by $\F$. Because of this, agnostic algorithms tend to have the advantages of being robust to model specification and noise.
% %Due to these virtues, agnostic algorithms have received much attention in the literature on interactive learning with proposed algorithms in areas such as active classification, best arm identification, and regret minimzation \jks{citations}.
% Due to these virtues, agnostic algorithms have received much attention in the literature on interactive learning such as active classification \cite{dasgupta2007general,balcan2009agnostic,hanneke2014theory}.
% By contrast, realizability-based algorithms assume that the model class $\F$ accurately models the real-world and leverages the structure in $\F$ to guide and potentially accelerate learning. Computationally efficient realizability-based algorithms have only been developed for specific model classes (essentially, linear models) for problems like best arm identification \cite{soare2014best,fiez2019sequential,katz2020empirical} and regret minimization \cite{lattimore_szepesvari_2020}.
At a high-level, there are two main algorithmic paradigms for interactive learning: \emph{agnostic} algorithms and \emph{realizability-based} algorithms. Agnostic algorithms may use a model class $\F$ to guide learning, but do not assume that the true data-generating process is well-modeled by $\F$. Because of this, agnostic algorithms tend to have the advantages of being robust to model misspecification and noise.
%Due to these virtues, agnostic algorithms have received much attention in the literature on interactive learning with proposed algorithms in areas such as active classification, best arm identification, and regret minimzation \jks{citations}.
Due to these virtues, agnostic algorithms have received much attention in the literature on interactive learning, e.g., in active classification \cite{dasgupta2007general,balcan2009agnostic,hanneke2014theory}.
By contrast, realizability-based algorithms assume that the model class $\F$ accurately models the real-world and leverages the structure in $\F$ to guide and potentially accelerate learning. Computationally efficient realizability-based algorithms have only been developed for \emph{specific} model classes
for problems like best arm identification \cite{soare2014best,fiez2019sequential,katz2020empirical} and regret minimization \cite{lattimore_szepesvari_2020}, and the literature lacks a general framework for developing computationally efficient minimax optimal algorithms for generic function classes in the realizable setting.
%\textcolor{magenta}{Rob comment: I searched a bit on term "realizability-based algorithm" and it seems uncommon and sometimes refers to implementable/practical algorithms, which is what it sounds like to me. So instead I tried out "realizability-based algorithm". I agree that it would be nice to have a short term that means an algorithm that assumes it is operating in a "realizable setting". }
The starting point of this paper is the basic question: in the realizable setting, can agnostic algorithms compete with realizability-based algorithms?
%In this paper, we give a series of negative results that demonstrate classes of instances on which any agnostic algorithm is minimax suboptimal.
In this paper, we begin by giving a series of negative results that demonstrate that agnostic algorithms pay a significant cost for their robustness to model mispecification. As an example, we show that \emph{any agnostic active classification algorithm} is minimax suboptimal for a class of realizable instances and thus has no hope of competing with realizability-based algorithms in the realizable setting. These results motivate us to develop \emph{a general framework for computationally efficient and sample-efficient algorithms for generic function classes in the realizable setting}. In doing so, we solve an open problem dating back to the work of \cite{amin2011bandits} on the Haystack Dimension, developing the first computationally efficient algorithm for best arm identification with generic function classes that matches the minimax lower bound up to logarithmic factors. Finally, we empirically demonstrate the generality and practicality of our new approach, \texttt{GRAILS}, for function classes ranging from vanilla kernel methods to kernel methods with side information and to the class of convex functions.
%We complement these results with novel realizability-based algorithms with general function classes for best arm identification and a new problem we call loss minimization. These algorithms are the first to be computationally efficient and minimax optimal for their respective settings.
%\begin{itemize}
%\item Empirical work suggests that realizability-based algorithms often perform better than agnostic algorithms in contextual bandits \cite{foster2018practical,krishnamurthy2016contextual}; \jks{other references with similar spirit?}.
%
%\begin{itemize}
%\item Often $\F$ is chosen in practice so that it can interpolate the true labels, and thus realizability actually holds.
%\end{itemize}
%
%\item There is a hole in the literature of computationally efficient minimax-optimal realizability-based algorithms for interactive learning problems like best-arm identification and regret minimization.
%
%\end{itemize}
%
%In this work, we make the following contributions:
%\begin{itemize}
%\item we provide new lower bounds that show that agnostic algorithm are necessarily suboptimal for problems like best arm identification and active classification under the realizability assumption. \kevin{Really? For best-arm? That's surprising}
%\item we give the first computationally efficient realizability-based algorithm for best arm identification (in addition to other problems such as identifying an $\epsilon$-good arm and an arm that is better than a threshold)
%\item we give a new computationally efficient, minimax-optimal algorithm for a loss minimization problem, a problem closely related to regret minimization.
%\end{itemize}
\section{Problem Setup}
Let $\X$ denote the input space and $\Y \subset \R$ the output space. We assume that $|\Y| < \infty$, but will relax this assumption later. Let $x_1,\ldots, x_n \in \X$ be a fixed pool of $n$ measurements (or arms) with associated scores $y_1,\ldots, y_n \in \Y$. At each round $t$, the learner selects (or queries) $I_t \in [n]$ and observes $y_{I_t}$. We assume that the learner is given a function class $\F \subset \Y^\X$ where $\Y^\X$ denotes the set of all functions mapping $\X$ to $\Y$. We say that \emph{realizability} holds if there exists $f^* \in \F$ such that $f^*(x_i) = y_i$ for all $i \in [n]$. An algorithm is \emph{realizability-based} if it assumes realizability. We focus on the noiseless setting here, but it is straightforward to extend our algorithms to handle the case where $y_{I_t}$ is perturbed by independent, additive noise by simple repeated sampling.
We consider the following three objectives:
\begin{itemize}[leftmargin=.5cm]
\item \textbf{Best arm identification:} The goal is to identify $\argmin_{i \in [n]} y_i$ using as few queries as possible.
\item \textbf{Cumulative Loss Minimization:} The goal is to identify $\argmin_{i \in [n]} y_i$ while minimizing the loss $\sum_{t=1}^T y_{I_t}$ incurred where $T$ is the round that the agent identifies $\argmin_{i \in [n]} f^*(x_i)$.
\item \textbf{Active Classification:} The goal is to identify $\argmin_{f \in \F} \sum_{i=1}^n \one\{y_i \neq f(x_i)\}$ using as few queries as possible.
\end{itemize}
% \jks{perhaps don't really emphasize loss minimization as a different problem.}
% \jks{relate to regret minimization}
Best-arm identification is a well-studied problem with applications ranging from clinical trials to A/B/n testing. Cumulative loss minimization is a new problem.
%\blake{Is it correct that cummulative loss minimization depends on a stopping time? In this sense, it seems different than regret minimization unless the horizon is large}
Applications include running a clinical trial to determine which of a collection of drugs is most effective while, due to ethical concerns about giving patients ineffective drugs, minimizing the number of participants with bad outcomes. It is closely related to regret minimization where the goal is instead to minimize $\sum_{t=1}^{T_0} y_{I_t} - \min_{i \in [n]} y_i$ for a fixed time horizon $T_0$. Finally, active classification is a mature field whose goal is to minimize the number of labels required to learn a high-quality classifier.
Our main focus in this work is on minimax optimality for a fixed $\F$ in the realizable setting:
\begin{itemize}[leftmargin=.5cm]
\itemsep0em
\item \textbf{Best arm identification:} $\bestopt(\F)$ is the smallest integer $q$ such that there exists some algorithm $\A$ such that for every $f^* \in \F$, $\A$ outputs an element of $\argmin_{i \in [n]} f^*(x_i)$ after at most $q$ queries.
\item \textbf{Cumulative Loss Minimization:} $\lossopt(\F)$ is the smallest real number $q$ such that there exists some algorithm $\A$ such that for every $f^* \in \F$, $\A$ outputs an element of $\argmin_{i \in [n]} f^*(x_i)$ after incurring a loss of at most $q$.
\item \textbf{Active Classification:} $\classopt(\F)$ is the smallest integer $q$ such that there exists some algorithm $\A$ such that for every $f^* \in \F$, $\A$ outputs $f^*$ after at most $q$ queries.
\end{itemize}
We emphasize that the above notion of minimax optimality is with respect to the class $\F$.
Next, we briefly summarize our contributions:
\begin{itemize}[leftmargin=.5cm]
% . Using similar techniques, we show that there exists a class of realizable instances on which there is a exponential gap between agnostic active classification and realizable active classification algorithms.
\item \textbf{Best arm identification:} Assuming we can sample efficiently from a distribution $\pi$ with support $\F$, we give a greedy and computationally efficient algorithm that obtains a sample complexity of $O(\log(n)\log(\frac{1}{\P_\pi(S_{f^*})} ) \bestext)$ where $\P_\pi(S_{f^*})$ is the probability of sampling $f^*$ and $\bestext$ is a combinatorial quantity related to the extended teaching dimension and is a minimax lower bound. This is the first computationally efficient and nearly minimax optimal algorithm for best arm identification with generic function classes that matches the minimax lower bound up to logarithmic factors.
%\kevin{Modified, still true? I feel theses contributions need to emphasize our theory more, and the connection to haystack and extended teaching dimension}
\item \textbf{Loss minimization:} We propose a new algorithm that achieves a loss of $O(\lossext \log(|\F|))$ in the worst case where $\lossext$ is the minimax lower bound. We show that when applied to the regret minimization setting with general function classes, our algorithm achieves a state-of-the-art regret bound that is always better than the prior state-of-the-art regret bound in \cite{amin2011bandits}, can be arbitrarily better, and for a large set of function classes matches the minimax lower bound up to logarithmic factors. Furthermore, using our techniques from our algorithm for best arm identification, we make this algorithm computationally efficient by leveraging a sampling oracle.
\item \textbf{Active Classification:} We show that there exists a class of realizable instances such that any agnostic algorithm must query $\tilde{\Omega}(\sqrt{n})$ arms to identify the best arm, but there is a realizability-based algorithm that requires $O(\log(n))$ queries.
This demonstrates an exponential gap between agnostic active classification and realizable active classification algorithms.
\end{itemize}
% \kevin{Related work may make sense here instead of later}
\section{Related Work}
Best arm identification has received much attention in the literature but predominantly for special classes of functions (e.g., linear) \cite{jamieson2014lil, kaufmann2016complexity, soare2014best,fiez2019sequential,katz2020empirical}.
% \blake{This attention, however, has been focused on identification with special, restricted classes of functions (e.g., linear functions). In this work we instead conside \emph{general function classes} which have received far less attention.}
By contrast, our work concerns best arm identification with \emph{general function classes}, which has received much less attention, and is most closely related to \cite{amin2011bandits}. \cite{amin2011bandits} introduce a combinatorial quantity $\hd(\F)$ known as the \emph{haystack dimension}, and propose a greedy algorithm that achieves a sample complexity of $\hd(\F) \log(|\F|)$, but is not computationally efficient. We build on this work by designing a greedy algorithm that achieves computational efficiency by appealing to \cite{lovasz2006fast} to sample functions from the function class $\F$. A key technical challenge is that the version space under a greedy algorithm is not convex and therefore standard sampling algorithms like ``hit and run'' cannot be directly applied to it.
%We overcome this challenge by sampling from a mixture of distributions, each supported on a convex set and show that the loss in sample complexity is only a $\log(n)$ factors
\cite{tosh2020diameter} gives a general computationally efficient algorithm that solves many pure exploration problems such as active classification and clustering. Their algorithm requires specifying a distance $d$ between functions in the function class $\F$ and proceeds by shrinking the average diameter of $\F$ as measured by $d$.
Unfortunately, for problems such as best arm or $\epsilon$-good arm identification, it is unclear how to construct the appropriate distance function to achieve optimal performance or even if such a function exists for a given application.
%\blake{For example, in the case of $\epsilon$-good arm identification, the natural sup-norm distance leads to poor behavior in simple examples, and it is unclear what the correct distance function should be.}
% In principle, their algorithm could be applied to problems like $\epsilon$-good arm identification, but it is not clear how to construct a distance function $d$ that would yield a minimax optimal sample complexity (see the supplementary material for a more in-depth discussion).\blake{Can we say something a bit stronger? How about, ``their result relies heavily on being able to construct a distance function $d$ that is appropriate for the objective, and beyond special cases such as blah and blah, it is unclear how to do this.'' I guess I have in mind the pushback for Gamification where the answer becomes ``sure this alg could do it, but no one knows how to compute the best response oracle in general''}
% To the best of our knowledge, our problem formulation of loss minimization is novel. It is closely related to regret minimization with general function classes, which \cite{amin2011bandits} studies. They propose an explore-and-then-commit algorithm. Since the exploration phase algorithm ignores the costs of acquiring information, their result is not minimax optimal for either loss minimization or regret minimization. Indeed, we show that the regret incurred by our algorithm is never worse than that of the algorithm in \cite{amin2011bandits} and there exist function classes where it is arbitrarily better.
% \textbf{Structured Bandits:} Recently, a series of works \cite{degenne2020structure,degenne2020gamification,tirinzoni2020asymptotically} have given results for structured bandits where $\theta$ is constrained. Their results are asymptotic for the asymptotic setting, but their nonasymptotic terms are large and suboptimal \jks{beef up.}
\textbf{Active Classification:} There have been many works on active classification \cite{hanneke2014theory,settles2009active}, both in the realizable setting (e.g., \cite{cohn1994improving,dasgupta2005coarse, tosh2017diameter}) and on agnostic algorithms (e.g., \cite{balcan2009agnostic,beygelzimer2010agnostic,dasgupta2007general,katz2021improved}).
% \kevin{Broken reference}
Proposition \ref{thm:agnostic_subopt} shows that agnostic algorithms are minimax suboptimal in the realizable setting, suggesting that they may never be able to match the performance of realizability-based algorithms.
%This is particularly relevant as model classes have become richer in light of the wild success of neural networks and the realizability assumption becomes more plausible. Furthermore, it has been observed in contextual bandits that realizability-based algorithms outperform agnostic algorithms \cite{krishnamurthy2016contextual, foster2018practical, bietti2018contextual}.
Our work is closely related to the problem of \emph{Exact learning} where a learning algorithm is required to identify the true $f^*$ with probability $1$ \cite{hegedHus1995generalized,balcazar2002new}, in contrast to the PAC requirement where a failure probability of $\delta$ is permitted. While this requirement is certainly strong, this framework enables the design of practical algorithms, as suggested by our experiments.
%Moreover, we stress that our ``No Free Lunch'' results, namely Propositions \ref{thm:agnostic_subopt} and \ref{thm:best_arm_agnostic_suboptimal} apply to algorithms that allow for a failure probability $\delta$.
%It is an open question whether allowing for a failure probability of $\delta$ enables the design of algorithms that sidestep extended-teaching dimension-like lower bounds.
%\kevin{I think we have to be a bit careful of distinguishing between PAC and exact learning results. Section 8.3 ``Exact Learning'' of Hanneke's disagreement based survey gives a nice disucssion and lit review.}
\textbf{Bayesian Optimization Work:} Our work is also related to the vast field of Bayesian optimization. We review a few relevant theoretical results and refer readers to \cite{shahriari2015taking} for a thorough survey. \cite{srinivas2009gaussian} propose and analyze \texttt{GP-UCB}. Although \texttt{GP-UCB} is applied often to best arm identification problems, \cite{srinivas2009gaussian} only give a regret bound and optimality is unclear. Other notable works include \cite{bull2011convergence} whose results are asymptotic and the recent paper \cite{camilleri2021high}, which gives a new algorithm for best arm identification for kernel bandits based on experimental design, but its optimality is unclear.
% \begin{itemize}
% \item \cite{srinivas2009gaussian} analyze GP-UCB in the regret setting and give Bayesian and frequentist bounds in terms of the information ratio. Optimality is unclear and it has a different focus than our work since pure exploration objectives are not considered
% \item \cite{de2012exponential} study a variant of UCB for regret minimization for the noise free case. They give exponentially decaying bounds (in contrast to \cite{srinivas2009gaussian}) for bayesian regret and no bounds for frequetist regret. Also no pure exploration.
% \item \cite{bull2011convergence} gives asymptotic matching upper and lower rates of convergence for a modification of expected improvement.
% \item \cite{frazier2018tutorial} says that despite the good performance of Bayesian optimization, there is no theory justifying its optimality. Our work gives a way to relate to the lower bound and is nearly optimal.
% \item The recent paper by \cite{camilleri2021high} gives a new approach to kernel bandits and studies pure exploration, but optimality is unclear.
% \end{itemize}
\section{No Free Lunch for Agnostic Algorithms}
Many agnostic algorithms have been proposed in the active classification literature (e.g. \cite{balcan2009agnostic,beygelzimer2010agnostic, dasgupta2007general}). We say an algorithm $\A$ is \emph{$\delta$-agnostic} if for any labeling of the data $y \in \Y^n$, $\A$ finds the best classifier, $\argmin_{f \in \F} \sum_{i=1}^n \one\{y \neq f(x_i)\}$, with probability at least $1-\delta$. Despite considering a much larger class of possible labelings, agnostic algorithms have been shown to achieve the lower bound in the \emph{realizable setting} for the well-studied problem of thresholds where $\F = \{f_1,\ldots, f_n\}$ and $f_i(x_j) = 1$ if $j \leq i$ and $f_i(x_j) =0$ otherwise \cite{jain2019new}. This raises the question: \emph{can agnostic algorithms be optimal over classes of realizable instances in general}? The following result shows that in fact the gap between agnostic algorithm and the minimax lower bound is exponential.
\begin{proposition}\label{thm:agnostic_subopt}
Let $\delta \in (0,\min(\tfrac{1}{20} , \tfrac{1}{n} ))$. Consider the active classification setting. There exists $x_1,\ldots, x_n$ and $\F$ forming a class of instances $\I = \{ (x_i,f(x_i))_{i=1}^n : f \in \F\}$ such that
\begin{itemize}[leftmargin=.5cm]
\item the expected number of samples of any $\delta$-agnostic algorithm is $\Omega\left(\log(1/\delta) \frac{\sqrt{n}}{\log(n/\delta)}\right)$ on one of the instances in $\I$, and
\item there exists a realizability-based algorithm that solves each instance in $\I$ in $O(\log(n))$ queries.
\end{itemize}
\end{proposition}
% \textbf{Regret Minimization:} Furthermore, we can also show that any agnostic algorithm cannot be minimax optimal for regret minimization either. We say that a regret minimization algorithm $\A$ is $\epsilon$-agnostic if for any $\tilde{y} \in \Y^n$ $\min_{j \in [n]} \tilde{y}_\ast - \tilde{y}_j \geq \epsilon$ where $\tilde{y}_\ast = \max_i \tilde{y}_i$, $\A$ suffers at most bounded regret $R(\A)$ independent of the time horizon. Note that this is a mild condition as even the algorithm that uniformly samples $x_1, \cdots, x_n$ then commits to the best achieves bounded regret for a sufficiently large time horizon.
It turns out agnostic algorithms are also minimax suboptimal for regret minimization, a problem closely related to loss minimization as we will discuss in more detail in Section \ref{sec:loss_min}. Here, we say that a regret minimization algorithm $\A$ is \emph{$\epsilon$-agnostic} if for any $y \in \Y^n$ such that $\min_{j \in [n] \setminus \{i_* \}} y_j-y_{i_*} \geq \epsilon$ where $i_\ast \in \argmin_{i \in [n]} y_i$, $\A$ suffers at most bounded regret independent of the time horizon.
\begin{proposition}\label{thm:regret_agnostic}
There exists $\F$ such that any $1$-agnostic algorithm suffers regret at least $O(|\F|)$ for some instance in $\F$ while there exists a realizability-based algorithm suffering regret at most $O(\log(|\F|))$.
%\jks{what definition of agnostic is used?}
\end{proposition}
For best-arm identification, a fully agnostic algorithm must consider any $y \in \Y^n$ and it is therefore trivial that it would need to query every $x_i$. Therefore, we consider a weaker notion of agnostic algorithm. For $k \in \N$ and $\delta \in (0,1)$, we say an algorithm $\A$ is \emph{$(\delta,k)$-agnostic} if for any $y \in \Y^n$ such that $\min_{f \in \F} \sum_{i=1}^n \one\{f(x_i) \neq y_i\} \leq k$, $\A$ identifies $\argmin_{i \in [n]} y_i$ with probability at least $1-\delta$. Despite only allowing for small amounts of mispecification, there is still an exponential gap between agnostic algorithms and the minimax lower bound.
\begin{proposition}\label{thm:best_arm_agnostic_suboptimal}
Let $\delta \in (0,\min(\tfrac{1}{40} , \tfrac{1}{n} ))$. There exists $\F$ such that for any algorithm $\A$ that is $(\delta,1)$-agnostic with respect to $\F$, $\A$ takes $\Omega\left(\frac{n}{\log(n/\delta)}\right)$ queries in expectation on some instance in $\F$, while there exists a realizability-based algorithm requiring $O(1)$ samples on all instances in $\F$.
\end{proposition}
%\jks{Show that CAL does badly on this instance}
% \begin{corollary}
% No $\delta$-agnostic algorithm wrt noiseless active binary classification is minimax optimal for realizable active learning in the pool-based setting.
% \end{corollary}
% The following theorem shows that there exists an instance such that if the algorithm allows for even slight mispecification, then it must query $\Omega(n)$ examples more than a realizability-based algorithm, in expectation.
% %\begin{theorem}
% %%Let $\delta \in (0,\min(1/8, \frac{1}{2n})$.
% %Let $\delta \in (0,1/16)$. Suppose that $\min_{i,j \in [L]} |w_i - w_j| \geq 1$. Let $\F = \{f^*\}$ with $f^* : \X \mapsto \{w_1,\ldots, w_L\}$ such that for all $i \in [n]$ $f^*(x_i) \in \{w_2,\ldots, w_{L-1}\}$ and $|\argmin_{i \in [n]} f^*(x_i)|=1$. Let $\A$ be a $(\delta,1)$-agnostic algorithm wrt $\F$. Then, $\A$ takes at least $n/2$ queries in expectation on the instance given by $f^*$.
% %\end{theorem}
% \begin{theorem}\label{thm:best_arm_agnostic_suboptimal}
% Let $\delta \in (0,\min(1/20,1/n))$. There exists $\F$ such that for any algorithm $\A$ that is $(\delta,1)$-agnostic with respect to $\F$, $\A$ takes at least $n/8$ queries in expectation on some instance in $\F$, while there exists a realizability-based algorithm requiring $O(1)$ samples on all instances in $\F$.
% \end{theorem}
% \textbf{Regret Minimization:} Furthermore, we can also show that any agnostic algorithm cannot be minimax optimal for regret minimization either. We say that a regret minimization algorithm $\A$ is $\epsilon$-agnostic if for any $\tilde{y} \in \Y^n$ $\min_{j \in [n]} \tilde{y}_\ast - \tilde{y}_j \geq \epsilon$ where $\tilde{y}_\ast = \max_i \tilde{y}_i$, $\A$ suffers at most bounded regret $R(\A)$ independent of the time horizon. Note that this is a mild condition as even the algorithm that uniformly samples $x_1, \cdots, x_n$ then commits to the best achieves bounded regret for a sufficiently large time horizon.
% \begin{theorem}\label{thm:regret_agnostic}
% There exists $\F$ such that any $1$-agnostic algorithm suffers regret at least $O(|\F|)$ for some instance in $\F$ while there exists a realizability-based algorithm suffering regret at most $O(\log(|\F|))$.
% %\jks{what definition of agnostic is used?}
% \end{theorem}
% \textbf{Active Classification:} Using similar techniques to the above, we obtain a negative result for agnostic algorithms for active classification. We say an algorithm $\A$ is $\delta$-agnostic if for any $y \in \Y^n$, $\A$ outputs $\argmin_{f \in \F} \sum_{i=1}^n \one\{y \neq f(x_i)\}$ with probability at least $1-\delta$. Many popular algorithms from the literature are $\delta$-agnostic (e.g. \cite{balcan2009agnostic,beygelzimer2010agnostic, dasgupta2007general}).
% \kevin{You're going to need pictures for to demontrate this separation. The instance from Dasgupta is easy for classification. What about best arm?}
\textbf{Techniques:} The above results rely on a novel approach for constucting instance-dependent lower bounds for the noiseless setting. The key idea is a reduction of the noiseless setting to a setting where observations are corrupted with a Gaussian random variable, that is, when the arm $i \in [n]$ is queried, the agent observes $y_i + \eta$ where $\eta \sim N(0,1)$, instead of $y_i$. This reduction enables the application of the transportation Lemma from the multi-armed bandit literature \cite{garivier2016optimal} to the noiseless setting and thereby to construct instances where agnostic algorithms \emph{necessarily} perform poorly.
%\jks{this should show cal is not minimax optimal for realizable. but we need new analysis}
%Recall the reduction in Section 3 of \cite{jain2019new} that shows that binary classification is an instance of combinatorial bandits. Now, consider the combinatorial bandit setting. The agent is given a set $\Z \subset \{0,1\}^n$. At each round $t$, an agent $\A$ selects a measurement $I_t \in [n]$ and observes $ \theta_{I_t}$ where $\theta \in \{-1,1\}^n$. We assume that $ \text{argmax}_{z \in \Z} z^\t \theta = \{z_*\}$, and the goal is to identify $z_*$.
%
%We call the above setup \emph{noiseless active binary classification}. If there exists $z \in \Z$ such that $2 z-1 = \theta$, then we say that the setting is \emph{realizable}.
%\begin{theorem}\label{thm:agnostic_subopt}
%There exists a class of realizable instances of noiseless active binary classification where
%\begin{itemize}
%\item any agnostic $\delta$-PAC algorithm wrt noiseless active binary classification requires $\Omega(\log(1/\delta) \frac{\sqrt{n}}{\log(n/\delta)})$ samples in expectation on one of these instances, and
%\item there exists an algorithm that assumes realizability and that solves each instance in $O(\log(n))$ queries.
%\end{itemize}
%\end{theorem}
%
%\jks{Show that CAL does badly on this instance}
%
%\begin{corollary}
%No agnostic $\delta$-PAC algorithm wrt noiseless active binary classification is minimax optimal for realizable active learning in the pool-based setting.
%\end{corollary}.
%
%\jks{this should show cal is not minimax optimal for realizable. but we need new analysis}
% \section{Sampling Oracles for Efficient Realizable Active Learning}
% %We suppose wlog that $w_1 < w_2 < \ldots < w_L$ and for simplicity that the $w_1,\ldots, w_L$ are evenly-spaced, but this latter assumption can be easily dropped with only superficial changes to algorithms and results.
% % \blake{Maybe title this section differently}
% In this section we introduce the concept of a \emph{sampling oracle}, a key tool that we will leverage to achieve computational efficiency.
% \blake{why is a sampling oracle good?}
% Let $\F \subset \R^\X $ be a convex compact set of functions where $\R^\X$ denotes the set of all functions $f: \X \mapsto \R$. For $z \in \R$, let $\round{z}$ denote the $y \in \Y$ that is closest to $z$, tiebreaking by rounding down. The set of regressors $\F$ induces a set of discretized functions
% \begin{align*}
% \F_{\calR} & = \{\round{f} : f \in \F\}.
% \end{align*}
% We make the realizability assumption that there exists $f^* \in \F$ such that $\round{f^*(x_i)} = y_i$ for all $i \in [n]$.
% %Define
% %\begin{align*}
% %\F_x & = \{(f(x_1),\ldots, f(x_n))^\t : f \in \F \} \\
% %\Fceil & = \{(\round{f(x_1},\ldots, \round{f(x_n)})^\t : f \in \F \}.
% %\end{align*}
% Given $\F \subset \R^\X$, we say that we have access to a \emph{sampling oracle} for $\F$ if there exists a distribution $\pi $ on $\F$
% \begin{itemize}
% \item such that we can draw $f \sim \pi$, and
% \item for any $\bar{\F} = \F \cap \H$ where $\H$ is the intersection of $\text{poly}(n,d)$ halfspaces of the form $\{x \in \R^n : w^\t x \leq b\}$, we can sample $f \sim \pi_{\bar{\F}}$ where for any measurable $A \subset \bar{\F}$ $\pi_{\bar{\F}}(A) = \frac{\pi(A)}{\pi(\bar{\F})}$.
% \end{itemize}
% \blake{We need way more context on that second bullet point. Kind of comes out of the blue. Idea: introduce $\H$ earlier before the bullet points and give context there.}
% % Given $\F \subset \R^\X$, we assume access to a \emph{sampling oracle}: that is, there exists a distribution $\pi $ on $\F$ and we can draw $f \sim \pi$. Furthermore, we assume that the sampling oracle allows us to cample from simple conditional distributions, precisely, if $\bar{\F} \subset \F$ and $\bar{\F}$ has the form $\bar{\F} = \F \cap \cap_{j=1}^m \{f : f(x_{i_j}) \in [a_j,b_j] \}$ for $a_j,b_j \in \R$ and $i_1,\ldots, i_m \in [n]$, then we can sample $f \sim \pi_{\bar{\F}}$ where $\pi_{\bar{\F}} = \frac{\pi(f)}{\pi(\bar{\F})}$.
% We show in the appendix that non-trivial sampling oracles are available for many useful function classes such as linear models, kernel methods, and convex function and leverage these oracles in our experiments.
\section{Efficient Algorithm for Best Arm Identification}
Given the limitations of agnostic algorithms for interactive learning in the realizable setting established in the prior section, we now turn to developing \emph{realizability-based} algorithms that are computationally efficient and match the minimax lower bound up to logarithmic factors. In this section, we examine the best arm identification problem in which the goal is to identify an $i_* \in \argmin_{i \in [n]} y_i $ using as few queries as possible. Practitioners may be willing to sacrifice a bit of optimality if that makes it easier to solve a problem and thus here we state some of our results and our algorithm in terms of $\epsilon$-good arm identification, a strict generalization of best arm identification. In this problem, we are given $\epsilon \geq 0$ and the goal is to identify an $\epsilon$-good arm, that is, an $i_* \in [n]$ such that $y_{i_*} \leq \min_{i \in [n]} y_i + \epsilon$. When $\epsilon = 0$, this reduces to best arm identification.
We begin by introducing a new quantity, inspired by the extended teaching dimension, for quantifying the difficulty of identifying an $\epsilon$-good arm in a worst-case sense.
\begin{align*}
\bestextep(\F) &= \max_{g : \X \mapsto \Y} \min_{I \subset [n]} |I| \\
& \text{ s.t. } \exists j \in [n] : \{f \in \F : f(x_i) = g(x_i) \, \forall i \in I\} \subset \left\{ f \in \F : f(x_j) \leq \min_{l \in [n]} f(x_l) +\epsilon\right\}.
\end{align*}
When $\epsilon = 0$, we simply write $\bestext(\F)$ instead of $\bestextep(\F)$. We occasionally write $\bestextep$ instead of $\bestextep(\F)$ when the context leaves no ambiguity. In words, $\bestext$ is the minimum number of samples required so that for any function $g : \X \mapsto \Y$, there is a subset of queries of size $\bestext$ that can make the best arm $i_* \in [n]$ unambiguous by eliminating all $f \in \F$ that do not put $i_*$ as the best. The following theorem establishes $\bestext$ as a lower bound to the optimal minimax sample complexity $\bestopt$ for best arm identification.
%\kevin{Is $g \in \mc{F}$? If not necessarily, does this definition still make sense? Perhaps, you need an if statement of if there does not exist any $f$ that agrees with $g$? Is another way to write this definition is to say there exists a $j \in [n]$ such that for all functions $f$ that agree with $g$, all of these functions have a minimum at index $j$}
% \begin{theorem}\label{thm:best_arm_lower_bound}
% For any deterministic algorithm $\A$, there exists an instance $f^* \in \F$ on which $\A$ requires at least $\bestext$ queries to identify $i_* \in [n]$ such that $i_* \in \argmin_{i \in [n]} y_i $.
% \end{theorem}
\begin{theorem}\label{thm:best_arm_lower_bound}
For any $\F \subset \Y^\X$, $\bestext(\F) \leq \bestopt(\F)$.
\end{theorem}
The setting of best arm identification in general function classes was previously studied in \cite{amin2011bandits} in which the sample complexity results were quantified in terms of the Haystack dimension. Specifically, letting $\F^\prime \subset \F$, define $\F^\prime((x_i,y)) = \{f \in \F : f(x_i) \neq y\}$, the subset of functions in $\F^\prime$ that disagree with the label $y$ on $x_i$, and $\F^\prime(x_i) = \{f \in \F : i \in \argmin_{j \in [n]} f(x_j) \}$, the set of functions in $\F^\prime$ that are minimized at $x_i$. Define $ \gamma(\F^\prime) := \sup_{i \in [n]} \inf_{y \in \Y} \tfrac{|\F^\prime(x_i) \cup \F^\prime((x_i,y))|}{|\F^\prime|}$. The Haystack dimension is defined as:
\begin{align*}
\hd(\F) := \frac{1}{\inf_{\F^\prime \subset \F} \gamma(\F^\prime)}.
\end{align*}
The following Proposition shows that $\bestext(\F)$ is never significantly less than the Haystack dimension $\hd(\F)$ and is never greater than $\hd(\F)$ by more than a $O(\log(|\F|))$ factor.
\begin{proposition}\label{prop:relate_HD_teach_dim}
For any $\F \subset \Y^\X$, $\hd(\F) -1 \leq \bestext(\F) \leq c\hd(\F) \log(|\F|) $, where $c$ is a positive universal constant.
\end{proposition}
%\blake{Can we borrow some language from the Haystack paper to better describe what these sets are? This is pretty tough...}
\subsection{Sampling Oracles for Efficient Realizable Active Learning}
%We suppose wlog that $w_1 < w_2 < \ldots < w_L$ and for simplicity that the $w_1,\ldots, w_L$ are evenly-spaced, but this latter assumption can be easily dropped with only superficial changes to algorithms and results.
% \blake{Maybe title this section differently}
In this section we introduce the concept of a \emph{sampling oracle}, a key tool for achieving computational efficiency.
In contrast to prior active methods that enumerate an intractably large version space (e.g., \cite{amin2011bandits,hegedHus1995generalized}), we instead place a measure over the version space to track its size without explicitly storing it and use sampling to approximate this measure.
Let $\reg \subset \R^\X $ be a set of regression functions where $\R^\X$ denotes the set of all functions $r: \X \mapsto \R$.
%Define
%\begin{align*}
%\F_x & = \{(f(x_1),\ldots, f(x_n))^\t : f \in \F \} \\
%\Fceil & = \{(\round{f(x_1},\ldots, \round{f(x_n)})^\t : f \in \reg \}.
%\end{align*}
Given $\reg \subset \R^\X$, we say that we have access to a \emph{sampling oracle} for $\reg$ if there exists a distribution $\pi $ on $\reg$.
\begin{itemize}[leftmargin=.5cm]
\item such that we can draw $r \sim \pi$, and
\item for any $\widetilde{\reg} = \reg \cap \H$ where $\H$ is the intersection of $\text{poly}(n,d)$ halfspaces of the form $\{x \in \R^n : w^\t x \leq b\}$, we can sample $r \sim \pi_{\widetilde{\reg}}$ where for any measurable $A \subset \widetilde{\reg}$ $\pi_{\widetilde{\reg}}(A) = \frac{\pi(A)}{\pi(\widetilde{\reg})}$.
\end{itemize}
By representing constraints on the version space as a set $\H$ in the second bullet, we can leverage sampling techniques to estimate the measure over the version space efficiently.
% \blake{We need way more context on that second bullet point. Kind of comes out of the blue. Idea: introduce $\H$ earlier before the bullet points and give context there.}
% Given $\reg \subset \R^\X$, we assume access to a \emph{sampling oracle}: that is, there exists a distribution $\pi $ on $\reg$ and we can draw $f \sim \pi$. Furthermore, we assume that the sampling oracle allows us to cample from simple conditional distributions, precisely, if $\bar{\reg} \subset \reg$ and $\bar{\reg}$ has the form $\bar{\reg} = \reg \cap \cap_{j=1}^m \{f : f(x_{i_j}) \in [a_j,b_j] \}$ for $a_j,b_j \in \R$ and $i_1,\ldots, i_m \in [n]$, then we can sample $f \sim \pi_{\bar{\reg}}$ where $\pi_{\bar{\reg}} = \frac{\pi(f)}{\pi(\bar{\reg})}$.
We show in Appendix~\ref{sec:sampling_examples} that non-trivial sampling oracles are available for many useful function classes such as linear models, kernel methods, and convex functions. To keep the presentation simple, we assume here that we can compute probabilities exactly and show that approximation suffices in the appendix.
For $z \in \R$, let $\round{z}$ denote the $y \in \Y$ that is closest to $z$, tiebreaking by rounding down. The set of regressors $\reg$, together with $D[\cdot]$, induces a set of discretized functions mapping $\X$ to $\Y$
\begin{align*}
\F_{\reg} & = \{\round{r} : r \in \reg\}.
\end{align*}
Our algorithmic approach is to use $\F_{\reg}$ as our predictors and to leverage the structure of $\reg$ to achieve computational efficiency. Our algorithm relies on \emph{greedy volume reduction}, which requires that the $f \in \F$ have nonzero measure and the discretized setting ensures this. In the appendix, we extend our algorithms to the continuous setting but at the cost of a potentially suboptimal sample complexity. We make the realizability assumption that there exists $r^* \in \reg$ such that $\round{r^*(x_i)} = y_i$ for all $i \in [n]$.
\subsection{The Algorithm}
In this section, we present the algorithm for $\epsilon$-good arm identification, a strict generalization of best arm identification. \texttt{GRAILS} (see Algorithm \ref{alg:effic_haystack_alg}) implicitly maintains a version space $\ver_t$ at each round over $\reg$ and thus we define
\begin{align*}
\ver_t(x_i,y) & = \{r \in \ver_t : \round{r(x_i)} \neq y \},\, \, \ver_{t,\epsilon}(x_i) = \{r \in \ver_t : \round{r(x_i)} \leq \min_{j \in [n]} \round{r(x_j)} + \epsilon\},
\end{align*}
defined analogously to the set $\F^\prime(x_i,y)$ and $\F^\prime(x_i)$ above. $\ver_t(x_i,y)$ consists of functions in the version space that are inconsistent with the observation $(x_i,y)$ and $ \ver_{t,\epsilon}(x_i)$ is the set of functions in the version space for which $x_i$ is an $\epsilon$-good arm and therefore no longer need to be considered. When $\epsilon = 0$, we write $ \ver_{t}(x_i)$ instead of $ \ver_{t,\epsilon}(x_i)$.
Algorithm \ref{alg:effic_haystack_alg} takes the greedy approach. At each round $t$, our algorithm queries a point $I_t \in [n]$ that maximizes the measure of the functions removed from the version space under a distribution $P_k$, namely $\P_{r \sim P_k}(r \in \ver_t(x_i) \cup \ver_{t,\epsilon}((x_i,y))$, for the worst case observation $y \in \Y$.
% \blake{Can we put these sentences in reverse order? Like, give a mathematical description of which point is sampled and then say why this is reasonable? Or maybe a sentence giving some intuition, then one with the definition of $I_t$, and then the idea why this is reasonable. Also the $\F$ sets haven't been defined yet in this sentence which makes it hard. }
Once the algorithm queries $I_t$, it adds $I_t$ to $O_t$, the set of observations up to time $t$.
Now, we describe our sampling approach. To keep the presentation simple and to capture the main ideas, we assume here $\reg$ is convex. Define $\texttt{better}_{\epsilon} = \max_{y < \min_{i \in O_t} y_i - \epsilon} y$, which is well-defined since $\Y$ is discrete. Note that $r \in \reg_t$ only if there exists $l \in [n]\setminus O_t$ such that $\round{r(x_l)} \leq \texttt{better}_{\epsilon}$. We may decompose the version space $\ver_t$ as the union of $O(n)$ sets:
\begin{align*}
\ver_t & =
% \{r \in \reg: \round{r(x_i)} = y_i \forall i \in O_t \wedge \round{r(x_i)} > \min_{j \in [n]} \round{r(x_j)} \, \forall i \in O_t \} \\
\cup_{l \in [n] \setminus O_t} \{r \in \reg: \round{r(x_i)} = y_i \forall i \in O_t\} \cap \{ \round{r(x_l)} \leq \texttt{better}_{\epsilon} \} = \cup_{l \in [n] \setminus O_t} C_l(O_t)
\end{align*}
where
\begin{align*}
C_l(O_t) := \{r \in \reg: \round{r(x_i)} = y_i \forall i \in O_t\} \cap \{ \round{r(x_l)} \leq \texttt{better}_{\epsilon} \}.
\end{align*}
Each set $C_l(O_t)$ is an intersection of $\reg$ with $O(n)$ halfspaces and is a convex set. Unfortunately, a union of convex sets need not be convex so one cannot hope to directly apply algorithms like hit-and-run to efficiently sample from $\ver_t$. To overcome this,
the algorithm samples from a mixture with each component supported on $C_l(O_t)$, a convex set. Operating in stages, in stage $k$, it puts the measure $P_k$ over the remaining functions in the version spaces where
\begin{align*}
\vspace{-3em}
P_k : = \frac{1}{n-|O_t|} \sum_{l \in [n] \setminus O_t} \pi_{C_l(O_t)}.
\end{align*}
Sampling from the mixture $P_k$ is a key algorithmic innovation in this work and enables efficient sampling from a non-convex version space.
We make two final remarks. First, \text{STOP}($\F_{\reg}$,$O_t$) is a subroutine for terminating the algorithm. It essentially checks whether the version space $\ver_t$ is empty by checking whether each of the $C_l(O_t)$ sets is feasible. For many function classes of interest such as linear models, kernel methods, and the class of convex functions, this can be formulated as a convex feasibility problem (see the Appendix for a concrete instance).
Second, in practice, one may not know the true model class precisely apriori, but it is straightforward to do model selection through the standard doubling technique. For example, for the class of Lipschitz functions, one may not know the true Lipschitz constant. In these situations, one can apply a standard doubling trick on the Lipschitz constant.
%Second, here, we focus on the noiseless case, but in the supplementary material we give an extension to deal with noisy observations. There, we consider functions with a continuous output space, but use a rounding scheme that is defined by the accuracy of the observations. That is, if the algorithm queries $x_i$ roughly $\epsilon^{-2}$ times, we round to an accuracy of $\epsilon$.
%\kevin{Can you extend this to a noisy setting where this rounding scheme is defined by the epsilon accuracy of the observations? That is, if you've sampled every query $\epsilon^{-2}$ times, can you round to accuracy $\epsilon$?}
% We briefly give some high-level comments on the algorithm. Suppose at round $t$, the algorithm has queried $O_t \subset [n]$ and observe $y_i$ for all $i \in O_t$. Then, the version space is
% \begin{align*}
% \ver_t & = \{f \in \reg: \round{f(x_i)} = y_i \forall i \in O_t \wedge \round{f(x_i)} > \min_{j \in [n]} \round{f(x_j)} + \epsilon \forall i \in O_t \} \\
% & = \cup_{l \in [n] \setminus O_t} \{f \in \reg: \round{f(x_i)} = y_i \forall i \in O_t\} \cap \{ \round{f(x_l)} +\epsilon < \round{f(x_i)} \forall i \in O_t \} \\
% & =: \cup_{l \in [n] \setminus O_t} C_l(O_t).
% \end{align*}
% where
% \begin{align*}
% C_l(O_t) := \{f \in \reg: \round{f(x_i)} = y_i \forall i \in O_t\} \cap \{ \round{f(x_l)} + \epsilon < \round{f(x_i)} \forall i \in O_t \}.
% \end{align*}
% Note that each set $C_l(O_t)$ is an intersection of $\F$ with $O(n)$ halfspaces. At each round $t$, we will sample from a mixture of distributions
% \begin{align*}
% \frac{1}{n-|O_t|} \sum_{l \in [n] \setminus O_t} \pi_{C_l(O_t)}.
% \end{align*}
%\jks{Note that we update the distribution in phases because this simplifies the argument that showing the reduction in the version space. This introduces a $\log(n)$ factor. There may be a way around this looseness.}
% Algorithm \ref{alg:effic_haystack_alg} can be viewed as maintaining a distinct version space for each $l \in [n]$. The algorithm greedily attempts to shrink all of these version spaces at once. We show that this is sufficient for shrinking each version space sufficiently rapidly. \text{STOP}($\F$,$O_t$) is a subroutine for terminating the algorithm. It essentially checks whether the version space $\ver_t$ is empty. For the function classes of interest in this paper, this can be formulated as a convex optimization problem; due to space constraints, we defer the details to the appendix.
\begin{algorithm}[t]
\small
$P_1 \longleftarrow \pi$, $\ver_1 \longleftarrow \reg$, $k \longleftarrow 1$, $t_1 = 1$, $O_1 \longleftarrow \emptyset$\;
\For{$t =1,2,\ldots$}{
Let $I_t \in \argmax_{i \in [n] \setminus O_t } \min_{y \in \Y} \P_{r \sim P_k}(r \in \ver_{t,\epsilon}(x_i) \cup \ver_t(x_i,y))$\;
Query $x_{I_t}$ and observe $y_{I_t}$ and set $O_{t+1} \longleftarrow O_t \cup \{I_t\}$ \;
Let $\ver_{t+1} \longleftarrow \ver_t \setminus (\ver_{t,\epsilon}(x_{I_t}) \cup \ver_t(x_{I_t},y_{I_t}))$\;
\If{$ \P_{r \sim P_k}(r \in \ver_{t+1}) \leq \frac{1}{2n}$}{
$k \longleftarrow k+1$\;
$t_k \longleftarrow t+1$, $P_{k+1} \longleftarrow \frac{1}{n-|O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k} } \pi_{C_l(O_{t_k})}$\;
}
\If{\text{STOP}($\F_{\reg}$,$O_t$)}{
\Return $\argmin_{i \in O_t} y_i$}
}
\caption{\texttt{GRAILS} (GReedy Algorithm for Interactive Learning using Sampling)}
\label{alg:effic_haystack_alg}
\end{algorithm}
\subsection{The Upper Bound}
We now introduce some notation necessary to state our upper bound.
%Define $f_x = (f(x_1),\ldots, f(x_n))^\t$ for $f \in \F$.
Define for every $f \in \F_{\reg}$ the set
\begin{align*}
S_f = \{r \in \reg : \round{r(x_i)} = f(x_i) \forall i \in [n] \}.
\end{align*}
The sets $S_f$ induce a partition of $\reg$.
\begin{theorem}\label{thm:haystack_eff}
Fix $\reg$. Let $\epsilon \geq 0$. There exists a universal constant $c >0$ such that if $t$ is greater than
% \begin{align*}
% \tilde{O}(\bestopt(\F_{\reg})) = c \bestextep(\F_{\reg}) \log(\frac{1}{\P_{\pi}(S_{f^*})}) \log(n),
% \end{align*}
\begin{align*}
c \bestextep(\F_{\reg}) \log(\frac{1}{\P_{\pi}(S_{f^*})}) \log(n),
\end{align*}
then Algorithm \ref{alg:effic_haystack_alg} has pulled an arm $I_s$ at some $s \leq t$ such that $y_{I_s} \leq \min_{i \in [n]} y_i + \epsilon$.
\end{theorem}
When $\epsilon = 0$, Algorithm \ref{alg:effic_haystack_alg} and Theorem \ref{thm:haystack_eff} together solve the open problem from \cite{amin2011bandits} of developing a computationally efficient optimal algorithm for best arm identification for generic function classes that matches the minimax lower bound up to logarithmic factors.
%\kevin{For the problems defined above, can $S_{f^*}$ be bounded?}
% \begin{remark}
% This algorithm can be extended to the case of nonconvex $\F$. Suppose that we have $\F_1,\ldots, \F_m$, each convex and compact. For example, each $\F_i$ may correspond to a different choice of kernel. Define $\F:= \cup_{i=1}^m \F_i$. Then, by using the same mixture of distributions idea above in Algorithm \ref{alg:effic_haystack_alg}, we can obtain an efficient algorithm that is minimax optimal wrt $\F$ at the cost of a $\log(m)$ factor in the sample complexity. This allows one to do active learning without committing to the bandwidth of the kernel, for example.
% \end{remark}
\textbf{Comparison to prior work:} \cite{amin2011bandits} proposes a \emph{computationally inefficient} algorithm for best arm identification that obtains a sample complexity of $O(\hd(\F_{\calR}) \log(|\F_{\calR}|))$. By Proposition \ref{prop:relate_HD_teach_dim}, $\bestext(\F_{\reg})$ is upper bounded by $\hd(\F_{\calR}) \log(|\F_{\calR}|)$, and thus our sample complexity is loose by a factor of $\log(\frac{1}{\P_{\pi}(S_{f^*})}) \log(n)$. Although our bound is indeed looser than the bound in \cite{amin2011bandits}, we note that computationally efficient algorithms for other active learning problems that match the minimax lower bound up to logarithmic factors have a similar logarithmic dependence on the inverse probability of sampling the true function \cite{tosh2017diameter,tosh2020diameter}. Thus, it is an important open question whether it is possible to develop computationally efficient and nearly minimax optimal algorithms for active learning that weaken or remove the dependence on $\log(\frac{1}{\P_{\pi}(S_{f^*})})$ in their sample complexity.
%While \cite{amin2011bandits} obtain a sample complexity of $\hd(\F_{\calR}) \log(|\F_{\calR}|)$, which has better logarithmic factors than our upper bound, unlike Algorithm \ref{alg:effic_haystack_alg}, their algorithm enumerates the version space and is therefore computationally inefficient.
%\textbf{Lower bounding $\P_\pi(S_{f})$.}
%Next, we lower bound $\P_\pi(\calS_{f})$ in a few simple settings.
We close this section with a simple instance of best arm identification in a linear function class that provides an easy instantiation of our upper bound.
\begin{proposition}\label{prop:conjun_class}
Let $x_1,\ldots, x_n \in \R^{n+1}$ such that $x_i = \frac{i}{n} e_1 + 10\cdot e_{i+1}$. Let $\reg_i = \{r(v) = v_1 - b + v_{i+1} : b \in \frac{1}{n}[i,i+1) \} $ where $v_i$ denotes the $i$th entry of $v$ and $\reg = \cup_{i=1}^n \reg_i$. Define $\Y = \{0,1,10\}$. Let $\pi $ be a uniform distribution over $[0,1]$. Fix $f^* \in \F_{\reg}$. Then, \texttt{GRAILS} returns the best arm in $\bestext(\F_{\reg}) \log(\frac{1}{\P_{\pi}(S_{f^*})})\log(n) \leq O(\log(n)^2)$ samples.
\end{proposition}
By contrast, any non-adaptive algorithm would require $\Omega(n)$ samples for the above function class. Furthermore, we suspect that the additional logarithmic factor stemming from Theorem \ref{thm:haystack_eff} is an artifact of the analysis.
% Since Theorem \ref{thm:haystack_eff} has an additional $\log(n)$ factor, the above Proposition implies that the sample complexity of Algorithm \ref{alg:effic_haystack_alg} scales as $O(\log(n)^2)$, though we suspect that the additional logarithmic factor is an artifact of the analysis. In contrast, any non-adaptive algorithm would require $\Omega(n)$ samples for the above function class.
We discuss other instances in the appendix.
%\begin{proposition}\label{prop:circle_bound}
%Let $\epsilon > 0$. Let $\Y = \Delta \N \cap [-1,1]$. Let $\reg = \{ r: r(\cdot) = \langle a, \cdot \rangle, \norm{a}_2 = 1 \}$. Let $x_{i} = (\cos(2\pi \frac{i-1}{n}), \sin (2\pi \frac{i-1}{n}))^\t$ for $i \in [n]$ with $n \geq 2^j$ for some integer $j \geq 1$. Then, if $\Delta \leq \frac{\epsilon}{4 \sqrt{2}}$, then for any $r_*(\cdot) = \langle a_*, \cdot \rangle$, we have that $\bestextep(\F_{\reg}) \log(\frac{1}{\P_{\pi}(S_{f^*})}) \leq O(\log(|\F_{\reg}|))$.
%\end{proposition}
%
%\begin{proposition}\label{prop:thresholds}
%Fix $\epsilon = 0$. Let $\X = \{1,\ldots, n\}$ and $\reg = \{ r : r(x) = x + b, b \in [0,n+1]\}$. Let $\pi$ be the uniform distribution over $[0,n+1]$. Let $\Y = \{-1,1\}$. Then, $\bestext(\F_{\reg}) \log(\frac{1}{\P_{\pi}(S_{f^*})}) \leq O(\log(n))$.
%\end{proposition}
%\begin{proposition}\label{prop:quadratic_example}
%Let $\X = [n]$. Let $\reg = \{r : r(x) = x^2 - 2ax, a \in [1,n]\}$. Let $\Y = \{-2n-1,-2n,\ldots,n^2-3,n^2-2\}$. Let $\pi$ be the uniform distribution over $[1,n]$. Let $f^* = x^2 - 2ax$ for some $a \in \{1,\ldots, n\}$. Then, $\bestext(\F_{\reg}) \log(\frac{1}{\P_{\pi}(S_{f^*})}) \leq O(\log(n))$.
%\end{proposition}
\section{An Efficient Algorithm for Cumulative Loss Minimization}\label{sec:loss_min}
In this Section, we consider the task of loss minimization. To interpret each $y_i$ as a loss, we assume $\min_{y \in \Y} y \geq 0$. Recall that in this setting when the learner identifies an element $i^\ast \in \argmin_{i \in [n]} f^*(x_i)$, she declares that the game is over.
We stress that the learner can identify an element $i^\ast \in \argmin_{i \in [n]} f^*(x_i)$ by eliminating all $f \in \F$ such that $i^\ast \not\in \arg\min_{i\in [n]}f(x_i)$.
%The goal is to identify $\argmin_{i \in [n]} f^*(x_i)$ while minimizing the loss $\sum_{t=1}^T f^*(x_{I_t})$ incurred where $T$ is the round that the agent identifies $\argmin_{i \in [n]} f^*(x_i)$.
To begin, we introduce the following novel quantity for quantifying the worst-case difficulty of cumulative loss minimization, inspired by the extended teaching dimension:
\begin{align*}
\lossext(\F) :=& \max_{g : \X \mapsto \Y} \min_{I \subset [n]} \min_{\widetilde{I} \subset I : |\widetilde{I}| = |I| -1} \sum_{i \in \widetilde{I}}^{} g(x_{i}) \\
& \text{ s.t. } \exists j \in [n] : \{f \in \F : f(x_i) = g(x_i) \, \forall i \in [I] \} \subset \{f \in \F : j \in \argmin_{i \in [n]} f(x_i) \}.
\end{align*}
We occasionally write $\lossext$ instead of $\lossext(\F)$ when the context leaves no ambiguity. In words, $\lossext$ is the loss that must be incurred (up to the penultimate round) for some scoring function $g: \X \mapsto \Y$ in order to identify the best arm. Next, we show that $\lossext$ is a minimax lower bound.
\begin{theorem}\label{thm:loss_min_lower_bound}
For all $\F \subset \Y^\X$, $\lossext(\F) \leq \lossopt(\F)$.
\end{theorem}
The following Proposition gives an upper bound of $\lossext$ in terms of $\bestext$.
\begin{proposition}\label{prop:relate_regret_td_best_td}
For all $\F \subset \Y^\X$, $\lossext(\F) \leq \bestext(\F) \max_{y \in \Y} y$.
\end{proposition}
In words, this Proposition reflects that one strategy for cumulative loss minimization is to minimize the number of queries to identify the best arm, ignoring the losses.
\subsection{The Algorithm and Upper Bound}
Algorithm \ref{alg:effic_haystack_alg_loss} is similar to Algorithm \ref{alg:effic_haystack_alg}: it is greedy and operates in phases, sampling from the mixture $P_k$ in the $k$th phase due to the nonconvexity of the version space. The main difference is the objective for selecting $I_t$: it queries the arm $I_t$ that for the worst case $y \in \Y$ minimizes the ratio of the loss incurred in round $t$ and the volume of the functions removed under the measure $P_k$. This objective is inspired by the greedy algorithm for weighted set cover \cite{williamson2011design}.
Define $\Smin = \argmin_{S_f : f \in \F_{\calR}, \P_\pi(S_f) > 0} \P_{\pi}( S_f) $. $\Smin$ is the subset of $\F_\reg$ in the partition that has the least nonzero probability under $\pi$. Recall $\reg $ is a set of regression functions.
\begin{algorithm}[t]
\small
$P_1 \longleftarrow \pi$, $\ver_1 \longleftarrow \reg$, $k \longleftarrow 1$, $t_1 = 1$, $O_1 \longleftarrow \emptyset$\;
\For{$t =1,2,\ldots$}{
Let $I_t \in \argmin_{i \in [n] \setminus O_t } \max_{y \in \Y} \frac{y}{\P_{r \sim P_k}(r \in \ver_t(x_i) \cup \ver_t((x_i,y)))}$\;
Query $x_{I_t}$ and observe $y_{I_t}$ and set $O_{t+1} \longleftarrow O_t \cup \{I_t\}$ \;
Let $\ver_{t+1} \longleftarrow \ver_t \setminus (\ver_t(x_{I_t}) \cup \ver_t((x_{I_t},y_{I_t})))$\;
\If{$ \P_{r \sim P_k}(r \in \ver_{t+1}) \leq \frac{1}{2n}$}{
$k \longleftarrow k+1$\;
$t_k \longleftarrow t+1$, $P_{k+1} \longleftarrow \frac{1}{n-|O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k} } \pi_{C_l(O_{t_k})}$\;
}
\If{\text{STOP}($\F_{\reg}$,$O_t$)}{
\Return $\argmin_{i \in O_t} y_i$}
}
\caption{\texttt{GRAILS} for Loss Minimization}
\label{alg:effic_haystack_alg_loss}
\end{algorithm}
\begin{theorem}\label{thm:loss_min_eff}
Fix $\reg$. Algorithm \ref{alg:effic_haystack_alg_loss} identifies $\argmin_{i \in [n]} f^*(x_i)$ after incurring a loss of at most
\begin{align*}
\widetilde{O}(\lossopt(\F_{\reg})) = 2(\lossext(\F_{\reg}) + \max_{i \in [n],f \in \F_{{\reg}}} f(x_i)) \log(n) \log\left(\frac{1}{\P(\Smin)}\right) .
\end{align*}
%$\log(\frac{1}{\P(\Smin)})$ is upper bounded by $ \log(\calN_{\bar{\epsilon}}(\norm{\cdot}_2, \F_x))$, where $\bar{\epsilon}$ is defined as in Theorem \ref{thm:haystack_eff}.
\end{theorem}
The above guarantee is optimal up to a multiplicative factor of $\log(n) \log(\frac{1}{\P(\Smin)})$ and an additive factor of $\max_{i \in [n],f \in \F} f(x_i)$, which is typically of lower order. We also give an algorithm that enumerates the function class and therefore is not efficient when $|\F_{\calR}|$ is exponential in problem-dependent parameters, but has a stronger guarantee. Due to space constraints and its similarity to Algorithm \ref{alg:effic_haystack_alg_loss}, we defer its presentation to the supplementary material and provide the result here.
\begin{theorem}\label{thm:loss_minimization_upper_bound_inefficient}
Fix $\F$. Algorithm \ref{alg:effic_haystack_alg_loss} identifies $\argmin_{i \in [n]} f^*(x_i)$ after incurring a loss of at most
\begin{align*}
\widetilde{O}(\lossopt(\F)) = 2(\lossext(\F) + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F|).
\end{align*}
\end{theorem}
%\blake{Make this a corollary, and remark on the importance of time horizon}
Theorems \ref{thm:loss_min_eff} and \ref{thm:loss_minimization_upper_bound_inefficient} are the first algorithms for cumulative loss minimization that match the minimax lower bound up to logarithmic factors. As a corollary, we obtain results for \emph{regret minimization}, in which the goal is to identify $\argmin_{i \in [n]} f^*(x_i)$ while minimizing the regret $\sum_{t=1}^T f^*(x_{I_t}) - \min_{i \in [n]} f^*(x_i)$ incurred where $T$ is the round that the agent identifies $\argmin_{i \in [n]} f^*(x_i)$.
\begin{remark}
If $\min_j f(x_j) = 0$ for all $f \in \F$, then loss minimization is equivalent to regret minimization. In this case, Theorem \ref{thm:loss_minimization_upper_bound_inefficient} gives nearly optimal minimax bounds for the regret minimization metric. Furthermore, if $\min_j f(x_j) = \min_j f^\prime(x_j) =: \text{opt}$ for all $f,f^\prime \in \F$, then by subtracting $\text{opt}$ from each $f$, we reduce to the setting where $\min_j f(x_j) = 0$ for all $f \in \F$ and obtain a nearly optimal minimax bound.
\end{remark}
To the best of our knowledge, \emph{these are the first regret bounds that match the minimax lower bound up to logarithmic factors for a large and general class of function classes.}
% \begin{remark}
% If $\min_j f(x_j) = 0$ for all $f \in \F$, then loss minimization is equivalent to regret minimization. Then, Theorem \ref{thm:loss_minimization_upper_bound_inefficient} gives a nearly optimal minimax optimal bound for regret minimization. Furthermore, if $\min_j f(x_j) = \min_j f^\prime(x_j) = \text{opt}$ for all $f,f^\prime \in \F$, then by subtracting $\text{opt}$ from each $f$, we reduce to the setting where $\min_j f(x_j) = 0$ for all $f \in \F$ and obtain a nearly optimal minimax optimal bound.
% \end{remark}
\textbf{Comparison to \cite{amin2011bandits}.} Next, we compare our algorithm in Theorem \ref{thm:loss_minimization_upper_bound_inefficient} to the \emph{computationally inefficient} algorithm from \cite{amin2011bandits} for regret minimization in the noiseless setting. Their regret scales as $O(\Delmax \hd(\F) \ln(|\F|))$, ignoring lower order terms. The following Proposition shows that our cumulative regret bound is never worse than theirs by more than a polylogarithmic factor. Let $y_* = \argmin_{y \in \Y} y$ and $\Delmax = \max_{y \in \Y} y - y_*$ and $\Delmin = \min_{y \in \Y : y \neq y_*} y - y_*$.
\begin{proposition}\label{prop:haystack_regret_comp}
Let $\F$ such that $\bestext(\F) \geq 1$.
%Suppose that Algorithm \ref{alg:loss_min} is run for $T_0 \geq c \Delmax \bestext \log(|\F|)$ rounds.
There exists a universal constant $c > 0$ such that for large enough $T_0 $, if Algorithm \ref{alg:loss_min} is run for $T_0$ rounds, the regret of Algorithm \ref{alg:loss_min} is bounded above by
\begin{align*}
c\bestext(\F) \ln(|\F|) \Delmax \leq c\hd(\F) \ln(|\F|)^2 \Delmax.
\end{align*}
\end{proposition}
The above Proposition says that if the horizon is long enough, then the cumulative regret in Theorem \ref{thm:loss_minimization_upper_bound_inefficient} is never that much worse than the cumulative regret of the algorithm in \cite{amin2011bandits}.
On the other hand, there exists an instance where the regret of our Algorithm in Theorem \ref{thm:loss_minimization_upper_bound_inefficient} is significantly better than the regret of the algorithm in \cite{amin2011bandits}.
\begin{proposition}\label{prop:haystack_regret_suboptimal}
For any $\xi > 0$, there exists an instance with $\Delmax \geq \Delmin + \xi$ where (ignoring logarithmic factors) the regret minimization algorithm from \cite{amin2011bandits} obtains a regret of at least $\Delmax n/2$ while the guarantee in Theorem \ref{thm:loss_minimization_upper_bound_inefficient} is $n \Delmin + \Delmax$.
\end{proposition}
Thus, the regret of \cite{amin2011bandits} scales as $\Delmax n$, while the regret of Algorithm \ref{alg:loss_min} scales as $\Delmin n$, and the gap between $\Delmax$ and $\Delmin$ can be made aribtrarily large. \emph{The key advantage of our algorithm over the work of \cite{amin2011bandits} is that their algorithm is an explore-and-then-commit algorithm, ignoring the cost of information gain, whereas our algorithm Algorithm \ref{alg:loss_min} (see Appendix) is cost-aware weighing the tradeoff between information gain and loss incurred.}
%\kevin{Can $\Delta_{\min}$ be made arbitrarily large with $\Delta_{\max} = \Omega(1)$?}
\textbf{Relation to Regret Minimization.} We conclude this section by discussing the relationship between loss minimization and regret minimization.
Our first observation is that any minimax-optimal regret-minimizing algorithm is also a minimax-optimal loss-minimizing algorithm, as shown by the following Proposition.
% This problem is closely related to a regret minimization metric, as defined above. Indeed, given a sufficiently long time horizon $T_0$, any minimax-optimal regret-minimizing algorithm is also a minimax-optimal loss-minimizing algorithm, as shown by the following Proposition.
% \blake{Agreed re Kevin's comment. Also, is it clear that a regret minimization algorithm needs to be identify $\arg\min f^\ast(x_i)$? what about short horizons?}
\begin{proposition}\label{prop:regret_loss_comparison}
Fix $T_0 \in \N$. Let $\text{loss}(\A;f;T_0) = \sum_{t=1 }^{T_0} f(x_{I_t}) $ and $\text{regret}(\A;f;T_0) = \sum_{t=1 }^{T_0} f(x_{I_t}) - \min_{j \in [n]} f(x_j)$ denote the loss and regret incurred by an algorithm $\A$ over $T_0$ rounds. Let $\bar{\A}$ be an algorithm such that $\max_{f \in \F} \text{regret}(\bar{\A};f;T_0) \leq c \min_{\A } \max_{f \in \F} \text{regret}(\A;f;T_0)$. Then,
\begin{align*}
\max_{f \in \F} \text{loss}(\bar{\A};f;T_0) \leq (c+1) \min_{\A} \max_{f \in \F} \text{loss}(\A;f;T_0) .
\end{align*}
\end{proposition}
On the other hand, a minimax optimal algorithm for loss minimization can have regret that is arbtrarily worse than the minimax lower bound, implying that regret minimization subsumes loss minimization.
\begin{proposition}\label{prop:regret_not_loss}
For any $\xi > 0$, there exists $\F$ such that the minimax regret is $1$, but a loss minimizing algorithm obtains a regret of $\xi$.
\end{proposition}
As discussed above, \emph{we provide the first upper bounds for loss minimization that match the minimax lower bound up to logarithmic factors, as well as state-of-the-art results for regret minimization with general function classes.}
\section{Experiments}
\begin{figure*}
\centering
\hfill
\begin{minipage}[b]{0.33\textwidth}
\centering
\includegraphics[width=\linewidth]{./figures/sig_comp_new.png}
\caption{Varying $\sigma$\label{fig:vary_sig}}
\end{minipage}
\hfill
\begin{minipage}[b]{0.335\textwidth}
\centering
\includegraphics[width=\linewidth]{./figures/pairwise_constraints.png}
\caption{Prior Knowledge}
\label{fig:prior_knowledge}
\end{minipage}
\begin{minipage}[b]{0.32\textwidth}
\centering
\includegraphics[width=\linewidth]{./figures/convex.png}
\caption{Convex Functions}
\label{fig:convex_functions}
\end{minipage}
\vspace{-2em}
\end{figure*}
In this Section, we present experiments comparing \texttt{GRAILS} to uniform sampling (\texttt{UNIF}), \texttt{GP-UCB} \citep{srinivas2009gaussian}, and \texttt{OFUL} \citep{abbasi2011improved}. We consider a version of \texttt{GRAILS} that is designed for continuous output spaces, and which we present and analyze in the appendix. For \texttt{OFUL}, we construct the lower confidence bound by solving a constrained optimization problem based on the prior feedback and the function class structure. Our algorithm outperforms the other algorithms and demonstrates the ability to seamlessly incorporate prior knowledge to accelerate learning, and works for function classes ranging from kernel methods to convex functions. While \texttt{OFUL} tends to perform only slightly worse than \texttt{GRAILS}, unlike \texttt{GRAILS}, \emph{it does not have general theoretical guarantees} that can handle the incorporation of constraints given by prior knowledge or specially structured spaces like the set of convex functions, and therefore we consider it a strong, if heuristic, baseline. \emph{Indeed, in the appendix, we show that \texttt{OFUL} is minimax suboptimal for some function classes.} Due to space constraints, we provide additional implementation details in the appendix.
\textbf{Varying $\sigma$.} In Figure~\ref{fig:vary_sig}, we plot the average number of samples necessary to achieve a simple regret of less than $0.01$ with the shaded region showing 1 standard error taken over 36 independent trials.
We generated random 2d functions living in a RBF Reproducing Kernel Hilbert Space (RKHS) with parameter $\sigma$ varying. We evaluate the function at $400$ points taken in a grid of $[0,1]^2$.
%In the case of \texttt{OFUL} we linearize the problem by computing the eigendecompostion of the gram matrix of the points. We provide additional implementation details in the appendix.
As $\sigma$ decreases and the effective dimension increases, the performance of all methods becomes similar to uniform sampling. Conversely, when $\sigma$ is large and the dimension decreases, the active methods improve markedly upon uniform sampling. In all cases, \texttt{GRAILS} matches or slightly exceeds the performance of the baseline methods.
\textbf{Kernel Methods with Prior Knowledge:} In many practical settings, prior knowledge is available to practitioners, e.g., of the form $f^*(x) \geq f^*(x^\prime)$ for some pairs $x,x^\prime \in X$. It is straightforward to incorporate into our algorithm any prior knowledge that can be expressed in terms of a polyhedron in the output space $\R^n$. By contrast, while there are a variety of approaches to incorporate constraints into GP regression methods \cite{swiler2020survey}, it is not immediately clear how to use constrained GP models in adaptive sampling methods.
In this experiment, we use $x_1,\ldots, x_n \in [0,6]$ equally spaced with $n = 250$. We use the Gaussian RBF kernel with $\sigma = 0.075$. For each of the 60 trials, we draw a random function from the RKHS. We vary the number of random pairwise constraints given to \texttt{GRAILS} and \texttt{OFUL}. Figure \ref{fig:prior_knowledge} depicts the average number of samples required to achieve a simple regret of $0.005$ and shows that as \texttt{GRAILS} and \texttt{OFUL} obtain more prior knowledge, their performance improves.
%GRAILS does slightly better than OFUL and has a theoretical guarantee, unlike OFUL.
\textbf{Convex Functions:} In this experiment, the algorithms use the function class $\F$ consisting of all convex functions. We generate $300$ points in an equally spaced grid on the interval $[0,1]$. $f^*(x) = 5(x-x_{\min})^2$ where for each of the $30$ trials $x_{\min}$ is drawn uniformly at random from $[0,1]$. We only consider \texttt{GRAILS}, \texttt{OFUL}, and \texttt{UNIF}. Figure~\ref{fig:convex_functions} depicts the average number of samples that each algorithm uses to obtain an $\epsilon$-accurate solution, showing that \texttt{GRAILS} does slightly better than \texttt{OFUL}, while, as expected, the sample complexity of \texttt{UNIF} blows up as $\epsilon$ decreases.
\section{Conclusion}
Our work leaves many open questions. First, while \texttt{GRAILS} is computationally efficient as evidenced by our experiments, it becomes computationally burdensome as $n$ grows very large. Further work is required to scale \texttt{GRAILS} to the regime where $n$ is extremely large (e.g., $n\approx 10,000$). Second, we have designed computationally efficient algorithms that match the minimax lower bound up to logarithmic factors in the \emph{noiseless setting} and it remains an open question how to achieve near minimax optimality in the noisy setting. Finally, it is an important future direction to develop computationally efficient and near minimax optimal algorithms in the continuous output setting.
\newpage
\begin{ack}
Kevin Jamieson was supported in part by the NSF TRIPODS II grant DMS 2023166. This work was partially supported by AFOSR grant FA9550-18-1-0166.
\end{ack}
\bibliographystyle{unsrt}
\bibliography{refs}
\newpage
\newpage
\appendix
\tableofcontents
\newpage
\section{Suboptimality of Agnostic Algorithms Proofs}
\subsection{Active Classification}
\begin{proof}[Proof of Proposition \ref{thm:agnostic_subopt}]
Let $m \in \N$ and $\ell = m+1$. Note $n = m \ell+m$. Define for all $i \in [m]$
\begin{align*}
f_i(x_j) & = \begin{cases}
1 & j \in \{\ell (i-1) + 1, \ldots, i\ell\} \cup \{m \ell +1, \ldots, m \ell+i\} \\
-1 & \text{o/w}
\end{cases}
\end{align*}
Further, define $f_0(x_i) = -1$ for all $i \in [n]$. Let $\F = \{f_0,f_1, \ldots, f_m\}$. This instance essentially couples disjoint sets and thresholds.
Suppose that $y_i = f_0(x_i)$ for all $i$, so that $f_0 = f^*$ (and note that the realizability assumption holds).
Note that if an algorithm assumes realizability, then it suffices to perform binary search on $x_{ml+1},\ldots, x_{m(l+1)}$ (which is a thresholds instance) and this procedure terminates and outputs $f_0$ after $O(\log(m))$ queries.
Next, we provide a lower bound for any $\delta$-agnostic algorithm wrt active classification on this instance. By corollary \ref{cor:noiseless_classification_lower_bound}, any $\delta$-agnostic algorithm wrt active classification requires
\begin{align*}
\frac{\log(1/\delta)}{\log(n/\delta)} \phi^*
\end{align*}
samples in expectation, so it suffices to lower bound $\phi^*$. Define the alternative instances for all $i \in [m]$.
\begin{align*}
\bar{y}_j^{(i)} = \begin{cases}
1 & \text{ if } j \in \{\ell (i-1) + 1, \ldots, i\ell\} \\
y_j & \text{ if } \text{ otherwise}
\end{cases}
\end{align*}
Note that since $\ell = m+1$, we have that
\begin{align*}
\sum_{i=1}^n \one \{f_j(x_i) \neq \bar{y}_i^{(j)}\} = j < m+1 = \sum_{i=1}^n \one \{f_0(x_i) \neq \bar{y}_i^{(j)}\}
\end{align*}
and thus $\bar{y}^{(j)}$ is an alternative instance. Therefore, we have that
\begin{align*}
\phi^* & = \min_{\lambda} \max_{\tilde{y} \in \{-1,1\}^n: \argmin_{f \in \F} \sum_{i=1}^n \one\{\tilde{y}_i \neq f(x_i)} \frac{1}{\sum_{i=1}^n \lambda_i \one \{y_i \neq \tilde{y}_i \}} \\
& \geq \min_{\lambda} \max_{ i \in [m]} \frac{1}{\sum_{j \in \{\ell (i-1) + 1, \ldots, i\ell\}} \lambda_j } \\
\end{align*}
The minimizing $\lambda$ puts $\frac{1}{\sum_{j \in \{\ell (i-1) + 1, \ldots, i\ell\}} \lambda_j } = C$ for all $i \in [m]$ for some constant $C \in \R$. We also have that
\begin{align*}
1 = \sum_{i=1}^{m \ell} \lambda_i = \sum_{i=1}^m \sum_{j \in \{\ell (i-1) + 1, \ldots, i\ell\}} \lambda_j = \sum_{i=1}^m \frac{1}{C} = \frac{m }{C}
\end{align*}
Therefore, we have shown that $\phi^* \geq m = \Omega(\sqrt{n})$. This completes the proof.
\end{proof}
%\begin{proof}[Proof of Theorem \ref{thm:agnostic_subopt}]
%Let $m \in \N$ and $\ell = m+1$. Define for all $i \in [m]$
%\begin{align*}
%z_i = \{\ell i + 1, \ldots, (i+1)\ell\} \cup \{m \ell +1, \ldots, m \ell+i\}.
%\end{align*}
%Let $\Z = \{\emptyset\} \cup \{z_i : i \in [m]\}$. This instance essentially couples disjoint sets and thresholds. Note $n = m \ell+m$.
%
%Suppose that $\theta = -\bm{1}$, so that $z_0 = \emptyset$ is the best (and note that the realizability assumption holds). For each $z_i$, let $z^{(1)}_i = \{\ell i + 1, \ldots, (i+1)\ell\}$ and $z^{(2)}_i = \{m \ell +1, \ldots, m \ell+i\}$.
%
%Note that if an algorithm assumes realizability, then it suffices to perform binary search on $z_0, z^{(2)}_1, \ldots, z^{(2)}_m$ (which is a thresholds instance) and this procedure terminates and outputs $z_0$ after $O(\log(m))$ queries.
%
%Next, we provide a lower bound for any $\delta$-PAC algorithm wrt noiseless binary classification on this instance. By corollary \ref{cor:noiseless_classification_lower_bound}, any $\delta$-PAC algorithm wrt noiseless binary classification requires
%\begin{align*}
%\frac{\log(1/\delta)}{\log(n/\delta)} \phi^*
%\end{align*}
%samples in expectation. Define the alternative instances for all $i \in [m]$.
%\begin{align*}
%\bar{\theta}_j^{(i)} = \begin{cases}
%1 & \text{ if } j \in \{\ell i + 1, \ldots, (i+1)\ell\} \\
%\theta_j & \text{ if } \text{ otherwise}
%\end{cases}
%\end{align*}
%Note that since $\ell = m+1$, we have that
%\begin{align*}
%( z_i-z_0)^\t \bar{\theta}^{(i)} = \sum_{j \in z^{(1)}_i} \bar{\theta}^{(i)}_j + \sum_{j \in z^{(2)}_i} \bar{\theta}^{(i)}_j \geq \ell - m > 0
%\end{align*}
%and thus $\bar{\theta}^{(i)}$ is an alternative instance. Therefore, we have that
%\begin{align*}
%\phi^* & = \min_{\lambda} \max_{\tilde{\theta} \in \{-1,1\}^n: \argmax_{z} \tilde{\theta}^\t z \neq z_*} \frac{1}{\sum_{i=1}^n \lambda_i \one \{\theta_i \neq \tilde{\theta}_i \}} \\
%& \geq \min_{\lambda} \max_{ i \in [m]} \frac{1}{\sum_{j \in \{\ell i + 1, \ldots, (i+1)\ell\}} \lambda_j } \\
%\end{align*}
%The minimizing $\lambda$ puts $\frac{1}{\sum_{j \in \{\ell i + 1, \ldots, (i+1)\ell\}} \lambda_j } = C$ for all $i \in [m]$ for some constant $C \in \R$. We also have that
%\begin{align*}
%1 = \sum_{i=1}^{m \ell} \lambda_i = \sum_{i=1}^m \sum_{j \in \{\ell i + 1, \ldots, (i+1)\ell\}} \lambda_j = \sum_{i=1}^m \frac{1}{C} = \frac{m }{C}
%\end{align*}
%Therefore, we have shown that $\phi^* \geq m = \Omega(\sqrt{n})$.
%
%\end{proof}
Recall that we say an algorithm $\A$ is $\delta$-agnostic wrt $\F$ if for any $y \in \Y^n$, $\A$ outputs $\argmin_{f \in \F} \sum_{i=1}^n \one\{y_i \neq f(x_i)\}$ with probability at least $1-\delta$.
We define the \emph{Gaussian Active Classification problem} as follows: when arm $i$ is pulled, $y_i + \eta$ where $\eta \sim N(0,1)$ is observed. The goal is to identify $\argmin_{f \in \F} \sum_{i=1}^n \one\{y_i \neq f(x_i)\}$ We call a pair $(y, \F)$ where $y \in \{-1,1\}^n$ an \emph{instance}.
%
%\begin{definition}
%In the \emph{constrained binary combinatorial bandit problem}, $\X = \{e_1,\ldots, e_n\}$, $\Z \subset \{0,1\}^n$, $\nu_i = N(\theta_i, 1)$, and the agent \emph{knows} that $\theta \in \{-1,1\}^n$.
%\end{definition}
\begin{proposition}\label{prop:reduction_lower_bound}
Let $\delta \in (0,\frac{1}{n})$. Fix $\Y = \{-1,1\}$, $\F \subset \Y^\X$, and $y \in \{-1,1\}$. If there exists a $\delta$-agnostic algorithm wrt $\F$ for noiseless active classification that achieves an expected sample complexity of $\tau$ on the instance $(y,\F)$, then there exists a $2 \delta$-agnostic algorithm wrt $\F$ for Gaussian active classification that achieves an expected sample complexity of $c\log(n/\delta) \tau $ samples on the instance $(y, \F)$, where $c >0$ is a universal constant.
\end{proposition}
% \jks{I think I can tighten the $\log(n/\delta)$ factor to $\log(\tau/\delta)$ using the doubling trick.}
\begin{proof}
We perform a reduction by constructing a new algorithm $\A^\prime$ for the Gaussian active classification problem instantiated by $\F$. Define $\A^\prime$ as follows: at round $t$, pull the arm $I_t$ that $\A$ would choose to pull in the noiseless active classification problem $O(\log(n/\delta) )$ times. If the resulting mean is positive, then tell $\A$ that $\widehat{y}_{I_t} = 1$ otherwise tell $\A$ that $\widehat{y}_{I_t} = -1$. If the algorithm $\A$ terminates and outputs an $f \in \F$, then output $f$ and terminate.
Define $f^* := \argmin_{f \in \F} \sum_{i=1}^n \one\{y_i \neq f(x_i)\}$ and let $\widehat{f}$ be the classifier returned by $\A^\prime$. Note that
\begin{align*}
\P( \widehat{f} \neq f^* ) & \leq \P(\{\exists i \in [n] : \widehat{y}_i \neq y_i \} \cup \widehat{f} \neq f^*) \\
& \leq 2 \delta
\end{align*}
using standard subGaussian bounds and the assumption that $\A$ is $\delta$-agnostic. Thus, $\A^\prime$ is $2 \delta$-agnostic algorithm wrt $\F$ for Gaussian active classification.
Let $T$ denote the (random) number of queries taken by $\A^\prime$ on the instance $(y, \F)$. Define the event $\Sigma = \{\widehat{y}_i \neq y_i \forall i \in [n]\}$. We have that
\begin{align*}
\E[T] & = \E[T|\Sigma]\P(\Sigma) + \E[T|\Sigma^c]\P(\Sigma^c) \leq \tau \log(n/\delta) + n\log(n/\delta) \frac{1}{n} \leq 2 \tau \log(n/\delta)
\end{align*}
\end{proof}
%\begin{proposition}\label{prop:reduction_lower_bound}
%Fix $\theta \in \{-1,1\}^n$ and $\Z \subset \{0,1\}^n$. If an algorithm $\A$ solves the noiseless active binary classification problem instantiated by $\theta$ and $\Z$ in $\tau$ samples with probability at least $1-\delta$, then there exists an Algorithm $\A^\prime$ that solves that constrained binary combinatorial bandit with $\theta$ and $\Z$ in $O(\log(n/\delta) \tau)$ samples with probability at least $1-2\delta$.
%\end{proposition}
%\jks{I think I can tighten the $\log(n/\delta)$ factor to $\log(\tau/\delta)$ using the doubling trick.}
%
%\begin{proof}
%Consider an instance of the constrained binary combinatorial bandit with $\theta$ and $\Z$. Define the following algorithm $\A^\prime$: at time $t=1$, pull the arm $I_t$ that $\A$ would choose to pull in the noiseless bernoulli problem $O(\log(n/\delta) )$ times. If the resulting mean is positive, then tell $\A$ that $\theta_{I_t} = 1$ otherwise tell $\A$ that $\theta_{I_t} = -1$. Continue in this way, giving $\A$ a subset of $\theta$ for which $\A^\prime$ has made a decision and asking which arm $\A$ would pull in the noiseless bernoulli problem and then pulling it $O(\log(n/\delta) )$ times. Terminate once $\A$ decides to terminate.
%
%By the union bound, with probability at least $1-\delta$, each $\theta_i$ fed to $\A$ is correctly identified. Since $\A$ terminates in $\tau$ rounds and is correct with probability at least $1-\delta$, $\A^\prime$ requires at most $O(\log(n/\delta) \tau)$ samples and is correct with probability at least $1-2\delta$.
%\end{proof}
%\begin{corollary}
%A lower bound of $L$ for the constrained binary combinatorial bandit implies a lower bound of $\frac{L}{\log(n/\delta) }$ for noiseless active binary classification.
%\end{corollary}
%
%\begin{corollary}\label{cor:pac_reduction}
%Suppose that there exists an algorithm that is $\delta$-PAC wrt any $\theta \in \{-1,1 \}^d$ on the noiseless active binary classification problem and has an expected sample complexity of $\E[\tau_\theta]$. Then, there exists an algorithm that is $\delta$-PAC wrt any $\theta \in \{-1,1\}^d$ on the constrained binary combinatorial bandit problem with expected sample complexity at most $O(\log(n/\delta) \tau_\theta)$.
%\end{corollary}
\begin{corollary}\label{cor:noiseless_classification_lower_bound}
Let $y \in \{-1,1\}^n$, and $\F \subset \{-1,1\}^\X$, and define $f^* := \argmin_{f \in \F} \sum_{i=1}^n \one\{y_i \neq f(x_i)\}$. Let $\A$ be a $\delta$-agnostic algorithm wrt $\F$ on the active classification problem. Suppose $\A$ an expected sample complexity of $\E[\tau_y]$. Then,
\begin{align*}
\E[\tau_y] \geq c\frac{\log(1/4.8 \delta)}{\log(n/\delta)} \phi^*
\end{align*}
where
\begin{align*}
\phi^* =\min_{\lambda} \max_{\tilde{y} \in \{-1,1\}^n: \argmin_{f \in \F} \sum_{i=1}^n \one\{\tilde{y}_i \neq f(x_i)\} \neq f^*} \frac{1}{\sum_{i=1}^n \lambda_i \one \{y_i \neq \tilde{y}_i \}}
\end{align*}
\end{corollary}
\begin{proof}
Since $\A$ is $\delta$-agnostic wrt $\F$ for active classification, by Proposition \ref{prop:reduction_lower_bound}, we have that there exists a $2\delta$-agnostic algorithm $\A^\prime$ wrt $\F$ for Gaussian active classification and that it has expected sample complexity of at most $c\log(n/\delta) \E[\tau_\theta]$. Define $\nu_{\bar{y}_i} = N(\bar{y}_i, 1)$. Using the Transportation Lemma from \cite{kaufmann2016complexity}, since $\A^\prime$ is $2 \delta$-agnostic wrt $\F$, we have that for any $\tilde{y} \in \{-1,1\}^n$ such that $\argmin_{f \in \F} \sum_{i=1}^n \one\{\tilde{y}_i \neq f(x_i)\} \neq f^*$,
\begin{align*}
\sum_{i=1}^n \E[T_i] \kl(\nu_{y_i} , \nu_{\tilde{y}_i} ) \geq \log(\frac{1}{4.8 \delta})
\end{align*}
where $T_i$ denotes the number of times that arm $i$ is pulled. Defining $\lambda_i =\frac{ \E[T_i]}{\sum_j \E[T_j]}$ and noting that
\begin{align*}
\kl(\nu_{y_i} , \nu_{\tilde{y}_i} ) =
\begin{cases}
2 & y_i = y_i \\
0 & \text{otherwise}
\end{cases}
\end{align*}
this implies that
\begin{align*}
\sum_{i=1}^n \E[T_i] \geq \frac{1}{2}\log(1/4.8 \delta)\frac{1}{\sum_{i=1}^n \lambda_i \one \{y_i \neq \tilde{y}_i \}}
\end{align*}
Maximizing over $\tilde{y} \in \{-1,1\}^n$ such that $\argmin_{f \in \F} \sum_{i=1}^n \one\{\tilde{y}_i \neq f(x_i)\} \neq f^*$ and then minimizing over $\lambda \in \simp_n$, we have that
\begin{align*}
c\E[\tau_y] & \log(n/\delta) \\
& \geq \sum_{i=1}^n \E[T_i] \\
& \geq \frac{1}{2} \log(1/4.8 \delta) \min_{\lambda} \max_{\tilde{y} \in \{-1,1\}^n: \argmin_{f \in \F} \sum_{i=1}^n \one\{\tilde{y}_i \neq f(x_i)\} \neq f^*} \frac{1}{\sum_{i=1}^n \lambda_i \one \{y_i \neq \tilde{y}_i \}} .
\end{align*}
Rearranging the above bound give the result.
\end{proof}
% \subsection{agnostic algorithms for regret minimization}
% \begin{theorem}\label{thm:regret_agnostic}
% There exist instances
% \end{theorem}
\subsection{Best Arm Identification}
Recall that for $k \in \N$ and $\delta \in (0,1)$, we say an algorithm $\A$ is $(\delta,k)$-agnostic if for any $y \in \Y^n$ such that $\min_{f \in \F} \sum_{i=1}^n \one\{f(x_i) \neq y_i\} \leq k$, $\A$ identifies $\argmin_{i \in [n]} y_i$ with probability at least $1-\delta$.
We define the \emph{Gaussian Best Arm Identification problem} as follows: when arm $i$ is pulled, $y_i + \eta$ where $\eta \sim N(0,1)$ is observed. The goal is to identify $\argmin_{i \in [n]} y_i$. We call a pair $(y, \F)$ where $y \in \{-1,1\}^n$ an \emph{instance}.
%
%\begin{definition}
%In the \emph{constrained binary combinatorial bandit problem}, $\X = \{e_1,\ldots, e_n\}$, $\Z \subset \{0,1\}^n$, $\nu_i = N(\theta_i, 1)$, and the agent \emph{knows} that $\theta \in \{-1,1\}^n$.
%\end{definition}
\begin{proposition}\label{prop:reduction_lower_bound_best_arm}
Let $\delta \in (0,\frac{1}{n})$. Fix $\Y = \{-1,1\}$, $\F \subset \Y^\X$, and $y \in \{-1,1\}$. Let $\A$ be a $(\delta,k)$-agnostic algorithm wrt $\F$ for noiseless best arm identification that achieves an expected sample complexity of $\tau$ on the instance $(y,\F)$ and has a probability $p_i$ of sampling $i \in [n]$. Then there exists a $(2 \delta,k)$-agnostic algorithm wrt $\F$ for Gaussian best arm identification that achieves an expected sample complexity of $c\log(n/\delta) \tau $ samples on the instance $(y, \F)$, where $c >0$ is a universal constant, and has a probability $p_i$ of sampling $i \in [n]$.
\end{proposition}
\begin{proof}
The proof is identical to the proof of Proposition \ref{prop:reduction_lower_bound}. The probability $p_i$ is the same for Gaussian best arm identification setting by the construction of $\A^\prime$ in the proof of Proposition \ref{prop:reduction_lower_bound}.
\end{proof}
\begin{proof}[Proof of Proposition \ref{thm:best_arm_agnostic_suboptimal}]
Let $\Y = \{w_1,\ldots, w_L\}$ with $w_1 < w_2 < \ldots < w_L$.
%Suppose that $\min_{i,j \in [L]} |w_i - w_j| \geq 1/2$.
Let $\F = \{f^*\}$ with $f^* : \X \mapsto \{w_1,\ldots, w_L\}$ such that for all $i \in [n]$ $f^*(x_i) \in \{w_2,\ldots, w_{L-1}\}$ and $\argmin_{i \in [n]} f^*(x_i) = \{i_* \}$.
%and suppose that $\min_{j \in [n]} f^*(x_j) - f^*(x_{i_*}) \leq 1$.
\textbf{Step 0: there exists an arm that is not queried with constant probability.} Let $\theta \in \{w_1,\ldots, w_L\}^n$ such that $\theta_i = f^*(x_i)$ for all $i \in [n]$. Let $T_i$ denote the number of times that arm $i$ is queried (a random variable). Towards a contradiction, suppose that $\E_\theta[\sum_{i=1}^n T_i] < n/8$. Since
\begin{align*}
n/8 > \E_{\theta}[\sum_{i=1}^n T_i] \geq \min_{i \in n} n \E_\theta [T_i].
\end{align*}
Thus, there exists $i_0 \in [n]$ such that $1/8 > \E_\theta [T_{i_0}] \geq \P_\theta (T_{i_0} > 0)$.
\textbf{Step 1: Reduction}
%Using the argument in Proposition \ref{prop:reduction_lower_bound}, we have that if there exists a $(\delta,1)$ agnostic algorithm with sample complexity $\tau$ for the noiseless problem, then there exists a $(2\delta,1)$ agnostic algorithm with sample complexity $\tau\log(n/\delta)$ for the following problem: if the $i$th example is queried, then $f^*(x_i)+ \eta$ with $\eta \sim N(0,1)$. Thus, it suffices to give a lower bound for the problem with Gaussian noise.
By Proposition \ref{prop:reduction_lower_bound_best_arm}, since $\delta < 1/n$, there exists a $(2 \delta,1)$-agnostic algorithm $\A^\prime$ wrt $\F$ for Gaussian best arm identification that achieves an expected sample complexity of $c\log(n/\delta) \tau $ samples on the instance $(y, \F)$ and the probability of querying $i_0$ is at most $1/8$.
\textbf{Step 2.1: Constructing a bad alternative} Let $\widehat{i}$ denote the arm returned at the end of the game. Suppose $i_0 \not \in \argmin_{j \in [n]} f^*(x_j)$. Then, since $\delta < \frac{1}{40}$ and $\A^\prime$ is $(2\delta,1)$-agnostic wrt $\F$ for Gaussian best arm identification
\begin{align*}
\P_\theta(\widehat{i} \neq i_0 ) \geq 19/20.
\end{align*}
%Let $W = \{i \in [n] : i \in \argmin_{j \in [n]} f^*(x_j)\}$ and note that $|W| \leq n$.
%\begin{align*}
%1-2 \delta \leq \P_\theta(\widehat{i} \in W) \leq \sum_{i \in W} \P_\theta(\widehat{i} = i) \leq |W| \max_{i \in W} \P_\theta(\widehat{i} = i) \leq n \max_{i \in W} \P_\theta(\widehat{i} = i).
%\end{align*}
%Thus, there exists $j_0 \in [n]$ such that
Define $B = \{T_{i_0} = 0\} \cap \{\widehat{i} \neq i_0 \}$. Then,
\begin{align*}
\P_{\theta}( B^c) \leq \P_{\theta}(T_{i_0} > 0) + \P_{\theta}(\widehat{i} = i_0) \leq 1/8 + 1/20 \leq 1/5.
\end{align*}
Define $\tilde{\theta} \in \{w_1,\ldots, w_L\}^n$ by
\begin{align*}
\tilde{\theta}_j & = \begin{cases}
f^*(x_j) & j \neq i_0 \\
w_1 & j = i_0
\end{cases}.
\end{align*}
Note that since by assumption $f^*(x_i) \in \{w_2,\ldots, w_{L-1}\}$ for all $i \in [n]$, under $\tilde{\theta}$, $i_0$ is the unique minimizer.
Define
\begin{align*}
\widehat{\text{kl}}_{i,T_i} = \sum_{s=1}^{T_i} \log(f_{\theta}(Z_s)/f_{\tilde{\theta}}(Z_s))
\end{align*}
where $Z_s$ is the observation on the $s$th pull of arm $i$, $f_{\bar{\theta}}$ denotes the density of the distribution associated with arm $i$ under $\bar{\theta}$, specifically $N(\bar{\theta}_i,1)$. Then, by the change of measure identity (Lemma 18) from \citep{kaufmann2016complexity},
\begin{align*}
\P_{\tilde{\theta}}(\widehat{i} \neq i_0) &\geq \P_{\tilde{\theta}}(B) \\
&= \E_\theta[\one\{B\} \exp(-T_i \widehat{\text{kl}}_{i,T_i})] \\
& = \P_\theta(B) \\
& \geq \frac{4}{5}.
\end{align*}
where we used the fact that the only difference between problem $\theta$ and problem $\tilde{\theta}$ is the mean of $i$th arm and on the event $A$, $T_i = 0$. But, this is a contradiction since under $\tilde{\theta}$, $i_0$ is the minimizer, $\sum_{i=1}^n \one\{\theta_i \neq \tilde{\theta}_i\} = 1$, and the algorithm $\A$ is $(2 \delta,1)$-agnostic with $\delta \in (0,1/20)$.
\textbf{Step 2.2: }
%Let $i_* \in \in \argmin_{j \in [n]} f^*(x_j)$. By assumption, there exists $j_0$ such that $|f^*(x_{i_*}) - f^*(x_{j_0})| \leq 1$. By the transportation Lemma from \cite{kaufmann2016complexity}, we have that
%\begin{align*}
% \E_\theta[T_{i_*}] (f^*(x_{i_*}) - f^*(x_{j_0})^2 \geq \E_\theta[T_{i_*}] (f^*(x_{i_*}) - f^*(x_{j_0})^2 \geq \log(1/4.8\delta) \geq 1
%\end{align*}
%where the last inequality follows since $\delta \leq 1/20$ by assumption. Thus, $i_0 \neq i_*$
Suppose that $i_0 = i_*$. Define $\bar{\theta} \in \{w_1,\ldots, w_L\}^n$ by
\begin{align*}
\bar{\theta}_j & = \begin{cases}
f^*(x_j) & j \neq i_* \\
w_L & j = i_*
\end{cases}.
\end{align*}
Define the events $\Sigma_1 = \{T_{i_*} = 0\}, \Sigma_2 = \{i_* = \widehat{i}\}, \Sigma = \Sigma_1 \cap \Sigma_2$.
%We have by the change-of-measure equality,
%\begin{align*}
%\P_{\bar{\theta}}(\Sigma_1) & = \E_\theta[\one\{\Sigma_1\}\exp(-T_{i_*} \widehat{\text{kl}}_{i,T_{i_*}})] \\
%& = \E_\theta[\one\{\Sigma_1\}] \\
%& \geq 7/8.
%\end{align*}
Note that
\begin{align*}
\P_\theta(\Sigma^c) & \leq \P_\theta(\Sigma^c_1) + \P_\theta(\Sigma^c_2) \\
& = \P_\theta (T_{i_*} > 0) + \P_\theta (\widehat{i}_* \neq \widehat{i})\\
& \leq 1/8 + 2\delta \\
& \leq 1/5
\end{align*}
This implies by a similar argument to the one made in Step 2.1 that
\begin{align*}
\P_{\bar{\theta}}(\Sigma_2) & \geq \P_{\bar{\theta}}(\Sigma)
= \E_\theta[\one\{\Sigma\}\exp(-T_{i_*} \widehat{\text{kl}}_{i,T_{i_*}})] \\
& = \P_{\theta}(\Sigma) \\
& \geq 4/5.
\end{align*}
But, this contradicts the assumption that $\A$ is $(2\delta,1)$-agnostic since $\sum_{i=1}^n \one\{\theta_i \neq \bar{\theta}_i\} = 1$ and $i_*$ is suboptimal in the instance given by $\bar{\theta}$ and $\A$ therefore makes a mistake with probability at least $4/5 > \delta$, a contradiction.
%Now, suppose that $i_0 \in \argmin_{j \in [n]} f^*(x_j)$. Then,
%\begin{align*}
%\P_\theta(\widehat{i} = i_0) \geq 1-2 \delta \geq 3/4.
%\end{align*}
%Define
%\begin{align*}
%C & = \{T_{i_0} > 0\} \cap \{i_0 = i_0 \}.
%\end{align*}
%Then,
%\begin{align*}
%\P_\theta(C^c) & \leq \P_\theta(T_{i_0} > 0) + \P_\theta(\widehat{i} \neq i_0 ) \leq \frac{1}{8} + 3/4 = 7/8.
%\end{align*}
%Define $\bar{\theta} \in \{w_1,\ldots, w_L\}^n$ by
%\begin{align*}
%\bar{\theta}_j & = \begin{cases}
%f^*(x_j) & j \neq i_0 \\
%w_L & j = i_0
%\end{cases}.
%\end{align*}
%Note that since by assumption $f^*(x_i) \in \{w_2,\ldots, w_{L-1}\}$ for all $i \in [n]$, under $\tilde{\theta}$, $i_0$ is not a minimizer. Then, using the same change of measure argument, we have that
%\begin{align*}
%\P_{\bar{\theta}}(\widehat{i} = i_0) \geq 1/8.
%\end{align*}
%But, this once again contradicts the assumption that algorithm $\A$ is $(2 \delta,1)$-Robust with $\delta \in (0,1/16)$.
\end{proof}
\subsection{Regret Minimization}
\begin{proof}[Proof of Proposition~\ref{thm:regret_agnostic}]
Consider an agnostic algorithm $\A$ for noiseless regret minimization. We assume there exist $n$ arms $x_1, \cdots, x_n =: \X$ and an unknown true function $f^\ast$ in a class $\F: \X\rightarrow \R$. In the noiseless regime, we assume that pulling $x_i$ returns $f^\ast(x_i)$. In the noisy regime, we assume that pulling $x_i$ returns $f^\ast(x_i) + \eta$ where $\eta$ is a $1$ sub-Gaussian random variable.
In the noiseless regime,
we assume that $\A$ achieves a bounded regret $R_{\A}(f^\ast, \X) < \infty$ independent of the horizon. This is a mild assumption as noise is not present in the responses. Indeed, the simple algorithm that samples each $x_i$ once and then chooses the $x^\ast$ achieving minimal loss $f(x)$ thereafter has bounded regret. Additionally, we may assume without loss of generality that $\A$ never re-samples a suboptimal arm as its value is known.
\textbf{Step 1}: Reduction from regret minimization in the noiseless regime to the noisy regime.
By \cite{lattimore2017end}, an algorithm $\A'$ for the noisy regime is said to be \emph{consistent} if its regret at time $T$, $R^{\A'}_{f^\ast}(T)$ obeys
\begin{align*}
\limsup_{T\rightarrow\infty} \frac{\log(R^{\A'}_{f^\ast}(T))}{\log(T)} \leq 0.
\end{align*}
We say that $f^\ast$ is $\epsilon$-separated on $x_1, \cdots, x_n$ if $\min_{i,j: f^\ast(x_i) \neq f^\ast(x_j)} |f^\ast(x_i) - f^\ast(x_j)| \geq \epsilon$. Next, we show how to use algorithm $\A$ in the noiseless regime to form a consistent algorithm $\A'$ for $1$-separated instances. For simplicity, we assume that $f^\ast$ takes values over the integers.
Let $T > 0$ denote the time horizon and assume that $\max |f(x_i) - f(x_j)| \leq B$.
At each time $t$, $\A'$
\begin{enumerate}
\item queries $\A$ for an arm $x_i$ to sample next
\item samples $x_i$ $c\log(nT)$ times for sufficiently large constant $c$.
\item Average the losses and round the number to the nearest integer.
\item Pass the rounded average to $\A$.
\end{enumerate}
Next, we argue that $\A'$ is consistent. For sufficiently large $c$ and a union bound over the $n$ arms, with probability $1 - 1/T$, we have that $f^\ast(x_i) = R\left(\frac{1}{c\log(nT)}\sum_{j=1}^{c\log(nT)}f(x_i) +\eta_j\right)$ where $\eta_j$ is the noise on the $j^\text{th}$ sample of $x_i$ and $R(\cdot)$ is a rounding function that rounds to the nearest integer value. In this case, since $\A$ has bounded regret and never re-pulls suboptimal arms, the total regret is $cR_{A}(f^\ast, \X)\log(nT)$. Otherwise, with probability $1/T$, $\A'$ receives regret no worse than $BT$. Hence the total regret is bounded by
\begin{align*}
R^{\A'}_{f^\ast}(T) \leq cR_{A}(f^\ast, \X)\log(nT) + B.
\end{align*}
Therefore,
\begin{align*}
\limsup_{T\rightarrow\infty} \frac{\log(R^{\A'}_{f^\ast}(T))}{\log(T)} & = \limsup_{T\rightarrow\infty} \frac{\log\left(cR_{A}(f^\ast, \X)\log(nT) + B\right)}{\log(T)} \\
& = \limsup_{T\rightarrow\infty} \frac{\log\left(cR_{A}(f^\ast, \X)\log(nT)\right)}{\log(T)}\\
&= 0
\end{align*}
which implies that $\A'$ is consistent.
\textbf{Step 2}: Instance dependent regret bounds for consistent algorithms.
For any consistent algorithm $\A'$, by Corollary 2 of \cite{lattimore2017end}
\begin{align*}
C(f^\ast, \X) \leq \limsup_{n\rightarrow\infty} \frac{R^{\A'}_{f^\ast}(T))}{\log(n)} = \limsup_{n\rightarrow\infty} \frac{cR_{A}(f^\ast, \X)\log(nT) + B}{\log(n)} = cR_{A}(f^\ast, \X)
\end{align*}
where $C(f^\ast, \X)$ is the optimal solution to
\begin{align*}
\min_{\alpha \in (0, \infty]^{\X}}\sum_{x \in \X^{-}} \alpha_i \Delta_i \ \text{ s.t } 1/\alpha_i \leq \Delta_i^2/2 \ \forall x_i \in \X^{-}
\end{align*}
where $\Delta_i := f^\ast(x_i)-\min_{x\in \X}f^\ast(x)$ and $\X^{-} := \{x\in \X: \Delta_x > 0\}$.
Choosing $\alpha_i = 2/\Delta_i^2$ to saturate the bound, we have that
\begin{align*}
C(f^\ast, \X) = \sum_{x \in \X^{-}} \frac{2}{\Delta_i}.
\end{align*}
\textbf{Step 3}: Constructing an instance where $R_{\A}(f^\ast, \X)$ is large.
Fix $n \in \N$ and consider a set of points $\X = \{x_1,\cdots, x_{2n}\}$.
We consider a family of $n$ functions $f_1, \cdots, f_n$ mapping from $\X$ to $\{-1, 0, 1\}$. By definition each $f_i$ is $1$-separated. Define
\begin{align*}
f_i(x_j) = \begin{cases}
1 & \text{ if } j = i\\
0 & \text{ if } j \leq n \text{ and } j \neq i \\
-1 & \text{ if } j \in [2n - i, 2n]
\end{cases}
\end{align*}
% Fix $n \in \N$ and consider a set of points $\X = \{x_1,\cdots, x_{n^2}, x_{n^2+1}, \cdots, x_{n^2 + n}\}$.
% We consider a family of $n$ functions $f_1, \cdots, f_n$ mapping from $\X$ to $\{-1, 0, 1\}$. By definition each $f_i$ is $1$-separated. Define
% \begin{align*}
% f_i(x_j) = \begin{cases}
% 1 & \text{ if } j \in ((i-1)\cdot n , i\cdot n]\\
% -1 & \text{ if } j \in [n^2 + n - i, n^2 + n]\\
% 0 & \text{ otherwise}
% \end{cases}
% \end{align*}
For any $f_i$, the set of suboptimal arms is given by $\X^{-} = [2n]\backslash \{i\}$. Choose $f^\ast \in \{f_1, \cdots, f_n\}$ uniformly. By step 2, we have that $R_{\A}(f^\ast, \X) \gtrsim (n-1)$ since $\Delta_i = O(1)$ where we have used the fact that $\A$ was an agnostic algorithm with bounded regret.
Conversely, a realizability-based algorithm that knows the version space $\{f_1, \cdots, f_n\}$ could perform binary search over $x_{n+1}, \cdots, x_{2n}$ and identify the maximizer of $f^\ast$ in $O(\log(n))$ pulls and achieving regret at most $O(\log(n))$.
\end{proof}
\section{Construction of Sampling Oracles for Efficient Active Learning}\label{sec:sampling_examples}
We now show that non-trivial sampling oracles are available for many useful function classes.
\textbf{Linear Models:} Define $\reg = \{ \langle a, \cdot \rangle : a \in \R^d, \norm{a} \leq B\}$ where $\norm{\cdot}$ is some norm and $B \in \R$. Let $\pi$ denote the uniform distribution on $D := \{a \in \R^d : \norm{a} \leq B\}$. Since $D$ a compact convex set, there exists a polynomial time algorithm to sample from $D$ \cite{lovasz2006fast}.
%Now, define $\bar{D} = D \cap \cap_{j \in [m]} \{a : \langle a, x_{i_j} \rangle \in [b_{i_j}^{(1)}, b_{i_j}^{(2)}] \}$ where $i_1,\ldots, i_m \in [n]$ and $b_{i_j}^{(1)}, b_{i_j}^{(2)} \in \R$.
Now, define $\bar{D} = D \cap \cap_{j \in [m]} \{a : \langle a,w_j \rangle \leq b_j \}$ where $w_j \'in \R^d$ and $b_j \in \R$.
Let $\bar{\pi}$ be the uniform distribution on $\bar{D}$. Note that $D$ has a membership oracle: for a given $x \in \R^d$, one can check whether $x \in D$ by computing $\norm{x}$. Since $\bar{D}$ is a convex body and has a membership oracle, there exists a polynomial time algorithm to sample from $\bar{D}$. Finally, note that $\unif(\bar{D})$ is the desired condition distribution:
\begin{align*}
\P_{z \sim \unif(\bar{D})}(z \in A) & = \frac{\P_{z \sim \unif(D)}(z \in A)}{\P_{z \sim \unif(D)}(z \in \bar{D})}.
\end{align*}
Thus, the sample oracle exists for $\reg$.
\textbf{Kernel Method:} Let $k : \X \times \X \mapsto \R$ be a kernel and let $\reg$ be the associated RKHS. Let $\reg_{B} = \{ f \in \reg : \norm{f}_{\reg} \leq B\}$. Fix $x_1,\ldots, x_n \in \X$. By standard arguments in kernel methods (e.g., Representer Theorem \cite{scholkopf2001generalized}), for every $f \in \reg_B$, there exists $f^\prime \in \reg_B$ such that $f^\prime(\cdot) = \sum_{i=1}^n \alpha_i k(\cdot,x_i)$ and $f^\prime(x_i) = f(x_i)$ for all $i \in [n]$.
%Let
%\begin{align*}
%\bar{\F} & = \spa(\{k(\cdot, x_1), \ldots, k(\cdot, x_n) \}) \\
%\bar{\F}^\perp & = \{ f \in \F : f(x_i) = 0 \, \, \forall i \in [n]\}.
%\end{align*}
%We claim that $\bar{\F}$ and $\bar{\F}^\perp$ are orthogonal. Let $f \in \bar{\F}^\perp$. Then,
%\begin{align*}
%\langle \sum_{i=1}^n \alpha_i k(\cdot, x_i), f \rangle = \sum_{i=1}^n \alpha_i \langle k(\cdot, x_i), f \rangle = \sum_{i=1}^n \alpha_i f(x_i) = 0.
%\end{align*}
Thus, it suffices to consider the function class
\begin{align*}
\tilde{\reg} = \{ \sum_{i=1}^n \alpha_i k(\cdot, x_i) : \norm{\alpha}_{K^{1/2}} = \norm{ \sum_{i=1}^n \alpha_i k(\cdot, x_i)}_{\reg} \leq B \}.
\end{align*}
Thus, it is enough to sample from the uniform distribution on
\begin{align*}
D & = \{\alpha \in \R^n : \norm{\alpha}_{K^{1/2}} \leq B \}
\end{align*}
which reduces to the linear case. Intersecting $D$ with halfspaces is also similar to the linear case.
% Define
%\begin{align*}
%\bar{D} & = D \cap \cap_{j=1}^m \{ \alpha : \sum_{i=1}^n \alpha_i k(x_{i_j}, x_i) \in [b_j^{(1)}, b_j^{(2)}]\}.
%\end{align*}
%This is also convex and thus can be sampled from. Thus, the sampling oracle exists for kernel methods.
%We note that for one-dimensional data, one can apply the above approach to $m$th order Sobolev space:
%\begin{align*}
%H^m[0,1] = \{f : [0,1] \mapsto \R : f^{(s)} \in L^2[0,1] \forall s =0,1,\ldots, m\}.
%\end{align*}
%This gives an approach to do active learning with smoothing splines.
\textbf{Convex Functions:} Let $\reg = \{f : \X \mapsto [-B_1,B_1], f \text{ convex}, \max_{x \in \X, g \in \partial f(x)} \norm{g}_2 \leq B_2 \}$ where $\partial f(x)$ denotes the subdifferential of $f$ at $x$. Let $f \in \reg$. Then, $f(x_i) \geq f(x_j) + g_j^\t (x_i - x_j)$ for all $i,j \in [n]$ where $g_j \in \partial f(x_j)$ with $\norm{g_j}_2 \leq B_2$. Thus, there exists $\widehat{y}_i = f(x_i)$ and $g_i \in \R^d$ with $\norm{g_i}_2 \leq R_2$ for $i=1,\ldots, n$ such that
\begin{align*}
\widehat{y}_i \geq \widehat{y}_j + g_j^\t (x_i - x_j).
\end{align*}
Note that $\bar{f}(x_i) = \max_{i=1,\ldots, n} \widehat{y}_i + g_i^\t(x - x_i)$ is convex and $\bar{f}(x_i) = f(x_i)$ for all $i$ \cite{boyd2004convex}. This shows that it suffices to sample from the set $D:=$
\begin{align*}
\{(\widehat{y}_i)_{i=1}^n (g_i)_{i=1}^n) \in \R^{n(d+1)} : \widehat{y}_i \geq \widehat{y}_j + g_j^\t (x_i - x_j) \wedge \widehat{y}_i \in [-B_1,B_1], \norm{g_i}_2 \leq B_2 \forall i \in [n] \}.
\end{align*}
Note that $D$ has a membership oracle: one can efficiently check whether $x \in D$ by checking whether it satisfies the inequalities defining $D$. Since $D$ is convex and compact, and has a membership oracle, one can construct a polynomial-time algorithm for sampling from it. Intersecting $D$ with halfspaces is similar to the linear case.
%Now, define
%\begin{align*}
%\bar{D} = \{(\widehat{y}_i)_{i=1}^n (g_i)_{i=1}^n) \in \R^{n(d+1)} & : \widehat{y}_i \geq \widehat{y}_j + g_j^\t (x_i - x_j) \\
%& \wedge \widehat{y}_i \in [-B_1,B_1], \norm{g_i}_2 \leq R_2 \forall i \in [n] \wedge \widehat{y}_{i_j} \in [a_j,b_j] \forall j \in [m] \}.
%\end{align*}
%$\bar{D}$ is a convex compact set, so there is a polynomial-time algorithm for sampling from it. Thus, there exists a sampling oracle for convex functions.
Note that one can construct a sampling oracle in a similar way for \textbf{$\alpha$-strongly convex functions}: one need only replace the constraint $\widehat{y}_i \geq \widehat{y}_j + g_j^\t (x_i - x_j)$ with $\widehat{y}_i \geq \widehat{y}_j + g_j^\t (x_i - x_j) + \frac{\alpha}{2} \norm{x_i - x_j}_2^2$.
%\textbf{Union of a small number of Convex Sets:} Let $k = O(n)$. Suppose that $\reg_1,\ldots, \reg_k$ each have sample oracles and let $\reg = \cup_{i=1}^k \reg_i$. Let $\pi_i$ denote the distribution associated with $\reg_i$. Then, we can construct a sampling oracle for $\reg$ by sampling from $\pi = \frac{1}{k} \sum_{i=1}^k \pi_i$.
\section{Best Arm Identification Proofs}
%\begin{proof}[Proof of Theorem \ref{thm:best_arm_lower_bound}]
%Consider an algorithm $\A$ that for all $f^* \in \F$ identifies $\argmin_{i \in [n]} f^*(x_i)$ using at most $C(\F)$ queries. Towards a contradiction, suppose $C(\F) < \bestext$. There exists some $g : \X \mapsto \Y$ achieving the maximum in $\bestext$. Let $I \subset [n]$ be the subset of arms queried by $\A$. Then, by definition of $\bestext$, $\forall j \in [n]$ there exist $f,f^\prime \in \F$ such that $f(x_i) = f^\prime(x_{i}) = g(x_{i})$ for all $i \in I $ such that either $j \not \in \argmin_{i \in [n]} f(x_i)$ or $j \not \in \argmin_{i \in [n]} f^\prime(x_i)$. Thus, $\A$ cannot distinguish between these two functions and determine the minimizer. Thus, if $\A$ outputs $j \in [n]$ after $C(\F) < \bestext$ queries, there exists some $f \in \F$ on which $\A$ would make a mistake, leading to a contradiction.
%\end{proof}
Here we state and prove a strictly more general result than Theorem \ref{thm:best_arm_lower_bound} for $\epsilon$-good arm identification. Theorem \ref{thm:best_arm_lower_bound} follows as a special case. First, we define:
\begin{definition}
$\bestoptep(\F)$ is the smallest integer $q$ such that there exists some algorithm $\A$ such that for every $f^* \in \F$, $\A$ outputs an element of $\{j \in [n] : f^*(x_j) \leq \min_{i \in [n]} f^*(x_i) + \epsilon\}$ after at most $q$ queries.
\end{definition}
\begin{theorem}\label{thm:epsilon_good_arm_lower_bound}
For any $\F \subset \Y^\X$, $\bestextep(\F) \leq \bestoptep(\F)$.
\end{theorem}
\begin{proof}[Proof of Theorem \ref{thm:epsilon_good_arm_lower_bound}]
Consider an algorithm $\A$ that for all $f^* \in \F$ identifies $i_0 \in [n]$ such that $f^*(x_{i_0}) \leq \min_{i \in [n]} f^*(x_i) + \epsilon$ using at most $C(\F)$ queries. Towards a contradiction, suppose $C(\F) < \bestextep$. There exists some $g : \X \mapsto \Y$ achieving the maximum in $\bestext$. Let $I \subset [n]$ be the subset of arms queried by $\A$. Then, by definition of $\bestext$, $\forall j \in [n]$ there exist $f,f^\prime \in \F$ such that $f(x_i) = f^\prime(x_{i}) = g(x_{i})$ for all $i \in I $ and either $f(x_j) > \min_{i \in [n]} f(x_i) + \epsilon$ or $f(x_j) > \min_{i \in [n]} f^\prime(x_i) + \epsilon$. Thus, $\A$ cannot distinguish between these two functions and determine the $\epsilon$-good arm. Therefore, if $\A$ outputs $j \in [n]$ after $C(\F) < \bestext$ queries, there exists some $f \in \F$ on which $\A$ would make a mistake, leading to a contradiction.
\end{proof}
\begin{proof}[Proof of Proposition \ref{prop:relate_HD_teach_dim}]
\textbf{The Lower Bound.} Let $F \subset \F$. Define $g : \X \mapsto \Y$ by
\begin{align*}
g(x_i) & = \argmin_{y \in \Y} | F(x_i) \cup F((x_i,y))|
\end{align*}
By definition of $\bestext$, there exists $\tilde{I} \subset [n]$ such that $|\tilde{I}| \leq \bestext$ and
\begin{align*}
\{f \in F : f(x_i) = g(x_i) \, \forall i \in \tilde{I} \} \subset \{ f \in F : j_0 \in \argmin_{k \in [n]} f(x_k) \}
\end{align*}
Then, defining $I = \tilde{I} \cup \{j_0\}$, we have that
\begin{align*}
F & = \{f \in F : \exists i \in I \text{ s.t. } f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{k \in [n]} f(x_k) \}.
\end{align*}
Therefore,
\begin{align*}
|F| & = | \{f \in F : \exists i \in I \text{ s.t. } f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{k \in [n]} f(x_k) \}| \\
& \leq \sum_{i \in I} | \{f \in F : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{k \in [n]} f(x_k) \}| \\
& \leq (\bestext+1) \max_{i \in I} | \{f \in F : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{k \in [n]} f(x_k) \}| \\
& \leq (\bestext+1) \max_{i \in [n]} | \{f \in F : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{k \in [n]} f(x_k) \}| \\
& = (\bestext+1) \max_{i \in [n]} \argmin_{y \in \Y} | \{f \in F : f(x_i) \neq y \text{ or } i \in \argmin_{k \in [n]} f(x_k) \}| \\
& = (\bestext+1) \max_{i \in [n]} \argmin_{y \in \Y} |F((x_i,y)) \cup F(x_i)|
\end{align*}
Rearranging, we have that
\begin{align*}
\frac{|F|}{\max_{i \in [n]} \argmin_{y \in \Y} |F((x_i,y)) \cup F(x_i)|} \leq \bestext+1.
\end{align*}
Since the above inequality holds for any $F \subset \F$, we have that
\begin{align*}
\hd(F) = \sup_{F \subset \F} \frac{|F|}{\max_{i \in [n]} \argmin_{y \in \Y} |F((x_i,y)) \cup F(x_i)|} \leq \bestext+1
\end{align*}
This completes the lower bound.
\textbf{The Upper Bound.} The proof is essentially identical to the proof from \cite{amin2011bandits}. Let $g : \X \mapsto \Y$ be arbitrary. Consider the algorithm from \cite{amin2011bandits} that sets $\F_1 = \F$, $\F_{t+1} = \F_t \setminus \F_t(x_{I_t}) \cup \F_t((x_{I_t}, g(x_{I_t}))$, and at each round $t$ chooses $I_t \in \argmax_{i \in [n]} \inf_{y \in \Y} \F_t(x_{i}) \cup \F_t((x_{I_t}, y))$. Consider round $t \leq c\hd(\F) \ln(|\F|)$ for some positive constant to be determine later. If $|\F_t| = 0$, then we are done. Thus, suppose that $|\F_t| > 1$. By definition, we have that
\begin{align*}
\hd(\F) \leq \gamma(\F_t) \leq \frac{|\F_t(x_{I_t}) \cup \F_t((x_{I_t}, f^*(x_{I_t}))|}{|\F_t|}.
\end{align*}
Therefore, using $|\F_t| = |\F_{t+1}| +|\F_t(x_{I_t}) \cup \F_t((x_{I_t}, f^*(x_{I_t}))|$, we have that
\begin{align*}
|\F_{t+1}| \leq (1-\hd(\F))|\F_{t}| .
\end{align*}
Unrolling the recurrence, we have that $|\F_{t+1}| \leq (1-\hd(\F))^t |\F|$. Plugging in $t = c\hd(\F) \ln(|\F|)$ for a large enough positive constant $c$ shows that $\F_{t} = \emptyset$. Therefore there trivially exists $j \in [n]$ such that for any $f \in \F$ such that $f(x_i) = g(x_i)$ for all $i \in \{I_1,\ldots, I_t\}$, $j \in \argmin_{k \in [n]} f(x_k)$. Thus, by definition of $\bestext$, we have that
\begin{align*}
\bestext \leq c\hd(\F) \ln(|\F|).
\end{align*}
\end{proof}
%
%
%\begin{proof}[Proof of Theorem \ref{thm:haystack_eff}]
%For the sake of abbreviation, let $\P(\cdot) := \P_{r \sim \pi}(\cdot)$ and $\P_k(\cdot) := \P_{r \sim P_k}(\cdot)$.
%
%\textbf{Step 1:} The Algorithm \ref{alg:effic_haystack_alg} breaks time into phases $[t_1,t_2]$, $[t_2,t_3]$, $\cdots$. In each phase $k$, the algorithm samples functions from a distinct distribution $P_k$. Consider round $k$, let $t \geq t_k$. Define $\theta \in \Y^n$ by
%\begin{align*}
%\theta_i & = \begin{cases}
%y_i & i \in O_t \\
%\argmin_{y \in \Y} \P_{k}(r \in \reg_t(x_i) \cup \reg_t((x_i,y)) & \text{o/w}
%\end{cases}.
%\end{align*}
%By definition of $\bestext$, there exists $\tilde{I} = \{i_1,\ldots, i_{\bestext}\}$ such that there exists $j_0 \in [n]$ such that
%\begin{align*}
% \{ r \in \reg_t : \round{r(x_i)} = \theta_i \, \, \forall i \in \tilde{I} \} \subset \{r \in \reg_t : j_0 \in \argmin_{l \in [n]} \round{r(x_l)} \}
%\end{align*}
%Define $I = \{i_1,\ldots, i_{\bestext}, j_0\}$. Then, we have that
%\begin{align*}
%\reg_t & = \{r \in \reg_t : \exists i \in I \text{ s.t. } i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}.
%\end{align*}
%From this, it follows that
%\begin{align}
%\P_k(\reg_t) & = \P_k (\{r \in \reg_t : \exists i \in I \text{ s.t. } i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
%& \leq \sum_{i \in I} \P_k (\{r \in \reg_t : i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
%& \leq (\bestext+1) \max_{i \in I} \P_k (\{r \in \reg_t : i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
%& = (\bestext+1) \P_k (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_{I_t} \}) \label{eq:efficient_I_t_def}\\
%& \leq (\bestext+1) \P_k (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq y_{I_t} \}) \label{eq:efficient_theta_def} \\
%& = (\bestext+1) \P_k (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \nonumber
%\end{align}
%where line \eqref{eq:efficient_I_t_def} comes from the definition of $I_t$ and the definition of $\theta$, and the line \eqref{eq:efficient_theta_def} comes from the definition of $\theta$. Noticing that
%\begin{align*}
%\reg_t & = (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \cup \reg_t \setminus (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \\
%& = (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \cup \reg_{t+1},
%\end{align*}
%we have that
%\begin{align*}
%\P_k(\reg_{t+1}) & = \P_k(\reg_t) - \P_k(\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \leq (1 - \frac{1}{\bestext+1})\P_k(\reg_t).
%\end{align*}
%Unraveling this recursive statement, we have that
%\begin{align*}
%\P_k(\reg_{t+1}) & \leq (1 - \frac{1}{\bestext+1})^{t-t_k}.
%\end{align*}
%Thus, we see that in phase $k$ at $t = t_k + s_k$ with $s_k := O(\ln(n) \bestext)$, we have that
%\begin{align}
%\P_k(\reg_{t_k+s_k}) \leq \frac{1}{2n}. \label{eq:shrink_objective}
%\end{align}
%
%\textbf{Step 2: Reduction in each version space.} Fix the first round $t$ such that $\P_k(r \in \reg_t) \leq \frac{1}{2n}$. Note that $t_{k+1} = t$. Fix $l_0 \in [n] \setminus O_{t_k}$. Then, we have that
%\begin{align}
%\frac{1}{2n} & \geq \frac{1}{n - |O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k}} \P_{f \sim \pi_{C_l(O_{t_k})}}(\reg_t) \nonumber \\
%& \geq \frac{1}{n} \sum_{l \in [n] \setminus O_{t_k}} \P_{f \sim \pi_{C_l(O_{t_k})}}(\reg_t) \nonumber \\
%& \geq \frac{1}{n} \P_{f \sim \pi_{C_{l_0}(O_{t_k})}}(\reg_t) \nonumber \\
%& = \frac{1}{n} \frac{\P(\reg_t \cap C_{l_0}(O_{t_k}))}{\P(C_{l_0}(O_{t_k}))} \label{eq:best_arm_cond_prob} \\
%& = \frac{1}{n} \frac{\P(\reg_{t_{k+1}} \cap C_{l_0}(O_{t_k}))}{\P(C_{l_0}(O_{t_k}))} \nonumber \\
%& \geq \frac{1}{n} \frac{\P(C_{l_0}(O_{t_{k+1}}))}{\P(C_{l_0}(O_{t_k}))} \label{eq:best_arm_containment}
%\end{align}
%where the equality \eqref{eq:best_arm_cond_prob} follows by the definition of conditional probability and in the inequality \eqref{eq:best_arm_containment} we used that
%\begin{align*}
%C_{l_0}(O_{t_{k+1}}) \subset \reg_t \cap C_{l_0}(O_{t_k}).
%\end{align*}
%The above display can be seen since by inspection of the definition for any $t_k \leq t_{k+1}$ and $l \in [n]$,
%\begin{align*}
%C_{l}(O_{t_{k+1} }) & = \{r \in \reg : \round{r(x_i)} = y_i \forall i \in O_{t_{k+1}}\} \cap \{ \round{r(x_l)} + \epsilon < \round{r(x_i)} \forall i \in O_{t_{k+1}} \} \\
%& \subset \{r \in \reg : \round{r(x_i)} = y_i \forall i \in O_{k}\} \cap \{ \round{r(x_l)} + \epsilon < \round{r(x_i)} \forall i \in O_{t_k} \} \\
%& \subset C_{l}(O_{t_{k} })
%\end{align*}
%and
%\begin{align*}
%C_l(O_{t_{k+1} }) \subset \cup_{j \in [n] \setminus O_{t_{k+1}}} C_j(O_{t_{k+1} }) = \reg_{t_{k+1}}.
%\end{align*}
%Thus, we have that
%\begin{align}
%\frac{\P(C_{l_0}(O_{t_k}))}{2} \geq \P(C_{l_0}(O_{t_{k+1}})). \label{eq:shrink_each_version}
%\end{align}
%
%\textbf{Step 3: Eliminating functions.} Let $f^* \in \F_{\reg}$ such that there exists $r^* \in \R$ such that $f^*(x_i) = \round{r^*(x_i)} = y_i$ for all $i \in [n]$. Thus, using \eqref{eq:shrink_objective} and \eqref{eq:shrink_each_version} in Steps 1 and 2 respectively, we see that after $\bar{t} : =c\bestext \log(\frac{1}{\P(S_{f^*})}) \log(n)$ queries we have that for all $l \in [n]$,
%\begin{align*}
%\P(C_l(O_{t_{\log(\frac{1}{\P(\S_{f^*})}) }}))) < \P(S_{f^*}).
%\end{align*}
%This implies that $S_{f^*} \not \subset C_l(O_{t_{\log(\frac{1}{\P(S_{f^*})}) }}))$ for all $l$. Therefore,
%\begin{align*}
%S_{f^*} \not \subset \cup_{l=1}^n C_l(O_{t_{\log(\frac{1}{\P(S_{f^*})}) }})) = \reg_{\bar{t}}.
%\end{align*}
%Therefore, $f^*$ was kicked out some round prior to $\bar{t}$. Since $\round{r^*(x_i)} = y_i$ for all $i \in [n]$, inspection of Algorithm \ref{alg:effic_haystack_alg} reveals that the only way that $f^*$ could be kicked out is if some $i \in \argmin_{j \in [n]} f^*(x_j)$ were queried. This completes the proof.
%
%
%
%\end{proof}
\begin{proof}[Proof of Theorem \ref{thm:haystack_eff}]
For the sake of abbreviation, let $\P(\cdot) := \P_{r \sim \pi}(\cdot)$ and $\P_k(\cdot) := \P_{r \sim P_k}(\cdot)$.
\textbf{Step 1:} The Algorithm \ref{alg:effic_haystack_alg} breaks time into phases $[t_1,t_2]$, $[t_2,t_3]$, $\cdots$. In each phase $k$, the algorithm samples functions from a distinct distribution $P_k$. Consider phase $k$, let $t \in [t_k,t_{k+1}]$. Define $\theta \in \Y^n$ by
\begin{align*}
\theta_i & = \begin{cases}
y_i & i \in O_t \\
\argmin_{y \in \Y} \P_{k}(r \in \reg_t(x_i) \cup \reg_{t,\epsilon}((x_i,y))) & \text{o/w}
\end{cases}.
\end{align*}
By definition of $\bestextep$, there exists $\tilde{I} = \{i_1,\ldots, i_{\bestextep}\}$ such that there exists $j_0 \in [n]$ satisfying
\begin{align*}
\{ r \in \reg_t : \round{r(x_i)} = \theta_i \, \, \forall i \in \tilde{I} \} \subset \{r \in \reg_t : \round{r(x_{j_0})} \leq \min_{l \in [n]} \round{r(x_l)} + \epsilon \}
\end{align*}
Define $I = \{i_1,\ldots, i_{\bestext}, j_0\}$. Then, we have that
\begin{align*}
\reg_t & = \{r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \leq \min_{l \in [n]} \round{r(x_l)}+ \epsilon \text{ or } \round{r(x_i)} \neq \theta_i \}.
\end{align*}
From this, it follows that
\begin{align}
\P_k(\reg_t) & = \P_k (\{r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \leq \min_{l \in [n]} \round{r(x_l)}+ \epsilon \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& \leq \sum_{i \in I} \P_k (\{r \in \reg_t : \round{r(x_i)} \leq \min_{l \in [n]} \round{r(x_l)}+ \epsilon\text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& \leq (\bestextep+1) \max_{i \in I} \P_k (\{r \in \reg_t : \round{r(x_i)} \leq \min_{l \in [n]} \round{r(x_l)}+ \epsilon \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& = (\bestextep+1) \P_k (\{r \in \reg_t : \round{r(x_{I_t})} \leq \min_{l \in [n]} \round{r(x_l)}+ \epsilon \text{ or } \round{r(x_i)} \neq \theta_{I_t} \}) \label{eq:efficient_I_t_def}\\
& \leq (\bestextep+1) \P_k (\{r \in \reg_t : \round{r(x_{I_t})} \leq \min_{l \in [n]} \round{r(x_l)}+ \epsilon \text{ or } \round{r(x_i)} \neq y_{I_t} \}) \label{eq:efficient_theta_def} \\
& = (\bestextep+1) \P_k (\reg_{t,\epsilon}(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t}))) \nonumber
\end{align}
where line \eqref{eq:efficient_I_t_def} comes from the definition of $I_t$ and the definition of $\theta$, and the line \eqref{eq:efficient_theta_def} comes from the definition of $\theta$. Noticing that
\begin{align*}
\reg_t & = (\reg_{t,\epsilon}(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \cup (\reg_t \setminus (\reg_{t,\epsilon}(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t}))) \\
& = (\reg_{t,\epsilon}(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \cup \reg_{t+1},
\end{align*}
we have that
\begin{align*}
\P_k(\reg_{t+1}) & = \P_k(\reg_t) - \P_k(\reg_{t,\epsilon}(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \leq (1 - \frac{1}{\bestext+1})\P_k(\reg_t).
\end{align*}
Unraveling this recursive statement, we have that
\begin{align*}
\P_k(\reg_{t+1}) & \leq (1 - \frac{1}{\bestextep+1})^{t-t_k}.
\end{align*}
Thus, we see that in phase $k$ at $t = t_k + s_k$ with $s_k := O(\ln(n) \bestext)$, we have that
\begin{align}
\P_k(\reg_{t_k+s_k}) \leq \frac{1}{2n}. \label{eq:shrink_objective}
\end{align}
\textbf{Step 2: Reduction in each version space.} Fix the first round $t$ such that $\P_k(r \in \reg_t) \leq \frac{1}{2n}$. Note that $t_{k+1} = t$. Fix $l_0 \in [n] \setminus O_{t_k}$. Then, we have that
\begin{align}
\frac{1}{2n} & \geq \frac{1}{n - |O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k}} \P_{f \sim \pi_{C_l(O_{t_k})}}(\reg_t) \nonumber \\
& \geq \frac{1}{n} \sum_{l \in [n] \setminus O_{t_k}} \P_{f \sim \pi_{C_l(O_{t_k})}}(\reg_t) \nonumber \\
& \geq \frac{1}{n} \P_{f \sim \pi_{C_{l_0}(O_{t_k})}}(\reg_t) \nonumber \\
& = \frac{1}{n} \frac{\P(\reg_t \cap C_{l_0}(O_{t_k}))}{\P(C_{l_0}(O_{t_k}))} \label{eq:best_arm_cond_prob} \\
& = \frac{1}{n} \frac{\P(\reg_{t_{k+1}} \cap C_{l_0}(O_{t_k}))}{\P(C_{l_0}(O_{t_k}))} \nonumber \\
& \geq \frac{1}{n} \frac{\P(C_{l_0}(O_{t_{k+1}}))}{\P(C_{l_0}(O_{t_k}))} \label{eq:best_arm_containment}
\end{align}
where the equality \eqref{eq:best_arm_cond_prob} follows by the definition of conditional probability and in the inequality \eqref{eq:best_arm_containment} we used that
\begin{align*}
C_{l_0}(O_{t_{k+1}}) \subset \reg_t \cap C_{l_0}(O_{t_k}).
\end{align*}
The above display can be seen since by inspection of the definition for any $t_k \leq t_{k+1}$ and $l \in [n]$,
\begin{align*}
C_{l}(O_{t_{k+1} }) & = \{r \in \reg : \round{r(x_i)} = y_i \forall i \in O_{t_{k+1}}\} \cap \{ \round{r(x_l)} + \epsilon < \round{r(x_i)} \forall i \in O_{t_{k+1}} \} \\
& \subset \{r \in \reg : \round{r(x_i)} = y_i \forall i \in O_{k}\} \cap \{ \round{r(x_l)} + \epsilon < \round{r(x_i)} \forall i \in O_{t_k} \} \\
& \subset C_{l}(O_{t_{k} })
\end{align*}
and
\begin{align*}
C_l(O_{t_{k+1} }) \subset \cup_{j \in [n] \setminus O_{t_{k+1}}} C_j(O_{t_{k+1} }) = \reg_{t_{k+1}}.
\end{align*}
Thus, we have that
\begin{align}
\frac{\P(C_{l_0}(O_{t_k}))}{2} \geq \P(C_{l_0}(O_{t_{k+1}})). \label{eq:shrink_each_version}
\end{align}
\textbf{Step 3: Eliminating functions.} Let $f^* \in \F_{\reg}$ such that there exists $r^* \in \R$ such that $f^*(x_i) = \round{r^*(x_i)} = y_i$ for all $i \in [n]$. Thus, using \eqref{eq:shrink_objective} and \eqref{eq:shrink_each_version} in Steps 1 and 2 respectively, we see that after $\bar{t} : =c\bestext \log(\frac{1}{\P(S_{f^*})}) \log(n)$ queries we have that for all $l \in [n]$,
\begin{align*}
\P(C_l(O_{\bar{t} }))) < \P(S_{f^*}).
\end{align*}
This implies that $S_{f^*} \not \subset C_l(O_{t_{\log(\frac{1}{\P(S_{f^*})}) }}))$ for all $l$. Therefore,
\begin{align*}
S_{f^*} \not \subset \cup_{l=1}^n C_l(O_{\bar{t} })) = \reg_{\bar{t}}.
\end{align*}
Therefore, $f^*$ was kicked out some round prior to $\bar{t}$. Since $\round{r^*(x_i)} = y_i$ for all $i \in [n]$, inspection of Algorithm \ref{alg:effic_haystack_alg} reveals that the only way that $f^*$ could be removed from the version space is there exists some $s \leq \bar{t}$ $I_s$ is queried where $f^*(x_{I_s}) \leq \min_{j \in [n]} f^*(x_j) +\epsilon$. This completes the proof.
\end{proof}
\section{Cumulative Loss Minimization Proofs}
\begin{proof}[Proof of Theorem \ref{thm:loss_min_lower_bound}]
Consider an algorithm $\A$ that after incurring cost at most $C(\F)$ declares that it has identified the minimizer. There exists some $\bar{g} : \X \mapsto \{w_1,\ldots, w_L\}$ achieving the maximum in $\lossext$. Suppose the algorithm queries $i_1,\ldots, i_{t_0}$ and receives feedback $g(x_{i_j})$ and time $t_0$ is the earliest point at which the algorithm can declare that it can identify $j_0 \in [n]$ such that
\begin{align*}
\{f \in \F \text{ s.t. } f(x_i) = \bar{g}(x_i) \forall i \in \{i_1,\ldots, i_{t_0}\} \} \subset \{f \in \F : j_0 \in \argmin_{i \in [n]} f(x_i) \}.
\end{align*}
We claim that there exists $\bar{f}$ such that
\begin{align*}
\{f \in \F \text{ s.t. } f(x_i) = \bar{f}(x_i) \forall i \in \{i_1,\ldots, i_{t_0-1}\} \} \neq \emptyset.
\end{align*}
If not, then the algorithm could terminate at time $t_0-1$, contradicting the definition of $t_0$. Thus, on the instance given by $\bar{f}$, the algorithm suffers
\begin{align*}
& \sum_{s=1}^{t_0-1} \bar{f}(x_{i_s}) \\
& = \sum_{s=1}^{t_0-1} \bar{g}(x_{i_s}) \\
& \geq \min_{I \subset [n]} \min_{\tilde{I} \subset I : |\tilde{I}| = |I| -1} \sum_{i \in \tilde{I}} \bar{g}(x_{i}) \\
& \qquad \text{ s.t. } \exists j \in [n] \text{ s.t. } \{f \in \F \text{ s.t. } f(x_i) = \bar{g}(x_i) \forall i \in [I] \} \subset \{f \in \F : j \in \argmin_{i \in [n]} f(x_i) \} \\
& = \max_{g : \X \mapsto \Y} \min_{I \subset [n]} \min_{\tilde{I} \subset I : |\tilde{I}| = |I| -1} \sum_{i \in \tilde{I}}^{} g(x_{i}) \\
& \qquad \text{ s.t. } \exists j \in [n] \text{ s.t. } \{f \in \F \text{ s.t. } f(x_i) = g(x_i) \forall i \in [I] \} \subset \{f \in \F : j \in \argmin_{i \in [n]} f(x_i) \} \\
& = \lossext
\end{align*}
where we used the definition of $\bar{g}$ as attaining the maximum in the definition of $\lossext$. This completes the proof.
% Then, by definition of $\lossext$, $\forall j \in [n]$ there are two $f,f^\prime$ such that $f(x_i) = f^\prime(x_{i}) = g(x_{i}) \forall i \in I $ such that either $j \not \in \argmin_{i \in [n]} f(x_i)$ or $j \not \in \argmin_{i \in [n]} f^\prime(x_i)$. Thus, $\A$ cannot distinguish between these two functions and determine the minimizer.
% Suppose that an algorithm queries points $x_{i_1},\ldots, x_{i_l}$ and declares that it has identified the maximizer. If
\end{proof}
% We introduce the loss minimization once again. Let $\X$ and let $x_1,\ldots, x_n \in \X$ be a fixed pool with $n$ potentially very large. Let $\F \subset {\{w_1,\ldots, w_L\}}^\X$ be set of functions where $0 \leq w_1 \leq \ldots \leq w_L$. At each round $t$, the learner chooses $I_t \in [n]$ and receives the loss $f^*(x_{I_t})$ where $f^* \in \F$. When the learner identifies an element in $\argmin_{i \in [n]} f^*(x_i)$, she declares that the game is over. The goal is to identify $\argmin_{i \in [n]} f^*(x_i)$ while minimizing the loss
% \begin{align*}
% \sum_{t=1}^T f^*(x_{I_t}).
% \end{align*}
% incurred where $T$ is the round that the agent identifies $\argmin_{i \in [n]} f^*(x_i)$.
Here, we give an inefficient algorithm for cumulative loss minimization, Algorithm \ref{alg:loss_min}.
% Define $\F((x_i,w_l)) = \{f \in \F : f(x_i) \neq w_l\}$ and $\F(x_i) = \{f \in \F : i \in \argmin_{l \in [n]} f(x_l) \}$.
\begin{algorithm}[t]
\small
$\F_1 \longleftarrow \F $ \;
\For{$t =1,2,\ldots,T$}{
Let $I_t \in \argmin_{i \in [n] \setminus \{I_1,\ldots, I_{t-1}\}} \max_{\bar{f} \in \F_t} \frac{\bar{f}(x_i)}{|\F_t(x_i) \cup \F((x_i,\bar{f}(x_i)) |}$\;
Query $x_{I_t}$ and observe $f^*(x_{I_t})$\;
Let $\F_{t+1} \longleftarrow \F_t \setminus \F_t(x_{I_t}) \cup \F((x_{I_t},f^*(x_{I_t}))$
}
\caption{\texttt{GRAILS} for Loss Minimization via Enumeration}
\label{alg:loss_min}
\end{algorithm}
\begin{proof}[Proof of Theorem \ref{thm:loss_minimization_upper_bound_inefficient}]
Consider round $t$. Define the function $g : \{x_1,\ldots, x_n\} \mapsto \Y$ in the following way:
%\begin{align*}
%g(x_i) = \begin{cases}
%\bar{f}(x_i) \text{ s.t. } \bar{f} = \argmin_{f \in \F_t} \max_{f \in \F_t} \frac{f(x_i)-\min_{j \in [n]} f(x_j)}{|\{f^\prime \in \F_t : f^\prime(x_i) \neq f(x_i)\}|} & \text{ if } i \not \in \{I_1,\ldots, I_{t-1}\} \\
%f^*(x_i) & \text{ o/w}
%\end{cases}
%\end{align*}
\begin{align*}
g(x_i) = \begin{cases}
\tilde{f}(x_i) \text{ s.t. } \tilde{f} = \argmax_{\bar{f} \in \F_t} \frac{\bar{f}(x_i)}{|\F_t(x_i) \cup \F((x_i,\bar{f}(x_i)) |} & i \not \in \{I_1,\ldots, I_{t-1}\} \\
f^*(x_i) & \text{ o/w}
\end{cases}
\end{align*}
% By definition of $g$, we have that
% \begin{align*}
% \frac{f^*(x_{I_t})}{| \F_t(x_{I_t}) \cup \F((x_i,f^*(x_{I_t})) |} \leq \frac{g(x_{I_t})}{|\F_t(x_{I_t}) \cup \F((x_i,g(x_{I_t})) |} .
% \end{align*}
Let $\tilde{I} = \{i_1,\ldots, i_{l}\} \subset [n]$ attain the optimum in the following optimization problem:
\begin{align}
\min_{l \in [n], i_1,\ldots, i_l \in [n]} & \sum_{j=1}^{l-1} g(x_{i_j}) \label{eq:loss_min_opt} \\
& \text{ s.t. } \exists j \in [n] \text{ s.t. } \{f \in \F_t : f(x_i) = g(x_i) \forall i \in \{i_1,\ldots, i_l\} \} \\
& \hspace{2.5cm}\subset \{f \in \F_t : j \in \argmin_{i \in [n]} f(x_i) \}
\end{align}
Define $\tilde{\F} = \{ f \in \F_t : f(x_i) = g(x_i) \forall i \in \tilde{I} \}$. If $\tilde{F}$ is empty, set $I = \tilde{I}$. If $\tilde{F}$ is nonempty, then by definition of the above optimization problem, there exists $j_0 \in [n]$ such that for all $f \in \tilde{F}$, $j_0 \in \argmin_{i \in [n]} f(x_i) $. Define $I = \tilde{I} \cup \{j_0\}$.
%Note that by definition of $i_1,\ldots, i_l$ as optimizing
%Let $k_1,\ldots, k_{l+1} $ be a permutation of $I$ such that
%\begin{align*}
%g(x_{k_0}) \leq g(x_{k_1}) \leq g(x_{k_2}) \leq \ldots \leq g(x_{k_{l}}).
%\end{align*}
\textbf{Step 1: Case 1.} Suppose that $|\F_t(x_{i_l}) \cup \F((x_i,g(x_{i_l})) | \leq \frac{|\F_t|}{2} $. Note that
\begin{align*}
\F_t \subset \{ f \in \F_t : \exists i \in I \text{ s.t. } f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} f(x_{j}) \}.
\end{align*}
Then,
\begin{align*}
|\F_t| & \leq |\{ f \in \F_t : \exists i \in I \text{ s.t. } f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} f(x_{j}) \}| \\
& \leq \sum_{i \in I } |\{ f \in \F_t : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} f(x_{j}) \}| \\
& \leq \sum_{i \in I \setminus \{i_l\} } |\{ f \in \F_t : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} f(x_{j}) \}| + \frac{|\F_t|}{2} \\
\end{align*}
which implies that
\begin{align}
\frac{|\F_t|}{2} \leq \sum_{i \in I \setminus \{i_l\} } |\{ f \in \F_t : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} f(x_{j}) \}| . \label{eq:loss_fact}
\end{align}
Then, we have that
\begin{align}
& \hspace{-1cm} \frac{f^*(x_{I_t})}{| \F_t(x_{I_t}) \cup \F((x_i,f^*(x_{I_t})) |} \nonumber \\
& \leq \frac{g(x_{I_t})}{|\F_t(x_{I_t}) \cup \F((x_i,g(x_{I_t})) |} \label{eq:loss_def_g} \\
& \leq \frac{\sum_{i \in I \setminus \{i_l\}} g(x_i)}{\sum_{i \in I \setminus \{i_l\} } |\{ f \in \F_t : f(x_i) \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} f(x_{j}) \}|} \label{eq:loss_math_fact} \\
& \leq 2\frac{\sum_{i \in I \setminus \{i_l\}} g(x_i)}{|\F_t|} \label{eq:loss_case_fact} \\
& \leq \frac{2(\lossext+ \max_{i \in [n],f \in \F_t }f(x))}{|\F_t|} \label{eq:loss_relate_tau}
\end{align}
where \eqref{eq:loss_def_g} follows by the definition of $g$, \eqref{eq:loss_math_fact} follows by definition of $I_t$ and the mathematical fact that given positive numbers $a_1,\ldots, a_k$ and $b_1,\ldots, b_k$
\begin{align*}
\min_{i=1,\ldots,k} \frac{a_i}{b_i} \leq \frac{\sum_{i=1}^k a_i}{\sum_{i=1}^k b_i},
\end{align*}
\eqref{eq:loss_case_fact} follows by \eqref{eq:loss_fact}, and \eqref{eq:loss_relate_tau} follows by
\begin{align*}
\sum_{i \in \tilde{I} \setminus \{l_l\} } g(x_i) \leq \lossext
\end{align*}
by the definition of the optimization problem in \eqref{eq:loss_min_opt} and we used $g(x_{j_0}) \leq \max_{i \in [n],f \in \F_t }f(x_i))$ by definition of $g$. Rearranging, we have that
\begin{align*}
f^*(x_{I_t}) & \leq \frac{2| \F_t(x_{I_t}) \cup \F((x_i,f^*(x_{I_t})) (\lossext+ \max_{i \in [n],f \in \F_t }f(x_i))}{ |\F_t|} \\
& = 2 \frac{(|\F_t| - |\F_{t+1}|) (\lossext+ \max_{i \in [n],f \in \F_t }f(x_i))}{|\F_t|}.
\end{align*}
\textbf{Step 2: Case 2.} On the other hand, suppose that $|\F_t(x_{i_l}) \cup \F((x_i,g(x_{i_l})) | \geq \frac{|\F_t|}{2} $. Then,
\begin{align}
\frac{f^*(x_{I_t})}{| \F_t(x_{I_t}) \cup \F((x_i,f^*(x_{I_t})) |} & \leq \frac{g(x_{I_t})}{|\F_t(x_{I_t}) \cup \F((x_i,g(x_{I_t})) |} \nonumber \\
& \leq \frac{g(x_{i_l})}{|\F_t(x_{i_l}) \cup \F((x_i,g(x_{i_l})) |} \label{eq:loss_def_I_t} \\
& \leq \frac{2 \max_{i \in [n],f \in \F_t }f(x_i)}{|\F_t|} \label{eq:loss_def_g_max}
\end{align}
where \eqref{eq:loss_def_I_t} follows by the definition of $I_t$ and \eqref{eq:loss_def_g_max} follows by the definition of $g$. Then, we have rearranging once again
\begin{align*}
f^*(x_{I_t}) & \leq 2 \frac{2| \F_t(x_{I_t}) \cup \F((x_i,f^*(x_{I_t})) | \max_{i \in [n],f \in \F_t }f(x)}{|\F_t|}
\end{align*}
\textbf{Step 3: Putting it together.} Thus, we have that
\begin{align*}
\sum_{t \geq 1} f^*(x_{I_t}) & \leq 2[\lossext+ \max_{i \in [n],f \in \F_t }f(x)] \sum_{t \geq 1} \frac{(|\F_t| - |\F_{t+1}|) }{|\F_t|} \\
& \leq 2[\lossext+ \max_{i \in [n],f \in \F_t }f(x)] \sum_{t\geq 1} \frac{1 }{|\F_t|} + \frac{1 }{|\F_t|-1} + \ldots + + \frac{1 }{|\F_{t+1}|+1} \\
& \leq c2[\lossext+ \max_{i \in [n],f \in \F_t }f(x)] \ln(|\F|)
\end{align*}
where we used that $\frac{1 }{|\F_t|} \leq \frac{1 }{i}$ for $i \leq |\F_t|$ and that $\sum_{l=1}^k \frac{1}{l} = \Theta(\ln(k))$.
\end{proof}
\begin{proof}[Proof of Theorem \ref{thm:loss_min_eff}]
For the sake of abbreviation, let $\P(\cdot) := \P_{r \sim \pi}(\cdot)$ and $\P_k(\cdot) := \P_{r \sim P_k}(\cdot)$.
\textbf{Step 1: Bounding the loss in each phase}
Consider round $t$ in phase $k$. Define the function $g : \{x_1,\ldots, x_n\} \mapsto \Y$ in the following way:
\begin{align*}
g(x_i) = \begin{cases}
\round{\tilde{r}(x_i)} \text{ s.t. } \tilde{r} = \argmax_{\bar{r} \in \reg_t} \frac{\bar{r}(x_i)}{\P_k(\reg_t(x_i) \cup \reg((x_i,\bar{f}(x_i)) )} & i \not \in \{I_1,\ldots, I_{t-1}\} \\
f^*(x_i) & \text{ o/w}
\end{cases}
\end{align*}
%By definition of $g$, we have that
%\begin{align*}
%\frac{f^*(x_{I_t})}{\P_k( \reg_t(x_{I_t}) \cup \reg((x_i,f^*(x_{I_t})) )} \leq \frac{g(x_{I_t})}{\P_k(\reg_t(x_{I_t}) \cup \reg((x_i,g(x_{I_t})) )} .
%\end{align*}
Let $\tilde{I} = \{i_1,\ldots, i_{l}\} \subset [n]$ attain the optimum in the following optimization problem:
\begin{align*}
\min_{l \in [n], i_1,\ldots, i_l \in [n]} & \sum_{j=1}^{l-1} g(x_{i_j}) \\
& \text{ s.t. } \exists j \in [n] : \{ r \in \reg_t : \round{r}(x_i) = g(x_i) \, \forall i \in \{i_1,\ldots, i_l\} \} \\
& \hspace{2cm} \subset \{r \in \R_t : j \in \argmin_{i \in [n]} \round{r(x_i)} \}
\end{align*}
Define $\tilde{\reg} = \{ r \in \reg_t : \round{r(x_i)} = g(x_i) \forall i \in \tilde{I} \}$. If $\tilde{\reg}$ is empty, set $I = \tilde{I}$. If $\tilde{\reg}$ is nonempty, then by definition of the above optimization problem, there exists $j_0 \in [n]$ such that for all $r \in \tilde{\reg}$, $j_0 \in \argmin_{i \in [n]} \round{r(x_i)} $. Define $I = \tilde{I} \cup \{j_0\}$.
%Let $k_1,\ldots, k_{l+1} $ be a permutation of $I$ such that
%\begin{align*}
%g(x_{k_0}) \leq g(x_{k_1}) \leq g(x_{k_2}) \leq \ldots \leq g(x_{k_{l}}).
%\end{align*}
\textbf{Step 2.1: Case 1.} Suppose that $\P_k(\reg_t(x_{i_l}) \cup \reg((x_i,g(x_{i_l})) ) \leq \frac{\P_k(\reg_t)}{2} $. Note that
\begin{align*}
\reg_t \subset \{ r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \}.
\end{align*}
Then,
\begin{align*}
\P_k(\reg_t) & \leq \P_k(\{ r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \}) \\
& \leq \sum_{i \in I } \P_k(\{ r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \}) \\
& \leq \sum_{i \in I \setminus \{i_l\} } \P_k(\{ r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \}) \\
& + \frac{\P_k(\reg_t)}{2} \\
\end{align*}
which implies that
\begin{align*}
\frac{\P_k(\reg_t)}{2} \leq \sum_{i \in I \setminus \{i_l\} } \P_k(\{ r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \}) .
\end{align*}
Then, we have that
\begin{align}
\hspace{1cm} & \frac{f^*(x_{I_t})}{\P_k( \reg_t(x_{I_t}) \cup \reg((x_i,f^*(x_{I_t})) )} \\
& \leq \frac{g(x_{I_t})}{\P_k(\reg_t(x_{I_t}) \cup \reg((x_i,g(x_{I_t})) )} \label{eq:loss_min_eff_diff_line_1} \\
& \leq \frac{\sum_{i \in I \setminus \{i_l\}} g(x_i)}{\sum_{i \in I \setminus \{i_l\} } \P_k(\{ r \in \reg_t : \exists i \in l \text{ s.t. } \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \})} \nonumber \\
& \leq 2\frac{\sum_{i \in I \setminus \{i_l\}} g(x_i)}{\P_k(\reg_t)} \nonumber \\
& = 2\frac{\sum_{i \in \tilde{I} \setminus \{i_l\} \cup \{j_0\}} g(x_i)}{\P_k(\reg_t)} \nonumber \\
& \leq \frac{2(\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)})}{\P_k(\reg_t)} \nonumber
\end{align}
where the above series of inequalities follows by the same arguments used to show the inequalities \eqref{eq:loss_def_g} to \eqref{eq:loss_relate_tau}
%where we used in the last inequality we used $ \sum_{i \in \tilde{I} \setminus \{k_l\} } g(x_i) \leq \sum_{j=1}^{l-1} g(x_{i_j}) \leq \lossext$ by the definition of $\tilde{I}$ and $\{k_l\}$ and $g(x_{j_0}) \leq \max_{i \in [n],r \in \reg_t }f(x))$ by definition of $g$.
Rearranging, we have that
\begin{align*}
f^*(x_{I_t}) & \leq \frac{2 \P_k( \reg_t(x_{I_t}) \cup \reg((x_i,f^*(x_{I_t})) ) [\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}]}{\P_k(\reg_t)} \\
& = 2 \frac{(\P_k(\reg_t) - \P_k(\reg_{t+1})) [\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}]}{\P_k(\reg_t)}.
\end{align*}
\textbf{Step 2.2: Case 2.} On the other hand, suppose that $\P_k(\reg_t(x_{i_l}) \cup \reg((x_i,g(x_{i_l})) ) \geq \frac{\P_k(\reg_t)}{2} $. Then,
\begin{align}
\frac{f^*(x_{I_t})}{\P_k( \reg_t(x_{I_t}) \cup \reg((x_i,f^*(x_{I_t})) )} & \leq \frac{g(x_{I_t})}{\P_k(\reg_t(x_{I_t}) \cup \reg((x_i,g(x_{I_t})) )} \label{eq:loss_min_eff_diff_line_2} \\
& \leq \frac{g(x_{i_l})}{\P_k(\reg_t(x_{i_l}) \cup \reg((x_i,g(x_{i_l})) )} \nonumber \\
& \leq \frac{2 \max_{i \in [n],r \in \reg_t }\round{r(x_i)}}{\P_k(\reg_t)} \nonumber
\end{align}
where the above series of inequalities follows by the same arguments used to show \eqref{eq:loss_def_I_t} to \eqref{eq:loss_def_g}. These imply
\begin{align*}
f^*(x_{I_t}) & \leq 2 \frac{ \P_k( \reg_t(x_{I_t}) \cup \reg((x_i,f^*(x_{I_t})) ) \max_{i \in [n],r \in \reg_t }\round{r(x_i)}}{\P_k(\reg_t)}
\end{align*}
Thus, we have that
\begin{align*}
\hspace{2cm} &\sum_{t \geq t_k : \P_k(\reg_t) \geq 1/2n} f^*(x_{I_t}) \\
& \leq 2[\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}] \sum_{t \geq t_k : \P_k(\reg_t) \geq 1/2n } \frac{\P_k(\reg_t) - \P_k(\reg_{t+1}) }{\P_k(\reg_t)} \\
& = 2[\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}] \sum_{t \geq t_k : \P_k(\reg_t) \geq 1/2n } \frac{2n(\P_k(\reg_t) - \P_k(\reg_{t+1})) }{2n \P_k(\reg_t)} \\
& \leq 2[\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}] \\
& \cdot [1+\sum_{t \geq t_k : \P_k(\reg_{t+1}) \geq 1/2n } \frac{1 }{2n \P_k(\reg_t)} + \frac{1 }{2n \P_k(\reg_t)-1} + \ldots + \frac{1 }{2n \P_k(\reg_{t+1})+1}] \\
& \leq c \ln(n) [\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}]
\end{align*}
where we used that $\frac{1 }{2n \P_k(\reg_t)} \leq \frac{1 }{i}$ for $i \leq 2n \P_k(\reg_t)$ and that $\sum_{l=1}^k \frac{1}{l} = \Theta(\ln(k))$. This bounds the cumulative loss incurred in round $k$.
\textbf{Step 2: Finish the proof.} We have that
\begin{align}
\frac{\P(C_{l_0}(O_{t_k}))}{2} \geq \P(C_{l_0}(O_{t_{k+1}}))
\end{align}
by the same argument used in the proof of Theorem \ref{thm:haystack_eff}. Thus, the loss incurred by the algorithm is at most
\begin{align*}
\ln(n) \ln(\frac{1}{\P(\Smin)}) [\lossext+ \max_{i \in [n],r \in \reg_t }\round{r(x_i)}] .
\end{align*}
%We can upper bound $\frac{1}{\P(\Smin)}$ by the covering number using the same argument from step 4 of Theorem \ref{thm:haystack_eff}.
\end{proof}
\subsection{Other Cumulative Loss Minimization Results}
\begin{proof}[Proof of Proposition \ref{prop:haystack_regret_comp}]
Fix $f^* \in \F$. Let $T_0 = n$. There exists a round $\bar{T} \leq T_0$ at which Algorithm \ref{alg:loss_min} identifies $i_* \in \min_{j \in [n]} f^*(x_j)$, at which point it only plays $i_*$. Note that since the algorithm is deterministic and there is no noise $\bar{T}$ is deterministic. We have that
\begin{align*}
\sum_{t=1}^{T_0} (f^*(x_{I_t}) - \min_{j \in [n]} f^*(x_j)) & = \sum_{t=1}^{\bar{T}} f^*(x_{I_t}) - \bar{T} \min_{j \in [n]} f^*(x_j)
\end{align*}
since no regret is incurred for $t > \bar{T}$.
\textbf{Case 1:} Suppose that $\bar{T} \geq 4 \bestext \ln(|\F|)$. First, we note that by Theorem \ref{thm:loss_minimization_upper_bound_inefficient}, we have that
\begin{align}
\sum_{t=1}^{\bar{T}} f^*(x_{I_t}) & \leq 2(\lossext + \max_{i \in [n], f \in \F} f^*(x_i)) \ln(|\F|) \\
& \leq 2(\bestext \max_{y \in \Y} y + \max_{i \in [n], f \in \F} f^*(x_i)) \ln(|\F|) \label{eq:regret_extended_teaching_bnd} \\
& \leq 4 \bestext \ln(|\F|)\max_{y \in \Y} y \label{eq:regret_extended_teaching_above_one}
\end{align}
where \eqref{eq:regret_extended_teaching_bnd} follows since $\lossext \leq \bestext \max_{y \in \Y} y $ by Proposition \ref{prop:relate_regret_td_best_td}, and \eqref{eq:regret_extended_teaching_above_one} follows since $\bestext \geq 1$ by assumption on $\F_{\calR}$. Since $\bar{T} \geq 4 \bestext \ln(|\F_{\calR}|)$ and $\min_{y \in \Y} y \geq 0$, we have that
\begin{align*}
\sum_{t=1}^{\bar{T}} f^*(x_{I_t}) - \bar{T} \min_{j \in [n]} f^*(x_j) & \leq 4 \bestext \ln(|\F_{\calR}|) \max_{y \in \Y} y - \bar{T} \min_{j \in [n]} f^*(x_j) \\
& \leq 4 \bestext \ln(|\F|)\max_{y \in \Y} y - 4 \bestext \ln(|\F|) \min_{y \in \Y} y \\
& = 4 \bestext \ln(|\F_{\calR}|) \Delmax.
\end{align*}
\textbf{Case 2:} Now, suppose that $\bar{T} \leq 4 \bestext \ln(|\F_{\calR}|)$. Then,
\begin{align*}
\sum_{t=1}^{\bar{T}} f^*(x_{I_t}) - \bar{T} \min_{j \in [n]} f^*(x_j) & \leq \bar{T}(\max_{y \in \Y} y - \min_{j \in [n]} f^*(x_j) ) \\
& \leq \bar{T}(\max_{y \in \Y} y - \min_{y \in \Y} y ) \\
& \leq 4 \bestext \ln(|\F|) \Delmax \\
\end{align*}
%
%Define
%\begin{align*}
%\lossext_{\min} :=& \max_{g : \X \mapsto \Y} \min_{l \in [n], i_1,\ldots, i_l \in [n]} \sum_{j=1}^{l-1} g(x_{i_j}) - \min_{y \in \Y} y \\
%& \text{ s.t. } \exists j \in [n] \text{ s.t. } f \in \F \text{ s.t. } f(x_i) = g(x_i) \forall i \in [I]
% \Longrightarrow j \in \argmin_{i \in [n]} f(x_i)
%\end{align*}
%It is not hard to see that
%\begin{align*}
%\sum_{t=1 }^{T_0} f(x_{I_t}) \leq \sum_{t=1 }^{T} f(x_{I_t}) + (T_0-T_{f}) \min_{j \in [n]} f(x_j) \leq 2(\lossext_{\min} + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) +(T_0-T_{f}) \min_{j \in [n]} f(x_j)
%\end{align*}
%
%\begin{align*}
%\text{regret}(\A;f) & = \sum_{t=1 }^{T_0} f(x_{I_t}) - \min_{j \in [n]} f(x_j) \\
%& \leq 2(\lossext_{\min} + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) +(T_0-T_{f}) \min_{j \in [n]} f(x_j)- T_0 \min_{j \in [n]} f(x_j) \\
%& \leq 2(\lossext_{\min} + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) +T_{f} \min_{j \in [n]} f(x_j) \\
%& \leq 2(\tau_* + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) - T_0 \min_{j \in [n]} f(x_j) \\
%& \leq 2(\tau_* + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) - T_0 \min_{y \in \Y} y \\
%& \leq 2( \max_{y \in \Y} y (\bestext+1) \log(|\F_{\calR}|)- T_0 \min_{y \in \Y} y \\
%& \leq 2( \max_{y \in \Y} y (\bestext+1) \log(|\F_{\calR}|)- 2(\bestext-1) \ln(|\F_{\calR}|) \min_{y \in \Y} y \\
%& = \Delmax2 (\bestext+1) \log(|\F_{\calR}|)
%\end{align*}
%
%\begin{align*}
%\text{regret}(\A;f) & = \sum_{t=1 }^{T_0} f(x_{I_t}) - \min_{j \in [n]} f(x_j) \\
%& \leq 2(\tau_* + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) - T_0 \min_{j \in [n]} f(x_j) \\
%& \leq 2(\tau_* + \max_{i \in [n],f \in \F} f(x_i)) \log(|\F_{\calR}|) - T_0 \min_{y \in \Y} y \\
%& \leq 2( \max_{y \in \Y} y (\bestext+1) \log(|\F_{\calR}|)- T_0 \min_{y \in \Y} y \\
%& \leq 2( \max_{y \in \Y} y (\bestext+1) \log(|\F_{\calR}|)- 2(\bestext-1) \ln(|\F_{\calR}|) \min_{y \in \Y} y \\
%& = \Delmax2 (\bestext+1) \log(|\F_{\calR}|)
%\end{align*}
\end{proof}
\begin{proof}[Proof of Proposition \ref{prop:haystack_regret_suboptimal}]
Let $\bar{y} > \tilde{y} > y_*$. We have $\Delmin = \tilde{y}-y_*$ and $\Delmax = \bar{y}-y_*$. Let $m$ be even and $n = m+m/2$. Let $\F = \{f_1,\ldots, f_m \}$ where
\begin{align*}
f_j(x_i) = \begin{cases}
\bar{y} & i \in [m/2] \setminus \{2j -1,2j\} \\
y_* & i \in \{2j -1,2j\} \\
\tilde{y} & i \in ([m+m/2] \setminus [m/2] )\setminus \{m/2 + j\} \\
y_* & i =m/2 + j
\end{cases}
\end{align*}
The greedy algorithm in the regret minimization algorithm from \cite{amin2011bandits} finds the best arm using a greedy best arm identification algorithm that greedily removes functions from the version space. Once if has found the best arm, it pulls the best arm for the remainder of the game. Each query in $[m/2]$ removes $2$ functions while each query in $[m+m/2]\setminus [m/2]$ removes $1$ function. Thus, the regret minimization algorithm from \cite{amin2011bandits} would use the queries in $[m/2]$ to remove functions, incurring a regret of $\Delmax$ for each query. In the worst case, it would incur a regret of $\Omega(\Delmax m) = \Omega(\Delmax n)$. On the other hand, Algorithm \ref{alg:loss_min} would query only in $[m+m/2]\setminus [m/2]$ if $\Delmax >> \Delmin$, incurring a regret of $O(m \Delmin) = O(n \Delmin)$.
\end{proof}
\begin{proof}[Proof of Proposition \ref{prop:regret_loss_comparison}]
\begin{align*}
\max_{f \in \F} \text{loss}(\bar{\A};f;T_0) & = \max_{f \in \F} \sum_{t=1 }^{T_0} f(x_{I_t}) \\
& = \max_{f \in \F} \sum_{t=1 }^{T_0} (f(x_{I_t}) - \min_{j \in [n]} f(x_j)) + {T_0} \min_{j \in [n]} f(x_j) \\
& = \max_{f \in \F} \text{regret}(\A;f;T_0)+ {T_0} \min_{j \in [n]} f(x_j) \\
& \leq \max_{f \in \F} \text{regret}(\A;f;T_0)+ \max_{f \in \F} {T_0} \min_{j \in [n]} f(x_j) \\
& \leq c \min_{\A } \max_{f \in \F} \text{regret}(\A;f;T_0) + \max_{f \in \F} {T_0} \min_{j \in [n]} f(x_j) \\
& \leq (c+1) \min_{\A } \max_{f \in \F} \text{loss}(\A;f;T_0)
\end{align*}
where in the last line we used that $ \text{regret}(\A;f;T_0) \leq \text{loss}(\A;f;T_0)$ since $\min_{y \in \Y} y \geq 0$ by assumption, which implies that $\min_{\A } \max_{f \in \F} \text{regret}(\A;f;T_0) \leq \min_{\A } \max_{f \in \F} \text{loss}(\A;f;T_0)$, and $\max_{f \in \F} {T_0} \min_{j \in [n]} f(x_j) \leq \min_{\A } \max_{f \in \F} \text{loss}(\A;f;T_0) $.
\end{proof}
\begin{proof}[Proof of Proposition \ref{prop:relate_regret_td_best_td}]
Fix $g : \X \mapsto \Y$. By definition of $\bestext$, there exists $\bar{I} \subset [n]$ such that $|\bar{I}| \leq \bestext$ and there exists $j_0 \in [n]$ such that
%for all $f \in \F$ such that $f(x_i)=g(x_i)$ for all $i \in \bar{I}$, $j_0 \in \argmin_{k \in [n]} f(x_k)$.
\begin{align*}
\{ f\in \F : f(x_i) = g(x_i) \, \forall i \in \bar{I}\} \subset \{f \in \F : j_0 \in \argmin_{k \in [n]} f(x_k)\}
\end{align*}
This immediately implies that
\begin{align*}
\min_{l \in [n], i_1,\ldots, i_l} & \sum_{j=1}^{l-1} g(x_{i_j}) \\
& \text{ s.t. } \exists j \in [n] \text{ s.t. } \{ f \in \F : f(x_i) = g(x_i) \forall i \in \{i_1,\ldots, i_l\} \\
& \hspace{2.5cm} \subset \{f \in \F : j \in \argmin_{i \in [n]} f(x_i) \} \\
& \leq \sum_{i \in \bar{I}} g(x_i) \\
& \leq |\bar{I}| \max_{y \in \Y} y \\
& \leq \bestext \max_{y \in \Y} y
\end{align*}
\end{proof}
\begin{proof}[Proof of Proposition \ref{prop:regret_not_loss}]
Suppose that $n = 2$ and $\F = \{f_1,f_2\}$ such that $f_1(x_1) = \Delta$, $f_1(x_2) = \Delta-1$, $f_2(x_1) = 1$, and $f_2(x_2) = \Delta/2+1$. Suppose $f_1$ is true. If one queries $x_1$, then one obtains a regret of $1$ and a loss of $\Delta$. If one queries $x_2$, one obtains a regret of $0$ and a loss of $\Delta-1$. Now, suppose $f_2$ is true. If one queries $x_1$, then one obtains a regret of $0$ and a loss of $1$. If one queries $x_2$, one obtains a regret of $\Delta/2$ and a loss of $\Delta/2+1$. Thus, to minimize worst-case regret, one must query $x_1$, obtaining a worst case regret of $1$. To minimize worst-case loss, one must query $x_2$, obtaining a worst-case loss of $\Delta-1$. But, querying $x_2$ has a regret of $\Delta/2$, completing the proof.
\end{proof}
\section{Extension to Continuous Output Space}
Now, we turn to the continuous output setting where we assume that the scores are continuous, that is, $y_1,\ldots, y_n \in [-1,1]$. Let $\F \subset [-1,1]^\X$ and suppose that there exists $f^* \in \F$ such that $f^*(x_i) = y_i$ for all $i =1,\ldots, n$. Let $\epsilon > 0$. The goal is to identify $j \in [n]$ such that $f^*(x_j) \leq \min_{i \in [n]} f^*(x_i) + \epsilon$.
Let $\delta > 0$ and $L_i= [-1 + (i-1) \delta,-1 + i \delta)$ for $i =1,2,\ldots, \frac{2}{\delta} =: p$. $L_1,\ldots, L_p$ is a grid on the output space $[-1,1]$. For $\tilde{y} \in [-1,1]$, let $L(\tilde{y})$ denote the interval $L_j$ such that $\tilde{y} \in L_j$. Furthermore, define $\bar{L}_j := \frac{1}{2}(-1 +(j-1) \delta + (-1 + j\delta) )$, the midpoint of the $L_j$ interval. Define
\begin{align*}
\F((x_i,L_j )) & = \{f \in \F : f(x_i) \not \in L_j \} \\
\F(x_i) & = \{f \in \F : f(x_i) \leq \min_{j \in [n]} f(x_j) + \epsilon \}.
\end{align*}
%At round $t$, the version space is
%\begin{align*}
%\F_t & = \{f \in \F : f(x_i) \in L(y_i) \forall i \in O_t \wedge f(x_i) + \epsilon < \min_{j \in [n]} f(x_j) \forall i \in O_t \} \\
%& = \cup_{l \in [n] \setminus O_t} \{f \in \F : f(x_i) \in L(y_i) \forall i \in O_t \} \cap \{ \min_{i \in O_t} f(x_i) > f(x_l) + \epsilon \} \\
%& =: \cup_{l \in [n] \setminus O_t} C_l(O_t).
%\end{align*}
%where
%\begin{align*}
%C_l(O_t) := \{f \in \F : f(x_i) \in L(y_i) \forall i \in O_t \} \cap \{ \min_{i \in O_t} f(x_i) > f(x_l) + \epsilon \} .
%\end{align*}
We overload notation, defining
\begin{align*}
C_l(O_t) := \{f \in \F : f(x_i) \in L(y_i) \, \forall i \in O_t \} \cap \{ f(x_i) \leq f(x_l) + \epsilon \, \forall i \in O_t \} .
\end{align*}
\begin{algorithm}[t]
\small
$k \longleftarrow 1$, $t_1 = 1$, $O_1 \longleftarrow \emptyset$\;
\For{$t =1,2,\ldots$}{
Let $I_t \in \argmax_{i \in [n] \setminus O_t } \min_{j \in [k]} \P_{f \sim P_k}(f \in \F_t(x_i) \cup \F_t((x_i,L_j)))$\;
Query $x_{I_t}$ and observe $y_{I_t}$ and set $O_{t+1} \longleftarrow O_t \cup \{I_t\}$ \;
Let $\F_{t+1} \longleftarrow \F_t \setminus (\F_t(x_{I_t}) \cup \F_t((x_{I_t},L(y_{I_t}))))$\;
\If{$ \P_{f \sim P_k}(f \in \F_{t+1}) \leq \frac{1}{2n}$}{
$P_{k+1} \longleftarrow \frac{1}{n-|O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k} } \pi_{C_l(O_{t_k})}$\;
$k \longleftarrow k+1$\;
$t_k \longleftarrow t+1$\;
}
}
\caption{\texttt{GRAILS} for Continuous Output Space}
\label{alg:effic_haystack_alg_cont}
\end{algorithm}
Define the following extended teaching dimension notion for the continuous output setting:
\begin{align*}
\upsilon_{\epsilon, \delta} & := \max_{g : \X \mapsto \R} \min_{I \subset [n]} |I| \\
& \text{ s.t. } \exists j \in [n] : \{f \in \F : |f(x_i) - g(x_i)| \leq \delta \forall i \in I \} \subset \{ f(x_j) \leq \min_{l \in [n]} f(x_l) + \epsilon \}.
\end{align*}
Define for every $f \in \F$ the set
\begin{align*}
S_f = \{ f^\prime \in \F : L(f^\prime(x_i)) = L(f(x_i)) \forall i \in [n] \}.
\end{align*}
The sets $S_f$ induce a partition of $\F$.
%Furthermore, define
%\begin{align*}
%\Smin & = \argmin_{S_f : f \in \F} \P_{\bar{f} \sim \pi}(\bar{f} \in S_f) \\
%\end{align*}
%$\Smin$ is the subset of $\F$ in the partition that has the least probability under$\pi$.
\begin{theorem}\label{thm:best_arm_ub_cont}
Let $f^* \in \F$ such that $f^*(x_i) = y_i$ for all $i \in [n]$. After $c \ln(\frac{1}{\P_\pi(S_{f^*})}) \upsilon^*_{\epsilon, \delta}$ queries, Algorithm \ref{alg:effic_haystack_alg_cont} has queried some $i \in [n]$ satisfying $f^*(x_i) \leq \min_{j \in [n]} f^*(x_j) + \epsilon$
\end{theorem}
We note that our algorithm for loss minimization can be extended to the continuous output space with a similar guarantee.
\begin{proof}[Proof of Theorem \ref{thm:best_arm_ub_cont}]
For the sake of abbreviation, let $\P(\cdot) := \P_{f \sim \pi}(\cdot)$ and $\P_k(\cdot) := \P_{f \sim P_k}(\cdot)$.
\textbf{Step 1:} The Algorithm \ref{alg:effic_haystack_alg} breaks time into phases $[t_1,t_2]$, $[t_2,t_3]$, $\cdots$. In each phase $k$, the algorithm samples functions from a distinct distribution $P_k$. Consider round $k$, let $t \geq t_k$. Define $\theta \in \{w_1,\ldots, w_L\}^n$ by
\begin{align*}
\theta_i & = \begin{cases}
y_i & i \in O_t \\
\argmin_{\tilde{y} \in \{ \bar{L}_1,\ldots, \bar{L}_p \}} \P_{k}(f \in \F_t(x_i) \cup \F_t((x_i,L(\tilde{y}))) & \text{o/w}
\end{cases}.
\end{align*}
By definition of $\upsilon^*_{\epsilon,\delta}$, there exists $\tilde{I} = \{i_1,\ldots, i_{\upsilon^*}\}$ such that there exists $j_0 \in [n]$ such that $\forall f \in \F_t$ that satisfy $|f(x_i) - \theta_i| \leq \delta$ for all $i \in \tilde{I}$, it holds that $f(x_{j_0}) - \epsilon \leq \min_{l \in [n]} f(x_l)$. Define $I = \{i_1,\ldots, i_{\upsilon^*}, j_0\}$. Then, we have that
\begin{align*}
\F_t & = \{f \in \F_t : \exists i \in I \text{ s.t. } f(x_{i}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } |f(x_i) -\theta_i| > \delta \} \\
& = \{f \in \F_t : \exists i \in I \text{ s.t. } f(x_{i}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } f(x_i) \not \in L(\theta_i) \}
\end{align*}
where the last equality follows since by definition of $L(\cdot)$, $|f(x_i) - \theta_i| > \delta$ implies that $f(x_i) \not \in L(\theta_i)$.
From this, it follows that
\begin{align}
\P_k(\F_t) & = \P_k (\{f \in \F_t : \exists i \in I \text{ s.t. } f(x_{i}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } f(x_i) \not \in L(\theta_i) \}) \nonumber \\
& \leq \sum_{i \in I} \P_k (\{ f \in \F_t : f(x_{i}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } f(x_i) \not \in L(\theta_i) \}) \nonumber \\
& \leq (\upsilon^*_{\epsilon,\delta}+1) \max_{i \in I} \P_k (\{ f \in \F_t : f(x_{i}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } f(x_i) \not \in L(\theta_i)\}) \nonumber \\
& = (\upsilon^*_{\epsilon,\delta}+1) \P_k (\{f \in \F_t : f(x_{I_t}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } f(x_{I_t}) \not \in L(\theta_{I_t}) \}) \label{eq:efficient_I_t_def_2}\\
& \leq (\upsilon^*_{\epsilon,\delta}+1) \P_k (\{f \in \F_t : f(x_{I_t}) \leq \min_{l \in [n]} f(x_l) + \epsilon \text{ or } f(x_{I_t}) \not \in L(y_{I_t}) \}) \label{eq:efficient_theta_def_2} \\
& = (\upsilon^*_{\epsilon,\delta}+1) \P_k (\F_t(x_{I_t}) \cup \F_t((x_{I_t}, L(y_{I_t}))) \nonumber
\end{align}
where line \eqref{eq:efficient_I_t_def_2} comes from the definition of $I_t$ and the line \eqref{eq:efficient_theta_def_2} comes from the definition of $\theta$. Noticing that
\begin{align*}
\F_t & = (\F_t(x_{I_t}) \cup \F_t((x_{I_t}, L(y_{I_t}))) \cup \F_t \setminus (\F_t(x_{I_t}) \cup \F_t((x_{I_t}, L(y_{I_t}))) \\
& = (\F_t(x_{I_t}) \cup \F_t((x_{I_t}, L(y_{I_t}))) \cup \F_{t+1},
\end{align*}
we have that
\begin{align*}
\P_k(\F_{t+1}) & = \P_k(\F_t) - \P_k(\F_t(x_{I_t}) \cup \F_t((x_{I_t}, L(y_{I_t}))) \leq (1 - \frac{1}{\upsilon^*_{\epsilon,\delta}+1})\P_k(\F_t).
\end{align*}
Unraveling this recursive statement, we have that
\begin{align*}
\P_k(\F_{t+1}) & \leq (1 - \frac{1}{\upsilon^*+1})^{t-t_k}.
\end{align*}
Thus, we see that in phase $k$ after $s_k := O(\ln(n) \upsilon^*_{\epsilon,\delta})$ rounds , we have that
\begin{align}
\P_k(\F_{t_k+s_k}) \leq \frac{1}{2n}. \label{eq:shrink_objective_2}
\end{align}
The rest of the proof is similar to the proof of Theorem \ref{thm:haystack_eff}.
\end{proof}
%\section{$\epsilon$-good arm identification examples}
%
%\begin{proof}[Proof of Proposition \ref{prop:ex_dis_eps_good_lin}]
%Fix $\delta_0 = \frac{\delta}{2 R_2}$. Fix $a_0 $ such that $\norm{a_0} \leq R_1$. Define
%\begin{align*}
%\bar{B}_{\alpha} & = B_{\delta_0}(\alpha a_0)
%\end{align*}
%where $B_\gamma(v) := \{v \in \R^d : \norm{v} \leq \gamma \}$.
%
%\textbf{Step 1.} We claim that for all $\alpha \in [0, 1-\delta_0/R_1]$, $\bar{B}_\alpha \subset \{a : \norm{a} \leq R_1\}$. Let $\alpha \in [0, 1-\epsilon/R_1]$. Let $v$ such that $\norm{v} \leq \delta_0$. Then,
%\begin{align*}
%\norm{(1-\delta_0/R_1) a_0 + v} \leq (1-\delta_0/R_1) \norm{a_0} + \norm{v} \leq R_1 - \delta_0 + \delta_0 = R_1,
%\end{align*}
%showing the claim.
%
%\textbf{Step 2.} Let $\alpha_0= \max(1-\delta_0/ \norm{a_0},0) \leq 1-\delta_0/ R_1 $. We claim that $a_0 \in \bar{B}_{\alpha_0}$. If $1-\delta_0/ \norm{a_0} \geq 0$, then
%\begin{align*}
%\norm{(\alpha_0 -1) a_0} = \frac{\delta_0}{\norm{a_0}} \norm{a_0} = \delta_0
%\end{align*}
%so that $a_0 \in \bar{B}_{\alpha_0}$. On the other hand, if $1-\delta_0/ \norm{a_0} < 0$, then $\norm{a_0} < \delta_0$, so that $a_0 \in \bar{B}_{\alpha_0}$ since $\alpha_0 = 0$.
%
%\textbf{Step 3.} Now, let $a_1 \in \bar{B}_{\alpha_0}$. Then,
%\begin{align*}
%\norm{a_0-a_1} \leq 2 \delta_0.
%\end{align*}
%Then, using Holder's inequality,
%\begin{align*}
%|a_0^\t x_i - (a_1^\t x_i )| \leq \norm{a_0-a_1} \max_{i} \norm{x_i}_* \leq 2 \delta_0 R_2 \leq \delta
%\end{align*}
%by definition of $\delta_0$. By definition of $\delta$, $|\round{\langle a_0, x_i \rangle} - \round{\langle a_1, x_i \rangle} | \leq \epsilon$ for all $i \in [n]$.
%
%\textbf{Step 4.} Since $|\round{\langle a_0, x_i \rangle} - \round{\langle a_1, x_i \rangle} | \leq \epsilon$ for all $i \in [n]$, $\bar{B}_{\alpha_0} \subset S_{f}$ where $f =\langle \cdot, a_0\rangle$. Thus,
%\begin{align*}
%\P_\pi(S_f) & \geq \P_{\pi}(\bar{B}_{\alpha_0} \}) \\
%& = \frac{\text{volume}(\{z \in \R^{d} : \norm{z} \leq \delta_0 \}}{\text{volume}(\{z \in \R^{d} : \norm{z} \leq R_1\}) }.
%\end{align*}
%Plugging in the value of $\delta_0$ gives the result.
%
%\end{proof}
%
%\begin{proof}[Proof of Proposition \ref{prop:ex_kernel_eff_dim}]
%
%Let $\delta_0 = \frac{\delta}{4 R_2}$. Fix $a_0 \in \R^n$ such that $\norm{a_0}_{K^{1/2}_{\deff}} \leq R_1$. Define
%\begin{align*}
%\bar{B}_\alpha & = \{P_{\deff} v \in \R^n : \norm{v-\alpha a_0}_{K_{\deff}^{1/2}} \leq \delta_0 \}.
%\end{align*}
%By the same argument in the proof of Proposition \ref{prop:ex_dis_eps_good_lin}, for all $\alpha \in [0,1-\frac{\delta_0}{R_1}]$, $\bar{B}_\alpha \subset \{ v \in \R^n : \norm{a_0}_{K^{1/2}_{\deff}} \leq R_1 \} $ and for $\alpha_0 = \max(0,1-\frac{\delta_0}{\norm{\alpha_0}_{K^{1/2}_{\deff} }})$, $P_{\deff} a_0 \in \bar{B}_{\alpha_0} $.
%
%Let $f(\cdot) = \langle K a_0, \cdot \rangle $ and $f^\prime(\cdot) = \langle K a^\prime, \cdot \rangle $ where $a^\prime \in \R^n$ such that $\norm{a^\prime}_{K^{1/2}} \leq R_1$ and $P_{\deff} a^\prime \in \bar{B}_{\alpha_0}$. Note that
%\begin{align}
%|f(x_i) - f^\prime(x_i)| & = |e_i^\t K(a_0-a^\prime)| \nonumber \\
%& = |e_i^\t (K - K_{\deff})(a_0-a^\prime) + e_i^\t K_{\deff}(a_0-a^\prime)| \nonumber \\
%& \leq |e_i^\t (K - K_{\deff})(a_0-a^\prime)| + |e_i^\t K_{\deff}(a_0-a^\prime)| \nonumber \\
%& \leq \frac{\delta}{2} + |e_i^\t K_{\deff}(a-a^\prime)| \label{eq:kernel_ex_deff_apply} \\
%& \leq \frac{\delta}{2} + \norm{K_{\deff}^{1/2} e_i}_2 \norm{K_{\deff}^{1/2} (a_0-a^\prime)}_2 \label{eq:kernel_ex_cs}\\
%& \leq \frac{\delta}{2} + \norm{ e_i}_{K^{1/2}} \norm{ (a_0-a^\prime)}_{K^{1/2}_{\deff}} \label{eq:kernel_ex_psd} \\
%& \leq \frac{\delta}{2} + \max_{i \in [n]} K(x_i,x_i) \norm{ a_0-a^\prime}_{K^{1/2}_{\deff}} \label{eq:kernel_ex_ball} \\
%& \leq \frac{\delta}{2} + \max_{i \in [n]} K(x_i,x_i) 2 \delta_0 \label{eq:kernel_ex_def_delta_0} \\
%& \leq \delta. \nonumber
%\end{align}
%where line \eqref{eq:kernel_ex_deff_apply} uses the definition of $\deff$, line \eqref{eq:kernel_ex_cs} uses Cauchy-Schwarz, line \eqref{eq:kernel_ex_psd} uses $K_{\deff} \preceq K$, and line \eqref{eq:kernel_ex_ball} uses the fact that $a_0, a^\prime \in \bar{B}_{\alpha_0}$, and line \eqref{eq:kernel_ex_def_delta_0} uses the definition of $\delta_0$. Thus, $|\round{\langle a_0, x_i \rangle} - \round{\langle a_1, x_i \rangle} | \leq \epsilon$ for all $i \in [n]$, and therefore $\bar{B}_{\alpha_0} \subset S_{f}$ where $f =\langle \cdot, a_0\rangle$. Thus,
%\begin{align*}
%\P_{\pi}(S_f) & \geq \P_\pi(\bar{B}_{\alpha_0}) \\
%& = \frac{\text{volume}(\{ z \in \R^{\deff} : \norm{z}_{K^{1/2}_{\deff}} \leq \delta_0 \}}{\text{volume}(\{z \in \R^{\deff} : \norm{z}_{K^{1/2}_{\deff}} \leq R_1\}) } \\
%& = (\frac{\delta_0}{R_1})^{\deff}
%\end{align*}
%where the last line follows by plugging in standard formulae for the volume of an ellipsoid. The result follows by plugging in the definition of $\delta_0$.
%
%
%\end{proof}
%
%
%\begin{proof}[Proof of Proposition \ref{prop:eff_dim}]
%We have that
%\begin{align*}
%|e_i^\t (\sum_{i=1}^n \lambda_i u_i u_i^\t - \sum_{i=1}^j \lambda_i u_i u_i^\t ) a| & = |e_i^\t (\sum_{i=j+1}^n \lambda_i u_i u_i^\t ) a| \\
%& \leq \norm{e_i}_{(\sum_{i=j+1}^n \lambda_i u_i u_i^\t )^{1/2}} \norm{a}_{(\sum_{i=j+1}^n \lambda_i u_i u_i^\t )^{1/2}} \\
%& \leq \norm{e_i}_{(\sum_{i=j+1}^n \lambda_i u_i u_i^\t )^{1/2}} \norm{a}_{K^{1/2}} \\
%& \leq \lambda_{j+1} R_1
%\end{align*}
%where we used that $\sum_{i=j+1}^n \lambda_i u_i u_i^\t \preceq K $ and the fact that $\lambda_{j+1}$ is the largest eigenvalue of $\sum_{i=j+1}^n \lambda_i u_i u_i^\t$. This shows the claim by definition of $\deff$.
%\end{proof}
\section{Active Classification and Extensions to other Objectives}
In this Section, we apply our framework to other problems such as active classification and a problem for identifying a sufficiently good arm, defined shortly. We give a result for active classification and give brief sketches for the other objectives since their treatment is similar to the problems already studied in this paper.
\textbf{Active Classification.} We recall the active classification setting. Let $\X$ denote the input space and $\Y = \{-1,1\}$ the output space. Let $x_1,\ldots, x_n \in \X$ be a fixed pool with associated scores $y_1,\ldots, y_n \in \Y$. At each round $t$, the learner selects (or queries) $I_t \in [n]$ and observes $y_{I_t}$. The goal is to identify $\argmin_{f \in \reg_{\calR}} \sum_{i=1}^n \one\{y_i \neq f(x_i)\}$ using as few queries as possible. Recall that under realizability there exists $f^* \in \F_{\reg}$ such that $f^*(x_i) = y_i$ for all $i \in [n]$.
We note that for active classification, the version space is convex, and so we may use a simpler version space. We also note that \text{STOP}($\F_\reg$,$O_t$) can check whether there are $f,f^\prime \in \F_t$ such that $f \neq f^\prime$ by solving the following optimization problem for $j \in [n] \setminus O_t$ and $y \in \Y$
\begin{align}
\exists f \text{ s.t. } f(x_i) = y_i \, \forall i \in O_t \wedge f(x_j) = y. \label{eq:act_class_stop}
\end{align}
If there exists $j$ such that there exists $f,f^\prime \in \F_t$ and $f(x_j) \neq f(x_j)$, then the Algorithm does not terminate. If not, then the algorithm terminates. We note that \eqref{eq:act_class_stop} is a convex optimization problem for function classes like linear functions, kernel methods, and convex functions.
\begin{algorithm}[t]
\small
$k \longleftarrow 1$, $t_1 = 1, O_1 \longleftarrow \emptyset$\;
\For{$t =1,2,\ldots$}{
Let $I_t \in \argmax_{i \in [n] \setminus O_t } \min_{y \in \Y} \P_{r \sim \pi_{\reg_t}}(\reg_t((x_i,y))$\;
Query $x_{I_t}$ and observe $y_{I_t}$ and set $O_{t+1} \longleftarrow O_t \cup \{I_t\}$ \;
Let $\reg_{t+1} \longleftarrow \reg_t \setminus \reg_t((x_{I_t},y_{I_t}))$\;
\If{\text{STOP}($\F_\reg$,$O_t$)}{
\Return $\argmin_{i \in O_t} y_i$}
}
\caption{\texttt{GRAILS} for Active Classification}
\label{alg:effic_active_class}
\end{algorithm}
We introduce the extended teaching dimension notion \cite{hegedHus1995generalized}, which has previously been shown to be a lower bound for active classification:
\begin{align*}
\classext(\F) &= \max_{g : \X \mapsto \Y} \min_{I \subset [n]} |I| \\
& \text{ s.t. } |\{f \in \reg : \round{f(x_i)} = g(x_i) \forall i \in I\}| \leq 1.
\end{align*}
Recall the definition $\Smin = \argmin_{S_f : f \in \F_{\calR}, \P_\pi(S_f) > 0} \P_{\pi}( S_f) $.
\begin{theorem}\label{thm:active_upper_bound}
After $c\log(\frac{1}{\P(S_{\min})}) \classext(\F_{\reg})$ queries, the version space only contains $S_{f^*}$.
\end{theorem}
\begin{proof}[Proof of Theorem \ref{thm:active_upper_bound}]
For the sake of abbreviation, let $\P(\cdot) := \P_{f \sim \pi}(\cdot)$ and $\P_t(\cdot) := \P_{f \sim \pi_t}(\cdot)$ where $\pi_t := \pi_{\reg_t}$.
\textbf{Step 1:} Consider round $t$. Define
\begin{align*}
\theta_i & = \begin{cases}
y_i & i \in O_t \\
\argmin_{y \in \Y} \P(r \in \reg_t((x_i,y))) & \text{o/w}
\end{cases}.
\end{align*}
By definition of the extended teaching dimension $\classext(\F_{\reg})$, there exists $I = \{i_1,\ldots, i_{\classext}\}$ such that
\begin{align*}
|\G_t | \leq 1
\end{align*}
where
\begin{align}
\G_t := \{ \round{r} \in \reg_t : \round{r(x_i)} = \theta_i \, \, \forall i \in I \}. \label{eq:class_def_G_t}
\end{align}
From this, it follows that
\begin{align}
\P(\reg_t \setminus \G_t) & = \P(\{r \in \reg_t : \exists i \in I \text{ s.t. } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& \leq \sum_{i \in I} \P (\{r \in \reg_t : \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& \leq \classext \max_{i \in I} \P (\{r \in \reg_t : \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& = \classext \P (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_{I_t} \}) \label{eq:efficient_I_t_def_class}\\
& \leq \classext \P (\{r \in \reg_t : \round{r(x_i)} \neq y_{I_t} \}) \label{eq:efficient_theta_def_class} \\
& = \classext \P ( \reg_t((x_{I_t}, y_{I_t}))) \nonumber
\end{align}
where line \eqref{eq:efficient_I_t_def_class} comes from the definition of $I_t$, the definition of $\theta$, and the fact that
\begin{align*}
\argmin_{i} \P(r \in \reg_t((x_i,\theta_i))) = \argmin_{i} \frac{\P(r \in \reg_t((x_i,\theta_i)))}{\P(\reg_t)} = \argmin_{i} \P_t(r \in \reg_t((x_i,\theta_i)))
\end{align*}
and the line \eqref{eq:efficient_theta_def_class} comes from the definition of $\theta$. Noticing that
\begin{align*}
\reg_t & = \reg_t((x_{I_t}, y_{I_t}) \cup \reg_t \setminus \reg_t((x_{I_t}, y_{I_t})) \\
& =\reg_t((x_{I_t}, y_{I_t}) \cup \reg_{t+1},
\end{align*}
we have that
\begin{align}
\P(\reg_{t+1}) & = \P(\reg_t) - \P(\reg_t((x_{I_t}, y_{I_t}))) \leq \P(\reg_t) - \frac{1}{\classext} \P(\reg_t \setminus \G_t). \label{eq:class_bound}
\end{align}
We have that
\begin{align*}
\sum_{t=1}^\infty \one\{\reg_t \neq S_{f^*}\} & = \sum_{t=1}^\infty \one\{\reg_t \neq S_{f^*} \wedge \P(\G_t) \leq \frac{1}{2} \P(\reg_t)\} \\
& \hspace{1cm} + \one\{\reg_t \neq S_{f^*} \wedge \P(\G_t) > \frac{1}{2} \P(\reg_t) \wedge \G_t \neq S_{f^*}\} \\
& \hspace{1cm} + \one\{\reg_t \neq S_{f^*} \wedge \P(\G_t) > \frac{1}{2} \P(\reg_t) \wedge \G_t = S_{f^*}\}.
\end{align*}
\textbf{Step 2: Bounding the first term.} We begin by bounding the first sum. Suppose $\P(\G_t) \leq \frac{1}{2} \P(\reg_t)$. Then, \eqref{eq:class_bound} implies that we have that
\begin{align*}
\P(\reg_{t+1}) & \leq\P(\reg_t) - \frac{1}{\classext} \P(\reg_t \setminus \G_t) = (1-\frac{1}{ \classext}) \P(\reg_t ) + \frac{1}{\classext}\P(\G_t) \leq (1-\frac{1}{ 2\classext}) \P(\reg_t ) .
\end{align*}
Thus, if for some $t^\prime \geq t$ $\one\{\reg_t \neq S_{f^*} \wedge \P(\G_t) \leq \frac{1}{2} \P(\reg_t)\}$ occurs $k$ times in the rounds between $t$ and $t^\prime$, we have that
\begin{align*}
\P(\reg_{t^\prime}) \leq (1-\frac{1}{ 2\classext})^k \P(\reg_t ).
\end{align*}
This implies that if this event occurs $c\log(\frac{1}{\P(S_{f^*})}) \classext$ times, we have that $\reg_{t^\prime} = S_{f^*}$. This completes bounding the second term.
\textbf{Step 3: Bounding the second term.} Fix round $t$ and suppose that event $\one\{\reg_t \neq S_{f^*} \wedge \P(\G_t) > \frac{1}{2} \P(\reg_t)\}$ occurs. Since $ \P(\G_t) > \frac{1}{2} \P(\reg_t)$, there exists $\bar{f} \in \F_{\reg}$ such that $\bar{f} \neq f^*$, $\G_t = S_{\bar{f}} $, and $\bar{r} \in \reg$ such that $\round{\bar{r}} = \bar{f}$. Let $t^\prime \geq t$ be the round at which $\bar{f}$ is kicked out or is the last remaining function. We claim that for all $s \in [t,t^\prime]$, $\G_s = S_{\bar{f}}$. Fix $i \in [n]$ and note that
\begin{align}
\P_{r \sim \pi_s}(\round{r(x_i)} = \round{\bar{r}(x_i)}) & \geq \P_{r \sim \pi_s}(\round{r} = \round{\bar{r}}) \label{eq:act_class_equal_one_equal_all} \\
& = \frac{\P(\round{r} = \round{\bar{r}})}{\P(\reg_s)} \label{eq:act_class_cond_prob}\\
& > \frac{1}{2} \frac{\P(\reg_t)}{\P(\reg_s)} \label{eq:act_class_large_prob} \\
& \geq \frac{1}{2} \label{eq:act_class_ver_space_contain}
\end{align}
where \eqref{eq:act_class_equal_one_equal_all} follows since if $\round{r} = \round{\bar{r}}$, then $\round{r(x_i)} = \round{\bar{r}(x_i)}$, \eqref{eq:act_class_cond_prob} follows by the definition of conditional probability, \eqref{eq:act_class_large_prob} follows since $\P(S_{\bar{f}}) =\P(\G_t) > \frac{1}{2} \P(\reg_t)$, and \eqref{eq:act_class_ver_space_contain} follows since $\reg_t \supset \reg_s$ because $s \geq t$ and the version spaces are monotonically decreasing. Thus, inspection of the definition of $\G_s$ from \eqref{eq:class_def_G_t} and the definition of $\theta$ as
\begin{align*}
\theta_i = \argmin_{y \in \Y} \P(r \in \reg_s : \round{r(x_i)} \neq y) = \argmax_{y \in \Y} \P(r \in \reg_s : \round{r(x_i)} = y)
\end{align*}
with $|\Y| = 2$ shows the claim that for all $s \in [t,t^\prime]$, $\G_s = S_{\bar{f}}$.
Now, consider round $t+1$. Then, by \eqref{eq:class_bound} and $\G_{t+1} = S_{\bar{f}}$, we have that
\begin{align*}
\P_t(\reg_{t+1} \setminus S_{\bar{f}}) + \Pr(S_{\bar{f}}) & \leq (1- \frac{1}{\classext}) \P_t(\reg_t \setminus S_{\bar{f}}) + \Pr(S_{\bar{f}}),
\end{align*}
which implies that
\begin{align*}
\P(\reg_{t+1} \setminus S_{\bar{f}}) & \leq (1- \frac{1}{\classext}) \P(\reg_t \setminus S_{\bar{f}}) .
\end{align*}
Unraveling this recurrence up to $t^\prime$, we have that
\begin{align*}
\P_t(\reg_{t^\prime} ) = \P_t(\reg_{t^\prime} \setminus S_{\bar{f}} ) \leq (1- \frac{1}{\classext})^{t^\prime -t} \P_t(\reg_t \setminus S_{\bar{f}}) \leq (1- \frac{1}{\classext})^{t^\prime -t} \P(\reg_t) .
\end{align*}
where in the first equality we used the fact that by definition of $t^\prime$, $\reg_{t^\prime} \cap S_{\bar{f}}= \emptyset$. Thus, we see that the second term is bounded above by at most $c\log(\frac{1}{\P(\calS_{f^*})}) \classext$.
\textbf{Step 4. Bounding the third term.} By the argument in the previous step, we have that $\G_t = \calS_{f^*}$ for the rest of the game. Thus, we have that from \eqref{eq:class_bound}
\begin{align*}
\P(\reg_{t+1}) & \leq \P(\reg_t) - \frac{1}{\classext} \P(\reg_t \setminus S_{f^*}) \\
& = (1- \frac{1}{\classext} )\P(\reg_t) + \frac{1}{\classext} \P( S_{f^*}).
\end{align*}
Thus, unravelling the recurrence, after $k$ rounds, we have that
\begin{align*}
\P(\reg_{t+k}) & \leq (1- \frac{1}{\classext} )^k\P(\reg_t) + \frac{1}{\classext} \sum_{l=1}^{k-1} (1- \frac{1}{\classext} )^l \P( S_{f^*}) \\
& \leq (1- \frac{1}{\classext} )^k\P(\reg_t) + \P( S_{f^*})
\end{align*}
where the last inequality followed by the geometric series. This implies that if this event occurs $c\log(\frac{1}{\P(S_{\min}}) \classext$ times, we have that $\reg_{t^\prime} = S_{f^*}$. This completes bounding the third term.
\end{proof}
\begin{corollary}
If the most probable classifier is output at each round, then after $c \log(\frac{1}{\P(S_{f^*}}) \classext(\F_{\reg})$ queries, only $f^*$ will be output for the rest of the game.
\end{corollary}
\begin{proof}
The only step that is slightly different is step 4. Here, it can be seen that after $c \log(\frac{1}{\P(S_{f^*}}) \classext(\F_{\reg})$ queries, $f^*$ is the most probable classifier and it will remain the most probable classifier for the remainder of the game.
\end{proof}
Next, we consider some other settings briefly.
% \textbf{$\epsilon$-good arm identification}. In $\epsilon$-good arm identification, the goal is to identify some $i_* \in [n]$ that satisfies $y_{i_*} \leq \min_{i \in [n]} y_i + \epsilon$. The extended teaching dimension notion is:
% \begin{align*}
%\upsilon^*_\epsilon &= \max_{g : \X \mapsto \Y} \min_{I \subset [n]} |I| \\
% & \text{ s.t. } \exists j \in [n] : \{f \in \F : \round{f(x_i)} = g(x_i) \forall i \in I\} \subset \{ f \in \F : \round{f(x_j)} \leq \min_{l \in [n]} \round{f(x_l)} + \epsilon \}.
%\end{align*}
%The version space would be defined as
%\begin{align*}
%\F_t & = \{f \in \F : \round{f(x_i)} = y_i \forall i \in O_t \wedge \round{f(x_i)} > \min_{j \in [n]} \round{f(x_j)} + \epsilon \forall i \in O_t \} \\
%& = \cup_{l \in [n] \setminus O_t} \{f \in \F : \round{f(x_i)} = y_i \forall i \in O_t\} \cap \{ \round{f(x_l)} + \epsilon < \round{f(x_i)} \forall i \in O_t \}
%\end{align*}
\textbf{Threshold/Satisficing Bandits.} Here we are given a threshold $\gamma \in \R$, and the goal is to identify $i \in [n]$ such that $y_i \leq \gamma$ (this problem was introduced and studied in the standard multi-armed bandit setting in \cite{katz2020true}). Then, the extended-teaching dimension notion is
\begin{align*}
\upsilon^*_\gamma &= \max_{g : \X \mapsto \Y} \min_{I \subset [n]} |I| \\
& \text{ s.t. } \exists j \in [n] : \{f \in \F : \round{f(x_i)} = g(x_i) \forall i \in I\} \subset \{ f \in \F : \round{f(x_j)} \leq \gamma \}.
\end{align*}
The version space is given by
\begin{align*}
\F_t & = \{f \in \F : \round{f(x_i)} = y_i \forall i \in O_t \}.
\end{align*}
The algorithm would query by the following rule
\begin{align*}
I_t \in \argmax_{i \in [n] \setminus O_t } \min_{y \in \Y} \P_{f \sim P_k}(f \in \F_{t,\gamma}(x_i) \cup \F_t((x_i,y)))
\end{align*}
where $\F_{t,\gamma}(x_i) = \{f \in \F_t : \round{f(x_i)} \leq \gamma \}$. We note that for this problem it is not necessary to sample from the mixture $P_k$ because the version space is convex.
We conjecture that there are extensions to settings where the feedback is multi-dimensional (e.g., \cite{katz2018feasible}) or the goal is constrained best arm identification (e.g., \cite{katz2019top}).
\section{Stopping Condition}
Here we briefly give a concrete case for the stopping condition for linear models. Let
\begin{align*}
\reg = \{ \langle a, \cdot \rangle : a \in \R^d, \norm{a} \leq 1\}
\end{align*}
and $\Y = [K]$ for some $K \in \N$. Suppose that at round $t$ $O_t = \{I_1,\ldots, I_t\}$ and $y_{I_1},\ldots, y_{I_t}$ have been observed. Suppose $y_{I_1},\ldots, y_{I_t} \neq 1$ since we are otherwise done and let $\bar{y} = \max_{y < \min_{s : } y_{I_s}} y$. Fix $l \in [n] \setminus \{I_1, \ldots, I_t\}$. Then, we have that
\begin{align*}
C_{l}(O_t) = \{ a \in \R^d : \norm{a} \leq 1, |a^\t x_i - y_i | \leq 1/2 \forall i \in O_t, a^\t x_l \leq \bar{y} + 1/2 \}.
\end{align*}
Determining whether $C_{l}(O_t)$ is nonempty is a simple convex feasibility problem.
% \begin{proof}[Proof of Proposition \ref{prop:quadratic_extended_teaching_dim}]
% Fix any $g: \{1,\ldots, n\} \mapsto \{w_1,\ldots, w_n\}$. If there exists $i \in [n]$ such that $g(x_i) \not \in \{w_1,\ldots, w_n\}$, then it suffices to query $i$, so wlog suppose that $g(x_i) \in \{w_1,\ldots, w_n\}$ for all $i \in [n]$.
% \textbf{Claim.} We claim that each $\round{f(x)}$ has a minimum value of $0$. Fix $a \in [1,n]$. Let $i \in \{1,\ldots, n\}$ that is closest to $a$. Then, $(a-x_i)^2 = (a-i)^2 \leq \frac{1}{4}$, which implies that $\round{(a-x_i)^2} = 0$ since $\round{\cdot}$ rounds to the nearest value of $w_i$ (and ties breaks by choosing the smaller one).
% Let $i_0 \in \argmin_{j \in [n]} g(x_j)$. Suppose that $g(x_{i_0}) = 0$. Then, only the functions that have value of $0$ at $i_0$ are left and since by the claim all the functions in the version space attain a minimum of $0$, the constraint in the definition of $\upsilon^*$ is satisfied.
% Now, suppose that $g(x_{i_0}) \neq 0$. Suppose that $i_0 \in (1,n)$ (the other cases are similar). Consider $I = \{i_0-1,i_0,i_0+1\}$. Then,
% \begin{align}
% \max(g(x_{i_0-1}),g(x_{i_0+1})) \geq g(x_{i_0}) > 0 \label{eq:quad_ex_contra}
% \end{align}
% Using the claim and the quasiconvexity of $\round{(a-x)^2}$, we see that there is no $a \in [1,n]$ such that $\round{(a-x_i)^2}$ satisfies the inequalities in \eqref{eq:quad_ex_contra} for $i \in I$. Thus, querying $\{i_0-1,i_0,i_0+1\}$ eliminates all function in the version space.This completes the proof.
% Now, we lower bound the probability. Fix $a \in [1,n]$. Let $b \in [1,n]$ such that $b \leq \frac{1}{n^2}$ and suppose $a + b \in [1,n]$ (the case $a-b \in [1,n]$ is similar. Then,
% \begin{align*}
% |(x-a )^2 - (x-(a+b))^2| = |b - 2(x-a)b| \leq |b| + 2|(x-a)b| \leq \frac{1}{n^2} + 2\frac{1}{n}.
% \end{align*}
% where we used the fact that $|x-a| \leq n $.
% $\round{(x-a)^2}$ rounds to the nearest integer and thus we see thatif $n \geq 5$, $\round{(x_i-a)^2}=\round{(x_i-(a+b))^2}$ for all $i \in [n]$. Thus, $\P_\pi(\S_f) \geq \frac{1/n^2}{n}= \frac{1}{n^3}$.
% \end{proof}
\section{The Algorithms with Approximation}
\subsection{Best Arm Identification}
Algorithm \ref{alg:effic_haystack_alg_approx} is the version of Algorithm \ref{alg:effic_haystack_alg} for best arm identification that estimates the events. It uses the subroutine EstimateEvent, Algorithm \ref{alg:est_event}, to estimate the events. Note that the sampling oracle is called a a number of times that is polynomial in $n$ and polynomial in $|\Y|$.
\begin{algorithm}[t]
\small
\textbf{Input:} distribution $P$ ; event $\Sigma$, accuracy $\epsilon$, failure probability $\delta$, \;
$N \longleftarrow 2 \epsilon^{-2} \log(1/\delta) $\;
Sample $z_1, \ldots, z_N \sim P$ \;
\Return $\frac{\sum_{i=1}^N \one \{ z_i \in \Sigma\} }{N}$
\caption{EstimateEvent}
\label{alg:est_event}
\end{algorithm}
\begin{algorithm}[t]
\small
$P_1 \longleftarrow \pi$, $\reg_1 \longleftarrow \reg, O_1 \longleftarrow \emptyset$ \;
$k \longleftarrow 1$, $t_1 = 1$, $\delta_t = \frac{\delta}{2t^2 |\Y| n}$ \;
\For{$t =1,2,\ldots$}{
Let $I_t \in \argmax_{i \in [n] \setminus O_t } \min_{y \in \Y} \text{EstimateEvent}(P_k, \reg_t(x_i) \cup \reg_t((x_i,y)), \frac{1}{32 n^2}, \delta_t )$\;
Query $x_{I_t}$ and observe $y_{I_t}$ and set $O_{t+1} \longleftarrow O_t \cup \{I_t\}$ \;
Let $\reg_{t+1} \longleftarrow \reg_t \setminus (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t},y_{I_t})))$\;
\If{$\text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) \leq \frac{1}{4n} $}{
$P_{k+1} \longleftarrow \frac{1}{n-|O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k} } \pi_{C_l(O_{t_k})}$\;
$k \longleftarrow k+1$\;
$t_k \longleftarrow t+1$\;
}
\If{\text{STOP}($\F_\reg$,$O_t$)}{
\Return $\argmin_{i \in O_t} y_i$}
}
\caption{\texttt{GRAILS} with Estimation}
\label{alg:effic_haystack_alg_approx}
\end{algorithm}
\begin{theorem}\label{thm:haystack_eff_approx}
With probability at least $1-\delta$, if $t \geq c \bestext \log(\frac{1}{\P_{\pi}(S_{f^*})}) \ln(n) $ where $c >0$ is a universal constant, Algorithm \ref{alg:effic_haystack_alg_approx} has pulled an arm $I_s$ for $s \leq t$ such that $I_s \in \argmin_{i \in [n]} y_i$.
\end{theorem}
\begin{proof}[Proof of Theorem \ref{thm:haystack_eff_approx}]
The proof is quite similar to the proof of Theorem \ref{thm:haystack_eff}.
\textbf{Step 0. Defining the good event.} Define
\begin{align*}
\Sigma_{1,t} & = \{ \forall y \in \Y, \forall i \in [n] \, : |\text{EstimateEvent}(P_k, \reg_t(x_i) \cup \reg_t((x_i,y)), \frac{1}{32 n^2}, \delta_t ) \\
& \hspace{2cm} - \P_{f \sim P_k}(r \in \reg_t(x_i) \cup \reg_t((x_i,y)))| \leq \frac{1}{32n^2} \} \\
\Sigma_{2,t} & = \{ |\text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) - \P_{f \sim P_k}(r \in \reg_{t+1})| \leq \frac{1}{8 n} \} \\
\Sigma_t & = \Sigma_{1,t} \cap \Sigma_{2,t} \\
\Sigma & = \cap_{t=1}^\infty \Sigma_t .
\end{align*}
By the total law of probability, we have that
\begin{align*}
\P(\Sigma^c) & \leq \sum_{t=1}^\infty \P(\Sigma_t^c | \cap_{s=1}^{t-1} \Sigma_{s} )\\
& \leq \sum_{t=1}^\infty \P(\Sigma_{t,1}^c | \cap_{s=1}^{t-1} \Sigma_{s} ) + \P(\Sigma_{t,2}^c | \cap_{s=1}^{t-1} \Sigma_{s} )\\
& \leq \frac{6}{\pi^2} \sum_{t=1}^\infty \frac{\delta}{t^2} \\
& \leq \delta
\end{align*}
where the last line follows by the definition of the Algorithm EstimateEvent and standard Hoeffding bounds. For the rest of the proof, we will condition on the event $\Sigma$.
\textbf{Step 1.} This step is very similar to step 1 in the proof of Theorem \ref{thm:haystack_eff}. Consider phase $k$, let $t \geq t_k$. Define $\theta \in \Y^n$ by
\begin{align*}
\theta_i & = \begin{cases}
y_i & i \in O_t \\
\argmin_{y \in \Y} \P_{k}(r \in \reg_t(x_i) \cup \reg_t((x_i,y))) & \text{o/w}
\end{cases}.
\end{align*}
By definition of $\bestext$, there exists $\tilde{I} = \{i_1,\ldots, i_{\bestext}\}$ such that there exists $j_0 \in [n] \setminus O_t$ such that
\begin{align*}
\{ r \in \reg_t : \round{r(x_i)} = \theta_i \, \, \forall i \in \tilde{I} \} \subset \{r \in \reg_t : j_0 \in \argmin_{l \in [n]} \round{r(x_l)} \}
\end{align*}
Define $I = \{i_1,\ldots, i_{\bestext}, j_0\}$. Then, we have that
\begin{align*}
\reg_t & = \{r \in \reg_t : \exists i \in I \text{ s.t. } i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}.
\end{align*}
Define
\begin{align*}
J_t := \argmax_{i \in I} \P_k (\{r \in \reg_t : i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \\
\bar{y} := \argmin_{y \in \Y} \text{EstimateEvent}(P_k, \reg_t(x_{I_t}) \cup \reg_t((x_{I_t},\bar{y}), \frac{1}{32n^2}, \delta_t )
\end{align*}
Note $J_t$ would be chosen if there were no noise from estimation.
By the event $\Sigma_{1,t}$,
\begin{align}
\P_k (\{r \in \reg_t : & J_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_{J_t})} \neq \theta_{J_t} \}) \nonumber \\
& \leq \text{EstimateEvent}(P_k, \reg_t(x_{J_t}) \cup \reg_t((x_{J_t},\theta_{J_t})), \frac{1}{32n^2}, \delta_t ) + \frac{1}{32n^2} \label{eq:best_arm_est_event_apply_1} \\
& \leq \text{EstimateEvent}(P_k, \reg_t(x_{I_t}) \cup \reg_t((x_{I_t},\bar{y})), \frac{1}{32n^2}, \delta_t ) + \frac{1}{32n^2} \label{eq:best_arm_est_def} \\
& \leq \P_k (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_{I_t})} \neq \bar{y} \}) + \frac{1}{16n^2} \label{eq:best_arm_est_event_apply_2} \\
& \leq \P_k (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_{I_t})} \neq \theta_{I_t} \}) + \frac{1}{16n^2} \label{eq:best_arm_est_loss}
\end{align}
where lines \eqref{eq:best_arm_est_event_apply_1} and \eqref{eq:best_arm_est_event_apply_2} follow from event $\Sigma_{1,t}$, line \eqref{eq:best_arm_est_def} follows from the definition of $I_t$ and $\bar{y}$, and line \eqref{eq:best_arm_est_loss} used the definition of $\theta_{I_t}$.
From this, it follows that
\begin{align}
\P_k(\reg_t) & = \P_k (\{r \in \reg_t : \exists i \in I \text{ s.t. } i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& \leq \sum_{i \in I} \P_k (\{r \in \reg_t : i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& \leq (\bestext+1) \max_{i \in I} \P_k (\{r \in \reg_t : i \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_i \}) \nonumber \\
& = (\bestext+1) \P_k (\{r \in \reg_t : J_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_{I_t} \}) \nonumber \\
& = (\bestext+1) [ \P_k (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq \theta_{I_t} \}) + \frac{1}{16 n^2}] \label{eq:efficient_I_t_def_approx}\\
& \leq (\bestext+1) [ \P_k (\{r \in \reg_t : I_t \in \argmin_{l \in [n]} \round{r(x_l)} \text{ or } \round{r(x_i)} \neq y_{I_t} \}) + \frac{1}{16n^2}]\label{eq:efficient_theta_def_approx} \\
& = (\bestext+1) [\P_k (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) + \frac{1}{16n^2}] \nonumber
\end{align}
where line \eqref{eq:efficient_I_t_def_approx} uses \eqref{eq:best_arm_est_loss}, and the line \eqref{eq:efficient_theta_def_approx} comes from the definition of $\theta$. Noticing that
\begin{align*}
\reg_t & = (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \cup \reg_t \setminus (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \\
& = (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \cup \reg_{t+1},
\end{align*}
we have that
\begin{align*}
\P_k(\reg_{t+1}) & = \P_k(\reg_t) - \P_k(\reg_t(x_{I_t}) \cup \reg_t((x_{I_t}, y_{I_t})) \leq (1 - \frac{1}{\bestext+1})\P_k(\reg_t) + \frac{1}{16 n^2}.
\end{align*}
Unraveling this recursive statement, we have that
\begin{align*}
\P_k(\reg_{t+1}) & \leq (1 - \frac{1}{\bestext+1})^{t-t_k} + (t-t_k) \frac{1}{16n^2} \\
& \leq (1 - \frac{1}{\bestext+1})^{t-t_k} + n\frac{1}{16n^2} \\
& \leq (1 - \frac{1}{\bestext+1})^{t-t_k} + \frac{1}{16n} .
\end{align*}
where we used the fact that the game lasts at most $n$ rounds because it is the noiseless setting. Thus, we see that in phase $k$ at $t = t_k + s_k$ with $s_k := O(\ln(n) \bestext)$, we have that
\begin{align}
\P_k(\reg_{t_k+s_k}) \leq \frac{1}{8n}. \label{eq:shrink_objective_approx}
\end{align}
\textbf{Step 2.1: Necessary condition for entering the next phase.} Now, we show that if Algorithm \ref{alg:effic_haystack_alg_approx} enters the phase $k+1$, then $\P_{f \sim P_k}(r \in \reg_{t+1}) < \frac{1}{2n}$. note that if $\frac{1}{4n} \geq \text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t )$, then
\begin{align*}
\frac{1}{4n} \geq \text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) \geq \P_{f \sim P_k}(r \in \reg_{t+1}) - \frac{1}{8 n}
\end{align*}
implying that
\begin{align*}
\P_{f \sim P_k}(r \in \reg_{t+1}) < \frac{1}{2n}.
\end{align*}
\textbf{Step 2.2: Necessary condition for not entering the next phase.} Now, we show that if $\P_{f \sim P_k}(r \in \reg_{t+1}) \geq \frac{1}{2n}$, then Algorithm \ref{alg:effic_haystack_alg_approx} does not the phase $k+1$. Suppose $\P_{f \sim P_k}(r \in \reg_{t+1}) \geq \frac{1}{2n}$. Then,
\begin{align*}
\frac{1}{2n} \leq \P_{f \sim P_k}(r \in \reg_{t+1}) \leq \text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) + \frac{1}{8 n} ,
\end{align*}
implying that $\text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) > \frac{1}{4n}$ and the algorithm does not update the phase.
Given these conditions, the rest of the proof is identical to the proof of Theorem \ref{thm:haystack_eff}.
\end{proof}
\subsection{Cumulative Loss Minimization}
For simplicity, in this section, we assume that $\min_{y \in \Y}y \geq 1$. One can simply add a constant to $\reg$ to arrive at this setting. We use the convention that $\frac{1}{0} = \infty$.
\begin{algorithm}[t]
\small
\textbf{Input:} distribution $P$ ; event $\Sigma$, multiplicative accuracy $\epsilon \in (0,1)$, failure probability $\delta \in (0,1)$\;
$s \longleftarrow 0$\;
\While{$ \sqrt{\frac{2\log(\frac{s^2 \pi^2}{\delta 6})}{s}} \geq \frac{1}{2n^2 \max_{y \in \Y} y}$}{
Sample $z_s \sim P$\;
Form $\widehat{\mu}_s = \frac{\sum_{i=1}^s \one \{ z_i \in \Sigma\} }{s} $\;
\If{$\epsilon \cdot \widehat{\mu}_s \geq \sqrt{2\log(\frac{s^2 \pi^2}{\delta 6})/s} $}{
\Return $\widehat{\mu}_s$}
$s \longleftarrow s +1$\;
}
\Return $0$
\caption{EstimateEventMult}
\label{alg:est_event_mult}
\end{algorithm}
\begin{algorithm}[t]
\small
$P_1 \longleftarrow \pi$, $\reg_1 \longleftarrow \reg$, $O_1 \longleftarrow \emptyset$ \;
$k \longleftarrow 1$, $t_1 = 1$, $\delta_t = \frac{\delta}{2t^2 |\Y| n}$\;
\For{$t =1,2,\ldots$}{
Let $I_t \in \argmin_{i \in [n] \setminus O_t } \max_{y \in \Y} \frac{y}{\text{EstimateEventMult}(P_k, \reg_t(x_i) \cup \reg_t((x_i,y)), \frac{1}{2}, \delta_t )}$\;
Query $x_{I_t}$ and observe $y_{I_t}$ and set $O_{t+1} \longleftarrow O_t \cup \{I_t\}$ \;
Let $\reg_{t+1} \longleftarrow \reg_t \setminus (\reg_t(x_{I_t}) \cup \reg_t((x_{I_t},y_{I_t})))$\;
\If{$\text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) \leq \frac{1}{4n} $}{
$P_{k+1} \longleftarrow \frac{1}{n-|O_{t_k}|} \sum_{l \in [n] \setminus O_{t_k} } \pi_{C_l(O_{t_k}))}$\;
$k \longleftarrow k+1$\;
$t_k \longleftarrow t+1$\;
}
\If{\text{STOP}($\F_\reg$,$O_t$)}{
\Return $\argmin_{i \in O_t} y_i$}
}
\caption{\texttt{GRAILS} for Loss Minimization with Estimation}
\label{alg:effic_haystack_alg_loss_approx}
\end{algorithm}
\begin{lemma}\label{lem:est_mult}
Let $\Sigma$ denote some event that has positive probability under $P$ such that $\epsilon(1-\epsilon) \P(\Sigma) \geq \frac{1}{2n^2 \max_{y \in \Y} y}$. With probability at least $1-\delta$, Algorithm \ref{alg:est_event_mult} satisfies
\begin{align*}
(1-\epsilon)\text{EstimateEventMult}(P, \Sigma, \epsilon, \delta ) \leq P(\Sigma) \leq (1+\epsilon ) \text{EstimateEventMult}(P, \Sigma, \epsilon, \delta )
\end{align*}
\end{lemma}
\begin{proof}[Proof of Lemma \ref{lem:est_mult}]
Define the event
\begin{align*}
E & = \{|P(\Sigma) - \widehat{\mu}_s| \leq \sqrt{\frac{2\log(\frac{s^2 \pi^2}{\delta 6})}{s}} \, \forall s \in \N \}
\end{align*}
By a union bound and a standard Hoeffding bound, $\P(E) \geq 1-\delta$. Suppose $E$ holds for the remainder of the proof.
\textbf{Step 1.} First, we show that if $\epsilon(1-\epsilon) \P(\Sigma) \geq \frac{1}{n^3 \max_{y \in \Y}}$, then Algorithm \ref{alg:est_event_mult} does not return $0$. Towards a contradiction, suppose the algorithm returns $0$ at round $s_0$. Then, we have on the event $E$ that
\begin{align*}
|P(\Sigma) - \widehat{\mu}_{s_0}| \leq \sqrt{\frac{2\log(\frac{s_0^2 \pi^2}{\delta 6})}{s_0}} \leq \frac{1}{2n^2 \max_{y \in \Y} y} \leq \epsilon(1-\epsilon) \P(\Sigma).
\end{align*}
Rearranging we have that
\begin{align*}
P(\Sigma) \leq \frac{1}{1-\epsilon} \widehat{\mu}_{s_0},
\end{align*}
which implies
\begin{align*}
\sqrt{\frac{2\log(\frac{s^2 \pi^2}{\delta 6})}{s}} \leq \frac{1}{2 n^2 \max_{y \in \Y}} \leq \epsilon(1-\epsilon) \P(\Sigma) \leq \epsilon \widehat{\mu}_{s_0}.
\end{align*}
But this implies the algorithm would not have returned $0$, yielding a contradiction.
\textbf{Step 2.} Suppose that Algorithm \ref{alg:est_event_mult} does not return $0$. Then, there exists a round $s_0 \in \N$ at which
\begin{align*}
\sqrt{\log(\frac{s_0^2 \pi^2}{\delta 6})/s_0} \leq \epsilon \widehat{\mu}_{s_0}.
\end{align*}
Then, by the event $E$, we have that
\begin{align*}
P(\Sigma) \leq (1+\epsilon) \widehat{\mu}_{s_0}.
\end{align*}
The other inequality follows as well, completing the proof.
\end{proof}
\begin{theorem}\label{thm:loss_min_eff_approx}
With probability at least $1-\delta$, Algorithm \ref{alg:effic_haystack_alg_loss_approx} incurs a loss of at most $2(\lossext + \max_{i \in [n],f \in \F_{\reg}} f(x_i)) \log(n) \log(\frac{1}{\P(\Smin)})$ to identify $\argmin_{i \in [n]} f^*(x_i)$.
\end{theorem}
\begin{proof}
The proof is very similar to the proof of Theorem \ref{thm:loss_min_eff}.
\textbf{Step 0. Defining the good event.} Define
\begin{align*}
\Sigma_{1,t} & = \{\forall y \in \Y, \forall i \in [n]: \P_{f \sim P_k}(f \in \reg_t(x_i) \cup \reg_t((x_i,y)) \geq \frac{2}{ n^2 \max_{y \in \Y} y} \Longrightarrow & \\
& \hspace{1cm} \frac{1}{2}\text{EstimateEventMult}(P_k, \reg_t(x_i) \cup \reg_t((x_i,y), \frac{1}{2}, \delta_t ) \\
& \hspace{1cm} \leq \P_{f \sim P_k}(f \in \reg_t(x_i) \cup \reg_t((x_i,y))) \\
& \hspace{1cm} \leq \frac{3}{2}\text{EstimateEventMult}(P_k, \reg_t(x_i) \cup \reg_t((x_i,y)), \frac{1}{2}, \delta_t ) \} \\
\Sigma_{2,t} & = \{ |\text{EstimateEvent}(P_k, \reg_{t+1}, \frac{1}{8n}, \delta_t ) - \P_{f \sim P_k}(f \in \reg_{t+1})| \leq \frac{1}{8 n} \} \\
\Sigma_t & = \Sigma_{1,t} \cap \Sigma_{2,t} \\
\Sigma & = \cap_{t=1}^\infty \Sigma_t .
\end{align*}
By the total law of probability, a Standard Hoeffding bound, and Lemma \ref{lem:est_mult}, we have that
\begin{align*}
\P(\Sigma^c) & \leq \sum_{t=1}^\infty \P(\Sigma_t^c | \cap_{s=1}^{t-1} \Sigma_{s} )\\
& \leq \sum_{t=1}^\infty \P(\Sigma_{t,1}^c | \cap_{s=1}^{t-1} \Sigma_{s} ) + \P(\Sigma_{t,2}^c | \cap_{s=1}^{t-1} \Sigma_{s} )\\
& \leq \frac{6}{\pi^2} \sum_{t=1}^\infty \frac{\delta}{t^2} \\
& \leq \delta.
\end{align*}
For the rest of the proof, we will condition on the event $\Sigma$.
\textbf{Step 1:} Define the function $g : \{x_1,\ldots, x_n\} \mapsto \Y$ in the following way:
\begin{align*}
g(x_i) = \begin{cases}
\round{\tilde{r}(x_i)} \text{ s.t. } \tilde{r} = \argmax_{\bar{r} \in \reg_t} \frac{\round{\bar{r}(x_i)}}{\P_k(\reg_t(x_i) \cup \reg((x_i,\bar{f}(x_i)) )} & i \not \in \{I_1,\ldots, I_{t-1}\} \\
f^*(x_i) & \text{ o/w}
\end{cases}
\end{align*}
Define
\begin{align*}
J_t & := \argmin_{j \in [n] \setminus O_t} \frac{g(x_{j})}{\P_k(\reg_t(x_{j}) \cup \reg_t((x_i,g(x_{j})) )},
\end{align*}
the arm that would be pulled at round $t$ if the probabilities were known exactly. Note that
We have that
\begin{align*}
& \hspace{-1.5cm} \frac{g(x_{J_t})}{\P_k(\reg_t(x_{J_t}) \cup \reg_t((x_i,g(x_{J_t})) )}
\\
& \leq \frac{\sum_{i \in [n] \setminus O_t} g(x_i)}{\sum_{i \in [n] \setminus O_t} \P_k(\{ r \in \reg_t : \round{r(x_i)} \neq g(x_i) \text{ or } i \in \argmin_{j \in [n]} \round{r(x_j)} \})} \\
& \leq \frac{\sum_{i \in [n] \setminus O_t} g(x_i)}{ \P_k(\reg_t)} \\
& \leq \frac{n \max_{y \in \Y} y}{\P_k(\reg_t)} \\
& \leq n^2 \max_{y \in \Y} y,
\end{align*}
where the last line uses Step 2.2 in the proof of Theorem \ref{thm:haystack_eff_approx}, which shows that $\P_k(\reg_t) \geq \frac{1}{2n}$. This implies that
\begin{align*}
\P_k(\reg_t(x_{J_t}) \cup \reg((x_i,g(x_{J_t})) ) \geq \frac{g(x_{J_t})}{c n^3 \max_{y \in \Y} y} \geq \frac{1}{c n^3 \max_{y \in \Y} y}
\end{align*}
where we used the fact that $\min_{y \in \Y} y \geq 1$ by assumption.
Thus, by event $\Sigma_{1,t}$, we have that
\begin{align*}
& \frac{1}{2}\text{EstimateEventMult}(P_k, \reg_t(x_{J_t}) \cup \reg_t((x_{J_t},g(x_{J_t})), \frac{1}{2}, \delta_t ) \\
\leq & \P_{f \sim P_k}(f \in \reg_t(x_{J_t}) \cup \reg_t((x_{J_t},g(x_{J_t})))) \\
\leq & \frac{3}{2}\text{EstimateEventMult}(P_k, \reg_t(x_{J_t}) \cup \reg_t((x_{J_t},g(x_{J_t})), \frac{1}{2}, \delta_t )
\end{align*}
Then, by standard arguments, it can be shown that Algorithm \ref{alg:effic_haystack_alg_loss_approx} chooses an arm $I_t$ that satisfies
\begin{align*}
\frac{g(x_{I_t})}{\P_{f \sim P_k}(f \in \reg_t(x_{I_t}) \cup \reg_t((x_{I_t},g(x_{I_t}))) }\leq c \frac{g(x_{J_t})}{\P_{f \sim P_k}(f \in \reg_t(x_{J_t}) \cup \reg_t((x_{J_t},g(x_{J_t})))}
\end{align*}
where $c$ is a universal constant.
\textbf{Step 2: } The only difference in the proof with Theorem \ref{thm:loss_min_eff} comes in lines \eqref{eq:loss_min_eff_diff_line_1} and \eqref{eq:loss_min_eff_diff_line_2}. In these lines, we lose a mulitiplicative constant factor from using $\text{EstimateEventMult}(P_k, \reg_t(x_i) \cup \reg_t((x_i,y), \frac{1}{2}, \delta_t ) $ instead of the exact probability, as shown in the previous step. The argument for changing phases is identical to the argument in the proof of Theorem \ref{thm:haystack_eff_approx}. The rest of the proof for bounding the cumulative loss incurred is identical.
\textbf{Step 3:} The total number of samples taken is a polynomial function of $n$, $\max_{y \in \Y} y$, and $\log(1/\delta)$ by definition of Algorithm \ref{alg:est_event_mult}.
\end{proof}
\section{Discussion of Function Classes}
\begin{proof}[Proof of Proposition \ref{prop:conjun_class}]
$\F_\reg$ consists of functions $f_1,\ldots, f_n$ of the following form
\begin{align*}
f_j(x_i) & \begin{cases}
10 & i = j+1 \\
1 & i \geq j \\
0 & i < j
\end{cases}.
\end{align*}
Suppose that $g:\X \mapsto \Y$ is such that $g \not \in \F_{\reg}$. Then, a simple computation shows that at most $3$ measurements are required to eliminate all functions from the version space. On the other hand, if $g = f_j$ for some $f_j \in \in \F_{\reg}$, then it suffices to query $x_j$ and $x_{j-1}$. Therefore, a simple computation shows that $\bestext(\F_{\reg}) \leq 3$.
Since $\pi$ is uniform over $[0,1]$, we have that $\P_{\pi}(S_{f^*}) \geq \frac{c}{n}$ for some positive universal constant. Thus, the result follows.
\end{proof}
\textbf{Linear Separators $\epsilon$-good arm identification:} We give an intuitive sufficient condition for lower bounding $\P_{\pi}(S_f)$.
\begin{assumption}\label{assum:pos_margin}
Fix $y_1, \ldots, y_n \in \Y$. Define the \emph{minimum margin}:
\begin{align*}
\margin := \max_{r^* \in \reg : \round{r^*(x_i)} = y_i \forall i \in [n]} \min_{i \in [n]} \min_{y \neq y_i} |r^*(x_i) - y| - |r^*(x_i) - y_i|.
\end{align*}
\end{assumption}
The following Proposition lower bounds $\P_\pi(S_{f^*})$ for linear functions.
\begin{proposition}\label{prop:ex_dis_eps_good_lin}
Let $\reg = \{ \langle a,\cdot \rangle : a \in \R^d, \norm{a} \leq R_1 \} $. Let $x_1,\ldots, x_n \in \R^d$ such that $\max_{i \in [n]} \norm{x_i}_* \leq R_2$ where $\norm{\cdot}_*$ denotes the dual norm of $\norm{\cdot}$. Let $y_1,\ldots, y_n \in \Y$. Suppose that $\gamma^* > 0$. Let $f^* = \round{\langle a_* , \cdot \rangle}$ for some $a_*$ with $\norm{a_*} \leq R_1$. Let $\pi$ be the uniform distribution over $\{a : \norm{a} \leq R_1\}$. Then,
\begin{align*}
\P_\pi(S_{f^*}) & \geq \frac{\text{volume}(\{z \in \R^{d} : \norm{z} \leq \frac{\gamma^*}{R_2} \}}{\text{volume}(\{z \in \R^{d} : \norm{z} \leq R_1\}) }.
\end{align*}
In particular, if $\norm{\cdot} = \norm{\cdot}_2$, then $\P_\pi(S_{f^*}) \geq (\frac{ \margin}{ R_1 R_2 })^{d}$.
\end{proposition}
The above Proposition shows that for linear separators with a Euclidean constraint and a margin $\gamma^*$, $\log(\frac{1}{\P_\pi(S_{f^*})}) \lesssim d \log(\gamma^*)$.
\begin{proof}[Proof of Proposition \ref{prop:ex_dis_eps_good_lin}]
By assumption $\margin > 0$. Define $\delta_0 = \frac{\margin}{6 R_2}$. Let $f^*(\cdot) = \langle a_*, \cdot \rangle$ where $a_* \in \R^d$ and $\norm{a_*} \leq R_1$. Define
\begin{align*}
\bar{B}_{\alpha} & = B_{\delta_0}(\alpha a_*)
\end{align*}
where $B_\gamma(v) := \{v \in \R^d : \norm{v} \leq \gamma \}$.
\textbf{Step 1. $\bar{B}_{\alpha}$ is feasible.} We claim that for all $\alpha \in [0, 1-\delta_0/R_1]$, $\bar{B}_\alpha \subset \{a : \norm{a} \leq R_1\}$. Let $\alpha \in [0, 1-\delta_0/R_1]$. Let $v$ such that $\norm{v} \leq \delta_0$. Then,
\begin{align*}
\norm{\alpha a_* + v} \leq \alpha \norm{a_*} + \norm{v} \leq (1-\delta_0/R_1) \norm{a_*} + \norm{v} \leq R_1 - \delta_0 + \delta_0 = R_1,
\end{align*}
showing the claim.
\textbf{Step 2. $a_* \in \bar{B}_{\alpha_0}$ for some $\alpha_0$} Let $\alpha_0= \max(1-\delta_0/ \norm{a_*},0) \leq 1-\delta_0/ R_1 $. We claim that $a_* \in \bar{B}_{\alpha_0}$. If $1-\delta_0/ \norm{a_*} \geq 0$, then
\begin{align*}
\norm{(\alpha_0 -1) a_*} = \frac{\delta_0}{\norm{a_*}} \norm{a_*} = \delta_0
\end{align*}
so that $a_* \in \bar{B}_{\alpha_0}$. On the other hand, if $1-\delta_0/ \norm{a_*} < 0$, then $\norm{a_*} < \delta_0$, so that $a_0 \in \bar{B}_{\alpha_0}$ since $\alpha_0 = 0$.
\textbf{Step 3. Putting it together.} Now, let $a \in \bar{B}_{\alpha_0}$. Then,
\begin{align*}
\norm{a_*-a} \leq 2 \delta_0.
\end{align*}
Then, using Holder's inequality,
\begin{align*}
|a_*^\t x_i - (a^\t x_i )| \leq \norm{a_*-a} \max_{i} \norm{x_i}_* \leq 2 \delta_0 R_2 \leq \frac{\margin }{3}
\end{align*}
by definition of $\delta_0$.
% Thus, if $\norm{a - a_*} < \frac{\margin}{2R_2}$, we have by Holder's inequality, for all $i \in [n]$
%\begin{align*}
%|\langle a -a_*, x_i \rangle| \leq \norm{a - a_*} \norm{x_i}_* < \frac{\margin}{R_2} R_2 = \frac{\margin}{2}.
%\end{align*}
Let $y \neq y_i$. Then,
\begin{align*}
|\langle a, x_i \rangle - y | - |\langle a, x_i \rangle - y_i | & > |\langle a_*, x_i \rangle - y_i | - |\langle a_*, x_i \rangle - y_i | - \frac{2}{3}\margin \\
& > 0
\end{align*}
where the last line uses the definition of $\margin$. Thus, we have that if $f = \langle a, \cdot, \rangle \in S_{f^*}$. Thus,
\begin{align*}
\P_\pi(S_{a_0}) & \geq \frac{\text{volume}(\{z \in \R^{d} : \norm{z} \leq \delta_0 \}}{\text{volume}(\{z \in \R^{d} : \norm{z} \leq R_1\}) }.
\end{align*}
The result for the case $\norm{\cdot} = \norm{\cdot}_2$ follows from standard formula for the volume of an $n$-dimensional ball.
%\begin{align*}
%|\langle a, x_i \rangle - y_i | - |\langle a, x_i \rangle - y_i | & = |\langle a, x_i \rangle - \langle a_*, x_i \rangle + \langle a_*, x_i \rangle - y_i | \\
%& \leq |\langle a-a_*, x_i \rangle| + |\langle a_*, x_i \rangle - y_i | \\
%& \leq \frac{\margin}{2} + |\langle a_*, x_i \rangle - y_i | \\
%& \leq \frac{\margin}{2} + |\langle a_*, x_i \rangle - y | - \margin \\
%& \leq |\langle a, x_i \rangle - y |-\frac{\margin}{2},
%\end{align*}
%which implies that $|\langle a, x_i \rangle - y | - |\langle a, x_i \rangle - y_i | \geq \frac{\margin}{2}$.
\end{proof}
Next, we give a simple bound on $\bestextep$ for linear functions. For simplicity, we let $|\Y|$ be countably infinite, but it could be extended to the $|\Y| < \infty$ case.
\begin{proposition}\label{prop:lin_eps_extended_bound}
Suppose $\Y = \{\ldots, -2\Delta, -\Delta, 0, \Delta, 2\Delta, \ldots \}$ where $\Delta > 0$. Let $\reg = \{ r : r(\cdot) = \langle a , \cdot \rangle, a \in \R^d\}$. Let $x_1,\ldots, x_n \in \R^d$ such that $\max_{i \in [n]} \norm{x_i}_i \leq 1$. Define
\begin{align*}
\phi := \min_{i_1,\ldots, i_d \in [n]} \sigma_d\left(\begin{pmatrix}
x_{i_1}^\t \\
\vdots \\
x_{i_d}
\end{pmatrix}\right)
\end{align*}
where $\sigma_d(A)$ denotes the $d$th singular value of a matrix $A$. Then if $\Delta \leq \min(\frac{\phi \epsilon}{ 4\sqrt{d}},\frac{\epsilon}{2})$, $\bestextep\leq d$.
\end{proposition}
This result, combined with Proposition \ref{prop:ex_dis_eps_good_lin}, suggests that \texttt{GRAILS} passes the sanity check of not sampling too many linearly dependent arms for the linear functions case.
\begin{proof}[Proof of Proposition \ref{prop:lin_eps_extended_bound}]
\textbf{Step 1:} Let $g : \R^d \mapsto \Y$ be an arbitrary mappying. By assumption, there exists $i_1,\ldots, i_d \in [n]$ such that defining $\bar{X} : = \begin{pmatrix}
x_{i_1}^\t \\
\vdots \\
x_{i_d}^\t
\end{pmatrix}$, we have that $\sigma_d(\bar{X}) = \phi$. Suppose that the algorithm queries $i_1,\ldots, i_d$. If there is no $a_* \in \R^d$ such that $g(x_{i_j}) = \round{a_*^\t x_{i_j}}$ for all $j \in [d]$, then we are done. So, assume that there exists $a_* \in \R^d$ such that $g(x_{i_j}) = \round{a_*^\t x_{i_j}}$ for all $j \in [d]$. Suppose wlog that $x_1 \in \argmin_{i \in [n]} a_*^\t x_i$. Now, we will show that all models consistent with $g(x_{i_j}) $ for all $j \in [d]$ put $x_1$ as an $\epsilon$-good arm. Let $a \in \R^d$ be another vector such that
\begin{align*}
\round{a^\t x_{i_j}} = g(x_{i_j})
\end{align*}
for all $j \in [d]$. Then, we have that
\begin{align*}
\Delta \sqrt{d} \geq \norm{\bar{X}(a_* - a)} \geq \sigma_d(\bar{X}) \norm{a -a_*}.
\end{align*}
Let $x_i \neq x_1$. Note that by definition of the rounding operator and $\Y = \{\ldots, -2\Delta, -\Delta, 0, \Delta, 2\Delta, \ldots \}$ ,
\begin{align*}
a_*^\t x - \frac{\Delta}{2} \leq \round{a_*^\t x} \leq a_*^\t x + \frac{\Delta}{2}
\end{align*}
for all $x$.
We have that
%\begin{align*}
% \round{a^\t x_1}-\round{a^\t x_i } - \Delta \leq a^\t (x_i - x_1) & \leq (a - a_* )^\t ( x_1 - x_i) + a_*^\t (x_1-x_i) \\
%& \leq \norm{a-a_*} \norm{x_i-x_1} + a_*^\t (x_1-x_i) \\
%& \leq \frac{\Delta \sqrt{d}}{ \phi} 2 + a_*^\t (x_1-x_i) \\
%& \leq \frac{\Delta \sqrt{d}}{ \phi} 2 + \Delta + \round{a_*^\t x_i } - \round{a_*^\t x_1} \\
%& \leq \frac{\Delta \sqrt{d}}{ \phi} 2 + \Delta \\
%& \leq \epsilon
%\end{align*}
\begin{align*}
\round{a^\t x_1}-\round{a^\t x_i } - \Delta \leq a^\t (x_i - x_1) & \leq (a - a_* )^\t ( x_1 - x_i) + a_*^\t (x_1-x_i) \\
& \leq \norm{a-a_*} \norm{x_i-x_1} + a_*^\t (x_1-x_i) \\
& \leq \frac{\Delta \sqrt{d}}{ \phi} 2 + a_*^\t (x_1-x_i) \\
& \leq \frac{\epsilon}{2}
\end{align*}
where we used $ a_*^\t x_1 - a_*^\t x_i \leq 0$ by definition of $x_1$ and the setting of $\Delta$. Thus, rearranging, we have that
\begin{align*}
\round{a^\t x_1}-\round{a^\t x_i } \leq \Delta + \frac{\epsilon}{2} \leq \epsilon
\end{align*}
implying that $x_1$ is an $\epsilon$-good arm for any $a$ such that
\begin{align*}
\round{a^\t x_{i_j}} = g(x_{i_j}).
\end{align*}
This completes the proof.
%\textbf{Step 2:} Now, suppose that $g : \R^d \mapsto \Y$ is such that there does not exist $a_* \in \R^d$ such that $g(x_i) = \round{a_*^\t x_i}$. Query $i_1,\ldots, i_d$ as in step 1. Let $
\end{proof}
\textbf{Active Classification with Linear Separators:} Next, we lower bound the minimum probability for linear separators under a certain geometric that can be interpreted as a quantitative version of the concept of general position. Define
\begin{align*}
\Gamma(\X) & = \{(y_1,\ldots, y_n)^\t \in \{-1,1\}^n : \exists w \in \R^d, \, b \in \R \text{ s.t. } \sign(w^\t x_i + b) = y_i \, \forall i \in [n] \}.
\end{align*}
\begin{proposition}\label{prop:class_prob_lower_bound}
Let $\gamma > 0$. Suppose $\X = \{x_1,\ldots, x_n\}$ satisfy $\forall S \subset \X$ such that $|S| \leq d-1$,
\begin{align*}
\min_{x \in \X \setminus S} \dist(x, \aff(S)) \geq \gamma.
\end{align*}
Suppose $\max_{i \in [n]} \norm{x_i} \leq R$. Let $\F_R = \{ \langle w , \cdot \rangle + b : (w^\t,b)^\t \in W \}$ and $W = \{(w^\t,b)^\t : \norm{w}_2^2 + b^2 \leq R^2 \}$.
%where $R$ is large enough that such that every for every $\X_+, \X_-$ such that $\X = \X_+ \cup \X_-$ and $\X_+ \cap \X_- = \emptyset$ such that there exists a max-margin linear separator $f$ of $\X_+$ and $\X_-$, then $f \in \F_{B}$. Define
where $R$ is large enough such that for every $(y_1,\ldots, y_n)^\t \in \Gamma(\X)$, the max-margin separator achieving $(y_1,\ldots, y_n)^\t $ belongs to $\F_R$. Let $\pi$ denote the uniform probability distribution over $W$. Then,
\begin{align*}
\min_{(y_1,\ldots, y_n)^\t \in \Gamma(\X)} \P_{(w^\t,b)^\t \sim \pi}(\sign( w^\t x_i + b) = y_i \, \forall i \in [n] ) \geq (\frac{\gamma}{R})^d
\end{align*}
\end{proposition}
Thus, under the condition in the Proposition \ref{prop:class_prob_lower_bound}, the sample complexity in Theorem \ref{thm:active_upper_bound} scales as $c d\log(\gamma) \classext(\F_{\reg})$. Often $\log(|\F_{\reg}|) = \Omega( d)$, in which case Theorem \ref{thm:active_upper_bound} is larger than the active classification upper bound in \cite{hegedHus1995generalized} by a factor of $\log(\gamma)$. On the other hand, Algorithm \ref{alg:effic_active_class} is computationally efficient.
\begin{proof}[Proof of Proposition \ref{prop:class_prob_lower_bound}]
Fix $(\bar{y}_1,\ldots, \bar{y}_n)^\t \in \Gamma(\X)$. Let $f = \langle \bar{w} , \cdot \rangle + \bar{b} \in \F_B $ attain the max-margin separator achieving $(\bar{y}_1,\ldots, \bar{y}_n)^\t $, which exists by assumption. Let $v_1,\ldots, v_l$ denote the support vectors with positive labels and $u_1, \ldots, u_k$ the support vectors with negative labels. By definition of support vectors, we have that
\begin{align*}
\aff(v_1,\ldots, v_l) = \aff(u_1,\ldots, u_k) + z
\end{align*}
for some $z \in \R^d$. We claim that $\norm{z}_2 \geq \gamma$. Towards a contradiction, suppose not. Then,
\begin{align*}
\dist(v_1, \aff(u_1,\ldots, u_k)) = \norm{z}_2 < \gamma
\end{align*}
which contradicts our assumption. This implies that the margin is at least $\gamma/2$ and that
\begin{align*}
|\bar{w}^\t x -\bar{b}| \geq \frac{\gamma}{2}
\end{align*}
for all $x \in \X$. Then, for any $w,b$ such that $\norm{w}^2_2 + b^2 \leq [\gamma/4]^2$, we have that
\begin{align*}
|\bar{w}^\t x - \bar{b} - [(\bar{w}+w)^\t x - \bar{b}-b ]| = |w^\t x -b| \leq 2\norm{(w^\t, b)^\t}_2 \leq \frac{\gamma}{2}
\end{align*}
The result follows by Proposition \ref{prop:ex_dis_eps_good_lin}.
\end{proof}
%\begin{lemma}
%Let $P = \{x \in \R^d : A x \leq b \}$ denote a polyhedron where $A \in \R^{m \times d}$ and $b \in \R^m$. Let $v \in \R^d$ be a vertex of $P$. Then, there exists vertices $v_1,\ldots, v_d \in \R^d$ of $P$ such that
%\begin{align*}
%P \subset \{\sum_{i=1}^d v + \alpha_i(v_i - v) : \alpha_i \geq 1 \forall i \in [d]\}.
%\end{align*}
%
%\end{lemma}
%
%\begin{proof}
%Since $v$ is a vertex, there exists rows of $A$ $a_1,\ldots, a_d$ that are linearly independent and such that $a^\t_i v = b_i$ for all $i \in [d]$ \jks{deal with the low rank case separately}.
%
%%To begin, suppose for some $i$ there is no $v_i \in P$ such that $v_i \neq v$ such that $v_i^\t a_j = b_j \forall j \in [d] \setminus \{i\}$. Now, let $z \in P$ such that $z^\t a_i \neq 0$. Then, $(1-z
%
%Suppose that for all $i \in [d]$, there exists a $v_i \neq v$ such that $v_i^\t a_j = b_j$ for all $j \in [d] \setminus \{i\}$ (\jks{figure out the other case}).
%
%\textbf{Claim: } $\{v_i - v\}_{i \in [d]}$ is a collection of linearly independent vectors.
%
%Note that since $A$ has full rank and $v_j \neq v$, for all $j \in [d]$ $A(v_j-v) = \beta_j e_j$ for some $\beta_j \in \R$. Therefore, $\{A(v_j-v) \}_{j \in [d]}$ are linearly independent. Therefore, we must have that $\{v_j-v\}_{j \in [d]}$ are linearly independent. This proves the claim.
%
%Let $u \in P$. Since $\{v_j-v\}_{j \in [d]}$ are linearly independent, we have that there exists $\alpha_1,\ldots, \alpha_d \in \R$ such that $u = v + \sum_{j=1}^d \alpha_j (v_j-v)$. Now, we claim that $\alpha_j \geq 1$. Suppose that there exists $i$ with $\alpha_i < 1$. Let $a_i^\t v_i = b_i - \epsilon$ for some $\epsilon > 0$. Then,
%\begin{align*}
%a_i^\t u = a_i^\t [v + \sum_{j=1}^d \alpha_j (v_j-v)] = b_i + \sum_{j\neq i} \alpha_j(b_i-b_i) + \alpha_i( b_i - \epsilon-b_i) = b_i + \alpha_i (-\epsilon) > b_i
%\end{align*}
%implying that $u \not \in P$, a contradiction. Thus, the result follows.
%
%
%
%%Define the matrix
%%\begin{align*}
%%A_j & = \begin{pmatrix}
%%a_1^\t \\
%%a_2^\t \\
%%\vdots \\
%%a_{j-1}^\t \\
%%a_{j+1}^\t \\
%%\vdots \\
%%a_d^\t
%%\end{pmatrix}
%%\end{align*}
%%Note that $A_j$ has rank $d-1$ and that $A_j(v_j-v) = 0$. We show the result inductively. Suppose that there exists $\alpha \neq 0$ such that $\alpha (v_1-v) = v_2-v$. Then, $A_1(v_2-v) = 0$ and $A_2(v_2-v) = 0$. But, this implies that $A(v_2-v) = 0$, which then implies by the full rank of $A$, $v_2=v$--a contradiction. Now, suppose that $\{v_i - v\}_{i \in [j-1]}$ are linearly independent. Towards a contradiction, suppose that there exists $\alpha_1,\ldots, \alpha_{j-1}$ not all zero such that
%%\begin{align*}
%%v_j-v = \sum_{i=1}^{j-1} \alpha_i (v_i-v).
%%\end{align*}
%%But, then, we have that
%%\begin{align*}
%%0 = A_j(v_j-v) = \sum_{i=1}^{j-1} \alpha_i A_j (v_i-v)
%%\end{align*}
%%which implies that one of
%
%
%\end{proof}
%\begin{proof}[Proof of Proposition \ref{prop:circle_bound}]
%We note that the conditions of Proposition \ref{prop:lin_eps_extended_bound} are satisfied, and hence $\bestext(\F_{\reg}) \leq 2$. Thus, it suffices to upper bound $\log(\frac{1}{\P_{\pi}(S_{f^*})}) $. Because of the symmetry of the problem, each $f \in \F_{\reg}$ has the same probability and the bound follows.
%\end{proof}
%\begin{proof}[Proof of Proposition \ref{prop:thresholds}]
%\textbf{Step 1:} First we bound $\bestext$. Fix $g: \X \mapsto \Y$ such that $g(x_i) = g(x_j) = -1 \forall i,j \in [n]$. Then, it suffices to query $1$ and $n$ to solve the problem. On the the other hand, suppose that $g: \X \mapsto \Y$ is such that there is some $x_i$ such that $g(x_i) = 1$. Then, it suffices to query $x_i$ to solve the problem. This shows that $\bestext(\F_{\reg}) \leq 2$.
%
%\textbf{Step 2:} Now, we bound the minimum probability. Fix $r(x) = x + b \in \reg$. $b$ belongs to $I_j = [j-1,j)$ for some $j \in [n+1]$. For any $r^\prime(x) = x + b^\prime$ with $b^\prime \in I_j$, we have that $\round{r^\prime(x)} = \round{r(x)}$ for all $x \in \X$. Thus, the minimum probability is bounded below by $\frac{1}{n+2}$. This completes the proof.
%\end{proof}
%
%\begin{proof}[Proof of Proposition \ref{prop:quadratic_example}]
%\textbf{Step 1:} First we bound $\bestext$. Fix $g: \X \mapsto \Y$ such that $g(x_i) = g(x_j) \forall i,j \in [n]$. By definition of $\reg$ and $\Y$ after two queries the version space is empty, completing this step. Now, suppose that that there exists $i,j \in [n]$ such that $g(x_i) \neq g(x_j)$. Then, let $i_0 \in \argmin_i g(x_i)$. Query $i_0$ and its two adjacent points. Then, by definition of $\Y$ and $\reg$ either all functions are eliminated or the only remaining functions in the version space put $i_0$ as the minimizer.
%
%\textbf{Step 2:} Fix $r_a(x) = x^2 -2a x\in \reg$ with $a \in \{1,\ldots, n\}$. Note that
%\begin{align*}
%|r_a(x) - r_b(x)| \leq 2n|a-b|
%\end{align*}
%for $x \in [1,n]$. Furthermore, we have $r_a(x_i) = y_i$ for all $i \in [n]$. Thus, vor any $r_b(x) = x^2 - 2b$ with $|b-a|\leq \frac{1}{4n}$, we have $|r_a(x) - r_b(x)| \leq 1/2$ so that $\round{r^\prime(x)} = \round{r(x)}$ for all $x \in \X$. Thus, $\P_{\pi}(S_{f^*}) \geq c\frac{1}{n}$ for some constant $c > 0$. This completes the proof.
%\end{proof}
%\begin{proposition}
%Let $K \in \N$ be even and let $\X = \{x \in \R^d : x_{i,j} \in [K] \}$. Let $\reg = \{ r : r \text{ is } \alpha \text{-strongly convex} \}$. Let $\Delta = \min_{y \neq y^\prime \in \Y} |y-y^\prime|$. If $\alpha \geq \frac{16 \Delta}{K}$ Then, $\bestext \leq d$.
%\end{proposition}
%
%\begin{proof}
%
%\textbf{Step 1:} Suppose that $g: \X \mapsto \Y$ such that there exists $y \in \Y$ such that $g(x) = y$ for all $x \in \X$. Now, let $x_0 = \frac{K}{2} \mathbf{1} \in \X$. Let $r$ be any $\alpha$-strongly convex function defined over $[1,K]^d$. Then, $x_0 \in \text{intdom}(r)$. Then, there exists a subgradient at $x$, $g$. Let $z_1 = \frac{K}{2} \mathbf{1} - e_1(\frac{K}{2}-1)$ and $z_2 = \frac{K}{2} \mathbf{1} + e_1\frac{K}{2}$. Then, either $g^\t(z_1 -x) \geq 0$ or $g^\t (z_2 - x_0) \geq 0$. Suppose wlog that $g^\t(z_1 -x_0) \geq 0$. Then, using $\alpha$-strong convexity we have that
%\begin{align*}
%r(z_1) \geq r(x_0) + g^\t (z_1 - x_0) + \frac{\alpha}{2} \norm{x_0 -z_1}^2 \geq r(x_0) + \frac{\alpha K}{8} \geq r(x) + 2 \Delta
%\end{align*}
%where we used $\alpha \geq \frac{16 \Delta}{K}$. This shows that for any $\alpha$-strongly convex function defined over $[1,K]^d$, either $\round{r(z_1)} \neq \round{r(x_0)}$ or $\round{r(z_1)} \neq \round{r(x_0)}$. Thus, it suffices to query $x_0, z_1, z_2$ to finish this case.
%
%\textbf{Step 2:}
%
%
%
%\end{proof}
%\begin{proposition}
%Let $K \in \N$ and let $\X = [K]$. Let $\reg = \{ r : r \text{ is } \alpha \text{-strongly convex} \}$. Let $\Delta = \min_{y \neq y^\prime \in \Y} |y-y^\prime|$. If $\alpha \geq 2 \Delta$ Then, $\bestext \leq d$.
%\end{proposition}
%
%\begin{proof}
%
%\textbf{Step 1:} Suppose that $g: \X \mapsto \Y$ such that there exists $y \in \Y$ such that $g(x) = y$ for all $x \in \X$. Now, let $x_0 = \floor{\frac{K}{2}} \in \X$. Let $r$ be any $\alpha$-strongly convex function defined over $[1,K]$. Then, $x_0 \in \text{intdom}(r)$. Then, there exists a subgradient at $x$, $g$. Let $z_1 = 1$ and $z_2 = K$. Then, either $g\cdot (z_1 -x) \geq 0$ or $g \cdot (z_2 - x_0) \geq 0$. Suppose wlog that $g \cdot (z_1 -x_0) \geq 0$. Then, using $\alpha$-strong convexity we have that
%\begin{align*}
%r(z_1) \geq r(x_0) + g^ \cdot (z_1 - x_0) + \frac{\alpha}{2} |x_0 -z_1| \geq r(x_0) + \frac{\alpha K}{8} \geq r(x) + 2 \Delta
%\end{align*}
%where we used $\alpha \geq 2\Delta$. This shows that for any $\alpha$-strongly convex function defined over $[1,K]^d$, either $\round{r(z_1)} \neq \round{r(x_0)}$ or $\round{r(z_1)} \neq \round{r(x_0)}$. Thus, it suffices to query $x_0, z_1, z_2$ to finish this case.
%
%\textbf{Step 2:} Now, suppose that $g: \X \mapsto \Y$ is such that there exists $x,x^\prime \in \X$ such that $g(x) \neq g(x^\prime)$. Then, let $I \in \argmin_{i \in [n]} g(x_i)$. It suffices to query $x_{I-1}, x_I$, and $x_{I+1}$. Either they
%
%
%
%\end{proof}
\textbf{Strongly Convex Functions:} Finally, we consider strongly convex functions. The following Proposition considers a setting of strongly convex functions where $\bestext \leq O(1)$.
\begin{proposition}
Let $K \in \N$ and let $\X = [K]$. Let $\reg = \{ r : r \text{ is } \alpha \text{-strongly convex} \}$. Let $\Delta = \min_{y \neq y^\prime \in \Y} |y-y^\prime|$. If $\alpha \geq 2 \Delta$ Then, $\bestext(\F_{\reg}) \leq 3$.
\end{proposition}
\begin{proof}
\textbf{Step 1:} Suppose that $g: \X \mapsto \Y$ such that there exists $y \in \Y$ such that $g(x) = y$ for all $x \in \X$. Now, let $x_0 \in \{2,\ldots, K-1\}$. Let $r$ be any $\alpha$-strongly convex function defined over $[1,K]$. Then, $x_0 \in \text{intdom}(r)$. Then, there exists a subgradient at $x$, $g$. Let $z_1 = x_0-1$ and $z_2 = x_0+1$. Then, either $g\cdot (z_1 -x) \geq 0$ or $g \cdot (z_2 - x_0) \geq 0$. Suppose wlog that $g \cdot (z_1 -x_0) \geq 0$. Then, using $\alpha$-strong convexity we have that
\begin{align*}
r(z_1) \geq r(x_0) + g^ \cdot (z_1 - x_0) + \frac{\alpha}{2} |x_0 -z_1| \geq r(x_0) +2 \Delta
\end{align*}
where we used $\alpha \geq 2\Delta$. This shows that for any $\alpha$-strongly convex function defined over $[1,K]^d$, either $\round{r(z_1)} \neq \round{r(x_0)}$ or $\round{r(z_1)} \neq \round{r(x_0)}$. Thus, it suffices to query $x_0, z_1, z_2$ to finish this case.
\textbf{Step 2:} Now, suppose that $g: \X \mapsto \Y$ is such that there exists $x,x^\prime \in \X$ such that $g(x) \neq g(x^\prime)$. Then, let $I \in \argmin_{i \in [n]} g(x_i)$. It suffices to query $x_{I-1}, x_I$, and $x_{I+1}$. By the prior step, we have that $g(x_{I-1}) > g(x_I) < g(x_{I+1})$ for any $\alpha$-strongly convex function such that $\round{r(x)} = g(x)$ for all $x \in \X$. Thus, this completes the case.
\end{proof}
\section{\texttt{OFUL} is not minimax-optimal}
Fix $x_1,\ldots, x_{2n}$ and let $\F = \{f_1,\ldots, f_n\}$ where
\begin{align*}
f_i(x_j) & = \begin{cases}
1 & j = i \\
.5 & j \in \{n+1,\ldots, n+i\} \\
0 & \text{o/w}
\end{cases}.
\end{align*}
Consider the instance given by $y_i = f_n(x_i)$ for all $i \in [2n]$. Then, inspection reveals that \texttt{OFUL} selects all of the arms in $\{1,\ldots, n\}$ before terminating, obtaining a sample complexity of $n$. On the other hand, it is possible to solve any instance $f \in \F$ in $O(\log(n))$ queries using a binary search procedure..
\section{Additional Experiment and Implementation Details}
All experiments were computed on a 36 core cluster machine, and all algorithms were implemented in python. To estimate the probabilities in the algorithm for sample selection, we used 300 independently drawn functions from the version space in the RKHS and constraints experiments and 500 in the case of convex functions. We used 50 iterations of hit and run for the RKHS and constraints experiments and 100 for the convex function experiment. Note that \texttt{GRAILS} divides the output space into intervals. In the RKHS experiments, we used 100 evenly spaced intervals in the output space of $f^\ast$
for computational efficiency. For the constraints experiment, we divided the output space of $f^\ast$ into 300 spaced for better sample complexity at the cost of computational efficiency. Lastly in the convex experiment, we used 1500 evenly spaced intervals.
Furthermore, note that hit and run must compute a projection onto the boundary of the convex set in order to choose the random step. As the step is along a uniformly chosen direction, we can compute this projection by bisection search. This was performed with tolerance 1e-6 for suitable performance with minimal computational overhead.
Lastly, our general implementation of hit and run requires a membership oracle for each function class. We implemented oracles for RKHS functions and 1d convex functions. In the case of the RKHS, we linearize the problem using the kernel trick by first computing the Grammian matrix and then projecting all data onto the span of the eigenvalues. Furthermore, in order to use hit and run, the MCMC iteration must begin with a point in the convex set. As our set is cut out by hyperplane and elliptical constraints, we achieve this by choosing a random starting point in and using CVXPy with the ECOS solver to compute the projection of that point into the convex set to generate a random stating point for hit and run.
For our implementation, we followed the update of \cite{srinivas2009gaussian} directly in the setting where the noise variance is $0$. For our implementation of OFUL, we track our constraint sets using equality constraints for the observed data and norm constraints for the norm of the unknown parameters. Then we use CVXPy to compute the upper and lower bound on the function value of any point $x_i$.
\newpage
\clearpage
%\bibliographystyle{unsrt}
%\bibliography{refs}
\end{document}
\begin{thebibliography}{10}
\bibitem{jamieson2018bandit}
Kevin Jamieson and Lalit Jain.
\newblock A bandit approach to multiple testing with false discovery control.
\newblock In {\em Advances in Neural Information Processing Systems}, 2018.
\bibitem{settles2009active}
Burr Settles.
\newblock Active learning literature survey.
\newblock 2009.
\bibitem{shahriari2015taking}
Bobak Shahriari, Kevin Swersky, Ziyu Wang, Ryan~P Adams, and Nando De~Freitas.
\newblock Taking the human out of the loop: A review of bayesian optimization.
\newblock {\em Proceedings of the IEEE}, 104(1):148--175, 2015.
\bibitem{dasgupta2007general}
Sanjoy Dasgupta, Daniel~J Hsu, and Claire Monteleoni.
\newblock {\em A general agnostic active learning algorithm}.
\newblock Citeseer, 2007.
\bibitem{balcan2009agnostic}
Maria-Florina Balcan, Alina Beygelzimer, and John Langford.
\newblock Agnostic active learning.
\newblock {\em Journal of Computer and System Sciences}, 75(1):78--89, 2009.
\bibitem{hanneke2014theory}
Steve Hanneke et~al.
\newblock Theory of disagreement-based active learning.
\newblock {\em Foundations and Trends{\textregistered} in Machine Learning},
7(2-3):131--309, 2014.
\bibitem{soare2014best}
Marta Soare, Alessandro Lazaric, and R{\'e}mi Munos.
\newblock Best-arm identification in linear bandits.
\newblock In {\em Advances in Neural Information Processing Systems}, pages
828--836, 2014.
\bibitem{fiez2019sequential}
Tanner Fiez, Lalit Jain, Kevin~G Jamieson, and Lillian Ratliff.
\newblock Sequential experimental design for transductive linear bandits.
\newblock In {\em Advances in Neural Information Processing Systems}, pages
10666--10676, 2019.
\bibitem{katz2020empirical}
Julian Katz-Samuels, Lalit Jain, Kevin~G Jamieson, et~al.
\newblock An empirical process approach to the union bound: Practical
algorithms for combinatorial and linear bandits.
\newblock {\em Advances in Neural Information Processing Systems}, 33, 2020.
\bibitem{lattimore_szepesvari_2020}
Tor Lattimore and Csaba Szepesvári.
\newblock {\em Bandit Algorithms}.
\newblock Cambridge University Press, 2020.
\bibitem{amin2011bandits}
Kareem Amin, Michael Kearns, and Umar Syed.
\newblock Bandits, query learning, and the haystack dimension.
\newblock In {\em Proceedings of the 24th Annual Conference on Learning
Theory}, pages 87--106. JMLR Workshop and Conference Proceedings, 2011.
\bibitem{jamieson2014lil}
Kevin Jamieson, Matthew Malloy, Robert Nowak, and S{\'e}bastien Bubeck.
\newblock lil’ucb: An optimal exploration algorithm for multi-armed bandits.
\newblock In {\em Conference on Learning Theory}, pages 423--439, 2014.
\bibitem{kaufmann2016complexity}
Emilie Kaufmann, Olivier Capp{\'e}, and Aur{\'e}lien Garivier.
\newblock On the complexity of best-arm identification in multi-armed bandit
models.
\newblock {\em The Journal of Machine Learning Research}, 17(1):1--42, 2016.
\bibitem{lovasz2006fast}
L{\'a}szl{\'o} Lov{\'a}sz and Santosh Vempala.
\newblock Fast algorithms for logconcave functions: Sampling, rounding,
integration and optimization.
\newblock In {\em 2006 47th Annual IEEE Symposium on Foundations of Computer
Science (FOCS'06)}, pages 57--68. IEEE, 2006.
\bibitem{tosh2020diameter}
Christopher Tosh and Daniel Hsu.
\newblock Diameter-based interactive structure discovery.
\newblock In {\em International Conference on Artificial Intelligence and
Statistics}, pages 580--590. PMLR, 2020.
\bibitem{cohn1994improving}
David Cohn, Les Atlas, and Richard Ladner.
\newblock Improving generalization with active learning.
\newblock {\em Machine learning}, 15(2):201--221, 1994.
\bibitem{dasgupta2005coarse}
Sanjoy Dasgupta.
\newblock Coarse sample complexity bounds for active learning.
\newblock In {\em NIPS}, volume~18, pages 235--242, 2005.
\bibitem{tosh2017diameter}
Christopher Tosh and Sanjoy Dasgupta.
\newblock Diameter-based active learning.
\newblock In {\em International Conference on Machine Learning}, pages
3444--3452. PMLR, 2017.
\bibitem{beygelzimer2010agnostic}
Alina Beygelzimer, Daniel Hsu, John Langford, and Tong Zhang.
\newblock Agnostic active learning without constraints.
\newblock {\em arXiv preprint arXiv:1006.2588}, 2010.
\bibitem{katz2021improved}
Julian Katz-Samuels, Jifan Zhang, Lalit Jain, and Kevin Jamieson.
\newblock Improved algorithms for agnostic pool-based active classification.
\newblock {\em International Conference on Machine Learning}, 2021.
\bibitem{hegedHus1995generalized}
Tibor Hegedus.
\newblock Generalized teaching dimensions and the query complexity of learning.
\newblock In {\em Proceedings of the eighth annual conference on Computational
learning theory}, pages 108--117, 1995.
\bibitem{balcazar2002new}
Jos{\'e}~L Balc{\'a}zar, Jorge Castro, and David Guijarro.
\newblock A new abstract combinatorial dimension for exact learning via
queries.
\newblock {\em Journal of Computer and System Sciences}, 64(1):2--21, 2002.
\bibitem{srinivas2009gaussian}
Niranjan Srinivas, Andreas Krause, Sham~M Kakade, and Matthias Seeger.
\newblock Gaussian process optimization in the bandit setting: No regret and
experimental design.
\newblock {\em arXiv preprint arXiv:0912.3995}, 2009.
\bibitem{bull2011convergence}
Adam~D Bull.
\newblock Convergence rates of efficient global optimization algorithms.
\newblock {\em Journal of Machine Learning Research}, 12(10), 2011.
\bibitem{camilleri2021high}
Romain Camilleri, Julian Katz-Samuels, and Kevin Jamieson.
\newblock High-dimensional experimental design and kernel bandits.
\newblock {\em International Conference on Machine Learning}, 2021.
\bibitem{jain2019new}
Lalit Jain and Kevin~G Jamieson.
\newblock A new perspective on pool-based active classification and
false-discovery control.
\newblock In {\em Advances in Neural Information Processing Systems}, pages
13992--14003, 2019.
\bibitem{garivier2016optimal}
Aur{\'e}lien Garivier and Emilie Kaufmann.
\newblock Optimal best arm identification with fixed confidence.
\newblock In {\em Conference on Learning Theory}, pages 998--1027, 2016.
\bibitem{williamson2011design}
David~P Williamson and David~B Shmoys.
\newblock {\em The design of approximation algorithms}.
\newblock Cambridge university press, 2011.
\bibitem{abbasi2011improved}
Yasin Abbasi-Yadkori, D{\'a}vid P{\'a}l, and Csaba Szepesv{\'a}ri.
\newblock Improved algorithms for linear stochastic bandits.
\newblock In {\em NIPS}, volume~11, pages 2312--2320, 2011.
\bibitem{swiler2020survey}
Laura~P Swiler, Mamikon Gulian, Ari~L Frankel, Cosmin Safta, and John~D
Jakeman.
\newblock A survey of constrained gaussian process regression: Approaches and
implementation challenges.
\newblock {\em Journal of Machine Learning for Modeling and Computing}, 1(2),
2020.
\bibitem{lattimore2017end}
Tor Lattimore and Csaba Szepesvari.
\newblock The end of optimism? an asymptotic analysis of finite-armed linear
bandits.
\newblock In {\em Artificial Intelligence and Statistics}, pages 728--737.
PMLR, 2017.
\bibitem{scholkopf2001generalized}
Bernhard Sch{\"o}lkopf, Ralf Herbrich, and Alex~J Smola.
\newblock A generalized representer theorem.
\newblock In {\em International conference on computational learning theory},
pages 416--426. Springer, 2001.
\bibitem{boyd2004convex}
Stephen Boyd, Stephen~P Boyd, and Lieven Vandenberghe.
\newblock {\em Convex optimization}.
\newblock Cambridge university press, 2004.
\bibitem{katz2020true}
Julian Katz-Samuels and Kevin Jamieson.
\newblock The true sample complexity of identifying good arms.
\newblock In {\em International Conference on Artificial Intelligence and
Statistics}, pages 1781--1791. PMLR, 2020.
\bibitem{katz2018feasible}
Julian Katz-Samuels and Clay Scott.
\newblock Feasible arm identification.
\newblock In {\em International Conference on Machine Learning}, pages
2535--2543. PMLR, 2018.
\bibitem{katz2019top}
Julian Katz-Samuels and Clayton Scott.
\newblock Top feasible arm identification.
\newblock In {\em The 22nd International Conference on Artificial Intelligence
and Statistics}, pages 1593--1601. PMLR, 2019.
\end{thebibliography}