Approximate Bayesian Computation via Classification
Extracted source text ยท arXiv 2111.11507
\documentclass[12pt]{article}
\usepackage{amsmath}
\usepackage{graphics}
\usepackage{enumerate}
\usepackage{natbib}
\usepackage{url} % not crucial - just used below for the URL
\usepackage{enumitem}
%\pdfminorversion=4
% NOTE: To produce blinded version, replace "0" with "1" below.
\newcommand{\blind}{1}
% DON'T change margins - should be 1 inch all around.
\addtolength{\oddsidemargin}{-.5in}%
\addtolength{\evensidemargin}{-1in}%
\addtolength{\textwidth}{1in}%
\addtolength{\textheight}{1.7in}%
\addtolength{\topmargin}{-1in}%
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage[nodisplayskipstretch]{setspace}
\usepackage{subfigure}
\usepackage{multirow}
\usepackage{color}
\usepackage{blkarray}
\usepackage{booktabs}
%\usepackage{geometry}
\usepackage{caption}
\usepackage{cleveref}
\crefname{assumption}{assumption}{assumptions}
%\usepackage[dvipsnames,table]{xcolor}
%\usepackage[]{hyperref}
%\hypersetup{colorlinks=true,
% linkcolor=red,
% citecolor=
%}
%% Please use the following statements for
%% managing the text and math fonts for your papers:
\usepackage{times}
%\usepackage[cmbold]{mathtime}
\usepackage{bm}
\usepackage[plain]{algorithm2e}
\makeatletter
\renewcommand{\algocf@captiontext}[2]{#1\algocf@typo. \AlCapFnt{}#2} % text of caption
\renewcommand{\AlTitleFnt}[1]{#1\unskip}% default definition
\def\@algocf@capt@plain{top}
\renewcommand{\algocf@makecaption}[2]{%
\addtolength{\hsize}{\algomargin}%
\sbox\@tempboxa{\algocf@captiontext{#1}{#2}}%
\ifdim\wd\@tempboxa >\hsize% % if caption is longer than a line
\hskip .5\algomargin%
\parbox[t]{\hsize}{\algocf@captiontext{#1}{#2}}% then caption is not centered
\else%
\global\@minipagefalse%
\hbox to\hsize{\box\@tempboxa}% else caption is centered
\fi%
\addtolength{\hsize}{-\algomargin}%
}
\makeatother
\captionsetup[table]{font={stretch=1}} %% change 1.2 as you like
\captionsetup[figure]{font={stretch=1}}
\setlength{\belowcaptionskip}{-10pt}
%%% User-defined macros should be placed here, but keep them to a minimum.
\newtheorem{definition}{Definition}[section]
\newtheorem{lemma}{Lemma}[section]
\newtheorem{lem}{Lemma}[section]
\newtheorem{theorem}{Theorem}[section]
\newtheorem{example}{Example}[section]
\newtheorem{remark}{Remark}[section]
\newtheorem{corollary}{Corollary}[section]
% \theoremstyle{assumption}
\newtheorem{assumption}{Assumption}
\DeclareMathOperator{\Thetabb}{\mathcal{C}}
\DeclareMathOperator*{\cart}{\times}
\newcommand{\utwi}[1]{\mbox{\boldmath $ #1$}}
%\newcommand{\bm}[1]{\boldsymbol{#1}}
\newcommand{\E}{\mathbb{E}}
\renewcommand{\d}{\mathrm{d}}
\renewcommand{\P}{\mathbb{P}}
\newcommand{\bbeta}{\bm{\beta}}
\newcommand{\Y}{\bm{Y}}
\newcommand{\bG}{\bm{\Gamma}}
\newcommand{\bmE}{E}
\newcommand{\bN}{\mathbb{N}}
\newcommand{\mB}{\mathcal{B}}
\newcommand{\mA}{\mathcal{A}}
\newcommand{\U}{\bm{U}}
\newcommand{\V}{\bm{V}}
\newcommand{\DD}{\bm{D}}
\newcommand{\mX}{\mathcal{X}}
\newcommand{\mT}{\mathcal{T}}
\newcommand{\wt}[1]{\widetilde{#1}}
\def\mS{\mathcal{S}}
\def\mC{\mathcal{C}}
\def\mTE{\mathcal{TE}}
\newcommand{\YY}{\bm{Y}}
\newcommand{\XX}{\bm{X}}
\newcommand{\Var}{\mathsf{Var\,}}
\newcommand{\1}{\mathbb{I}}
\newcommand{\iid}{\stackrel{iid}{\sim}}
\newcommand{\expect}[1]{\langle\mbox{$ #1$}\rangle}
\newcommand{\Expect}[1]{\langle\mbox{$ #1$}\rangle}
\newcommand{\wh}[1]{\smash{\widehat{#1}}}
\newcommand{\QED}{\flushright{QED}\flushleft}
\newcommand{\st}[1]{{\color{blue}(#1)}}
\def\C {\,|\:}
\def\D{\bm{\Delta}}
\def\t{\bm{\theta}}
\def\bomega{\bm{\omega}}
\def\C {\,|\:}
\def\g{\gamma}
\def\G{\bm{\Gamma}}
\def\mF{\mathcal{F}}
\def\mI{\mathcal{I}}
\def\B{\bm{B}}
\def\b{\bm{\beta}}
\def\Y{\bm{Y}}
\def\Z{\bm{Z}}
\def\q{\bm{q}}
\def\A{\bm{A}}
\def\S{\bm{\Sigma}}
\def\Set{\mathcal{S}}
\def\X{\bm{X}}
\def\x{\bm{x}}
\def\y{\bm{y}}
\def\by{\bm{y}}
\def\u{\bm{u}}
\def\bg{\bm{\gamma}}
\def\mV{\mathcal{V}}
\def\DM{\bm{DM}}
\def\Bphi{\bm{\Phi}}
\def\z{\bm{\zeta}}
\def\BD{\bm{\Delta}}
\def\BOmega{\bm{\Omega}}
\def\BS{\bm{\Sigma}}
\def\t{\theta}
\def\bt{\bm{\theta}}
\def\b{\bm{\beta}}
\def\bmu{\bm{\mu}}
\def\bK{\bm{K}}
\def\bmS{\bm{\mathcal{S}}}
\def\btheta{\bm{\theta}}
\def\bTheta{\bm{\Theta}}
\newcommand{\e}{\mathrm{e}}
\newcommand{\supp}{\mathrm{supp}}
\newcommand{\norm}[1]{\left\Vert#1\right\Vert}
\newcommand{\abs}[1]{\left\vert#1\right\vert}
\newcommand{\set}[1]{\left\{#1\right\}}
\newcommand{\Real}{\mathbb R}
\newcommand{\eps}{\varepsilon}
\newcommand{\To}{\longrightarrow}
\newcommand{\prob}{\mathbb{P}}
\newcommand{\p}{\mathbf{p}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\M}{\mathbb{M}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Ha}{\mathcal{H}^\alpha}
\newcommand{\Hg}{\mathcal{H}^\gamma}
\newcommand{\T}{\mathcal{T}}
\newcommand{\mM}{\mathcal{M}}
\newcommand{\mH}{\mathcal{H}}
\newcommand{\mN}{\mathcal{N}}
\newcommand{\mG}{\mathcal{G}}
\newcommand{\mP}{\mathcal{P}}
\newcommand{\mE}{\mathcal{E}}
\newcommand{\mU}{\mathcal{U}}
\newcommand{\mD}{\mathcal{D}}
\begin{document}
\def\spacingset#1{\renewcommand{\baselinestretch}%
{#1}\small\normalsize} \spacingset{1.2}
%% Here are the title, author names and addresses
%\title{ Approximate Bayesian Computation via Classification}
%\author{Yuexi Wang, Tetsuya Kaji and Veronika Ro\v{c}kov\'{a}\\
%\textit{\normalsize Booth School of Business, University of Chicago}}
%\date{
%\today
%}
%\maketitle
%\if1\blind
%{
% \title{\bf Approximate Bayesian Computation via Classification}
% \author{Yuexi Wang \footnote{$4^{th}$ year PhD student at \textsl{\small Booth school of Business, University of Chicago}}, \, Tetsuya Kaji \footnote{Assistant Professor in Econometrics and Statistics; Liew Family Junior Faculty Fellow and Richard N. Rosett Faculty Fellow at the \textsl{\small Booth School of Business, University of Chicago}} \, and Veronika Ro\v{c}kov\'{a}
% \thanks{
% Associate Professor in Econometrics and Statistics and James S. Kemper Faculty Scholar at the \textsl{\small Booth School of Business, University of Chicago}. The author gratefully acknowledges the support from the James S. Kemper Faculty Fund at the Booth School of Business and the National Science Foundation (Grant No. NSF DMS-1944740)}}
% \maketitle
%} \fi
\if1\blind
{
\title{\bf Approximate Bayesian Computation via Classification}
\author{Yuexi Wang, \, Tetsuya Kaji \, and Veronika Ro\v{c}kov\'{a}
\thanks{The author gratefully acknowledges the support from the James S. Kemper Faculty Fund at the Booth School of Business and the National Science Foundation (Grant No. NSF DMS-1944740)}\hspace{.2cm}\\
\textsl{Booth School of Business, University of Chicago}}
\date{\today}
\maketitle
} \fi
\if0\blind
{
\bigskip
\bigskip
\bigskip
\begin{center}
{\LARGE\bf Approximate Bayesian Computation via Classification}
\end{center}
\medskip
} \fi
\bigskip
\begin{abstract}
Approximate Bayesian Computation (ABC) enables statistical inference in complex models whose likelihoods are difficult to calculate but easy to simulate from. ABC constructs a kernel-type approximation to the posterior distribution through an accept/reject mechanism which compares summary statistics of real and simulated data. To obviate the need for summary statistics, we directly compare empirical distributions with a Kullback-Leibler (KL) divergence estimator obtained via classification. In particular, we blend flexible machine learning classifiers within ABC to automate fake/real data comparisons.
We consider the traditional accept/reject kernel as well as an exponential weighting scheme which does not require the ABC acceptance threshold.
Our theoretical results show that the rate at which our ABC posterior distributions concentrate around the true parameter depends on the estimation error of the classifier. We derive limiting posterior shape results and find that, with a properly scaled exponential kernel, asymptotic normality holds. We demonstrate
the usefulness of our approach on simulated examples as well as real data in the context of stock volatility estimation.
\end{abstract}
\noindent%
{\bf Keywords: }{\em
Approximate Bayesian Computation; Classification; Likelihood-free Inference; \\
Kullback-Leibler Divergence; Posterior Concentration
}
\newpage
\spacingset{1.5} % DON'T change the spacing!
\section{Introduction}
\vspace{-0.3cm}
Implicit in the deployment of many Bayesian methods is the assumption that the likelihood function can be numerically evaluated. More often than not, however, modern statistical models are prescribed by intractable likelihoods or are defined through an underlying generating mechanism. Fortunately, it may still be possible to simulate synthetic datasets from the model. The ability to simulate from the likelihood has opened up new opportunities for simulating from the posterior. For example, Approximate Bayesian Computation (ABC) \citep{pritchard1999population, beaumont2002approximate} emerged as a default likelihood-free Bayesian inferential tool. It is an accept/reject posterior sampling mechanism which obviates likelihood evaluations. Each iteration proceeds by (1) simulating prior parameter guesses and fake data from the likelihood, and then (2) accepting those parameter values whose fake data were close to the observed data. A big challenge with ABC has been gauging the similitude between observed and fake data.
Measures of similarity between data sets have traditionally been based on summary statistics (see \citet{blum2013comparative} for an overview within the ABC context). In other words, two datasets are considered similar if their summary statistics are close.
In the absence of expert knowledge, however,
constructing effective summary statistics can be challenging \citep{joyce2008approximately, nunes2010optimal,blum2013comparative} and one may need to resort to automated strategies.
One possibility is regressing parameter values onto (functionals of) fake data in a pilot ABC run to train a flexible mapping which can be substituted for summary statistics \citep{fearnhead2011constructing,jiang2017learning,aakesson2020convolutional}. Another possibility, related to indirect inference, is to construct summary statistics from an auxiliary model \citep{drovandi2011approximate, wood2010statistical}. One can also choose a subset of candidate summary statistics that satisfy some optimality criterion \citep{joyce2008approximately,nunes2010optimal} or find an optimal projection of a set of summary statistics onto a lower-dimensional map \citep{fearnhead2011constructing}.
Alternatively, one can view the observed and synthetic data sets as empirical distributions and directly use a distance between them inside ABC (such as Kullback-Leibler \citep{jiang2018approximate} or Wasserstein \citep{bernton2019approximate} or Maximum Mean (MM) discrepancy \citep{park2016k2}). Our work fortifies this last point of view by focusing on the Kullback-Leibler discrepancy estimated via classification.
The KL divergence is one of the most widely used discrepancy metrics. It expresses the average information per observation to discriminate between two probabilistic models \citep{kullback1958information}.
In large deviations, for example, it characterizes the exponential decay rate at which empirical measures converge to their probabilities (see Sanov's theorem in \citet{den2008large}) and the rate of decay of the probability of error in a binary hypothesis testing problem (see Stein's Lemma in \citet{cover1991entropy}). KL also naturally connects to maximum likelihood estimation through its interpretation as the expectation of the log-likelihood ratio. There exist many methods for estimating the KL divergence.
For example, \citet{wang2009divergence} proposed nearest-neighbor techniques to obtain a mean-square consistent estimator. \citet{wang2005divergence} proposed a histogram-based KL estimator based on partitioning of the space into statistically equivalent intervals. \citet{silva2007universal} and \citet{silva2010information} went a step further and proposed using data-driven partitions (including multivariate recursive partitioning) and formulated sufficient consistency conditions. Alternatively, \citet{nguyen2007estimating} proposed a variational approach by turning KL estimation into a penalized convex risk minimization problem. Our work is different from the approaches above as we propose a KL estimator based on classification.
We suggest embedding a machine learning classifier inside ABC to determine whether or not fake and observed data are similar and, thereby, whether or not the underlying parameter value should be kept in the ABC reference table. The fundamental premise of this proposal is as follows: parameter values that yield indistinguishable simulated datasets can be deemed close. Bayesian inference via classification has been suggested before.
\citet{kaji2021mh} developed a version of the Metropolis-Hastings algorithm, called MHC, based on classification-based estimators of likelihood ratios. \citet{gutmann2018likelihood} proposed a classification strategy related to ours using a different discrepancy metric. Our paper reframes the method of \citet{gutmann2018likelihood} as a genuine ABC algorithm with a KL divergence discriminator and provides supporting theory which justifies its inferential potential.
In particular, we study statistical properties of the approximate posterior which, in part, depend on the properties of the KL divergence estimator.
We consider the traditional accept/reject ABC version (with a uniform kernel) as well as an exponential kernel variant which does not require the ABC tolerance threshold.
%For the uniform kernel, we extend the results in \citet{frazier2018asymptotic}. Since we view data discrepancies through the empirical KL divergence, we loosen the restriction that the dimensionality of summary statistics needs to be at least as large as the dimensionality of the parameter.
Similar to \citet{frazier2018asymptotic}, we show that the choice of the ABC acceptance threshold $\epsilon$ plays a critical role in the convergence rate and in the limiting posterior shape. In practice, it is often not obvious what the optimal threshold $\epsilon$ should be.
Motivated by the connections with the MHC algorithm of Kaji and Rockova (2021), we propose an exponential kernel which yields ABC posteriors that correspond to the stationary distribution of MHC.
Our ABC kernel method can be thus regarded as a parallelizable counterpart to the sequential MHC sampling, targeting the same posterior approximation.
The concentration and asymptotic shape behavior of the ABC posterior, which can be derived from \citet{kaji2021mh}, theoretically justify our exponential weighting scheme.
In addition, the exponential kernel can be motivated as an instantiation of General Bayesian Inference (GBI) \citep{bissiri2016general} with the KL estimator as the loss function.
\iffalse
We study the statistical properties of the approximate posterior which, in part, depend on the properties of the KL divergence estimator.
We focus on two types of aggregation kernels: uniform (which corresponds to the accept/reject ABC rule) and exponential kernels. For the uniform kernel, we extend results in \citet{frazier2018asymptotic}. Since we distill data through KL divergence, we loosen the restriction that the dimensionality of the summary statistics needs to be at least as large as dimensionality of the parameter. Similar to \citet{frazier2018asymptotic}, we show that the choice of ABC acceptance threshold $\epsilon$ is critical in the convergence rate and the limiting posterior shape. However, it is not obvious what the optimal threshold $\epsilon$ is in most cases, especially for complex models. Thus, motivated by the connection of KL divergence to likelihood ratio, we resort to a scaled exponential kernel that requires no ad hoc threshold. This can be viewed as a special case of General Bayesian Inference (GBI) \citep{bissiri2016general}, where we plug in an estimator of KL divergence as the loss function. Our approach naturally relates to the classical Bayesian convergence and asymptotic theories developed by \citet{ghosal2007convergence} and \citet{kleijn2012bernstein}. We show that with this continuous exponential kernel, the concentration and asymptotic behavior of the ABC posteriors can be justified. And the resulting distributions coincide with the ergodic stable Markov chains generated by Metropolis-Hastings via Classification (MHC) \citep{kaji2021mh}.
Our work can also be regarded as a supplement to ABC theories on model misspecification. \citet{frazier2019model} studies the concentration and asymptotic behavior of the accept-reject ABC procedures and the popular local regression adjustments to ABC. The ABC posterior no longer yield credible sets with valid frequentist coverage and has non-standard asymptotic behavior. For our method, the ABC posterior concentrates around the KL projection of actual data-generating process onto the model space. And the concentration results and asymptotic behavior are still valid.
\fi
The remaining of the paper is structured as follows. In Section 2, we flesh out the basic idea of ABC and introduce our framework with classification. In Section 3, we investigate the posterior concentration and limiting shape behaviors of the ABC posteriors. Section 4 shows performance on simulated datasets and Section 5 further highlights the practical value of our approach on real data. In Section 6, we conclude with a discussion.
\smallskip
{\em Notation.} We employ the operator notation for expectation, e.g., $P_0 f=\int f dP_0$. The $\epsilon$-bracketing number $N_{[]}(\epsilon, \mF, d)$ of a set $\mF$ with respect to a premetric $d$ is the minimal number of $\epsilon$-brackets in $d$ needed to cover $\mF$\footnote{A premetric on $\mF$ is a function $d: \mF\times \mF \to \R$ such that $d(f,f)=0$ and $d(f,g)=d(g,f)\geq 0$.}. The $\delta$-bracketing entropy integral of $\mF$ with respect to $d$ is $J_{[]}(\delta, \mF, d)=\int_0^{\delta} \sqrt{1+\log N_{[]}(\epsilon, \mF,d)} d\epsilon$. Next, $K(f,g)=\int f\log(f/g)d\mu$ denotes the Kullback-Leibler divergence between two density functions and $V_2(f,g)=\int f\abs{\log(f/g)}^2 d\mu$. For real-valued sequences $\{a_n\}_{n\geq 1}$ and $\{b_n\}_{n\geq 1}$, $a_n\lesssim b_n$ means that $a_n\leq C\,b_n$ for some generic constant $C>0$, $a_n\asymp b_n$ means that $a_n\lesssim b_n\lesssim a_n$, and $a_n\gg b_n$ indicates a greater order of magnitude. For a sequence of random variables $x_n$, $x_n=o_P(a_n)$ if $\lim_{n\to\infty }P(\abs{x_n/a_n}\geq C)=0$ for every $C>0$, and $x_n=O_P(a_n)$ if for every $C>0$ there exists a finite $M>0$ and a finite $N$ such that $P(\abs{x_n/a_n}\geq M)\leq C$ for all $n>N$. All limits are taken as $n\to \infty$. Take $\norm{\cdot}$ to be the Euclidean norm.
\vspace{-0.5cm}
\section{ABC without Summary Statistics}
\vspace{-0.3cm}
Our framework consists of a collection of i.i.d. observations $\X=(X_1, \ldots, X_n)'$ where each $X_i\in \mX$ is realized from a parametric model $\{P_{\theta_0}: \theta_0\in \Theta\subset\R^d\}$. We assume that $P_\theta$, for each $\theta\in \Theta$, admits a density $p_\theta$.
Hereafter, we will use the shorthand notation $p_0=p_{\theta_0}$ and $P_0=P_{\theta_0}$.
We are interested in Bayesian inference about $\theta_0$ based on the posterior distribution
\begin{equation}\label{eq:posterior}
\pi_n(\theta\mid \X)\propto p_\theta^{(n)}(\X)\pi(\theta)
\end{equation}
prescribed by the likelihood $p_\theta^{(n)}(\X)$ and the prior density $\pi(\theta)$.
We are particularly interested in scenarios when the likelihood function cannot be directly expressed/evaluated (such as discretely observed diffusions \citep{sorensen2004parametric} or generative models) but can be sampled from.
The now default ABC method for Bayesian likelihood-free inference constructs a nested kernel-type approximation to the posterior distribution. The first approximation occurs when the data is distilled into summary statistics to obtain $\pi(\theta\C S_X) \propto \pi(S_X\C\theta)\pi(\theta)$, where $S_X= S(\X)$ is a vector of summary statistics. The quality of this approximation depends crucially on the informativeness of $S_X$. The actual ABC approximation to the posterior \eqref{eq:posterior}
is then constructed via a kernel function as
$\pi_{ABC}(\theta\C S_X) =\int \pi(\theta,S\C S_X) dS$ with
$\pi(\theta, S)\propto T_\epsilon(\| S-S_Y\|) \pi (S\C\theta)\pi(\theta)$, where $T_\epsilon(\|u\|)=T(\|u\|/\epsilon)$
is a standard (smoothing) kernel with a scale parameter $\epsilon>0$.
The key two challenges with ABC are (1) deriving low-dimensional summary statistics with a minimal loss of information and (2) selecting the kernel and its the tolerance level $\epsilon$.
To remediate the reliance of ABC on summary statistics, we focus on viewing the observed and fake data as empirical distributions and gauge the distance between them.
Regarding the choice of aggregation kernels, we consider the traditional uniform kernel yielding an accept/reject algorithm and a smoothing kernel free from $\epsilon$-tuning.
\vspace{-0.3cm}
\subsection{ABC with KL Divergence}
Instead of summary statistics, we use the estimated KL divergence inside the ABC algorithm.
Our interest in the KL divergence as a discrepancy metric stems partially from the following connection to the generalized Bayesian inference \citep{bissiri2016general}. The posterior distribution \eqref{eq:posterior} can be rewritten as a generalized posterior
$
\pi_n(\theta\mid \X) \propto \pi(\theta)\exp( - n\times KL(p_0^{(n)},p_\theta^{(n)}))
$
where the parameter $\theta$ is linked to data through the empirical Kullback-Liebler (KL) divergence $KL(p_0^{(n)},p_\theta^{(n)})\equiv \frac{1}{n}\sum_{i=1}^n \log ({p_0}/{p_\theta})(X_i)$.
For when the KL divergence cannot be easily evaluated, we consider various estimators in the next section.
We denote a generic KL divergence estimator obtained from observed data $\X \sim P_0^{(n)}$ and pseudo-data $\tilde \X^\theta\sim P_\theta^{(n)}$ as $\hat K(\X, \tilde \X^\theta)$.
Adopting $\hat K(\cdot, \cdot)$ as the ABC summary statistics, we consider a simple accept/reject ABC mechanism detailed in Algorithm \ref{alg:abc_plain} below.
While \citet{jiang2017learning} used a nearest-neighbor estimator of the KL divergence, we devise a different estimator based on classification in Section \ref{sec:clas_KL}.
\begin{algorithm}[!h]
\caption{KL-ABC with Accept-Reject}\label{alg:abc_plain}
For a pre-determined tolerance level $\epsilon>0$ repeat for $j=1,\dots, N$:
\begin{enumerate}
\item Simulate $\theta_j$ from $\pi(\theta)$.
\item Simulate $\tilde \X^{\theta_j} =(\tilde X_1^{\theta_j}, \ldots ,\tilde X_m^{\theta_j})'$ through i.i.d sampling from the model $p_{\theta_j}$.
\item Accept $\theta_j$ when $\hat K(\X, \tilde \X^{\theta_j})\leq \epsilon$.
\end{enumerate}
\vspace{-0.2in}
\end{algorithm}
%The algorithm differs from other conventional ABC methods in that the statistic $\hat K(\X, \tilde \X^\theta)$ directly quantifying the ``distance" between simulated data and observed data.
Algorithm \ref{alg:abc_plain} simulates pairs of parameter values and pseudo-data $\{\theta, \tilde \X^\theta\}$ from the joint posterior density
\begin{equation}\label{eq:abc_post}
\hat \pi^{AR}\left(\theta, \tilde \X^\theta\mid \hat K(\X, \tilde \X^\theta)\leq \epsilon\right)=\frac{\pi(\theta)p_\theta^{(n)}(\tilde \X^\theta)\1\big(\hat K(\X, \tilde \X^\theta)\leq \epsilon\big)}{\int \pi(\theta)p_\theta^{(n)}(\tilde \X^\theta)\1\big(\hat K(\X, \tilde \X^\theta)\leq \epsilon\big)\d\tilde \X^\theta \d\theta},
\end{equation}
which margins towards the following approximate (Accept/Reject) posterior density
\begin{align}
\hat \pi^{AR}_\epsilon(\theta \mid \X)&=\int \hat \pi^{AR}\left(\theta, \tilde \X^\theta\mid \hat K(\X, \tilde \X^\theta)\leq \epsilon\right) \d \tilde \X^\theta \nonumber \\
& \equiv\frac{\pi(\theta)P_\theta^{(n)} \big(\hat K(\X, \tilde \X^\theta)\leq \epsilon\big)}{\int \pi(\theta)P_\theta^{(n)} \big(\hat K(\X, \tilde \X^\theta)\leq \epsilon\big) \d\theta}\label{eq:alg_post1}.
\end{align}
The approximation \eqref{eq:alg_post1} will be scrutinized theoretically later in \Cref{sec:post_rate}.
In particular, we will later see that the convergence rate of \eqref{eq:alg_post1} around $\theta_0$ depends on the choice of $\epsilon$ \citep{frazier2018asymptotic}.
Algorithm \ref{alg:abc_plain} uses the uniform kernel which corresponds to the indicator function $T_\epsilon(\|u\|)=\mathbb I(\|u\|\leq \epsilon)$.
In practice, it is difficult to balance out conflicting demands of smaller $\epsilon$ (yielding good approximability) and larger acceptance rates (yielding more posterior samples).
As a remedy, we propose a way to aggregate the ABC samples through a scaled exponential kernel
motivated by the connection between KL and the log-likelihood ratio. This ABC variant requires no ad-hoc thresholding and is summarized in Algorithm \ref{alg:abc_exp}.
\begin{algorithm}[!h]
\caption{KL-ABC with Exponential Weighting}\label{alg:abc_exp}
Repeat for $j=1,\dots, N$:
\begin{enumerate}
\item Simulate $\theta_j$ from $\pi(\theta)$.
\item Simulate $\tilde \X^{\theta_j} =(\tilde X_1^{\theta_j}, \ldots ,\tilde X_m^{\theta_j})'$ through i.i.d sampling from the model $p_{\theta_j}$.
\item Assign $\theta_j$ a weight proportional to $\exp\big(-n \hat K(\X, \tilde \X^\theta)\big)$.
\end{enumerate}
\vspace{-0.2in}
\end{algorithm}
Algorithm \ref{alg:abc_exp} generates draws for the pair $\{\theta, \tilde \X^\theta\}$ from a joint posterior density
\begin{equation}
\hat \pi^{EK}(\theta, \tilde \X^\theta\mid \X)=\frac{\pi(\theta)p_\theta^{(n)}(\tilde \X^\theta) \exp\big(-n\hat K(\X, \tilde \X^\theta)\big)}{\int \pi(\theta)p_\theta^{(n)}(\tilde \X^\theta)\exp\big(-n\hat K(\X, \tilde \X^\theta)\big)\d\tilde \X^\theta \d\theta},
\end{equation}
which leads to the approximated Bayesian posterior as
\begin{align}
\hat \pi^{EK}(\theta \mid \X)& = \int \hat \pi^{EK}(\theta, \tilde \X^\theta\mid \X) \d\tilde\X^\theta=\frac{\pi(\theta)P_\theta^{(n)}\exp\big(-n\hat K(\X, \tilde \X^\theta)\big)}{\int \pi(\theta)P_\theta^{(n)} \exp\big(-n\hat K(\X, \tilde \X^\theta)\big)\d\theta}. \label{eq:alg_post2}
\end{align}
{
Smooth kernels have been used inside ABC before to rescale the acceptance probability (e.g. \citet{beaumont2002approximate} employed the Epanechnikov kernel). \citet{wilkinson2013approximate} and \citet{sisson2018overview} provide a thorough overview and comparisons of the commonly used kernels. Our smoothed weights are directly interpretable due to their linkage between the KL divergence and the log-likelihood ratio.} We later show in \Cref{sec:post_rate} that, with the scaled exponential kernel, the ABC posterior corresponds to the stationary distribution of the MHC algorithm of Kaji and Rockova (2021) and can be regarded as a posterior under a misspecified model.
\begin{remark}(Generating Fake Data)\label{rem:generate}
We assume $\tilde \X^\theta=g_\theta(\tilde\X)$, where $\tilde \X\in\R^m$ are random variables arriving from $\tilde P^{(m)}$ and where $g_\theta:\R^m\rightarrow\R^m$ is a deterministic mapping.
Generating random variable draws by passing $\tilde \X$ through some mapping is commonly done in practice. For example, Gaussian random variables $\tilde\X=\{\tilde X^\theta_i\}_{i=1}^m$ that follow i.i.d. $N(\mu, \sigma^2)$ distribution can be obtained by transforming $\{\tilde X_i\}_{i=1}^m\overset{i.i.d.}{\sim} N(0,1)$ via $\tilde X^\theta_i = \mu+\sigma \tilde X_i$. In other cases, one can use uniform draws $\tilde \X$ and the inverse transform sampling.
\end{remark}
\vspace{-0.5cm}
\subsection{Estimating KL Divergence via Classification}\label{sec:clas_KL}
We adopt the `$-\log D$' trick to estimate the KL divergence \citep{goodfellow2014generative}.
More precisely, a flexible discriminator $D$ (such as a neural network or logistic regression) is trained to maximize
\begin{equation}\label{eq:bce_obj}
\M_{n,m}^\theta(D)= \P_n \log D+\P_m^\theta \log (1-D),
\end{equation}
where we employ the operator notation for expectation, e.g., $\P_n f=\frac{1}{n}\sum_{i=1}^n f(X_i)$ and $\P_m^\theta f=\frac{1}{m}\sum_{i=1}^m f(\tilde X_i^\theta)$. This can also be regarded as a classification problem where we label $\{X_i\}_{i=1}^n$ (`real' data) with 1 and $\{\tilde X_i^\theta\}_{i=1}^m$ (`fake' data) with 0. The oracle maximizer to \eqref{eq:bce_obj} can be shown to be \citep[Proposition 1]{goodfellow2014generative}\begin{equation}\label{eq:oracle}
D_\theta(X)=\frac{p_0(X)}{p_0(X)+p_\theta(X)}.
\end{equation}
The functional form of the oracle solution in \eqref{eq:oracle} naturally suggests the following KL estimator obtained from a trained discriminator $\hat D^\theta_{n,m}$
\begin{equation}\label{eq:KL_dnn}
\hat K(\X, \tilde \X^\theta)=\P_n\log\frac{\hat D^\theta_{n,m}}{1-\hat D^\theta_{n,m}}=\frac{1}{n}\sum_{i=1}^n\log\frac{\hat D^\theta_{n,m}(X_i)}{1-\hat D^\theta_{n,m}(X_i)}.
\end{equation}
Next, we show that our classification-based KL estimator \eqref{eq:KL_dnn} converges to a well-defined limit counterpart $K(p_0,p_\theta)$ under mild conditions.
We assume that the set of considered classifiers $\mD$ resides in a sieve $\mD_n$ that expands with the sample size and its size is measured by bracketing entropy $N_{[]}(\epsilon, \mF, d)$. \citet{kaji2020adversarial} prove the rate of convergence of such a classifier (under assumptions reviewed later) with a Hellinger-type distance defined as
\[
d_\theta(D_1, D_2)=\sqrt{h_\theta(D_1, D_2)^2+ h_\theta(1-D, 1-D_\theta)^2},
\]
where $h_\theta(D_1, D_2)=\sqrt{(P_0+P_\theta)(\sqrt D_1-\sqrt D_2)^2}$.
\begin{assumption} \label{ass:entropy} \citep[Assumption 3]{kaji2020adversarial}
Assume that $n/m$ converges and that an estimator $\hat D^\theta_{n,m}$ exists that satisfies $\M_{n,m}(\hat D^\theta_{n,m})\geq \M_{n,m}^\theta(D_\theta)-O_P(\delta_n^2)$ for a nonnegative sequence $\delta_n$. Moreover, assume that the bracketing entropy integral satisfies $J_{[]}(\delta_n, \mD^\theta_{n,\delta_n},d_\theta)\lesssim \delta_n^2\sqrt n$ and that there exists $\alpha<2$ such that $J_{[]}(\delta_n, \mD^\theta_{n,\delta_n},d_\theta)/\delta^\alpha$ is increasing in $\delta$. Here $\mD^\theta_{n,\delta_n}= \{D\in \mD_n: d_\theta(D, D_\theta)\leq \delta\}$.
\end{assumption}
\begin{lemma}\label{lem:delta_n} \citep[Theorem 1']{kaji2020adversarial} Under Assumption \ref{ass:entropy}, $d_\theta(\hat D_{n,m}^\theta, D_\theta)=O_P^*(\delta_n)$. \footnote{We use $P^*$ to denote outer expectation (definition see Section 1.2 of \citet{van1996weak}), here is the expectation of a ``a smallest measureable function g that dominates $d_\theta(\hat D_{n,m}^\theta, D_\theta)$".}
\end{lemma}
In Assumption \ref{ass:entropy}, we only assume that $\delta_n$ be a nonnegative sequence, not necessarily vanishing. To ensure that the approximate posterior converges, we require $\delta_n\to 0$ as $n\to\infty$. \citet{kaji2020adversarial} show that if the true likelihood ratio has a low-dimensional representation and an appropriate neural network is used for the discriminator, the rate $\delta_n$ depends only on the underlying dimension and it vanishes when $n\to\infty$. This result can also be generalized to other network configurations, including \citet{schmidt2020nonparametric} and \citet{yarotsky2017error}. Hence, we implicitly assume $\delta_n=o(1)$ and note that $\delta_n$ will typically be slower than $n^{-1/2}$.
To establish the rate of convergence of our approximated posterior, we have the following assumption.
\begin{assumption}\label{ass:ratio_bd} \citep[Assumption 2]{kaji2021mh}
There exists $\Lambda>0$ such that for every $\theta\in \Theta$, $P_0(p_0/p_\theta)$ and $P_0(p_0/p_\theta)^2$ are bounded by $\Lambda$ and
\[
\sup_{D\in \mD^\theta_{n,\delta_n} }P_0\left(\frac{D_\theta}{D}\middle\vert \frac{D_\theta}{D} \geq \frac{25}{16}\right)<\Lambda, \quad \sup_{D\in \mD^\theta_{n,\delta_n} }P_0\left(\frac{1-D_\theta}{1-D}\middle\vert \frac{1-D_\theta}{1-D} \geq \frac{25}{16}\right)<\Lambda,
\]
for $\delta_n$ in Assumption \ref{ass:entropy}. The brackets in Assumption \ref{ass:entropy} can be taken so that $P_0(\sqrt{u/l}-1)^2=O(d_\theta(u,l)^2)$ and $P_0(\sqrt{(1-l)/(1-u)}-1)^2=o(d_\theta(u,l))$.
\end{assumption}
The following theorem quantifies the rate of convergence of our estimator \eqref{eq:KL_dnn} towards the {\em empirical} KL divergence $\P_n\log \frac{p_0}{p_\theta}$.
\begin{lemma} \label{thm:conv_moments}
(Convergence Rate of Estimation Errors) Under Assumptions \ref{ass:entropy} and \ref{ass:ratio_bd},
\begin{align}
&\abs{\hat K(\X, \tilde \X^\theta)-\P_n\log \frac{p_0}{p_\theta}}=O_{P^*}(\delta_n).\label{eq:approx_error}
%&\exp\Big(\abs{\hat K(\X, \tilde \X^\theta)-\P_n\log \frac{p_0}{p_\theta}}\Big)=1+O^*_P(\delta_n)=e^{O^*_P(\delta_n)}.\nonumber
\end{align}
\end{lemma}
\textbf{Proof. }
Since the estimation error can be rewritten as
\begin{align*}
\hat K(\X, \tilde \X^\theta)-\P_n \log \frac{p_0}{p_\theta} = - \P_n\Big(\log \frac{1-\hat D^\theta_{n,m}}{1-D_\theta}- \log\frac{\hat D^\theta_{n,m}}{D_\theta}\Big),
\end{align*}
it follows from \citet[Theorem 4.1]{kaji2021mh}.
\begin{remark}\label{rmk:worst_rate} (Uniform Convergence Rate)
The rate of estimation in \Cref{lem:delta_n} and \Cref{thm:conv_moments} is characterized as point-wise. While for each $\theta \in \Theta$ the estimation error is shrinking at the rate $\delta_n$, the multiplication constant in front the rate could potentially depend on $\theta$. Assuming a compact parameter support (which is not a-typical in deep learning models; see e.g. \citet{schmidt2020nonparametric} or \citet{polson2018posterior} and \citet{wang2020uncertainty}) and continuity of the multiplication constant, we can essentially regard the rate as uniform. Hereafter, we thereby abuse the notation and tacitly assume that $\delta_n$ is the worst rate over all $\theta \in \Theta\subset\R^d$, i.e. the rate with the largest multiplication constant.
\end{remark}
Next, we investigate convergence around the {\em actual} KL divergence $K(p_0, p_\theta)$. The next lemma will be utilized later in the proof of \Cref{thm:post_rate}. However, it is of independent interest as it shows how
the speed at which the joint error probability (accounting for randomness of both the observed and fake data $(\X,\tilde\X)$) decays in terms of the estimation error. Below, the probability $P$ corresponds to $P_0^{(n)}\otimes \tilde P^{(m)}$, where $\tilde P^{(m)}$ is the measure for $\tilde\X$ (see Remark \ref{rem:generate}).
\begin{theorem}\label{thm:est_error_conc}
For a given $\theta\in\Theta$, we define for $u>0$ and $\delta_n>0$ as in Lemma \ref{lem:delta_n} and for an arbitrarily slowly increasing sequence $C_n>0$
\begin{equation}\label{eq:est_error_bigO}
\rho_{n,\theta}(u; C_n;\delta_n)\equiv P\biggl(\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} >2u, d_\theta(\hat D^\theta_{n,m}, D^\theta)\leq C_n\delta_n \biggr).
\end{equation}
Under Assumptions \ref{ass:entropy} and \ref{ass:ratio_bd}, we then have
$$
\rho_n(u;C_n;\delta_n)\equiv\sup_{\theta\in \Theta}\rho_{n,\theta}(u;C_n;\delta_n)=O\Big(\frac{C_n\delta_n}{u} +\frac{1}{nu^2}\Big).
$$
\end{theorem}
The proof can be found in \Cref{pf:est_error_conc}. {Note that the intersecting event above has probability converging to one, according to Lemma \ref{lem:delta_n}.}
\vspace{-0.3cm}
\subsection{Other KL Estimators} Our ABC results, presented later in Section \ref{sec:freq_ABC}, can be extended to other types of KL estimators if similar estimation error results as in \Cref{thm:est_error_conc} can be shown. One example is the k-Nearest Neighbor (kNN) estimator proposed in \citet{perez2008kullback}. \citet{wang2009divergence} showed that this estimator is asymptotically unbiased and mean-square consistent and they propose a data-dependent choice of $k$ which can improve the convergence speed. \citet{jiang2018approximate} assess data discrepancy inside ABC with the special case of 1-nearest neighbor, which is defined as
\begin{equation}\label{eq:KL_knn}
\hat K(\X, \tilde \X^\theta)=\frac{d}{n} \sum_{i=1}^n \log \frac{\min_j \norm{X_i-\tilde X^\theta_j}}{\min_{j \neq i} \norm{X_i-X_j}}+\log \frac{m}{n-1}.
\end{equation}
where $d$ is the number of covariates in $\X$. \citet{zhao2020analysis} provide estimation error rate results for this kNN estimator.
%The bias is shrinking at rate $n^{-1/d}\log^{1/d}n$ when the distributions have bounded support and the densities are bounded away from zero. When the density is smooth everywhere and can be arbitrarily close to zero, the convergence rate of the bias is $n^{-2\gamma/(2+d)}\log n$ where $\gamma \in (0,1]$ is a constant related to the tail behavior.
Another route to estimate the KL divergence is via (data-dependent) partitioning methods. \citet{wang2005divergence} proposed to estimate the Radon-Nikodym derivative $\d P_0/ \d P_\theta$ using frequency counts on a statistically equivalent partition of $\R^d$. However, the computational complexity of their method is exponential in $d$ and the estimation accuracy deteriorates quickly as the dimension increases. \citet{silva2010information} further contributed to multivariate data-driven partition schemes by using a Barron-type histogram-based density estimate. They provide sufficient conditions on the partitions scheme to make the estimator strongly consistent.
Lastly, \citet{nguyen2007estimating} adopted a variational approach to estimate KL by reframing the estimation problem as a penalized convex risk minimization problem, where they restrict the estimate to a bounded subset of a Reproducing Kernel Hilbert Space (RKHS). Convergence rates are then obtained from empirical process theory on nonparametric M-estimators \citep{geer2000empirical}. In an independent contribution, \citet{ghimire2020reliable} used a discriminator in RKHS to estimate KL using a similar approach to ours. They showed that the estimator error bound is related to the complexity of the discriminator in RKHS.
\vspace{-0.3cm}
\subsection{Generalizations to GAN-style Discrepancy Metrics }
Beyond the KL divergence, our classification framework allows us to consider other discrepancy metrics. Alternatively to \eqref{eq:KL_dnn}, we could instead estimate the reversed KL divergence
\[
\hat K_{\text{reverse}}( \tilde \X^\theta,\X )= \frac{1}{m}\sum_{i=1}^m \log \frac{1-\hat D^\theta_{n,m}}{\hat D^\theta_{n,m}}(\tilde X_i^\theta)
\]
which converges to $K(p_\theta, p_0)$ and which is still uniquely minimized at $p_\theta=p_0$. One can show that the estimation error of this reversed KL estimator is still $O_{P^*}(\delta_n)$ by following the same techniques used in \Cref{thm:conv_moments}. The reversed KL divergence is widely used in variational inference \citep{jordan1999introduction, wainwright2008graphical}. Forward and reversed KL's could perform differently when the function class inside the variational approach is not rich enough. The reversed KL is zero-forcing/mode-seeking, while the forward KL is mass-covering/mean-seeking \citep{bishop2006pattern}.
Another related metric, deployed by \citet{gutmann2018likelihood}, is the classification accuracy (CA) defined as
\begin{equation}\label{eq:ca}
\text{CA}(\X, \tilde \X^\theta)=\frac{1}{n+m} \Big(\sum_{i=1}^n \hat D^\theta_{n,m}(X_i)+\sum_{i=1}^m \big(1-\hat D^\theta_{n,m}(\tilde X^\theta_i) \big)\Big).
\end{equation}
{Since $\hat D(\cdot)$ and $\log \frac{\hat D}{1-\hat D}(\cdot)$ are linked by a logistic transformation which is Lipschitz-continous, CA can be roughly regarded as a weighted average of the forward KL divergence and the reversed KL divergence.} All three metrics are associated with Jensen-Shannon (JS) divergence, each having a slightly different emphasis.
There are many other divergences that can be estimated using different configurations of \eqref{eq:bce_obj}. Given a set $\mF$ of functions from $\mX$ to $\R$, we can define
\begin{equation}\label{eq:ipm}
d_\mF(p_0, p_\theta)=\sup_{f\in\mF} \P_n f(X_i)-\P_m^\theta f(\tilde X^\theta_i),
\end{equation}
as the Integral Probability Metric (IPM) \citep{muller1997integral} associated with the function class $\mF$. \citet{arjovsky2017wasserstein} show that different choices of the function class $\mF$ can dramatically change the topology, as we discuss below, and the regularity of $d_\mF(p_0, p_\theta)$ as a loss function. Several widely used discrepancy metrics can be obtained from IPMs. For example, the 1-Wasserstein distance or the Earth-Mover distance \citep{villani2009optimal} are obtained when $\mF$ is the set of 1-Lipschitz functions. \citet{arjovsky2017wasserstein} proposed Wasserstein GANs, which approximate the 1-Wasserstein distance with gradient-clipped neural networks. Another example is the Total Variation (TV) distance, obtained when $\mF$ is the set of all measurable functions bounded between -1 and 1. Next, the Maximum Mean (MM) discrepancy is obtained when $\mF=\{f\in \mH: \norm{f}_\infty\leq 1\}$ for $\mH$ some Reproducing Kernel Hilbert Space (RKHS) associated with a given kernel $k: \mX \times \mX \to \R$. Our classification-based GAN framework thus provides an alternative route towards implementing ABCs with the Wasserstein distance \citep{bernton2019approximate} as well as MM \citep{park2016k2}.
One important aspect of these various metrics is that they induce topologies of different strengths. A divergence makes it easier for a sequence of distributions to converge when it induces a weaker topology. \citet{arjovsky2017wasserstein} show that KL induces the strongest one, followed by JS and TV, with Wasserstein distance being the weakest. However, the computation costs of the Wasserstein distance are high, i.e. $O((n+m)^3\log (n+m))$ to calculate it exactly \citep{burkard2012assignment}. The GAN version approximation also requires weight clipping to enforce the Lipschitz constraint which can be problematic to implement. Our KL estimator, on the other hand, is more scalable and places no constraints on the discriminator. {The neural network discriminator can converge faster than traditional nonparametric methods if some low-dimensionality assumption is satisfied \citep{kaji2020adversarial, bauer2019deep}}. In addition, the KL estimator has a natural interpretation through its link to the log-likelihood ratio.
The KL divergence appears in many theoretical results in Bayesian non-parametrics. For example, Schwartz's Theorem \citep{schwartz1965bayes} requires that, for posterior consistency, the prior should assign positive probability to any KL neighborhood of $p_0$. By adopting the KL estimator within our framework, we can relate ABC to classical posterior convergence rate results.
\vspace{-0.5cm}
\section{Frequentist Analysis of the ABC Posterior}\label{sec:freq_ABC}
\vspace{-0.3cm}
One way to assess the quality of the posterior distribution is through the speed at which it contracts around the truth $\theta_0$ as $n\rightarrow\infty$. While the ABC posterior is ultimately an approximation, it might still concentrate about $\theta_0$ at a reasonable rate.
Concentration rates are typically quantified in terms of a prior concentration (measured in terms of a combination of the KL divergence and the KL variation) and the entropy of the model. We have a similar prior mass condition (see (8.4) in Section 8.2 in \citet{ghosal2017fundamentals}). We denote the KL-neighborhood of $p_0$ by
\begin{equation}\label{eq:KL_ball}
B_2(p_0, \epsilon)=\{{\theta}: K(p_0,p_\theta)<\epsilon^2\}.
\end{equation}
\begin{assumption}\label{ass:prior_mass} (Prior Mass)
There exist some constants $\kappa>0$ and $\xi>0$ such that for every $0<\epsilon<\xi$ and some constant $C>0$, the prior probability satisfies $\Pi[B_2(p_0, \epsilon)]\geq C\epsilon^\kappa$.
\end{assumption}
Next, we assume that the parameter $\theta$ is identifiable in the sense that the KL divergence is locally compatible with the Euclidean norm. This assumption is adopted from Assumption 3(ii) of \citet{frazier2018asymptotic}.
%Since KL divergence is uniquely minimized at $p_0$, we remove the injectivity condition.
\vspace{-0.1cm}
\begin{assumption}\label{ass:identification} (Identification)
The density function $p_\theta$ is continuous in $\theta$ and for every $\theta$ in some open neighborhood of $\theta_0$ satisfies
\[
\norm{\theta-\theta_0}\leq L\times K(p_0, p_\theta)^\alpha
\]
for some $L>0$ and $\alpha>0$.
\end{assumption}
\vspace{-0.1cm}
Similarly as in (5.1) in \citet{kleijn2006misspecification},
\Cref{ass:identification} ensures posterior concentration around $\theta_0$ when $K(p_0, p_\theta)\to 0$. This holds for many distributions. For example, for the exponential distribution with a rate parameter $\theta$, we have $K(p_0, p_\theta)=\frac{\theta}{\theta_0}-\log(\frac{\theta}{\theta_0})-1$. Since $\log(1+x)=x-\frac{x^2}{2}+o(x^2)$ when $x\to 0$, we have
$K(p_0, p_\theta)\geq \frac{1}{2}\theta_0^{-2}(\theta-\theta_0)^2$. For multivariate normal distribution with a known variance $\Sigma$ and an unknown location $\mu$, we have
$K(p_0, p_\theta)=\frac{1}{2}(\mu-\mu_0)\Sigma^{-1}(\mu-\mu_0)\geq \frac{1}{2}\rho(\Sigma)^{-1}\norm{\mu-\mu_0}^2$, where $\rho(\Sigma)$ is the spectral radius, i.e., the largest eigenvalue, of a matrix $\Sigma$.
\vspace{-0.3cm}
\subsection{Posterior Concentration Rate}\label{sec:post_rate}
First, we focus on the uniform kernel $T_{\epsilon}(x)=\1(\abs{x}\leq \epsilon)$ used in Algorithm \ref{alg:abc_plain}.
Recall (from Remark \ref{rem:generate}) that $\tilde \X^\theta=g_\theta(\tilde \X)$ for some suitable mapping $g_\theta(\tilde \X)$ where $\tilde\X\sim\tilde P^{(m)}$.
The ABC joint posterior \eqref{eq:abc_post} is a weighted aggregation of uniform kernels, i.e
\begin{equation}
\hat \pi^{AR}\left(\theta,\tilde\X\mid \hat K[\X, g_\theta(\tilde \X)]\leq \epsilon\right)=\frac{\tilde\pi(\tilde\X)\pi(\theta)\1\big(\hat K[\X, g_\theta(\tilde \X)]\leq \epsilon\big)}{\int \tilde\pi(\tilde\X) \pi(\theta)\1\big(\hat K[\X, g_\theta(\tilde \X)]\leq \epsilon\big)\d \tilde\X \d\theta},
\end{equation}
which yields the following ABC posterior distribution
\begin{align}\label{eq:ABC_accept_post}
%\pi(\theta\mid \hat K(\X, \tilde \X^\theta) \leq \epsilon_n ) &=\frac{ \pi(\theta)P_\theta \big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)}{\int_{\Theta} \pi(\theta)P_\theta\big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)\d \theta},\\
\hat \Pi^{AR}_{\epsilon_n}(A\mid \X) &=\frac{ \int_{\theta\in A} \pi(\theta)\tilde P^{(m)} \big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)\d \theta}{\int_{\Theta} \pi(\theta)\tilde P^{(m)}\big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)\d \theta}
\quad \text{for a Borel-measurable $A\subset \Theta$}.
\end{align}
\iffalse
margins towards the following approximate (Accept/Reject) posterior density
\begin{align}
\hat \pi^{AR}_\epsilon(\theta \mid \X)&=\int \hat \pi^{AR}\left(\theta, \tilde \X \mid \hat K[\X, g_\theta(\tilde \X)]\leq \epsilon\right) d \tilde \X \nonumber \\
& =\frac{\pi(\theta) \tilde P^{(m)}\big(\hat K[\X, g_\theta(\tilde \X)]\leq \epsilon\big)}{\int \pi(\theta)\tilde P^{(m)} \big(\hat K[\X, g_\theta(\tilde \X)]\leq \epsilon\big) d\theta}\label{eq:alg_post1}.
\end{align}
In the fixed design, we would have, conditionally on a given $\tilde\X$ and $\X$,
\begin{equation}
\hat \pi^{AR}_\epsilon\left(\theta\mid \X,\tilde \X \right)=\frac{\pi(\theta)\1\big(\hat K(\X, \tilde \X^\theta)\leq \epsilon\big)}{\int \pi(\theta)\1\big(\hat K(\X, \tilde \X^\theta)\leq \epsilon\big) d\theta}.
\end{equation}
The corresponding ABC posterior distribution is defined through the following expression implied by \eqref{eq:alg_post1}
\begin{align}
%\pi(\theta\mid \hat K(\X, \tilde \X^\theta) \leq \epsilon_n ) &=\frac{ \pi(\theta)P_\theta \big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)}{\int_{\Theta} \pi(\theta)P_\theta\big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)\d \theta},\\
\hat \Pi^{AR}_{\epsilon_n}(A\mid \X) &=\frac{ \int_{\theta\in A} \pi(\theta)P_\theta^{(n)} \big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)\d \theta}{\int_{\Theta} \pi(\theta)P_\theta^{(n)}\big( \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\big)\d \theta}
\quad \text{for a Borel-measurable $A\subset \Theta$}.
\end{align}
\fi
The following theorem (a modification of Theorem 1 in \citet{frazier2018asymptotic}) quantifies the concentration rate in terms of the tolerance threshold $\epsilon_n$ as well as the rate at which the classification-based KL estimator can estimate $\P_n\log(p_0/p_\theta)$ (as formulated in \eqref{eq:approx_error}).
\begin{theorem} \label{thm:post_rate}
Let Assumptions \ref{ass:entropy}, \ref{ass:ratio_bd} and \ref{ass:prior_mass} hold and take $\delta_n$ as in \eqref{eq:approx_error} in \Cref{thm:conv_moments}. Then, as $n\to \infty$ and with $\epsilon_n=o(1)$ such that $n\epsilon_n^2\rightarrow\infty$ and $C_n\delta_n=o(\epsilon_n)$ for some arbitrarily slowly increasing sequence $C_n>0$ we have
\begin{equation}\label{eq:kl_rate}
P_0^{(n)}\Pi[K(p_0,p_\theta)>\lambda_n \mid\hat K(\X, \tilde \X^\theta) \leq \epsilon_n]=o(1),
\end{equation}
where $\lambda_n =\epsilon_n+M_nC_n \delta_n\epsilon_n^{-\kappa} +\sqrt{M_n} n^{-1/2}\epsilon_n^{-\kappa/2}$ for some arbitrarily slowly increasing sequence $M_n>0$. Moreover, if Assumption \ref{ass:identification} also holds, as $n\to \infty$, we have
\begin{equation}\label{eq:theta_abc_rate}
P_0^{(n)}\Pi[\norm{\theta-\theta_0}>L\lambda_n^\alpha \mid\hat K(\X, \tilde \X^\theta)\leq \epsilon_n]= o(1).
\end{equation}
\end{theorem}
The proof of the theorem is provided in \Cref{pf:post_rate}. Thus, the convergence rate of our ABC posterior depends on three components: the accept-reject threshold $\epsilon_n$, the estimation error of the KL estimator $\delta_n$ and the rate of discrepancy $n^{-1/2}$ between the empirical and true KL divergence. Since $\delta_n$ will typically be greater than the parametric rate $n^{-1/2}$, the overall convergence rate is then driven by $\lambda_n =\epsilon_n+ \wt M_n \delta_n\epsilon_n^{-\kappa}$, where $\wt M_n$
is an arbitrarily slowly increasing sequence.
\iffalse
\begin{remark} (Mean versus Sum)
One may wonder if it matters to aggregate the logit scores as the sum or as the mean. In practice, they should approximate the same posterior. The main difference lies in the choice of decision threshold inside the accept-reject process. If we use the sum of the logit scores as summary statistic, then $\sum_{i=1}^n \log \hat D^\theta_{n,m}/(1-\hat D^\theta_{n,m})(X_i)$ is a diverging function and its limit is $n K(p_0, p_\theta)$, leaving $\epsilon_n$ and $\lambda_n$ diverge as well. We are still able to recover the same theoretical rate if we modify the Lipschitz condition in Assumption 3 as $\norm{\theta-\theta_0}\leq Ln^{-\alpha}{K(p_0, p_\theta)}^\alpha$.
\end{remark}
\fi
\vspace{0.1in}
In practice, it is unclear how to properly choose $\epsilon_n$. In Algorithm \ref{alg:abc_exp}, we proposed to weight the draws using a scaled exponential kernel $\exp(-n\hat K(\X, \tilde \X^\theta) )$. We denote the ABC posterior under the exponential kernel as $\hat \Pi^{EK}(\cdot \mid \X)$ where
\begin{align}
\hat \Pi^{EK}(A\mid \X) &=\frac{\int_A \tilde P^{(m)} \exp\big(-n\hat K(\X, \tilde \X^\theta)\big)\pi(\theta) \d \theta}{\int_\Theta \tilde P^{(m)} \exp\big(-n\hat K(\X, \tilde \X^\theta)\big)\pi(\theta) \d \theta}. \label{eq:abc_post_exp}
\end{align}
To gain more insights into the ABC posterior behavior under the exponential kernel, we take a closer look at the ``likelihood function" above
\begin{align*}
\tilde P^{(m)} \exp\big(- n\hat K(\X, \tilde \X^\theta)\big) %&=\tilde P^{(m)}\exp\Big[- n\left(\hat K(\X, \tilde \X^\theta)-\P_n\log\frac{p_0}{p_\theta}\right)-n\P_n\log\frac{p_0}{p_\theta}\Big]\\
&=\frac{p_\theta^{(n)}}{p_0^{(n)}}\tilde P^{(m)} \e^{u_\theta},
\end{align*}
where
$
u_\theta(\X, \tilde\X^\theta)=- n\times \Big(\hat K(\X, \tilde \X^\theta)-\P_n\log\frac{p_0}{p_\theta}\Big).
$
From the equations above, we can write
\begin{equation}\label{eq:abc_post_exp_u}
\hat \pi^{EK} (\theta\mid \X) \propto p_\theta^{(n)}(\X)\times \e^{\hat u_\theta(\X)} \times \pi(\theta)\quad \text{with}\quad \hat u_\theta (\X)=\log \int \e^{u_\theta(\X,\tilde \X^\theta)}\d \tilde P^{(m)}(\tilde\X).
\end{equation}
When $\hat K(\X, \tilde\X^\theta)$ is the classification-based estimator, $\hat u_\theta(\X)$ can be related to the random generator setting of the Metropolis-Hastings MHC algorithm in \citet{kaji2021mh} which has \eqref{eq:abc_post_exp_u} as its stationary distribution. Similarly as in {Appendix Section 5} of \citet{kaji2021mh}, we can regard the posterior approximation in \eqref{eq:abc_post_exp_u} as a posterior $\hat \pi^{EK}(\theta \mid \X) \propto q_\theta^{(n)} \tilde \pi(\theta)$ under a misspecified likelihood
\begin{equation}\label{eq:tilde_p}
q_\theta^{(n)}=\frac{p_\theta^{(n)}(\X)\e^{\hat u_\theta(\X)}}{C_\theta} \quad \text{ where }\quad C_\theta=\int_\mX p_\theta^{(n)}(\X)\e^{\hat u_\theta(\X)} \d\X
\end{equation}
and a modified prior $\tilde \pi(\theta)\propto \pi(\theta)C_\theta$.
Since the likelihood is misspecified, the ABC posterior concentrates around a projection point $\theta^*$ defined as
\begin{equation}\label{eq:theta_star}
\theta^*=\arg\min_{\theta\in \Theta} -P_{0}^{(n)}\log[q_\theta^{(n)}/p_0^{(n)}],
\end{equation}
which corresponds to the mis-specified model that is closest to $P_0^{(n)}$ in the KL sense \citep{kleijn2006misspecification}.
\citet{kaji2021mh} study posterior concentration of \eqref{eq:abc_post_exp_u}. We recall their Theorem 4.5 in \Cref{sec:exp_kernel_post}.
Unlike in Theorem \ref{thm:post_rate}, the posterior concentration rate here depends both on the estimation error $\delta_n$ and the actual concentration rate of the true posterior, not the acceptance threshold.
\iffalse
\begin{remark}(Extension to i.i.d Case) The likelihood in \eqref{eq:tilde_p} does not separate due to the nested dependence of $u_\theta$ and $\X$. When $n\to \infty$, one can also extend the theorem to i.i.d. data case as studied in \citet{kleijn2006misspecification}. This can be achieved by splitting $\X$ into two halfs, one for training ($\X^{train}$) and one for prediction ($\X^{pred}$). Then $u_\theta(\cdot)$ only depends on $\X^{train}$, and thus $u_\theta(\X^{pred})$ is separable.
\end{remark}
\fi
{
\begin{remark} (Vanishing Bias) To better understand the severity of the centering bias of the misspecified model, we note
\begin{align*}
-P_{0}^{(n)}\log[q_{\theta^*}^{(n)}/p_0^{(n)}] &\leq -P_{0}^{(n)}\log[q_{\theta_0}^{(n)}/p_0^{(n)}]
= P_0^{(n)} \log \frac{p_0^{(n)}}{p_0^{(n)} \e^{\hat u_{\theta_0}(\X) }/C_{\theta_0}} \\
&= \log C_{\theta_0}- P_0^{(n)} {\hat u_{\theta_0}(\X)} = \log P_0^{(n)} \e^{\hat u_{\theta_0}(\X)} - P_0^{(n)} {\hat u_{\theta_0}(\X)}.
\end{align*}
This is essentially the Jensen gap. If we have this Jensen gap vanishing when $n\to \infty$, then we can conclude that the centering bias is also vanishing, and the ABC posterior in \eqref{eq:abc_post_exp} will eventually concentrate at the right location.
\end{remark}
}
\begin{remark} (Connection to Generalized Bayesian Inference)
The exponential kernel has an alternative interpretation as a generalized posterior within the Generalized Bayesian Inference (GBI) framework \citep{bissiri2016general}.
\citet{schmon2020generalized} show that one can view the ABC posterior constructed from the scaled exponential kernel as
\[
p_l(\theta\mid\X) \propto \int \exp(-w\cdot l(\X, \tilde \X^\theta)) \pi(\theta)\d\tilde P^{(m)}(\tilde\X),
\]
where $\l(\cdot, \cdot)$ is a ``loss function" which can be any distance between the summary statistics or a data discrepancy metric used in ABC, and where the weight $w >0$ can be chosen to adjust the influence of the loss on the posterior.
The exponential kernel can thus be applied more generally to other summary statistics or data discrepancy measures.
\end{remark}
\vspace{-0.3cm}
\subsection{Shape of the Limiting ABC Posterior Distribution}
We now analyze the limiting shape of $\hat \Pi^{AR}_{\epsilon_n}(\cdot\mid \X)$ defined in \eqref{eq:ABC_accept_post}. We focus on the case when $\epsilon_n\gg \delta_n^{1/(\kappa+1)}$, where $\kappa$ was defined in Assumption \ref{ass:prior_mass}, since the posterior is not guaranteed to converge when the decision threshold $\epsilon_n$ is smaller than the estimation error $\delta_n$ of the KL estimator.
\begin{assumption}\label{ass:KL_taylor}
Assume that for every $\varepsilon>0$, we have $\inf_{\|\theta-\theta_0\|>\varepsilon}K(p_0,p_\theta)>0$. In addition, assume that $\log p_\theta$ is twice differentiable with respect to $\theta$ and that, for every $\theta$ in some neighborhood of $\theta_0$, the remainder of the second order Taylor expansion of $K(p_0, p_\theta)= P_0 \log\frac{p_0}{p_\theta}(x)$ around $\theta_0$ is comparatively small relative to the second-order term, i.e.
\begin{align*}
K(p_0,p_\theta)&=\nabla_{\theta=\theta_0} K(p_0, p_\theta) (\theta-\theta_0)+\frac{1}{2}(\theta-\theta_0)'\nabla^2_{\theta=\theta_0} K(p_0, p_\theta) (\theta-\theta_0)\{1+o(1)\} \\
&=\frac{1}{2} (\theta-\theta_0)'I(\theta_0) (\theta-\theta_0)\{1+o(1)\},
\end{align*}
where $I(\theta)=\nabla^2_{\theta} K(p_0, p_\theta)= P_0[(\nabla_\theta \log p_\theta)^2]$ is the Fisher information matrix.
\end{assumption}
{When \Cref{ass:KL_taylor} is satisfied, the condition in \Cref{ass:identification} is immediately satisfied as well and the identification of $\theta_0$ is guaranteed. In the open-neighborhood of $\theta_0$, since we have
\[
(\theta-\theta_0)'I(\theta_0) (\theta-\theta_0) = 2K(p_0, p_\theta)\{1+o(1)\},
\]
and $I(\theta_0)$ is positive definite, the convergence in the bilinear form $ (\theta-\theta_0)'I(\theta_0) (\theta-\theta_0)$ ensures the convergence of the parameters.}
\begin{theorem}\label{thm:unif_ci}
Assume that the prior function $\pi(\cdot)$ is continuous around $\theta_0$. Then, under Assumptions \ref{ass:entropy}, \ref{ass:ratio_bd}, \ref{ass:prior_mass} and \ref{ass:KL_taylor}, if $\lim_n \delta_n/\epsilon_n^{\kappa+1}\to 0$, the average posterior distribution of $\epsilon_n^{-1/2}(\theta-\theta_0)$ converges to the uniform distribution over the ellipse $\{w: w' I(\theta_0) w\leq 2\}$ where $I(\theta)$ is the Fisher information matrix defined in \Cref{ass:KL_taylor}. In particular, as $n\to 0$, we have
\[
P_0^{(n)}\int f\Big(\epsilon_n^{-1/2}(\theta-\theta_0)\Big) \hat \Pi^{AR}_{\epsilon_n}(\theta \mid \X) \to \int_{u'I(\theta_0)u\leq 2} f(u) \d u/\int_{u'I(\theta_0)u\leq 2} \d u
\]
for every continuous and bounded function $f(\cdot):\mX\rightarrow\R$.
\end{theorem}
The proof is provided in \Cref{sec:proof_unif_ci}.
\begin{remark}
\Cref{thm:unif_ci} is adapted from the case (i) in Theorem 2 of \citet{frazier2018asymptotic}. We only consider situations when $\epsilon_n \gg\delta_n^{1/(\kappa+1)}$ with the prior shrinkage parameter $\kappa$ defined in Assumption \ref{ass:prior_mass}. In other words, we assume that the ABC decision threshold $\epsilon_n$ is dominating both the estimation error $\delta_n$ and the asymptotic error $n^{-1/2}$ and, thereby, determines the posterior concentration rate. It is not entirely obvious how the posterior would behave when the threshold $\epsilon_n$ shrinks faster than the estimation error $\delta_n$.
\iffalse
\Cref{thm:unif_ci} is adapted from case (i) in Theorem 2 of \citet{frazier2018asymptotic}. We only consider the case where $\epsilon_n \gg\delta_n^{1/(\kappa+1)}$ with the prior shrinkage parameter $\kappa$ from Assumption \ref{ass:prior_mass}. In other words, we assume that the ABC decision threshold $\epsilon_n$ is dominating both the estimation error $\delta_n$ and the asymptotic error $n^{-1/2}$ and, thereby, determines the posterior concentration rate. It is not entirely obvious how the posterior would behave when the threshold $\epsilon_n$ shrinks faster than the estimation error $\delta_n$. In addition, we study the posterior behavior of $\epsilon_n^{-1/2}(\theta-\theta_0)$ rather than $\epsilon_n^{-1}(\theta-\theta_0)$. This inefficiency is the result of $\nabla_{\theta=\theta_0} K(p_0,p_\theta) =0$ and the use of second-order expansion.
\fi
\end{remark}
For the asymptotic behavior of the ABC posterior \eqref{eq:abc_post_exp_u} induced by the exponential kernel, we resort to BvM characterizations under misspefication in LAN models \citep{kleijn2012bernstein}. {When the posterior \eqref{eq:abc_post_exp_u} concentrates around $\theta^*$ in \eqref{eq:theta_star} at the rate $\epsilon_n^*$, one can show under a suitable LAN condition that the approximate posterior converges to a sequence of normal distributions in total variation at the rate $\epsilon_n^*$. The centering and the asymptotic covariance matrix both depend on $\theta^*$. The formal statement and the proof is in \citet{kaji2021mh}.
}
\vspace{-0.5cm}
\section{Simulations}
\vspace{-0.3cm}
In this section, we illustrate our approach and make comparisons with other likelihood-free inference techniques. Within our KL-ABC framework, we include two types of KL estimators. One is obtained with the logit discriminator score, which we refer to as KL estimation via classification (KLC), and the other one is estimated via the kNN method (kNN) with $k=1$ \citep{jiang2018approximate}. For both estimators, we aggregate ABC samples with the accept-reject kernel as in Algorithm \ref{alg:abc_plain} and the exponential kernel as in Algorithm \ref{alg:abc_exp}. The latter will be denoted with a suffix `exp', e.g. KLC-exp or kNN-exp. The discriminator used for each dataset will be specified later. We provide more discussions on discriminator calibrations in \Cref{sec:dhat_training}. Besides the two examples presented here, we have another analysis on the g-and-k distribution in \Cref{sec:gk}.
The ABC discrepancy metrics we choose for comparisons are (1) the classification accuracy (CA) \citep{gutmann2018likelihood} defined as \eqref{eq:ca}; (2) the Wasserstein (WA) distance under the Euclidean metric \citep{bernton2019approximate} defined as $\text{WA}(\X,\tilde \X^\theta)= \min_\gamma [\sum_{i=1}^n \sum_{j=1}^m \gamma_{ij}\norm{X_i-\tilde X^\theta_j}^2]^{1/2}$ s.t. $\gamma' {\bf 1}_m={\bf 1}_n, \gamma' {\bf 1}_n={\bf 1}_m$ with $0\leq \gamma_{ij}\leq 1$; (3) $\ell_2$-distance between summary statistics (SS) and we use the semi-automatic (SA) method \citep{fearnhead2011constructing} if no candidate summary statistics are given; (4) auxiliary likelihood $AL=\frac{1}{m} \ln p_A(\tilde\X^\theta\mid \hat \phi(\tilde\X^\theta)) -\frac{1}{m} \ln p_A(\tilde\X^\theta\mid \hat \phi(\X)) $ proposed by \citet{drovandi2011approximate}, where $p_A(x\mid \phi)$ is a $d$-dimensional Gaussian distribution with $\phi$ being the sample mean and covariance. For the classification accuracy (CA), we use the same discriminator as the one in KL estimation.
In each experiment, unless otherwise noted, we set the tolerance threshold $\epsilon$ adaptively such that $1\,000$ of $100\,000$ (i.e. the top 1\%) proposed ABC samples are accepted.
\vspace{-0.5cm}
\subsection{M/G/1-queuing Model}
Because queuing models are usually easy to simulate from, but have no tractable likelihoods, they have been frequently used as test cases in the ABC literature, see e.g. \citet{fearnhead2011constructing} and \citet{bernton2019approximate}.
Here, we choose the same setup as in \citet{jiang2017learning}. Each datum is a 5-dimensional vector consisting of the first five inter-departure times $x_i=(x_{i1},x_{i2},x_{i3},x_{i4},x_{i5})'$. In the model, the service times $u_{ik}$ follow a uniform distribution $U[\theta_1, \theta_2]$, and the arrival times $w_{ik}$ are exponentially distributed with the rate $\theta_3$. We only observe the interdeparture times $x_i$, given by the process $x_{ik}=u_{ik}+\max(0, \sum_{j=1}^k w_{ij} -\sum_{j=1}^{k-1} x_{ij})$. We perform ABC on $n=500$ observed samples which are generated from the true parameter $\theta_0=(1, 5, 0.2)$. The prior on $(\theta_1, \theta_2-\theta_1, \theta_3)$ is uniform on $[0,10]^2\times [0, 0.5]$. \footnote{We place the uniform prior on $\theta_2-\theta_1$ instead of $\theta_2$, since $\theta_2$ must be larger than $\theta_1$. This is used in \citet{jiang2017learning}.}
We train the discriminator as a $\ell_1$-penalized logistic regressor onto degree-2 polynomials of the data, including quadratic and interaction terms. The penalty parameter $\lambda$ is selected via 5-fold cross-validation. We implement the discriminator with the R package \texttt{glmnet}.
\begin{figure}[!ht]
\begin{center}
\includegraphics[width=0.9\textwidth]{fig/mg1_n500_thetas_001.eps}
\end{center}
\caption{ABC reconstructed posteriors under M/G/1-queuing model with $\theta_0=(1, 5, 0.2)'$. The vertical black lines mark the true values. The blue curves in kNN and KLC boxes mark a smoothed density calculated from the exponential kernel.}\label{fig:mg1_thetas}
\end{figure}
The shape of the ABC posteriors is given in \Cref{fig:mg1_thetas}. The plot reveals that, among the three parameters, $\theta_1$ is the hardest one to estimate where all methods, except for the semi-automatic ABC (SA), gave relatively flat posterior estimates. Regarding $\theta_2$, our methods, SA and W2 seem to center well around the truth and W2 gives the tightest estimation. Regarding $\theta_3$, it is obvious that SA does not provide enough information. Next, we repeat the experiments on 10 different datasets, and summarize the average squared estimation errors and the width of the 95\% credible intervals in \Cref{tab:mg1_stats}. Overall, we see that although SA captures $\theta_1$ and $\theta_2$ well, it fails to identify the correct location of $\theta_3$. Our KLC method outperforms the kNN estimator in estimating all three parameters. The two variants of KLC and CA give relatively good and robust estimates for all three parameters. These methods tend to be similar in both the point estimate and credible intervals, which is not entirely unexpected since they are derived from the same discriminators.
\begin{table}[!ht]
\centering
%\renewcommand{\arraystretch}{1.1}
\scalebox{0.9}{
\begin{tabular}{l | *{2}{c} | *{2}{c} |*{2}{c}} \toprule
& \multicolumn{2}{c|}{$\theta_1=1$} & \multicolumn{2}{c|}{$\theta_2=5$} & \multicolumn{2}{c}{$\theta_3=0.2$} \\
{Method} &{$(\hat\theta_1-\theta_1)^2$} &{95\% CI width} & {$(\hat\theta_2-\theta_2)^2$} & {95\% CI width} &{$(\hat\theta_3-\theta_3)^2$} &{95\%CI width} \\
\midrule
KLC & 0.197 & 3.116 & 0.217 & 4.599 & 0.308 & 0.064 \\
KLC-exp & \bf{0.169} & 2.851 & 0.312 & 3.708 & \bf{0.234} & \bf{0.030} \\
kNN & 0.525 & 3.135 & 0.106 & 3.986 & 3.659 &0.094 \\
kNN-exp & 1.057 & 2.664 (0.8) & 0.431 & {\bf 3.331} (0.9)& 2.634 & 0.072 \\
\vspace{-0.15in} &&&&&&\\
AL & 0.259 & 3.423 & 0.057 & 5.651 & 0.575 & 0.044 \\
CA & 0.191 & 3.107 & 0.235 & 4.602 & 0.280 & 0.064 \\
SA & 0.180 & \bf{2.457} & 0.355 & 3.514 & 45.297 & 0.446 \\
W2 & 0.595 & 3.631 & \bf{0.039} & 4.871 & 3.846 &0.052 (0.8)\\
\bottomrule
\end{tabular}
}
\caption{ABC performance on the M/G/1 queuing model over 10 repetitions with top 1\% ABC samples selected. Most of the 95\% CIs have full coverage with the rest having their coverage marked in brackets. The bold fonts mark the best model in each metric.}\label{tab:mg1_stats}
\end{table}
\vspace{-0.3cm}
\subsection{Lotka-Volterra Model}
The Lotka-Volterra (LV) predator-prey model \citep{wilkinson2018stochastic} describes population evolutions in ecosystems where predators interact with prey. It is one of the classical stochastic kinetic network model examples. The state of the population is prescribed deterministically via a system of ordinary differential equations (ODEs). Inference for such models is challenging because the transition density is intractable. However, simulation from the model is possible, which makes it a natural candidate for ABC methods.
The model monitors population sizes of predators $X_t$ and prey $Y_t$ over time $t$. The changes in states are determined by four parameters $\theta=(\theta_1, \ldots, \theta_4)'$ controlling: (1) the rate $r_1^t=\theta_1 X_t Y_t$ of a predator being born; (2) the rate $r_2^t=\theta_2 X_t$ of a predator dying; (3) the rate $r_3^t=\theta_3Y_t$ of a prey being born; (4) the rate $r_4^t=\theta_4X_tY_t$ of a prey dying. Given the initial population sizes $(X_0, Y_0)$ at time $t=0$, the dynamics can be simulated using the Gilliespie algorithm \citep{gillespie1977exact}. The algorithm samples times to an event from an exponential distribution (with a rate $\sum_{j=1}^4 r_j^t$) and picks one of the four reactions with probabilities proportional to their individual rates $r_j^t$.
We use the same simulation setup as \citet{kaji2021mh}. Each simulation is started at $X_0=50$ and $Y_0=100$ and state observations are recorded every 0.1 time units for a period of 20 time units, resulting in a series of $T=201$ observations each. The real data ($n=20$ time series) are generated with true values $\theta_0=(0.01, 0.5, 1, 0.01)'$. The predator-prey interaction dynamic is very sensitive to parameter changes. For example, Figure 4 of \citet{kaji2021mh} shows that a slight perturbation in $\theta_2$ leads to significant changes in the population renewal cycle. We rely on the ability of the discriminator to tell such different patterns apart. The sensitivity of the model to minor parameter changes is confirmed with heat-map plots of the estimated KL divergence as a function of $(\theta_1, \theta_4)'$ in \Cref{fig:lv_heatmap}. \Cref{fig:lv_zoomout} provides a plot of the estimated KL over the region $[0,0.1]^2$ where, apparently, the majority of the region is flat and uninformative with a sharp spike around the true values at $\theta_1=\theta_4=0.01$. We thus narrow the investigation down to a smaller region $[0,0.02]^2$ in \Cref{fig:lv_zoomin}. Again, the curvature in the estimated KL around the truth is quite steep. This may pose some issues for Metropolis-Hasting algorithms, since the majority of the prior region is uninformative and improper initialization could lead to extremely slow convergence.
%The results from our ABC analysis can serve as a pioneer search for the initialization point for MH in this scenario.
\begin{figure}[!ht]
\centering
\subfigure[$(\theta_1, \theta_4) \in (0,0.1)^2$]{
\includegraphics[width=0.4\textwidth]{fig/lotka3.png}\label{fig:lv_zoomout}
}
\subfigure[$(\theta_1, \theta_4) \in (0,0.02)^2$ ]{
\includegraphics[width=0.4\textwidth]{fig/lotka1.png}\label{fig:lv_zoomin}
}
\caption{Estimated $\hat K(\X, \tilde \X^\theta)$ (via Classification) over a grid of $(\theta_1, \theta_4)$ values. The other two parameters are fixed as $\theta_2=0.5, \theta_3=1$.} \label{fig:lv_heatmap}
\end{figure}
Previous ABC analyses of this model suggested various summary statistics including the mean, log-variance, autocorrelation (at lag 1 and 2) of each series as well as their cross-correlation \citep{papamakarios2016fast}. For the discriminator of our method, we choose the $\ell_1$-penalized (LASSO) logistic regression classifier with $m=n$ and with a penalty $\lambda$ selected via 5-fold cross-validation (as implemented in the R package \texttt{glmnet}).
Similar to \citet{kaji2021mh}, we use an informative prior $\theta \in U(\Xi)$ with a restricted domain $\Xi =[0, 0.1] \times [0, 1] \times [0,2] \times [0, 0.1]$ so that the computation is more economic and efficient. A typical snapshot of the ABC posteriors is plotted in \Cref{fig:lv_post}. From the plot, we can see that the difficulty in estimating different parameters varies a lot and $\theta_3$ is the most challenging among all. While other methods give relatively flat posteriors for $\theta_3$, our method (KLC) combined with the scaled exponential kernel identifies the correct location of the parameter with a much tighter posterior. The average performance of ABC methods under this model is summarized in \Cref{tab:lv_comp}. We can see that KLC with the exponential kernel gives the tightest CI most of the time, while maintaining relatively small estimation errors.
\begin{figure}[!ht]
\centering
\includegraphics[width=0.85\textwidth]{fig/lotka_abc_002}
\caption{ABC posterior densities for the Lotka-Volterra model with $\theta_0=(0.01, 0.5, 1, 0.01)'$. The black vertical lines mark the true parameter values. The blue curves in kNN and KLC boxes represent the smoothed density calculated from the exponential kernel. The ABC posteriors are plotted from the top 1\% out of $10^5$ samples.}\label{fig:lv_post}
\end{figure}
\begin{table}[!ht]
\centering
%\renewcommand{\arraystretch}{1}
\scalebox{0.75}{
\begin{tabular}{l | *{2}{c} | *{2}{c} | *{2}{c} | *{2}{c} }
\toprule
& \multicolumn{2}{c|}{$\theta_1=0.01$} & \multicolumn{2}{c|}{$\theta_2=0.5$} & \multicolumn{2}{c|}{$\theta_3=1$} & \multicolumn{2}{c}{$\theta_4=0.01$} \\
{Method} & {$(\hat\theta_1-\theta_1)^2$} & {95\% CI} width & {$(\hat\theta_2-\theta_2)^2$} & { 95\% CI} width & {$(\hat\theta_3-\theta_3)^2$} & { 95\% CI} width & {$(\hat\theta_4-\theta_4)^2$} & {95\% CI} width\\
(scale) & {\small$(10^{-5})$} & & & & {\small$(10^{-2})$} & & {\small$(10^{-5})$ }& \\
\midrule
KLC & 42.949 & 0.0447 & 0.0169 & 0.626 & 0.119 & 0.945 & 17.159 & 0.0472 \\
KLC-exp & 0.066 & \bf{0.0035} & 0.0025 & \bf{0.285} & 0.405 & \bf{0.643} & \bf{0.021} & \bf{0.0063} \\
kNN & 0.563 & 0.0146 & 0.0011 & 0.602 & 0.054 & 0.934 & 0.203 & 0.0133 \\
kNN-exp & \bf{0.049} & 0.0054 (0.9) & \bf{0.0004} & 0.243 (0.9) & 0.346 & \bf{0.599} (0.9) & 0.066 & \bf{0.0062} (0.9)\\
\vspace{-0.14in} &&&&&&&&\\
CA & 36.477 & 0.0438 & 0.0160 & 0.629 & 0.118 & 0.945 & 15.684 & 0.0462 \\
Stats & 1.639 & 0.0178 & 0.0226 & 0.721 & 1.019 & 0.901 & 3.767 & 0.0166 \\
W2 & 1.265 & 0.0167 & 0.0048 & 0.599 & \bf{0.011} & 0.917 & 4.080 & 0.0205\\
\bottomrule
\end{tabular}}
\caption{ABC performance evaluations on the Lotka-Volterra Model, averaged over 10 repetitions, with the top 1\% ABC samples selected. Most of the 95\% CIs have full coverage with the rest having their coverage marked in brackets. The bold fonts mark the best model in each metric.}\label{tab:lv_comp}
\end{table}
\section{Empirical Analysis}
\vspace{-0.3cm}
We further demonstrate our approach on the nontrivial problem of estimating stock volatility using merely daily observations on high, low and closing prices. All of these price observations are typically available to investors. We use a similar data generating process as in \citet{magdon2003maximum}, assuming that the assets follow a Brownian motion with a constant drift and volatility. % and estimate the parameters using the maximum likelihood approach.
In particular, suppose that the log-price processes $X_j(t), i=1, \ldots, d$, are correlated Brownian motions, that is $E[X_i(s)X_j(t)]=\sigma_{ij}\min\{s,t\}$,
and that the joint movement of the log-price processes $\X(t)=(X_1(t), \ldots, X_d(t))'$ follows a multivariate Brownian motion as
\begin{equation}
d\X(t)= \bmu dt+\Sigma d \bm{W}(t),
\end{equation}
where $\bmu=(\mu_1, \ldots, \mu_d)'$ and $\Sigma=[\sigma_{ij}]_{1\leq i,j\leq d}$ denote the drift and the volatility of the log processes, respectively. We write
\[
H_j =\max_{0\leq t\leq 1} X_j(t), \qquad L_j=\min_{0\leq t\leq 1} X_j(t), \qquad S_j=X_j(1),
\]
for the high, low and final log price, respectively, over a fixed time interval $[0,1]$. We want to estimate the drift $\bmu$ and the volatility matrix $\Sigma$ merely from observing these three prices over a period of time.
We impose a normal-inverse-Wishart prior $(\bmu,\Sigma )\sim NIW (\bmu_0, \lambda, \Phi, \nu)$. This distribution can be sampled from in two steps: (1) sample $\Sigma$ from an inverse Wishart distribution $\Sigma\mid \Phi, \nu \sim W^{-1} (\Phi, \nu)$; (2) sample $\bmu$ from a multivariate normal distribution $\bmu\mid\bmu_0 , \lambda, \Sigma \sim N(\bmu_0, \frac{1}{\lambda}\Sigma)$. Since $\Sigma$ is a semi-positive definite matrix, we model the parameters through its Cholesky root $\Sigma^{1/2}$. Without loss of generality, we only consider the case $W(0)=(0, \ldots, 0)'$, since the closing prices on the previous day or the opening prices of today are usually known.
\subsection{Synthetic Data}
To compare various likelihood-free estimators, we first generate synthetic data for 1\,000 trading days. For each day (of length $t=1$), we simulate the Brownian motion using 500 time steps to obtain the high, low and closing price data for each particular window.
We first restrict our attention to the case of just two assets, which leaves us with $5$ parameters to estimate: $\mu_1, \mu_2$ and the upper triangular root of $\Sigma$, denoted with
$
L={\scriptsize\left[\begin{array}{cc}
l_{11} & 0 \\
l_{12}& l_{22}
\end{array}\right]}$.
We first illustrate how the covariance parameter $\sigma_{12}$ impacts the co-movement of asset prices. Holding $\mu_1=\mu_2=0$ and $\sigma_{11}=\sigma_{22}=1$ fixed, we plot time series realizations of the closing prices for three particular choices of $\sigma_{12}$ in \Cref{fig:AP_s12}. The patterns are as expected where the prices tend to co-fluctuate when $\sigma_{12}$ is closer to one.
The success of our method depends on how well the discriminator can tell apart these trajectories.
\vspace{-0.3cm}
\begin{figure}[!ht]
\subfigure[$\sigma_{12}=0.05$]{
\includegraphics[width=0.3\textwidth]{fig/AP1}
}
\subfigure[$\sigma_{12}=0.5$]{
\includegraphics[width=0.3\textwidth]{fig/AP2}
}
\subfigure[$\sigma_{12}=0.95$]{
\includegraphics[width=0.3\textwidth]{fig/AP3}
}
\caption{Closing prices time series for three choices of $ \sigma_{12}$ ($\mu_1=\mu_2=0, \sigma_{11}=\sigma_{22}=1$).}
\label{fig:AP_s12}
\end{figure}
For our simulation, we set $\mu=(0,0)'$ and $\sigma_{11}=\sigma_{22}=1, \sigma_{12}=0.5$. We choose relatively non-informative prior hyper-parameters $\bmu_0=(0,0)',\lambda=1, \Phi=I_d$ and $\nu=d$.
Posterior distributions reconstructed with different ABC methods are given in \Cref{fig:brownian_simu} and the averaged performance over 10 repetitions is summarized in \Cref{tab:brownian_simu}. It is clear that our exponential kernel methodsplace more mass around the true location of the parameters, and that the shape of kNN-exp is slightly less regular than KLC-exp. Although KLC-exp induces a larger bias in estimating the drift $(\mu_1, \mu_2)$, it does a better job at capturing the correct location of the volatilities.
%The differences in posterior shapes and the CI lengths indicate that the shapes of our methods with an exponential kernel could be closer to the shape of the true posterior, while the other methods have flatter posteriors due to the use of accept-reject kernels.
\begin{figure}[!ht]
\centering
\subfigure[ABC]{
\includegraphics[width=0.85\textwidth]{fig/brownian_d2_n1000_abc_001}}
\caption{Posterior densities on simulated volatility data ($d=2$)}\label{fig:brownian_simu}
\end{figure}
% Please add the following required packages to your document preamble:
% \usepackage{multirow}
\begin{table}[!ht]
\centering
%\renewcommand{\arraystretch}{0.67}
\scalebox{0.85}{
\begin{tabular}{c c | *{7}{c}}
\toprule
& & KLC & KLC-exp & kNN & kNN-exp & CA & SS & WA \\
\midrule
\multirow{2}{*}{$\mu_1=0$} & MSE$(\times 10^{-4})$ & 2.004 & 2.113 & 0.638 & \bf{0.159} & 1.497 & 1.216 & 0.170 \\
& 95\% CI width & 1.760 & \bf{0.331} & 1.480 & 0.442 & 1.660 & 1.734& 0.831 \\
\vspace{-0.14in} & & & & & & & & \\
\multirow{2}{*}{ $\mu_2=0$} & MSE$(\times 10^{-4})$ & 0.734 & 7.439 & 0.363 & 0.259 & 0.142 & \bf{0.021} & 0.277 \\
&95\% CI width & 1.628 & \bf{0.431} & 1.435 & 0.538 & 1.564 & 1.569 & 0.813 \\
\vspace{-0.14in} & & & & & & & & \\
\multirow{2}{*}{$\sigma_{11}=1$} & MSE$(\times 10^{-3})$ & 0.311 & 0.111 & 0.167 & 0.460 & \bf{0.002} & 1.404 & 2.634 \\
& 95\% CI width & 0.963 & 0.371 & 0.954 & \bf{0.329 } & 0.896 & 0.763 & 1.140 \\
\vspace{-0.14in} & & & & & & & & \\
\multirow{2}{*}{$\sigma_{12}=0.5$} & MSE$(\times 10^{-3})$ & 1.035 & \bf{0.103} & 0.427 & 0.107 & 0.861 & 0.109 & 1.528 \\
&95\% CI width & 1.193 & 0.355 & 1.092 & \bf{0.324} & 1.122 & 0.628 & 0.988 \\
\vspace{-0.14in} & & & & & & & & \\
\multirow{2}{*}{$\sigma_{22}=1$} & MSE$(\times 10^{-3})$ & 0.416 & 0.304 & 0.124 & 0.018 & \bf{0.007} & 3.687 & 4.244 \\
& 95\% CI width & 0.959 & \bf{0.252} & 0.974 & 0.314 & 0.920 & 0.783 & 1.157 \\
\bottomrule
\end{tabular}}
\caption{Performance on the stock volatility estimation example, averaged over 10 repetitions, with top 1\% selected. All 95\% CIs have full coverage of the true parameters. The bold fonts mark the best model in each row.}\label{tab:brownian_simu}
\end{table}
\subsection{Real Data Analysis}
Following the example in \citet{rogers2008estimating}, we examine a small data set of stock prices focusing on two stocks: Boeing (BA) and Proctor \& Gamble (PG). The prices were obtained from NYSE (Yahoo Finance), starting from 3rd January 2011 and consisting of $1\,000$ trading days. Since the off-market trades follow a different mechanism than the market trade, we only model price changes from the opening price to the closing price each day where the log price differences $X(t)$ are all computed based on the opening prices of that day.
We observe that the fluctuations in the prices are much smaller than in our simulated time series and we thereby choose the hyperparameters based on the mean and covariance of the closing prices. \Cref{fig:brownian_v2_d2} gives the ABC posterior distributions estimated from different methods and the corresponding summaries of the distributions are provided in \Cref{tab:brownian_v2_d2}. We again observe that our methods with the exponential kernels return much narrower posterior distributions. The estimates on the volatilities are quite close except for the ABC with summary statistics (SS). The drifts of both BA and PG are not significantly different from zero.
\begin{figure}[!ht]
\centering
\includegraphics[width=0.8\textwidth]{fig/brownian_v2_d2_abc_post}
\caption{Posterior densities estimated from log-prices of BA and PG}\label{fig:brownian_v2_d2}
\end{figure}
% Please add the following required packages to your document preamble:
% \usepackage{multirow}
\begin{table}[!ht]
\centering
%\renewcommand{\arraystretch}{0.67}
\scalebox{0.8}{
\begin{tabular}{rr | *{7}{c} }
\toprule
& & KLC & KLC-exp & kNN & kNN-exp & CA & SS & W2 \\
\midrule
\multirow{3}{*}{$\mu_1$
$(\times 10^{-3})$} & $\bar\mu_1$& -0.774 & 1.354 & 0.250 & 0.779 & 0.342 & -0.071 & -0.126 \\
&$l$ & -10.380 & -1.981 & -6.816 & -1.145 & -7.099 & -9.532 & -3.987 \\
& $u$& 8.202 & 3.514 & 7.229 & 3.385 & 7.395 & 10.121 & 3.736 \\
\vspace{-0.12in} \\
\multirow{3}{*}{$\mu_2$ $(\times 10^{-3})$} & $\bar\mu_2$ & 0.009 & -0.039 & 0.491 & 0.822 & 0.445 & 0.450 & 0.453 \\
& $l$ & -6.591 & -1.718 & -4.329 & -1.406 & -4.604 & -9.831 & -3.491 \\
& $u$ & 6.531 & 4.563 & 5.503 & 2.166 & 5.538 & 10.171 & 4.450 \\
\vspace{-0.12in} \\
\multirow{3}{*}{$\sigma_{11}$$(\times 10^{-4})$} & $\bar \sigma_{11}$ & 1.406 & 1.142 & 1.049 &1.054 & 1.101 & 0.738 & 1.130 \\
& $l$ & 0.824 & 0.076 & 0.633 & 0.844 & 0.634 & 0.155 & 0.626 \\
& $u$ & 2.362 & 1.444 & 1.640 & 1.202 & 1.819 & 1.976 & 1.938 \\
\vspace{-0.12in} \\
\multirow{3}{*}{$\sigma_{12}$$(\times 10^{-5})$} & $\bar \sigma_{12}$& 2.811 & 2.446 & 1.767 & 2.084 & 1.906 & -0.376 & 2.052 \\
&$l$& -2.379 & -0.37 & -2.359 & 0.494 & -2.390 & -6.660 & -3.137 \\
&$u$& 8.226 & 3.629 & 6.037 & 4.298 & 6.442 & 6.065 & 7.253 \\
\vspace{-0.12in} \\
\multirow{3}{*}{$\sigma_{22}$$(\times 10^{-4})$} &$\bar \sigma_{22}$ & 0.692 & 0.526 & 0.492 & 0.482 & 0.547 & 0.552 & 0.495 \\
& $l$ & 0.407 & 0.089 & 0.277 & 0.365 & 0.315 & 0.133 & 0.173 \\
& $u$ & 1.105 & 0.698 & 0.802 & 0.559 & 0.883 & 1.285 & 0.984 \\
\bottomrule
\end{tabular}}
\caption{Posterior estimates on analysis of BA and PG. For each parameter, we report three summary statistics, the posterior means, the lower limit of the 95\% CI intervals ($l$) and the upper limit of the 95\% CI intervals ($u$).}\label{tab:brownian_v2_d2}
\end{table}
\vspace{-0.3cm}
\section{Discussion}
\vspace{-0.3cm}
This paper develops an ABC variant using a classification-based KL estimator as a discrepancy measure. By deploying a flexible classifier, the empirical KL divergence can be estimated with a vanishing error. In addition, inspired by the connection between the KL divergence and the log-likelihood ratio, we propose a scaled exponential kernel to aggregate ABC samples. This smoothing variant avoids the need for choosing the ad hoc threshold $\epsilon_n$ and fully utilizes information returned from all ABC samples. Under mild conditions, we show that the posterior concentration rate of the accept-reject ABC depends on the estimation error $\delta_n$ and the accept-reject threshold $\epsilon_n$, while the rate of the smooth version depends on the estimation error $\delta_n$ and the contraction rate of the actual posterior distribution. Our methodology can also be related to many other likelihood-free inference methods, including ABC with Classification Accuracy \citep{gutmann2018likelihood}, Wasserstein distance ABC \citep{bernton2019approximate}, and Generalized Posteriors \citep{schmon2020generalized}. Along with our theoretical investigations, we demonstrate competitive performance of our methods on benchmark examples. Our theoretical analysis provides theoretical justifications for the method of \cite{gutmann2018likelihood}.
%We find out in our simulation analysis that KL and CA tend to maintain similar ABC rankings of parameters. However, our estimator KL permits a more profound analysis because of its connection to the log-likelihood ratio.
%\section*{Acknowledgements}
%The authors gratefully acknowledge support from the James S. Kemper Faculty Fund at the Booth School of Business of the University of Chicago and the National Science Foundation (Grant No. NSF DMS-1944740).
\bibliographystyle{apa}
\bibliography{abc-gan}
\clearpage
\appendix
\section{Frequentist's analysis on the exponential kernel}\label{sec:exp_kernel_post}
We thus study the concentration in terms of a KL neighborhood around $Q^{(n)}_{\theta^*}$ defined as
\begin{equation}\label{eq:B_tildeP}
B(\epsilon, Q^{(n)}_{\theta^*}; P_0^{(n)}) = \{Q^{(n)}_{\theta^*} \in \mathcal{Q}^{(n)}: \tilde K(\theta^*,\theta)\leq n\epsilon^2, \tilde V(\theta^*,\theta)\leq n \epsilon^2\},
\end{equation}
where $\tilde K(\theta^*,\theta) \equiv P_0^{(n)}\log \frac{q_{\theta^*}^{(n)}}{q_{\theta}^{(n)}}$ and $\tilde V(\theta^*,\theta)\equiv P_0^{(n)} \abs{\log \frac{q_{\theta^*}^{(n)}}{q_{\theta}^{(n)}} -\tilde K(\theta^*,\theta)}^2$.
The following corollary is directly adopted from {Theorem 5.1} of \citet{kaji2021mh}.
\begin{corollary}
Denote with $\wt Q_\theta^{(n)}$ a measure defined through $\d \wt Q_\theta^{(n)}= ({p_0^{(n)}}/{q_{\theta^*}^{(n)}})\d P_\theta^{(n)}$ and let $d(\cdot, \cdot)$ be a semi-metric on $ \mP^{(n)}$. Suppose that there exists a sequence $\epsilon_n>0$ satisfying $\epsilon_n \to 0$ and $n\epsilon_n^2\to \infty$ such that for every $\epsilon>\epsilon_n$ there exists a test $\phi_n$ (depending on $\epsilon$) such that for every $J \in \N_0$
\begin{equation}
P_0^{(n)}\phi_n \lesssim e^{-n\epsilon^2/4} \quad \text{ and } \sup_{Q_\theta^{(n)}: d(Q_\theta^{(n)}, P_{\theta^*}^{(n)})> J\epsilon} \wt Q_\theta^{(n)}(1-\phi_n)\leq e^{-n J^2\epsilon^2/4}.
\end{equation}
Let $B(\epsilon,Q^{(n)}_{\theta^*}; P_0^{(n)})$ be as in \eqref{eq:B_tildeP} and let $\tilde \Pi_n(\theta)$ be a prior distribution with a density $\tilde \pi(\theta)\propto C_\theta\pi(\theta)$ with $C_\theta$ as in \eqref{eq:tilde_p}. Assume that there exists a constant $L>0$ such that, for all $n$ and $j \in \N$,
\begin{equation}
\frac{\tilde \Pi_n\left( \theta\in \Theta: j\epsilon_n < d(Q_\theta^{(n)}, P_{\theta^*}^{(n)}) \leq (j+1)\epsilon_n \right)}{\tilde \Pi_n \left(B(\epsilon_n, Q^{(n)}_{\theta^*}; P_0^{(n)}) \right)}\leq e^{n\epsilon_n^2 j^2/8}.
\end{equation}
Then for every sufficiently large constant $M$, as $n\to \infty$,
\begin{equation}
P_0^{(n)}\Pi^{EK}\left(Q_\theta^{(n)}: d(Q_\theta^{(n)}, P_{\theta^*}^{(n)}) \geq M \epsilon_n\mid X^{(n)}\right)\to 0.
\end{equation}
\end{corollary}
Next, we want to show the shape of the posterior is actually asymptotically gaussian around $\theta^*$. The following corollary follows from Theorem 2.1 of \citet{kleijn2012bernstein} and {Lemma 8.1} of \citet{kaji2021mh}.
\begin{corollary} (Bernstein von-Mises) Assume that the posterior \eqref{eq:tilde_p} concentrates around $\theta^*$ at the rate $\epsilon_n^*$ and that for every compact $K\in \R^d$
\begin{equation}
\sup_{h\in K} \abs{\log \frac{q_{\theta^*+\epsilon_n^* h}^{(n)}(\X)}{q_{\theta^*}^{(n)}(\X)}- h' \tilde V_{\theta^*}\tilde\delta_{n, \theta^*}-\frac{1}{2}h' \tilde V_{\theta^*} h}\to 0
\end{equation}
for some random vector $\tilde \delta_{n, \theta^*}$ and a non-singular matrix $\tilde V_{\theta^*}$. Then the approximated posterior $\Pi^{EK}(\cdot)$ converges to a sequence of normal distributions in total variation at the rate $\epsilon_n^*$, i.e.
\begin{equation}
\sup_B \abs{\Pi^{EK}\left({\epsilon_n^*}^{-1}(\theta-\theta^*\right)\in B\mid \X)- N_{\tilde \delta_{n, \theta^*},\tilde V_{\theta^*}^{-1}(B)}}\to 0 \quad \text{ in } P_0^{(n)}-probability.
\end{equation}
\end{corollary}
\section{Simulation: Univariate g-and-k Distributions}\label{sec:gk}
Another classical example in the ABC literature is the univariate $g$-and-$k$ distribution. It is defined implicitly by its inverse distribution function
\[
F^{-1}(x)=A+B\Big[1+0.8\frac{1-\e^{-gz_x}}{1+\e^{-gz_x}}\Big](1+z_x^2)^k z_x,
\]
where $z_x$ is the $x$-th quantile of the standard normal distribution, and parameters $A,B,g,k$ are related to location, scale, skewness and kurtosis. The probability density function has no analytical form but can be numerically calculated with high precision since it only involves one-dimensional inversions and differentiations of the quantile function. Therefore, Bayesian inference can be carried out.
We generate $n=500$ observations from the model by letting $A=3, B=1, g=2, k=0.5$. Among the four parameters, $g$ is the hardest to identify, as observed in previous analyses. From a pilot run,
narrow the range of our uniform priors to $A\sim U([2,4]), B\sim U([0,0.25]), g\sim U([0,10])$ and $k\sim U([0,2.5])$.
We estimate the KL divergence by a neural network with two hidden layers with 10 nodes each. The first layer is activated with the rectified linear unit (ReLU) function, while the second layer is equipped with a hyperbolic tangent function (tanh). In addition to observed data $\X$, we also include higher-order terms $\X^2$ and $\X^3$ in the input.\footnote{We are trying to manually create more features for this univariate data. We also explore even higher order terms and find the including up to $\X^3$ works the best in practice.}
We compare the posteriors obtained from different ABC methods in \Cref{fig:gk_post}. We can see that, indeed, the parameter $g$ is the most difficult one to estimate. While the majority of methods return a flat posterior, KLC and kNN with the exponential kernel still place most of the posterior mass around the true value. AL and SA misidentify the location of one or two parameters. While CA and KLC give relatively similar ranking on the ABC sampled parameters, KLC with the exponential kernel can utilize all the samples and re-adjust the shape according to weights. In addition, KLC-exp returns more spiky posteriors than kNN-exp. This is also reflected in the performance summaries in \Cref{tab:gk_comp} which show that KLC-exp has a narrower CI than kNN-exp on average in terms of estimating $A, B$ and $k$. We observe that kNN could be advantageous than KLC in estimating $g$-and-$k$ distributions, since the data is univariate and the use of classifiers may not be necessary.
\begin{figure}[!ht]
\centering
\includegraphics[width=0.95\textwidth]{fig/gk_n500_thetas_001}
\caption{Posterior densities for the $g$-and-$k$ distributions $(A=3, B=1, g=2, k=0.5)$. The black vertical lines mark the true parameter values. The blue curves in kNN and KLC boxes represent the weighted density calculated from the exponential kernel. The ABC posteriors were plotted with the top 1\% of $10^5$ samples.} \label{fig:gk_post}
\end{figure}
\begin{table}[!ht]
\centering
%\renewcommand{\arraystretch}{0.67}
\scalebox{0.75}{
\begin{tabular}{l | *{2}{c} | *{2}{c} | *{2}{c} | *{2}{c} }
\toprule
& \multicolumn{2}{c|}{$A=3$} & \multicolumn{2}{c|}{$B=1$} & \multicolumn{2}{c|}{$g=2$} & \multicolumn{2}{c}{$k=0.5$} \\ \
{Method} &{$(\hat A-A)^2$} & {95\% CI} width & {$(\hat B-B)^2$} & {95\% CI} width & {$(\hat g-g)^2$} &{ 95\% CI} width & {$(\hat k-k)^2$} &{95 \%CI} width \\
\midrule
KLC & 0.0143 & 0.535 & 0.0369 & 1.528 & 12.840 & 8.356 & \bf{0.00341} & 1.044 \\
KLC-exp & 0.0156 & \bf{0.455} & 0.0339 & 1.229 & 12.516 & 8.191 & 0.00800 & 0.784 \\
kNN & \bf{0.0024} & 0.843 & 0.0486 & 1.770 & 2.695 & 7.558 & 0.00421 & 1.086 \\
kNN-exp & 0.0032 & 0.610 & 0.0410 & 1.333 & \bf{1.673} & \bf{4.109} & 0.00480 & 0.805 \\
\vspace{-0.14in} &&&&&&&&\\
AL & 0.0176 & 1.325 & 0.2211 & 1.593 & 8.704 & 9.480 & 0.04451 & 0.835 \\
CA & 0.0125 & 0.503 & 0.0466 & 1.568 & 13.077 & 8.162 & 0.00574 & 1.080 \\
SA & 0.1191 & 0.424 (0) & 0.0598 & 2.379 & 8.800 & 9.478 & 0.56399 & 2.379 \\
WA & 0.0145 & 0.627 & \bf{0.0240} & \bf{1.034} & 11.621 & 8.566 & 0.01060 & \bf{0.776}\\
\bottomrule
\end{tabular}}
\caption{Performance summaries on $g$-and-$k$ distributions over $10$ repetitions with top 1\% ABC samples selected. Most of the 95\% CIs have full coverage with the rest having their coverage marked after the CI width. The bold fonts mark the best model in each metric.}\label{tab:gk_comp}
\end{table}
\section{Discriminator calibration}\label{sec:dhat_training}
We provide more intuition for how the choice of the discriminator, including the $m/n$ ratio and the number of sets of latent variables needed to approximate the KL divergence, affects our results. We also discuss possible modifications we make to the KL estimation to eliminate bias. We use the M/G/1-queuing model as an example.
First, we want to investigate how the choice of the discriminator and the $m/n$ ratio could impact the estimation of the KL divergence. We consider three discriminators: (1) logistic regression on degree-2 polynomial terms of $\X$ (LRD); (2) a neural network with one layer of 10 ReLu nodes and two layers of 10 tanh nodes with input $\X$ (NND1); (3) a neural network with one layer of 10 ReLu nodes and one layer of 10 tanh nodes with input as the degree-2 polynomial terms of $\X$ (NND2). For each discriminator, we do a grid search over $m/n\in\{1,2,3,5\}$. We plot the estimated KL divergence (rescaled) in \Cref{fig:mg1_param1}. From the plot we can see it is quite obvious that LRD best identifies the location of the true values, and the fluctuations/variance in its estimations are the smallest. It also worth noting that the difficulties in estimating different parameters are significantly different here. All discriminators are able to learn the location of $\theta_3$ with little variance, but only LRD is able to learn $\theta_1$ and $\theta_2$ well. The $m/n$ ratio does not seem to have significant impact for LRD, but we observe for NND1, NND2 and some other models that the variance in the estimation tends to get smaller when $m/n>1$ but not overly large to avoid bias. We continue our experiments with LRD and $m/n=3$.
\begin{figure}[!ht]
\begin{center}
\includegraphics[width=0.75\textwidth]{fig/mg1_param1}
\end{center}
\caption{Estimated KL divergence (rescaled) under different discriminators (nlatent=10). We plot the marginal changes in estimated KL divergence with respect to different parameters and different $m/n$ ratios here. The estimated values are rescaled by subtracting the minimum in each scenario so the changes are more comparable. }\label{fig:mg1_param1}
\end{figure}
Next, we want to know how many different sets of latent variables are needed to control the variance in estimation. We focus on the best discriminator we have previously found (i.e. LRD with $m/n=3$). We include the cases where $nlatent\in\{1,5,10\}$ here. The results are shown in \Cref{fig:mg1_param2}. It is expected that with more sets of latent variables to average over, the variance in estimation gets smaller. The main challenge for the M/G/1 model lies in estimating $\theta_1$ correctly and stably. With only one or five sets of latent variable, it seems hopeless to identify $\theta_1$. To trade off variance and computation costs, we set $\text{nlatent}=10$ in our experiments.
\begin{figure}[!ht]
\begin{center}
\includegraphics[width=0.75\textwidth]{fig/mg1_param2}
\end{center}
\caption{Estimated KL divergence (rescaled) using LRD and $m/n=3$. We plot the marginal changes in estimated KL divergence with respect to different parameters and different number of sets of latent variables. The estimated values are rescaled by subtracting the minimum in each scenario so the changes are more comparable.}\label{fig:mg1_param2}
\end{figure}
\clearpage
\section{Proofs}
\subsection{Proof of \Cref{thm:est_error_conc}} \label{pf:est_error_conc}
Denote $K_n=\P_n\log\frac{p_0}{p_\theta}$.
Using Chebyshev's inequality and \Cref{ass:ratio_bd} as
\begin{align*}
P_0^{(n)}[\abs{K_n-K(p_0, p_\theta)}>u]&=P_0^{(n)}\Big(\abs{(\P_n- P_0) \log \frac{p_0}{p_\theta} }>u \Big) \leq \frac{1}{u^2}P_0^{(n)}\Big[\abs{(\P_n- P_0) \log \frac{p_0}{p_\theta}}^2\Big]\\
&=\frac{1}{u^2}P_0^{(n)}\Big[\abs{\P_n \big(\log \frac{p_0}{p_\theta}-P_0\log\frac{p_0}{p_\theta}\big)}^2\Big] = \frac{1}{nu^2}P_0 \Big[\abs{ \log \frac{p_0}{p_\theta}-P_0\log\frac{p_0}{p_\theta}}^2\Big] \\
&\leq \frac{16(2+\Lambda)h(p_0, p_\theta)}{nu^2},
\end{align*}
where the last inequality follows from {Lemma 2.1} (iii) of \citet{kaji2021mh}.
Next, note
$$
\hat K(\X, \tilde \X^\theta)-K_n=- \P_n\Big(\log \frac{1-\hat D^\theta_{n,m}}{1-D_\theta}- \log\frac{\hat D^\theta_{n,m}}{D_\theta}\Big).
$$
{Recall the outer expectation $P^*$ in \Cref{lem:delta_n} and the set of classifiers $D^\theta_{n, \delta}$ defined in \Cref{ass:entropy},} we can bound
\begin{align*}
& P\biggl(\abs{\P_n\Big(\log \frac{1-\hat D^\theta_{n,m}}{1-D_\theta}- \log\frac{\hat D^\theta_{n,m}}{D_\theta}\Big)}>u, d_\theta(\hat D^\theta_{n,m}, D_\theta)\leq C_n\delta_n\biggr) \\
&\qquad\qquad \leq P^*\Big( \sup_{D\in \mD^\theta_{C_n\delta_n}}\abs{\P_n\Big(\log \frac{1-D}{1-D_\theta}- \log\frac{D}{D_\theta}\Big)}>u \Big)\\
&\qquad \qquad \leq \frac{1}{u}\E^* \sup_{D\in \mD^\theta_{C_n\delta_n}} \abs{\P_n\Big(\log \frac{1-D}{1-D_\theta}- \log\frac{D}{D_\theta}\Big)}
\end{align*}
by Markov's inequality. The proof of {Theorem 4.1} of \citet{kaji2021mh} shows that the expectation is $O(C_n\delta_n)$.
Using the triangle inequality and the Bonferroni inequality, since $h(p_0, p_\theta)\leq 2$, we can then write
\begin{align*}
&P\biggl(\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} >2u, d_\theta(\hat D^\theta_{n,m}, D^\theta)\leq C_n\delta_n\biggr) \\
&\quad \leq P\biggl(\abs{\hat K(\X, \tilde \X^\theta)-K_n}+\abs{K_n-K(p_0, p_\theta)}>2u, d_\theta(\hat D^\theta_{n,m}, D_\theta)\leq C_n\delta_n \biggr)\\
&\quad \leq P\biggl(\abs{\hat K(\X, \tilde \X^\theta)-K_n}>u, d_\theta(\hat D^\theta_{n,m}, D_\theta)\leq C_n\delta_n \biggr)
+ P_0^{(n)}[\abs{K_n-K(p_0, p_\theta)}>u]\\
&\quad \leq O\Big(\frac{C_n\delta_u}{u}\Big)+\frac{32(2+\Lambda)}{nu^2}.
\end{align*}
\subsection{Proof of \Cref{thm:post_rate}}\label{pf:post_rate}
Throughout, we continue to assume that $\tilde \X^\theta=g_\theta(\tilde \X)$ and we denote with $P= P_0^{(n)}\otimes \tilde P^{(m)}$ the joint measure for $(\X,\tilde\X)$. Below, we will be using the notation $\Pi(\cdot)$ to denote the generic probability, i.e. for $(\theta,\tilde\X)$ or for the conditional probability $\theta$ given $\tilde \X$.
Later, we will define a high-probability event $\Omega_n(C,\varepsilon_n)$ such that $P_0^{(n)}[\Omega_n(C,\varepsilon_n)^c]=o(1)$ for some $C\in(0,1)$.
Given $\delta_n>0$ from our assumptions, we can write for every $\lambda_n>0$ and $\epsilon_n>0$ and for every arbitrarily slowly increasing sequence $C_n>0$
\begin{multline}\label{eq:conc_cond}
P_0^{(n)}\Pi\left(K(p_0, p_\theta)>\lambda_n\middle\vert \hat K(\X,\tilde\X^\theta) \leq \epsilon_n \right)\leq \Pi_1+o(1)+ \\
P_0^{(n)} \Pi\left(K(p_0, p_\theta)>\lambda_n\middle\vert \hat K(\X,\tilde\X^\theta) \leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n\right)\1[\Omega_n(C,\varepsilon_n)],
\end{multline}
where, using Lemma \ref{lem:delta_n} and the fact that the rate $\delta_n$ is uniform (see Remark \ref{rmk:worst_rate}),
$$
\Pi_1\equiv P_0^{(n)} \Pi(d(\hat D^\theta_{n,m}, D_\theta)> C_n\delta_n)\leq \sup_{\theta\in\Theta}P(d(\hat D^\theta_{n,m}, D_\theta)> C_n\delta_n)=o(1).
$$
Consider the joint event, for some $\delta'>0$,
$$
A_{\epsilon_n}(\delta')=\{(\tilde \X ,\theta):\hat K[\X, g_\theta(\tilde \X)] \leq \epsilon_n\}\cap\{ K(p_0, p_\theta)>\delta'\}.
$$
For every $(\tilde \X, \theta)\in A_{\epsilon_n}(\delta')$ we have
\begin{align*}
K(p_0,p_\theta)\leq\hat K(\X, g_\theta(\tilde \X)) +\abs{\hat K(\X, g_\theta(\tilde \X))-K(p_0, p_\theta)} \leq\epsilon_n+ \abs{\hat K(\X, g_\theta(\tilde \X))-K(p_0, p_\theta)}.
\end{align*}
Hence $(\tilde \X , \theta)\in A_{\epsilon_n}(\delta')$ implies that
\[
\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} > \delta'-\epsilon_n,
\]
and choosing $\delta'\geq \epsilon_n+t_\epsilon$ leads to
\[
\Pi [A_{\epsilon_n}(\delta')]\leq \int_{\Theta} \tilde P^{(m)} \left[ \abs{\hat K(\X,\tilde\X^\theta)-K(p_0, p_\theta)} >t_\epsilon \right] d\Pi(\theta).
\]
Using \eqref{eq:conc_cond}, we now focus on the conditional probability, given $d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n$,
\begin{multline}\label{eq:conc_frac}
\Pi\left(K(p_0, p_\theta)>\epsilon_n+t_\epsilon \middle\vert \hat K(\X,\tilde\X^\theta) \leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right) \\
\leq \frac{\int_\Theta \tilde P^{(m)} \left[\abs{\hat K(\X,\tilde\X^\theta)-K(p_0, p_\theta)} >t_\epsilon\mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right] d\Pi(\theta)}{\int_\Theta \tilde P^{(m)} \left[\hat K(\X,\tilde\X^\theta) \leq \epsilon_n \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right]d\Pi(\theta)}.
\end{multline}
We now find a lower bound for the denominator.
Recall the KL neighborhood $B_2(p_0,\epsilon_n)$ defined in \eqref{eq:KL_ball}. Since
\[
\hat K(\X, \tilde \X^\theta) \leq K(p_0,p_\theta)+\abs{\hat K(\X, \tilde \X^\theta)- K(p_0, p_\theta)}\leq \epsilon_n/2+ K(p_0, p_\theta),
\]
provided that $\abs{\hat K(\X, \tilde \X^\theta) -K(p_0, p_\theta)} \leq \epsilon_n/2$. The denominator can be then bounded by
\begin{align*}
& \int_\Theta \tilde P^{(m)} [\hat K(\X, \tilde \X^\theta) \leq \epsilon_n \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n ]d\Pi(\theta)\\
&\qquad \geq \int_{B_2(p_0, \epsilon_n/{2})} \tilde P^{(m)} \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} \leq \epsilon_n/2 \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right] d\Pi(\theta) \\
& \qquad \geq \Pi[{B_2(p_0, \epsilon_n/{2})} ] -\int_{B_2(p_0, \epsilon_n/{2})} \tilde P^{(m)} \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} > \epsilon_n/2 \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right] d\Pi(\theta).
\end{align*}
Denoting
$$
Z(\X)\equiv \int_{B_2(p_0, \epsilon_n/{2})} \tilde P^{(m)} \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} > \epsilon_n/2 \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right] d\Pi(\theta)
$$
we can write, for every $C>0$, using Fubini's theorem and Markov's inequality
\begin{align*}
P_0^{(n)}\left(Z(\X)> C\right)&\leq \frac{1}{C} \int_{B_2(p_0, \epsilon_n/{2})} P \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} > \epsilon_n/2 \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right]d\Pi(\theta)\\
&=\frac{\Pi(B_2(p_0, \epsilon_n/{2}))}{C}\frac{\sup\limits_{\theta\in\Theta} P \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0, p_\theta)} > \epsilon_n/2, d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right]}{\sup\limits_{\theta\in\Theta} P \left[ d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right]}\\
&=\frac{\Pi(B_2(p_0, \epsilon_n/{2}))}{C(1+o(1))}\sup\limits_{\theta\in\Theta}\rho_{n,\theta}(\varepsilon_n/2;C_n;\delta_n),
%&P_0^{(n)}\left[\left(\int_\Theta \tilde P^{(m)}\left[\hat K[\X, g_\theta(\tilde \X)] > u, d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right]d\Pi(\theta)>C-1-o(1)\right]\\
%&\leq P_0^{(n)}\left(\int_{\Theta} P\left[\hat K[\X, g_\theta(\tilde \X)] > u, d(\hat D^\theta_{n,m}, D_\theta) < C_n\delta_n \right] d\Pi(\theta) > 1+o(1)-C_u \right) \\
%&= \frac{O(C_n\delta_n/u)}{ 1+o(1)-C_u}=o(1)
\end{align*}
where we have used Theorem \ref{thm:est_error_conc} and the fact that $P(d(\hat D^\theta_{n,m}, D_\theta)\leq C_n\delta_n)=1+o(1)$ for every $\theta\in\Theta$ from Lemma \ref{lem:delta_n}.
We now define an event, for some $0<C<1$ and $\epsilon_n>0$,
$$
\Omega_n(C,\epsilon_n)=\left\{\X: Z(\X) \leq C \times \Pi(B_2(p_0, \epsilon_n/{2}))\right\}.
$$
Using Theorem \ref{thm:est_error_conc}, we have
$$
\rho_{n,\theta}(\epsilon_n/2;C_n;\delta_n)=O\Big(\frac{C_n\delta_n}{\epsilon_n} +\frac{1}{n\epsilon_n^2}\Big)\quad\text{ for every}\quad \theta\in\Theta.
$$
Choosing $\epsilon_n>0$ such that $\epsilon_n=o(1)$ and $n\epsilon_n^2\rightarrow\infty$ and $C_n\delta_n=o(\epsilon_n)$ we have $P_0^{(n)}[\Omega_n(C,\epsilon_n)^c]=o(1)$ for every $C\in(0,1)$.
On the event $\Omega_n(C,\epsilon_n)$, for some $0<C<1$, we can lower-bound the denominator with
$$
\int_\Theta \tilde P^{(m)} [\hat K(\X, \tilde \X^\theta) \leq \epsilon_n \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n ]d\Pi(\theta)> (1-C)\times \Pi(B_2(p_0, \epsilon_n/{2}).
$$
Using this bound and applying Fubini's theorem, we can further write
\begin{multline}\label{eq:frac_exp}
P_0^{(n)} \Pi\left(K(p_0, p_\theta)>\epsilon_n+t_\epsilon \mid \hat K(\X,\tilde\X^\theta) \leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right)\1[\Omega_n(C,\varepsilon_n)] \\
\leq \frac{\int_\Theta P\left[\abs{\hat K(\X,\tilde\X^\theta)-K(p_0, p_\theta)} >t_\epsilon \mid d(\hat D^\theta_{n,m}, D_\theta) \leq C_n\delta_n \right] d\Pi(\theta)}{(1-C)\times \Pi(B_2(p_0, \epsilon_n/{2})}
\end{multline}
Using Theorem \ref{thm:est_error_conc} again, we obtain an upper bound for the display above with
$$
\frac{ \rho_n(t_\epsilon;C_n;\delta_n)}{(1-C)\times \Pi(B_2(p_0, \epsilon_n/{2})(1+o(1))}.
$$
Using the prior Assumption \ref{ass:prior_mass}, we can choose $t_\epsilon$ such that
$$
\Big(\frac{C_n\delta_n}{t_\epsilon} +\frac{1}{nt_\epsilon^2}\Big)/\epsilon_n^{\kappa}=1/M_n
$$
for some arbitrarily slowly increasing sequence $M_n>0$.
We choose $t_\epsilon=M_n C_n\delta_n/\epsilon_n^{\kappa}+\sqrt M_n n^{-1/2}/\epsilon_n^{\kappa/2}$. Since $\delta_n\gtrsim n^{-1/2}$ and $\epsilon_n^{-\kappa}\geq \epsilon_n^{-\kappa/2}$, the overall rate is then driven by
$\epsilon_n+t_\epsilon=\epsilon_n+\wt M_n\delta_n \epsilon_n^{-\kappa}$, where $\wt M_n=M_nC_n$.
\subsection{Proof of \Cref{thm:unif_ci}}\label{sec:proof_unif_ci}
Recall from \Cref{thm:post_rate} that with the accept-reject strategy, the true KL divergence $K(p_0, p_\theta)$ is contracting at the rate $\lambda_n=\epsilon_n+ \wt M_n \delta_n \epsilon_n^{-\kappa}$, where $\wt M_n$ is a slowly increasing sequence that diverges faster than $C_n$.
Consider the case when $\epsilon_n \gg \wt M_n \delta_n \epsilon_n^{-\kappa}$ or, equivalently, $\epsilon_n \gg \delta_n^{1/(\kappa+1)}$. Denote
\[
x(\theta)=\epsilon_n^{-1}K(p_0,p_\theta)\quad\text{ and}\quad f_n(\theta-\theta_0)=f\big(\epsilon_n^{-1/2}(\theta-\theta_0)\big).
\]
We express the ABC posterior expectation of $f_n (\theta-\theta_0)$ for a non-negative and bounded function $f_n(\cdot)$ by
\begin{align*}
& P_0^{(n)}E_{\hat \Pi^{AR}_{\epsilon_n}} \Big[f_n(\theta-\theta_0)\Big] =P_0^{(n)} \int f_n(\theta-\theta_0) \d \hat \Pi^{AR}_{\epsilon_n}(\theta\mid \X)\\
&\quad =\underbrace{P_0^{(n)} \int f_n(\theta-\theta_0) \1[K(p_0,p_\theta)\leq \lambda_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n)] \d \hat \Pi^{AR}_{\epsilon_n}(\theta\mid \X)}_{(\text{I})}\\
&\qquad + \underbrace{P_0^{(n)} \int f_n(\theta-\theta_0) \1[K(p_0,p_\theta)\leq \lambda_n, d(\hat D^\theta_{n,m}, D_\theta )> C_n\delta_n)] \d \hat \Pi^{AR}_{\epsilon_n}(\theta\mid \X)}_{(\text{II})} \\
& \qquad +
\underbrace{P_0^{(n)} \int f_n(\theta-\theta_0) \1[K(p_0,p_\theta)> \lambda_n] \d \hat \Pi^{AR}_{\epsilon_n}(\theta\mid \X)}_{(\text{III})}
\end{align*}
where the term $(\text{III})$ can be controlled using Fubini's theorem and the concentration result in \Cref{thm:post_rate} as follows
\begin{align*}
(\text{III})& = \int f_n(\theta-\theta_0) P_0^{(n)} \1[K(p_0,p_\theta)> \lambda_n] \d\hat \Pi^{AR}_{\epsilon_n}(\theta\mid \X)\\
& \leq \norm{f}_\infty P_0^{(n)}\Pi\Big(K(p_0, p_\theta)>\lambda_n\mid \hat K(\X, \tilde \X^\theta)\leq \epsilon_n\Big)=o(1).
\end{align*}
The second term $(\text{II})$ can be bounded similarly as
\begin{align*}
(\text{II})&= P_0^{(n)} \int_{K(p_0, p_\theta)\leq \lambda_n } f_n(\theta-\theta_0) \1 [d(\hat D^\theta_{n,m}, D_\theta )> C_n\delta_n)]\d \hat \Pi^{AR}_{\epsilon_n}(\theta\mid \X) \\
& \leq \norm{f}_\infty \int_{K(p_0, p_\theta)\leq \lambda_n}P(d(\hat D^\theta_{n,m}, D_\theta )> C_n\delta_n)\d\Pi(\theta) \\
&\leq \norm{f}_\infty \sup_{\theta \in \Theta} P(d(\hat D^\theta_{n,m}, D_\theta )> C_n\delta_n) = o(1)
\end{align*}
where we use the fact that $\sup_{\theta\in \Theta} P(d(\hat D^\theta_{n,m}, D_\theta )> C_n\delta_n) = o(1)$ from \Cref{lem:delta_n}.
Thus, the asymptotic behavior is mainly determined by the term $(\text{I})$. Combined with the continuity of $\pi(\theta)$ at $\theta_0$ we can re-write $(\text{I})$ as
\begin{align*}
(\text{I})&= P_0^{(n)} \frac{\int_{K(p_0, p_\theta)\leq \lambda_n} \pi(\theta) f_n(\theta-\theta_0)\tilde P^{(m)}\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n , d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta}{\int_{K(p_0, p_\theta)\leq \lambda_n}\pi(\theta) \tilde P^{(m)}\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n,d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta}\\
&= P_0^{(n)} \frac{\int_{K(p_0, p_\theta)\leq \lambda_n} f_n(\theta-\theta_0)\tilde P^{(m)}\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n , d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta}{\int_{K(p_0, p_\theta)\leq \lambda_n}\tilde P^{(m)}\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n,d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta} (1+o(1))
\end{align*}
We have $x(\theta)\geq 0$ for all $\theta\in \Theta$ and since
$$
\hat K(\X, \tilde \X^\theta)=\hat K(\X, \tilde \X^\theta)-K(p_0,p_\theta)+\epsilon_n x(\theta),
$$
we can write
\begin{multline}\label{eq:N_khat}
\tilde P^{(m)} \big[\hat K(\X, \tilde \X^\theta )\leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\\
\geq \tilde P^{(m)} \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0,p_\theta)}\leq \epsilon_n(1-x(\theta)), d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \right]\\
= 1- \tilde P^{(m)} \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0,p_\theta)}> \epsilon_n(1-x(\theta)), d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \right].
\end{multline}
Denoting with
\[
\tilde Z(\X )=\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \tilde P^{(m)} \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0,p_\theta)}>\epsilon_n\big(1-x(\theta)\big), d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \right] \d \theta.
\]
Using Markov's inequality and Fubini's theorem we have, for every $\tilde C>0$,
\begin{align*}
&P_0^{(n)}(\tilde Z(\X)>\tilde C)\\
&\quad \leq \frac{1}{\tilde C}\int\limits_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} P \left[\abs{\hat K(\X, \tilde \X^\theta)-K(p_0,p_\theta)}>\epsilon_n\big(1-x(\theta)\big), d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \right] \d\theta \\
&\quad \leq \frac{1}{\tilde C} \int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \rho_n\big(\epsilon_n(1-x(\theta));C_n;\delta_n\big) \d\theta \\
&\quad \leq \frac{1}{\tilde C} \times\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta\times \sup_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n} } \rho_n\big(\epsilon_n(1-x(\theta));C_n;\delta_n\big) \\
&\quad \leq \frac{\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta}{\tilde C} \times \rho_n(M_nC_n\delta_n; C_n; \delta_n) \\
&\quad= \frac{\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta}{\tilde C}\times O\left(\frac{1}{M_n}+\frac{1}{n M_n^2C_n^2\delta_n^2}\right). %=\frac{\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta}{\tilde C}\times o(1).
\end{align*}
where $\rho_n(\cdot; C_n; \delta_n)$ is defined in \Cref{thm:est_error_conc}.
Thus, we can define a set $\tilde\Omega_{n}(\tilde C_n)$, for some
for some arbitrarily slowly increasing sequence $\tilde C_n>0$, and $\tilde C_n =O(1/M_n)$, as
\[
\tilde\Omega_{n}(\tilde C_n)=\left\{\X: \tilde Z(\X)\leq \tilde C_n\times \int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta \right\}.
\]
Then we have that $P_0^{(n)}(\tilde \Omega_n(\tilde C_n)^c)=o(1)$. Recall the inequality in \eqref{eq:N_khat}, on $\tilde \Omega_n(\tilde C_n)$, we have that
\begin{multline*}
\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \tilde P^{(m)} \big[\hat K(\X, \tilde \X^\theta )\leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big] \d\theta \\
\geq \int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta - \tilde Z(\X)= \int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta \times (1-o(1))
\end{multline*}
And this quantity is upper-bounded by $\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta$. Therefore, we can conclude that
\[
\int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \tilde P^{(m)} \big[\hat K(\X, \tilde \X^\theta )\leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big] \d\theta = \int_{x(\theta)\leq 1- \frac{M_nC_n\delta_n}{\epsilon_n}} \d\theta \big(1+o(1)\big).
\]
Note that $x(\theta)\leq 1- M_nC_n\delta_n/\epsilon_n$ implies that
$K(p_0, p_\theta)\leq \epsilon_n -M_nC_n\delta_n$. On this set $\tilde \Omega_n(\tilde C_n)$, we can further lower bound the denominator as
\begin{align*}
&\int_{K(p_0, p_\theta)\leq \lambda_n} \tilde P^{(m)}\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta \\
&\qquad = \underbrace{\int_{K(p_0, p_\theta)\leq \epsilon_n-M_nC_n\delta_n} \d \theta}_{D_1}(1+o(1))\\
&\qquad \quad +\underbrace{\int_{\epsilon_n-M_nC_n\delta_n<K(p_0, p_\theta)\leq \lambda_n}\tilde P^{(m)}\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta}_{D_2}.
\end{align*}
Next, we show that the second term $D_2$ is $o(D_1)$. Let $u=\epsilon_n^{-1/2}(\theta-\theta_0)$. Under Assumption \ref{ass:KL_taylor}, we have
$K(p_0,p_\theta)=\frac{1}{2} (\theta-\theta_0)'I(\theta_0) (\theta-\theta_0)\{1+o(1)\}$, which is $x(\theta_0+\sqrt \epsilon_n u)=\frac{1}{2}u'I(\theta_0)u$. Since $I(\theta_0)$ is positive definite we can write
\begin{align}\label{eq:D1D2}
\frac{D_2}{D_1}& \leq \frac{\int_{1-M_nC_n\delta_n/\epsilon_n<x(\theta)\leq 1+\wt M_n\delta_n/\epsilon_n^{\kappa+1} } \d \theta}{\int_{x(\theta)\leq 1-M_nC_n\delta_n/\epsilon_n} \d \theta}\\
&\leq \frac{\int_{2(1-M_nC_n\delta_n/\epsilon_n)\leq u'I(\theta_0)u
\leq 2(1+\wt M_n\delta_n/\epsilon_n^{\kappa+1})}\d\theta}{\int_{u'I(\theta_0)u \leq 2(1-M_nC_n\delta_n/\epsilon_n)}\d\theta}
\lesssim \frac{M_nC_n\delta_n}{\epsilon_n}+\frac{\wt M_n\delta_n}{\epsilon_n^{\kappa+1}} = o(1).
\end{align}
{where the approximation follows from the fact that $\epsilon_n \gg \delta_n^{1/(\kappa+1)}$ and because the denominator is approximating the integral $\int_{u'I(\theta_0)u \leq 2}\d\theta$ and the numerator is the length of shrinking intervals.}
Combining the above results, we find that, on the set $\tilde \Omega_n(\tilde C_n)$, the denominator can be lower-bounded by
$
\int_{x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}} \d \theta \{1+o(1)\}.
$
This implies
\begin{align}
(\text{I})&= \frac{\int_{K(p_0, p_\theta)\leq \lambda_n} f_n(\theta-\theta_0) P\big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n , d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n \big]\d\theta}{(1+o(1))\int_{x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}} \d \theta} (1+o(1))+ o(1)\nonumber\\
&=(N_1+N_2)\{1+o(1)\}+o(1).
\end{align}
with
\begin{equation}
N_1\equiv{\frac{\int\limits_{x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}} f\Big(\epsilon_n^{-1/2}(\theta-\theta_0)\Big) \d \theta}{\int\limits_{x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}} \d \theta} }
\end{equation}
and
\begin{equation}
N_2\equiv\frac{\int\limits_{K(p_0, p_\theta)\leq \lambda_n} \1\left[{x(\theta)> 1-\frac{M_nC_n\delta_n}{\epsilon_n}}\right] f\Big(\epsilon_n^{-1/2}(\theta-\theta_0)\Big) P \Big[\hat K(\X, \tilde \X^\theta) \leq \epsilon_n, d(\hat D^\theta_{n,m}, D_\theta )\leq C_n\delta_n\Big] \d \theta}{\int\limits_{x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}} \d \theta}\label{eq:N2},
\end{equation}
where the second equality follows from the fact that $x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}$ leads to $K(p_0, p_\theta)\leq \epsilon_n -M_nC_n\delta_n$ and then $K(p_0, p_\theta)\leq \lambda_n$ is trivially satisfied and where the last $o(1)$ comes from the set $\tilde \Omega_n(\tilde C_n)^c$.
Since we have $\epsilon_n\gg \delta_n$, with $u=\epsilon_n^{-1/2}(\theta-\theta_0)$, the first term is approximately equal to
\[
N_1=\frac{\int_{K(p_0,p_{\theta_0+\sqrt{\epsilon_n} u})\leq \epsilon_n} f(u) \d u}{ \int_{K(p_0,p_{\theta_0+\sqrt{\epsilon_n} u})\leq \epsilon_n} \d u}\left(1+o(1)\right).
\]
Using Assumption \ref{ass:KL_taylor} again, we have
\[
\int_{K(p_0, p_{\theta_0+\sqrt{\epsilon_n} u})\leq \epsilon_n} \d u = \int_{\frac{1}{2}\sqrt{\epsilon_n} u' I(\theta)\sqrt{\epsilon_n} u \leq \epsilon_n} \d u+o(1)=\int_{u'I(\theta_0)u\leq 2}\d u+o(1).
\]
This leads to
\[
N_1=\frac{\int_{u'I(\theta_0)u \leq 2}f(u) \d u}{\int_{u'I(\theta_0)u\leq 2} \d u } \left(1+o(1)\right).
\]
Next, we show that the fraction $N_2$ converges to 0. The numerator can be simplified as an integral over $1-\frac{M_nC_n\delta_n}{\epsilon_n}<x(\theta)\leq 1+\frac{\wt M_n\delta_n}{\epsilon_n^{\kappa+1}}$, which can be bounded by \eqref{eq:D1D2} as
\begin{align*}\label{eq:N2}
N_2&\leq \frac{\norm{f}_\infty \int_{1-\frac{M_nC_n\delta_n}{\epsilon_n}<x(\theta)\leq 1+\frac{\wt M_n\delta_n}{\epsilon_n^{\kappa+1}}} \d \theta }{
\int_{x(\theta)\leq 1-\frac{M_nC_n\delta_n}{\epsilon_n}} \d \theta}\\
& \leq \norm{f}_\infty \frac{D_2}{D_1}=o(1).
\end{align*}
Since we have $\epsilon_n^{\kappa+1}\gg \delta_n$, putting all the terms together, we obtain that the $P_0^{(n)}$-averaged ABC posterior distribution of $\epsilon_n^{-1/2}(\theta-\theta_0)$ is asymptotically uniform over the ellipsoid $\{u: u'I(\theta_0)u\leq 2\}$.
\end{document}
\begin{thebibliography}{}
\bibitem[\protect\astroncite{{\AA}kesson
et~al.}{2020}]{aakesson2020convolutional}
{\AA}kesson, M., Singh, P., Wrede, F., and Hellander, A. (2020).
\newblock Convolutional neural networks as summary statistics for approximate
{B}ayesian computation.
\newblock {\em arXiv}.
\bibitem[\protect\astroncite{Arjovsky et~al.}{2017}]{arjovsky2017wasserstein}
Arjovsky, M., Chintala, S., and Bottou, L. (2017).
\newblock Wasserstein generative adversarial networks.
\newblock In {\em International conference on machine learning}, pages
214--223. PMLR.
\bibitem[\protect\astroncite{Bauer and Kohler}{2019}]{bauer2019deep}
Bauer, B. and Kohler, M. (2019).
\newblock On deep learning as a remedy for the curse of dimensionality in
nonparametric regression.
\newblock {\em The Annals of Statistics}, 47(4):2261--2285.
\bibitem[\protect\astroncite{Beaumont et~al.}{2002}]{beaumont2002approximate}
Beaumont, M.~A., Zhang, W., and Balding, D.~J. (2002).
\newblock Approximate {B}ayesian computation in population genetics.
\newblock {\em Genetics}, 162(4):2025--2035.
\bibitem[\protect\astroncite{Bernton et~al.}{2019}]{bernton2019approximate}
Bernton, E., Jacob, P.~E., Gerber, M., and Robert, C.~P. (2019).
\newblock Approximate {B}ayesian computation with the {W}asserstein distance.
\newblock {\em Journal of the Royal Statistical Society: Series B (Statistical
Methodology)}, 81(2):235--269.
\bibitem[\protect\astroncite{Bishop}{2006}]{bishop2006pattern}
Bishop, C.~M. (2006).
\newblock {\em Pattern recognition and Machine Learning}.
\newblock Springer-Verlag, Berlin, Heidelberg.
\bibitem[\protect\astroncite{Bissiri et~al.}{2016}]{bissiri2016general}
Bissiri, P.~G., Holmes, C.~C., and Walker, S.~G. (2016).
\newblock A general framework for updating belief distributions.
\newblock {\em Journal of the Royal Statistical Society. Series B, Statistical
methodology}, 78(5):1103.
\bibitem[\protect\astroncite{Blum et~al.}{2013}]{blum2013comparative}
Blum, M.~G., Nunes, M.~A., Prangle, D., and Sisson, S.~A. (2013).
\newblock A comparative review of dimension reduction methods in approximate
{B}ayesian computation.
\newblock {\em Statistical Science}, 28(2):189--208.
\bibitem[\protect\astroncite{Burkard et~al.}{2012}]{burkard2012assignment}
Burkard, R., Dell'Amico, M., and Martello, S. (2012).
\newblock {\em Assignment problems: revised reprint}.
\newblock SIAM.
\bibitem[\protect\astroncite{Cover and Thomas}{1991}]{cover1991entropy}
Cover, T.~M. and Thomas, J.~A. (1991).
\newblock Entropy, relative entropy and mutual information.
\newblock {\em Elements of information theory}, 2(1):12--13.
\bibitem[\protect\astroncite{Den~Hollander}{2008}]{den2008large}
Den~Hollander, F. (2008).
\newblock {\em Large deviations}, volume~14.
\newblock American Mathematical Soc.
\bibitem[\protect\astroncite{Drovandi et~al.}{2011}]{drovandi2011approximate}
Drovandi, C.~C., Pettitt, A.~N., and Faddy, M.~J. (2011).
\newblock Approximate {B}ayesian computation using indirect inference.
\newblock {\em Journal of the Royal Statistical Society: Series C (Applied
Statistics)}, 60(3):317--337.
\bibitem[\protect\astroncite{Fearnhead and
Prangle}{2012}]{fearnhead2011constructing}
Fearnhead, P. and Prangle, D. (2012).
\newblock Constructing summary statistics for approximate bayesian computation:
Semi-automatic approximate bayesian computation.
\newblock {\em Journal of the Royal Statistical Society: Series B (Statistical
Methodology)}, 74(3):419--474.
\bibitem[\protect\astroncite{Frazier et~al.}{2018}]{frazier2018asymptotic}
Frazier, D.~T., Martin, G.~M., Robert, C.~P., and Rousseau, J. (2018).
\newblock Asymptotic properties of approximate bayesian computation.
\newblock {\em Biometrika}, 105(3):593--607.
\bibitem[\protect\astroncite{Ghimire et~al.}{2020}]{ghimire2020reliable}
Ghimire, S., Gyawali, P.~K., and Wang, L. (2020).
\newblock Reliable estimation of kullback-leibler divergence by controlling
discriminator complexity in the reproducing kernel hilbert space.
\newblock {\em arXiv}.
\bibitem[\protect\astroncite{Ghosal and Van~der
Vaart}{2017}]{ghosal2017fundamentals}
Ghosal, S. and Van~der Vaart, A. (2017).
\newblock {\em Fundamentals of nonparametric Bayesian inference}, volume~44.
\newblock Cambridge University Press.
\bibitem[\protect\astroncite{Gillespie}{1977}]{gillespie1977exact}
Gillespie, D.~T. (1977).
\newblock Exact stochastic simulation of coupled chemical reactions.
\newblock {\em The journal of physical chemistry}, 81(25):2340--2361.
\bibitem[\protect\astroncite{Goodfellow
et~al.}{2014}]{goodfellow2014generative}
Goodfellow, I.~J., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D.,
Ozair, S., Courville, A.~C., and Bengio, Y. (2014).
\newblock Generative adversarial nets.
\newblock In {\em Advances in Neural Information Processing Systems}, pages
2672--2680.
\bibitem[\protect\astroncite{Gutmann et~al.}{2018}]{gutmann2018likelihood}
Gutmann, M.~U., Dutta, R., Kaski, S., and Corander, J. (2018).
\newblock Likelihood-free inference via classification.
\newblock {\em Statistics and Computing}, 28(2):411--425.
\bibitem[\protect\astroncite{Jiang et~al.}{2018}]{jiang2018approximate}
Jiang, B., Wu, T.-Y., and Wong, W.~H. (2018).
\newblock Approximate {B}ayesian computation with {Kullback-Leibler} divergence
as data discrepancy.
\newblock In {\em International Conference on Artificial Intelligence and
Statistics}, pages 1711--1721. PMLR.
\bibitem[\protect\astroncite{Jiang et~al.}{2017}]{jiang2017learning}
Jiang, B., Wu, T.-y., Zheng, C., and Wong, W.~H. (2017).
\newblock Learning summary statistic for approximate {B}ayesian computation via
deep neural network.
\newblock {\em Statistica Sinica}, pages 1595--1618.
\bibitem[\protect\astroncite{Jordan et~al.}{1999}]{jordan1999introduction}
Jordan, M.~I., Ghahramani, Z., Jaakkola, T.~S., and Saul, L.~K. (1999).
\newblock An introduction to variational methods for graphical models.
\newblock {\em Machine learning}, 37(2):183--233.
\bibitem[\protect\astroncite{Joyce and Marjoram}{2008}]{joyce2008approximately}
Joyce, P. and Marjoram, P. (2008).
\newblock Approximately sufficient statistics and {B}ayesian computation.
\newblock {\em Statistical applications in genetics and molecular biology},
7(1).
\bibitem[\protect\astroncite{Kaji et~al.}{2020}]{kaji2020adversarial}
Kaji, T., Manresa, E., and Pouliot, G. (2020).
\newblock An adversarial approach to structural estimation.
\newblock {\em arXiv}.
\bibitem[\protect\astroncite{Kaji and Rockova}{2021}]{kaji2021mh}
Kaji, T. and Rockova, V. (2021).
\newblock Metropolis-hastings via classification.
\newblock {\em arXiv}.
\bibitem[\protect\astroncite{Kleijn and van~der
Vaart}{2006}]{kleijn2006misspecification}
Kleijn, B.~J. and van~der Vaart, A.~W. (2006).
\newblock Misspecification in infinite-dimensional bayesian statistics.
\newblock {\em The Annals of Statistics}, 34(2):837--877.
\bibitem[\protect\astroncite{Kleijn and van~der
Vaart}{2012}]{kleijn2012bernstein}
Kleijn, B.~J. and van~der Vaart, A.~W. (2012).
\newblock The bernstein-von-mises theorem under misspecification.
\newblock {\em Electronic Journal of Statistics}, 6:354--381.
\bibitem[\protect\astroncite{Kullback}{1958}]{kullback1958information}
Kullback, S. (1958).
\newblock {\em Information theory and statistics}.
\newblock Wiley, New York.
\bibitem[\protect\astroncite{Magdon-Ismail and Atiya}{2003}]{magdon2003maximum}
Magdon-Ismail, M. and Atiya, A.~F. (2003).
\newblock A maximum likelihood approach to volatility estimation for a
{B}rownian motion using high, low and close price data.
\newblock {\em Quantitative Finance}, 3(5):376.
\bibitem[\protect\astroncite{M{\"u}ller}{1997}]{muller1997integral}
M{\"u}ller, A. (1997).
\newblock Integral probability metrics and their generating classes of
functions.
\newblock {\em Advances in Applied Probability}, pages 429--443.
\bibitem[\protect\astroncite{Nguyen et~al.}{2007}]{nguyen2007estimating}
Nguyen, X., Wainwright, M.~J., and Jordan, M.~I. (2007).
\newblock Estimating divergence functionals and the likelihood ratio by
penalized convex risk minimization.
\newblock In {\em Proceedings of the 20th International Conference on Neural
Information Processing Systems}, pages 1089--1096.
\bibitem[\protect\astroncite{Nunes and Balding}{2010}]{nunes2010optimal}
Nunes, M.~A. and Balding, D.~J. (2010).
\newblock On optimal selection of summary statistics for approximate {B}ayesian
computation.
\newblock {\em Statistical applications in genetics and molecular biology},
9(1).
\bibitem[\protect\astroncite{Papamakarios and
Murray}{2016}]{papamakarios2016fast}
Papamakarios, G. and Murray, I. (2016).
\newblock Fast $\varepsilon$-free inference of simulation models with bayesian
conditional density estimation.
\newblock In {\em Proceedings of the 30th International Conference on Neural
Information Processing Systems}, pages 1036--1044.
\bibitem[\protect\astroncite{Park et~al.}{2016}]{park2016k2}
Park, M., Jitkrittum, W., and Sejdinovic, D. (2016).
\newblock {K2-ABC}: {A}pproximate {B}ayesian computation with kernel
embeddings.
\newblock In {\em Artificial Intelligence and Statistics}, pages 398--407.
PMLR.
\bibitem[\protect\astroncite{P{\'e}rez-Cruz}{2008}]{perez2008kullback}
P{\'e}rez-Cruz, F. (2008).
\newblock Kullback-{L}eibler divergence estimation of continuous distributions.
\newblock In {\em 2008 IEEE international symposium on information theory},
pages 1666--1670. IEEE.
\bibitem[\protect\astroncite{Polson and Rockova}{2018}]{polson2018posterior}
Polson, N.~G. and Rockova, V. (2018).
\newblock Posterior concentration for sparse deep learning.
\newblock In {\em Proceedings of the 32nd International Conference on Neural
Information Processing Systems}, pages 938--949.
\bibitem[\protect\astroncite{Pritchard et~al.}{1999}]{pritchard1999population}
Pritchard, J.~K., Seielstad, M.~T., Perez-Lezaun, A., and Feldman, M.~W.
(1999).
\newblock Population growth of human {Y} chromosomes: a study of {Y} chromosome
microsatellites.
\newblock {\em Molecular biology and evolution}, 16(12):1791--1798.
\bibitem[\protect\astroncite{Rogers and Zhou}{2008}]{rogers2008estimating}
Rogers, L.~C. and Zhou, F. (2008).
\newblock Estimating correlation from high, low, opening and closing prices.
\newblock {\em The Annals of Applied Probability}, 18(2):813--823.
\bibitem[\protect\astroncite{Schmidt-Hieber}{2020}]{schmidt2020nonparametric}
Schmidt-Hieber, J. (2020).
\newblock Nonparametric regression using deep neural networks with relu
activation function.
\newblock {\em The Annals of Statistics}, 48(4):1875--1897.
\bibitem[\protect\astroncite{Schmon et~al.}{2020}]{schmon2020generalized}
Schmon, S.~M., Cannon, P.~W., and Knoblauch, J. (2020).
\newblock Generalized posteriors in approximate {B}ayesian computation.
\newblock In {\em Third Symposium on Advances in Approximate Bayesian
Inference}.
\bibitem[\protect\astroncite{Schwartz}{1965}]{schwartz1965bayes}
Schwartz, L. (1965).
\newblock On {B}ayes procedures.
\newblock {\em Zeitschrift f{\"u}r Wahrscheinlichkeitstheorie und verwandte
Gebiete}, 4(1):10--26.
\bibitem[\protect\astroncite{Silva and Narayanan}{2007}]{silva2007universal}
Silva, J. and Narayanan, S. (2007).
\newblock Universal consistency of data-driven partitions for divergence
estimation.
\newblock In {\em 2007 IEEE International Symposium on Information Theory},
pages 2021--2025. IEEE.
\bibitem[\protect\astroncite{Silva and Narayanan}{2010}]{silva2010information}
Silva, J. and Narayanan, S.~S. (2010).
\newblock Information divergence estimation based on data-dependent partitions.
\newblock {\em Journal of Statistical Planning and Inference},
140(11):3180--3198.
\bibitem[\protect\astroncite{Sisson et~al.}{2018}]{sisson2018overview}
Sisson, S.~A., Fan, Y., and Beaumont, M.~A. (2018).
\newblock Overview of abc.
\newblock In {\em Handbook of approximate Bayesian computation}, pages 3--54.
Chapman and Hall/CRC.
\bibitem[\protect\astroncite{S{\o}rensen}{2004}]{sorensen2004parametric}
S{\o}rensen, H. (2004).
\newblock Parametric inference for diffusion processes observed at discrete
points in time: a survey.
\newblock {\em International Statistical Review}, 72(3):337--354.
\bibitem[\protect\astroncite{van~de Geer}{2000}]{geer2000empirical}
van~de Geer, S.~A. (2000).
\newblock {\em Empirical Processes in M-estimation}, volume~6.
\newblock Cambridge university press.
\bibitem[\protect\astroncite{van~der Vaart and Wellner}{1996}]{van1996weak}
van~der Vaart, A.~W. and Wellner, J.~A. (1996).
\newblock {\em Weak convergence and empirical processes: with applications to
statistics}.
\newblock Springer.
\bibitem[\protect\astroncite{Villani}{2009}]{villani2009optimal}
Villani, C. (2009).
\newblock {\em Optimal transport: old and new}, volume 338.
\newblock Springer.
\bibitem[\protect\astroncite{Wainwright and
Jordan}{2008}]{wainwright2008graphical}
Wainwright, M.~J. and Jordan, M.~I. (2008).
\newblock {\em Graphical models, exponential families, and variational
inference}.
\newblock Now Publishers Inc.
\bibitem[\protect\astroncite{Wang et~al.}{2005}]{wang2005divergence}
Wang, Q., Kulkarni, S.~R., and Verd{\'u}, S. (2005).
\newblock Divergence estimation of continuous distributions based on
data-dependent partitions.
\newblock {\em IEEE Transactions on Information Theory}, 51(9):3064--3074.
\bibitem[\protect\astroncite{Wang et~al.}{2009}]{wang2009divergence}
Wang, Q., Kulkarni, S.~R., and Verd{\'u}, S. (2009).
\newblock Divergence estimation for multidimensional densities via $ k
$-nearest-neighbor distances.
\newblock {\em IEEE Transactions on Information Theory}, 55(5):2392--2405.
\bibitem[\protect\astroncite{Wang and Rockova}{2020}]{wang2020uncertainty}
Wang, Y. and Rockova, V. (2020).
\newblock Uncertainty quantification for sparse deep learning.
\newblock In {\em International Conference on Artificial Intelligence and
Statistics}, pages 298--308. PMLR.
\bibitem[\protect\astroncite{Wilkinson}{2018}]{wilkinson2018stochastic}
Wilkinson, D.~J. (2018).
\newblock {\em Stochastic modelling for systems biology}.
\newblock CRC press.
\bibitem[\protect\astroncite{Wilkinson}{2013}]{wilkinson2013approximate}
Wilkinson, R.~D. (2013).
\newblock Approximate bayesian computation (abc) gives exact results under the
assumption of model error.
\newblock {\em Statistical applications in genetics and molecular biology},
12(2):129--141.
\bibitem[\protect\astroncite{Wood}{2010}]{wood2010statistical}
Wood, S.~N. (2010).
\newblock Statistical inference for noisy nonlinear ecological dynamic systems.
\newblock {\em Nature}, 466(7310):1102--1104.
\bibitem[\protect\astroncite{Yarotsky}{2017}]{yarotsky2017error}
Yarotsky, D. (2017).
\newblock Error bounds for approximations with deep relu networks.
\newblock {\em Neural Networks}, 94:103--114.
\bibitem[\protect\astroncite{Zhao and Lai}{2020}]{zhao2020analysis}
Zhao, P. and Lai, L. (2020).
\newblock Analysis of k nearest neighbor {KL} divergence estimation for
continuous distributions.
\newblock In {\em 2020 IEEE International Symposium on Information Theory
(ISIT)}, pages 2562--2567. IEEE.
\end{thebibliography}