Estimating the Value of Evidence-Based Decision Making
Extracted source text ยท arXiv 2306.13681
\documentclass[letterpaper,12pt,unknownkeysallowed, dvipsnames]{article}
\usepackage{algorithm}
\usepackage[noend]{algpseudocode}
\usepackage{booktabs}
\usepackage{filemod}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{enumitem}
\usepackage{bbm}
\usepackage{fullpage}
\usepackage{amsmath}
\usepackage{mathabx}
\usepackage{mathtools}
\usepackage{scalerel}
\usepackage{xcolor}
\usepackage{array}
\usepackage{tablefootnote}
\usepackage{threeparttable}
\newcolumntype{L}[1]{>{\raggedright\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\usepackage{multirow}
\mathtoolsset{showonlyrefs}
\usepackage[round,longnamesfirst]{natbib}
\bibliographystyle{apalike}
\renewcommand\bibpreamble{\vspace{.2\baselineskip}}
\newtheorem{assumption}{Assumption}
\newtheorem{lemma}{Lemma}
\newtheorem{remark}{Remark}
\newtheorem{theorem}{Theorem}
\renewcommand{\appendix}{\small\parindent 0cm\parskip 5pt\setcounter{equation}{0}
\setcounter{section}{0}
\renewcommand{\thesection}{A.\arabic{section}}
\renewcommand{\theequation}{A.\arabic{equation}}
\setcounter{lemma}{0}\renewcommand{\thelemma}{A.\arabic{lemma}}
\setcounter{theorem}{0}\renewcommand{\thetheorem}{A.\arabic{theorem}}
}
\makeatletter
\def\blfootnote{\xdef\@thefnmark{}\@footnotetext}
\makeatother
\makeatletter
\def\@sect#1#2#3#4#5#6[#7]#8{\ifnum #2>\c@secnumdepth
\let\@svsec\@empty\else
\refstepcounter{#1}\edef\@svsec{\csname the#1\endcsname. \hskip 0.4em}\fi
\@tempskipa #5\relax
\ifdim \@tempskipa>\z@
\begingroup #6\relax
\@hangfrom{\hskip #3\relax\@svsec}{\interlinepenalty \@M #8\par}
\endgroup
\csname #1mark\endcsname{#7}\addcontentsline
{toc}{#1}{\ifnum #2>\c@secnumdepth \else
\protect\numberline{\csname the#1\endcsname}\fi
#7}\else
\def\@svsechd{#6\hskip #3\relax
\@svsec #8\csname #1mark\endcsname
{#7}\addcontentsline
{toc}{#1}{\ifnum #2>\c@secnumdepth \else
\protect\numberline{\csname the#1\endcsname}\fi
#7}}\fi
\@xsect{#5}}
\makeatother
\makeatletter
\renewcommand{\section}{\@startsection{section}{1}{0mm}{-\baselineskip}{0.25\baselineskip}{\centering\normalfont\normalsize\bf}}
\renewcommand{\subsection}{\@startsection{subsection}{2}{0mm}{-\baselineskip}{0.25\baselineskip}{\raggedright\normalfont\normalsize\bf}}
\renewcommand{\subsubsection}{\@startsection{subsubsection}{3}{0mm}{-\baselineskip}{0.25\baselineskip}{\raggedright\itshape\small}}
\def\@begintheorem#1#2{\trivlist \item[\hskip \labelsep{\bf #1\ #2}]\it}
\makeatother
\makeatletter
\renewenvironment{description}
{\list{}{\leftmargin=0pt
\labelwidth\z@ \itemindent-\leftmargin
\let\makelabel\descriptionlabel}}
{\endlist}
\makeatother
\renewcommand{\thesection}{\arabic{section}}
\renewcommand{\thesubsection}{\arabic{section}.\arabic{subsection}}
\renewenvironment{abstract}
{\begin{center}\normalsize\bf\text{Abstract}
\end{center}\begin{quote}\normalsize}
{\end{quote}}
\newcommand*\mwidebar[1]{%
\hbox{%
\vbox{%
\hrule height 0.5pt %
\kern0.30ex% %
\hbox{%
\kern-0.02em% %
\ensuremath{#1}%
\kern-0.02em% %
}%
}%
}%
}
\newcommand*\lowwidebar[1]{%
\hbox{%
\vbox{%
\hrule height 0.5pt %
\kern0.20ex% %
\hbox{%
\kern-0.02em% %
\ensuremath{#1}%
\kern-0.02em% %
}%
}%
}%
}
\usepackage{tikz,amsmath,environ}
\usetikzlibrary{decorations.pathreplacing,calc}
\tikzset{
ncbar angle/.initial=90,
ncbar/.style={
to path=(\tikztostart)
-- ($(\tikztostart)!#1!\pgfkeysvalueof{/tikz/ncbar angle}:(\tikztotarget)$)
-- ($(\tikztotarget)!($(\tikztostart)!#1!\pgfkeysvalueof{/tikz/ncbar angle}:(\tikztotarget)$)!\pgfkeysvalueof{/tikz/ncbar angle}:(\tikztostart)$)
-- (\tikztotarget)
},
ncbar/.default=0.5cm,
freaky dim/.default=4pt,
freaky/.style={
to path={let \p1=(\tikztostart),\p2=(\tikztotarget) in (\tikztostart)
-- ($(\tikztostart)!#1!\pgfkeysvalueof{/tikz/ncbar angle}:(\tikztotarget)$)
-- ({\x1-#1},{(\y1+\y2)/2-abs(#1)})
-- ({\x1-2*#1},{(\y1+\y2)/2})
-- ({\x1-#1},{(\y1+\y2)/2+abs(#1)})
-- ($(\tikztotarget)!($(\tikztostart)!#1!\pgfkeysvalueof{/tikz/ncbar angle}:(\tikztotarget)$)!\pgfkeysvalueof{/tikz/ncbar angle}:(\tikztostart)$)
-- (\tikztotarget)}
},
freaky/.default=0.5cm,
}
\tikzset{freaky left brace/.style={freaky=0.5ex}}
\tikzset{freaky right brace/.style={freaky=-0.5ex}}
\tikzset{square left brace/.style={ncbar=0.5ex}}
\tikzset{square right brace/.style={ncbar=-0.5ex}}
\tikzset{round left paren/.style={ncbar=0.3cm,out=115,in=-115}}
\tikzset{round right paren/.style={ncbar=0.3cm,out=65,in=-65}}
%
\NewEnviron{flcases}{\setbox0=\hbox{$\,\begin{matrix}\BODY\end{matrix}\,$}%
\setbox2=\hbox{\begin{tikzpicture}
\draw [thick] (0,\botdim) to [freaky left brace] (0,\topdim);
\copy0
\end{tikzpicture}}
\vcenter{\hbox{\copy2}}%
}
\def\topdim{\the\dimexpr+\ht0+.5\ht\strutbox-.5\dp\strutbox-3pt\relax}
\def\botdim{\the\dimexpr-\ht0+.5\ht\strutbox-.5\dp\strutbox+3pt\relax}
\allowdisplaybreaks
\renewcommand{\topfraction}{1}
\usepackage{fancyheadings}
\pagestyle{fancy}
\fancyhf{}
\renewcommand{\headrulewidth}{0.0pt}% To remove the rule on the top of the page
\cfoot{\thepage}
\begin{document}
\vspace*{0.2cm}
\blfootnote{\hspace*{-0.25in}We are grateful to Brad Efron for making a subset of the Cochrane data available to us.}
%\thispagestyle{empty}
\vskip 20pt
\centerline{\large\bf Estimating the Value of Evidence-Based Decision Making}
\begin{center}
\vskip 10pt
% \blfootnote{\hspace*{-0.25in}\!8.}
{\large
\lineskip .5em
\begin{tabular}[t]{ccccc}
Alberto Abadie&&Anish Agarwal&&Guido Imbens\\[.2ex]
MIT&&Amazon.com&&Stanford\\[1.5ex]
Siwei Jia&&James McQueen&&Serguei Stepaniants\\[.2ex]
Amazon.com&&Amazon.com&&Amazon.com\\[1ex]
\end{tabular}
\par}
\vskip 1em
{\large September 9, 2023} \par
\vskip 1em
\end{center}\par
\bigskip
\begin{center}\normalsize\bf\text{Abstract}
\end{center}\begin{quote}\normalsize
\noindent{\small Business/policy decisions are often based on evidence from randomized experiments and observational studies. In this article we propose an empirical framework to estimate the value of evidence-based decision making (EBDM) and the return on the investment in statistical precision.}
\end{quote}
\addtolength{\baselineskip}{0.5\baselineskip}
\section{Introduction} \label{sec:introduction}
Many organizations use randomized experiments and observational studies to improve their decision making. For example, \citet{gupta2019top} write ``Together [Airbnb, Amazon, Booking.com, Facebook, Google, LinkedIn, Lyft, Microsoft, Netflix, Twitter, Uber, Yandex, and Stanford University] have tested more than one hundred thousand experiment treatments last year.'' The fact that so many organizations conduct so many experiments
suggests that data evidence provides a value for guiding business/policy decisions.
However, we are unaware of the existence of empirical tools that organizations can use to assess the value of their EBDM practices.
In the absence of such tools, it is difficult to assess whether too much experimentation is being done, or too little, whether experiments are too large or too small, and whether the right experiments are done. Part of the challenge in evaluating the value of EBDM is that it requires a description of the role of evidence in the business/policy decision process. In other words, it requires assumptions on what organizations will do with and without various amounts of evidence that they can choose to generate at some cost.
In this article, we propose an empirical Bayes estimator of the value of EBDM. We study the problem of a decision maker deciding whether to adopt a particular policy intervention. We use the term ``agent'' to refer to the decision maker, and ``policy'', ``intervention'' and ``treatment'' interchangeably to refer to the policy intervention under scrutiny. The agent can implement the intervention based on prior information or gather additional information at some cost, for example, by running an experimental or observational evaluation of the effect of the intervention. At the stage where the decision maker decides to implement the intervention or not, she does so to maximize her utility given the available information. We derive expressions for the value of the additional information and show how to estimate this value using meta data on estimates of the effects of business/policy interventions and their standard errors.
Our framework allows decision makers to assess the value of experimental and nonexperimental studies, and how this value changes with the precision of the studies. Currently, many organizations decide on the precision of their studies based on power calculations, which do not take into account the cost and benefits of EBDM.
\section{The value of EBDM}
We use the notation $X\sim (\theta,\sigma^2)$ to indicate that the random variable $X$ has mean $\theta$ and variance $\sigma^2$. We use $X\sim N(\theta,\sigma^2)$ to indicate that the distribution of $X$ is Gaussian with mean and variance $(\theta,\sigma^2)$. We use $f_X(\cdot)$ for the density of a random variable, $X$, and $f_{X|W}(\cdot|w)$ for the conditional density of $X$ given $W=w$.
$\phi(\cdot)$ and $\Phi(\cdot)$ are the probability density function and the cumulative distribution function of the standard Gaussian distribution.
\subsection{Setup}
Consider the problem of a decision maker who has to choose whether to adopt a particular policy on a population of units. The per-unit payoff of the policy, $\tau$, follows a distribution with probability density function $f_\tau(\cdot)$ and mean $\mu$. We assume that the agent has risk-neutral preferences.
As a result, in the absence of additional information, the agent launches the policy as long as the expected payoff from launching is positive,
\[
\mu-c_L> 0,
\]
where $c_L$ is the cost of launching per-unit. The expected value of this decision is
$\max\{\mu-c_L,0\}$.
Suppose, however, that the agent can choose to obtain additional information on the policy payoff, at some cost. In particular, the agent can obtain a signal, $\widehat\tau$, with distribution
\begin{equation}
\widehat\tau\,|\,\tau \sim N(\tau,\sigma^2),
\label{equation:GaussAprox}
\end{equation}
at cost $c_F + c(\sigma^2)$, with $c_F\geq 0$, $c(\cdot)\geq 0$ and $c'(\cdot)\leq 0$. This aspect of the model aims to capture information obtained from studies that estimate policy effects based on experimental or observational data. The constant $c_F$ reflects the fixed cost of a data-driven policy evaluation, and the function $c(\cdot)$ measures the costs of precision, which depends in part on the sample size of the study.
The restriction on the derivative of $c(\cdot)$ conveys the notion that more precise information is weakly more costly.
The Gaussianity assumption on the distribution of $\widehat\tau\,|\,\tau$ is motivated by approximate Gaussianity of the large sample distributions of many commonly used estimators of treatment effects. After observing the signal $\widehat\tau$, the expected payoff of the policy is
\begin{align*}
E[\tau|\widehat\tau=t] &= \int u f_{\tau|\widehat\tau}(u|t) du\\
&= \frac{\displaystyle\int \frac{u}{\sigma}\phi((t-u)/\sigma)f_{\tau}(u) du}{\displaystyle\int \frac{1}{\sigma}\phi((t-u)/\sigma)f_\tau(u) du}.
\end{align*}
In this case, the agent launches the policy if
\[
E[\tau|\widehat\tau] - c_L> 0.
\]
For any scalar, $x$, let $I_{(0,\infty)}(x)$ be the function that takes value one if $x$ is greater than zero and value zero otherwise. The expected payoff with EBDM is
\begin{align}
V(\sigma^2)
&=E\left[(\tau - c_L) I_{(0,\infty)}(E[\tau|\widehat\tau] - c_L)\right]\nonumber\\
&=E\left[I_{(0,\infty)}(E[\tau|\widehat\tau] - c_L)E[\tau-c_L|\widehat\tau]\right]\nonumber\\
&=E\left[I_{(0,\infty)}(E[\tau|\widehat\tau] - c_L)(E[\tau|\widehat\tau]-c_L)\right]\nonumber\\
&=E\left[\max\left\{E[\tau|\widehat\tau]-c_L,0\right\}\right].
\end{align}
Let
\[
V(\infty) = \max\{\mu-c_L,0\},
\]
which is the expected payoff under no additional information besides the information in the distribution of $\tau$. Because $\max\{x,0\}$ is a convex function of $x$, Jensen's Inequality implies,
\[
V(\sigma^2)\geq \max\{\mu-c_L,0\}=V(\infty).
\]
The value of EBDM is the difference in expected payoffs $V(\sigma^2)$ and $V(\infty)$, which is nonnegative, minus the cost of acquiring the information, which is generally positive:
\begin{align}\label{eq:VoE}
\textsf{VoE}(\sigma^2) = V(\sigma^2) - V(\infty) - (c_F + c(\sigma^2)).
\end{align}
A second version of $\textsf{VoE}$, which we term $\textsf{VoID}$ is obtained when, in the absence of additional information about the effect of the intervention, the intervention is always deployed:
\begin{align}\label{eq:VoEL}
\textsf{VoID}(\sigma^2) = V(\sigma^2) - (\mu -c_L) - (c_F + c(\sigma^2)).
\end{align}
$\textsf{VoID}(\sigma^2)$ is motivated by settings with ex-ante (pre-evaluation) ambiguity on the distribution of $\tau$, and agents who have a bias for action in the presence of such ambiguity.
\subsection{A motivating example}
A common instance of the setting described above is one where the decision maker obtains experimental evidence on the effect of the policy. Consider an experiment with $N$ units: $i=1, \ldots, N$. The experimenter assigns $N_1$ units at random to treatment and the remaining $N_0=N-N_1$ to control. If unit $i$ is treated, an outcome is drawn
\begin{align*}
Y_i(1) \sim (\theta_1, \sigma_1^2).
\end{align*}
If unit $i$ is untreated, the outcome is drawn
\begin{align*}
Y_i(0) \sim (\theta_0, \sigma_0^2).
\end{align*}
Let $W_i$ be an indicator of treatment for unit $i$. We observe $Y_i=Y_i(1)W_i+Y_i(0)(1-W_i)$.
The average effect of the treatment is
\begin{align*}
\tau=\theta_1-\theta_0.
\end{align*}
A simple estimator of $\tau$ is the difference in mean outcomes between treated and nontreated,
\begin{align*}
\hat{\tau} = \frac{1}{N_1}\sum^{N}_{i = 1} W_i Y_i - \frac{1}{N_0}\sum^{N}_{i = 1} (1-W_i)Y_i.
\end{align*}
Then, for large $N_0$ and $N_1$, equation \eqref{equation:GaussAprox} holds approximately, with
\begin{align*}
\sigma^2 = \frac{\sigma_1^2}{N_1} + \frac{\sigma_0^2}{N_0}. \label{eq:variance_defn}
\end{align*}
In some settings, researchers favor treatment effect parameters free of units of measurement, such as lift $\tau = (\theta_1-\theta_0)/\theta_0$. Let
\[
\widehat\tau = \frac{\displaystyle\frac{1}{N_1}\displaystyle\sum^{N}_{i = 1} W_i Y_i - \displaystyle\frac{1}{N_0}\displaystyle\sum^{N}_{i = 1} (1-W_i)Y_i}{\displaystyle\frac{1}{N_0}\displaystyle\sum^{N}_{i = 1} (1-W_i)Y_i}.
\]
Then, for large $N_0$ and $N_1$, equation \eqref{equation:GaussAprox} holds with
\[
\sigma^2 = \frac{1}{\theta_0^2}\left(\frac{\sigma_1^2}{N_1}+(1+\tau)^2\frac{\sigma_0^2}{N_0}\right).
\]
For values of the lift parameters close to zero, as is common in some online experimentation settings, we can approximate
\[
\sigma^2 \approx \frac{1}{\theta_0^2}\left(\frac{\sigma_1^2}{N_1}+\frac{\sigma_0^2}{N_0}\right).
\]
\subsection{A Gaussian distribution for $\tau$}
\label{section:gaussian_prior}
In this section, we obtain simple closed-form formulas for the case when the distribution of $\tau$ can be approximated with a Gaussian distribution,
\begin{align}
\tau\sim N(\mu, \gamma^2).
\label{equation:tau_sigma}
\end{align}
While equation \eqref{equation:GaussAprox} is supported by the Central Limit Theorem for large sample studies, equation \eqref{equation:tau_sigma} imposes two important restrictions. First, $\tau$ is Gaussian. Second, this distribution is independent of $\sigma^2$. The first is a strong parametric restriction.
Gaussianity of $\tau$ could be a suitable approximation in some settings, but not in others. The second restriction could be violated, for example, if experimenters have some additional information about $\tau$ for each treatment, and adapt the power of their experimental or observational studies accordingly through the choice of sample size. We dispose of these two restrictions later in the article. We adopt them in this section, however, to obtain closed-form formulas for the value of EBDM.
If equations \eqref{equation:GaussAprox} and \eqref{equation:tau_sigma} hold, the marginal distribution of $\widehat{\tau}$ is $\widehat\tau\sim N(\mu, \gamma^2 + \sigma^2)$.
The posterior for $\tau$ is given by
\begin{align*}
\tau \,|\, \widehat{\tau} \sim N\left(
\frac{\mu / \gamma^2 + \hat{\tau} / \sigma^2}{1 / \gamma^2 + 1 / \sigma^2},
\frac{1}{1 / \gamma^2 + 1 / \sigma^2}
\right).
\end{align*}
The expected payoff with EBDM is
\begin{align*}
V(\sigma^2)
&=E\left[\max\left\{\frac{\mu / \gamma^2 + \widehat\tau / \sigma^2}{1 / \gamma^2 + 1 / \sigma^2}-c_L,0\right\}\right].
\end{align*}
Let
\[
Z= \frac{\mu / \gamma^2 + \widehat\tau / \sigma^2}{1 / \gamma^2 + 1 / \sigma^2}-c_L.
\]
Recall that the marginal distribution of $\widehat\tau$ is Gaussian with mean $\mu$ and variance $\gamma^2+\sigma^2$. As a result,
\begin{equation}
Z\sim N \left(\mu-c_L,\frac{\gamma^4}{\gamma^2+\sigma^2}\right).
\label{equation:distribution_Z}
\end{equation}
Now, $V(\sigma^2)$ is the the first moment of the Gaussian distribution in \eqref{equation:distribution_Z} censored from below at zero,
\begin{align}
V(\sigma^2)
&=(\mu - c_L) \Phi\left(\frac{\mu - c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}} \right) + \frac{\gamma^2}{\sqrt{\gamma^2+\sigma^2}}\phi\left(\frac{\mu - c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}}\right).
\label{equation:Vsigma}
\end{align}
$V(\sigma^2)$ is decreasing in $\sigma^2$ and increasing in $\gamma^2$ (see appendix). This conforms to our intuition that higher precision of the signal $\widehat\tau|\tau$ and higher uncertainty regarding $\tau$ increase the payoff from EBDM.
Notice that
\begin{equation*}
\lim_{\sigma^2\rightarrow \infty} V(\sigma^2) = \max\{\mu-c_L,0\}=V(\infty)
\end{equation*}
and
\[
\lim_{\sigma^2\rightarrow 0} V(\sigma^2) =
(\mu -c_L)\Phi\left(\frac{\mu-c_L}{\gamma}\right)+\gamma\phi\left(\frac{\mu-c_L}{\gamma}\right).
\]%\label{eq:value_limit_0}
%
As $\sigma^2\rightarrow\infty$, we lose any additional information about the value of $\tau$ beyond its distribution, and $V(\sigma^2)$ converges to $V(\infty)$. As $\sigma^2\rightarrow 0$, we learn the value of $\tau$. In this case, $V(\sigma^2)$ converges to $E[\max\{\tau - c_L, 0\}]$, the mean of the distribution of $\tau - c_L$ censored at zero.
So far, we have treated $\sigma^2$ as a constant. We now allow $\sigma^2$ to have a non-degenerate distribution, independent of $\tau$.
In this case, the average payoff of EBDM is
\begin{align}
V&=E\Big[\max\{E\big[\tau|\widehat\tau,\sigma\big]-c_L,0\}\Big]\nonumber\\&=E\Big[E\big[\max\{E[\tau|\widehat\tau,\sigma]-c_L,0\}|\sigma\big]\Big]\nonumber\\
&=E\left[(\mu - c_L) \Phi\left(\frac{\mu - c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}} \right) + \frac{\gamma^2}{\sqrt{\gamma^2+\sigma^2}}\phi\left(\frac{\mu - c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}}\right)\right].
\label{equation:V}
\end{align}
\subsection{A Gaussian mixture distribution for $\tau$}
\label{section:mixture_prior}
In section \ref{section:Simulations} we use a Gaussian mixture distribution of $\tau$ to evaluate the effects of ... Suppose $\tau$ follows a mixture of $k$ Gaussian distributions with parameters $(\mu_1,\gamma^2_1), \ldots, (\mu_k,\gamma^2_k)$, and mixture probabilities $p_1, \ldots, p_k$. Let $\widehat\tau=\tau + \varepsilon$, where $\varepsilon$ is independent Gaussian noise with variance $\sigma^2$. Conditional on $\tau\sim N(\mu_j,\gamma^2_j)$, we have
\[
E[\tau|\widehat\tau, \tau\sim N(\mu_j,\gamma^2_j)] = \frac{\mu_j / \gamma_j^2 + \widehat{\tau} / \sigma^2}{1 / \gamma_j^2 + 1 / \sigma^2}.
\]
As a result,
\begin{align}
E[\tau|\widehat\tau] &= \sum_{j=1}^k E[\tau|\widehat\tau, \tau\sim N(\mu_k,\gamma^2_j)]
\Pr(\tau\sim N(\mu_j,\gamma^2_j)|\widehat\tau)\\
&=\frac{\displaystyle\sum_{j=1}^k\left(\displaystyle\frac{\mu_j / \gamma^2_j + \widehat{\tau} / \sigma^2}{1 / \gamma^2_j + 1 / \sigma^2}\right)\frac{1}{\sqrt{\gamma_j^2+\sigma^2}}\phi\left(\displaystyle\frac{\widehat\tau-\mu_j}{\sqrt{\gamma^2_j+\sigma^2}}\right)p_j}{\displaystyle\sum_{j=1}^k\frac{1}{\sqrt{\gamma_j^2+\sigma^2}}\phi\left(\displaystyle\frac{\widehat\tau-\mu_j}{\sqrt{\gamma^2_j+\sigma^2}}\right)p_j}.
\label{equation:mixture_posterior_mean}
\end{align}
\section{Empirical Bayes estimation}
\label{section:empirical_Bayes}
In this section, we consider a setting with multiple realizations from the distribution of $(\tau,\sigma,\widehat\tau,\widehat\sigma)$, where only $\widehat\tau$ and $\widehat\sigma$ are observed. In this setting, $(\widehat\tau,\widehat\sigma)$ are estimates and the corresponding standard errors for a set of policy evaluations in the data set. We consider the homoskedastic case, with $\sigma^2$ constant, and the heteroskedastic case with $\mbox{var}(\sigma^2)>0$. In all our calculations we approximate the per-unit launch cost as $c_L\approx 0$. Alternatively we can think of the $\tau$ as capturing the net benefits of the treatment after taking out the launch cost.
We illustrate these calculations using a subset of the Cochrane database \citep{collaboration2002cochrane, starr2009origins} containing information on 8821 randomized experiments. For each experiment we observe the point estimate, $\widehat\tau$, as well as the standard error of $\widehat\tau$, which can be seen as an estimate of $\sigma$. We use the Cochrane dataset as a benchmark, without attempting to interpret the magnitude or direction of the resulting estimates. Figure \ref{figure:Cochrane} shows the distribution of $\widehat\tau$ across the experiments in the dataset. The average value of $\widehat\tau$ is $-0.1421$ with range $[-8.4763, 7.1663]$. Standard errors have mean $0.7471$ and range $[0.0099,2.1232]$.
\begin{figure}[!tp]
\centering
\includegraphics[width=.5\textwidth]{hist_tau_hat_crop.pdf}
\caption{Distribution of estimated treatment effects in the Cochrane data}
\label{figure:Cochrane}
\end{figure}
\subsection{Parametric empirical Bayes}
\label{section:peB}
In this section, we adopt a Gaussian specification for the distribution of $\tau$. Because $\widehat\tau$ is unbiased, we can estimate $\mu$---the mean of the distribution of $\tau$---as the mean of $\widehat\tau$ across evaluations. To estimate $\gamma^2$---the variance of the distribution of $\tau$---we deconvolute the distribution of $\widehat\tau$ as follows. By the Total Law of Variance,
\[
\mbox{var}(\widehat\tau)=E[\mbox{var}(\widehat\tau|\tau)]+\mbox{var}(E[\widehat\tau|\tau]).
\]
Unbiasedness of $\widehat\tau$ conditional on $\tau$ implies
\begin{align*}
\gamma^2&=\mbox{var}(\tau)\\
&=\mbox{var}(E[\widehat\tau|\tau])\\
&=\mbox{var}(\widehat\tau)-E[\mbox{var}(\widehat\tau|\tau)].
\end{align*}
As a result, we define $\widehat\gamma^2$ as the difference between the variance $\widehat\tau$ across experiments in the data minus the mean of the squares of the standard errors. This estimator is not guaranteed to be positive. In the Cochrane dataset, $\widehat\gamma^2=1.6677-0.7641=0.9036$.
\subsubsection{Homoskedastic case}
For the homoskedastic case, we estimate $\sigma^2$ as the average of the squares of the standard deviations of $\widehat\tau$ across studies. In the Cochrane data, this estimate is 0.7641. Plugging in this value in \eqref{equation:Vsigma} along with estimates of $\mu$ and $\gamma^2$, we obtain $V(0.7641)=0.2138$.
\subsubsection{Heteroskedastic case}
\label{section:peB_het}
We now relax the assumption that $\sigma^2$ is constant. It can be shown (see appendix) that $V(\sigma^2)$ is convex. Then, by Jensen's Inequality, $V(E[\sigma^2])\leq E[V(\sigma^2)]$. This result implies that the assumption of homoskedasticity may lead to underestimation of the average payoff when $\sigma^2$ is not constant. Under heteroskedasticity, we evaluate the expression in \eqref{equation:V} plugging in study-specific estimates of $\sigma^2$. Relative to the calculations in the previous section, now the value of experimentation is computed for each value of $\sigma^2$ and then integrated over the distribution of $\sigma^2$. An estimator of $V$ based on a set of policy estimates can be calculated in two steps: {\it (i)} use the square of the standard error of $\widehat\tau$ to approximate $\sigma^2$, and estimate the value of each study separately, and {\it (ii)} take the average over all studies in the sample. Doing this for the Cochrane data, we obtain $V=0.2351$.
\subsection{Nonparametric empirical Bayes}
We now relax the parametric restriction $\tau\sim N(\mu,\gamma^2)$ of section \ref{section:peB} and consider a nonparametric distribution.
\subsubsection{Homoskedastic case}
\label{section:npeB_hom}
Suppose we have $\tau/\sigma|\sigma \sim G_\sigma$,
where $G_\sigma$ is some unspecified distribution. Then,
\begin{equation}
\widehat\tau/\sigma \big|\tau/\sigma, \sigma \sim N(\tau/\sigma,1).
\end{equation}
Therefore, the marginal density of $\widehat\tau/\sigma$ is
\begin{equation}
f_{{\hat\tau}/\sigma}(z)=\int\left(\int \phi(z-t) dG_{s}(t)\right)dP(s),
\end{equation}
where $P$ is the distribution of $\sigma$.
If $\tau/\sigma|\sigma \sim G$, independent of $\sigma$ (a very strong assumption), then
\[
f_{{\hat\tau}/\sigma}(z)=\int \phi(z-t) dG(t),
\]
and the distribution $G$ can be estimated via nonparametric empirical Bayes methods (NPEB). Suppose we have $n$ experiments, indexed $i=1, \ldots, n$.
Following \cite{jiang2009general}, we can estimate $G$ as
\begin{equation}
\max_{G\in \mathcal G} \sum_{i=1}^n \log \int \phi(\widehat\tau/\sigma-t) dG(t).
\label{equation:npml}
\end{equation}
\cite{jiang2009general} uses fixed-point methods to solve the nonparametric maximum likelihood problem in \eqref{equation:npml}. The solution is computed over a grid $u_1, \ldots, u_m$ representing points of mass of $G$, with respective probabilities, $\widehat f_1, \ldots, \widehat f_m$.
Notice that,
\begin{align}
E[\tau/\sigma|\hat\tau/\sigma=z,\sigma=s]&=\frac{\displaystyle\int t\, \phi(z-t)dG(t)}{\displaystyle\int \phi(z-t)dG(t)}.
\end{align}
The sample analog of $E[\tau|\hat\tau/\sigma=z,\sigma=s]$ is
\[
\sigma_i\,\frac{\displaystyle\sum_{j=1}^m u_j\phi(\widehat\tau_i/\sigma_i-u_j)\widehat f_j}{\displaystyle\sum_{j=1}^m \phi(\widehat\tau_i/\sigma_i-u_j)\widehat f_j}.
\]
\cite{koenker2014convex} propose an algorithm to solve \eqref{equation:npml} based on convex optimization procedures. For the Cocharne data set, using the standard errors of the estimates to approximate $\sigma_1, \ldots, \sigma_n$ and the algorithm in \cite{koenker2014convex} to estimate $G$, we obtain the estimate $\widehat V=0.2226$.
\subsubsection{Heteroskedastic case}
\label{section:npeB_het}
We relax the strong assumption that $\tau/\sigma$ is independent of $\sigma$ by dividing the support of $\sigma$ into 5 intervals and doing the NPEB calculations of the previous section interval by interval. Applying this method to the Cochrane data set, we obtain the estimate $\widehat V=0.2897$.
\begin{table}[!tp]
\caption{Expected payoff of EBDM}
\centering
\begin{tabular}{lccccc}\\[-1ex]\hline\hline\\[-2ex]
&&&&\multicolumn{2}{c}{empirical Bayes}\\
\hspace*{-0.2cm}\raisebox{-1ex}{distribution of $\tau$:}&&\hspace*{2cm}true value &¶metric &nonparametric\\ \cline{5-6}\\[-2ex]
%\hspace*{.15cm}
\multirow{2}{*}{\begin{tabular}{l}\hspace*{1.28cm}{Gaussian}\ $\begin{flcases}\mbox{expected payoff}\\ \mbox{95\% interval}\end{flcases}$\end{tabular}} &&\hspace*{2cm}0.3970& &0.3970 &0.3970\\
&&&&$[0.3429,0.4510]$&$[0.3423,0.4517]$\\[2ex]
\multirow{2}{*}{\begin{tabular}{c}\hspace*{1.5cm}{mixture}\ $\begin{flcases}\mbox{expected payoff}\\ $\mbox{95\% interval}$\end{flcases}$\end{tabular}} &&\hspace*{2cm}0.3225 &&0.3942 &0.3227\\
&&&&$[0.3168,0.4717]$&$[0.2685,0.3768]$\\[1ex] \hline
\end{tabular}
\label{table:simulations}
\end{table}
\section{Simulations}
\label{section:Simulations}
We consider two data generating processes (DGP). In DGP1, the parameters $\tau$ have a standard Gaussian distribution $\tau\sim N(0,1)$, so the parametric empirical Bayes model of section \ref{section:gaussian_prior} applies. In DGP2, the parameters $\tau$ follow the mixture distribution as in section \ref{section:mixture_prior}. In particular, in DGP2,
\[
\tau \sim \left\{\begin{array}{ll}N(-5,1/2)&\mbox{with prob. } 0.01,\\
N(0,1/2)&\mbox{with prob. } 0.98,\\
N(5,1/2)&\mbox{with prob. } 0.01.\\\end{array}\right.
\]
DGP1 and DGP2 both produce a distribution of $\tau$ with mean zero and variance one. We generate $\widehat\tau$ as $\widehat\tau=\tau + \sigma u$, where $u$ is independent standard Gaussian and $\sigma=0.1$. To calculate the expected payoff of EBDM, we consider the case of $c_L=0$.
We run 1000 simulations for DGP1 and DGP2 with $n=500$. Equation \eqref{equation:Vsigma} with $\mu=0$, $\gamma^2=1$, $\sigma=0.1$, and $c_L=0$ gives the true expected payoff of EBDM under DGP1. To calculate the true expected payoff of EBDM under DGP2,
we first use equation \eqref{equation:mixture_posterior_mean} to compute $E[\tau|\widehat\tau]$ over the $n\times 1000=500,000$ realizations of $\widehat\tau$ in the simulations, and report the average of $\max\{E[\tau|\widehat\tau],0\}$. In each of the simulations, we calculate parametric and nonparametric empirical Bayes estimates of the average payoff of EBDM. The parametric empirical Bayes estimator is the sample analog of equation \eqref{equation:V}. This estimator is valid only under the assumption that the true distribution of $\tau$ is Gaussian. The nonparametric empirical Bayes estimator is as in section \ref{section:npeB_hom}. For the simulations in this section, we treat $\sigma$ as known.
%Because the simulation does not generate standard errors estimates for the different %realizations of $\widehat\tau$.
Table \ref{table:simulations} reports the true values of the expected payoff of EBDM along with means and 95\% intervals for the distribution of the estimates across simulations. When the distribution of $\tau$ is Gaussian, the distribution of the parametric and nonparametric empirical Bayes estimates across simulations are both centered at the true value of the expected EBDM payoff. Moreover, there is no evidence of substantial gains from knowledge of the parametric form of the distribution of $\tau$. The 95\% interval for the nonparametric estimator is only 1.1\% wider than the interval for the parametric estimator.
For the case when the distribution of $\tau$ is a mixture, the results for parametric estimator reveal a clear bias, while the distribution of the nonparametric estimator remains centered at the true value of the expected payoff. Moreover, the 95\% interval for the nonparametric estimator is 30.1\% narrower than the interval for the parametric estimator.
\begin{table}[t]
\caption{Results for the Cochrane data}
\centering
\begin{tabular}{lccccc}\\[-1ex]\hline\hline\\[-2ex]
{\it Distribution of $\widehat\tau$:}\\
\multicolumn{6}{c}{$\widehat\mu = -0.1421$}\\[1ex]
{\it Value of EBDM:}\\
&¶metric &&\multicolumn{2}{c}{nonparametric}\\
&& &&homoskedastic &heteroskedastic\\ \cline{5-6}\\[-2ex]
%\hspace*{.15cm}
\multirow{2}{*}{\begin{tabular}{l}$\textsf{VoE}$\hspace*{.3cm}$\begin{flcases}\mbox{estimate}\\ \mbox{95\% interval}\end{flcases}$\end{tabular}} &&0.2351 &&0.2226 &0.2055\\
&&$[0.2164,0.2537]$&&$[0.2060,0.2392]$&$[0.1890,0.2220]$\\[1ex]
\multirow{2}{*}{\begin{tabular}{c}$\textsf{VoID}$\ $\begin{flcases}\mbox{estimate}\\ \mbox{95\% interval}\end{flcases}$\end{tabular}} &&0.3772 &&0.3647 &0.3477\\
&&$[0.3572,0.3972]$&&$[0.3443,0.3852]$&$[0.3267,0.3686]$\\[1ex] \hline
\end{tabular}
\label{table:efron}
\end{table}
\section{Application to the Cochrane dataset}
Table \ref{table:efron} collects parametric and nonparametric empirical Bayes estimates of the value of EBDM in the Cochrane data. For the parametric case, Table \ref{table:efron} reports estimates computed under heteroskedasticity. For the nonparametric case, Table \ref{table:efron} reports estimates computed under homoskedasticity ($\mbox{var}(\sigma^2)=0$) and under heteroskedasticity ($\mbox{var}(\sigma^2)>0$). Below each of the estimates of the value of EBDM, Table \ref{table:efron} reports 95\% intervals computed over 1000 bootstrap draws from the distribution of $(\widehat\tau,\widehat\sigma^2)$ in the data. In our calculations, we impose $c_L+c_F+c(\sigma^2)=0$. The average value of $\widehat\tau$ across experiments in the Cochrane data is -0.1421 and the variance of $\widehat\tau$ across experiments is 1.6677.
Because $\widehat\tau$ has a negative mean and a variance that is large relative to the mean, this is a setting where we expect to have substantial gains from EBDM. Indeed, both the parametric and the nonparametric empirical Bayes estimates show positive values of about twice the magnitude of $\widehat\mu$. For this application, parametric empirical Bayes and nonparametric empirical Bayes under homoskedasticity produce similar estimates. Allowing for heteroskedasticity decreases the magnitudes of nonparametric empirical Bayes estimates by 7.7\% for the case of $\textsf{VoE}$ and $4.7$ for the case of $\textsf{VoID}$.
\section{Estimation of counterfactual EBDM values}
This section provides estimates of the value of EBDM under alternative levels of statistical precision. That is, we estimate how the value of EBDM would change as a result of a change in $\sigma^2$. In the resulting counterfactuals, the variance of the estimators is equal to the variance estimates for $\widehat\tau_1, \ldots, \widehat\tau_n$ in the original sample multiplied by $\lambda$. That is, $\lambda=0.5$ represents a counterfactual scenario where the variances of the estimators are 50\% smaller than the variance estimates in the original sample, while for $\lambda=1.5$ the variances of the estimators are 50\% larger than in the original sample.
For simplicity, we consider only counterfactual scenarios such that $\tau$ is independent of estimation variance, $\sigma^2$, and estimate the value of EBDM using the parametric empirical Bayes estimator of section \ref{section:peB}. It is conceptually straightforward to extend our procedure to more general settings (e.g., by modeling the dependence between $\tau$ and $\sigma^2$ and/or using nonparametric empirical Bayes estimators).
For each estimate, $i=1, \ldots, n$, in our sample we draw a value from the empirical Bayes estimate of the distribution of $\tau$. Let $\tau_1^*, \ldots , \tau_n^*$ be the resulting values for the draws. Next, for $i=1, \ldots, n$, we obtain $\widehat\tau_i^*=\tau_i^*+\sigma_i^* U_i$, where $U_1, \ldots, U_n$ are independent draws from the standard Gaussian distribution, and $\sigma_i^*=\sqrt{\lambda}\widehat\sigma_i$. We use the new sample $(\widehat\tau_1^*,\widehat\sigma_1^*), \ldots, (\widehat\tau_n^*,\widehat\sigma_n^*)$ to compute an estimate of the value of EBDM. We repeat this procedure multiple times to obtain the distribution of EBDM-value estimates for a particular value of $\lambda$. The average of this distribution is our estimate of the value of EBDM under variance modification factor $\lambda$. For the parametric empirical Bayes case, this average can also be computed directly using a empirical counterpart of equation \eqref{equation:V} that applies the variance modification factor $\lambda$ to $\sigma^2$.
Figure \ref{figure:counterfactual_cochrane} reports the results obtained from applying the procedure describe above to the Cochrane data. The solid line represents the value of EBDM as a function of the variance modification factor, $\lambda$. The shaded area represents 95\% intervals from the distribution of EBDM estimates. An investment that reduces estimation variance by half (about a 29.3\% decrease in standard errors) leads to an increase in the value of EBDM by 11.6\%, from 0.2351 to 0.2623. Conversely, an increase in estimation variance by half (about a 22.5\% increase in standard errors) decreases the value of EBDM by 8.1\%, from 0.2351 to 0.2161.
\begin{figure}[!tp]
\centering
\caption{Counterfactual values of EBDM}
\includegraphics[width=.7\textwidth]{counterfactual_cochrane.pdf}
\label{figure:counterfactual_cochrane}
\end{figure}
\bigskip
\centerline{\bf Appendix}
\appendix
The derivative of $V(\sigma^2)$ with respect to $\sigma^2$ is
\begin{align}\frac{\partial V(\sigma^2)}{\partial \sigma^2}
&= - \frac{\gamma^2}{2(\gamma^2+\sigma^2)^{3/2}}\phi\left(\frac{\mu-c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}}\right)\leq 0.
\end{align}
This conforms to our intuition that more experimental precision should increase the value of experimentation.
The derivative of $V(\sigma^2)$ with respect to $\gamma^2$ is
\begin{equation}
\frac{\partial V(\sigma^2)}{\partial \gamma^2} = \frac{\gamma^2+2\sigma^2}{2(\gamma^2+\sigma^2)^{3/2}}\phi\left(\frac{\mu-c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}}\right)\geq 0.
\label{equation:der_V_gamma}
\end{equation}
This conforms to our intuition that more concentrated priors should reduce the value of experimentation.
The second derivative of $V(\sigma^2)$ with respect to $\sigma^2$ is
\begin{equation}
\frac{\partial^2 V(\sigma^2)}{\partial \sigma^2\partial\sigma^2} =
\frac{3\gamma^4 + (\mu-c_L)^2(\gamma^2+\sigma^2)}{4\gamma^2(\gamma^2+\sigma^2)^{5/2}}\phi\left(\frac{\mu-c_L}{\gamma^2/\sqrt{\gamma^2+\sigma^2}}\right)\geq 0.
\end{equation}
Now, Jensen's Inequality implies $V(E[\sigma^2])\leq E[V(\sigma^2)]$.
\bigskip
\bibliography{bib}
\end{document}
\begin{thebibliography}{}
\bibitem[{Cochrane Collaboration}, 2002]{collaboration2002cochrane}
{Cochrane Collaboration} (2002).
\newblock The {C}ochrane library.
\newblock Database available on disk and CDROM. Oxford, UK, Update Software.
\bibitem[Gupta et~al., 2019]{gupta2019top}
Gupta, S., Kohavi, R., Tang, D., Xu, Y., Andersen, R., Bakshy, E., Cardin, N.,
Chandran, S., Chen, N., Coey, D., et~al. (2019).
\newblock Top challenges from the first practical online controlled experiments
summit.
\newblock {\em ACM SIGKDD Explorations Newsletter}, 21(1):20--35.
\bibitem[Jiang and Zhang, 2009]{jiang2009general}
Jiang, W. and Zhang, C.-H. (2009).
\newblock General maximum likelihood empirical {B}ayes estimation of normal
means.
\newblock {\em The Annals of Statistics}, 37(4):1647--1684.
\bibitem[Koenker and Mizera, 2014]{koenker2014convex}
Koenker, R. and Mizera, I. (2014).
\newblock Convex optimization, shape constraints, compound decisions, and
empirical {B}ayes rules.
\newblock {\em Journal of the American Statistical Association},
109(506):674--685.
\bibitem[Starr et~al., 2009]{starr2009origins}
Starr, M., Chalmers, I., Clarke, M., and Oxman, A.~D. (2009).
\newblock The origins, evolution, and future of {The Cochrane Database of
Systematic Reviews}.
\newblock {\em International Journal of Technology Assessment in Health Care},
25(S1):182--195.
\end{thebibliography}