Meta Answering for Machine Reading
Extracted source text · arXiv 1911.04156
\documentclass[table,times,11pt,a4paper]{article}
\usepackage{style/emnlp2020}
\newif\ifcomment\commenttrue
\aclfinalcopy
\input{style/math_commands}
\usepackage{hyphsubst}
\HyphSubstLet{english}{usenglishmax}
\usepackage[english]{babel}
\usepackage{xcolor}
\usepackage{amsmath}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{mdwlist}
\usepackage{dsfont}
\usepackage{times}
\usepackage{url}
\usepackage[pdftex]{graphicx}
\usepackage[]{algorithm2e}
\usepackage{multirow}
\usepackage{subcaption}
\SetKwInOut{Parameter}{Parameters}
\title{Meta Answering for Machine Reading}
\author{\normalsize
Benjamin B\"orschinger\textsuperscript{1},
Jordan Boyd-Graber\textsuperscript{1,4},
Christian Buck\textsuperscript{1},
Jannis Bulian\textsuperscript{1}, \\
{\bf \normalsize
Michelle Chen Huebscher\textsuperscript{1},
Massimiliano Ciaramita\textsuperscript{1},
Wojciech Gajewski\textsuperscript{1},} \\
{\bf \normalsize
Yannic Kilcher\textsuperscript{2}\thanks{\, Contribution partially while as an intern at Google.}\hphantom{,},
Rodrigo Nogueira\textsuperscript{3}\thanks{\, Contribution while an intern at Google.}\hphantom{,},
Lierni Sestorain Saralegui\textsuperscript{1}}\\
\normalsize{
\textsuperscript{1}Google Research \quad
\textsuperscript{2}ETH Z\"urich \quad
\textsuperscript{3}New York University \quad
\textsuperscript{4}University of Maryland}\\
\texttt{\small \{bboerschinger, jbg, cbuck, jbulian, michellechen, massi, wgaj, lierni\}@google.com} \\
\texttt{\small yannic.kilcher@inf.ethz.ch} \quad \texttt{\small rodrigonogueira@nyu.edu}}
\newcommand{\g}{\, | \,}
\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}
\newcommand{\explain}[2]{\underbrace{#2}_{\mbox{\footnotesize{#1}}}}
\newcommand{\bs}[1]{\boldsymbol{#1}}
\newcommand{\name}{meta-answerer}
\newcommand{\clstok}{\texttt{{[}CLS{]}}}
\newcommand{\septok}{\texttt{{[}SEP{]}}}
\newcommand{\verminusone}{\textsc{AnswerOnly}}
\newcommand{\verzero}{\textsc{Context}}
\newcommand{\verthree}{\textsc{RewriteQues}}
\newcommand{\fone}{F1}
\newcommand{\bert}{\textsc{bert}}
\newcommand{\qa}{QA}
\newcommand{\history}{evidence}
\newcommand{\History}{Evidence}
\newcommand{\inputa}{q, t, a_{i}, h}
\newcommand{\inputh}{q, t, h}
\newcommand{\nbest}{$M$-best}
\begin{document}
\maketitle
\input{2020_emnlp_metaanswer/sections/00-abstract}
\input{2020_emnlp_metaanswer/sections/10-introduction}
\input{2020_emnlp_metaanswer/sections/20-metaanswer}
\input{2020_emnlp_metaanswer/sections/30-human}
\input{2020_emnlp_metaanswer/sections/40-model}
\input{2020_emnlp_metaanswer/sections/50-computer_exper}
\input{2020_emnlp_metaanswer/sections/60-related}
\input{2020_emnlp_metaanswer/sections/70-conc}
\bibliography{bib/iclr2020_conference,bib/journal-full,bib/jbg}
\bibliographystyle{style/acl_natbib}
\clearpage
\setcounter{section}{0}
\renewcommand{\thesection}{\Alph{section}}
\input{2020_emnlp_metaanswer/sections/appendix}
\end{document}
%%%%% NEW MATH DEFINITIONS %%%%%
\usepackage{amsmath,amsfonts,bm}
\ifcomment
\newcommand{\pinaforecomment}[3]{\colorbox{#1}{\parbox{.8\linewidth}{#2: #3}}}
\else
\newcommand{\pinaforecomment}[3]{}
\fi
\newcommand{\jbgcomment}[1]{\pinaforecomment{green}{JBG}{#1}}
\newcommand{\massicomment}[1]{\pinaforecomment{yellow}{MC}{#1}}
\newcommand{\bencomment}[1]{\pinaforecomment{magenta}{BB}{#1}}
% Mark sections of captions for referring to divisions of figures
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
% Highlight a newly defined term
\newcommand{\newterm}[1]{{\bf #1}}
% Figure reference, lower-case.
\def\figref#1{figure~\ref{#1}}
% Figure reference, capital. For start of sentence
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
% Section reference, lower-case.
\def\secref#1{section~\ref{#1}}
% Section reference, capital.
\def\Secref#1{Section~\ref{#1}}
% Reference to two sections.
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
% Reference to three sections.
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
% Reference to an equation, lower-case.
\def\eqref#1{equation~\ref{#1}}
% Reference to an equation, upper case
\def\Eqref#1{Equation~\ref{#1}}
% A raw reference to an equation---avoid using if possible
\def\plaineqref#1{\ref{#1}}
% Reference to a chapter, lower-case.
\def\chapref#1{chapter~\ref{#1}}
% Reference to an equation, upper case.
\def\Chapref#1{Chapter~\ref{#1}}
% Reference to a range of chapters
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
% Reference to an algorithm, lower-case.
\def\algref#1{algorithm~\ref{#1}}
% Reference to an algorithm, upper case.
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
% Reference to a part, lower case
\def\partref#1{part~\ref{#1}}
% Reference to a part, upper case
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
% Random variables
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
% rm is already a command, just don't name any random variables m
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
% Random vectors
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
% Elements of random vectors
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
% Random matrices
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
% Elements of random matrices
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
% Vectors
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
% Elements of vectors
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
% Matrix
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
% Tensor
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
% Graph
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
% Sets
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
% Don't use a set called E, because this would be the same as our symbol
% for expectation.
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
% Entries of a matrix
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
% entries of a tensor
% Same font as tensor, without \bm wrapper
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
% The true underlying data generating distribution
\newcommand{\pdata}{p_{\rm{data}}}
% The empirical distribution defined by the training set
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
% The model distribution
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
% Stochastic autoencoder distributions
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} % Laplace distribution
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\ind}[1]{\mathds{1}\left[ #1 \right] }
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
% Wolfram Mathworld says $L^2$ is for function spaces and $\ell^2$ is for vectors
% But then they seem to use $L^2$ for vectors throughout the site, and so does
% wikipedia.
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} % See usage in notation.tex. Chosen to match Daphne's book.
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
\begin{abstract}
We investigate a framework for machine reading inspired by real-world information-seeking tasks. Similarly to scrolling through web search results, a \name{} inspects the candidate answers to a question provided by a machine reading QA system.
We find that with just a small snippet of text around an answer, humans can outperform the underlying QA system.
Similarly, a simple machine \name{} outperforms the reader on the Natural Questions dataset with equally impoverished information.
However, humans and machines seem to differ crucially, in terms of strategies and capacity to use contextual information.
Thus, the task suggests a challenge to probe the understanding of context in NLU.
\end{abstract}
\section{Introduction}
Question Answering (QA) is a benchmark task in Natural Language
Understanding (NLU) that has evolved from a niche
evaluation~\cite{voorhees-00} to a yardstick for measuring human
vs. computer intelligence~\cite{ferruci-10}.
The dominant format for QA is \emph{machine
reading}~\citep{rajpurkar-etal-2016-squad}: answering a question
either by extracting a span of text identifying the \emph{answer} from
a single textual \emph{context}, or abstaining~\cite{rajpurkar-18}.
Machine learning can answer many questions using machine reading, but the price of this setting is artifice, as both the data and the task
fail to represent an ideal question answering setup.
On the data side, systems can shirk
understanding language in favor of artifact-enabled shortcuts, thus
making them vulnerable to adversarial
attacks~\citep{jia-liang-2017-adversarial,mudrakarta-etal-2018-model,niven-kao-2019-probing}.
On the task side, real-world information-seeking problems such as web search, have different properties; e.g., they are mediated by
machines which provide relevant but incomplete and noisy
information to a \emph{human user}.
Computers and humans have different skills and different
resources~\citep{Xie:2002:PII:566014.566017,Marchionini:2006,joy-of-search}.
Humans can synthesize discrete pieces of information, create
long-range strategies, and understand nuanced language.
Computers, in contrast, have nigh infinite memory and can take advantage of
subtle patterns in their inputs~\cite{wallace-19}.
To put humans and computers on an even playing field, we re-purpose the machine reading task as \emph{meta answering} (Section~\ref{sec:meta}). A \name{}, when presented with the output of a question answering system, must decide which (if any) of the top candidate answers is correct.
This task is characterized by imperfect information, since the QA system predictions are noisy. Information is also asymmetric: the \name{} and QA system have different views of the data. At the same time, the task is backward-compatible with respect to its original evaluation setup, hence, results are comparable.
We study how humans and computers perform, in both qualitative and quantitative terms, with an existing machine reading (MR) dataset---Natural
Questions~\citep{nq}---which embeds real users' information seeking needs in a realistic information retrieval context.
Our comparison recapitulates the divergent skills of humans and machines.
Humans do better when provided with more information (Section~\ref{sec:human}) because they discern whether
answer candidates are responsive to the question, can solve ambiguous
references, and can spot irrelevant distractors that can vex brittle
QA systems.
In contrast, the machine \name{} is overall better at the task
(Section~\ref{sec:exp}), besting a BERT~\citep{devlin2018bert} reader by two F1 points.
It falters, however, when provided with
richer contexts.
Thus, meta answering surfaces limitations of NLU and provides insights on different processing strategies: machines are risk-averse, while humans seem naturally inclined to explore. Section~\ref{sec:conc} discusses how synergies between them could further improve NLU in information-seeking tasks.
\section{From Question Answering to Meta-answering}
\label{sec:meta}
A \name{} takes as input the highest scoring answers of an extractive QA system for a given question. The \name{} then tries to improve on the underlying system's result by selecting an answer that possibly differs from the system's top-scoring one.
Formally, an extractive QA system maps a question-context pair $\langle q, d\rangle$
to a set of $M$ answer candidates and their scores, $\{a_{i}, \sigma_{i}\}_{i=1}^M$;
all $a_{i}$ are subspans of $d$.
In our experiments we use Natural Questions~\citep{nq}, where $q$ is a web search query, $d$ is the highest
ranking Wikipedia page returned by Google for $q$, and each answer candidate is a short
span from $d$.
In web search, a user sees less information, the search results only, compared to the search engine that uses the full document collection.
Similarly, a \name{} has access to strictly less information than the underlying QA system and
lacks direct access to $d$. Its goal is to find the best answer $\hat{a}$ within QA's \nbest{} candidate list.
Again, this resembles a human confronted with a search results composed of links and tiny snippets. Analogously, the QA system returns `snippets' of $d$ centered on the answer
candidates: a window of $K$ tokens to either side of the answer context.
For example, given the question ``who did the 49ers beat in the super bowls'', the \name{} must
decide whether the candidate answer "San Diego Chargers" is the right answer, given the context ``The 49ers steamrolled the \underline{San Diego Chargers} 49 -- 26 , at''.
One of the issues we aim to evaluate is the importance of this additional context for humans and machines.
%In the next section we show that humans can successfully deal with the meta-answering task.
\begin{table*}[t]
\centering
\begin{tabular}{lrrrrrrrrr}
\toprule
& \multicolumn{3}{c}{\verminusone{}}& \multicolumn{3}{c}{\verzero{}} & \multicolumn{3}{c}{\verthree{}} \\
& Pr. & Rec. & F1 & Pr. & Rec. & F1& Pr. & Rec. & F1 \\
\midrule
NQ annotator & {\bf 57.9}& 46.4& {\bf 51.5}& {\bf 64.2} & 51.6 & {\bf 57.2} & 57.7 & 45.5 & 50.9\\
\bert{}~\citep{bert-baseline} &56.4& 45.7 & 50.5& 60.1 & 47.2 & 52.9 & {\bf 67.2}& 56.4 & 61.4 \\
avg. Human & 40.7 & 41.1 & 40.6& 48.9 & 50.9 & 49.8 & 54.5 & 56.7 & 55.5\\
best Human & 39.4 & {\bf 48.8} & 43.6& 52.7 & {\bf 59.1} & 55.7& 60.1 & {\bf 66.8} &{\bf 63.3}\\
$\hookrightarrow$ vs. NQ annotator & -18.5 & 2.4 & -7.9& -11.5 & 7.5 & -1.5 & 2.4 & 21.3 & 12.4 \\
$\hookrightarrow$ vs. \texttt{BERT} & -17.1 & 3.1 & -6.9& -7.5&11.9&2.8&-7.1 &10.4& 1.9 \\
\bottomrule
\end{tabular}
\caption{A boostrap evaluation to fairly compare original NQ annotators, human \name{}s, and the baseline QA system. In the context with the richest interaction, the best human \name{} can improve over both the average NQ annotator and the baseline QA system, mostly through improving recall.}
\label{table:string_match_results}
\end{table*}
\begin{table}[t]
\centering
\rowcolors{2}{gray!25}{white}
\begin{tabular}{p{2cm}cc}
& Answerable & Unanswerable \\
\toprule
{\bf Answer Correctly} & \textit{Right} & N/A \\
{\bf Answer Wrong} & Neg & Fool \\
{\bf No Answer} & Dead & \textit{Abstain} \\
\bottomrule
\end{tabular}
\caption{Ontology for \name{} outcomes based on whether a question is answerable. Right and Abstain are ``good'' outcomes for \name{}, while the other outcomes are incorrect. Humans often provide answers when they should not.}
\label{tab:ontology}
\end{table}
\section{Humans as \name{}s}
\label{sec:human}
To better understand the task and provide a benchmark, we place humans in the role of the \name{}.
Like the machine \name{} in the next section, they see the top candidates from a strong \bert{}-based~\cite{alberti-etal-2019-synthetic} system's \nbest{} list and must select which (if any) to provide as the final answer. On one hand, humans have world knowledge that computers do not.
However, they are restricted to the same context as computers; they may instead be burdened by their innate knowledge rather than aided.
This section examines how well human \name{}s can find correct answers in settings with increasing complexity and information; a subset of these settings also correspond to those of our machine \name{}.
{\bf \verminusone{}} shows only the question and candidate answers \emph{without context}; the \name{} needs to decide whether \underline{Central Germany} is a reasonable answer to ``what culture region is Germany a part of''.
{\bf \verzero{}} adds surrounding context: is ``Charles Osgood as the Narrator \underline{Jesse McCartney} as JoJo, the Mayor's son'' a good answer to ``who is JoJo in \textit{Horton Hears a Who}''.
This is identical to the information the machine \name{} uses in the next section.
{\bf \verthree{}} goes beyond the machine \name{}: users can ask questions to two other QA systems: \citet{lee-etal-2019-latent} over all of Wikipedia or ask the system that generated the \nbest{} list a \emph{different} question over the NQ source page.
For example, the user asks ``Who did Jesse McCartney play in \textit{Horton Hears a Who}'' to verify (the system believes) he plays JoJo.
\subsection{Human Answering Framework}
A human \name{} interacts with the underlying QA system through a text-based interface (Figure~\ref{fig:interface} in Appendix).
They first see a prompt; they can then request an answer from the underlying QA system.
After requesting up to 20 examples, the user can either abstain or select one of the answer candidates.
For each condition, the same five human \name{}s (results are averages with error bars) play episodes for random samples of 100 questions from NQ dev questions.
In addition, in \verthree{}, the human \name{} can ask a different question as an action.
\subsection{Comparing Human \name{}s to NQ Annotators and the QA system}
\label{sec:human-vs-machine}
First, however, we need to discuss how to compare human \name{}s to a traditional QA system or to the humans who created the dataset.
NQ uses \emph{exact span} as an error metric,\footnote{Both human and computer \name{}s also cannot answer questions with disjoint spans; e.g., names from a cast or
binary yes/no questions.} however this does not work for human \name{}s: our interface shows a span out of context so humans do not know where the string came from.
Thus, selecting the correct answer at the wrong position is
still counted as a miss (e.g., \underline{Kevin Kline} at token 13 or 47 is wrong but \underline{Kevin Kline} at token 30 is correct).
To compare the accuracy of the QA system and original NQ annotators (both see the whole source document) with the humans operating on partial knowledge, we
compare both \emph{exact string} matches and partial string matches (\emph{surface F1} measuring token overlap).
The first question is whether our human \name{} are as good as the original NQ annotators (who see the whole document).
NQ provides five annotations of potential answers; and answer ``counts'' if two NQ annotators agree.
For a fair apples to apples comparison, we would need to hold out one NQ annotation and compare it to our human \name{}s.
Rather than rely on a deficient gold set, bootstrap sample
from the remaining four annotations to get back to five annotations.
But it would unfair to only evaluate NQ annotators using this, hence we evaluate our humans and the baseline \bert{} QA system using the same metric (a bootstrap sample to get five annotations).
This is a fair comparison between the NQ annotator, our
human annotators, and the baseline QA system.
Human \name{}s have lower F1 than both the original annotators as well as the baseline QA system (Table~\ref{table:string_match_results}), in \verzero{} and especially \verminusone{}.
All humans improve between \verminusone{} and \verzero{}.
There is also an improvement from \verzero{} to \verthree{}, but formulating new questions also introduces more variance,\footnote{The highest agreement between raters is near 0.7 chance-adjusted $\kappa$, while agreement between the users and the system is typically between 0.5 and 0.6.} which is a function of skill.
The better human \name{}s improve over the baseline system.
The best human (consistent across all conditions) was $-7.9$ worse on \fone{} in the \verminusone{}, the best human \name{} has a higher \fone{} than the average NQ annotator in \verthree{} with substantially less information.
\subsection{A Taxonomy of Outcomes}
\begin{figure}[t]
\centering
\includegraphics[width=1.0\linewidth]{2020_emnlp_metaanswer/figures/user_accuracy}
\caption{Humans improve over the underlying QA as they see more information. they abstain less, but this is often balanced out by being ``fooled'' (tricked into answering questions where they should abstain). Numbers not comparable to bootstrap evaluation in Table~\ref{table:string_match_results}.}
\label{fig:improve_over_qa}
\end{figure}
The human \name{}s improve recall far more than precision.
To better understand where these gains come from, it's helpful to go beyond these broad outcomes.
To see why human \name{}s have better recall, we describe possible outcomes of a \name{} (Table~\ref{tab:ontology}).
\paragraph{\bf Right and Neg}
The clearest result is that the \name{} selects a \emph{right} answer from the QA system and provides it.
This can either be confirming the answer that the QA system would have presented, answering when the QA system would have abstained, or selecting a different answer.
The most human \name{} improvement is to answer instead of abstaining: \underline{1967} to the question ``when did colour tv come out in uk'', the answer is at the top of the baseline QA's $n$-best list, but below the threshold.
Sometimes humans select a wrong answer when an answer is available.
We call this a negative selection, or ``neg'' for short.
For example, the question ``when did the crucifix become the symbol of christianity'' has gold answers \underline{the 4th century}, \underline{in the 2nd century}, and \underline{4th century}.
A human \name{} selects \underline{the 2nd century}, which was not an acceptable answer.
\paragraph{\bf Abstain}
Many NQ instances cannot be answered\footnote{E.g., ``universal social services are provided to members of society based on their income or means'' is not a question, the NQ source provides an article about Paralympics for an Olympics question, or no Wikipedia page answers ``where am i on the steelers waiting list''.} the next most common outcome is for a \name{} to correctly recognize it should not provide a response.
The QA system abstains more than humans, which leads to human \name{}s biggest failing\dots
\paragraph{\bf Fool}
The flipside is being ``fooled'' into answering instead of abstaining; for example answering \underline{3 September} to the question ``when did the united kingdom entered world war 2'': humans are enticed by the context ``after a British ultimatum''.
While humans have a much higher rate of being fooled than computers (humans are reluctant to say ``I don't know''), some of this is attributable to annotation problems with NQ.
\paragraph{\bf Dead}
Sometimes a question is answerable, but the \name{} falsely abstains.
We call this result a ``dead'' question.
Sometimes the \name{} is unsure of an answer. For example, for the question ``who was fighting in the civil war in england and about what'', the gold answer is ``Parliamentarians and Royalists over, principally, the manner of England's government''.
However, humans rejected that answer (perhaps as being too superficial).
Humans improve as we move to the \verthree{} setting, adding the ability to ask new questions.
Human \name{}s can improve recall by converting baseline abstentions into right answers (and a smaller number of baseline negs). However, humans are more often fooled, resulting in lower average precision (Figure~\ref{fig:improve_over_qa}).
Because the human \name{} is at the mercy of the baseline QA system, if the baseline system does not surface the answer, the question will go dead without the ability of the human to find the answer.
\subsection{Human Strategies}
Without contexts in the \verminusone{} setting, humans are limited to examining the question answer combination (e.g., knowing that ``Germany'' is not a part of \underline{Central Germany} or that \underline{Jennifer} is not the ``meaning of the name Sinead'').
However, these cases are rare enough that human \name{}s do not overall improve the system.
With \verzero{}, human \name{}s use context to select better answers than the system.
For example, seeing that \underline{near Arenosa Creek and Matagorda Bay} was settled by explorer Robert Cavelier de La Salle in the Wikipedia page context, allowing to convert a dead model question into a correct one.
With \verthree{}, humans can more thoroughly probe the source document to establish whether an answer is correct.
E.g., while the baseline system answers the question ``who plays eddie's father on blue bloods'' with \underline{Eric Laneuville}, humans can explore outside the source document to find the correct answer (William Sadler, who plays Armin Janko, only appears in the sixth season, while the NQ document is about an earlier season) or within the page to establish that Eric Laneuville directed several episodes of \textit{Blue Bloods}.
\begin{table*}[t]
\centering
\includegraphics[width=1.0\linewidth]{2020_emnlp_metaanswer/figures/ma_embedding}
\caption{Our \name{} has three primary components to determine answers, \history{} sequences, and impossibility. Each uses a representation from a BERT representation that distinguishes (Segment embedding) the Wikipedia page that contains the answer, the answer candidate, and \history{} that encodes alternative candidates as well as the internal structure of the answer and \history{} (Segment Embedding). The feature embedding layer captures the strength of the original QA model's prediction.} \label{table:state_representation}
\end{table*}
\section{Machines as \name{}}
\label{sec:machine}
Like the humans, machine \name{}s inspect and evaluate candidates, based on the available context, and then provide an answer (or abstain).
The most salient difference from traditional machine reading systems is that the \name{} does not have access to the entire text---it must choose from discrete pieces of \history{} to justify which (if any) answer to return.
We break the task into three subtasks with their own losses: an answer candidate selector~$P_A$, an \history{} selector~$P_H$ to decide which evidence to pick,
and an impossibility classifier~$P_I$ to decide whether to answer at all.
The {\bf \history{}} are the candidate answers and the surrounding context around the answer (as with the human \name{} in Section~\ref{sec:human}).
While humans can skim over other answers and their contexts to use salient hints, we force the computer to explicitly build a set of \history{}.
\subsection{System architecture}
Our classifiers are output layers on top of a BERT encoder~\citep{devlin2018bert} which translates a semi-structured input into a single dense vector.
The $P_A(\cdot \mid \inputa{})$ is
the probability that candidate~$a$ is a correct answer to question~$q$, assuming that the (unobserved)
Wikipedia page from which $a$ is extracted has title~$t$, and an \history{} set of hints~$h = \langle o_{1}, \dots, o_{K} \rangle$;
these inputs are encoded into a dense vector using a BERT model parameterized via $\bs{W}_E$
\begin{equation}
\bs{e}_{A}^{(\clstok)} = \textsc{BERT}_{\bs{W}_E}(\bs{E}(\inputa{}))
\label{eq:inputa}
\end{equation}
and then becomes a probability as the output of the feed-forward neural network (FFNN):
\begin{equation}
P_{A}(\cdot \mid \inputa{}) = \textsc{FFNN}_{\bs{W}_A}(\bs{e}_{A}^{(\clstok)}).
\label{eq:pa}
\end{equation}
The individual hints in the \history{} set are chosen by $P_H$ as `relevant' for answering through a similar process.
The inputs to evaluate a piece of \history{} are the inputs to evaluate a candidate except there is no answer~$a_i$, as the goal is for the \history{} set to be useful for \emph{all} candidates.
Given this input of question, title, and set of \history{}~$h$, we create a dense representation
\begin{equation}
\bs{e}_{H}^{(\clstok)} = \textsc{BERT}(\bs{E}(\inputh{})),
\label{eq:inputb}
\end{equation}
which, as above, is turned into a probability via a feed-forward network,
\begin{equation}
P_{H}(\cdot \mid \inputh{}) = \textsc{FFNN}_{\bs{W}_H}(\bs{e}_{H}^{(\clstok)}).
\end{equation}
\subsection{Semi-structured Embeddings}
\label{sect:sse}
An effective input representation for \bert{} must encode not just individual words but also the role those words play.
In Equation~\ref{eq:inputa} and~\ref{eq:inputb}, we differentiate the words in the context around an answer and the title of a source page.
In addition to an embedding matrix for text tokens~$\bs{E^{T}}$, we use a top-level segment embedding to distinguish questions~$Q$, title~$T$, answer candidates~$A$, and \history{}~$O$ (row $S$ in Table~\ref{table:state_representation}).
There is a finer distinction within the answer candidates and \history{}.
We need to separate the context around an answer candidate and the candidate itself (e.g., distinguishing the token ``Nigel'' from its context in ``he voiced \underline{Nigel} in Rio'').
We use sub-segment embedding~$\bs{E^{S'}}$ to distinguish the answer span~$a$ and its surrounding left ($c^{(l)}$) and right ($c^{(r)}$) context, within each primary segment type (either Answer~$A$ or \History{}~$O$).
To the text inputs we concatenate the score of the candidate~$\sigma_i$ (row $f$ in Table~\ref{table:state_representation}) and finally the position embeddings $\bs{E^{P}}$:
\begin{equation}
\bs{E}_{:,i} = (\bs{E^{T}}_{:,T_{i}}, \bs{E^{S}}_{:,S_{i}},
\bs{E^{S'}}_{:, S'_{i}}, F_{i}\times \bs{e^{f}}, \bs{E^{P}}_{:,i}).
\end{equation}
When encoding inputs that should not contain the answers (see below) we mask the respective tokens.
\subsection{Answer Candidate Selector}
\label{sec:training}
We train $P_{A}(\cdot \mid \inputa{})$ with a binary cross-entropy loss, assuming examples of the form
$\langle y, q, t, o, h\rangle$, where $q$ is a question, $t$ a page title, $o=(c_i^{(l)},a_i,c_i^{(r)})$ a candidate
answer in context, $h$ the \history{}, and a binary label~$y$ indicating whether the answer is a correct answer.
We train this classifier using the \nbest{} list from \citet{bert-baseline}.
Because most candidates are wrong, we downsample negative examples.\footnote{Different negatives are picked at each epoch so that in expectation all negative examples are seen.}
Selecting the \history{} sequence is less straightforward; we describe this in the next section.
For the moment, however, let us assume that we have an \history{} sequence~$h$ for each question.
Then for a $\mathbb{D}$, the answer selector loss $\mathcal{L}_{A,\mathbb{D}}(\bs{W}_{A}, \bs{W}_{E})$ is
\begin{equation}
\sum_{\langle y, q, t, o, h\rangle \in \mathbb{D}} -\log P_{A}(y \mid \inputa{}; \bs{W}_{A}, \bs{W}_{E}). \notag
\end{equation}
\subsection{\History{} Selector}
The \history{} selector uses $P_{H}(\,\cdot \mid \inputh{})$---the probability that $h$ is a useful sequence
of observations to evaluate answers for question~$q$ and title~$t$ using $P_{A}$---to pick, among a set
of candidate \history{}s $\mathcal{H}$, the sequence~$h$ of length $M$ that scores the highest.
Training $P_H$ requires knowing the answer because we want it to score good \history{} sequences higher.
But we don't want $P_H$ to directly depend on the answer.
To create an implicit training signal, we induce a pseudo-label that does depend on the answer that is 1 whenever $h'$ provides better \history{} for determining the correct answer $d(P_A, q, t, o, y, h, h') \equiv$
\begin{equation}
\label{eq:pseudo-label}
\ind{P_{A}(y\mid q, t, o, h) < P_{A}(y\mid q, t, o, h') }
\end{equation}
For readability, we abbreviate Equation~\ref{eq:pseudo-label} as $d(P_A,h,h')$.
Given this pseudo-label for training, we train $P_H$ with the answer candidate~$a_{i}$ masked (Table~\ref{table:state_representation}) to allow $P_H$ to discover useful properties of the \history{} to find the correct answer.
For
a pair of candidate evidences $h$ and $h'$, this induces a \history{} loss $\mathcal{L}_{H,\mathbb{D}}(\bs{W}_{H}, \bs{W}_{E})$ to train $P_H$,
\begin{align*}
& \sum_{\langle y, q, t, o, h\rangle \in \mathbb{D},h'}\!\!\!\!\!\!\!\!\!\!\!\left[-\log P_{H}(d(P_A,h,h')\mid q, t, h; \bs{W}_{H}, \bs{W}_{E}) \right. \notag \\
& \hphantom{\dots} \left. -\log P_{H}(d(P_A,h',h)\mid q, t, h'; \bs{W}_{H}, \bs{W}_{E})\right].
\end{align*}
We generate an alternate \history{}~$h'$ by randomly replacing one of the observations in $h$.\footnote{Decoding also selects $h, h'$ with unit Hamming distance.}
The \history{} loss thus maximizes the expected reduction in entropy for $P_A$ and $q$ produced by substituting $h$ with $h'$.
While the full expectation would require summing over all possible answers for $q$, training on a single answer defines an unbiased (although noisy) estimate of this expectation.
The loss and, consequently, the training signal for $P_H$ implicitly depends on
a $P_A$.
However, $P_A$ and $P_H$ can be co-trained from scratch.
\subsection{Auxiliary impossibility loss}
51\% of the questions in the NQ dataset are `unanswerable', i.e., there is no gold answer in the context. As demonstrated in the human \name{} experiments, learning to abstain is crucial for doing well on NQ.
To help our model detect unanswerable questions, we add an additional impossibility classifier~$P_I$.
The impossibility classifier is a single feed-forward layer on top of $\bs{e}^{\clstok}_A$ (Equation~\ref{eq:inputa}) with loss $\mathcal{L}_{I,\mathbb{D'}}(\bs{W}_{I}, \bs{W}_{E}) =$
\begin{equation}
\sum_{\langle q, t, b, o, h\rangle \in \mathbb{D'}} -\log P_{I}(b\mid q, t, o, h; \Theta_{I}, \Theta_{E}),
\end{equation}
where $b$ is a binary label that is $1$ for answerable examples and $0$ otherwise.
\subsection{Training}
We initialize \bert{} from the public checkpoint. Because \bert{} is most effective with masked language model pre-training (MLM),
we ensure our encoder is used to the input described in Table~\ref{table:state_representation} using examples
in $\mathbb{D}$ to compute input sequences and randomly masking 30 tokens in each. We run the pre-training
for 200,000 steps, using a batch size of 32.
Our pre-training produces better models, as does cotraining $P_A$, $P_H$, $P_I$ and MLM: masking one random token from every input.
We combine the four losses into a single weighted loss and treat the per-loss weights
as hyper-parameters:
\begin{equation*}
\mathcal{L} = \lambda_{A}\mathcal{L}_{A} + \lambda_{H}\mathcal{L}_{H} + \lambda_{I}\mathcal{L}_{I} + \lambda_{MLM}\mathcal{L}_{MLM}
\end{equation*}
\paragraph{Making predictions}
\begin{algorithm}[t]
\SetKwFunction{GetCandidates}{GetCandidates}
\SetKwFunction{ComputeAnswerScores}{ComputeAnswerScores}
\SetKwFunction{MaxScoreAnswer}{MaxScoreAnswer}
\SetAlgoLined
\KwIn{$q, t, \mathrm{QA}, M$}
\Parameter{$k,P_A,P_H$}
\KwOut{$\vec{s}_{a_i}$}
$\mathcal{A} \leftarrow$ \GetCandidates{$q,t, \mathrm{QA}, M$}\;
$\displaystyle \evh_{1:k}\leftarrow \mathcal{A}_{1:k}$\;
\For{$t=k+1$ \KwTo $M$}{
$i \leftarrow \argmax_{i\in \{1,\dots,k\}} P_H(\mathrm{good} \g q, t, h_{i/t})$\;
\If{$P_H(\mathrm{good} \g q, t, h_{i/t})>P_H(\mathrm{good} \g q, t, h)$}{
$h_i\leftarrow h_t$\;
}
}
\For{$i=1$ \KwTo $M$}{
$\vec{s}_{a_i}[i] \leftarrow P_A(\mathrm{correct} \g q, t, a_i, h) $\\
}
\caption{Scoring algorithm for the \name{} paired with MR system QA.}
\label{alg:greedy}
\end{algorithm}
We now describe
how, at test time, we generate answer predictions.
Algorithm~\ref{alg:greedy} uses $P_A$ and $P_H$ to serve as a \name{} on top of an existing QA system.
It first builds a size-$k$ \history{} set from the top-$k$ answer candidates
from the QA system. It then iterates over the remaining $M-k$ candidates using $P_H$ to greedily decide whether to replace observations in the \history{}. For example, $h_{i/j}$ is the \history{} when the $i^{th}$ element of $h$ has been replaced with observation $o_j$.
Once all $M$ candidates have been processed and a $k$-size history $h$ has been selected, we score
all candidates using $P_{A}$ and return their scores.
We do not use $P_I$ at test time.\footnote{Nevertheless, co-training $P_I$ is beneficial as this auxiliary training loss helps create representations that better capture whether a question is impossible.}
Following \citet{alberti-etal-2019-synthetic}, we always predict the best answer if the score is above a threshold picked on development data.
%
\begin{table*}[t!]
\begin{center}
\begin{tabular}{lcccccc}
\toprule
\multirow{2}{*}{\hphantom{\dots\dots\dots\dots\dots\dots} System} &\multicolumn{3}{c}{Short Answer Dev}&\multicolumn{3}{c}{Short Answer Test}\\
&P&R&Span F1&P&R&Span F1\\\midrule
Single annotator~\citep{nq} & 63.4 & 52.6 & 57.5 & - & - & - \\
\texttt{BERT}~\citep{bert-baseline}& 59.5 & 47.3 & 52.7 & 63.8 & 44.0 & 52.1\\
\texttt{BERT\textsubscript{WWM}} & 59.5 & 51.9 & 55.4 & 63.1 & 51.4 & 56.6 \\
\texttt{MMA\textsubscript{Base}} &56.7&49.5&52.9&55.5&49.4&52.3\\
\texttt{MMA}$_{\verminusone{}}$ & 63.1 & \textbf{54.3} & 58.4 & \textbf{67.3} & 52.4 & \textbf{58.9} \\
\texttt{MMA}$_{\verzero{}}$ & \textbf{64.5} & 53.5 & \textbf{58.5} & 66.2 & \textbf{52.7} & 58.7 \\
\bottomrule
\end{tabular}
\end{center}
\caption{Results on the Short Answer task of the Natural Questions dataset.}\label{tab:results}
\end{table*}
\section{Machine Experiments}
\label{sec:exp}
\subsection{Machine Meta Answering systems}
We evaluate three versions of the machine \name{}s.
The first, \texttt{MMA\textsubscript{Base}}, is a baseline to see if a model can pick out the correct answer without context and minimal textual evidence. For each $a_i$ in the \nbest{} list of \texttt{BERT\textsubscript{WWM}}, we encode an observation as $(a_i,q,a_1,a_2,\ldots,a_M)$. We annotate $a_i$ with segment id “A”, the rest with “B”. Answers and question are separated by [SEP] tokens. Overflowing text is truncated from the end. On the last \clstok vector we add a binary classifier, corresponding to $P_A$ in Equation~(\ref{eq:pa}), trained to identify the gold answer labels for the candidate $a_i$. \texttt{MMA\textsubscript{Base}} is a vanilla BERT implementation; i.e., without the semi-structured embeddings corresponding to the top two rows of Table~\ref{table:state_representation}, without evidence selection and multiple losses described in Sections~(4.1-4.6).
We then focus on two full versions of the machine \name{}s, \texttt{MMA}$_{\verminusone{}{}}$ and \texttt{MMA}$_{\verzero{}}$. As in the human experiments, the former can only see the text of the candidate answer, while the latter can also see a snippet of five tokens to each side of the answer.
All losses are trained jointly for 200,000 steps, initializing the encoder with our
custom pre-trained BERT large model and randomly initializing $\bs{W}_{A}, \bs{W}_{I}$ and $\bs{W}_{H}$.
After hyper-parameter search, for \texttt{MMA}$_{\verminusone{}{}}$ we set the $M$-best list size $M=5$, evidence size $k=3$, and $w_{A}=3, w_{H}=0.1, w_{I}=10, w_{MLM}=0$, for the weighted training loss. For \texttt{MMA}$_{\verzero{}}$ we set $M=4$, $k=3$, $w_{A}=3, w_{H}=10, w_{I}=3, w_{MLM}=1$.
\subsection{Results}
Full results on the NQ short answer task are reported in Table~\ref{tab:results}.
We take as reference \citet{bert-baseline}'s QA model trained on NQ and
refer to it as \texttt{BERT}. Adding whole word masking~\citep{roberta_paper} adds
4.5 \fone{} points, and we use this \texttt{BERT\textsubscript{WWM}} as the
QA system. The baseline, \texttt{MMA\textsubscript{Base}}, is worse than \texttt{BERT\textsubscript{WWM}}. This shows that a na\"ive approach to encode evidential information is not comparable with the original answer context. The more careful handling of evidence, embeddings, and multiple losses lead the full \texttt{MMA} systems to outperforming \texttt{MMA\textsubscript{Base}} by more than 6 F1 points on test. They also improve over \texttt{BERT\textsubscript{WWM}} by about 2 F1 points. Interestingly, \texttt{MMA}$_{\verminusone{}}$ outperforms \texttt{MMA}$_{\verzero{}}$ on test, particularly on precision (see Appendix~\ref{sec:analysis} for breakdown), while the latter has slightly better recall.
This is counter intuitive, from a human perspective, as humans are better with more information.
Together with the additional finding that this is flipped on dev---\texttt{MMA}$_{\verzero}$ has higher precision and F1 than \texttt{MMA}$_{\verminusone}$---indicates that more work is needed to properly model context.
\subsection{Comparing Computers and Humans}
In both cases, the \name{} works with a stream of answer candidates and tries to improve the top prediction.
The computer \name{} slightly improves both precision and recall. Qualitatively, the changes are minor: tweaking a span, adding a word, favoring earlier spans, improving the abstention threshold.
Human \name{}s, however, were more bold. This allows them to greatly improve recall, digging deep in the \nbest{} list to find the answers they believe best answer the question. This exploratory behavior comes at the cost of lower precision; they often get fooled by plausible sounding answers that the NQ annotators did not agree with.
Interestingly, while humans do better when given more context, machine \name{}s did not. We suspect that the models are still not rich enough for the computer to reason with using the augmented evidence provided in the contexts.
\section{Related Work}
\label{sec:related}
The work carried out in the early 2000s in the context of the TREC conference,\footnote{\url{https://trec.nist.gov/}.}, QA track~\cite{voorhees-00} is relevant here. The evaluation of the evidence (candidate answers and their context) is an important aspect of meta answering. The TREC QA framework envisions accounting for answer support as a key part of the task~\citep{voorhees-03} and successful systems relied crucially on notions of “answer justification”~\citep{Harabagiu00falcon:boosting}. A substantial difference\footnote{Besides the evolution of the machine learning paradigm, and associated size requirements for datasets.} is that a \name{} has no way of affecting the production of evidence (the retrieval step) which is provided by the reader. Also relevant from this body of work, \citet{pizzato-molla-2005-extracting} propose a metasearch QA framework, which relies on several Web search and QA systems.
Another relevant line of research involves query reformulation, a popular method among search users~\citep{jansen2009}, mastered also by children~\citep{rutter-etal-2015}, which often relies on understanding and reusing the search results context~\citep{huang2009}. Automatic query reformulation has been applied with some success to machine reading.
In \citet{Nogueira:2017} and \citet{aqa-iclr:2018}, reinforcement learning (RL) trained agents seek good answers while learning to reformulate questions.
\citet{das2018multistep} propose to perform query reformulation in embedding (continuous) space, and find that it can outperform the surface language reformulations of~\citet{aqa-iclr:2018}.
We do not consider reformulation here, although it may be a promising direction to explore. More generally, a \name{} differs from the methods discussed above in that it does not try to change the candidate list content.In this sense, it is akin to evaluating search results by scrolling down a static result list.
A practical solution to the limited encoding capacity of BERT is to summarize documents. \citet{han-etal-2019-episodic} propose an episodic reader that summarizes using RL. \citet{nishida-etal-2019-answering} combine answering and summarization to justify answers, which is effective when explanations are annotated~\citep{yang-etal-2018-hotpotqa,thorne-etal-2018-fact}. Yet, there is no evidence that summarization-based readers are competitive in major MR tasks.
Similarly related is skim-reading~\citep{seo2018neural, hansen2018neural}, where documents are only partially encoded for prediction. Skip actions are not supervised and systems are trained with RL. The main advantage is, again, efficiency.
\citet{yuan2019imrc} extend MR by re-purposing existing datasets such as SQuAD and NewsQA~\citep{trischler-etal-2017-newsqa} for a partially-observed, incremental, QA task. Here an agent is trained with RL for information-gathering and answering actions However, the performance of such agents is far from competitive.
%This is an important aspect of language-based information-seeking tasks.
\section{Conclusion}
\label{sec:conc}
Meta-answering is a framework for QA that attempts to simulate real-world---imperfect---information-seeking tasks, where humans look for answers in settings mediated by machines, using natural language.
Human \name{}s can compete with a BERT-based single system with access to full documents, by only looking at a five token window around candidates.
A machine \name{} built on BERT can improve the environment's QA system, thus proving that it is possible to investigate MR in imperfect information settings in high-performance regimes.
Further, the task brings to the surface, yet again but from a novel perspective, limitations of the current NLU paradigm.
MMA cannot use the contextual information that is effortlessly exploited by humans. Thus, it might prove a suitable framework to advance on these challenges.
For future work, we plan to research the use of reinforcement learning for meta answering. The motivation is two-fold. First, the sequential aspect of information-gathering and answering lend itself naturally to episodic formalization characterized by delayed rewards. Secondly, information seeking tasks might offer challenging and realistic problems for RL.\footnote{Preliminary experiments with RL to train a MMA with advantage actor critic~\citep{pmlr-v80-espeholt18a} failed to generalize beyond the trivial policy of copying the top prediction.} As rewards are sparse, credit assignment is hard.
Language and cognition could help boost progress on more natural learning algorithms (cf.~\citet{Hung2019}), for example using imitation learning from human \name{}s to teach computers more intricate answering and verification strategies. Together, this may allow computers to combine thoroughness with human flexibility and insight.
\section{Appendix}
\subsection{Differences between \texttt{MMA$_\verminusone$} and \texttt{BERT\textsubscript{WWM}}}
\label{sec:analysis}
Table~\ref{tab:analysis} breaks down the abstain and answer decisions for \texttt{MMA$_\verminusone$} and \texttt{BERT\textsubscript{WWM}}.
We find that \texttt{MMA$_\verminusone$} is a slightly more accurate abstainer, but that by and large, its tendency to answer
or abstain matches that of \texttt{BERT\textsubscript{WWM}}. However, it is a considerably more accurate answerer -- even though
it answers $38$ times fewer than \texttt{BERT\textsubscript{WWM}}, it increases the absolute number of correct answers
given to by $83$ and decreases the number of incorrect answers given by $121$, explaining both the higher precision
and recall for the NQ evaluation metric.
\begin{table}[ht!]
\centering
\resizebox{\columnwidth}{!}{
\begin{tabular}{|c|c|c|c|c|}
\hline
\multirow{2}{*}{} & \multicolumn{2}{c|}{ABSTAIN} & \multicolumn{2}{c|}{ANSWER}\tabularnewline
\cline{2-5} \cline{3-5} \cline{4-5} \cline{5-5}
& Correct & Incorrect & Correct & Incorrect\tabularnewline
\hline
\multirow{3}{*}{\texttt{MMA}$_{\verminusone}$} & 3803 & 1050 & 1879 & 1098\tabularnewline
\cline{2-5} \cline{3-5} \cline{4-5} \cline{5-5}
& \multicolumn{2}{c|}{78.36\%} & \multicolumn{2}{c|}{63.12\%}\tabularnewline
\cline{2-5} \cline{3-5} \cline{4-5} \cline{5-5}
& \multicolumn{2}{c|}{4853} & \multicolumn{2}{c|}{2977}\tabularnewline
\hline
\multirow{3}{*}{\texttt{BERT\textsubscript{WWM}}} & 3764 & 1051 & 1796 & 1219\tabularnewline
\cline{2-5} \cline{3-5} \cline{4-5} \cline{5-5}
& \multicolumn{2}{c|}{78.17\%} & \multicolumn{2}{c|}{59.57\%}\tabularnewline
\cline{2-5} \cline{3-5} \cline{4-5} \cline{5-5}
& \multicolumn{2}{c|}{4815} & \multicolumn{2}{c|}{3015}\tabularnewline
\hline
\multirow{2}{*}{Difference} & 39 & -1 & 83 & -121\tabularnewline
\cline{2-5} \cline{3-5} \cline{4-5} \cline{5-5}
& \multicolumn{2}{c|}{38} & \multicolumn{2}{c|}{-38}\tabularnewline
\hline
\end{tabular}
}
\caption{Overall accuracy for the original model and the agent, split by action type.}\label{tab:analysis}
\end{table}
\subsection{Qualitative analysis of differences between \texttt{MMA$_\verminusone$} and \texttt{BERT\textsubscript{WWM}}}
In this section, we will drill down into cases where the \texttt{MMA$_\verminusone$} answer has flipped from incorrect to correct, or vice-versa. Additionally, these cases are analyzed with respect to their nature, and grouped into three categories:
\begin{enumerate}
\item \emph{Entity change}: a chosen answer contains a different entity,
e.g. a different time-span, geographic name, or person. For the question:
\emph{When was the first time the Internet was used?} a correct answer: \textbf{1969} is chosen instead of the previously incorrect choice of \textbf{the 1980s}.
\item \emph{Different span/same entity}, e.g. for question: \emph{The vascular layer of the eye is the }A
different text span quoting \textbf{uvea} as an answer is chosen.
\item \emph{Incorrect span}: for a question the same entity in the same
text fragment is chosen, however, the original answer provides fewer or more details judged unnecessary for the answer by the raters, e.g. for the question:
\emph{Who is Tinker Air Force Base named after? }the span: \textbf{Oklahoma native Major General Clarence L. Tinker} is chosen instead of \textbf{Major General Clarence L. Tinker}.
\end{enumerate}
Statistics of the answer changes flips from incorrect to correct ("Loss to Win"), and from correct to incorrect ("Win to Loss") are listed in Table~\ref{tab:analysis_qualitative}.
\begin{table}[h]
\centering
\begin{tabular}{|c|c|c|}
\hline
Modification &Loss to Win&Win to Loss\tabularnewline\hline
Entity Change &10 & 6\\
Different Span & 1 & 0 \\
Incorrect Span & 25 & 10\\\hline
\textbf{Overall} & 36 & 16\\
\hline
\end{tabular}
\caption{Classification of cases where the agent's answer changes correctness.}
\label{tab:analysis_qualitative}
\end{table}
Additionally, in case of Correct to Incorrect label change, we have found three answers to be incorrectly or doubtfully marked by the raters.
Moreover, among all of the cases (35 of them in total) where the algorithm has chosen to pick a different span with the original answer, in 31 of cases the span is shorter. That may indicate that the algorithm finds it more beneficial to avoid all of the unnecessary information from the answer, leading to an overall improvement for the answer statistics. In most of these cases, however, a human would probably judge both of the answers correct. For example, for question \emph{What class of ship is the Carnival Glory?} the original and deemed correct answer is \textbf{Conquest - class cruise ship} while the one picked by \texttt{MMA$_\verminusone$} - \textbf{Conquest - class} is labeled as incorrect by the raters.
\begin{figure*}
\includegraphics[width=\linewidth]{2020_emnlp_metaanswer/figures/interface}
\caption{Our interface for human \name{} experiments. Users see a question from NQ, see candidates from base QA system, and find their meta answer.}
\label{fig:interface}
\end{figure*}
%\end{figure*}
\begin{thebibliography}{36}
\expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi
\bibitem[{Alberti et~al.(2019{\natexlab{a}})Alberti, Andor, Pitler, Devlin, and
Collins}]{alberti-etal-2019-synthetic}
Chris Alberti, Daniel Andor, Emily Pitler, Jacob Devlin, and Michael Collins.
2019{\natexlab{a}}.
\newblock Synthetic {QA} corpora generation with roundtrip consistency.
\newblock In \emph{Proceedings of the 57th Annual Meeting of the Association
for Computational Linguistics}, pages 6168--6173.
\bibitem[{Alberti et~al.(2019{\natexlab{b}})Alberti, Lee, and
Collins}]{bert-baseline}
Chris Alberti, Kenton Lee, and Michael Collins. 2019{\natexlab{b}}.
\newblock A bert baseline for the natural questions.
\newblock \emph{https://arxiv.org/abs/1901.08634}.
\bibitem[{Buck et~al.(2018)Buck, Bulian, Ciaramita, Gesmundo, Houlsby,
Gajewski, and Wang}]{aqa-iclr:2018}
Christian Buck, Jannis Bulian, Massimiliano Ciaramita, Andrea Gesmundo, Neil
Houlsby, Wojciech Gajewski, and Wei Wang. 2018.
\newblock Ask the right questions: Active question reformulation with
reinforcement learning.
\newblock In \emph{Proceedings of ICLR}.
\bibitem[{Das et~al.(2019)Das, Dhuliawala, Zaheer, and
McCallum}]{das2018multistep}
Rajarshi Das, Shehzaad Dhuliawala, Manzil Zaheer, and Andrew McCallum. 2019.
\newblock \href {https://openreview.net/forum?id=HkfPSh05K7} {Multi-step
retriever-reader interaction for scalable open-domain question answering}.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Devlin et~al.(2018)Devlin, Chang, Lee, and
Toutanova}]{devlin2018bert}
Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. 2018.
\newblock Bert: Pre-training of deep bidirectional transformers for language
understanding.
\newblock \emph{arXiv preprint arXiv:1810.04805}.
\bibitem[{Espeholt et~al.(2018)Espeholt, Soyer, Munos, Simonyan, Mnih, Ward,
Doron, Firoiu, Harley, Dunning, Legg, and Kavukcuoglu}]{pmlr-v80-espeholt18a}
Lasse Espeholt, Hubert Soyer, Remi Munos, Karen Simonyan, Vlad Mnih, Tom Ward,
Yotam Doron, Vlad Firoiu, Tim Harley, Iain Dunning, Shane Legg, and Koray
Kavukcuoglu. 2018.
\newblock {IMPALA}: Scalable distributed deep-{RL} with importance weighted
actor-learner architectures.
\newblock In \emph{Proceedings of the 35th International Conference on Machine
Learning}, pages 1407--1416.
\bibitem[{Ferrucci et~al.(2010)Ferrucci, Brown, Chu-Carroll, Fan, Gondek,
Kalyanpur, Lally, Murdock, Nyberg, Prager, Schlaefer, and Welty}]{ferruci-10}
David Ferrucci, Eric Brown, Jennifer Chu-Carroll, James Fan, David Gondek,
Aditya~A. Kalyanpur, Adam Lally, J.~William Murdock, Eric Nyberg, John
Prager, Nico Schlaefer, and Chris Welty. 2010.
\newblock {Building Watson: An Overview of the DeepQA Project}.
\newblock \emph{AI Magazine}, 31(3).
\bibitem[{Han et~al.(2019)Han, Kang, Jung, and Hwang}]{han-etal-2019-episodic}
Moonsu Han, Minki Kang, Hyunwoo Jung, and Sung~Ju Hwang. 2019.
\newblock Episodic memory reader: Learning what to remember for question
answering from streaming data.
\newblock In \emph{Proceedings of the 57th Annual Meeting of the Association
for Computational Linguistics}, pages 4407--4417.
\bibitem[{Hansen et~al.(2019)Hansen, Hansen, Alstrup, Simonsen, and
Lioma}]{hansen2018neural}
Christian Hansen, Casper Hansen, Stephen Alstrup, Jakob~Grue Simonsen, and
Christina Lioma. 2019.
\newblock \href {https://openreview.net/forum?id=B1xf9jAqFQ} {Neural speed
reading with structural-jump-{LSTM}}.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Harabagiu et~al.(2000)Harabagiu, Moldovan, Pasca, Mihalcea, Surdeanu,
Bunescu, Girju, Rus, and Morarescu}]{Harabagiu00falcon:boosting}
Sanda Harabagiu, Dan Moldovan, Marius Pasca, Rada Mihalcea, Mihai Surdeanu,
Razvan Bunescu, Roxana Girju, Vasile Rus, and Paul Morarescu. 2000.
\newblock Falcon: Boosting knowledge for answer engines.
\newblock In \emph{TREC}, pages 479--488.
\bibitem[{Huang and Efthimiadis(2009)}]{huang2009}
Jeff Huang and Efthimis Efthimiadis. 2009.
\newblock Analyzing and evaluating query reformulation strategies in web search
logs.
\newblock In \emph{CIKM}, pages 77--86.
\bibitem[{Hung et~al.(2019)Hung, Lillicrap, Abramson, Wu, Mirza, Carnevale,
Ahuja, and Wayne}]{Hung2019}
Chia-Chun Hung, Timothy Lillicrap, Josh Abramson, Yan Wu, Mehdi Mirza, Federico
Carnevale, Arun Ahuja, and Greg Wayne. 2019.
\newblock Optimizing agent behavior over long time scales by transporting
value.
\newblock \emph{Nature Communications}, 10.
\bibitem[{Jansen et~al.(2009)Jansen, Booth, and Spink}]{jansen2009}
B.~J. Jansen, D.~L. Booth, and A.~Spink. 2009.
\newblock Patterns of query reformulation during web searching.
\newblock \emph{Journal of the American Society for Information Science and
Technology}, 60(7):1358--1371.
\bibitem[{Jia and Liang(2017)}]{jia-liang-2017-adversarial}
Robin Jia and Percy Liang. 2017.
\newblock Adversarial examples for evaluating reading comprehension systems.
\newblock In \emph{Proceedings of the 2017 Conference on Empirical Methods in
Natural Language Processing}, pages 2021--2031.
\bibitem[{Kwiatkowski et~al.(2019)Kwiatkowski, Palomaki, Redfield, Collins,
Parikh, Alberti, Epstein, Polosukhin, Kelcey, Devlin, Lee, Toutanova, Jones,
Chang, Dai, Uszkoreit, Le, and Petrov}]{nq}
Tom Kwiatkowski, Jennimaria Palomaki, Olivia Redfield, Michael Collins, Ankur
Parikh, Chris Alberti, Danielle Epstein, Illia Polosukhin, Matthew Kelcey,
Jacob Devlin, Kenton Lee, Kristina~N. Toutanova, Llion Jones, Ming-Wei Chang,
Andrew Dai, Jakob Uszkoreit, Quoc Le, and Slav Petrov. 2019.
\newblock Natural questions: a benchmark for question answering research.
\newblock \emph{Transactions of the Association of Computational Linguistics}.
\bibitem[{Lee et~al.(2019)Lee, Chang, and Toutanova}]{lee-etal-2019-latent}
Kenton Lee, Ming-Wei Chang, and Kristina Toutanova. 2019.
\newblock Latent retrieval for weakly supervised open domain question
answering.
\newblock In \emph{Proceedings of the 57th Annual Meeting of the Association
for Computational Linguistics}, pages 6086--6096.
\bibitem[{Liu et~al.(2019)Liu, Ott, Goyal, Du, Joshi, Chen, Levy, Lewis,
Zettlemoyer, and Stoyanov}]{roberta_paper}
Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer
Levy, Mike Lewis, Luke Zettlemoyer, and Veselin Stoyanov. 2019.
\newblock \href {http://arxiv.org/abs/1907.11692} {Roberta: {A} robustly
optimized {BERT} pretraining approach}.
\newblock \emph{CoRR}, abs/1907.11692.
\bibitem[{Marchionini(2006)}]{Marchionini:2006}
Gary Marchionini. 2006.
\newblock Exploratory search: From finding to understanding.
\newblock \emph{Commun. ACM}, 49(4).
\bibitem[{Mudrakarta et~al.(2018)Mudrakarta, Taly, Sundararajan, and
Dhamdhere}]{mudrakarta-etal-2018-model}
Pramod~Kaushik Mudrakarta, Ankur Taly, Mukund Sundararajan, and Kedar
Dhamdhere. 2018.
\newblock Did the model understand the question?
\newblock In \emph{Proceedings of the 56th Annual Meeting of the Association
for Computational Linguistics (Volume 1: Long Papers)}, pages 1896--1906.
\bibitem[{Nishida et~al.(2019)Nishida, Nishida, Nagata, Otsuka, Saito, Asano,
and Tomita}]{nishida-etal-2019-answering}
Kosuke Nishida, Kyosuke Nishida, Masaaki Nagata, Atsushi Otsuka, Itsumi Saito,
Hisako Asano, and Junji Tomita. 2019.
\newblock Answering while summarizing: Multi-task learning for multi-hop {QA}
with evidence extraction.
\newblock In \emph{Proceedings of the 57th Annual Meeting of the Association
for Computational Linguistics}, pages 2335--2345.
\bibitem[{Niven and Kao(2019)}]{niven-kao-2019-probing}
Timothy Niven and Hung-Yu Kao. 2019.
\newblock Probing neural network comprehension of natural language arguments.
\newblock In \emph{Proceedings of the 57th Annual Meeting of the Association
for Computational Linguistics}, pages 4658--4664.
\bibitem[{Nogueira and Cho(2017)}]{Nogueira:2017}
Rodrigo Nogueira and Kyunghyun Cho. 2017.
\newblock Task-oriented query reformulation with reinforcement learning.
\newblock In \emph{Proceedings of EMNLP}.
\bibitem[{Pizzato and Molla(2005)}]{pizzato-molla-2005-extracting}
Luiz~Augusto Pizzato and Diego Molla. 2005.
\newblock \href {https://www.aclweb.org/anthology/U05-1016} {Extracting exact
answers using a meta question answering system}.
\newblock In \emph{Proceedings of the Australasian Language Technology Workshop
2005}, pages 105--112.
\bibitem[{Rajpurkar et~al.(2018)Rajpurkar, Jia, and Liang}]{rajpurkar-18}
Pranav Rajpurkar, Robin Jia, and Percy Liang. 2018.
\newblock \href {https://doi.org/10.18653/v1/P18-2124} {Know what you don{'}t
know: Unanswerable questions for {SQ}u{AD}}.
\newblock In \emph{Proceedings of the Association for Computational
Linguistics}.
\bibitem[{Rajpurkar et~al.(2016)Rajpurkar, Zhang, Lopyrev, and
Liang}]{rajpurkar-etal-2016-squad}
Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, and Percy Liang. 2016.
\newblock {SQ}u{AD}: 100,000+ questions for machine comprehension of text.
\newblock In \emph{Proceedings of the 2016 Conference on Empirical Methods in
Natural Language Processing}, pages 2383--2392.
\bibitem[{Russell(2019)}]{joy-of-search}
Daniel~M. Russell. 2019.
\newblock \emph{{The Joy of Search: A Google Insider's Guide to Going Beyond
the Basics}}.
\newblock The MIT Press.
\bibitem[{Rutter et~al.(2015)Rutter, Ford, and Clough}]{rutter-etal-2015}
Sophie Rutter, Nigel Ford, and Paul Clough. 2015.
\newblock How do children reformulate their search queries?
\newblock \emph{Information Research}, 20(1).
\bibitem[{Seo et~al.(2018)Seo, Min, Farhadi, and Hajishirzi}]{seo2018neural}
Minjoon Seo, Sewon Min, Ali Farhadi, and Hannaneh Hajishirzi. 2018.
\newblock \href {https://openreview.net/forum?id=Sy-dQG-Rb} {Neural speed
reading via skim-{RNN}}.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Thorne et~al.(2018)Thorne, Vlachos, Cocarascu, Christodoulopoulos,
and Mittal}]{thorne-etal-2018-fact}
James Thorne, Andreas Vlachos, Oana Cocarascu, Christos Christodoulopoulos, and
Arpit Mittal. 2018.
\newblock The fact extraction and {VER}ification ({FEVER}) shared task.
\newblock In \emph{Proceedings of the First Workshop on Fact Extraction and
{VER}ification ({FEVER})}.
\bibitem[{Trischler et~al.(2017)Trischler, Wang, Yuan, Harris, Sordoni,
Bachman, and Suleman}]{trischler-etal-2017-newsqa}
Adam Trischler, Tong Wang, Xingdi Yuan, Justin Harris, Alessandro Sordoni,
Philip Bachman, and Kaheer Suleman. 2017.
\newblock {N}ews{QA}: A machine comprehension dataset.
\newblock In \emph{Proceedings of the 2nd Workshop on Representation Learning
for {NLP}}.
\bibitem[{Voorhees(2003)}]{voorhees-03}
Ellen~M. Voorhees. 2003.
\newblock \href {https://www.aclweb.org/anthology/N03-1034} {Evaluating the
evaluation: A case study using the {TREC} 2002 question answering track}.
\bibitem[{Voorhees and Tice(2000)}]{voorhees-00}
Ellen~M Voorhees and Dawn~M Tice. 2000.
\newblock Building a question answering test collection.
\bibitem[{Wallace et~al.(2019)Wallace, Rodriguez, Feng, Yamada, and
Boyd-Graber}]{wallace-19}
Eric Wallace, Pedro Rodriguez, Shi Feng, Ikuya Yamada, and Jordan Boyd-Graber.
2019.
\newblock Trick me if you can: Human-in-the-loop generation of adversarial
question answering examples.
\newblock \emph{Transactions of the Association of Computational Linguistics},
10.
\bibitem[{Xie(2002)}]{Xie:2002:PII:566014.566017}
Hong~(Iris) Xie. 2002.
\newblock Patterns between interactive intentions and information-seeking
strategies.
\newblock \emph{Inf. Process. Manage.}, 38(1):55--77.
\bibitem[{Yang et~al.(2018)Yang, Qi, Zhang, Bengio, Cohen, Salakhutdinov, and
Manning}]{yang-etal-2018-hotpotqa}
Zhilin Yang, Peng Qi, Saizheng Zhang, Yoshua Bengio, William Cohen, Ruslan
Salakhutdinov, and Christopher~D. Manning. 2018.
\newblock {H}otpot{QA}: A dataset for diverse, explainable multi-hop question
answering.
\newblock In \emph{Proceedings of the 2018 Conference on Empirical Methods in
Natural Language Processing}.
\bibitem[{Yuan et~al.(2019)Yuan, Fu, C\^ot\'{e}, Tay, Pal, and
Trischler}]{yuan2019imrc}
Xingdi Yuan, Jie Fu, Marc-Alexandre C\^ot\'{e}, Yi~Tay, Christopher Pal, and
Adam Trischler. 2019.
\newblock \href {http://arxiv.org/abs/1908.10449} {Interactive machine
comprehension with information seeking agents}.
\newblock \emph{CoRR}, abs/1908.10449.
\end{thebibliography}