Personal Entity, Concept, and Named Entity Linking in Conversations
Extracted source text · arXiv 2206.07836
%%
%% This is file `sample-sigconf.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigconf')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%%
%% Commands for TeXCount
%TC:macro~\cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf]{acmart}
\usepackage{hyperref}
%%%%% Table %%%%%
\usepackage{tabularx}
\usepackage{multirow}
\usepackage[usestackEOL]{stackengine} % Too use shortstack with a buffer
\usepackage{cellspace}
\setlength\cellspacetoplimit{4pt}
\setlength\cellspacebottomlimit{4pt}
\usepackage{fixltx2e}
\usepackage{pifont}
\newcommand{\cmark}{\ding{51}}
\usepackage{makecell}
\renewcommand\theadfont{}
%%%%% Fig %%%%%%
\usepackage{graphicx}
\graphicspath{ {./images/} }
\usepackage{pifont}
\usepackage{balance}
%% Item list indentation
\usepackage{enumitem}
\usepackage{makecell}
\usepackage{mathtools}
\usepackage{booktabs}
\newcommand{\hi}[1]{\textcolor{black}{#1}}
\newcommand{\fa}[1]{\textcolor{black}{#1}}
\newcommand{\todo}[1]{\textcolor{red}{#1}}
\newcommand{\note}[1]{\textcolor{brown}{#1}}
\newcommand{\tocheck}[1]{\textcolor{blue}{#1}}
\newcommand{\hicr}[1]{\textcolor{red}{#1}} % Hideaki Camera Review
\newcommand*\OK{\ding{51}}
\newcommand{\hjscalebox}{\scalebox{1.0}[1.0]}
\newcommand{\seconv}{S2E$_{\text{conv}}$}
\newcommand{\seconvminus}{S2E$^{-}_{\text{conv}}$}
%Shrinking
\newcommand{\miniskip}{\vspace*{-.9\baselineskip}}
\newcommand{\shrink}{\vspace*{-.9\baselineskip}}
% Vertically aligning textsuperscript and textsubscript together
% Example: \def\spsb#1#2{\rlap{\textsuperscript{#1}}\textsubscript{#2}}
\def\wast#1{\rlap{*}\textsubscript{#1}} % With asterisk
\def\spsb#1#2{\rlap{\textsuperscript{#1}}\textsubscript{#2}} % superscript and subscript
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
%% ========================= Common license from here =================================
%% Author: Download image from
%% https://urldefense.com/v3/__http://www.scomminc.com/pp/acmsig/cc_by4acm.png__;!!HJOPV4FYYWzcc1jazlU!94aCD-b18aEpf6sfjQQNk2dbSqns3k7cakN2-qGX8T32dH_KI0tDERyUe0KPzYHXRct8e4vEKS2l4_Us2W_1RuUR$
%% and place into a figures/figs folder related
%% to your .tex source files
% Overwrite copyright field outside from cls file.
\copyrightyear{2022}
\acmYear{2022}
\setcopyright{rightsretained}
%% These commands are specific for your submission.
\acmConference[CIKM '22]{Proceedings of the 31st ACM International Conference on Information and Knowledge Management}{October 17--21, 2022}{Atlanta, GA, USA}
\acmBooktitle{Proceedings of the 31st ACM Int'l Conference on Information and Knowledge Management (CIKM '22), Oct. 17--21, 2022, Atlanta, GA, USA}
\acmISBN{978-1-4503-9236-5/22/10}
\acmDOI{10.1145/3511808.3557667}
% Overwrite copyright field outside from cls file.
\usepackage{etoolbox}
\makeatletter
\patchcmd{\maketitle}{\@copyrightpermission}{
\begin{minipage}{0.3\columnwidth}
\href{http://creativecommons.org/licenses/by/4.0/}{\includegraphics[width=0.90\textwidth]{cc_by4acm.png}}
\end{minipage}\hfill
\begin{minipage}{0.7\columnwidth}
\href{http://creativecommons.org/licenses/by/4.0/}{This work is licensed under a Creative Commons Attribution International 4.0 License.}
\end{minipage}
\vspace{5pt}
}{}{}
\makeatother
%% ========================= Until here =================================
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{Personal Entity, Concept, and Named Entity Linking in Conversations} % NOTE: ``Resource:'' is needed to identify this is a resource type short paper. See here: https://www.cikm2022.org/calls/call-for-short-papers
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
% \author{Ben Trovato}
% \authornote{Both authors contributed equally to this research.}
% \email{trovato@corporation.com}
% \orcid{1234-5678-9012}
% \author{G.K.M. Tobin}
% \authornotemark[1]
% \email{webmaster@marysville-ohio.com}
% \affiliation{%
% \institution{Institute for Clarity in Documentation}
% \streetaddress{P.O. Box 1212}
% \city{Dublin}
% \state{Ohio}
% \country{USA}
% \postcode{43017-6221}
% }
\author{Hideaki Joko}
\affiliation{%
\institution{Radboud University}
\country{}
}
\email{hideaki.joko@ru.nl}
\author{Faegheh Hasibi}
\affiliation{%
\institution{Radboud University}
\country{}
}
\email{f.hasibi@cs.ru.nl}
%\author{Arjen P. de Vries}
%\affiliation{%
% \institution{Radboud University}
% \country{}
%}
%\email{a.devries@cs.ru.nl}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
%%
%% 1. To make the ACM headers consistent for this year's conference publication, kindly use the following for the even page headers:
\renewcommand{\shortauthors}{Hideaki Joko \& Faegheh Hasibi}
%% No italics
%% If needed use a foot or author note to identify equal contribution
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
Building conversational agents that can have natural and knowledge-grounded interactions with humans requires understanding user utterances.
% in conversational systems is important to satisfy users' needs.
Entity Linking (EL) is an effective technique for understanding natural language text and connecting it to external knowledge. It is, however, shown that the existing EL methods developed for annotating documents are suboptimal for conversations, where concepts and personal entities (e.g., ``my cars'') are essential for understanding user utterances.
In this paper, we introduce a collection and a tool for entity linking in conversations.
We provide EL annotations for 1,327 conversational utterances, consisting of links to named entities, concepts, and personal entities.
The dataset is used for training our toolkit for conversational entity linking, CREL. Unlike existing EL methods, CREL is developed to identify both named entities and concepts. It also utilizes coreference resolution techniques to identify personal entities and their references to the explicit entity mentions in the conversations. We compare CREL with state-of-the-art techniques and show that it outperforms all existing baselines.
% reported that traditional EL methods for documents are suboptimal for conversations, where concepts and personal entities (e.g., ``my neighborhood'' and ``my guitar'') are also important.
% In this paper, we develop an EL tool for conversations, that can link named entities, concepts, and personal entities.
% To this end, we collect EL annotations for conversations, which makes it possible to train the mention detection method to deal with both concepts and named entities.
% Additionally, we modify coreference resolution method to link personal entities, and analyze how much coreference resolution can help to deal with personal entities.
% Our EL tool (CREL) and annotated dataset (CEL) are provided as resources, allowing our community to further research on conversations.
% \note{NOTE: CEL? ConEL22? or another? The reason for the CEL is that it is short and easy to fit in the tables.}
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10002951.10003317.10003331</concept_id>
<concept_desc>Information systems~Users and interactive retrieval</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10002951.10003317.10003347.10003348</concept_id>
<concept_desc>Information systems~Question answering</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10002951.10003317.10003347.10003352</concept_id>
<concept_desc>Information systems~Information extraction</concept_desc>
<concept_significance>100</concept_significance>git
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Information systems~Users and interactive retrieval}
\ccsdesc[500]{Information systems~Question answering}
\ccsdesc[100]{Information systems~Information extraction}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{Entity Linking; Conversational System; Datasets}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\input{introduction}
\input{method}
\input{annotation}
\input{results}
\input{conclusion}
%%% The acknowledgments
%\begin{acks}
%To Robert, for the bagels and explaining CMYK and color spaces.
%\end{acks}
\balance
\bibliographystyle{ACM-Reference-Format}
\newpage
\bibliography{references-base}
\end{document}
\endinput
%%
%% End of file `sample-sigconf.tex'.
\section{Introduction}
\label{sec:introduction}
% Fig is here: https://docs.google.com/presentation/d/1LyvB0d8lFY6bVeus64txdMsPWupeRonwmLuMaeQ5KSo/edit?usp=sharing
\begin{figure}[t]
\includegraphics[width=1.0\linewidth]{example}
\caption{Example of entity linking in conversations.
% The green and gray speech bubbles represent user and system turns, respectively.
% In conversations, not only name entities, but concepts (e.g., ``tomatoes'') and personal entities (e.g., ``my neighborhood'') are important.
}
\label{fig:example}
\vspace{-1em}
\end{figure}
Understanding user utterances in conversational systems is fundamental to creating natural and informative interactions with users.
Entity Linking (EL) is a powerful text understanding technique that has been extensively explored in previous studies~\cite{Shang:2021:ERO, Joko:2021:CEL} and has been shown to be effective in question-answering and retrieval tasks~\cite{Yamada:2018:SOQ, Gerritse:2022:ETE, Dalton:2014:EQF, Hasibi:2016:EEL, Xiong:2017:WED, Dargahi:2018:QUE}.
However, the EL approaches developed for documents prove to be suboptimal for conversations, to the extent that the state-of-the-art EL models are outperformed by the outdated models when evaluated on conversational EL datasets~\cite{Joko:2021:CEL}. The reasons are three-fold:
%(i) no support for annotating personal entity mentions in conversations, (ii) limiting annotation to named entities instead of both concepts and named entities, (iii) training EL models on documents and well-written texts instead of conversations.
%
\textbf{(i) Personal Entity Annotation}: Conversations often contain personal statements, referring to entities by their pronouns; e.g., ``my cars.'' Identifying personal entities is essential for building personal knowledge graphs~\cite{Balog:2019:PKG, Li:2014:PKG, Tigunova:2019:LBL, Tigunova:2020:CHA} and also generating personalized responses to users~\cite{Joshi:2017:PGD, Luo:2019:LPE}. Existing approaches for annotating documents, or even conversations~\cite{Laskar:2022:BEE, Xu:2022:OCR, Shang:2021:ERO}, do not identify and link personal entity mentions.
\textbf{(ii) Concept Annotation}: EL models developed for documents or short texts focus mainly on annotating named entities~\cite{vanHulst:2020:REL, Cao:2021:GENRE}. In conversations, however, both named entities and concepts contribute to machine understanding of text~\cite{Joko:2021:CEL} and should be annotated.
\textbf{(iii) Training Data}: Existing EL approaches are trained on well-written texts, while conversations have multiple (informal) turns with information spread over the turns.
In this paper, we overcome the above challenges and provide a toolkit and a training dataset as two resources for EL in conversations. The challenges are addressed in the following ways.
%
\paragraph{Personal Entity Annotation}
We detect \emph{personal entity mentions} (e.g., ``my cars''), find their corresponding \emph{explicit entity mentions} (e.g., ``Life'' and ``Pilot''), and link them to the corresponding entities in the knowledge graph (see Figure~\ref{fig:example}).
This task and coreference resolution both connect and stand in contrast to each other: they both aim to find mentions referring to the same entity~\cite{Singh:2011:LCD, Lee:2018:HOC, Joshi:2019:BFC, Kirstain:2021:CRS}, but personal entity mentions are sparse and different from common pronominal expressions that are resolved in coreference resolution.
% An analogy can be drawn between this task and coreference resolution, as they both aim to find mentions referring to the same entity~\cite{Singh:2011:LCD, Lee:2018:HOC, Joshi:2019:BFC, Kirstain:2021:CRS}.
%Differently from each other, personal entity mentions are sparse and different from common pronominal expressions, there by making
% The two task, however, stand in contrast to each other: personal entity mentions are sparse and different from common pronominal expressions that are resolved in coreference resolution, which implies
This implies that state-of-the-art coreference resolution methods cannot be used out-of-box for resolving personal entities. We, therefore, identify personal entity mentions in a separate module and modify coreference resolution techniques to map personal entities to their explicit entity mentions.
\paragraph{Concept Annotation}
While open-domain knowledge graphs (e.g., Wikipedia) contain both named entities and concepts, conventional EL approaches are optimized for annotating only named entities, which hampers their performance for annotating conversations~\cite{vanHulst:2020:REL, Cao:2021:GENRE, Wu:2020:BLINK, Zhang:2022:ENT}.
Specifically, the performance of named entity and concept linking in the widely used REL entity linker~\cite{vanHulst:2020:REL} varies dramatically (F-score of 43.1 and 2.5, respectively).
We mitigate this problem by training a mention detection model that can identify both concepts and named entity mentions in conversations and optimize the entity disambiguation model of REL~\cite{vanHulst:2020:REL} for all entity types.
Our methods for personal entity, concept and named entity linking are integrated into our conversational EL tool, which is provided as a resource to the community. We show that this method outperforms state-of-the-art EL models.
\paragraph{Training Data}
A key to solving the aforementioned problems is training data. Existing studies on conversational entity linking either use private proprietary datasets~\cite{Shang:2021:ERO} or public datasets with insufficient number of conversations for both training and evaluation purposes~\cite{Joko:2021:CEL}.
In this work, we build on the existing ConEL dataset~\cite{Joko:2021:CEL} and address its shortcomings in our new dataset: (i) In ConEL, each personal entity mention is mapped to only a single explicit entity,
while in real conversations, personal entity mentions can have multiple corresponding entities (see Figure~\ref{fig:example}); (ii) ConEL contains a limited number of conversations with personal entities, as it synthesizes conversations mainly from question-answering and task-oriented benchmarks with a limited number of personal statements. Additionally, the ConEL-PE dataset~\cite{Joko:2021:CEL}, generated by annotating real social chats, contains only 25 annotated conversations, which cannot be used for training purposes.
Our newly developed dataset, referred to as \emph{ConEL-2}, consists of 1,327 utterances, annotated with personal entities, concepts, and named entities. This dataset has been used for training and evaluation of our conversational EL tool.
In summary, this work makes the following resources available to the community:\footnote{\url{https://github.com/informagi/conversational-entity-linking-2022}}
\begin{itemize}[leftmargin=0.9em]
\item An open-source tool for entity linking in conversations, which can uniquely identify personal entities, concepts, and named entities. Our developed conversational EL methods outperform state-of-the-art baselines.
\item A dataset containing EL annotations for conversations, which can be used for training and evaluation purposes.
% \item Our EL tool (CREL) and collected annotations (CEL) are publicly available as resources with detailed instructions, which allows our community to conduct further research in conversations.
\end{itemize}
%In conversational systems, understanding the user utterances is important to provide a satisfactory user experience.
%Entity Linking (EL) is known to be a powerful technique for the various downstream tasks for text understanding~\cite{Dalton:2014:EQF, Hasibi:2016:EEL, Xiong:2017:WED, Shang:2021:ERO, Hasibi:2017:DFS, Dargahi:2018:QUE},
%however, current EL tools are not effective for annotating conversations because of the three main reasons:
%(1) they are unable to consider personal entities, where references to entities are made by their pronouns,
%(2) only focusing on named entities (NEs) and not suitable for dealing with concepts, and
%(3) are trained on official well-written documents, which is not suitable for conversations.
%
%\emph{\textbf{Personal Entities.}}
%One of the challenges for EL in conversations is linking personal entities (hereinafter, personal entity linking), where the task is detecting \emph{personal entity mentions} (e.g., ``my cars''), finding the corresponding \emph{explicit entity mentions} (e.g., ``London'') in the conversation history, and link it to the corresponding entities (e.g., ``London, England'') in the knowledge graph (Fig.~\ref{fig:example}).
%Given the similarity between personal entity linking and coreference resolution (coref), where both are finding the links between mentions which represent the same entity, one of the intuitive ways to solve this task is applying the coref method, however, directly applying the state-of-the-art coref model fails because of the different characteristics of the task.
%The recent state-of-the-art coref models such as~\cite{Lee:2018:HOC, Joshi:2019:BFC, Kirstain:2021:CRS} utilize the end-to-end model, where the assumption is having plenty of coreferent mentions in a single document, however, personal entity mentions are sparse in the conversations, leading to a huge fail in the mention detection (MD) part of the end-to-end coref.
%% \note{NOTE: For example, we tried to apply S2E coref model directly to personal entity linking tasks, however, it fails without getting any meaningful results mainly due to not being able to find candidate coreference mentions (F1 was 0.)}
%Moreover, even for the simple pronoun coreferences, the difficulty of handling them is reported~\cite{Joshi:2019:BFC}.
%To this end, we develop the personal entity linking method, where we combine the rule-based personal entity mention detection method and state-of-the-art coref method.
%By having a separate module for personal entity mention detection, we can avoid the above-mentioned end-to-end coref problem, making it possible to apply the coref method to the personal entity linking task.
%
%
%\emph{\textbf{Concepts.}}
%
%In conversations, not only NEs but also concepts are important~\cite{Joko:2021:CEL}, however, traditional EL tools are optimized for documents, where recognizing concepts was not paid much attention.\citet{Becker:2021:TLC} proposed a method which performs concept entity linking using the ConceptNet as a knowledge graph.
%Although it is reported to be effective for ConceptNet, their method is based on the deep observation of its characteristics, making it difficult to directly apply to our task where the knowledge graph contains both concepts and NEs.
%We mitigate this problem by training our BERT-based MD model and the existing entity disambiguation (ED) model~\cite{vanHulst:2020:REL} on the conversational EL dataset (described below), to be able to perform EL for concepts and NEs in the conversations.
%The concept and NE linking method, and personal entity linking method are integrated into our conversational EL tool, which is provided as a resource to the community.
%\emph{\textbf{Training data.}}
%The key to solving the abovementioned problems is to have enough training data, however, there is also a challenge here - lack of datasets.
%The datasets which are used in traditional EL systems are mainly focused on documents, not on conversations; the widely used datasets such as~\cite{Hoffart:2011:RDN, Hoffart:2012:KOR} are mainly focused on NEs, not concepts and personal entities.
%The related work from \citet{Joko:2021:CEL} provides a collection of evaluation data, however, the number of data is insufficient for training purposes.
%Additionally, their personal entity annotation only contains the specific case of the personal entities; they focus on the case where personal entity mentions have only one corresponding entity, whereas, in actual conversations, we can observe the case where personal entity mentions have multiple corresponding entities and no corresponding entity.
%Moreover, their dataset does not contain the evaluation data for personal entity mention detection, which makes it impossible to evaluate the MD part of personal entity linking on their dataset.
%To tackle these problems, we collect EL annotations on an existing conversational dataset, which can be used to train and evaluate conversational EL models.
%The annotations, named \emph{Conversational Entity Linking dataset 2022 (CEL)}, which consist of 290 dialogues (1,327 utterances with 2,833 annotations in total) sampled from the Wizard of Wikipedia dataset~\cite{Dinan:2018:WWK}, are publicly available as a resource for the further research.
%
%In summary, this work makes the following contributions:
%\begin{itemize}
% \item We develop a conversational EL tool that can link all the important types of entities in conversational settings: personal entities, concepts, and NEs.
% \item We collect EL annotations for conversations, named CEL dataset, which can be used for training and evaluating conversational EL methods. In addition, we report the detailed procedures of annotation collection and the baseline results by the different EL methods.
% \item Our EL tool (CREL) and collected annotations (CEL) are publicly available as resources with detailed instructions, which allows our community to conduct further research in conversations.
%\end{itemize}
\section{Method}
\label{sec:method}
Our conversational entity linking approach consists of two components;
(1) personal entity linking, and (2) concept and named entity linking.
% The technical overview of the components is as follows.
% For the first and second module, we use rule-based heuristic methods and existing EL methods, respectively.
% For the third module, we modify the state-of-the-art coref model to support this task.
% For the last module, we use the entity annotations performed by above-mentioned EL tools.
% In the subsections below, we will discuss the details of each component.
\subsection{Personal Entity Linking}
\label{sec:method_pel}
Inspired by coreference resolution approaches~\cite{Joshi:2019:BFC, Kirstain:2021:CRS, Lee:2017:E2E, Lee:2018:HOC}, our model identifies personal entity mentions and their entity antecedents from conversation history. We disentangle mention detection from entity antecedent assignment and employ a BERT-based model (cf. Sect.~\ref{sec:method_cne}) for detecting entity mentions and a rule-based approach~\cite{Li:2014:PKG} for identifying personal entity mentions. Specifically, we follow \citet{Joko:2021:CEL} to identify personal mentions, starting with ``my'' or ``our'' tokens followed by adjectives, common nouns, proper nouns, pronouns, numbers, particles, and articles. Also, ``of'' and ``in'' are allowed to be part of the mention.
While admittedly this approach cannot capture all forms of personal entity mentions, we believe that it covers a reasonably large number of cases to be used for early experimentation in this direction. We leave the comprehensive research on different forms of personal entity mention for future work.
%This component has two steps: detecting personal entity mentions and finding corresponding explicit entities.
%
%\medskip \noindent \emph{\textbf{Personal Entity Mention Detection.}}
%% \subsection{Detecting Personal Entity Mentions}
%Although simple, the heuristics are powerful in detecting personal entity mentions.
%Due to the importance of understanding the system users, personal entity mentions beginning with the first-person pronouns are included in this study.
%More specifically, inspired by~\citet{Joko:2021:CEL}, as personal entity mentions, we detect the spans start with ``my'' and ``our'' followed by adjectives, common nouns, proper nouns, pronouns, numbers, particles, and articles.
%Additionally, ``of'' and ``in'' are allowed to be part of the mention.
%These rules are set using our annotations from stage 1 in Sect.~\ref{sec:annot}.
%Our empirical results, which we describe in Sect.~\ref{sec:results}, show that this simple heuristic approach is effective in detecting personal entity mentions.
%
% NOTE: $s$ and $e$ represents the start and end tokens, and m^{s} and m^{e} represent the start and end token *representations*.
Similar to~\cite{Kirstain:2021:CRS}, we denote the start ($s$) and end token ($e$) representations of mentions as:
%
\begin{gather}
m^{s} = GeLU(\mathbf{W}^{s}v) \quad m^{e} = GeLU(\mathbf{W}^{e}v), %\nonumber
\end{gather}
%
where $\mathbf{W}^{s}$ and $\mathbf{W}^{e}$ are trainable weights and $v$ is contextualized representation of a token.
We use LongFormer~\cite{Beltagy:2020:Longformer} to obtain contextualized embeddings as it can accommodate the long conversation history. The input to the LongFormer model is the current turn and conversational history. The hidden state in the output of the last layer of the model is used as $v$.
Assuming mention $m_{bf}$ appears before mention $m_{af}$, the compatibility score between the two mentions is computed by:
%\medskip \noindent \emph{\textbf{Finding Corresponding Explicit Entities.}}
%We modify the start-to-end coref method (s2e-coref) proposed in~\cite{Kirstain:2021:CRS} to find corresponding explicit entities.
%The modification we make is about the MD part, which is trained end-to-end manner in the original s2e-coref, whereas our method has a separate module.
%Following~\cite{Kirstain:2021:CRS}, in our method, the start and end positions of the mention are used to calculate personal entity linking scores.
%The mention could be either personal entity mention or explicit entity mention, which have already been detected in above-mentioned separate modules.
%The start and end token representations of mentions are as follows:
%\begin{gather}
% m^{s} = GeLU(\mathbf{W}^{s}v) \quad m^{e} = GeLU(\mathbf{W}^{e}v) \nonumber
%\end{gather}
%\noindent where, $GeLU$ is the Gaussian Error Linear Unit activation function~\cite{Hendrycks:2016:GEL}, $W^{s}$ and $W^{e}$ are the trainable weights of the start and end token of the candidate mention, and $v$ is token contextualized embedding taken from the underlying LongFormer model~\cite{Beltagy:2020:Longformer}.
%The input for the LongFormer model is the all tokens from the current turn and conversational history, and the hidden-state at the output of the last layer of the model is used as $v$.
%The personal entity linking score for two mentions is calculated as follows:
%
\begin{align}
score(m_{bf}, m_{af}) & = m^{s}_{bf} \cdot B^{ss} \cdot m^{s}_{af}
+ m^{s}_{bf} \cdot B^{se} \cdot m^{e}_{af} \nonumber \\
& + m^{e}_{bf} \cdot B^{es} \cdot m^{s}_{af}
+ m^{e}_{bf} \cdot B^{ee} \cdot m^{e}_{af}, %\nonumber
\label{eq:score}
\end{align}
%
where $B$ represents trainable weights and the superscripts $s$ and $e$ denote the start and end tokens of a mention.
% subscripts $bf$ and $af$ represent that the mention $m_{bf}$ appears before $m_{af}$ in the conversation, and $B$ is the trainable weights, where each superscript represents the combination of the start and end token of the candidate mention.
Equation~\ref{eq:score} is computed for all pairs of personal entity mentions and explicit entity mentions.
%During training, the weights $W$ and $B$ are trained so that scores can classify mention pairs by whether they are in a relationship between personal and explicit entity mention or not.
During inference, personal and explicit entity mention pairs with the $score$ higher than threshold $\tau$ are selected.
%which a higher $score$ than a threshold, which is searched by validation data, are selected as the personal entity linking result.
% For the training procedure, to mitigate the small amount of training data, we take out pre-trained BERT model which is adapted to \underline{conv}ersations domain, and fine-tune it on our CEL \underline{t}raining \underline{d}ataset (hereinafter the model is called CREL\wast{CONV}).
% More specifically, we use is bert-base-cased-conversational\footnote{\url{https://huggingface.co/DeepPavlov/bert-base-cased-conversational}} as a pre-trained model, which is domain adapted model with fine-tuned on conversational datasets such as DailyDialogues~\cite{Li:2017:DDM}, OpenSubtitles~\cite{Lison:2016:OSE}, Debates~\cite{Zhang:2016:CFO}, and Blogs~\cite{Schler:2006:EAG}.
% In our work, the other training procedures are also evaluated, however, it is found that this procedure achieved higher performance than others (see Sect.~\ref{sec:results_cne}).
% We employ our method (Sect.~\ref{sec:method}) in three different fine-tuning procedures: S2E*, S2E\textsubscript{ON}, and S2E\wast{ON}.
% Following~\citet{Kirstain:2021:CRS}, S2E* uses longformer-large\footnote{\url{https://huggingface.co/allenai/longformer-large-4096}} as a pre-trained model and fine-tuned on our training data.
% S2E\textsubscript{ON} uses the coref model\footnote{\url{https://github.com/yuvalkirstain/s2e-coref}}, which uses the same pre-trained model as above and is trained on the coref dataset, OntoNotes Release 5.0~\cite{Hovy:2006:ON9}.
% In other words, the model is not fine-tuned on our personal entity linking task.
% S2E\wast{ON} is the joint approach of S2E* and S2E\textsubscript{ON}: the model uses the same coref pre-trained model as S2E\textsubscript{ON}, and fine-tuned on our training data.
% The batch size of 8 and learning rate of 1e-5 are used for training, and the validation set is used to threshold and decided the training steps from which we take out the model.
\if 0
For the training procedure, because of the similarity of the task between coref and personal entity linking, we utilize the model pre-trained on the coref dataset.
More specifically, we take out LongFormer model~\cite{Beltagy:2020:Longformer} pre-trained on OntoNote Release 5.0~\cite{Hovy:2006:ON9}, and fine-tuned it on our training data (hereinafter the model is called S2E\wast{ON}, where * and ON represents fine-tuning using our CNE dataset and OntoNote, respectively).
Following~\citet{Kirstain:2021:CRS}, LongFormer is used here to accommodate the long conversation history.
We also evaluate other training procedures in Sect.~\ref{sec:results_pe}, and show the effectiveness of utilizing the model pre-trained on the coref dataset.
\note{NOTE: This paragraph was added to align with Concept and Named Entity section's training procedure part.}
\fi
% The first modification is that we do not use the mention score for the personal entity, which is used in the original s2e-coref method.
% This is because we have separate module for detecting the personal entity mention (Sect.~\ref{sec:detecting_personal_entity_mentions}).
% The second modification is that we only consider detected explicit entity mentions (Sect.~\ref{sec:linking_ne_and_concepts}) as candidates.
% This is also because we have a separate explicit entity MD module.
\subsection{Concept and Named Entity Linking}
\label{sec:method_cne}
Entity linking typically involves two subtasks: (i) identifying mention boundaries and (ii) disambiguating mentions using a knowledge graph.
In order to utilize existing advancements on these two subtasks, we need to adapt them to the conversational setup.
For mention detection (MD), this translates to adapting MD models to the conversational domain and identifying mentions of both named entities and concepts.
We employ the vanilla BERT~\cite{Devlin:2019:BER} architecture and fine-tune it to predict BIO labels, corresponding to begin, inside, and outside of mentions. The BERT model utilized in this work is pre-trained on conversational datasets such as DailyDialogues~\cite{Li:2017:DDM}, OpenSubtitles~\cite{Lison:2016:OSE}, Debates~\cite{Zhang:2016:CFO}, and Blogs~\cite{Schler:2006:EAG}.
For entity disambiguation, we train REL~\cite{vanHulst:2020:REL} disambiguation approach on our ConEL-2 dataset.
% by following the instruction\footnote{\url{https://github.com/informagi/REL}}.
REL is one of the state-of-the-art EL tools that is optimized for document annotation, but its disambiguation approach, considering both the local and global context of mentions, can be seamlessly adapted to conversations. Our entity linking method, consisting of the aforementioned MD and ED models is named CREL, which is an extension of REL tool for conversations.
%This component consists of two steps: mention detection (MD) and entity disambiguation (ED).
%
%\medskip \noindent \emph{\textbf{Mention Detection.}}
%We use BERT~\cite{Devlin:2019:BER} for MD; although it has proposed a lot of modifications to the BERT architecture, it is reported that most of them do not meaningfully improve performance~\cite{Narang:2021:DTM}.
%In addition, the original BERT is highly scalable due to its well-researched architecture and high availability of pre-trained models, which makes the tool more suitable as a resource for the community.
%Thus, we use the original BERT~\cite{Devlin:2019:BER} architecture as our MD.
%Following the implementation of BERT token classification\footnote{\url{https://huggingface.co/docs/transformers/tasks/token_classification}}, our BERT-based MD model is trained to predict BIO labels that correspond to begin, inside, and outside the mentions.
%The BERT model size that we use is the \emph{BERT\textsubscript{BASE}}; we also evaluated the performance for \emph{BERT\textsubscript{LARGE}}, however, we could not observe a significant improvement in the performance, mainly due to the relatively smaller size of the training data.
%
%For the training procedure, to mitigate the small amount of training data, we take out pre-trained BERT model which is adapted to \underline{conv}ersations domain, and fine-tune it on our CEL training dataset (hereinafter the model is called CREL\wast{CONV}).
%More specifically, we use is bert-base-cased-conversational\footnote{\url{https://huggingface.co/DeepPavlov/bert-base-cased-conversational}} as a pre-trained model, which is domain adapted model with fine-tuned on conversational datasets such as DailyDialogues~\cite{Li:2017:DDM}, OpenSubtitles~\cite{Lison:2016:OSE}, Debates~\cite{Zhang:2016:CFO}, and Blogs~\cite{Schler:2006:EAG}.
%% In our work, the other training procedures are also evaluated, however, it is found that this procedure achieved higher performance than others (see Sect.~\ref{sec:results_cne}).
%We also evaluate other training procedures in Sect.~\ref{sec:results_cne}, and show the effectiveness of using this domain adapted pre-trained model.
%
%\medskip \noindent \emph{\textbf{Entity Disambiguation.}}
%For ED, we train REL~\cite{vanHulst:2020:REL} on our CEL dataset by following the instruction\footnote{\url{https://github.com/informagi/REL}}.
%The REL is one of the state-of-the-art EL tools, which employs modular architecture.
%Although the provided REL only focuses on documents, thanks to the modular architecture, we can only use disambiguation modules trained on our CEL dataset.
% We use existing entity linking (EL) methods to detect candidates for explicit entity:
% TagMe~\cite{Ferragina:2010:TAG}, WAT~\cite{Piccinno:2014:FTM}, REL~\cite{vanHulst:2020:REL}, and GENRE~\cite{Cao:2021:GENRE}.
% Since explicit entity mentions can be either concepts or NEs, traditional EL methods, which mainly focusing on NEs, reported to be suboptimal~\cite{Joko:2021:CEL}.
% As we pointed out in \todo{Sect.X}, our focus here is on concepts, since concepts have not gotten much attention in the traditional entity linking for documents.
% We fine tune BERT-NER \todo{[x]} on Wikipedia abstract dataset used in~\todo{[x]} to detect the concept mentions.
% Each detected concept mention is then linked to the entity by using \todo{xxx}.
\section{The ConEL-2 Collection}
\label{sec:annot}
To Facilitate the development of EL approaches for conversations, we construct the ConEL-2 collection.
We draw the conversations from an existing conversational dataset. To select the dataset, we focus on social chat datasets, as personal entities are mainly found in social chat conversations~\cite{Joko:2021:CEL}.
We choose the Wizard of Wikipedia (WoW)~\cite{Dinan:2018:WWK} dataset, which contains multi-domain multi-turn conversations collected from actual interactions between two people. The selected dialogues are annotated in three stages: (i) identifying personal entity mentions, (ii) pairing personal and explicit entity mentions, and (iii) entity linking. Unless stated otherwise, each HIT is annotated by three turkers, and additional turkers are added when needed based on our quality check strategies.
%
%To tackle the problem of lack of training data for conversational EL (see Sect.~\ref{sec:introduction}), we collect EL annotations for the existing conversational dataset.
%For selecting the dataset, we focus on the social chat as personal entities are mainly found in the social chat conversations.
%Furthermore, we focus on real-world conversational situations and set our criteria as multi-domain, multi-turn, and collected from actual interactions between two people.
%With these criteria, following~\citet{Joko:2021:CEL}, we select the Wizard of Wikipedia~\cite{Dinan:2018:WWK} as the dataset.
%The annotation collection process contains three stages: (1) personal entity mention annotation stage, (2) personal and explicit entity pair annotation stage, and (3) entity linking stage.
%The annotations are collected using the Amazon Mechanical Turk (MTurk).
%To reduce the cognitive load of the turkers, our annotation interfaces employ multiple-choice questions, i.e., give turkers multiple mentions or entities to annotate.
%These pools are created by the rule that word sequences begin with ``my'' or ``our'' (Stage 1), or by existing EL tools (Stages 2 and 3).
%Each HIT is annotated by three turkers, however, if additional annotations are needed based on our quality check, then additional turkers are added.
\begin{figure}[t]
\includegraphics[width=1.0\linewidth]{example_stage1}
\caption{Annotation interface for identifying personal entity mentions (stage 1 of annotation process).}
\label{fig:example_stage1}
\end{figure}
\medskip \noindent \emph{\textbf{Stage 1: Identifying personal entity mentions.}}
In this stage, we ask turkers to identify text spans referring to personal entities (i.e., personal entity mentions).
First, we extract conversations that contain ``my'' or ``our'' in at least one user utterance, which amounts to 7,888 out of 22,311 conversations in the WoW dataset.
%Although it could be extended to other pronouns, this study focuses only on the pronouns of ``my'' and ``our'', since the first-person pronouns are crucial to understand the users of the system.
%\note{To Faegheh: Regarding your comment: Here, I simply did a keyword matching with ``my'' and ``our''. The explanation of subsequent words is described below.}
To make the annotation process technically and financially feasible, we randomly select 1,000 conversations from the extracted dialogues.
We then show the dialogue history and a set of candidate mentions to the turkers and ask them to find the best personal entity mention; the interface is shown in Fig.~\ref{fig:example_stage1}. Candidate mentions are text spans starting with ``my'' or ``our,'' followed by a maximum of 10 subsequent words. Our statistics on the collected annotations show that 98.9\% of the HITs were agreed upon by two or more turkers, with a Fleiss' kappa of 0.864.
We move forward with the 985 conversations, where at least two turkers agreed on all personal entity mentions of the conversation.
%The annotation interface is shown in Fig.~\ref{fig:example_stage1}.
%We show 10 candidate personal entity mentions to turkers.
%These candidates start with ``my'' or ``our'' and are followed by a maximum of 10 subsequent words.
%Our statistics on the collected annotations show the effectiveness of our MTurk annotation design; 98.9\% of the HITs were agreed upon by two or more turkers, with a Fleiss' kappa of 0.864.
%%% NOTE: For more detail on annotation statistics, see here https://docs.google.com/document/d/16VX6kvQ0zV_HuCmB817BBl3S68ihD6PRAxx0OIpqMmk/edit#heading=h.2wkgesd3dita
%As a result of the stage 1, we select 985 conversations, where at least two turkers are agreed on one personal entity mention.
%We further split these selected conversations into train, val, and test sets in the ratio of 6:2:2.
%~\note{<-- NOTE: I keep this sentence here since I could not figure out where this sentence can be placed the best.}
%% Additionally, we create a personal entity mention dataset from these annotated conversations; the conversations was split into training, validation, and test sets at a ratio of 0.6, 0.2, and 0.2, respectively.
%% The test set is used to evaluate our rule-based personal entity mention approach in Sect.~\ref{sec:results}.
%All of these 985 conversations are passed to the next stage.
% \begin{table}
% \centering
% \caption{Summary of the collected annotations for personal entity mention detection (corresponding stage 1 MTurk annotations).}
% \label{tbl:annotation_statistics_pem}
% % \shrink
% \begin{tabular}{@{~}l||@{~~}l|@{~~}l|@{~~}l}
% \Xhline{1.5pt}
% & Train & Val & Test \\
% \Xhline{1.5pt}
% Conversations & 591 & 197 & 197 \\
% User utterances & 587 & 195 & 195 \\
% Personal entity mentions & 803 & 286 & 280 \\
% \Xhline{1.5pt}
% \end{tabular}
% % \shrink
% \end{table}
\begin{table}[t]
\centering
\caption{Statistics of the ConEL-2 collection.
% Summary of the collected annotations for entity linking (corresponding stage 2 and 3 MTurk annotations).
% NE and concept annotations represents the number of annotations for concepts and NEs.
Personal entity annotations represent the number of annotations for personal entity mentions, with or without corresponding explicit entity mention in the dialogue.
% \note{NOTE: The ``personal entity annotations'' include ``not-in-the-dialogue'' pattern, where PEM represents the entity but corresponding EEMs cannot be found in the dialogue history. Details are \url{https://gitlab.science.ru.nl/hjoko/conel-2022/-/tree/main/data/ConEL_Personal_Entity}}
}
\label{tbl:annotation_statistics}
% \shrink
\begin{tabular}{@{~}l|@{~~}l|@{~~}l|@{~~}l}
\hline
& Train & Val & Test \\
\Xhline{1.5pt}
Conversations & 174 & 58 & 58 \\
User utterances & 800 & 267 & 260 \\
NE and concept annotations & 1428 & 523 & 452 \\
% % personal entity annotations & 190 & 64 & 59 \\
% \hline
% PEM w/ single-EEM & 186 & 63 & 57 \\
% PEM w/ multi-EEM & 5 & 1 & 3 \\
% PEM w/o EEM & 77 & 25 & 13 \\
Personal entity annotations & 268 & 89 & 73 \\
\hline
\end{tabular}
% \shrink
\end{table}
\medskip \noindent \emph{\textbf{Stage 2: Pairing personal and explicit entity mentions.}}
In this stage, we ask turkers to find the corresponding explicit entity mention for a given personal entity mention. For example, in Figure~\ref{fig:example}, the personal entity mention ``\emph{my cars}'' is shown to turkers, and they are asked to find the corresponding explicit entity mention ``\emph{Life}'' and ``\emph{Pilot}.''
We create a pool of candidate explicit entity mentions by using existing EL tools: TagMe~\cite{Ferragina:2010:TAG}, WAT~\cite{Piccinno:2014:FTM}, REL~\cite{vanHulst:2020:REL}, and GENRE~\cite{Cao:2021:GENRE}.
The detected mentions are manually processed and meaningless duplicates such as ``a restaurant'' and ``restaurant'' are resolved.
Turkers are then given the option to select one corresponding explicit entity mention or the ``not in dialogue'' option if the corresponding entity mention is not found in the dialogue.
Fleiss' kappa for the inter-annotator agreement is 0.434, which is considered moderate.
%Since a Fleiss' kappa of the MTurk annotation results is 0.434 of moderate agreement,
To improve the annotation quality, two extra annotations are collected for HITs where two turkers agreed on one option. We then select all conversations with equal or more than three turkers agreed on one option (either an explicit entity mention or the ``not in dialogue'' option). This selection amounts to 290 conversations, which are passed to the next step.
%After this extra annotations, we select 290 conversations, where equal or more than three turkers are agreed upon one explicit entity or not having the corresponding explicit entity in the conversation history.
%\note{To Faegheh: Regarding your comment about kappa, I am not sure whether it is better to talk about kappa value for these 290 conversations, since the conversations are extracted \emph{in a way} that they should have a high agreement, meaning this cannot represent the annotation quality.
%For example, thinking about the extreme case, even if the annotation quality is really low, if we select only fully-agreed annotations and calculate the kappa value only for them, the kappa value becomes 1.
%Additionally, as far as I understand, Fleiss' kappa can only be calculated for the fixed number of judges (i.e., in this case, \#turkers for each HIT). However, in our situation, 290 dialogues have different \#turkers (5 for the ones where extra annotations are collected, and 3 for the others), so it would be difficult to calculate the single kappa value which represents the annotation quality of stage 2 annotations.
%}
We note that one of the shortcomings of the existing ConEL dataset is identifying only one explicit entity mention for each personal mention (cf. Section~\ref{sec:introduction}).
In ConEL-2, we address this issue and ask turkers to find another corresponding explicit entity mention for all extracted conversations if any.
%As discussed in Section~\ref{sec:introduction}, one of the missing pieces of the existing dataset is to consider the case where there are multiple explicit entity mentions for a given personal entity mention.
%To address this issue, we ask turkers to find another corresponding explicit entity mention for all extracted conversations if any.
% Finally, collected conversations are split into $6:2:2$ to create train / val / test sets. See Table.~\ref{tbl:annotation_statistics} for the summary of collected annotations.
% personal entity annotations represent personal entity annotations.
% PEM w/ single-EEM, w/ multi-EEM, and w/o EEM represents,
% cases of having only one corresponding explicit entity mention for each personal entity mention,
% having multiple corresponding explicit entity mentions for each personal entity mention,
% and having no corresponding explicit entity mention for each personal entity mention, respectively.
\medskip \noindent \emph{\textbf{Stage 3: Entity linking}}
In this stage, we ask turkers to select the corresponding Wikipedia article to the given entity mention,
e.g., mapping the mention ``car dealers'' to the entity \emph{Car dealership} in Figure~\ref{fig:example}.
%the user utterance ``\emph{I am staying next to Waterloo Station in \underline{London}}'' is given to turkers, and they are asked to select ``\emph{London, England}'' not ``\emph{London, Ontario}''.
We first use the EL tools described in the stage 2 to create the pool of candidate mention-entity pairs.
Noisy mentions such as ``please'' and ``I am'' are manually removed from the pool.
We also include top-$10$ Wikipedia search\footnote{\url{https://www.mediawiki.org/wiki/API:Main_page}} results using mentions as queries.
% For these detected mentions, nonsense detected mentions (e.g., ``please'' and ``I am'') are removed or corrected heuristically.
Turkers are then asked to select one of the candidate entities for each entity mention if any.
%These candidates contain the entities detected by the EL tools, as well as top-$10$ Wikipedia search\footnote{\url{https://www.mediawiki.org/wiki/API:Main_page}} results with using mentions as queries.
The statistics show 98.7\% of the HITs were agreed by two or more turkers (Fleiss' kappa of 0.839), which highlights the high quality of our annotation process.
%annotations in the stage 3; 98.7\% of the HITs were agreed upon by two or more turkers with a Fleiss' kappa of 0.839.
We therefore include all annotation results which are agreed by at least two turkers.
For the remaining 1.3\% of the non-agreed HITs, we recollected two extra annotations for each and select the entity that is selected by the majority of turkers.
%Finally, as post-processing, we delete mention overlaps by selecting the longest mention.
The annotation results are shown in Table~\ref{tbl:annotation_statistics}. We split the dataset to train, validation, and test set with the 60\%-20\%-20\% ratio.
%The conversations with our annotations are split into 6:2:2 to create train, val, and test sets.
%We get 290 conversations in total with 2,403 concepts and NE and 430 personal entity annotations.
%\note{To Faegheh: Regarding your comment, the note below is correct and the sentence about ``... remove the dialogue where ....'' was wrong, so I dropped it.
%(Original NOTE: This contains the case where ``not in the dialogue'' case described in stage 2. Detail statistics are commented out in the Table~\ref{tbl:annotation_statistics}.)}
% NOTE: "delete mention overlaps" --> this is technically wrong since I leave two cases of "partial overlap" patterns as it is.
% NOTE: Why did we handle overlaps in the last step? --> Some longest mentions are non-sense, thus we need to eliminate these patterns by human annotators.
\section{Results}
\label{sec:results}
\subsection{Personal Entity Linking}
\label{sec:results_pe}
\begin{table}[t]
\addtolength{\tabcolsep}{-1.9pt}
\centering
\miniskip
\caption{Personal entity linking results. Micro-averaged precision, recall, and F1 scores are computed.
% Personal entity linking results. W2V and S2E represent the baseline Wikipedia2vec-based method~\cite{Joko:2021:CEL} and our modified s2e-coref based method, respectively.
% Each subscript for the CREL model represents fine-tuning datasets. S2E* and S2E\textsubscript{ON} represent the model is trained on our training data and OntoNote dataset, respectively (Sect.~\ref{sec:experiment_pel}).
% S2E\wast{ON} represents first the model is fine-tuned on OntoNote and further on our training data.
}
\shrink
\label{tbl:pe}
\hjscalebox{
\begin{tabular}{l|ccc|ccc|ccc}
\hline
\multicolumn{1}{Sl|}{}
& \multicolumn{3}{Sc|}{ConEL-2 \footnotesize{(Val)}} & \multicolumn{3}{Sc|}{ConEL-2 \footnotesize{(Test)}} & \multicolumn{3}{Sc}{ConEL-PE} \\
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc}{F} \\
\Xhline{2pt}
S2E~\cite{Kirstain:2021:CRS} & .375 & .415 & .394 & .292 & .302 & .297 & .317 & .481 & .382 \\
W2V~\cite{Joko:2021:CEL} & .324 & .508 & .395 & .458 & .429 & .443 & .436 & .630 & .515 \\
% S2E*
% S2E
S2E$^{-}_{\text{conv}}$ & .494 & \textbf{.615} & .548 & .450 & \textbf{.571} & .503 & .400 & \textbf{.741} & .519 \\
% S2E\wast{ON}
S2E\textsubscript{conv}& \textbf{.714} & .538 & \textbf{.614} & \textbf{.673} & .524 & \textbf{.589} & \textbf{.613} & .704 & \textbf{.655} \\
\hline
\end{tabular}
}
\end{table}
\begin{table}[t]
\shrink
\addtolength{\tabcolsep}{-1.4pt}
\centering
\caption{Concept and named entity linking results.
% Mention detection and entity linking results for concepts and NEs.
F\textsubscript{MD} and F\textsubscript{EL} represent the micro-averaged F1 scores for mention detection and entity disambiguation, respectively.}
\miniskip
\label{tbl:md}
\hjscalebox{
\begin{tabular}{l|c@{~}c|c@{~}c|cc cc}
\hline
& \multicolumn{2}{Sc|}{ConEL-2 \footnotesize{(Val)}} & \multicolumn{2}{Sc|}{ConEL-2 \footnotesize{(Test)}} &
% \multicolumn{2}{Sc|}{ConEL-PE} &
\multicolumn{2}{Sc}{ConEL} \\
& F\textsubscript{MD} & F\textsubscript{EL} & F\textsubscript{MD} & F\textsubscript{EL} &
% F\textsubscript{MD} & F\textsubscript{EL} &
F\textsubscript{MD} & F\textsubscript{EL} \\
\Xhline{2pt}
GENRE \cite{Cao:2021:GENRE} & .290 & .252 & .320 & .299 & .350 & .211 \\
REL \cite{vanHulst:2020:REL} & .304 & .244 & .279 & .231 & .462 & .245 \\
TagMe \cite{Ferragina:2010:TAG} & .559 & .478 & .611 & .504 & .510 & .375 \\
WAT \cite{Piccinno:2014:FTM} & .616 & .539 & .613 & .519 & .416 & .336 \\
\hline
REL*(BERT$_{\text{conv}}$) & \textbf{.748} & \textbf{.652} & .716 & .587 & .551 & .424 \\
REL*(BERT$_{\text{WP-conv}}$) & .697 & .624 & .708 & .592 & .508 & .399 \\
REL*(BERT$_{\text{NER-conv}}$) & .723 & .635 & .693 & .576 & .548 & .407 \\
% CREL\wast{CONV}
CREL & .742 & .651 & \textbf{.729} & \textbf{.597} & \textbf{.559} & \textbf{.429} \\
\hline
\end{tabular}
}
\end{table}
\begin{table}[t]
\small\addtolength{\tabcolsep}{-2pt}
\centering
\caption{Overall entity linking results, annotating both personal entities (PE) and concepts and named entities (EL).}
\label{tbl:overall}
\hjscalebox{
\begin{tabular}{l l|ccc|ccc|ccc}
\hline
\multicolumn{1}{Sl }{} & \multicolumn{1}{Sl |}{}
& \multicolumn{3}{Sc|}{ConEL-2 \footnotesize{(Val)}} & \multicolumn{3}{Sc|}{ConEL-2 \footnotesize{(Test)}} & \multicolumn{3}{Sc}{ConEL-PE} \\
EL & PE & \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc}{F} \\
\Xhline{2pt}
% REL & \underline{.631} & .136 & .224 & \underline{.598} & .129 & .212 & \underline{.561} & .161 & .250 \\
% GENRE & \underline{.586} & .145 & .232 & \underline{\textbf{.672}} & .177 & .280 & \underline{.633} & .191 & .293 \\
% TagMe & .599 & .353 & .444 & .606 & .380 & .467 & .622 & .447 & .520 \\
% WAT & \underline{.654} & .503 & .568 & .635 & .464 & .536 & .595 & .598 & .596 \\
% CREL\wast{CONV} & \underline{\textbf{.712}} & .531 & .608 & .629 & .500 & .557 & .598 & .628 & .613 \\
% \hline
REL & W2V & .523 & .159 & .244 & .516 & .159 & .243 & .488 & .206 & \underline{.290} \\
REL & S2E\textsubscript{conv} & .523 & .162 & .248 & .522 & .165 & \underline{.250} & .466 & .206 & .286 \\
\hline
GENRE & W2V & .468 & .180 & .260 & .569 & .223 & .320 & .537 & .256 & .347 \\
GENRE & S2E\textsubscript{conv} & .589 & .169 & .263 & \textbf{.675} & .217 & \underline{.328} & \textbf{.646} & .256 & \underline{.367} \\
\hline
TagMe & W2V & .504 & .407 & .451 & .530 & .440 & .481 & .528 & .518 & .523 \\
TagMe & S2E\textsubscript{conv} & .612 & .395 & .480 & .621 & .434 & \underline{.511} & .636 & .518 & \underline{.571} \\
\hline
WAT & W2V & .506 & .489 & .497 & .545 & .464 & .501 & .536 & .628 & .579 \\
WAT & S2E\textsubscript{conv} & .585 & .490 & .534 & .582 & .464 & \underline{.516} & .571 & .648 & \underline{.607} \\
\hline
CREL & W2V & .636 & .589 & .612 & .607 & .554 & .579 & .573 & .714 & .635 \\
CREL & S2E\textsubscript{conv} & \textbf{.712} & \textbf{.593} & \textbf{.647} & .634 & \textbf{.566} & \underline{\textbf{.598}} & .600 & \textbf{.724} & \underline{\textbf{.656}} \\
\hline
\end{tabular}
}
\end{table}
\paragraph{Experimental Setup}
We take the LongFormer-large model\footnote{\url{https://huggingface.co/allenai/longformer-large-4096}}~\cite{Beltagy:2020:Longformer} pre-trained on OntoNote Release 5.0~\cite{Hovy:2006:ON9}, and fine-tune it on the training set of the ConEL-2 dataset. This model is referred to as \seconv{}.
%We set the threshold $\tau$ to \hi{0.0} and
We perform training with the batch size of 8 and learning rate of 1e-5 using AdamW optimizer.
For evaluation, we use validation and test sets of the ConEL-2 datasets as well as the ConEL-PE dataset~\cite{Joko:2021:CEL}. ConEL-PE contains 25 dialogues from the WoW dataset. We only use personal entity annotations for this evaluation.
\paragraph{Baselines}
We compare our model with the Wikipedia2Vec-based method (W2V)~\cite{Joko:2021:CEL}, where antecedent assignment is based on similarity between Wikipedia2vec~\cite{Yamada:2020:W2V} embeddings of entities and personal entity mentions.
We also report on the official S2E model\footnote{\url{https://github.com/yuvalkirstain/s2e-coref}}~\cite{Kirstain:2021:CRS} and its variation \seconvminus{}, where fine-tuning is performed only on our conversational dataset (without using OntoNote dataset).
\paragraph{Results}
We measured the performance of our rule-based personal entity mention detection approach and obtained micro-averaged precision, recall, and F1 of 0.908, 0.898, and 0.903, respectively. This indicates that identifying personal entity mentions can be handled effectively by a simple approach. Assigning entity antecedents, on the hand, is a challenging task.
Table~\ref{tbl:pe} shows the results for personal entity linking. We observe that \seconv{} fine-tuned on both OntoNote and conversational datasets outperforms all baselines. It also indicates that, fine-tuning on conversational data substantially improves personal entity linking performance (S2E vs. \seconv{} results).
%
%\medskip \noindent \emph{\textbf{Personal Entity Mention Detection.}}
%The performance of our rule-based personal entity mention detection approach (Sect.~\ref{sec:method_pel}) is evaluated on the test set of our personal entity mention dataset (Stage 1 of Sect.~\ref{sec:annot}).
%The performance results for precision, recall, and F1 are 0.908, 0.898, and 0.903, respectively, which represent the effectiveness of the simple rule-based approach for personal entity mention detection.
%
%
%
%\medskip \noindent \emph{\textbf{Personal Entity Linking.}}
%% We evaluate personal entity linking results with the baseline method of W2V and our method with different training procedures.
%% In this experiment, explicit entity mentions are detected and linked using our concepts and NE linking method (Sect.~\ref{sec:method_cne}).
%The personal entity linking results are reported in Table~\ref{tbl:pe}.
%The results show that the S2E\textsubscript{ON} has the lowest F1 score on all sets, representing simply using coref trained model is not sufficient for personal entity linking.
%The S2E* achieves the higher performance than S2E\textsubscript{ON}, which represents the importance of fine-tuning the model on our personal entity linking annotations.
%Overall, S2E\wast{ON} achieves the highest F1 score on all sets, suggesting the knowledge transferability of coref in the personal entity linking.
\subsection{Concept and Named Entity Linking}
\label{sec:results_cne}
\paragraph{Experimental Setup}
For mention detection, we fine-tune bert-base-cased-conversational\footnote{\url{https://huggingface.co/DeepPavlov/bert-base-cased-conversational}} on our training data with the batch size of 16, learning rate of 5e-5, and AdamW optimizer. For ED, we train the REL ED model, following the same training procedure as~\citet{vanHulst:2020:REL}.
For evaluation, we use the ConEL-2 and ConEL~\cite{Joko:2021:CEL} datasets. ConEL consists of 100 conversations that are collected from three different conversational tasks (question answering, task-oriented, and social chat) and annotated with concepts and named entities.
Only concept and named entity annotations are used for this evaluation.
\paragraph{Baselines}
% There is no publicly available method for entity linking in conversations.
REL~\cite{vanHulst:2020:REL}, GENRE~\cite{Cao:2021:GENRE}, TagMe~\cite{Ferragina:2010:TAG}, and WAT~\cite{Piccinno:2014:FTM} are strong publicly available EL tools that are used as our baselines.
Additionally, we train three BERT-based MD models:
(i) \emph{BERT$_{\text{conv}}$}: The bert-base-uncased model\footnote{\url{https://huggingface.co/bert-base-uncased}}, fine-tuned on our training data,
(ii) BERT$_{\text{WP-conv}}$: Similar to BERT$_{\text{conv}}$, but fine-tuned on the Wikipedia anchor links from~\cite{Cao:2021:GENRE} before fine-tuning on ConEL-2 training data, and
(iii) BERT$_{\text{NER-conv}}$: The bert-base-NER model\footnote{\url{https://huggingface.co/dslim/bert-base-NER}}, fine-tuned on our training data.
The fine-tuning procedures are the same as CREL. We combine these models with the REL ED model trained on conversations (denoted as REL*) and report on their results.
\paragraph{Results}
The results are reported in Table~\ref{tbl:md}.
%The first block shows the results for existing entity linking tools and the second block shows our EL tool (CREL).
F\textsubscript{MD} and F\textsubscript{EL} are the F1 scores for MD and EL, respectively.
The results show that the REL and GENRE, while being state-of-the-art EL toolkits, perform worse than TagMe and WAT.
%From the first block, we can see a performance gap between the first two rows (REL and GENRE) and the rest of the rows in top block (TagMe and WAT).
This is because REL and GENRE are intended to detect only NEs, while TagMe and WAT are able to identify concepts as well.
Comparing CREL with REL* methods, we observe that adaptation of BERT for conversational domain plays an important role in conversational entity linking, reflected by the highest score for ConEL-2 (test) and ConEL datasets.
%From the comparison between the first and second blocks, we can see our method achieves higher performance.
%Given that our methods are trained on conversational EL data (which is denoted as *), this result show the importance of training data for the conversational EL.
%Overall, for both MD and EL, CREL\wast{CONV} achieves the highest F1 scores for three out of four sets, showing the domain adaptation for conversations helps the model to understand the characteristics of conversations.
\subsection{Overall Performance}
\label{sec:results_overall}
Putting all the pieces together, we report on the overall performance of our model for linking personal entities, concepts and NEs.
For this experiment, we report only on the ConEL-2 and ConEL-PE datasets, as ConEL does not have annotations for personal entities. The results are shown in Table~\ref{tbl:overall}.
We notice at first glance that our proposed personal entity linking extension (S2E\textsubscript{conv}) leads to improved F-scores compared to W2V in most cases. The results also show that our EL tool (CREL with S2E\textsubscript{conv}) achieves the highest F-score for all sets, reinforcing the importance of adapting EL approaches for conversations.
%Table~\ref{tbl:overall} shows the entity linking results.
%The first line for each block shows the results where personal entities are not linked.
%The second and third line for each block shows the results where personal entities are linked by the baseline W2V method and our fine-tuned S2E\wast{ON} method, respectively.
%By comparing the first line of each block, we can observe that our CREL\wast{CONV} achieves the highest F1 score for all sets, reinforcing the importance of training on our conversational entity linking data for this task.
%The results also show that the personal entity linking extension (i.e., +S2E\wast{ON} and +W2V) leads to better F1 scores in most of the cases, representing the extension contributes to the performance improvements.
%Overall, our EL tool, which is CREL\wast{CONV} with S2E\wast{ON}-based personal entity linking extension, achieves the highest F1 score on all sets, which shows the effectiveness of our EL tool for conversations.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% Tables %%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \begin{table}[t]
% \addtolength{\tabcolsep}{-1.4pt}
% \centering
% \caption{Mention detection and entity linking results for concepts and NEs.
% F\textsubscript{MD} and F\textsubscript{EL} represent the F1 scores for mention detection and entity linking, respectively.
% % The first block shows the existing entity linking models and the bottom block shows CREL model which is trained on our training data.
% % Each subscript for the CREL model represents fine-tuning datasets. TD, WP, NER, and CON represent our training data, Wikipedia, NE recognition dataset, and conversations (Sect.~\ref{sec:experiment_cne}).
% % The arrow represents the order of the finetuning, e.g., WP$\rightarrow$TD represents that the CREL is first fine-tuned on Wikipedia and further on the training data.
% % TD, WP, NER, and CON represents the \underline{T}raining \underline{d}ata, \underline{W}iki\underline{p}edia abstract, \underline{N}amed \underline{E}ntity \underline{R}ecognition dataset, and \underline{Conv}ersational dataset, respectively.
% }
% \label{tbl:md}
% \hjscalebox{
% \begin{tabular}{l|cc|cc|cc|cc}
% \hline
% & \multicolumn{2}{Sc|}{ConEL-2 (Val)} & \multicolumn{2}{Sc|}{ConEL-2 (Test)} & \multicolumn{2}{Sc|}{ConEL-PE} & \multicolumn{2}{Sc}{ConEL} \\
% & F\textsubscript{MD} & F\textsubscript{EL} & F\textsubscript{MD} & F\textsubscript{EL} & F\textsubscript{MD} & F\textsubscript{EL} & F\textsubscript{MD} & F\textsubscript{EL} \\
% \Xhline{2pt}
% GENRE \cite{Cao:2021:GENRE} & .290 & .252 & .320 & .299 & .346 & .328 & .350 & .211 \\
% REL \cite{vanHulst:2020:REL} & .304 & .244 & .279 & .231 & .360 & .279 & .462 & .245 \\
% TagMe \cite{Ferragina:2010:TAG} & .559 & .478 & .611 & .504 & .611 & .565 & .510 & .375 \\
% WAT \cite{Piccinno:2014:FTM} & .616 & .539 & .613 & .519 & .642 & .602 & .416 & .336 \\
% \hline
% CREL* & \textbf{.748} & \textbf{.652} & .716 & .587 & .705 & .630 & .551 & .424 \\
% CREL\wast{WP} & .697 & .624 & .708 & .592 & .720 & .646 & .508 & .399 \\
% CREL\wast{NER} & .723 & .635 & .693 & .576 & .719 & .636 & .548 & .407 \\
%% CREL\wast{CONV}
% CREL & .742 & .651 & \textbf{.729} & \textbf{.597} & \textbf{.737} & \textbf{.656} & \textbf{.559} & \textbf{.429} \\
% \hline
% \end{tabular}
% }
% \end{table}
%
% \begin{table}[t]
% \small\addtolength{\tabcolsep}{-2pt}
% \centering
% \caption{\todo{Format based on our last discussion. Difficult to highlight the ``adding PE brings better performance'' claim. TODO: Update the values.}
% }
% \hjscalebox{
% \begin{tabular}{l|ccc|ccc|ccc}
% \hline
% \multicolumn{1}{Sl|}{}
% & \multicolumn{3}{Sc|}{CEL (Val)} & \multicolumn{3}{Sc|}{CEL (Test)} & \multicolumn{3}{Sc}{ConEL-PE} \\
% & \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
% & \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
% & \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc}{F} \\
% \Xhline{2pt}
% % REL & \underline{.631} & .136 & .224 & \underline{.598} & .129 & .212 & \underline{.561} & .161 & .250 \\
% % GENRE & \underline{.586} & .145 & .232 & \underline{\textbf{.672}} & .177 & .280 & \underline{.633} & .191 & .293 \\
% % TagMe & .599 & .353 & .444 & .606 & .380 & .467 & .622 & .447 & .520 \\
% % WAT & \underline{.654} & .503 & .568 & .635 & .464 & .536 & .595 & .598 & .596 \\
% % CREL\wast{CONV} & \underline{\textbf{.712}} & .531 & .608 & .629 & .500 & .557 & .598 & .628 & .613 \\
% % \hline
% REL + W2V & .529 & .160 & .246 & .523 & .161 & .246 & .488 & \underline{.206} & \underline{.290} \\
% GENRE + W2V & .472 & \underline{.182} & .262 & .585 & .229 & .329 & .537 & \underline{.256} & \underline{.347} \\
% TagMe + W2V & .504 & \underline{.407} & .451 & .530 & \underline{.440} & .481 & .528 & \underline{.518} & .523 \\
% WAT + W2V & .569 & .545 & .557 & .603 & \underline{.510} & .553 & .567 & .663 & .611 \\
% CREL\wast{CONV} + W2V & .636 & .589 & .612 & .607 & .554 & .579 & .573 & .714 & .635 \\
% \hline
% REL + S2E\wast{ON} & .531 & \underline{.164} & \underline{.251} & .529 & \underline{.167} & \underline{.253} & .466 & \underline{.206} & .286 \\
% GENRE + S2E\wast{ON} & .515 & \underline{.182} & \underline{.269} & .612 & \underline{.231} & \underline{.335} & .537 & \underline{.256} & \underline{.347} \\
% TagMe + S2E\wast{ON} & \underline{.612} & .395 & \underline{.480} & \underline{.621} & .434 & \underline{.511} & \underline{\textbf{.636}} & \underline{.518} & \underline{.571} \\
% WAT + S2E\wast{ON} & \underline{.654} & \underline{.547} & \underline{.596} & \underline{.641} & \underline{.510} & \underline{.568} & \underline{.602} & \underline{.683} & \underline{.640} \\
% CREL\wast{CONV} + S2E\wast{ON} & \underline{\textbf{.712}} & \underline{\textbf{.593}} & \underline{\textbf{.647}} & \underline{.634} & \underline{\textbf{.566}} & \underline{\textbf{.598}} & \underline{.600} & \underline{\textbf{.724}} & \underline{\textbf{.656}} \\
% \hline
%
% \end{tabular}
% }
% \end{table}
\if 0
\begin{table}[t]
\addtolength{\tabcolsep}{-2.5pt}
\centering
\caption{Overall entity linking results.
The first row for each block represents the method where only concepts and NEs are annotated.
The other rows represent the methods where personal entities are also annotated by either baseline W2V or our modified S2E\wast{ON}, respectively.
The highest score per block is marked with underlined.
\note{NOTE1: It fits without deleting the CEL (Val) column.}
\note{NOTE2: There is still a naming problem. From the table below, the reader might think that CREL is a method \emph{without a PE linking}, but that is not our intention.}
}
\label{tbl:overall}
\hjscalebox{
\begin{tabular}{l|ccc|ccc|ccc}
\hline
\multicolumn{1}{Sl|}{}
& \multicolumn{3}{Sc|}{CEL (Val)} & \multicolumn{3}{Sc|}{CEL (Test)} & \multicolumn{3}{Sc}{ConEL-PE} \\
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc}{F} \\
\Xhline{2pt}
REL & \underline{.623} & .134 & .221 & \underline{.589} & .127 & .208 & \underline{.561} & .161 & .250 \\
\hspace{1mm}+W2V & .523 & .159 & .244 & .516 & .159 & .243 & .488 & \underline{.206} & \underline{.290} \\
\hspace{1mm}+S2E\wast{ON} & .523 & \underline{.162} & \underline{.248} & .522 & \underline{.165} & \underline{.250} & .466 & \underline{.206} & .286 \\
\hline
GENRE & .579 & .143 & .229 & .649 & .171 & .270 & .633 & .191 & .293 \\
\hspace{1mm}+W2V & .468 & \underline{.180} & .260 & .569 & \underline{.223} & .320 & .537 & \underline{.256} & .347 \\
\hspace{1mm}+S2E\wast{ON} & \underline{.589} & .169 & \underline{.263} & \underline{\textbf{.675}} & .217 & \underline{.328} & \underline{\textbf{.646}} & \underline{.256} & \underline{.367} \\
\hline
TagMe & .599 & .353 & .444 & .606 & .380 & .467 & .622 & .447 & .520 \\
\hspace{1mm}+W2V & .504 & \underline{.407} & .451 & .530 & \underline{.440} & .481 & .528 & \underline{.518} & .523 \\
\hspace{1mm}+S2E\wast{ON} & \underline{.612} & .395 & \underline{.480} & \underline{.621} & .434 & \underline{.511} & \underline{.636} & \underline{.518} & \underline{.571} \\
\hline
WAT & .579 & .446 & .504 & .571 & .418 & .483 & .560 & .563 & .561 \\
\hspace{1mm}+W2V & .506 & .489 & .497 & .545 & \underline{.464} & .501 & .536 & .628 & .579 \\
\hspace{1mm}+S2E\wast{ON} & \underline{.585} & \underline{.490} & \underline{.534} & \underline{.582} & \underline{.464} & \underline{.516} & \underline{.571} & \underline{.648} & \underline{.607} \\
\hline
CREL\wast{CONV} & \underline{.712} & .531 & .608 & .629 & .500 & .557 & .598 & .628 & .613 \\
\hspace{1mm}+W2V & .636 & .589 & .612 & .607 & .554 & .579 & .573 & .714 & .635 \\
\hspace{1mm}+S2E\wast{ON} & \underline{\textbf{.712}} & \underline{\textbf{.593}} & \underline{\textbf{.647}} & \underline{.634} & \underline{\textbf{.566}} & \underline{\textbf{.598}} & \underline{.600} & \underline{\textbf{.724}} & \underline{\textbf{.656}} \\
\hline
\end{tabular}
}
\end{table}
\begin{table*}[t]
\small\addtolength{\tabcolsep}{+2pt}
\centering
\caption{\todo{Another idea. Here, we focus on using the name of our EL tool (here I use ``CREL'' as a placeholder). We can also add ``+W2V'' here. In any cases, I think this looks like unorganized. TODO: Update the values.}
}
\hjscalebox{
\begin{tabular}{l|ccc|ccc|ccc}
\hline
\multicolumn{1}{Sl|}{}
& \multicolumn{3}{Sc|}{CEL (Val)} & \multicolumn{3}{Sc|}{CEL (Test)} & \multicolumn{3}{Sc}{ConEL-PE} \\
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc|}{F}
& \multicolumn{1}{Sc}{P} & \multicolumn{1}{Sc}{R} & \multicolumn{1}{Sc}{F} \\
\Xhline{2pt}
REL & \underline{.631} & .136 & .224 & \underline{.598} & .129 & .212 & \underline{.561} & .161 & .250 \\
GENRE & \underline{.586} & .145 & .232 & \underline{\textbf{.672}} & .177 & .280 & \underline{.633} & .191 & .293 \\
TagMe & .599 & .353 & .444 & .606 & .380 & .467 & .622 & .447 & .520 \\
WAT & \underline{.654} & .503 & .568 & .635 & .464 & .536 & .595 & .598 & .596 \\
CREL\wast{CONV} & \underline{\textbf{.712}} & .531 & .608 & .629 & .500 & .557 & .598 & .628 & .613 \\
\hline
WAT + S2E\wast{ON} & \underline{.654} & \underline{.547} & \underline{.596} & \underline{.641} & \underline{.510} & \underline{.568} & \underline{.602} & \underline{.683} & \underline{.640} \\
CREL (CREL\wast{CONV} + S2E\wast{ON}) & \underline{\textbf{.712}} & \underline{\textbf{.593}} & \underline{\textbf{.647}} & \underline{.634} & \underline{\textbf{.566}} & \underline{\textbf{.598}} & \underline{.600} & \underline{\textbf{.724}} & \underline{\textbf{.656}} \\
\hline
\end{tabular}
}
\end{table*}
\fi
\section{Conclusion}
In this paper, we tackled the problem of entity linking in conversational settings, where not only named entities, but also concepts and personal entities are important.
To this end, we collected conversational entity annotations, which allows us to train and evaluate entity linking in conversations.
Additionally, based on the collected annotations, we develop a conversational EL tool.
Our empirical results show that our EL tool achieves the highest performance over conventional EL tools for documents and short texts.
The collected annotations and our EL tool with detailed instructions are publicly available as a research resource for further study in conversation.
Future work includes incorporating coreference resolution methods to identify more diverse personal entity mentions.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{36}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Balog and Kenter(2019)]%
{Balog:2019:PKG}
\bibfield{author}{\bibinfo{person}{Krisztian Balog} {and} \bibinfo{person}{Tom
Kenter}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Personal Knowledge Graphs: A Research Agenda}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2019 ACM SIGIR International
Conference on Theory of Information Retrieval}} \emph{(\bibinfo{series}{ICTIR
'19})}. \bibinfo{pages}{217--220}.
\newblock
\bibitem[Beltagy et~al\mbox{.}(2020)]%
{Beltagy:2020:Longformer}
\bibfield{author}{\bibinfo{person}{Iz Beltagy}, \bibinfo{person}{Matthew~E.
Peters}, {and} \bibinfo{person}{Arman Cohan}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Longformer: The Long-Document Transformer}.
\newblock \bibinfo{journal}{\emph{arXiv:2004.05150}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Cao et~al\mbox{.}(2021)]%
{Cao:2021:GENRE}
\bibfield{author}{\bibinfo{person}{Nicola~De Cao}, \bibinfo{person}{Gautier
Izacard}, \bibinfo{person}{Sebastian Riedel}, {and} \bibinfo{person}{Fabio
Petroni}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Autoregressive Entity Retrieval}. In
\bibinfo{booktitle}{\emph{Proceedings of International Conference on Learning
Representations}} \emph{(\bibinfo{series}{ICLR '21})}.
\newblock
\bibitem[Dalton et~al\mbox{.}(2014)]%
{Dalton:2014:EQF}
\bibfield{author}{\bibinfo{person}{Jeffrey Dalton}, \bibinfo{person}{Laura
Dietz}, {and} \bibinfo{person}{James Allan}.}
\bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{{Entity Query Feature Expansion Using Knowledge
Base Links}}. In \bibinfo{booktitle}{\emph{Proceedings of the 37th
International ACM SIGIR Conference on Research and Development in Information
Retrieval}} \emph{(\bibinfo{series}{SIGIR '14})}. \bibinfo{pages}{365--374}.
\newblock
\bibitem[Dargahi~Nobari et~al\mbox{.}(2018)]%
{Dargahi:2018:QUE}
\bibfield{author}{\bibinfo{person}{Arash Dargahi~Nobari},
\bibinfo{person}{Arian Askari}, \bibinfo{person}{Faegheh Hasibi}, {and}
\bibinfo{person}{Mahmood Neshati}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Query Understanding via Entity Attribute
Identification}. In \bibinfo{booktitle}{\emph{Proceedings of the 27th ACM
International Conference on Information and Knowledge Management}}
\emph{(\bibinfo{series}{CIKM '18})}. \bibinfo{pages}{1759--1762}.
\newblock
\bibitem[Devlin et~al\mbox{.}(2019)]%
{Devlin:2019:BER}
\bibfield{author}{\bibinfo{person}{Jacob Devlin}, \bibinfo{person}{Ming-Wei
Chang}, \bibinfo{person}{Kenton Lee}, {and} \bibinfo{person}{Kristina
Toutanova}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{BERT}: Pre-training of Deep Bidirectional
Transformers for Language Understanding}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2019 Conference of the North
{A}merican Chapter of the Association for Computational Linguistics: Human
Language Technologies}} \emph{(\bibinfo{series}{NAACL-HLT '19})}.
\bibinfo{pages}{4171--4186}.
\newblock
\bibitem[Dinan et~al\mbox{.}(2019)]%
{Dinan:2018:WWK}
\bibfield{author}{\bibinfo{person}{Emily Dinan}, \bibinfo{person}{Stephen
Roller}, \bibinfo{person}{Kurt Shuster}, \bibinfo{person}{Angela Fan},
\bibinfo{person}{Michael Auli}, {and} \bibinfo{person}{Jason Weston}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Wizard of Wikipedia: Knowledge-Powered
Conversational Agents}. In \bibinfo{booktitle}{\emph{International Conference
on Learning Representations}} \emph{(\bibinfo{series}{ICLR '19})}.
\newblock
\bibitem[Ferragina and Scaiella(2010)]%
{Ferragina:2010:TAG}
\bibfield{author}{\bibinfo{person}{Paolo Ferragina} {and} \bibinfo{person}{Ugo
Scaiella}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{TAGME: On-the-fly Annotation of Short Text
Fragments (by Wikipedia Entities)}. In \bibinfo{booktitle}{\emph{Proceedings
of the 19th ACM International Conference on Information and Knowledge
Management}} \emph{(\bibinfo{series}{CIKM '10})}.
\bibinfo{pages}{1625--1628}.
\newblock
\bibitem[Gerritse et~al\mbox{.}(2022)]%
{Gerritse:2022:ETE}
\bibfield{author}{\bibinfo{person}{Emma~J Gerritse}, \bibinfo{person}{Faegheh
Hasibi}, {and} \bibinfo{person}{Arjen~P. de Vries}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Entity-aware Transformers for Entity Search}. In
\bibinfo{booktitle}{\emph{Proceedings of the 45th International ACM SIGIR
Conference on Research and Development in Information Retrieval}}
\emph{(\bibinfo{series}{SIGIR '22})}. \bibinfo{pages}{1455–1465}.
\newblock
\bibitem[Hasibi et~al\mbox{.}(2016)]%
{Hasibi:2016:EEL}
\bibfield{author}{\bibinfo{person}{Faegheh Hasibi}, \bibinfo{person}{Krisztian
Balog}, {and} \bibinfo{person}{Svein~Erik Bratsberg}.}
\bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{{Exploiting Entity Linking in Queries for Entity
Retrieval}}. In \bibinfo{booktitle}{\emph{Proceedings of the 2016 ACM on
International Conference on the Theory of Information Retrieval}}
\emph{(\bibinfo{series}{ICTIR '16})}. \bibinfo{pages}{209--218}.
\newblock
\bibitem[Hovy et~al\mbox{.}(2006)]%
{Hovy:2006:ON9}
\bibfield{author}{\bibinfo{person}{Eduard Hovy}, \bibinfo{person}{Mitchell
Marcus}, \bibinfo{person}{Martha Palmer}, \bibinfo{person}{Lance Ramshaw},
{and} \bibinfo{person}{Ralph Weischedel}.} \bibinfo{year}{2006}\natexlab{}.
\newblock \showarticletitle{{O}nto{N}otes: The 90{\%} Solution}. In
\bibinfo{booktitle}{\emph{Proceedings of the Human Language Technology
Conference of the {NAACL}}}. \bibinfo{pages}{57--60}.
\newblock
\bibitem[Joko et~al\mbox{.}(2021)]%
{Joko:2021:CEL}
\bibfield{author}{\bibinfo{person}{Hideaki Joko}, \bibinfo{person}{Faegheh
Hasibi}, \bibinfo{person}{Krisztian Balog}, {and} \bibinfo{person}{Arjen~P.
de Vries}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Conversational Entity Linking: Problem Definition
and Datasets}. In \bibinfo{booktitle}{\emph{Proceedings of the 44rd
International ACM SIGIR Conference on Research and Development in Information
Retrieval}} \emph{(\bibinfo{series}{SIGIR '21})}.
\bibinfo{pages}{2390–2397}.
\newblock
\bibitem[Joshi et~al\mbox{.}(2017)]%
{Joshi:2017:PGD}
\bibfield{author}{\bibinfo{person}{Chaitanya~K Joshi}, \bibinfo{person}{Fei
Mi}, {and} \bibinfo{person}{Boi Faltings}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Personalization in Goal-oriented Dialog}. In
\bibinfo{booktitle}{\emph{Proceedings of the NeurIPS’17 Workshop on
Conversational AI}}.
\newblock
\bibitem[Joshi et~al\mbox{.}(2019)]%
{Joshi:2019:BFC}
\bibfield{author}{\bibinfo{person}{Mandar Joshi}, \bibinfo{person}{Omer Levy},
\bibinfo{person}{Luke Zettlemoyer}, {and} \bibinfo{person}{Daniel Weld}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{BERT} for Coreference Resolution: Baselines and
Analysis}. In \bibinfo{booktitle}{\emph{Proceedings of the 2019 Conference on
Empirical Methods in Natural Language Processing and the 9th International
Joint Conference on Natural Language Processing}}
\emph{(\bibinfo{series}{EMNLP-IJCNLP '19})}. \bibinfo{pages}{5802--5807}.
\newblock
\bibitem[Kirstain et~al\mbox{.}(2021)]%
{Kirstain:2021:CRS}
\bibfield{author}{\bibinfo{person}{Yuval Kirstain}, \bibinfo{person}{Ori Ram},
{and} \bibinfo{person}{Omer Levy}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Coreference Resolution without Span
Representations}. In \bibinfo{booktitle}{\emph{Proceedings of the 59th Annual
Meeting of the Association for Computational Linguistics and the 11th
International Joint Conference on Natural Language Processing}}
\emph{(\bibinfo{series}{ACL-IJCNLP '21})}. \bibinfo{pages}{14--19}.
\newblock
\bibitem[Laskar et~al\mbox{.}(2022)]%
{Laskar:2022:BEE}
\bibfield{author}{\bibinfo{person}{Md~Tahmid~Rahman Laskar},
\bibinfo{person}{Cheng Chen}, \bibinfo{person}{Aliaksandr Martsinovich},
\bibinfo{person}{Jonathan Johnston}, \bibinfo{person}{Xue-Yong Fu},
\bibinfo{person}{Shashi~Bhushan Tn}, {and} \bibinfo{person}{Simon
Corston-Oliver}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{{BLINK} with {E}lasticsearch for Efficient Entity
Linking in Business Conversations}. In \bibinfo{booktitle}{\emph{Proceedings
of the 2022 Conference of the North American Chapter of the Association for
Computational Linguistics: Human Language Technologies}}
\emph{(\bibinfo{series}{NAACL-HLT '22})}. \bibinfo{pages}{344--352}.
\newblock
\bibitem[Lee et~al\mbox{.}(2017)]%
{Lee:2017:E2E}
\bibfield{author}{\bibinfo{person}{Kenton Lee}, \bibinfo{person}{Luheng He},
\bibinfo{person}{Mike Lewis}, {and} \bibinfo{person}{Luke Zettlemoyer}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{End-to-end Neural Coreference Resolution}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2017 Conference on Empirical
Methods in Natural Language Processing}}. \bibinfo{pages}{188--197}.
\newblock
\bibitem[Lee et~al\mbox{.}(2018)]%
{Lee:2018:HOC}
\bibfield{author}{\bibinfo{person}{Kenton Lee}, \bibinfo{person}{Luheng He},
{and} \bibinfo{person}{Luke Zettlemoyer}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Higher-Order Coreference Resolution with
Coarse-to-Fine Inference}. In \bibinfo{booktitle}{\emph{Proceedings of the
2018 Conference of the North {A}merican Chapter of the Association for
Computational Linguistics: Human Language Technologies}}
\emph{(\bibinfo{series}{NAACL-HLT '18})}. \bibinfo{pages}{687--692}.
\newblock
\bibitem[{Li} et~al\mbox{.}(2014)]%
{Li:2014:PKG}
\bibfield{author}{\bibinfo{person}{X. {Li}}, \bibinfo{person}{G. {Tur}},
\bibinfo{person}{D. {Hakkani-Tür}}, {and} \bibinfo{person}{Q. {Li}}.}
\bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Personal knowledge graph population from user
utterances in conversational understanding}. In
\bibinfo{booktitle}{\emph{2014 IEEE Spoken Language Technology Workshop}}.
\bibinfo{pages}{224--229}.
\newblock
\bibitem[Li et~al\mbox{.}(2017)]%
{Li:2017:DDM}
\bibfield{author}{\bibinfo{person}{Yanran Li}, \bibinfo{person}{Hui Su},
\bibinfo{person}{Xiaoyu Shen}, \bibinfo{person}{Wenjie Li},
\bibinfo{person}{Ziqiang Cao}, {and} \bibinfo{person}{Shuzi Niu}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{{D}aily{D}ialog: A Manually Labelled Multi-turn
Dialogue Dataset}. In \bibinfo{booktitle}{\emph{Proceedings of the Eighth
International Joint Conference on Natural Language Processing}}
\emph{(\bibinfo{series}{IJCNLP '17})}. \bibinfo{pages}{986--995}.
\newblock
\bibitem[Lison and Tiedemann(2016)]%
{Lison:2016:OSE}
\bibfield{author}{\bibinfo{person}{Pierre Lison} {and}
\bibinfo{person}{J{\"o}rg Tiedemann}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{{O}pen{S}ubtitles2016: Extracting Large Parallel
Corpora from Movie and {TV} Subtitles}. In
\bibinfo{booktitle}{\emph{Proceedings of the Tenth International Conference
on Language Resources and Evaluation ({LREC}'16)}}.
\bibinfo{pages}{923--929}.
\newblock
\bibitem[Luo et~al\mbox{.}(2019)]%
{Luo:2019:LPE}
\bibfield{author}{\bibinfo{person}{Liangchen Luo}, \bibinfo{person}{Wenhao
Huang}, \bibinfo{person}{Qi Zeng}, \bibinfo{person}{Zaiqing Nie}, {and}
\bibinfo{person}{Xu Sun}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Learning Personalized End-to-end Goal-oriented
Dialog}. In \bibinfo{booktitle}{\emph{Proceedings of the AAAI Conference on
Artificial Intelligence}} \emph{(\bibinfo{series}{AAAI '19})}.
\bibinfo{pages}{6794--6801}.
\newblock
\bibitem[Piccinno and Ferragina(2014)]%
{Piccinno:2014:FTM}
\bibfield{author}{\bibinfo{person}{Francesco Piccinno} {and}
\bibinfo{person}{Paolo Ferragina}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{From TagME to WAT: A New Entity Annotator}. In
\bibinfo{booktitle}{\emph{Proceedings of the First International Workshop on
Entity Recognition and Disambiguation}}. \bibinfo{pages}{55--62}.
\newblock
\bibitem[Schler et~al\mbox{.}(2006)]%
{Schler:2006:EAG}
\bibfield{author}{\bibinfo{person}{J Schler}, \bibinfo{person}{M Koppel},
\bibinfo{person}{S Argamon}, {and} \bibinfo{person}{JW Pennebaker}.}
\bibinfo{year}{2006}\natexlab{}.
\newblock \showarticletitle{Effects of Age and Gender on Blogging}. In
\bibinfo{booktitle}{\emph{Proceedings of 2006 AAAI Spring Symposium on
Computational Approaches for Analyzing Weblogs}}.
\newblock
\bibitem[Shang et~al\mbox{.}(2021)]%
{Shang:2021:ERO}
\bibfield{author}{\bibinfo{person}{Mingyue Shang}, \bibinfo{person}{Tong Wang},
\bibinfo{person}{Mihail Eric}, \bibinfo{person}{Jiangning Chen},
\bibinfo{person}{Jiyang Wang}, \bibinfo{person}{Matthew Welch},
\bibinfo{person}{Tiantong Deng}, \bibinfo{person}{Akshay Grewal},
\bibinfo{person}{Han Wang}, \bibinfo{person}{Yue Liu}, \bibinfo{person}{Yang
Liu}, {and} \bibinfo{person}{Dilek Hakkani-Tur}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Entity Resolution in Open-domain Conversations}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2021 Conference of the North
American Chapter of the Association for Computational Linguistics: Human
Language Technologies}} \emph{(\bibinfo{series}{NAACL-HLT '21})}.
\bibinfo{pages}{26--33}.
\newblock
\bibitem[Singh et~al\mbox{.}(2011)]%
{Singh:2011:LCD}
\bibfield{author}{\bibinfo{person}{Sameer Singh}, \bibinfo{person}{Amarnag
Subramanya}, \bibinfo{person}{Fernando Pereira}, {and}
\bibinfo{person}{Andrew McCallum}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{{Large-Scale Cross-Document Coreference Using
Distributed Inference and Hierarchical Models}}. In
\bibinfo{booktitle}{\emph{Proceedings of the 49th Annual Meeting of the
Association for Computational Linguistics: Human Language Technologies}}
\emph{(\bibinfo{series}{ACL-HLT '11})}. \bibinfo{pages}{793--803}.
\newblock
\bibitem[Tigunova et~al\mbox{.}(2019)]%
{Tigunova:2019:LBL}
\bibfield{author}{\bibinfo{person}{Anna Tigunova}, \bibinfo{person}{Andrew
Yates}, \bibinfo{person}{Paramita Mirza}, {and} \bibinfo{person}{Gerhard
Weikum}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Listening between the Lines: Learning Personal
Attributes from Conversations}. In \bibinfo{booktitle}{\emph{Proceedings of
the World Wide Web Conference}} \emph{(\bibinfo{series}{WWW '19})}.
\bibinfo{pages}{1818--1828}.
\newblock
\bibitem[Tigunova et~al\mbox{.}(2020)]%
{Tigunova:2020:CHA}
\bibfield{author}{\bibinfo{person}{Anna Tigunova}, \bibinfo{person}{Andrew
Yates}, \bibinfo{person}{Paramita Mirza}, {and} \bibinfo{person}{Gerhard
Weikum}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{{CHARM}: Inferring Personal Attributes from
Conversations}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020
Conference on Empirical Methods in Natural Language Processing (EMNLP)}}.
\bibinfo{pages}{5391--5404}.
\newblock
\bibitem[van Hulst et~al\mbox{.}(2020)]%
{vanHulst:2020:REL}
\bibfield{author}{\bibinfo{person}{Johannes~M. van Hulst},
\bibinfo{person}{Faegheh Hasibi}, \bibinfo{person}{Koen Dercksen},
\bibinfo{person}{Krisztian Balog}, {and} \bibinfo{person}{Arjen~P. de
Vries}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{REL: An Entity Linker Standing on the Shoulders of
Giants}. In \bibinfo{booktitle}{\emph{Proceedings of the 43rd International
ACM SIGIR Conference on Research and Development in Information Retrieval}}
\emph{(\bibinfo{series}{SIGIR '20})}. \bibinfo{pages}{2197--2200}.
\newblock
\bibitem[Wu et~al\mbox{.}(2020)]%
{Wu:2020:BLINK}
\bibfield{author}{\bibinfo{person}{Ledell Wu}, \bibinfo{person}{Fabio Petroni},
\bibinfo{person}{Martin Josifoski}, \bibinfo{person}{Sebastian Riedel}, {and}
\bibinfo{person}{Luke Zettlemoyer}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{{Scalable Zero-shot Entity Linking with Dense
Entity Retrieval}}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020
Conference on Empirical Methods in Natural Language Processing (EMNLP)}}.
\bibinfo{pages}{6397--6407}.
\newblock
\bibitem[Xiong et~al\mbox{.}(2017)]%
{Xiong:2017:WED}
\bibfield{author}{\bibinfo{person}{Chenyan Xiong}, \bibinfo{person}{Jamie
Callan}, {and} \bibinfo{person}{Tie-Yan Liu}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{{Word-Entity Duet Representations for Document
Ranking}}. In \bibinfo{booktitle}{\emph{Proceedings of the 40th International
ACM SIGIR Conference on Research and Development in Information Retrieval}}
\emph{(\bibinfo{series}{SIGIR '17})}. \bibinfo{pages}{763--772}.
\newblock
\bibitem[Xu and Choi(2022)]%
{Xu:2022:OCR}
\bibfield{author}{\bibinfo{person}{Liyan Xu} {and} \bibinfo{person}{Jinho~D
Choi}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Online Coreference Resolution for Dialogue
Processing: Improving Mention-Linking on Real-Time Conversations}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2205.10670}}
(\bibinfo{year}{2022}).
\newblock
\bibitem[Yamada et~al\mbox{.}(2020)]%
{Yamada:2020:W2V}
\bibfield{author}{\bibinfo{person}{Ikuya Yamada}, \bibinfo{person}{Akari Asai},
\bibinfo{person}{Jin Sakuma}, \bibinfo{person}{Hiroyuki Shindo},
\bibinfo{person}{Hideaki Takeda}, \bibinfo{person}{Yoshiyasu Takefuji}, {and}
\bibinfo{person}{Yuji Matsumoto}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{{W}ikipedia2{V}ec: An Efficient Toolkit for
Learning and Visualizing the Embeddings of Words and Entities from
{W}ikipedia}. In \bibinfo{booktitle}{\emph{Conference on Empirical Methods in
Natural Language Processing}} \emph{(\bibinfo{series}{EMNLP '20})}.
\bibinfo{pages}{23--30}.
\newblock
\bibitem[Yamada et~al\mbox{.}(2018)]%
{Yamada:2018:SOQ}
\bibfield{author}{\bibinfo{person}{Ikuya Yamada}, \bibinfo{person}{Ryuji
Tamaki}, \bibinfo{person}{Hiroyuki Shindo}, {and} \bibinfo{person}{Yoshiyasu
Takefuji}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Studio Ousia's Quiz Bowl Question Answering
System}. In \bibinfo{booktitle}{\emph{Proceedings of the NIPS '17
Competition: Building Intelligent Systems}}. \bibinfo{pages}{181--194}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2016)]%
{Zhang:2016:CFO}
\bibfield{author}{\bibinfo{person}{Justine Zhang}, \bibinfo{person}{Ravi
Kumar}, \bibinfo{person}{Sujith Ravi}, {and} \bibinfo{person}{Cristian
Danescu-Niculescu-Mizil}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Conversational Flow in {O}xford-style Debates}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2016 Conference of the North
{A}merican Chapter of the Association for Computational Linguistics: Human
Language Technologies}} \emph{(\bibinfo{series}{NAACL-HLT '16})}.
\bibinfo{pages}{136--141}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2022)]%
{Zhang:2022:ENT}
\bibfield{author}{\bibinfo{person}{Wenzheng Zhang}, \bibinfo{person}{Wenyue
Hua}, {and} \bibinfo{person}{Karl Stratos}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{{Ent{QA}: Entity Linking as Question Answering}}.
In \bibinfo{booktitle}{\emph{Proceedings of the International Conference on
Learning Representations}} \emph{(\bibinfo{series}{ICLR '22})}.
\newblock
\end{thebibliography}