ANEA: Automated (Named) Entity Annotation for German Domain-Specific Texts
Extracted source text · arXiv 2112.06724
%%
%% This is file `sample-manuscript.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `manuscript')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-manuscript.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% The first command in your LaTeX source must be the \documentclass command.
%%%% Small single column format, used for CIE, CSUR, DTRAP, JACM, JDIQ, JEA, JERIC, JETC, PACMCGIT, TAAS, TACCESS, TACO, TALG, TALLIP (formerly TALIP), TCPS, TDSCI, TEAC, TECS, TELO, THRI, TIIS, TIOT, TISSEC, TIST, TKDD, TMIS, TOCE, TOCHI, TOCL, TOCS, TOCT, TODAES, TODS, TOIS, TOIT, TOMACS, TOMM (formerly TOMCCAP), TOMPECS, TOMS, TOPC, TOPLAS, TOPS, TOS, TOSEM, TOSN, TQC, TRETS, TSAS, TSC, TSLP, TWEB.
% \documentclass[acmsmall]{acmart}
%%%% Large single column format, used for IMWUT, JOCCH, PACMPL, POMACS, TAP, PACMHCI
% \documentclass[acmlarge,screen]{acmart}
%%%% Large double column format, used for TOG
% \documentclass[acmtog, authorversion]{acmart}
%%%% Generic manuscript mode, required for submission
%%%% and peer review
\documentclass[manuscript,screen,review]{acmart}
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmcopyright}
\copyrightyear{2018}
\acmYear{2018}
\acmDOI{10.1145/1122445.1122456}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[Woodstock '18]{Woodstock '18: ACM Symposium on Neural
Gaze Detection}{June 03--05, 2018}{Woodstock, NY}
\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
June 03--05, 2018, Woodstock, NY}
\acmPrice{15.00}
\acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{The Name of the Title is Hope}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Ben Trovato}
\authornote{Both authors contributed equally to this research.}
\email{trovato@corporation.com}
\orcid{1234-5678-9012}
\author{G.K.M. Tobin}
\authornotemark[1]
\email{webmaster@marysville-ohio.com}
\affiliation{%
\institution{Institute for Clarity in Documentation}
\streetaddress{P.O. Box 1212}
\city{Dublin}
\state{Ohio}
\country{USA}
\postcode{43017-6221}
}
\author{Lars Th{\o}rv{\"a}ld}
\affiliation{%
\institution{The Th{\o}rv{\"a}ld Group}
\streetaddress{1 Th{\o}rv{\"a}ld Circle}
\city{Hekla}
\country{Iceland}}
\email{larst@affiliation.org}
\author{Valerie B\'eranger}
\affiliation{%
\institution{Inria Paris-Rocquencourt}
\city{Rocquencourt}
\country{France}
}
\author{Aparna Patel}
\affiliation{%
\institution{Rajiv Gandhi University}
\streetaddress{Rono-Hills}
\city{Doimukh}
\state{Arunachal Pradesh}
\country{India}}
\author{Huifen Chan}
\affiliation{%
\institution{Tsinghua University}
\streetaddress{30 Shuangqing Rd}
\city{Haidian Qu}
\state{Beijing Shi}
\country{China}}
\author{Charles Palmer}
\affiliation{%
\institution{Palmer Research Laboratories}
\streetaddress{8600 Datapoint Drive}
\city{San Antonio}
\state{Texas}
\country{USA}
\postcode{78229}}
\email{cpalmer@prl.com}
\author{John Smith}
\affiliation{%
\institution{The Th{\o}rv{\"a}ld Group}
\streetaddress{1 Th{\o}rv{\"a}ld Circle}
\city{Hekla}
\country{Iceland}}
\email{jsmith@affiliation.org}
\author{Julius P. Kumquat}
\affiliation{%
\institution{The Kumquat Consortium}
\city{New York}
\country{USA}}
\email{jpkumquat@consortium.net}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Trovato and Tobin, et al.}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
A clear and well-documented \LaTeX\ document is presented as an
article formatted for publication by ACM in a conference proceedings
or journal publication. Based on the ``acmart'' document class, this
article presents and explains many of the common variations, as well
as many of the formatting elements an author may use in the
preparation of the documentation of their work.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10010520.10010553.10010562</concept_id>
<concept_desc>Computer systems organization~Embedded systems</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010575.10010755</concept_id>
<concept_desc>Computer systems organization~Redundancy</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010553.10010554</concept_id>
<concept_desc>Computer systems organization~Robotics</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10003033.10003083.10003095</concept_id>
<concept_desc>Networks~Network reliability</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Computer systems organization~Embedded systems}
\ccsdesc[300]{Computer systems organization~Redundancy}
\ccsdesc{Computer systems organization~Robotics}
\ccsdesc[100]{Networks~Network reliability}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{datasets, neural networks, gaze detection, text tagging}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
\begin{teaserfigure}
\includegraphics[width=\textwidth]{sampleteaser}
\caption{Seattle Mariners at Spring Training, 2010.}
\Description{Enjoying the baseball game from the third-base
seats. Ichiro Suzuki preparing to bat.}
\label{fig:teaser}
\end{teaserfigure}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
ACM's consolidated article template, introduced in 2017, provides a
consistent \LaTeX\ style for use across ACM publications, and
incorporates accessibility and metadata-extraction functionality
necessary for future Digital Library endeavors. Numerous ACM and
SIG-specific \LaTeX\ templates have been examined, and their unique
features incorporated into this single new template.
If you are new to publishing with ACM, this document is a valuable
guide to the process of preparing your work for publication. If you
have published with ACM before, this document provides insight and
instruction into more recent changes to the article template.
The ``\verb|acmart|'' document class can be used to prepare articles
for any ACM publication --- conference or journal, and for any stage
of publication, from review to final ``camera-ready'' copy, to the
author's own version, with {\itshape very} few changes to the source.
\section{Template Overview}
As noted in the introduction, the ``\verb|acmart|'' document class can
be used to prepare many different kinds of documentation --- a
double-blind initial submission of a full-length technical paper, a
two-page SIGGRAPH Emerging Technologies abstract, a ``camera-ready''
journal article, a SIGCHI Extended Abstract, and more --- all by
selecting the appropriate {\itshape template style} and {\itshape
template parameters}.
This document will explain the major features of the document
class. For further information, the {\itshape \LaTeX\ User's Guide} is
available from
\url{https://www.acm.org/publications/proceedings-template}.
\subsection{Template Styles}
The primary parameter given to the ``\verb|acmart|'' document class is
the {\itshape template style} which corresponds to the kind of publication
or SIG publishing the work. This parameter is enclosed in square
brackets and is a part of the {\verb|documentclass|} command:
\begin{verbatim}
\documentclass[STYLE]{acmart}
\end{verbatim}
Journals use one of three template styles. All but three ACM journals
use the {\verb|acmsmall|} template style:
\begin{itemize}
\item {\verb|acmsmall|}: The default journal template style.
\item {\verb|acmlarge|}: Used by JOCCH and TAP.
\item {\verb|acmtog|}: Used by TOG.
\end{itemize}
The majority of conference proceedings documentation will use the {\verb|acmconf|} template style.
\begin{itemize}
\item {\verb|acmconf|}: The default proceedings template style.
\item{\verb|sigchi|}: Used for SIGCHI conference articles.
\item{\verb|sigchi-a|}: Used for SIGCHI ``Extended Abstract'' articles.
\item{\verb|sigplan|}: Used for SIGPLAN conference articles.
\end{itemize}
\subsection{Template Parameters}
In addition to specifying the {\itshape template style} to be used in
formatting your work, there are a number of {\itshape template parameters}
which modify some part of the applied template style. A complete list
of these parameters can be found in the {\itshape \LaTeX\ User's Guide.}
Frequently-used parameters, or combinations of parameters, include:
\begin{itemize}
\item {\verb|anonymous,review|}: Suitable for a ``double-blind''
conference submission. Anonymizes the work and includes line
numbers. Use with the \verb|\acmSubmissionID| command to print the
submission's unique ID on each page of the work.
\item{\verb|authorversion|}: Produces a version of the work suitable
for posting by the author.
\item{\verb|screen|}: Produces colored hyperlinks.
\end{itemize}
This document uses the following string as the first command in the
source file:
\begin{verbatim}
\documentclass[sigconf,authordraft]{acmart}
\end{verbatim}
\section{Modifications}
Modifying the template --- including but not limited to: adjusting
margins, typeface sizes, line spacing, paragraph and list definitions,
and the use of the \verb|\vspace| command to manually adjust the
vertical spacing between elements of your work --- is not allowed.
{\bfseries Your document will be returned to you for revision if
modifications are discovered.}
\section{Typefaces}
The ``\verb|acmart|'' document class requires the use of the
``Libertine'' typeface family. Your \TeX\ installation should include
this set of packages. Please do not substitute other typefaces. The
``\verb|lmodern|'' and ``\verb|ltimes|'' packages should not be used,
as they will override the built-in typeface families.
\section{Title Information}
The title of your work should use capital letters appropriately -
\url{https://capitalizemytitle.com/} has useful rules for
capitalization. Use the {\verb|title|} command to define the title of
your work. If your work has a subtitle, define it with the
{\verb|subtitle|} command. Do not insert line breaks in your title.
If your title is lengthy, you must define a short version to be used
in the page headers, to prevent overlapping text. The \verb|title|
command has a ``short title'' parameter:
\begin{verbatim}
\title[short title]{full title}
\end{verbatim}
\section{Authors and Affiliations}
Each author must be defined separately for accurate metadata
identification. Multiple authors may share one affiliation. Authors'
names should not be abbreviated; use full first names wherever
possible. Include authors' e-mail addresses whenever possible.
Grouping authors' names or e-mail addresses, or providing an ``e-mail
alias,'' as shown below, is not acceptable:
\begin{verbatim}
\author{Brooke Aster, David Mehldau}
\email{dave,judy,steve@university.edu}
\email{firstname.lastname@phillips.org}
\end{verbatim}
The \verb|authornote| and \verb|authornotemark| commands allow a note
to apply to multiple authors --- for example, if the first two authors
of an article contributed equally to the work.
If your author list is lengthy, you must define a shortened version of
the list of authors to be used in the page headers, to prevent
overlapping text. The following command should be placed just after
the last \verb|\author{}| definition:
\begin{verbatim}
\renewcommand{\shortauthors}{McCartney, et al.}
\end{verbatim}
Omitting this command will force the use of a concatenated list of all
of the authors' names, which may result in overlapping text in the
page headers.
The article template's documentation, available at
\url{https://www.acm.org/publications/proceedings-template}, has a
complete explanation of these commands and tips for their effective
use.
Note that authors' addresses are mandatory for journal articles.
\section{Rights Information}
Authors of any work published by ACM will need to complete a rights
form. Depending on the kind of work, and the rights management choice
made by the author, this may be copyright transfer, permission,
license, or an OA (open access) agreement.
Regardless of the rights management choice, the author will receive a
copy of the completed rights form once it has been submitted. This
form contains \LaTeX\ commands that must be copied into the source
document. When the document source is compiled, these commands and
their parameters add formatted text to several areas of the final
document:
\begin{itemize}
\item the ``ACM Reference Format'' text on the first page.
\item the ``rights management'' text on the first page.
\item the conference information in the page header(s).
\end{itemize}
Rights information is unique to the work; if you are preparing several
works for an event, make sure to use the correct set of commands with
each of the works.
The ACM Reference Format text is required for all articles over one
page in length, and is optional for one-page articles (abstracts).
\section{CCS Concepts and User-Defined Keywords}
Two elements of the ``acmart'' document class provide powerful
taxonomic tools for you to help readers find your work in an online
search.
The ACM Computing Classification System ---
\url{https://www.acm.org/publications/class-2012} --- is a set of
classifiers and concepts that describe the computing
discipline. Authors can select entries from this classification
system, via \url{https://dl.acm.org/ccs/ccs.cfm}, and generate the
commands to be included in the \LaTeX\ source.
User-defined keywords are a comma-separated list of words and phrases
of the authors' choosing, providing a more flexible way of describing
the research being presented.
CCS concepts and user-defined keywords are required for for all
articles over two pages in length, and are optional for one- and
two-page articles (or abstracts).
\section{Sectioning Commands}
Your work should use standard \LaTeX\ sectioning commands:
\verb|section|, \verb|subsection|, \verb|subsubsection|, and
\verb|paragraph|. They should be numbered; do not remove the numbering
from the commands.
Simulating a sectioning command by setting the first word or words of
a paragraph in boldface or italicized text is {\bfseries not allowed.}
\section{Tables}
The ``\verb|acmart|'' document class includes the ``\verb|booktabs|''
package --- \url{https://ctan.org/pkg/booktabs} --- for preparing
high-quality tables.
Table captions are placed {\itshape above} the table.
Because tables cannot be split across pages, the best placement for
them is typically the top of the page nearest their initial cite. To
ensure this proper ``floating'' placement of tables, use the
environment \textbf{table} to enclose the table's contents and the
table caption. The contents of the table itself must go in the
\textbf{tabular} environment, to be aligned properly in rows and
columns, with the desired horizontal and vertical rules. Again,
detailed instructions on \textbf{tabular} material are found in the
\textit{\LaTeX\ User's Guide}.
Immediately following this sentence is the point at which
Table~\ref{tab:freq} is included in the input file; compare the
placement of the table here with the table in the printed output of
this document.
\begin{table}
\caption{Frequency of Special Characters}
\label{tab:freq}
\begin{tabular}{ccl}
\toprule
Non-English or Math&Frequency&Comments\\
\midrule
\O & 1 in 1,000& For Swedish names\\
$\pi$ & 1 in 5& Common in math\\
\$ & 4 in 5 & Used in business\\
$\Psi^2_1$ & 1 in 40,000& Unexplained usage\\
\bottomrule
\end{tabular}
\end{table}
To set a wider table, which takes up the whole width of the page's
live area, use the environment \textbf{table*} to enclose the table's
contents and the table caption. As with a single-column table, this
wide table will ``float'' to a location deemed more
desirable. Immediately following this sentence is the point at which
Table~\ref{tab:commands} is included in the input file; again, it is
instructive to compare the placement of the table here with the table
in the printed output of this document.
\begin{table*}
\caption{Some Typical Commands}
\label{tab:commands}
\begin{tabular}{ccl}
\toprule
Command &A Number & Comments\\
\midrule
\texttt{{\char'134}author} & 100& Author \\
\texttt{{\char'134}table}& 300 & For tables\\
\texttt{{\char'134}table*}& 400& For wider tables\\
\bottomrule
\end{tabular}
\end{table*}
Always use midrule to separate table header rows from data rows, and
use it only for this purpose. This enables assistive technologies to
recognise table headers and support their users in navigating tables
more easily.
\section{Math Equations}
You may want to display math equations in three distinct styles:
inline, numbered or non-numbered display. Each of the three are
discussed in the next sections.
\subsection{Inline (In-text) Equations}
A formula that appears in the running text is called an inline or
in-text formula. It is produced by the \textbf{math} environment,
which can be invoked with the usual
\texttt{{\char'134}begin\,\ldots{\char'134}end} construction or with
the short form \texttt{\$\,\ldots\$}. You can use any of the symbols
and structures, from $\alpha$ to $\omega$, available in
\LaTeX~\cite{Lamport:LaTeX}; this section will simply show a few
examples of in-text equations in context. Notice how this equation:
\begin{math}
\lim_{n\rightarrow \infty}x=0
\end{math},
set here in in-line math style, looks slightly different when
set in display style. (See next section).
\subsection{Display Equations}
A numbered display equation---one set off by vertical space from the
text and centered horizontally---is produced by the \textbf{equation}
environment. An unnumbered display equation is produced by the
\textbf{displaymath} environment.
Again, in either environment, you can use any of the symbols and
structures available in \LaTeX\@; this section will just give a couple
of examples of display equations in context. First, consider the
equation, shown as an inline equation above:
\begin{equation}
\lim_{n\rightarrow \infty}x=0
\end{equation}
Notice how it is formatted somewhat differently in
the \textbf{displaymath}
environment. Now, we'll enter an unnumbered equation:
\begin{displaymath}
\sum_{i=0}^{\infty} x + 1
\end{displaymath}
and follow it with another numbered equation:
\begin{equation}
\sum_{i=0}^{\infty}x_i=\int_{0}^{\pi+2} f
\end{equation}
just to demonstrate \LaTeX's able handling of numbering.
\section{Figures}
The ``\verb|figure|'' environment should be used for figures. One or
more images can be placed within a figure. If your figure contains
third-party material, you must clearly identify it as such, as shown
in the example below.
\begin{figure}[h]
\centering
\includegraphics[width=\linewidth]{sample-franklin}
\caption{1907 Franklin Model D roadster. Photograph by Harris \&
Ewing, Inc. [Public domain], via Wikimedia
Commons. (\url{https://goo.gl/VLCRBB}).}
\Description{A woman and a girl in white dresses sit in an open car.}
\end{figure}
Your figures should contain a caption which describes the figure to
the reader.
Figure captions are placed {\itshape below} the figure.
Every figure should also have a figure description unless it is purely
decorative. These descriptions convey what’s in the image to someone
who cannot see it. They are also used by search engine crawlers for
indexing images, and when images cannot be loaded.
A figure description must be unformatted plain text less than 2000
characters long (including spaces). {\bfseries Figure descriptions
should not repeat the figure caption – their purpose is to capture
important information that is not already provided in the caption or
the main text of the paper.} For figures that convey important and
complex new information, a short text description may not be
adequate. More complex alternative descriptions can be placed in an
appendix and referenced in a short figure description. For example,
provide a data table capturing the information in a bar chart, or a
structured list representing a graph. For additional information
regarding how best to write figure descriptions and why doing this is
so important, please see
\url{https://www.acm.org/publications/taps/describing-figures/}.
\subsection{The ``Teaser Figure''}
A ``teaser figure'' is an image, or set of images in one figure, that
are placed after all author and affiliation information, and before
the body of the article, spanning the page. If you wish to have such a
figure in your article, place the command immediately before the
\verb|\maketitle| command:
\begin{verbatim}
\begin{teaserfigure}
\includegraphics[width=\textwidth]{sampleteaser}
\caption{figure caption}
\Description{figure description}
\end{teaserfigure}
\end{verbatim}
\section{Citations and Bibliographies}
The use of \BibTeX\ for the preparation and formatting of one's
references is strongly recommended. Authors' names should be complete
--- use full first names (``Donald E. Knuth'') not initials
(``D. E. Knuth'') --- and the salient identifying features of a
reference should be included: title, year, volume, number, pages,
article DOI, etc.
The bibliography is included in your source document with these two
commands, placed just before the \verb|\end{document}| command:
\begin{verbatim}
\bibliographystyle{ACM-Reference-Format}
\bibliography{bibfile}
\end{verbatim}
where ``\verb|bibfile|'' is the name, without the ``\verb|.bib|''
suffix, of the \BibTeX\ file.
Citations and references are numbered by default. A small number of
ACM publications have citations and references formatted in the
``author year'' style; for these exceptions, please include this
command in the {\bfseries preamble} (before the command
``\verb|\begin{document}|'') of your \LaTeX\ source:
\begin{verbatim}
\citestyle{acmauthoryear}
\end{verbatim}
Some examples. A paginated journal article \cite{Abril07}, an
enumerated journal article \cite{Cohen07}, a reference to an entire
issue \cite{JCohen96}, a monograph (whole book) \cite{Kosiur01}, a
monograph/whole book in a series (see 2a in spec. document)
\cite{Harel79}, a divisible-book such as an anthology or compilation
\cite{Editor00} followed by the same example, however we only output
the series if the volume number is given \cite{Editor00a} (so
Editor00a's series should NOT be present since it has no vol. no.),
a chapter in a divisible book \cite{Spector90}, a chapter in a
divisible book in a series \cite{Douglass98}, a multi-volume work as
book \cite{Knuth97}, a couple of articles in a proceedings (of a
conference, symposium, workshop for example) (paginated proceedings
article) \cite{Andler79, Hagerup1993}, a proceedings article with
all possible elements \cite{Smith10}, an example of an enumerated
proceedings article \cite{VanGundy07}, an informally published work
\cite{Harel78}, a couple of preprints \cite{Bornmann2019,
AnzarootPBM14}, a doctoral dissertation \cite{Clarkson85}, a
master's thesis: \cite{anisi03}, an online document / world wide web
resource \cite{Thornburg01, Ablamowicz07, Poker06}, a video game
(Case 1) \cite{Obama08} and (Case 2) \cite{Novak03} and \cite{Lee05}
and (Case 3) a patent \cite{JoeScientist001}, work accepted for
publication \cite{rous08}, 'YYYYb'-test for prolific author
\cite{SaeediMEJ10} and \cite{SaeediJETC10}. Other cites might
contain 'duplicate' DOI and URLs (some SIAM articles)
\cite{Kirschmer:2010:AEI:1958016.1958018}. Boris / Barbara Beeton:
multi-volume works as books \cite{MR781536} and \cite{MR781537}. A
couple of citations with DOIs:
\cite{2004:ITE:1009386.1010128,Kirschmer:2010:AEI:1958016.1958018}. Online
citations: \cite{TUGInstmem, Thornburg01, CTANacmart}. Artifacts:
\cite{R} and \cite{UMassCitations}.
\section{Acknowledgments}
Identification of funding sources and other support, and thanks to
individuals and groups that assisted in the research and the
preparation of the work should be included in an acknowledgment
section, which is placed just before the reference section in your
document.
This section has a special environment:
\begin{verbatim}
\begin{acks}
...
\end{acks}
\end{verbatim}
so that the information contained therein can be more easily collected
during the article metadata extraction phase, and to ensure
consistency in the spelling of the section heading.
Authors should not prepare this section as a numbered or unnumbered {\verb|\section|}; please use the ``{\verb|acks|}'' environment.
\section{Appendices}
If your work needs an appendix, add it before the
``\verb|\end{document}|'' command at the conclusion of your source
document.
Start the appendix with the ``\verb|appendix|'' command:
\begin{verbatim}
\appendix
\end{verbatim}
and note that in the appendix, sections are lettered, not
numbered. This document has two appendices, demonstrating the section
and subsection identification method.
\section{SIGCHI Extended Abstracts}
The ``\verb|sigchi-a|'' template style (available only in \LaTeX\ and
not in Word) produces a landscape-orientation formatted article, with
a wide left margin. Three environments are available for use with the
``\verb|sigchi-a|'' template style, and produce formatted output in
the margin:
\begin{itemize}
\item {\verb|sidebar|}: Place formatted text in the margin.
\item {\verb|marginfigure|}: Place a figure in the margin.
\item {\verb|margintable|}: Place a table in the margin.
\end{itemize}
%%
%% The acknowledgments section is defined using the "acks" environment
%% (and NOT an unnumbered section). This ensures the proper
%% identification of the section in the article metadata, and the
%% consistent spelling of the heading.
\begin{acks}
To Robert, for the bagels and explaining CMYK and color spaces.
\end{acks}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
%%
%% If your work has an appendix, this is the place to put it.
\appendix
\section{Research Methods}
\subsection{Part One}
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi
malesuada, quam in pulvinar varius, metus nunc fermentum urna, id
sollicitudin purus odio sit amet enim. Aliquam ullamcorper eu ipsum
vel mollis. Curabitur quis dictum nisl. Phasellus vel semper risus, et
lacinia dolor. Integer ultricies commodo sem nec semper.
\subsection{Part Two}
Etiam commodo feugiat nisl pulvinar pellentesque. Etiam auctor sodales
ligula, non varius nibh pulvinar semper. Suspendisse nec lectus non
ipsum convallis congue hendrerit vitae sapien. Donec at laoreet
eros. Vivamus non purus placerat, scelerisque diam eu, cursus
ante. Etiam aliquam tortor auctor efficitur mattis.
\section{Online Resources}
Nam id fermentum dui. Suspendisse sagittis tortor a nulla mollis, in
pulvinar ex pretium. Sed interdum orci quis metus euismod, et sagittis
enim maximus. Vestibulum gravida massa ut felis suscipit
congue. Quisque mattis elit a risus ultrices commodo venenatis eget
dui. Etiam sagittis eleifend elementum.
Nam interdum magna at lectus dignissim, ac dignissim lorem
rhoncus. Maecenas eu arcu ac neque placerat aliquam. Nunc pulvinar
massa et mattis lacinia.
\end{document}
\endinput
%%
%% End of file `sample-authordraft.tex'.
%%
%% This is file `sample-lualatex.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigconf')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf]{acmart}
% \usepackage{cite}
% \usepackage{amsmath,amssymb,amsfonts}
\usepackage{algorithmic}
\usepackage{graphicx}
\usepackage{textcomp}
\usepackage{filecontents}
% \usepackage[noadjust]{cite}
\usepackage{multirow}
\usepackage{makecell}
\usepackage{xcolor}
\usepackage{etoolbox}
\usepackage{hyperref}
\makeatletter
\renewcommand{\ceur}{EEKE 2021 - Workshop on Extraction and Evaluation of Knowledge Entities from Scientific Documents}
\renewcommand{\ceurLicense}{Copyright 2021 for this paper by its authors. Use permitted under Creative Commons License Attribution 4.0 International (CC BY 4.0).}
% Add the line below
% \usepackage{comgipp}
% \newcommand{\theReferenceText}{\fullcite{ \small A. Zhukova, F. Hamborg, B.Gipp, ``ANEA: Automated (Named) Entity Annotation for German Domain-Specific Texts'', 2nd Workshop on Extraction and Evaluation of Knowledge Entities from Scientific Documents (EEKE2021) at the ACM/IEEE Joint Conference on Digital Libraries 2021 (JCDL2021)} }
% \fancyhead{}
%% NOTE that a single column version may be required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}
}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
% \setcopyright{acmcopyright}
% \copyrightyear{2021}
% \acmYear{2021}
% \acmDOI{10.1145/1122445.1122456}
% %% These commands are for a PROCEEDINGS abstract or paper.
% \acmConference[EEKE 2021 - Workshop on Extraction and Evaluation of Knowledge Entities from Scientific Documents]{2nd Workshop on Extraction and Evaluation of Knowledge Entities from Scientific Documents (EEKE2021)}{September 30}{2021}
% \acmBooktitle{2nd Workshop on Extraction and Evaluation of Knowledge Entities from Scientific Documents (EEKE2021),
% September 30, 2021, Online}
% \acmPrice{15.00}
% \acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{ANEA: Automated (Named) Entity Annotation for German Domain-Specific Texts}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Anastasia Zhukova}
% \authornote{Both authors contributed equally to this research.}
\email{zhukova@uni-wuppertal.de}
\orcid{0000-0001-9084-2890}
% \author{G.K.M. Tobin}
\authornotemark[1]
% \email{webmaster@marysville-ohio.com}
\affiliation{%
\institution{University of Wuppertal}
% \streetaddress{P.O. Box 1212}
% \city{Wuppertal}
% \state{Ohio}
\country{Germany}
% \postcode{43017-6221}
}
\author{Felix Hamborg}
% \authornote{Both authors contributed equally to this research.}
\email{felix.hamborg@uni-konstanz.de}
% \orcid{1234-5678-9012}
% \author{G.K.M. Tobin}
\authornotemark[2]
% \email{webmaster@marysville-ohio.com}
\affiliation{%
\institution{University of Konstanz}
% \streetaddress{P.O. Box 1212}
% \city{Konstanz}
% \state{Ohio}
\country{Germany}
% \postcode{43017-6221}
}
\author{Bela Gipp}
% \authornote{Both authors contributed equally to this research.}
\email{gipp@uni-wuppertal.de}
% \orcid{1234-5678-9012}
% \author{G.K.M. Tobin}
\authornotemark[1]
% \email{webmaster@marysville-ohio.com}
\affiliation{%
\institution{University of Wuppertal}
% \streetaddress{P.O. Box 1212}
% \city{Wuppertal}
% \state{Ohio}
\country{Germany}
% \postcode{43017-6221}
}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
% \renewcommand{\shortauthors}{Zhukova et al.}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
Named entity recognition (NER) is an important task that aims to resolve universal categories of named entities, e.g., persons, locations, organizations, and times. Despite its common and viable use in many use cases, NER is barely applicable in domains where general categories are suboptimal, such as engineering or medicine. To facilitate NER of domain-specific types, we propose ANEA, an automated (named) entity annotator to assist human annotators in creating domain-specific NER corpora for German text collections when given a set of domain-specific texts. In our evaluation, we find that ANEA automatically identifies terms that best represent the texts' content, identifies groups of coherent terms, and extracts and assigns descriptive labels to these groups, i.e., annotates text datasets into the domain (named) entities.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
% TODO
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10002951.10003317.10003347.10003352</concept_id>
<concept_desc>Information systems~Information extraction</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010147.10010178.10010179.10003352</concept_id>
<concept_desc>Computing methodologies~Information extraction</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010147.10010178.10010179.10010186</concept_id>
<concept_desc>Computing methodologies~Language resources</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010147.10010257.10010258.10010260.10003697</concept_id>
<concept_desc>Computing methodologies~Cluster analysis</concept_desc>
<concept_significance>300</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[300]{Information systems~Information extraction}
\ccsdesc[300]{Computing methodologies~Information extraction}
\ccsdesc[300]{Computing methodologies~Language resources}
\ccsdesc[300]{Computing methodologies~Cluster analysis}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{information extraction, low-resource languages, named entity recognition, domain-specific texts}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
% \begin{teaserfigure}
% \includegraphics[width=\textwidth]{sampleteaser}
% \caption{Seattle Mariners at Spring Training, 2010.}
% \Description{Enjoying the baseball game from the third-base
% seats. Ichiro Suzuki preparing to bat.}
% \label{fig:teaser}
% \end{teaserfigure}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
%insert this line directly after maketitle
% \thispagestyle{firststyle}
% \newpage
% optional re-enable header on the second page (do not execute this command on the first page)
% \pagestyle{standardpagestyle}
% \maketitle
\section{Introduction}
Named entity recognition (NER), a common preprocessing step in natural language processing (NLP) for various tasks, such as information extraction, summarization, question and answering, and text understanding, is often criticized for capably representing only datasets with few general categories, e.g., person, location, organization, and time (including their subcategories) \cite{zhang2013unsupervised, eltyeb2014chemical}. While the original NER task contains only few categories, a rapidly increasing number of NER applications show a high demand for the datasets with \textit{domain-specific named entities} \cite{rocktaschel2012chemspot}.
% Creation of the domain-specific datasets requires costly manual annotation. Training supervised NER systems requires often large datasets but their creation is a time-consuming process.
To (semi-)automatically create large general-purpose NER corpora, recent research projects extensively use structured domain sources, such as dictionaries, knowledge graphs, and Wikipedia or other knowledge bases \cite{richman2008mining, simon2012automatically, zhou2015automatically}.
In this paper, we propose ANEA, an unsupervised Wiktionary-based approach that automatically derives domain entities from German texts, i.e., low-resource language, by (1) extracting terms from topically-related domain texts, (2) identifying the most domain-representative, i.e., semantically distinct, terms of the analyzed texts, and (3) automatically annotating the terms, i.e., ANEA extracts labels from Wiktionary and assigns them to the identified groups of terms. Not all of the domain categories may be named, e.g., machinery or process.
By automating the most labor-intense parts, the proposed unsupervised approach minimizes the cost of expensive and laborious annotations required for the creation of domain-specific NER datasets. Typical manual tasks in annotations include (1) reading the domain text multiple times, (2) deriving entities based on the text content, and (3) manually selecting terms that match the derived categories. ANEA substitutes the most time-consuming task of deriving a coding book and automatically defines categories and annotates the most representative terms (nouns) into these categories. We evaluate the approach with user studies on multiple domain datasets against multiple silver datasets and discuss a default input configuration for ANEA to annotate other domain NER datasets\footnote{ \url{https://github.com/anastasia-zhukova/ANEA}}.
\section{Related work}
% no domain tagging, only annotation
NER datasets usually contain standard types, e.g., person, location, organization, and are manually annotated \cite{sang2003introduction, benikova2014nosta} or automatically extracted \cite{richman2008mining, simon2012automatically, zhou2015automatically}.
% , and \textit{resolution (NER)}, i.e., automatic identification of the NE spans in text and assignment of the category labels, e.g., for German language \cite{rocktaschel2012chemspot, loster2017improving, ruppenhoferfine2020}.
Domain-specific NER typically needs to introduce domain-specific (sub-)categories of the established named entity (NE) categories or entirely new categories. This is because domain-specific texts contain NE categories that are (1) detailed variants of the standard NE categories, e.g., ``Person'' is replaced with the domain-specific sub-categories ``Players'' and ``Coaches'' \cite{weber2006web}, (2) standard NE categories extended with a small number of new categories, e.g., ``Trigger of a traffic jam'' \cite{schiersch2018german, loster2018curex, ruppenhoferfine2020}, and (3) domain-derived NE categories, e.g., ``Proteins'' in biology or ``Reactions'' in chemistry domains \cite{rocktaschel2012chemspot, zhang2013unsupervised, heuss2014comparison, tomori2016domain}. Most domain-derived NE categories originate from structured classifications or dictionaries \cite{heuss2014comparison, tomori2016domain, loster2017improving} or are derived by manually unifying multiple of them \cite{eltyeb2014chemical}. In sum, creating domain-specific datasets for NER requires expert knowledge and is time-consuming.
To minimize such efforts, some NER approaches use seed-NEs, i.e., a small number of manually provided terms and their NE-categories \cite{nadeau2006unsupervised, zhang2013unsupervised, foley2018named}.
Such approaches use the seed-NEs as examples to extract patterns of NE definitions and apply them to the full text suggested for annotation. %The seed-based NER systems often rely on the domain knowledge bases (KB) (e.g., medical) or lexical bases, e.g., WordNet.
% The typically underlying domain knowledge bases (KB), e.g., medical, ensure domain undertandng of such NER systems. On the contrary, such systems suffer from the slow-pace KB update and lead to low recall on the texts with unknown terms.
These NER approaches suffer from the slow updates of the underlying domain knowledge bases (KB) \cite{yadav2018survey} and perform worse on lower-resource languages than on English \cite{heuss2014comparison}.
An alternative to domain-KBs is community KBs, such as Wikipedia and Wiktionary, which are constantly updated by their communities. They prove to contain a sufficient amount of domain information \cite{zesch2008extracting, meyer2011psycholinguists}.
% Wikipedia has also proven to be a valid resource to creation large silver, i.e., automatically created, datasets for NER of standard types (person, location, organization) in many languages \cite{richman2008mining, simon2012automatically, zhou2015automatically}.
Unlike the existing supervised approaches for annotating domain-specific named entities \cite{mansouri2008named, etzioni2005unsupervised}, in this paper, we explore ANEA, an unsupervised method to support researchers and users during the creation of a coding book. Given a set of domain- or use case-specific documents, ANEA automatically derives domain-specific categories and exemplary terms within. This way, ANEA automates the most time-intensive, previously manual tasks. As a consequence, users only need to revise these terms, e.g., by renaming the categories or re-annotating the not matching to the categories terms.
% To the best of our knowledge, only \citet{etzioni2005unsupervised} proposed and tested a syntactic-based system to extract terms and assign domain-NE-labels.
\section{Methodology}
We propose an unsupervised approach for annotation of domain-specific (named) entities (ANEA) on a lower-resource language. The goal of ANEA is to fully automatically derive entity categories (later in the text: categories) by selecting groups of related terms and extract and assign a meaningful label to these terms. To do so, ANEA, first, links terms extracted from domain-specific texts to pages in Wiktionary \cite{meyer2011psycholinguists, simon2012automatically}. Second, ANEA automatically identifies groups of related terms and automatically labels them by performing a double optimization task of both maximizations of cross-similarity of terms in a group and the average similarity of these terms to a candidate label. That is, the approach consists of two main steps: (1) text preprocessing, i.e., term mapping to Wiktionary pages (WPs) and construction of a domain graph, and (2) identification of related terms and label assignment.
\subsection{Preprocessing and domain graph}
\label{sec:prepr}
\subsubsection{Preprocessing}
The goal of preprocessing is to extract terms from the set of texts and maximize the number of terms aligned to the Wiktionary structure, i.e., map the domain-specific terminology to the structured knowledge base. The mapping of the extracted terms to the knowledge graph enables using their semantic information, such as term definitions, areas, hypernyms, and hyponyms (see Figure \ref{img:example}).
\begin{figure}[h]
\centering
\includegraphics[width=0.45\textwidth]{motor}
\caption{\label{img:example} An example of a Wiktionary page (WP).}
\end{figure}
The preprocessing steps include parsing and part-of-speech (POS) tagging using spaCy \cite{spacy2}. We define a \textit{term} as any unique noun phrase that does not contain any digits \cite{foley2018named}. After extraction, terms are mapped to their respective German WP, if any, i.e., each term gets assigned to a link of a WP.
In German texts, we find that many domain-specific terms are compound words, i.e., words that consist of more than one noun component, for example, ``Sechszylindermotor'' = ``sechs'' + ``Zylinder'' + ``Motor'' (six-cylinder motor).
% (F: nice example :-))
Typically, such complex domain compound words are not described in Wiktionary since they are too rare or specific. On the contrary, we observe that compound words' \textit{heads}, i.e., the part of a composition term that bears the core meaning of the phrase, e.g., ``Motor'' in the example above, are highly likely to have a WP in Wiktionary.
\begin{figure*}[h]
\centering
\includegraphics[width=0.95\textwidth]{wikt}
\caption{\label{img:wikt} (a) A domain graph is a combination of Wiktionary and domain terminology extracted from text.
(b) Domain graph construction: initialization and expansion of the graph.}
\end{figure*}
To map rare domain-specific terms to WPs, we extract the heads of the extracted terms with a compound splitter, i.e., a model that splits terms into two parts, compound- and head-parts, \cite{Tuggener2016} and attempt to map the heads to Wiktionary. If a (multi-token) term has a corresponding WP, we set a full term as a term's head. If the compound splitter outputs a head that is not a part of Wiktionary, we continue recursively search for a head that can be mapped to a WP. If no heads have corresponding WPs, then we do not assign a head to a term.
% TODOA
The preprocessing could be changed to include the terms with digits, but for now, we focused on the noun phrases as terms. If a term or its head of compound phrase do not have a WP, they are excluded from annotation because the absence of a link to Wiktionary leads to an inability to map terms to potential category labels. Later, such discarded terms can be manually classified by human annotators or automatically with state-of-the-art NER models trained on the automatically created domain datasets.
We use fastText to vectorize extracted terms and candidate labels \cite{grave2018learning}. We chose fastText due to its ability to vectorize out-of-vocabulary words, which often happens with domain-specific terminology.
% \subsection{Domain graph}
\subsubsection{Domain graph} The \textit{domain graph} is a locally stored knowledge graph where the leaves are the extracted domain terms. Nodes are all terms obtained from the WPs linked to the leaves and to each other with hyponymy-hypernymy relations. Figure~\ref{img:wikt}a depicts the principle of a domain graph. The construction of the domain graph includes three steps: (1) graph initialization, i.e., extraction of the WP properties, e.g., definitions, with the scraping of the WPs assigned to the domain terms and their heads; (2) determination of pruning criteria of Wiktionary graph to scrape only domain related pages; (3) expansion of the domain graph, i.e., scrapping of the hypernym pages, to create a pool of candidate labels to later annotated the identified groups of terms. Figure~\ref{img:wikt}b shows the process of domain graph construction.
\subsubsection{Initialization}
% (F: add 1-2 sentences: what is the input to the initialization step, what will be the output, and why is it necessary; similar as you did for the a priori pruning :-) )
To initialize the graph, we use the extracted terms to which we mapped WPs and scrap the mapped WPs to extract WPs' properties, e.g., hypernyms. As a preliminary step of the graph initialization, we group the extracted terms by their head. The \textit{head grouping} aims at the extraction of the initial hyponym-hypernym relation for the domain graph. Then, we sort the list of heads in decreasing order by (1) the number of unique terms with each head, (2) the frequency of the overall in-text occurrence of words with such head.
To maximize the descriptiveness and generalization of the terms that will become annotated into categories, we initialize the domain graph with the terms-to-annotate that belong to the top $M$ largest head groups, i.e., containing the most lexically diverse and/or frequent terms. Section~\ref{sec:eval} determines an optimal value of terms-to-annotate and the largest head groups the series of experiments. This filtering procedure reduces the size of the domain graph to minimize the time of the execution and extract the most representative candidate labels, i.e., the most closely located hypernyms.
% As opposed to selecting the most frequent extracted terms to initialize a graph, the selection of the frequent groups of heads guarantees that most of representative terms due the high frequency and initial relations between terms will become a part of the domain graph.
Each term without a hyponym is a \textit{leaf} of the domain graph; a \textit{node} is a head that aggregates more than one term. We scrape WPs of all leave- and node-terms to extract the text and links from definitions, hypernyms and hyponyms (see Figure~\ref{img:example}).
% (F: just fyi, you can use the cleverref package to use backslashCref to not need to write Figure or Table :-))
We extract hyponym terms from the corresponding WP's section.
% (F: i suggest to use the english terms here. i would suggest to keep the description conceptual, because in principle this approach would work for any language that has a lot of compounds; so i would abstract this from technical (what exactly is the name of the section on the webpage) to conceptual (which section is the one we are interested in semantically))
We extract hypernym terms from two WP's parts: (1) the hypernym section, (2) the definition section by parsing the text of term's definitions and ensuring that the extracted word has its WP\footnote{We extract the tokens that have one of the following dependency tags: ``ROOT'', ``oa''= accusative object, ``oa2'' = second accusative object, ``app'' = apposition, ``cj'' = conjunct.}. For example, in Figure~\ref{img:example}, the word ``Maschine'' will be extracted as an additional (in-text) hypernym to those listed in the hypernym section.
The extracted properties are assigned to each node. The hypernyms' links point at the WPs that may later become nodes of the domain graph.
% we do not instantly scrap the WP of the extracted hypernyms and hyponyms and add them as nodes.
Extraction and assignment of the WP's properties bridge the domain terms and heads to the Wiktionary's knowledge graph.
\subsubsection{A priori pruning}
Most of the terms of WPs have more than one sense and some of them may be associated with the different semantic areas, e.g., technology, medicine, sport, law, etc. If a sense belongs to only one area, the title of the area precedes the definition explaining the sense, e.g., ``Technik'' in Figure~\ref{img:example}. The step \textit{a priori pruning} determines which senses of yet to add hypernyms need to comply with the senses of the previously added terms.
% (F: i understand what this does, but i think the phrase "determines the selection criteria" is not optimal. maybe better remove it eg: The step \textit{a priori pruning} determines which senses of hypernyms comply with the senses of the previously added terms)
% for hypernyms in the domain graph, i.e., which senses of hypernyms comply with the senses of the previously added terms. (F: not sure about the following sentence, but it was not grammatically correct, tried to rewrite it but unsure whether it still has the meaning you intended)
Hypernyms will become properties of a node in the domain graph if and only if their areas belong to a predefined list of areas or if they do not list any domain areas.
To identify which Wiktionary's areas determine the graph's domain, we select the most frequent and semantically similar areas extracted from the senses' definitions of the previously added leaves and nodes. To find the most semantically similar and frequent areas, we cluster all titles using hierarchical clustering and select the areas from the most representative clusters.
% (F: i think you really like clustering :-) it is in every paper we work on :-D)
As parameters of the hierarchical clustering, we use Euclidean distance, average linkage criterion, and optimize the number of clusters. To represent areas' titles in the vector space, we apply the fastText word embeddings model \cite{grave2018learning}.
We extract three most representative clusters by: (1) selecting a cluster with the average cosine similarity across all words in a cluster being the highest among all clusters ($C_s$); (2) selecting a cluster with the count of all words in a cluster is the highest among all clusters ($C_f$); (3) forming an extra cluster with the $K$ most frequent areas $C_k$ ($K = 5$). To identify the Wiktionary areas $A$ forming the domain of the graph, we intersect all three representative clusters: $ A = \{\cup a: a \in C_s \cap C_f \cap C_k\} $. Finally, we select a clustering configuration that outputs the best domain-defining areas as $A_{best} = \operatorname{arg\,max}_{6\leq i \leq 12} (s(A_i) \cdot f(A_i))$, where $A_i$ are the areas identifies at $i$ the number of clusters, $s(A_i)$ is an average cross similarity of the areas in $A_i$ and $f(A_i)$ is sum of $A_i$' frequencies.
\subsubsection{Graph growing}
The goal of NEA is to assign the most generalizing yet still representative labels to groups of semantically related terms, e.g., ``Person'' for ``Trump'' and ``Einstein.'' To ensure the generalization property of label candidates, we \textit{``grow''}, i.e., expand the domain graph up, by adding new nodes on the top of the graph from the scrapped hypernym WPs.
To grow the graph, we iterate over the top nodes and create new nodes for each of the hypernym terms. To obtain node's properties, we scrape WPs of the hypernym terms and extract term definitions, hyponyms, and hypernyms. For each new node, we add hypernym-hyponym edges between this new node and the matching previously added nodes while also removing any edges creating cycles in the domain graph. To avoid over-generalizing candidate labels, we perform only one or two iterations of the graph growing.
\subsection{Automated term grouping and labeling}
\label{sec:anea}
The goal of ANEA is to obtain (named) \textit{entity categories}, i.e., few clusters of generally related terms of high cross-similarity and assign descriptive labels to these clusters. To do so, we maximize two parameters at the same time: cross-term group similarity and similarity between a group of terms and a label.
ANEA consists of the initial setup of the categories and three subsequent optimization steps to improve the representativeness of the terms and the assigned labels in the groups of terms.
\subsubsection{Setup}
We initialize ANEA by collecting all candidate categories, i.e., groups of potentially related terms and the assigned labels to them. Figure~\ref{img:anea} depicts the process of candidate label collection.
\begin{figure}[h]
\centering
\includegraphics[width=0.4\textwidth]{setup}
\caption{\label{img:anea} ANEA setup: a collection of candidate entity categories. }
\end{figure}
\begin{figure}[h]
\centering
\includegraphics[width=0.45\textwidth]{steps}
\caption{\label{img:steps} \centering (a) Resolution of the full overlaps.
(b) Resolution of the substantial overlaps. (c) Resolution of the conflicting terms. }
\end{figure}
First, we iterate over all term-nodes, i.e., the domain graph's leaves and nodes that were created from the extracted terms not the hypernym WPs. For each term-node, we collect candidate labels extracted from the names of their hypernym-nodes. Each term obtains a list of candidate labels with various \textit{distances}, i.e., the number of edges between a term-node and a label-node. We recursively traverse the domain graph as long as the distance between a term-node and a label-node is $d \leq d_{max}$, $d_{max}=5$, or there are hypernyms to the current node in the domain graph. During the experiments, we notice that label-nodes with larger distances are often rather abstract and do not characterize a term well.
Second, we ``transpose'' all terms and their candidate labels to obtain one label assigned to a group of terms. That is, we create a collection of categories among which we seek to find the most representative categories of the analyzed domain-specific text collection.
The selection of the most optimal categories among the candidates is a double-optimization process towards two requirements: generalization and specification. On the one hand, \textit{generalization} aims at covering categories' broader semantics, e.g., a category with a more general label ``Person'' is better than categories such as ``Actor,'' ``Politician,'' etc. On the other hand, \textit{specification} aims at selecting the category with more narrow semantics, e.g., categories such as ``Country,'' ``City,'' ``State'' provide more details about its terms than a category ``Location''.
We use a \textit{quality score} $Q_i$ to evaluate each (entity) category $EC_i$ in a list of candidates: % F. i like the equations! :-)
\[ Q_i = T_i \cdot L_i \cdot O_i \cdot \operatorname*{max}({\log_{2} |EC_i|, 1 }) \cdot d_{avg\_i} \]
where $T_i$ is a mean cross-term cosine similarity; $L_i$ is a mean label-terms similarity; $O_i$ is the overall similarity, i.e., $O_i = T_i + L_i$; $|EC_i|$ is a size of a category, i.e., the number of the terms in the class; $d_{avg\_i}$ is an average of non-zero distances between category's terms and label $l$: $d_{avg\_i} = \frac{1}{|D_{nn\_i|}} \sum_{d \in D_{nn\_i}} d $, where $D_{nn\_i} = \{D_{i,l}| \, \forall i \in EC: D_{i,l} > 0\}$ and $D$ is a distance matrix\footnote{Rows are the names of term-nodes and the columns are the names of term-nodes and label-nodes. The columns contain also the term-nodes because some of the term-nodes may not be leaves but nodes of the domain graph (see Figure~\ref{img:wikt}). }. If $|D_{nn\_i}|= 0$, then $d_{avg\_i}=1$.
To calculate cosine similarities, we represent each term and label in a vector space with fastText word embeddings \cite{grave2018learning}. We chose fastText for the representation of the out-of-vocabulary words, which often occur in domain-specific texts.
% \footnote{We plan to use ELMo and other context-depended word embeddings in future work. In this work, we experimented with the static semantic representation of each term. }.
% this footnote could be removed, esp in case we need space, because it kind of is a trivial statement and doesnt really add value to the paper
Requiring a large mean cross-term cosine similarity $T_i$ increases the specificity of a category. Typically, the smaller the number of related terms, the larger the mean cross-similarity is. A larger mean label-terms similarity $L_i$ also increases the specificity, i.e., a large similarity value is equivalent to a narrow descriptiveness of the terms of a label.
The overall sum $O_i$ facilitates balancing potentially small values of either $T_i$ or $L_i$ if the other item is still large. The large size of a category increases its generalizing and descriptive properties, i.e., one label needs to describe as many terms as possible. Lastly, the average distance $d_{avg}\_i$ acts as an amplifying factor for the generalization: the higher the label $l$ is in the domain graph, the more general is its meaning to the terms in this category.
Before the optimization steps, we perform filtering of the candidate categories to remove low quality categories from the candidate list. We remove an $EC_i$ if: (1) $T_i < 0.2$, (2) $L_i < 0.3$, (3) $|EC_i| > 0.15 \cdot |TTA| \land |EC_i| < 5$, where $|TTA|$ is a number of terms-to-annotate, i.e., a number of the term-nodes in the domain graph. In other words, we remove too vaguely related, very large or small categories.
\subsubsection{Resolution of full overlaps}
Figure~\ref{img:steps}a depicts that if two categories have the same terms but different labels, we sort the classes by their quality scores $Q_i$ and keep the categories with the highest $Q_i$.
\subsubsection{Resolution of substantial overlaps}
%F: rather than describing some characteristics i would recommend to describe a problem that motivates why we have this optimization step that is now being described.
Typically, categories have overlaps between their terms albeit we find that cross-terms and terms-label combinations of a single category are more semantically coherent than combinations of another category. We define that categories have substantial overlap if they share more than 50\% of their terms.
Figure~\ref{img:steps}b depicts the process of conflict resolution. We construct a matrix of replacements $R$, i.e., a matrix indicating the quality of a category measured by $Q$ compared to those categories with substantial overlaps (values of $R$ initialized with 0). The matrix is used to identify if an $EC_A$ contains the best terms-label combination or there is a better $EC_{repl}$ to replace $EC_A$. Since ANEA's goal is to annotate as many generally related terms as possible yet find as specific categories as possible, we challenge both the size of $EC_a$ and its descriptive properties.
First, we sort categories by their $Q$ score in decreasing order.
Second, we intersect all categories with each other.
% Second, we iterate in a double loop over the entity classes and overlap the terms from the outer loop $A$ (which also denotes all entity classes) with the inner loop $B$.
If $|EC_a \cap EC_b| \geq 0.5 \cdot |EC_a|$, then we consider the overlaps substantial and add the quality score $Q_b$ to the matrix of replacements as a $R_{a,b}$ value. Note that the matrix is squared but asymmetrical because we calculate 50\% of $EC_a$'s size and not of a pairwise function of two categories, e.g., $\operatorname*{min}(|EC_a|, |EC_b|)$.
%F: uff, this might sound a bit to technical / close to the actual implementation (eg "double loop", i think it be good to remove these programming terms and try to abstract <way from the implemnentation and conceptually describe it :)
Finally, for each $EC_a: \forall a \in A$ represented by a row $r_a$ in $R$, we select a replacement $EC_{repl}$:
\begin{multline*}
EC_{repl}(EC_a) = \{EC_c | \quad \exists c \in A :\\ \quad \operatorname*{arg\,max} r_a = EC_c\, \land \operatorname*{arg\,max} r_c = EC_c \}
\end{multline*}
That is, we call $EC_c$ a replacement to $EC_a$ if $EC_c$ is the best among all comparable categories to $EC_a$ and also $EC_c$ is the best among all categories compared to itself. Also, a category can be a replacement for itself. We keep only the unique categories that are the best replacements $\{EC_{repl}(E_a): \forall a \in A\}$.
\subsubsection{Resolution of conflicting terms}
After resolution of substantially overlapping terms, some categories contain minor conflicting terms, i.e., that are present in more than one category (Figure~\ref{img:steps}c).
% Figure~\ref{img:anea}d depicts the last optimization step ensures a one-to-one assignment of the terms already used in EC to the categories.
To resolve conflicting terms, first, we create a list of ``clean'' categories, i.e., from each category we remove all conflicting terms and record categories' labels from which the conflicting terms were removed. Additionally, we resolve the terms of the categories that may be also labels of another category, e.g., such a term as ``h'' in Figure~\ref{img:anea}, i.e., move these label-terms to the categories with corresponding labels. We keep all categories even if some categories may afterward have no terms, i.e., if all their terms conflicted with other categories.
Second, to resolve conflicting terms, we estimate the quality of all ``clean'' categories. We calculate a quality score $Q$ for each category (see Section~\ref{sec:anea}; if $|EC| = 0$, then $Q=0$) and sort categories by decreasing $Q$. Sorting brings forward categories that are the most probable to become final categories.
Third, we resolve all conflicting terms by beginning with those that belong to the categories with the highest $Q$. For each conflicting term $t_j$ and all $EC_i$ from which the term originated, we calculate a similarity score $S$:
$S(t_j, EC_i) = T + L$, where $T$ is the mean cosine similarity between the vector representation of $t_j$ and the remaining terms in a ``clean'' $EC_i$; $L$ is the cosine similarity between $t_j$ and a label of $EC_i$. Even if a ``clean'' $EC_i$ contains no terms, i.e., $T=0$, $L$ will always yield $S>0$. We select the best category for a given term $t_j$ as:
\[EC_{best}(t_j) = \{EC_i| \exists i \in A : \operatorname*{arg\,max} S_i(t_j)\}\]
We add the resolved terms to their best matching category. The final categories are where $|EC| \geq 5$
% \footnote{Figure~\ref{img:anea}d depicts the smaller allowed $|EC|$ due to a very small number of initial terms in the example.}
, i.e., that represent a sufficiently large number of extracted terms from the given domain-specific texts.
%F: a summarizing sentence of this whole section / approach would help to set in context what was the beinning and what is the output and to bringg the reader from the low-level technical description to a higher level of the overall approach, before the readers continue reading the eval
\section{Experiments}
\label{sec:eval}
The evaluation goals are twofold. First, we seek to quantitatively assess the quality of the automatically extracted and annotated terms of both ANEA and a baseline using ratings from domain experts. Second, we seek to identify a recommendation for ANEA's default configuration to automatically annotate texts also of other domains by evaluating the annotated and human-assessed annotated datasets against silver quality datasets.
Due to the lack of German datasets for the analysis of domain-specific NER, we assess the quality of the produced categories through user studies where we ask users to rate the quality of the entities extracted by our system. We test ANEA and compare it to a baseline on four text datasets with four configurations.
% TODO evaluation of the domain upsupervised method on the domain annotated english corpus \cite{\cite{zhang2013unsupervised}}
%F: caption is incomplete currently
\subsection{User study}
Our user study aimed at human assessment of the semantic quality of the produced categories due to different configurations. We collect feedback from human assessors for multiple configurations of two methods: ANEA and hierarchical clustering (see Section \ref{sec:hc}). First, we use this feedback to automatically construct silver-quality datasets and evaluate the proposed input configurations against them. Second, we use these silver datasets and evaluate the obtained configurations of the dataset for ANEA to find parameters for default configuration with which ANEA could be used to annotate other domain datasets.
% if space permits (eg camera ready variant) we should point out the limitations of this type of evaluation, e.g., we effectively measure only "precision" if i understand it correctly? but if a term is not extracted in the first place, it wont be evaluated.
\subsubsection{Test datasets}
We create four text datasets of comparable size from three different domains: processing industry (P), computer science, and traveling (T). To enable both cross- and intra-domain evaluation, we create two text datasets related to the computer science domain: databases (D) and software development (S). Table~\ref{tab:events} provides an overview of the datasets' parameters, such as the overall number of words, the number of unique terms and heads of terms (see Sec.~\ref{sec:prepr}), and the number of human assessors per each dataset. The table shows that the number of the unique heads may vary even given the identical number of unique extracted terms (cf. datasets S and T ).
%F: changed it so that all should be right aligned except for the first column. somehow, D and T arent, could not fix it though
\begin{table}[h]
\small
\caption{Dataset statistics: databases (D), software development (S), travelling (T), and processing industry (P). The histograms show the distribution of the user-assigned relatedness score to categories extracted with various input configurations per dataset. Bold scores indicate a threshold used to construct silver datasets. }
\centering
\begin{tabular}{lllll}
\hline
\textbf{Dataset} & \textbf{D} & \textbf{S} & \textbf{T} & \textbf{P} \\
\hline
All words & 8161 & 8581 & 6293 & 7984\\
Terms & 1209 & 1041 & 1040 & 552\\
Heads& 713 & 673 & 801 & 328 \\
Assessors & 3 & 3 & 2 & 4 \\
\hline
\makecell[l]{User study: \\ 1. cross-term \\relatedness \\ scores} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_db.png}} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_sd.png}} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_travel.png}} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_pr.png}}
\\
\makecell[l]{2. label-terms \\relatedness \\ scores} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_db_l.png}} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_sd_l.png}} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_travel_l.png}} & \makecell[l]{\includegraphics[width=0.07\textwidth, height=0.07\textwidth]{bins_pr_l.png}}
\\
\hline
\end{tabular}
% \caption{Characteristics of the test datasets. }
\label{tab:events}
\end{table}
To test the applicability of the approach on different domains, we use the publicly available data from Wikipedia and a dataset built on private text data from a real-world production line in the processing industry. Specifically, the first three datasets (databases, software development, and traveling) originate from German Wikipedia articles dedicated to the respective categories. For each dataset, we searched for related articles in Wikipedia using a query ``incategory:\textit{category}'', where ``category'' is ``Datenbanken,'' ``Programmierung,'' or ``Reise''. We iterated over the list of the search results sorted by relevance and extracted the texts of the articles if the articles had a specific number of words $W$: $220 \leq W \leq 2500$, i.e., articles of medium size. The last dataset consists of reports about the daily operations of a company in the processing industry. Such reports include texts about statuses of the machinery, processes in the production lines, and problems that occurred throughout the daily routines. The dataset consists of approximately 200 short texts, each of 20-100 words.
% We stopped adding texts as soon as the number of words from the new articles would exceed 9000 or 7000.
\begin{figure*}[h]
\centering
\includegraphics[width=0.95\textwidth]{silver.png}
\caption{\label{img:silver} \centering Construction of a silver dataset through collecting of information from user studies.}
\end{figure*}
\subsubsection{Experiment setup}
For the human assessment, we recruited nine native-German speaking participants (4 f, 5 m, aged between 23-60). Each participant is familiar with the domain of the assigned dataset(s) through their job, education, and/or hobbies.
We assigned 3-4 participants to each dataset, and each participant evaluated one or two datasets. Albeit the processing industry dataset has the smallest number of unique terms, we assigned the largest number of assessors to it due to the high relevance of obtaining valid results for such complex, expert domains as chemistry and technology. The vocabulary of these domains is typically strongly underrepresented in general text corpora used to train word embedding models \cite{grave2018learning}.
The evaluation included two tasks for the participants: (1) assess the cross-term relatedness within the identified groups of terms and (2) assess the relatedness of the labels automatically assigned to the identified groups of terms. Per dataset, each participant needed to perform an assessment of eight sheets with automated annotation results: four identical input configurations per both ANEA and a baseline. Each participant needed to assign a semantic relatedness score between 0 and 9, where 0 meant no similarity and 9 - the highest similarity.
The input configuration included four different numbers of the input terms, i.e., \textit{terms-to-annotate (TTAs)}, among which the algorithms needed to extract the most representative terms that can form a separate semantic concept, i.e., an category. To vary the size of TTAs, we selected $1/Z \cdot 100\%$ most frequent heads of phrases: $Z\in[2,3,4,5]$ for the datasets with a number of unique terms $< 1000$, else $Z\in[3,4,5,7]$. By selecting only terms that share all most frequent heads, we ensure that these terms are the most representative of each domain-specific text.
Table \ref{tab:events} reports the results of the user studies and shows that the relatedness of the groups of terms is higher than assigned labels. That is, the cross-term relatedness score was biased towards higher values, the label relatedness had a more uniform score distribution. Additionally, the mean and maximum of the relatedness scores vary across the datasets. We noticed that the relatedness scores were biased toward the size of the identified categories, i.e., a smaller number of terms in categories tend to have higher scores since it is easier for a human to assess a smaller number of items. However, we did not find any correlation between individual datasets and any of the outlined numeric characteristics.
To estimate which input configuration and approach yielded the most coherent categories, we require a silver dataset, which will average the assigned scores and extract the highly rated combinations of terms into categories, and assigned the highly rated labels to them.
\subsubsection{``Silver'' datasets} The goal of a silver dataset is to ensure a fair and unified evaluation strategy of the approaches for all topics. We constructed a silver dataset for each topic by aggregating information from the human assessment sheets following the identical procedure.
First, for each dataset, we constructed term-to-term and label-to-terms score matrices between the vocabulary of each topic and the extracted and assigned labels (Figure \ref{img:silver}). The matrices were initialized with zeros. We iterated over the relatedness scores across two approaches, four input configurations, and two-four human assessors. For every two terms in a term group, we added an assigned cross-term relatedness score to a value in a term-to-term matrix. This score demonstrates how two terms are evaluated in various combinations with other terms across different setups. After the summation was completed, we normalized each value in the matrix by the number of times two terms occurred together. We performed a similar procedure with the label-to-term relatedness scores: for each term in a category and an assigned label to a category, we added a label-to-terms relatedness score assigned to a category and then normalized by the number of times a label was applied to a term.
To identify a threshold of relatedness of two terms belonging to a category, we built a histogram of all scores used to evaluate each dataset (see Table~\ref{tab:events}). In a score range of 0-9, we decided that a threshold of sufficient relevance of terms needs to lie higher than the mean score and not equal to the maximum value, i.e., between scores 6-8. Thus, for each dataset, we chose the most frequent score as a threshold and if a preceding score was less frequent by 1, then we calculated a mean of these scores.
We collected silver groups of terms, by choosing a term and merging it with other terms, a normalized relatedness score to which is higher or equal to the threshold of this dataset. We expanded this list of terms with the terms that are related to any of the merged terms compared to the threshold. Note that relatedness of all terms to each other exceeded a relatedness threshold, but relatedness of at least two terms needed to exceed a threshold. If a group of terms contained at least five terms, we form a silver category. We assigned a label to a silver category by (1) calculating mean label-to-terms scores of all labels applied to at least two identified terms of a group, (2) selecting a label with the maximum mean score.
\subsection{Evaluation}
To evaluate the coherence and semantic quality of the produced categories at various input configurations, we introduce the evaluation methodology to evaluate ANEA and a representative baseline against the silver datasets. By identification the input configurations that yielded the best results, we sought to propose an optimal default ANEA's input configuration for any dataset.
\subsubsection{Baseline: hierarchical clustering}
\label{sec:hc} We selected hierarchical clustering (HC) \cite{murtagh2012algorithms} as a baseline to ANEA, since it successfully identifies semantically related terms that refer to identical entities \cite{cambria2016senticnet}. Although HC does not have the functionality of automated extraction and assignment of a label of a cluster of terms, we observed that HC's clusters could form meaningful categories. Therefore, we selected HC as a baseline to compare the quality of the produced groups of terms.
To ensure the best performance of HC per each dataset and each input configuration, we created an optimization of HC that selects the best clustering results and outputs clusters that contain maximum terms with maximum cross-term similarity. For each group of terms, we ran HC four times with fixed parameters of cosine similarity and average linkage criterion. We chose the linkage method, distance metric, and the optimization of the hyperparameters for the HC that are the most similar to the ANEA.
We built clustering configurations by varying the similarity threshold value between $[0.5; 0.8]$ with a step of 0.1. For each clustering configuration $j$, we selected only clusters $CL_{j,i}$ with more than 5 terms in each ($|CL_{j,i}| \geq 5$), i.e., impose the same minimum size requirements as for ANEA. Then, we calculated a weighted similarity score of each parameter configuration $WS_j$:
\[WS_j = \frac{1}{\sum_{i=0}^I |CL_{j,i}|} \sum_{i=0}^I T_{j,i} \cdot |CL_{j,i}|\]
where $I$ is the number of clusters larger than 5 produced at a run $j$, and $T_{j,i}$ is a cross-term similarity within a cluster. We selected the best configuration as
$C_{best} = \operatorname*{arg\,max}_j (WS_j \cdot \sum_{i=0}^I |CL_{j,i}|)$, i.e., a configuration that clusters the most terms and in the most semantically coherent way.
Since HC does not have label extraction and assigning functionality, the human assessors received only one task of assessment of the cross-term relatedness of clusters produced by HC.
\subsubsection{Metrics} To evaluate the quality of the identified categories, we use five parameters: (1) number of categories: a larger number indicates diverse and narrowly defined categories, a smaller number - generalizing categories, (2) number of annotated terms (AT): property of identified relations between more terms, (3) the average size of categories: a smaller size indicates more narrowly-defined categories, whereas a larger size indicates more generally-related terms in categories; (4) average cross-term score (TS), (5) average label-to-terms score (LS), and (6) average score (AS) between TS and LS: the high scores indicate the higher relatedness of the extracted terms and extracted and assigned labels. The main goal of our evaluation is to identify which input configurations lead to the highest average score between cross-term and label-to-terms relatedness while annotating more TTA into more general categories.
\begin{table}[]
\caption{Evaluation of the entity identification methods in various input configuration. Z is the denominator to choose terms-to-annotate from the most $1/Z$ frequent terms' heads; TTA is the number of \textit{terms-to-annotate}, EC is the number of the \textit{(entity) categories} that were produced by each approach; AT is the number of \textit{annotated terms}, i.e., that belong to the identified categories; TS is the mean cross-\textit{term relatedness score} among the categories; LS is the mean \textit{label-to-terms relatedness score}; AS is the \textit{average score} between TS and LS (* means that AS is equal to TS because HC does not assign labels to groups of terms).}
\begin{tabular}{l|l|l|l|l|l|l|l|l|l}
topic & Appr. & Z & TTA & EC & AT & Size & TS & LS & AS\\
\hline
\multirow{9}{*}{D} & silver & 3 & 420 & 5 & 113 & \textbf{23} & 7.2 & 7.0 & \textbf{7.2}\\
\cline{2-10}
% & freq.heads & 3 & 420 & 13 & 88 & 6.6 & 3.7 & 5.1\\
% \cline{2-9}
& \multirow{4}{*}{HC} & 3 & 420 & 14 & 108 & 8 & 6.9 & -- & 6.9* \\
& & 4 & 363 & 12 & 87 & 7 & 7.0 & -- & 7.0*\\
& & 5 & 316 & 10 & 73 & 7 & 7.0 & -- & 7.0*\\
& & 7 & 253 & 8 & 52 & 7 & 7.2 & -- & \textbf{7.2}* \\
\cline{2-10}
& \multirow{4}{*}{ANEA} & 3 & 420 & 26 & \textbf{306} & 12 & 4.7 & 4.2 & 4.4 \\
& & 4 & 363 & 22 & 255 & 12 & 5.3 & 4.6 & 5.0 \\
& & 5 & 316 & 21 & 234 & 11 & 5.6 & 5.0 & 5.3\\
& & 7 & 253 & 18 & 179 & 10 & 5.7 & 5.0 & 5.4\\
\hline
\multirow{9}{*}{S} & silver & 3 & 356 & 6 & 57 & 10 & 6.2 & 6.0 & \textbf{6.1}\\
\cline{2-10}
% & freq.heads & 3 & 356 & 4 & 24 & 5.6 & 5.8 & 5.7\\
% \cline{2-9}
& \multirow{4}{*}{HC} & 3 & 356 & 17 & 190 & 11 & 5.3 & -- & 5.3*\\
& & 4 & 303 & 15 & 152 & 10 & 5.5 & -- & 5.5*\\
& & 5 & 255 & 15 & 137 & 9 & 5.4 & -- & 5.4*\\
& & 7 & 191 & 12 & 103 & 9 & 5.4 & -- & 5.4*\\
\cline{2-10}
& \multirow{4}{*}{ANEA} & 3 & 356 & 21 & \textbf{242} & 12 & 4.1 & 4.4 & 4.2\\
& & 4 & 303 & 18 & 186 & 10 & 4.2 & 3.8 & 4.0\\
& & 5 & 255 & 15 & 164 & 11 & 4.2 & 4.4 & 4.3\\
& & 7 & 191 & 10 & 119 & 12 & 5.0 & 5.3 & 5.2\\
\hline
\multirow{9}{*}{T} & silver & 3 & 363 & 6 & 115 & 19 & 7.8 & 6.7 & \textbf{7.3}\\
\cline{2-10}
% & freq.heads & 3 & 363 & 2 & 10 & 7.7 & 8.0 & 7.8\\
% \cline{2-9}
& \multirow{4}{*}{HC} & 3 & 363 & 19 & 156 & 8 & 7.3 & -- & \textbf{7.3}*\\
& & 4 & 297 & 16 & 133 & 8 & 7.3 & -- & \textbf{7.3}*\\
& & 5 & 258 & 12 & 105 & 9 & 7.3 & -- & \textbf{7.3}*\\
& & 7 & 211 & 9 & 76 & 8 & 7.2 & -- & 7.2*\\
\cline{2-10}
& \multirow{4}{*}{ANEA} & 3 & 363 & 22 & \textbf{239} & 11 & 5.4 & 4.8 & 5.1\\
& & 4 & 297 & 17 & 191 & 11 & 5.4 & 4.4 & 4.9 \\
& & 5 & 258 & 14 & 161 & 12 & 5.3 & 4.4 & 4.9 \\
& & 7 & 211 & 14 & 137 & 10 & 5.6 & 4.0 & 4.8 \\
\hline
\multirow{9}{*}{P} & silver & 2 & 282 & 7 & 102 & 15 & 6.6 & 6.2 & \textbf{6.4}\\
\cline{2-10}
% & freq.heads & 2 & 282 & 7 & 58 & 6.4 & 2.5 & 4.5\\
% \cline{2-9}
& \multirow{4}{*}{HC} & 2 & 282 & 9 & 65 & 7 & 5.2 & -- & 5.2* \\
& & 3 & 227 & 8 & 61 & 8 & 6.0 & -- & 6.0*\\
& & 4 & 200 & 8 & 61 & 8 & 6.0 & -- & 6.0*\\
& & 5 & 183 & 7 & 56 & 8 & 6.1 & -- & 6.1*\\
\cline{2-10}
& \multirow{4}{*}{ANEA} & 2 & 282 & 18 & \textbf{213} & 12 & 4.7 & 4.5 & 4.6 \\
& & 3 & 227 & 16 & 172 & 11 & 5.3 & 4.9 & 5.1\\
& & 4 & 200 & 16 & 163 & 10 & 5.3 & 4.8 & 5.1\\
& & 5 & 183 & 15 & 149 & 10 & 5.2 & 4.5 & 4.8 \\
\end{tabular}
\label{table:eval1}
\end{table}
\subsubsection{Results}
To calculate average relatedness scores, we assigned the scores from the normalized score matrices to the terms and labels of the identified EC and average these scores. Table \ref{table:eval1} reports the evaluation results for four datasets and four input configurations. The table shows that HC gets the highest average relatedness score ($AS_{avg, HC} = 6.5$) that almost reaches silver dataset ($AS_{avg,silver} = 6.7$) but at the same time produces categories of the size smaller than silver categories ($Size_{avg, HC} = 6$ and $Size_{avg, silver} = 16$). While on average, ANEA annotates the largest number of terms ($AT_{mean, ANEA} = 175$), it also yields the lowest average relatedness score ($AS_{mean, ANEA} = 5.2$) both compared to the silver dataset and HC. When creating a coding book, multiple human coders first annotate, and then the majority voting decides which excepts and labels describe a dataset the best. We applied a similar strategy to improve the performance of ANEA.
\begin{table}[]
\caption{Evaluation of the entity identification methods when (entity) categories (EC) are created by majority voting between input configurations. In each dataset, the majority voting improves relatedness score for each dataset. The highlighted configurations show a range of terms-to-annotate (TTA) and the resulted best average score (AS). }
\begin{tabular}{l|l|l|l|l|l|l|l|l}
& Conf. Z & TTA & EC & AT & Size & TS & LS & AS \\
\hline
\multirow{8}{*}{D} & silver& 420 & 5 & 113 &23 & 7.2 & 7.0 & 7.2\\
& prev best: 7 & 253 & 18 & 179 & 10 & 5.7 & 5.0 & 5.4\\
\cline{2-9}
& 3+4 & 363-420 & 16 & 160 & 10 & 5.7 & 5.1 & 5.4 \\
& 3+4+5 & 316-420 & 16 & 230 & 14 & 5.5 & 4.9 & 5.2 \\
& 3+4+5+7 & 253-316 & 16 & 247 & 15 & 5.4 & 4.8 & 5.1 \\
& 4+5 & 316-363 & 16 & 172 & 11 & 6.0 & 5.5 & 5.7 \\
& 4+5+7 & 253-363 & 15 & 202 & 13 & 5.7 & 5.1 & 5.4 \\
& 5+7 & \underline{\textbf{253-316}} & 12 & 122 & 10 & 6.3 & 5.9 & \underline{\textbf{6.1}} \\
\hline
\multirow{8}{*}{S} & silver & 356 & 6 & 57 & 10 & 6.2 & 6.0 & 6.1\\
& prev best: 7 & 191 & 10 & 119 & 12 & 5.0 & 5.3 & 5.2\\
\cline{2-9}
& 3+4 & 303-356 & 10 & 95 & 10 & 4.5 & 5.0 & 4.8 \\
& 3+4+5 & 255-356 & 13 & 173 & 13 & 4.4 & 4.6 & 4.5 \\
& 3+4+5+7 & 191-356 & 11 & 185 & 17 & 4.4 & 4.4 & 4.4 \\
& 4+5 & 255-303 & 8 & 89 & 11 & 4.4 & 4.5 & 4.4 \\
& 4+5+7 & 191-303 & 10 & 126 & 13 & 4.2 & 4.8 & 4.5 \\
& 5+7 & \underline{\textbf{191-255}} & 4 & 44 & 11 & 5.6 & 6.5 & \underline{\textbf{6.0}} \\
\hline
\multirow{8}{*}{T} & silver & 363 & 6 & 115 & 19 & 7.8 & 6.7 & 7.3\\
& prev best: 3 & 363 & 22 & 239 & 11 & 5.4 & 4.8 & 5.1\\
\cline{2-9}
& 3+4 & 297-363 & 14 & 119 & 9 & 6.1 & 5.3 & 5.7 \\
& 3+4+5 & \underline{\textbf{258-363}} & 12 & 146 & 12 & 6.2 & 5.6 & \underline{\textbf{5.9}} \\
& 3+4+5+7 & 211-363 & 9 & 171 & 19 & 6.1 & 5.6 & 5.8 \\
& 4+5 & 258-297 & 10 & 101 & 10 & 6.1 & 5.0 & 5.6 \\
& 4+5+7 & 211-297 & 12 & 137 & 11 & 5.8 & 5.1 & 5.4 \\
& 5+7 & 211-258 & 9 & 86 & 10 & 5.9 & 5.2 & 5.6 \\
\hline
\multirow{8}{*}{P} & silver & 282 & 7 & 102 & 15 & 6.6 & 6.2 & 6.4\\
& prev best: 3 & 227 & 16 & 172 & 11 & 5.3 & 4.9 & 5.1\\
\cline{2-9}
& 2+3 & 227-282 & 12 & 133 & 11 & 5.6 & 5.6 & 5.6 \\
& 2+3+4 & 200-282 & 14 & 160 & 11 & 5.5 & 5.2 & 5.4 \\
& 2+3+4+5 & \underline{\textbf{181-282}} & 9 & 157 & 17 & 5.7 & 5.6 & \underline{\textbf{5.6}} \\
& 3+4 & 200-227 & 12 & 128 & 11 & 5.5 & 4.9 & 5.2 \\
& 3+4+5 & 183-227 & 14 & 146 & 10 & 5.4 & 4.7 & 5.0 \\
& 4+5 & 200-227 & 13 & 121 & 9 & 5.4 & 5.0 & 5.2 \\
\end{tabular}
\label{table:eval2}
\end{table}
\subsubsection{Voting strategy and default input configuration}
Ensemble learning is a common approach in machine learning to improve the results of a classifier, i.e., by combining predictions of multiple classifiers to achieve a boost in the overall accuracy through collecting ``the wisdom of the crowd'' \cite{berthold2010guide}.
We followed this principle to improve the quality of the extracted by ANEA categories and combine results of 2-4 configurations in each dataset. Similar to the construction of silver datasets, we created a category of at least five terms if these terms co-occurred in at least two input configurations. We assigned a label that describes the majority of the terms in the identified group.
Table \ref{table:eval2} reports that at least one of the combination of ANEA with multiple input configurations increases the average relatedness score compared to ANEA without a voting strategy on average by 0.7 ($AS_{avg, ANEA_{vote}} = 5.9$). Although the voting approach does not exceed the relatedness scores of the silver datasets (reaches 87.9\% of the silver score), it increases a number of annotated terms per category ($AS_{avg, ANEA} = 11$ and $AS_{avg, ANEA_{vote}} = 13$) and also identifies more generalizing categories ($EC_{avg, ANEA} = 15$ and $EC_{avg, ANEA_{vote}} = 9$).
To identify the best default configuration for the voting strategy of ANEA, we selected the best performing voting strategy configurations per each dataset and deduced a default input configuration by generalizing these configurations. We took the minimum and the maximum number of terms-to-annotate (TTAs) from the best voting configuration and plot against a number of unique heads in each dataset (see Table \ref{tab:events}).
Figure \ref{img:trend} depicts a linear trend between the TTAs and the unique heads from the datasets. Based on this trend, for any other dataset, we recommend annotating only the first $y=158 + 0.167x$ TTAs that belong to the most frequent heads, where $x$ is a number of unique heads in a dataset. For the voting strategy, we recommend using the input configurations of the $y$, $y-40$, and $y+40$ number of terms that share the most frequent heads of terms.
\begin{figure}[h]
\centering
\includegraphics[width=0.45\textwidth]{inpreq.png}
\caption{\label{img:trend} \centering Default input configuration: a number of terms-to-annotate linearly depends on the number of unique NP heads per dataset. }
\end{figure}
\section{Discussion and future work}
Our evaluation shows that ANEA facilitates a faster annotation process. Specifically, ANEA automatically performs the most time-consuming tasks of deriving a coding book for the annotation of a dataset for NER. ANEA imitates the first two stages of a manual annotation process. First, a small set of articles ($\approx$6000-8000 words) is used to automatically identify categories relevant to the data of the current domain, i.e., identify and extract related terms and assign a label to each of them (identified 4-12 categories with 44-157 assigned terms). Second, a voting strategy is applied, which aims at increasing the validity of the derived categories by following the idea of ensemble learning and intercoder agreement (relatedness score improved on average from 77.4\% of the silver average relatedness score to 87.9\%). To continue with manual annotation of a (N)ER dataset, next, researchers manually validate their coding book. If they find that the coding book sufficiently represents the dataset, they annotate the remaining texts to create a large corpus for NER.
Therefore, the primary use cases of ANEA are as follows. First, extraction of domain categories from a subset of a large text dataset and improve their quality with the voting strategy. Second, manual validation and improvement of the identified categories by moving terms between the categories and suggesting better labels to them.
The final stage of annotation of a NER dataset is to apply a coding book to a large dataset, i.e., read the text and assign categories to text excerpts following guidelines or examples of the coding book. Although, such manual text annotation is a standard approach to create ``gold''-standard datasets, the recent semi-supervised learning neural network models (e.g., DART \cite{chang-etal-2020-dart}) show high potential in reliable annotating of large text collections. We plan to use the models like DART will complete the automation of creating domain-specific NER datasets.
Future work directions include the creation of manually annotated datasets from scratch for multiple domains to calculate accuracy metrics, e.g., precision, recall, and F1, to evaluate the effectiveness of identifying categories by ANEA. Further, to improve the quality and meaningfulness of the assigned labels, we plan to test ANEA on other knowledge graphs, e.g., Wikidata or BabelNet. To test the applicability of ANEA, we also plan to evaluate the approach in other languages, e.g., English, with an additional module for the identification of multi-word expressions similar to compound-based German words \cite{salehi2014detecting}. Further, to improve the semantic quality of both categories' terms and labels in a specific domain, we plan to use a language model, e.g., BERT, use the quality score as a learning objective. Lastly, we seek to build a semi-supervised NER model to complete automated annotation of NER datasets, i.e., automatically annotate large datasets suitable for training neural network models. We will use terms and labels from the derived categories as the seed-terms and seed-labels and perform named entity tagging and classify more domain terms \cite{shang2018learning, chang-etal-2020-dart}.
\section{Conclusion}
In this paper, we propose ANEA, an automatic approach to derive domain entity categories from a subset of domain input texts, i.e., create a small dataset to train a NER model. Specifically, ANEA identifies related domain-representative terms and automatically extracts and assigns descriptive and generalizing labels to them based on Wiktionary. In our user assessment and evaluation, ANEA could not outperform a silver dataset on the relatedness scores assigned to the groups of terms and labels describing these groups. However, ANEA produced more generalizing domain categories compared to a strong baseline. We showed that our voting strategy of combining terms and labels from the categories identified at multiple input configurations significantly improved the quality of the final categories. Additionally, we suggested a default input configuration that can be applied to derive categories from German domain text datasets. Finally, we think that the best application of ANEA is to annotate and use a small dataset in semi-supervised learning. Moreover, we plan to improve and validate the annotations with a domain expert, and use this small domain dataset to train state-of-the-art NER models.
\section*{Acknowledgement}
The research for this paper has been conducted in collaboration with the company eschbach (\url{https://eschbach.com}) supported by the Central Innovation Programme (ZIM) of the German Federal Ministry for Economic Affairs and Energy.
We thank all study participants for their significant contribution to this publication.
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
\end{document}
\endinput
%%
%% End of file `sample-lualatex.tex'.
%%
%% This is file `sample-sigconf.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigconf')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf]{acmart}
%% NOTE that a single column version may be required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmcopyright}
\copyrightyear{2018}
\acmYear{2018}
\acmDOI{10.1145/1122445.1122456}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[Woodstock '18]{Woodstock '18: ACM Symposium on Neural
Gaze Detection}{June 03--05, 2018}{Woodstock, NY}
\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
June 03--05, 2018, Woodstock, NY}
\acmPrice{15.00}
\acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{The Name of the Title is Hope}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Ben Trovato}
\authornote{Both authors contributed equally to this research.}
\email{trovato@corporation.com}
\orcid{1234-5678-9012}
\author{G.K.M. Tobin}
\authornotemark[1]
\email{webmaster@marysville-ohio.com}
\affiliation{%
\institution{Institute for Clarity in Documentation}
\streetaddress{P.O. Box 1212}
\city{Dublin}
\state{Ohio}
\country{USA}
\postcode{43017-6221}
}
\author{Lars Th{\o}rv{\"a}ld}
\affiliation{%
\institution{The Th{\o}rv{\"a}ld Group}
\streetaddress{1 Th{\o}rv{\"a}ld Circle}
\city{Hekla}
\country{Iceland}}
\email{larst@affiliation.org}
\author{Valerie B\'eranger}
\affiliation{%
\institution{Inria Paris-Rocquencourt}
\city{Rocquencourt}
\country{France}
}
\author{Aparna Patel}
\affiliation{%
\institution{Rajiv Gandhi University}
\streetaddress{Rono-Hills}
\city{Doimukh}
\state{Arunachal Pradesh}
\country{India}}
\author{Huifen Chan}
\affiliation{%
\institution{Tsinghua University}
\streetaddress{30 Shuangqing Rd}
\city{Haidian Qu}
\state{Beijing Shi}
\country{China}}
\author{Charles Palmer}
\affiliation{%
\institution{Palmer Research Laboratories}
\streetaddress{8600 Datapoint Drive}
\city{San Antonio}
\state{Texas}
\country{USA}
\postcode{78229}}
\email{cpalmer@prl.com}
\author{John Smith}
\affiliation{%
\institution{The Th{\o}rv{\"a}ld Group}
\streetaddress{1 Th{\o}rv{\"a}ld Circle}
\city{Hekla}
\country{Iceland}}
\email{jsmith@affiliation.org}
\author{Julius P. Kumquat}
\affiliation{%
\institution{The Kumquat Consortium}
\city{New York}
\country{USA}}
\email{jpkumquat@consortium.net}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Trovato and Tobin, et al.}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
A clear and well-documented \LaTeX\ document is presented as an
article formatted for publication by ACM in a conference proceedings
or journal publication. Based on the ``acmart'' document class, this
article presents and explains many of the common variations, as well
as many of the formatting elements an author may use in the
preparation of the documentation of their work.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10010520.10010553.10010562</concept_id>
<concept_desc>Computer systems organization~Embedded systems</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010575.10010755</concept_id>
<concept_desc>Computer systems organization~Redundancy</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010553.10010554</concept_id>
<concept_desc>Computer systems organization~Robotics</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10003033.10003083.10003095</concept_id>
<concept_desc>Networks~Network reliability</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Computer systems organization~Embedded systems}
\ccsdesc[300]{Computer systems organization~Redundancy}
\ccsdesc{Computer systems organization~Robotics}
\ccsdesc[100]{Networks~Network reliability}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{datasets, neural networks, gaze detection, text tagging}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
\begin{teaserfigure}
\includegraphics[width=\textwidth]{sampleteaser}
\caption{Seattle Mariners at Spring Training, 2010.}
\Description{Enjoying the baseball game from the third-base
seats. Ichiro Suzuki preparing to bat.}
\label{fig:teaser}
\end{teaserfigure}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
ACM's consolidated article template, introduced in 2017, provides a
consistent \LaTeX\ style for use across ACM publications, and
incorporates accessibility and metadata-extraction functionality
necessary for future Digital Library endeavors. Numerous ACM and
SIG-specific \LaTeX\ templates have been examined, and their unique
features incorporated into this single new template.
If you are new to publishing with ACM, this document is a valuable
guide to the process of preparing your work for publication. If you
have published with ACM before, this document provides insight and
instruction into more recent changes to the article template.
The ``\verb|acmart|'' document class can be used to prepare articles
for any ACM publication --- conference or journal, and for any stage
of publication, from review to final ``camera-ready'' copy, to the
author's own version, with {\itshape very} few changes to the source.
\section{Template Overview}
As noted in the introduction, the ``\verb|acmart|'' document class can
be used to prepare many different kinds of documentation --- a
double-blind initial submission of a full-length technical paper, a
two-page SIGGRAPH Emerging Technologies abstract, a ``camera-ready''
journal article, a SIGCHI Extended Abstract, and more --- all by
selecting the appropriate {\itshape template style} and {\itshape
template parameters}.
This document will explain the major features of the document
class. For further information, the {\itshape \LaTeX\ User's Guide} is
available from
\url{https://www.acm.org/publications/proceedings-template}.
\subsection{Template Styles}
The primary parameter given to the ``\verb|acmart|'' document class is
the {\itshape template style} which corresponds to the kind of publication
or SIG publishing the work. This parameter is enclosed in square
brackets and is a part of the {\verb|documentclass|} command:
\begin{verbatim}
\documentclass[STYLE]{acmart}
\end{verbatim}
Journals use one of three template styles. All but three ACM journals
use the {\verb|acmsmall|} template style:
\begin{itemize}
\item {\verb|acmsmall|}: The default journal template style.
\item {\verb|acmlarge|}: Used by JOCCH and TAP.
\item {\verb|acmtog|}: Used by TOG.
\end{itemize}
The majority of conference proceedings documentation will use the {\verb|acmconf|} template style.
\begin{itemize}
\item {\verb|acmconf|}: The default proceedings template style.
\item{\verb|sigchi|}: Used for SIGCHI conference articles.
\item{\verb|sigchi-a|}: Used for SIGCHI ``Extended Abstract'' articles.
\item{\verb|sigplan|}: Used for SIGPLAN conference articles.
\end{itemize}
\subsection{Template Parameters}
In addition to specifying the {\itshape template style} to be used in
formatting your work, there are a number of {\itshape template parameters}
which modify some part of the applied template style. A complete list
of these parameters can be found in the {\itshape \LaTeX\ User's Guide.}
Frequently-used parameters, or combinations of parameters, include:
\begin{itemize}
\item {\verb|anonymous,review|}: Suitable for a ``double-blind''
conference submission. Anonymizes the work and includes line
numbers. Use with the \verb|\acmSubmissionID| command to print the
submission's unique ID on each page of the work.
\item{\verb|authorversion|}: Produces a version of the work suitable
for posting by the author.
\item{\verb|screen|}: Produces colored hyperlinks.
\end{itemize}
This document uses the following string as the first command in the
source file:
\begin{verbatim}
\documentclass[sigconf]{acmart}
\end{verbatim}
\section{Modifications}
Modifying the template --- including but not limited to: adjusting
margins, typeface sizes, line spacing, paragraph and list definitions,
and the use of the \verb|\vspace| command to manually adjust the
vertical spacing between elements of your work --- is not allowed.
{\bfseries Your document will be returned to you for revision if
modifications are discovered.}
\section{Typefaces}
The ``\verb|acmart|'' document class requires the use of the
``Libertine'' typeface family. Your \TeX\ installation should include
this set of packages. Please do not substitute other typefaces. The
``\verb|lmodern|'' and ``\verb|ltimes|'' packages should not be used,
as they will override the built-in typeface families.
\section{Title Information}
The title of your work should use capital letters appropriately -
\url{https://capitalizemytitle.com/} has useful rules for
capitalization. Use the {\verb|title|} command to define the title of
your work. If your work has a subtitle, define it with the
{\verb|subtitle|} command. Do not insert line breaks in your title.
If your title is lengthy, you must define a short version to be used
in the page headers, to prevent overlapping text. The \verb|title|
command has a ``short title'' parameter:
\begin{verbatim}
\title[short title]{full title}
\end{verbatim}
\section{Authors and Affiliations}
Each author must be defined separately for accurate metadata
identification. Multiple authors may share one affiliation. Authors'
names should not be abbreviated; use full first names wherever
possible. Include authors' e-mail addresses whenever possible.
Grouping authors' names or e-mail addresses, or providing an ``e-mail
alias,'' as shown below, is not acceptable:
\begin{verbatim}
\author{Brooke Aster, David Mehldau}
\email{dave,judy,steve@university.edu}
\email{firstname.lastname@phillips.org}
\end{verbatim}
The \verb|authornote| and \verb|authornotemark| commands allow a note
to apply to multiple authors --- for example, if the first two authors
of an article contributed equally to the work.
If your author list is lengthy, you must define a shortened version of
the list of authors to be used in the page headers, to prevent
overlapping text. The following command should be placed just after
the last \verb|\author{}| definition:
\begin{verbatim}
\renewcommand{\shortauthors}{McCartney, et al.}
\end{verbatim}
Omitting this command will force the use of a concatenated list of all
of the authors' names, which may result in overlapping text in the
page headers.
The article template's documentation, available at
\url{https://www.acm.org/publications/proceedings-template}, has a
complete explanation of these commands and tips for their effective
use.
Note that authors' addresses are mandatory for journal articles.
\section{Rights Information}
Authors of any work published by ACM will need to complete a rights
form. Depending on the kind of work, and the rights management choice
made by the author, this may be copyright transfer, permission,
license, or an OA (open access) agreement.
Regardless of the rights management choice, the author will receive a
copy of the completed rights form once it has been submitted. This
form contains \LaTeX\ commands that must be copied into the source
document. When the document source is compiled, these commands and
their parameters add formatted text to several areas of the final
document:
\begin{itemize}
\item the ``ACM Reference Format'' text on the first page.
\item the ``rights management'' text on the first page.
\item the conference information in the page header(s).
\end{itemize}
Rights information is unique to the work; if you are preparing several
works for an event, make sure to use the correct set of commands with
each of the works.
The ACM Reference Format text is required for all articles over one
page in length, and is optional for one-page articles (abstracts).
\section{CCS Concepts and User-Defined Keywords}
Two elements of the ``acmart'' document class provide powerful
taxonomic tools for you to help readers find your work in an online
search.
The ACM Computing Classification System ---
\url{https://www.acm.org/publications/class-2012} --- is a set of
classifiers and concepts that describe the computing
discipline. Authors can select entries from this classification
system, via \url{https://dl.acm.org/ccs/ccs.cfm}, and generate the
commands to be included in the \LaTeX\ source.
User-defined keywords are a comma-separated list of words and phrases
of the authors' choosing, providing a more flexible way of describing
the research being presented.
CCS concepts and user-defined keywords are required for for all
articles over two pages in length, and are optional for one- and
two-page articles (or abstracts).
\section{Sectioning Commands}
Your work should use standard \LaTeX\ sectioning commands:
\verb|section|, \verb|subsection|, \verb|subsubsection|, and
\verb|paragraph|. They should be numbered; do not remove the numbering
from the commands.
Simulating a sectioning command by setting the first word or words of
a paragraph in boldface or italicized text is {\bfseries not allowed.}
\section{Tables}
The ``\verb|acmart|'' document class includes the ``\verb|booktabs|''
package --- \url{https://ctan.org/pkg/booktabs} --- for preparing
high-quality tables.
Table captions are placed {\itshape above} the table.
Because tables cannot be split across pages, the best placement for
them is typically the top of the page nearest their initial cite. To
ensure this proper ``floating'' placement of tables, use the
environment \textbf{table} to enclose the table's contents and the
table caption. The contents of the table itself must go in the
\textbf{tabular} environment, to be aligned properly in rows and
columns, with the desired horizontal and vertical rules. Again,
detailed instructions on \textbf{tabular} material are found in the
\textit{\LaTeX\ User's Guide}.
Immediately following this sentence is the point at which
Table~\ref{tab:freq} is included in the input file; compare the
placement of the table here with the table in the printed output of
this document.
\begin{table}
\caption{Frequency of Special Characters}
\label{tab:freq}
\begin{tabular}{ccl}
\toprule
Non-English or Math&Frequency&Comments\\
\midrule
\O & 1 in 1,000& For Swedish names\\
$\pi$ & 1 in 5& Common in math\\
\$ & 4 in 5 & Used in business\\
$\Psi^2_1$ & 1 in 40,000& Unexplained usage\\
\bottomrule
\end{tabular}
\end{table}
To set a wider table, which takes up the whole width of the page's
live area, use the environment \textbf{table*} to enclose the table's
contents and the table caption. As with a single-column table, this
wide table will ``float'' to a location deemed more
desirable. Immediately following this sentence is the point at which
Table~\ref{tab:commands} is included in the input file; again, it is
instructive to compare the placement of the table here with the table
in the printed output of this document.
\begin{table*}
\caption{Some Typical Commands}
\label{tab:commands}
\begin{tabular}{ccl}
\toprule
Command &A Number & Comments\\
\midrule
\texttt{{\char'134}author} & 100& Author \\
\texttt{{\char'134}table}& 300 & For tables\\
\texttt{{\char'134}table*}& 400& For wider tables\\
\bottomrule
\end{tabular}
\end{table*}
Always use midrule to separate table header rows from data rows, and
use it only for this purpose. This enables assistive technologies to
recognise table headers and support their users in navigating tables
more easily.
\section{Math Equations}
You may want to display math equations in three distinct styles:
inline, numbered or non-numbered display. Each of the three are
discussed in the next sections.
\subsection{Inline (In-text) Equations}
A formula that appears in the running text is called an inline or
in-text formula. It is produced by the \textbf{math} environment,
which can be invoked with the usual
\texttt{{\char'134}begin\,\ldots{\char'134}end} construction or with
the short form \texttt{\$\,\ldots\$}. You can use any of the symbols
and structures, from $\alpha$ to $\omega$, available in
\LaTeX~\cite{Lamport:LaTeX}; this section will simply show a few
examples of in-text equations in context. Notice how this equation:
\begin{math}
\lim_{n\rightarrow \infty}x=0
\end{math},
set here in in-line math style, looks slightly different when
set in display style. (See next section).
\subsection{Display Equations}
A numbered display equation---one set off by vertical space from the
text and centered horizontally---is produced by the \textbf{equation}
environment. An unnumbered display equation is produced by the
\textbf{displaymath} environment.
Again, in either environment, you can use any of the symbols and
structures available in \LaTeX\@; this section will just give a couple
of examples of display equations in context. First, consider the
equation, shown as an inline equation above:
\begin{equation}
\lim_{n\rightarrow \infty}x=0
\end{equation}
Notice how it is formatted somewhat differently in
the \textbf{displaymath}
environment. Now, we'll enter an unnumbered equation:
\begin{displaymath}
\sum_{i=0}^{\infty} x + 1
\end{displaymath}
and follow it with another numbered equation:
\begin{equation}
\sum_{i=0}^{\infty}x_i=\int_{0}^{\pi+2} f
\end{equation}
just to demonstrate \LaTeX's able handling of numbering.
\section{Figures}
The ``\verb|figure|'' environment should be used for figures. One or
more images can be placed within a figure. If your figure contains
third-party material, you must clearly identify it as such, as shown
in the example below.
\begin{figure}[h]
\centering
\includegraphics[width=\linewidth]{sample-franklin}
\caption{1907 Franklin Model D roadster. Photograph by Harris \&
Ewing, Inc. [Public domain], via Wikimedia
Commons. (\url{https://goo.gl/VLCRBB}).}
\Description{A woman and a girl in white dresses sit in an open car.}
\end{figure}
Your figures should contain a caption which describes the figure to
the reader.
Figure captions are placed {\itshape below} the figure.
Every figure should also have a figure description unless it is purely
decorative. These descriptions convey what’s in the image to someone
who cannot see it. They are also used by search engine crawlers for
indexing images, and when images cannot be loaded.
A figure description must be unformatted plain text less than 2000
characters long (including spaces). {\bfseries Figure descriptions
should not repeat the figure caption – their purpose is to capture
important information that is not already provided in the caption or
the main text of the paper.} For figures that convey important and
complex new information, a short text description may not be
adequate. More complex alternative descriptions can be placed in an
appendix and referenced in a short figure description. For example,
provide a data table capturing the information in a bar chart, or a
structured list representing a graph. For additional information
regarding how best to write figure descriptions and why doing this is
so important, please see
\url{https://www.acm.org/publications/taps/describing-figures/}.
\subsection{The ``Teaser Figure''}
A ``teaser figure'' is an image, or set of images in one figure, that
are placed after all author and affiliation information, and before
the body of the article, spanning the page. If you wish to have such a
figure in your article, place the command immediately before the
\verb|\maketitle| command:
\begin{verbatim}
\begin{teaserfigure}
\includegraphics[width=\textwidth]{sampleteaser}
\caption{figure caption}
\Description{figure description}
\end{teaserfigure}
\end{verbatim}
\section{Citations and Bibliographies}
The use of \BibTeX\ for the preparation and formatting of one's
references is strongly recommended. Authors' names should be complete
--- use full first names (``Donald E. Knuth'') not initials
(``D. E. Knuth'') --- and the salient identifying features of a
reference should be included: title, year, volume, number, pages,
article DOI, etc.
The bibliography is included in your source document with these two
commands, placed just before the \verb|\end{document}| command:
\begin{verbatim}
\bibliographystyle{ACM-Reference-Format}
\bibliography{bibfile}
\end{verbatim}
where ``\verb|bibfile|'' is the name, without the ``\verb|.bib|''
suffix, of the \BibTeX\ file.
Citations and references are numbered by default. A small number of
ACM publications have citations and references formatted in the
``author year'' style; for these exceptions, please include this
command in the {\bfseries preamble} (before the command
``\verb|\begin{document}|'') of your \LaTeX\ source:
\begin{verbatim}
\citestyle{acmauthoryear}
\end{verbatim}
Some examples. A paginated journal article \cite{Abril07}, an
enumerated journal article \cite{Cohen07}, a reference to an entire
issue \cite{JCohen96}, a monograph (whole book) \cite{Kosiur01}, a
monograph/whole book in a series (see 2a in spec. document)
\cite{Harel79}, a divisible-book such as an anthology or compilation
\cite{Editor00} followed by the same example, however we only output
the series if the volume number is given \cite{Editor00a} (so
Editor00a's series should NOT be present since it has no vol. no.),
a chapter in a divisible book \cite{Spector90}, a chapter in a
divisible book in a series \cite{Douglass98}, a multi-volume work as
book \cite{Knuth97}, a couple of articles in a proceedings (of a
conference, symposium, workshop for example) (paginated proceedings
article) \cite{Andler79, Hagerup1993}, a proceedings article with
all possible elements \cite{Smith10}, an example of an enumerated
proceedings article \cite{VanGundy07}, an informally published work
\cite{Harel78}, a couple of preprints \cite{Bornmann2019,
AnzarootPBM14}, a doctoral dissertation \cite{Clarkson85}, a
master's thesis: \cite{anisi03}, an online document / world wide web
resource \cite{Thornburg01, Ablamowicz07, Poker06}, a video game
(Case 1) \cite{Obama08} and (Case 2) \cite{Novak03} and \cite{Lee05}
and (Case 3) a patent \cite{JoeScientist001}, work accepted for
publication \cite{rous08}, 'YYYYb'-test for prolific author
\cite{SaeediMEJ10} and \cite{SaeediJETC10}. Other cites might
contain 'duplicate' DOI and URLs (some SIAM articles)
\cite{Kirschmer:2010:AEI:1958016.1958018}. Boris / Barbara Beeton:
multi-volume works as books \cite{MR781536} and \cite{MR781537}. A
couple of citations with DOIs:
\cite{2004:ITE:1009386.1010128,Kirschmer:2010:AEI:1958016.1958018}. Online
citations: \cite{TUGInstmem, Thornburg01, CTANacmart}. Artifacts:
\cite{R} and \cite{UMassCitations}.
\section{Acknowledgments}
Identification of funding sources and other support, and thanks to
individuals and groups that assisted in the research and the
preparation of the work should be included in an acknowledgment
section, which is placed just before the reference section in your
document.
This section has a special environment:
\begin{verbatim}
\begin{acks}
...
\end{acks}
\end{verbatim}
so that the information contained therein can be more easily collected
during the article metadata extraction phase, and to ensure
consistency in the spelling of the section heading.
Authors should not prepare this section as a numbered or unnumbered {\verb|\section|}; please use the ``{\verb|acks|}'' environment.
\section{Appendices}
If your work needs an appendix, add it before the
``\verb|\end{document}|'' command at the conclusion of your source
document.
Start the appendix with the ``\verb|appendix|'' command:
\begin{verbatim}
\appendix
\end{verbatim}
and note that in the appendix, sections are lettered, not
numbered. This document has two appendices, demonstrating the section
and subsection identification method.
\section{SIGCHI Extended Abstracts}
The ``\verb|sigchi-a|'' template style (available only in \LaTeX\ and
not in Word) produces a landscape-orientation formatted article, with
a wide left margin. Three environments are available for use with the
``\verb|sigchi-a|'' template style, and produce formatted output in
the margin:
\begin{itemize}
\item {\verb|sidebar|}: Place formatted text in the margin.
\item {\verb|marginfigure|}: Place a figure in the margin.
\item {\verb|margintable|}: Place a table in the margin.
\end{itemize}
%%
%% The acknowledgments section is defined using the "acks" environment
%% (and NOT an unnumbered section). This ensures the proper
%% identification of the section in the article metadata, and the
%% consistent spelling of the heading.
\begin{acks}
To Robert, for the bagels and explaining CMYK and color spaces.
\end{acks}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
%%
%% If your work has an appendix, this is the place to put it.
\appendix
\section{Research Methods}
\subsection{Part One}
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi
malesuada, quam in pulvinar varius, metus nunc fermentum urna, id
sollicitudin purus odio sit amet enim. Aliquam ullamcorper eu ipsum
vel mollis. Curabitur quis dictum nisl. Phasellus vel semper risus, et
lacinia dolor. Integer ultricies commodo sem nec semper.
\subsection{Part Two}
Etiam commodo feugiat nisl pulvinar pellentesque. Etiam auctor sodales
ligula, non varius nibh pulvinar semper. Suspendisse nec lectus non
ipsum convallis congue hendrerit vitae sapien. Donec at laoreet
eros. Vivamus non purus placerat, scelerisque diam eu, cursus
ante. Etiam aliquam tortor auctor efficitur mattis.
\section{Online Resources}
Nam id fermentum dui. Suspendisse sagittis tortor a nulla mollis, in
pulvinar ex pretium. Sed interdum orci quis metus euismod, et sagittis
enim maximus. Vestibulum gravida massa ut felis suscipit
congue. Quisque mattis elit a risus ultrices commodo venenatis eget
dui. Etiam sagittis eleifend elementum.
Nam interdum magna at lectus dignissim, ac dignissim lorem
rhoncus. Maecenas eu arcu ac neque placerat aliquam. Nunc pulvinar
massa et mattis lacinia.
\end{document}
\endinput
%%
%% End of file `sample-sigconf.tex'.
%%
%% This is file `sample-xelatex.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigconf')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf]{acmart}
%% NOTE that a single column version may be required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmcopyright}
\copyrightyear{2018}
\acmYear{2018}
\acmDOI{10.1145/1122445.1122456}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[Woodstock '18]{Woodstock '18: ACM Symposium on Neural
Gaze Detection}{June 03--05, 2018}{Woodstock, NY}
\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
June 03--05, 2018, Woodstock, NY}
\acmPrice{15.00}
\acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{The Name of the Title is Hope}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Ben Trovato}
\authornote{Both authors contributed equally to this research.}
\email{trovato@corporation.com}
\orcid{1234-5678-9012}
\author{G.K.M. Tobin}
\authornotemark[1]
\email{webmaster@marysville-ohio.com}
\affiliation{%
\institution{Institute for Clarity in Documentation}
\streetaddress{P.O. Box 1212}
\city{Dublin}
\state{Ohio}
\country{USA}
\postcode{43017-6221}
}
\author{Lars Th{\o}rv{\"a}ld}
\affiliation{%
\institution{The Th{\o}rv{\"a}ld Group}
\streetaddress{1 Th{\o}rv{\"a}ld Circle}
\city{Hekla}
\country{Iceland}}
\email{larst@affiliation.org}
\author{Valerie B\'eranger}
\affiliation{%
\institution{Inria Paris-Rocquencourt}
\city{Rocquencourt}
\country{France}
}
\author{Aparna Patel}
\affiliation{%
\institution{Rajiv Gandhi University}
\streetaddress{Rono-Hills}
\city{Doimukh}
\state{Arunachal Pradesh}
\country{India}}
\author{Huifen Chan}
\affiliation{%
\institution{Tsinghua University}
\streetaddress{30 Shuangqing Rd}
\city{Haidian Qu}
\state{Beijing Shi}
\country{China}}
\author{Charles Palmer}
\affiliation{%
\institution{Palmer Research Laboratories}
\streetaddress{8600 Datapoint Drive}
\city{San Antonio}
\state{Texas}
\country{USA}
\postcode{78229}}
\email{cpalmer@prl.com}
\author{John Smith}
\affiliation{%
\institution{The Th{\o}rv{\"a}ld Group}
\streetaddress{1 Th{\o}rv{\"a}ld Circle}
\city{Hekla}
\country{Iceland}}
\email{jsmith@affiliation.org}
\author{Julius P. Kumquat}
\affiliation{%
\institution{The Kumquat Consortium}
\city{New York}
\country{USA}}
\email{jpkumquat@consortium.net}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Trovato and Tobin, et al.}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
A clear and well-documented \LaTeX\ document is presented as an
article formatted for publication by ACM in a conference proceedings
or journal publication. Based on the ``acmart'' document class, this
article presents and explains many of the common variations, as well
as many of the formatting elements an author may use in the
preparation of the documentation of their work.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10010520.10010553.10010562</concept_id>
<concept_desc>Computer systems organization~Embedded systems</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010575.10010755</concept_id>
<concept_desc>Computer systems organization~Redundancy</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010553.10010554</concept_id>
<concept_desc>Computer systems organization~Robotics</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10003033.10003083.10003095</concept_id>
<concept_desc>Networks~Network reliability</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Computer systems organization~Embedded systems}
\ccsdesc[300]{Computer systems organization~Redundancy}
\ccsdesc{Computer systems organization~Robotics}
\ccsdesc[100]{Networks~Network reliability}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{datasets, neural networks, gaze detection, text tagging}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
\begin{teaserfigure}
\includegraphics[width=\textwidth]{sampleteaser}
\caption{Seattle Mariners at Spring Training, 2010.}
\Description{Enjoying the baseball game from the third-base
seats. Ichiro Suzuki preparing to bat.}
\label{fig:teaser}
\end{teaserfigure}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
ACM's consolidated article template, introduced in 2017, provides a
consistent \LaTeX\ style for use across ACM publications, and
incorporates accessibility and metadata-extraction functionality
necessary for future Digital Library endeavors. Numerous ACM and
SIG-specific \LaTeX\ templates have been examined, and their unique
features incorporated into this single new template.
If you are new to publishing with ACM, this document is a valuable
guide to the process of preparing your work for publication. If you
have published with ACM before, this document provides insight and
instruction into more recent changes to the article template.
The ``\verb|acmart|'' document class can be used to prepare articles
for any ACM publication --- conference or journal, and for any stage
of publication, from review to final ``camera-ready'' copy, to the
author's own version, with {\itshape very} few changes to the source.
\section{Template Overview}
As noted in the introduction, the ``\verb|acmart|'' document class can
be used to prepare many different kinds of documentation --- a
double-blind initial submission of a full-length technical paper, a
two-page SIGGRAPH Emerging Technologies abstract, a ``camera-ready''
journal article, a SIGCHI Extended Abstract, and more --- all by
selecting the appropriate {\itshape template style} and {\itshape
template parameters}.
This document will explain the major features of the document
class. For further information, the {\itshape \LaTeX\ User's Guide} is
available from
\url{https://www.acm.org/publications/proceedings-template}.
\subsection{Template Styles}
The primary parameter given to the ``\verb|acmart|'' document class is
the {\itshape template style} which corresponds to the kind of publication
or SIG publishing the work. This parameter is enclosed in square
brackets and is a part of the {\verb|documentclass|} command:
\begin{verbatim}
\documentclass[STYLE]{acmart}
\end{verbatim}
Journals use one of three template styles. All but three ACM journals
use the {\verb|acmsmall|} template style:
\begin{itemize}
\item {\verb|acmsmall|}: The default journal template style.
\item {\verb|acmlarge|}: Used by JOCCH and TAP.
\item {\verb|acmtog|}: Used by TOG.
\end{itemize}
The majority of conference proceedings documentation will use the {\verb|acmconf|} template style.
\begin{itemize}
\item {\verb|acmconf|}: The default proceedings template style.
\item{\verb|sigchi|}: Used for SIGCHI conference articles.
\item{\verb|sigchi-a|}: Used for SIGCHI ``Extended Abstract'' articles.
\item{\verb|sigplan|}: Used for SIGPLAN conference articles.
\end{itemize}
\subsection{Template Parameters}
In addition to specifying the {\itshape template style} to be used in
formatting your work, there are a number of {\itshape template parameters}
which modify some part of the applied template style. A complete list
of these parameters can be found in the {\itshape \LaTeX\ User's Guide.}
Frequently-used parameters, or combinations of parameters, include:
\begin{itemize}
\item {\verb|anonymous,review|}: Suitable for a ``double-blind''
conference submission. Anonymizes the work and includes line
numbers. Use with the \verb|\acmSubmissionID| command to print the
submission's unique ID on each page of the work.
\item{\verb|authorversion|}: Produces a version of the work suitable
for posting by the author.
\item{\verb|screen|}: Produces colored hyperlinks.
\end{itemize}
This document uses the following string as the first command in the
source file:
\begin{verbatim}
\documentclass[sigconf]{acmart}
\end{verbatim}
\section{Modifications}
Modifying the template --- including but not limited to: adjusting
margins, typeface sizes, line spacing, paragraph and list definitions,
and the use of the \verb|\vspace| command to manually adjust the
vertical spacing between elements of your work --- is not allowed.
{\bfseries Your document will be returned to you for revision if
modifications are discovered.}
\section{Typefaces}
The ``\verb|acmart|'' document class requires the use of the
``Libertine'' typeface family. Your \TeX\ installation should include
this set of packages. Please do not substitute other typefaces. The
``\verb|lmodern|'' and ``\verb|ltimes|'' packages should not be used,
as they will override the built-in typeface families.
\section{Title Information}
The title of your work should use capital letters appropriately -
\url{https://capitalizemytitle.com/} has useful rules for
capitalization. Use the {\verb|title|} command to define the title of
your work. If your work has a subtitle, define it with the
{\verb|subtitle|} command. Do not insert line breaks in your title.
If your title is lengthy, you must define a short version to be used
in the page headers, to prevent overlapping text. The \verb|title|
command has a ``short title'' parameter:
\begin{verbatim}
\title[short title]{full title}
\end{verbatim}
\section{Authors and Affiliations}
Each author must be defined separately for accurate metadata
identification. Multiple authors may share one affiliation. Authors'
names should not be abbreviated; use full first names wherever
possible. Include authors' e-mail addresses whenever possible.
Grouping authors' names or e-mail addresses, or providing an ``e-mail
alias,'' as shown below, is not acceptable:
\begin{verbatim}
\author{Brooke Aster, David Mehldau}
\email{dave,judy,steve@university.edu}
\email{firstname.lastname@phillips.org}
\end{verbatim}
The \verb|authornote| and \verb|authornotemark| commands allow a note
to apply to multiple authors --- for example, if the first two authors
of an article contributed equally to the work.
If your author list is lengthy, you must define a shortened version of
the list of authors to be used in the page headers, to prevent
overlapping text. The following command should be placed just after
the last \verb|\author{}| definition:
\begin{verbatim}
\renewcommand{\shortauthors}{McCartney, et al.}
\end{verbatim}
Omitting this command will force the use of a concatenated list of all
of the authors' names, which may result in overlapping text in the
page headers.
The article template's documentation, available at
\url{https://www.acm.org/publications/proceedings-template}, has a
complete explanation of these commands and tips for their effective
use.
Note that authors' addresses are mandatory for journal articles.
\section{Rights Information}
Authors of any work published by ACM will need to complete a rights
form. Depending on the kind of work, and the rights management choice
made by the author, this may be copyright transfer, permission,
license, or an OA (open access) agreement.
Regardless of the rights management choice, the author will receive a
copy of the completed rights form once it has been submitted. This
form contains \LaTeX\ commands that must be copied into the source
document. When the document source is compiled, these commands and
their parameters add formatted text to several areas of the final
document:
\begin{itemize}
\item the ``ACM Reference Format'' text on the first page.
\item the ``rights management'' text on the first page.
\item the conference information in the page header(s).
\end{itemize}
Rights information is unique to the work; if you are preparing several
works for an event, make sure to use the correct set of commands with
each of the works.
The ACM Reference Format text is required for all articles over one
page in length, and is optional for one-page articles (abstracts).
\section{CCS Concepts and User-Defined Keywords}
Two elements of the ``acmart'' document class provide powerful
taxonomic tools for you to help readers find your work in an online
search.
The ACM Computing Classification System ---
\url{https://www.acm.org/publications/class-2012} --- is a set of
classifiers and concepts that describe the computing
discipline. Authors can select entries from this classification
system, via \url{https://dl.acm.org/ccs/ccs.cfm}, and generate the
commands to be included in the \LaTeX\ source.
User-defined keywords are a comma-separated list of words and phrases
of the authors' choosing, providing a more flexible way of describing
the research being presented.
CCS concepts and user-defined keywords are required for for all
articles over two pages in length, and are optional for one- and
two-page articles (or abstracts).
\section{Sectioning Commands}
Your work should use standard \LaTeX\ sectioning commands:
\verb|section|, \verb|subsection|, \verb|subsubsection|, and
\verb|paragraph|. They should be numbered; do not remove the numbering
from the commands.
Simulating a sectioning command by setting the first word or words of
a paragraph in boldface or italicized text is {\bfseries not allowed.}
\section{Tables}
The ``\verb|acmart|'' document class includes the ``\verb|booktabs|''
package --- \url{https://ctan.org/pkg/booktabs} --- for preparing
high-quality tables.
Table captions are placed {\itshape above} the table.
Because tables cannot be split across pages, the best placement for
them is typically the top of the page nearest their initial cite. To
ensure this proper ``floating'' placement of tables, use the
environment \textbf{table} to enclose the table's contents and the
table caption. The contents of the table itself must go in the
\textbf{tabular} environment, to be aligned properly in rows and
columns, with the desired horizontal and vertical rules. Again,
detailed instructions on \textbf{tabular} material are found in the
\textit{\LaTeX\ User's Guide}.
Immediately following this sentence is the point at which
Table~\ref{tab:freq} is included in the input file; compare the
placement of the table here with the table in the printed output of
this document.
\begin{table}
\caption{Frequency of Special Characters}
\label{tab:freq}
\begin{tabular}{ccl}
\toprule
Non-English or Math&Frequency&Comments\\
\midrule
\O & 1 in 1,000& For Swedish names\\
$\pi$ & 1 in 5& Common in math\\
\$ & 4 in 5 & Used in business\\
$\Psi^2_1$ & 1 in 40,000& Unexplained usage\\
\bottomrule
\end{tabular}
\end{table}
To set a wider table, which takes up the whole width of the page's
live area, use the environment \textbf{table*} to enclose the table's
contents and the table caption. As with a single-column table, this
wide table will ``float'' to a location deemed more
desirable. Immediately following this sentence is the point at which
Table~\ref{tab:commands} is included in the input file; again, it is
instructive to compare the placement of the table here with the table
in the printed output of this document.
\begin{table*}
\caption{Some Typical Commands}
\label{tab:commands}
\begin{tabular}{ccl}
\toprule
Command &A Number & Comments\\
\midrule
\texttt{{\char'134}author} & 100& Author \\
\texttt{{\char'134}table}& 300 & For tables\\
\texttt{{\char'134}table*}& 400& For wider tables\\
\bottomrule
\end{tabular}
\end{table*}
Always use midrule to separate table header rows from data rows, and
use it only for this purpose. This enables assistive technologies to
recognise table headers and support their users in navigating tables
more easily.
\section{Math Equations}
You may want to display math equations in three distinct styles:
inline, numbered or non-numbered display. Each of the three are
discussed in the next sections.
\subsection{Inline (In-text) Equations}
A formula that appears in the running text is called an inline or
in-text formula. It is produced by the \textbf{math} environment,
which can be invoked with the usual
\texttt{{\char'134}begin\,\ldots{\char'134}end} construction or with
the short form \texttt{\$\,\ldots\$}. You can use any of the symbols
and structures, from $\alpha$ to $\omega$, available in
\LaTeX~\cite{Lamport:LaTeX}; this section will simply show a few
examples of in-text equations in context. Notice how this equation:
\begin{math}
\lim_{n\rightarrow \infty}x=0
\end{math},
set here in in-line math style, looks slightly different when
set in display style. (See next section).
\subsection{Display Equations}
A numbered display equation---one set off by vertical space from the
text and centered horizontally---is produced by the \textbf{equation}
environment. An unnumbered display equation is produced by the
\textbf{displaymath} environment.
Again, in either environment, you can use any of the symbols and
structures available in \LaTeX\@; this section will just give a couple
of examples of display equations in context. First, consider the
equation, shown as an inline equation above:
\begin{equation}
\lim_{n\rightarrow \infty}x=0
\end{equation}
Notice how it is formatted somewhat differently in
the \textbf{displaymath}
environment. Now, we'll enter an unnumbered equation:
\begin{displaymath}
\sum_{i=0}^{\infty} x + 1
\end{displaymath}
and follow it with another numbered equation:
\begin{equation}
\sum_{i=0}^{\infty}x_i=\int_{0}^{\pi+2} f
\end{equation}
just to demonstrate \LaTeX's able handling of numbering.
\section{Figures}
The ``\verb|figure|'' environment should be used for figures. One or
more images can be placed within a figure. If your figure contains
third-party material, you must clearly identify it as such, as shown
in the example below.
\begin{figure}[h]
\centering
\includegraphics[width=\linewidth]{sample-franklin}
\caption{1907 Franklin Model D roadster. Photograph by Harris \&
Ewing, Inc. [Public domain], via Wikimedia
Commons. (\url{https://goo.gl/VLCRBB}).}
\Description{A woman and a girl in white dresses sit in an open car.}
\end{figure}
Your figures should contain a caption which describes the figure to
the reader.
Figure captions are placed {\itshape below} the figure.
Every figure should also have a figure description unless it is purely
decorative. These descriptions convey what’s in the image to someone
who cannot see it. They are also used by search engine crawlers for
indexing images, and when images cannot be loaded.
A figure description must be unformatted plain text less than 2000
characters long (including spaces). {\bfseries Figure descriptions
should not repeat the figure caption – their purpose is to capture
important information that is not already provided in the caption or
the main text of the paper.} For figures that convey important and
complex new information, a short text description may not be
adequate. More complex alternative descriptions can be placed in an
appendix and referenced in a short figure description. For example,
provide a data table capturing the information in a bar chart, or a
structured list representing a graph. For additional information
regarding how best to write figure descriptions and why doing this is
so important, please see
\url{https://www.acm.org/publications/taps/describing-figures/}.
\subsection{The ``Teaser Figure''}
A ``teaser figure'' is an image, or set of images in one figure, that
are placed after all author and affiliation information, and before
the body of the article, spanning the page. If you wish to have such a
figure in your article, place the command immediately before the
\verb|\maketitle| command:
\begin{verbatim}
\begin{teaserfigure}
\includegraphics[width=\textwidth]{sampleteaser}
\caption{figure caption}
\Description{figure description}
\end{teaserfigure}
\end{verbatim}
\section{Citations and Bibliographies}
The use of \BibTeX\ for the preparation and formatting of one's
references is strongly recommended. Authors' names should be complete
--- use full first names (``Donald E. Knuth'') not initials
(``D. E. Knuth'') --- and the salient identifying features of a
reference should be included: title, year, volume, number, pages,
article DOI, etc.
The bibliography is included in your source document with these two
commands, placed just before the \verb|\end{document}| command:
\begin{verbatim}
\bibliographystyle{ACM-Reference-Format}
\bibliography{bibfile}
\end{verbatim}
where ``\verb|bibfile|'' is the name, without the ``\verb|.bib|''
suffix, of the \BibTeX\ file.
Citations and references are numbered by default. A small number of
ACM publications have citations and references formatted in the
``author year'' style; for these exceptions, please include this
command in the {\bfseries preamble} (before the command
``\verb|\begin{document}|'') of your \LaTeX\ source:
\begin{verbatim}
\citestyle{acmauthoryear}
\end{verbatim}
Some examples. A paginated journal article \cite{Abril07}, an
enumerated journal article \cite{Cohen07}, a reference to an entire
issue \cite{JCohen96}, a monograph (whole book) \cite{Kosiur01}, a
monograph/whole book in a series (see 2a in spec. document)
\cite{Harel79}, a divisible-book such as an anthology or compilation
\cite{Editor00} followed by the same example, however we only output
the series if the volume number is given \cite{Editor00a} (so
Editor00a's series should NOT be present since it has no vol. no.),
a chapter in a divisible book \cite{Spector90}, a chapter in a
divisible book in a series \cite{Douglass98}, a multi-volume work as
book \cite{Knuth97}, a couple of articles in a proceedings (of a
conference, symposium, workshop for example) (paginated proceedings
article) \cite{Andler79, Hagerup1993}, a proceedings article with
all possible elements \cite{Smith10}, an example of an enumerated
proceedings article \cite{VanGundy07}, an informally published work
\cite{Harel78}, a couple of preprints \cite{Bornmann2019,
AnzarootPBM14}, a doctoral dissertation \cite{Clarkson85}, a
master's thesis: \cite{anisi03}, an online document / world wide web
resource \cite{Thornburg01, Ablamowicz07, Poker06}, a video game
(Case 1) \cite{Obama08} and (Case 2) \cite{Novak03} and \cite{Lee05}
and (Case 3) a patent \cite{JoeScientist001}, work accepted for
publication \cite{rous08}, 'YYYYb'-test for prolific author
\cite{SaeediMEJ10} and \cite{SaeediJETC10}. Other cites might
contain 'duplicate' DOI and URLs (some SIAM articles)
\cite{Kirschmer:2010:AEI:1958016.1958018}. Boris / Barbara Beeton:
multi-volume works as books \cite{MR781536} and \cite{MR781537}. A
couple of citations with DOIs:
\cite{2004:ITE:1009386.1010128,Kirschmer:2010:AEI:1958016.1958018}. Online
citations: \cite{TUGInstmem, Thornburg01, CTANacmart}. Artifacts:
\cite{R} and \cite{UMassCitations}.
\section{Acknowledgments}
Identification of funding sources and other support, and thanks to
individuals and groups that assisted in the research and the
preparation of the work should be included in an acknowledgment
section, which is placed just before the reference section in your
document.
This section has a special environment:
\begin{verbatim}
\begin{acks}
...
\end{acks}
\end{verbatim}
so that the information contained therein can be more easily collected
during the article metadata extraction phase, and to ensure
consistency in the spelling of the section heading.
Authors should not prepare this section as a numbered or unnumbered {\verb|\section|}; please use the ``{\verb|acks|}'' environment.
\section{Appendices}
If your work needs an appendix, add it before the
``\verb|\end{document}|'' command at the conclusion of your source
document.
Start the appendix with the ``\verb|appendix|'' command:
\begin{verbatim}
\appendix
\end{verbatim}
and note that in the appendix, sections are lettered, not
numbered. This document has two appendices, demonstrating the section
and subsection identification method.
\section{SIGCHI Extended Abstracts}
The ``\verb|sigchi-a|'' template style (available only in \LaTeX\ and
not in Word) produces a landscape-orientation formatted article, with
a wide left margin. Three environments are available for use with the
``\verb|sigchi-a|'' template style, and produce formatted output in
the margin:
\begin{itemize}
\item {\verb|sidebar|}: Place formatted text in the margin.
\item {\verb|marginfigure|}: Place a figure in the margin.
\item {\verb|margintable|}: Place a table in the margin.
\end{itemize}
%%
%% The acknowledgments section is defined using the "acks" environment
%% (and NOT an unnumbered section). This ensures the proper
%% identification of the section in the article metadata, and the
%% consistent spelling of the heading.
\begin{acks}
To Robert, for the bagels and explaining CMYK and color spaces.
\end{acks}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
%%
%% If your work has an appendix, this is the place to put it.
\appendix
\section{Research Methods}
\subsection{Part One}
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi
malesuada, quam in pulvinar varius, metus nunc fermentum urna, id
sollicitudin purus odio sit amet enim. Aliquam ullamcorper eu ipsum
vel mollis. Curabitur quis dictum nisl. Phasellus vel semper risus, et
lacinia dolor. Integer ultricies commodo sem nec semper.
\subsection{Part Two}
Etiam commodo feugiat nisl pulvinar pellentesque. Etiam auctor sodales
ligula, non varius nibh pulvinar semper. Suspendisse nec lectus non
ipsum convallis congue hendrerit vitae sapien. Donec at laoreet
eros. Vivamus non purus placerat, scelerisque diam eu, cursus
ante. Etiam aliquam tortor auctor efficitur mattis.
\section{Online Resources}
Nam id fermentum dui. Suspendisse sagittis tortor a nulla mollis, in
pulvinar ex pretium. Sed interdum orci quis metus euismod, et sagittis
enim maximus. Vestibulum gravida massa ut felis suscipit
congue. Quisque mattis elit a risus ultrices commodo venenatis eget
dui. Etiam sagittis eleifend elementum.
Nam interdum magna at lectus dignissim, ac dignissim lorem
rhoncus. Maecenas eu arcu ac neque placerat aliquam. Nunc pulvinar
massa et mattis lacinia.
\end{document}
\endinput
%%
%% End of file `sample-xelatex.tex'.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{31}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[\protect\citeauthoryear{Benikova, Biemann, and Reznicek}{Benikova
et~al\mbox{.}}{2014}]%
{benikova2014nosta}
\bibfield{author}{\bibinfo{person}{Darina Benikova}, \bibinfo{person}{Chris
Biemann}, {and} \bibinfo{person}{Marc Reznicek}.}
\bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{NoSta-D Named Entity Annotation for German:
Guidelines and Dataset.}. In \bibinfo{booktitle}{\emph{LREC}}.
\bibinfo{pages}{2524--2531}.
\newblock
\bibitem[\protect\citeauthoryear{Berthold, Borgelt, H{\"o}ppner, and
Klawonn}{Berthold et~al\mbox{.}}{2010}]%
{berthold2010guide}
\bibfield{author}{\bibinfo{person}{Michael~R Berthold},
\bibinfo{person}{Christian Borgelt}, \bibinfo{person}{Frank H{\"o}ppner},
{and} \bibinfo{person}{Frank Klawonn}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Guide to intelligent data analysis: how to
intelligently make sense of real data}}.
\newblock \bibinfo{publisher}{Springer Science \& Business Media}.
\newblock
\bibitem[\protect\citeauthoryear{Cambria, Poria, Bajpai, and Schuller}{Cambria
et~al\mbox{.}}{2016}]%
{cambria2016senticnet}
\bibfield{author}{\bibinfo{person}{Erik Cambria}, \bibinfo{person}{Soujanya
Poria}, \bibinfo{person}{Rajiv Bajpai}, {and} \bibinfo{person}{Bj{\"o}rn
Schuller}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{SenticNet 4: A semantic resource for sentiment
analysis based on conceptual primitives}. In
\bibinfo{booktitle}{\emph{Proceedings of COLING 2016, the 26th international
conference on computational linguistics: Technical papers}}.
\bibinfo{pages}{2666--2677}.
\newblock
\bibitem[\protect\citeauthoryear{Chang, Caplinger, Marin, Shen, and
Demberg}{Chang et~al\mbox{.}}{2020}]%
{chang-etal-2020-dart}
\bibfield{author}{\bibinfo{person}{Ernie Chang}, \bibinfo{person}{Jeriah
Caplinger}, \bibinfo{person}{Alex Marin}, \bibinfo{person}{Xiaoyu Shen},
{and} \bibinfo{person}{Vera Demberg}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{{DART}: A Lightweight Quality-Suggestive
Data-to-Text Annotation Tool}. In \bibinfo{booktitle}{\emph{Proceedings of
the 28th International Conference on Computational Linguistics: System
Demonstrations}}. \bibinfo{publisher}{International Committee on
Computational Linguistics (ICCL)}, \bibinfo{address}{Barcelona, Spain
(Online)}, \bibinfo{pages}{12--17}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.18653/v1/2020.coling-demos.3}
\showDOI{\tempurl}
\bibitem[\protect\citeauthoryear{Eltyeb and Salim}{Eltyeb and Salim}{2014}]%
{eltyeb2014chemical}
\bibfield{author}{\bibinfo{person}{Safaa Eltyeb} {and} \bibinfo{person}{Naomie
Salim}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Chemical named entities recognition: a review on
approaches and applications}.
\newblock \bibinfo{journal}{\emph{Journal of cheminformatics}}
\bibinfo{volume}{6}, \bibinfo{number}{1} (\bibinfo{year}{2014}),
\bibinfo{pages}{17}.
\newblock
\bibitem[\protect\citeauthoryear{Etzioni, Cafarella, Downey, Popescu, Shaked,
Soderland, Weld, and Yates}{Etzioni et~al\mbox{.}}{2005}]%
{etzioni2005unsupervised}
\bibfield{author}{\bibinfo{person}{Oren Etzioni}, \bibinfo{person}{Michael
Cafarella}, \bibinfo{person}{Doug Downey}, \bibinfo{person}{Ana-Maria
Popescu}, \bibinfo{person}{Tal Shaked}, \bibinfo{person}{Stephen Soderland},
\bibinfo{person}{Daniel~S Weld}, {and} \bibinfo{person}{Alexander Yates}.}
\bibinfo{year}{2005}\natexlab{}.
\newblock \showarticletitle{Unsupervised named-entity extraction from the web:
An experimental study}.
\newblock \bibinfo{journal}{\emph{Artificial intelligence}}
\bibinfo{volume}{165}, \bibinfo{number}{1} (\bibinfo{year}{2005}),
\bibinfo{pages}{91--134}.
\newblock
\bibitem[\protect\citeauthoryear{Foley, Sarwar, and Allan}{Foley
et~al\mbox{.}}{2018}]%
{foley2018named}
\bibfield{author}{\bibinfo{person}{John Foley},
\bibinfo{person}{Sheikh~Muhammad Sarwar}, {and} \bibinfo{person}{James
Allan}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Named Entity Recognition with Extremely Limited
Data}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1806.04411}}
(\bibinfo{year}{2018}).
\newblock
\bibitem[\protect\citeauthoryear{Grave, Bojanowski, Gupta, Joulin, and
Mikolov}{Grave et~al\mbox{.}}{2018}]%
{grave2018learning}
\bibfield{author}{\bibinfo{person}{{\'E}douard Grave}, \bibinfo{person}{Piotr
Bojanowski}, \bibinfo{person}{Prakhar Gupta}, \bibinfo{person}{Armand
Joulin}, {and} \bibinfo{person}{Tom{\'a}{\v{s}} Mikolov}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Learning Word Vectors for 157 Languages}. In
\bibinfo{booktitle}{\emph{Proceedings of the Eleventh International
Conference on Language Resources and Evaluation (LREC 2018)}}.
\newblock
\bibitem[\protect\citeauthoryear{Heuss, Humm, Henninger, and Rippl}{Heuss
et~al\mbox{.}}{2014}]%
{heuss2014comparison}
\bibfield{author}{\bibinfo{person}{Timm Heuss}, \bibinfo{person}{Bernhard
Humm}, \bibinfo{person}{Christian Henninger}, {and} \bibinfo{person}{Thomas
Rippl}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{A comparison of NER tools wrt a domain-specific
vocabulary}. In \bibinfo{booktitle}{\emph{Proceedings of the 10th
International Conference on Semantic Systems}}. \bibinfo{pages}{100--107}.
\newblock
\bibitem[\protect\citeauthoryear{Honnibal and Montani}{Honnibal and
Montani}{2017}]%
{spacy2}
\bibfield{author}{\bibinfo{person}{Matthew Honnibal} {and}
\bibinfo{person}{Ines Montani}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \bibinfo{title}{{spaCy 2}: Natural language understanding with
{B}loom embeddings, convolutional neural networks and incremental parsing}.
(\bibinfo{year}{2017}).
\newblock
\newblock
\shownote{To appear}.
\bibitem[\protect\citeauthoryear{Loster, Naumann, Ehmueller, and
Feldmann}{Loster et~al\mbox{.}}{2018}]%
{loster2018curex}
\bibfield{author}{\bibinfo{person}{Michael Loster}, \bibinfo{person}{Felix
Naumann}, \bibinfo{person}{Jan Ehmueller}, {and} \bibinfo{person}{Benjamin
Feldmann}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Curex: a system for extracting, curating, and
exploring domain-specific knowledge graphs from text}. In
\bibinfo{booktitle}{\emph{Proceedings of the 27th ACM International
Conference on Information and Knowledge Management}}.
\bibinfo{pages}{1883--1886}.
\newblock
\bibitem[\protect\citeauthoryear{Loster, Zuo, Naumann, Maspfuhl, and
Thomas}{Loster et~al\mbox{.}}{2017}]%
{loster2017improving}
\bibfield{author}{\bibinfo{person}{Michael Loster}, \bibinfo{person}{Zhe Zuo},
\bibinfo{person}{Felix Naumann}, \bibinfo{person}{Oliver Maspfuhl}, {and}
\bibinfo{person}{Dirk Thomas}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Improving Company Recognition from Unstructured
Text by using Dictionaries.}. In \bibinfo{booktitle}{\emph{EDBT}}.
\bibinfo{pages}{610--619}.
\newblock
\bibitem[\protect\citeauthoryear{Mansouri, Affendey, and Mamat}{Mansouri
et~al\mbox{.}}{2008}]%
{mansouri2008named}
\bibfield{author}{\bibinfo{person}{Alireza Mansouri},
\bibinfo{person}{Lilly~Suriani Affendey}, {and} \bibinfo{person}{Ali Mamat}.}
\bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Named entity recognition approaches}.
\newblock \bibinfo{journal}{\emph{International Journal of Computer Science and
Network Security}} \bibinfo{volume}{8}, \bibinfo{number}{2}
(\bibinfo{year}{2008}), \bibinfo{pages}{339--344}.
\newblock
\bibitem[\protect\citeauthoryear{Meyer and Gurevych}{Meyer and
Gurevych}{2011}]%
{meyer2011psycholinguists}
\bibfield{author}{\bibinfo{person}{Christian~M Meyer} {and}
\bibinfo{person}{Iryna Gurevych}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{What psycholinguists know about chemistry: Aligning
Wiktionary and WordNet for increased domain coverage}. In
\bibinfo{booktitle}{\emph{Proceedings of 5th International Joint Conference
on Natural Language Processing}}. \bibinfo{pages}{883--892}.
\newblock
\bibitem[\protect\citeauthoryear{Murtagh and Contreras}{Murtagh and
Contreras}{2012}]%
{murtagh2012algorithms}
\bibfield{author}{\bibinfo{person}{Fionn Murtagh} {and} \bibinfo{person}{Pedro
Contreras}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Algorithms for hierarchical clustering: an
overview}.
\newblock \bibinfo{journal}{\emph{Wiley Interdisciplinary Reviews: Data Mining
and Knowledge Discovery}} \bibinfo{volume}{2}, \bibinfo{number}{1}
(\bibinfo{year}{2012}), \bibinfo{pages}{86--97}.
\newblock
\bibitem[\protect\citeauthoryear{Nadeau, Turney, and Matwin}{Nadeau
et~al\mbox{.}}{2006}]%
{nadeau2006unsupervised}
\bibfield{author}{\bibinfo{person}{David Nadeau}, \bibinfo{person}{Peter~D
Turney}, {and} \bibinfo{person}{Stan Matwin}.}
\bibinfo{year}{2006}\natexlab{}.
\newblock \showarticletitle{Unsupervised named-entity recognition: Generating
gazetteers and resolving ambiguity}. In \bibinfo{booktitle}{\emph{Conference
of the Canadian society for computational studies of intelligence}}.
Springer, \bibinfo{pages}{266--277}.
\newblock
\bibitem[\protect\citeauthoryear{Richman and Schone}{Richman and
Schone}{2008}]%
{richman2008mining}
\bibfield{author}{\bibinfo{person}{Alexander~E Richman} {and}
\bibinfo{person}{Patrick Schone}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Mining wiki resources for multilingual named entity
recognition}. In \bibinfo{booktitle}{\emph{Proceedings of ACL-08: HLT}}.
\bibinfo{pages}{1--9}.
\newblock
\bibitem[\protect\citeauthoryear{Rockt{\"a}schel, Weidlich, and
Leser}{Rockt{\"a}schel et~al\mbox{.}}{2012}]%
{rocktaschel2012chemspot}
\bibfield{author}{\bibinfo{person}{Tim Rockt{\"a}schel},
\bibinfo{person}{Michael Weidlich}, {and} \bibinfo{person}{Ulf Leser}.}
\bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{ChemSpot: a hybrid system for chemical named entity
recognition}.
\newblock \bibinfo{journal}{\emph{Bioinformatics}} \bibinfo{volume}{28},
\bibinfo{number}{12} (\bibinfo{year}{2012}), \bibinfo{pages}{1633--1640}.
\newblock
\bibitem[\protect\citeauthoryear{Ruppenhofer, Rehbein, and Flinz}{Ruppenhofer
et~al\mbox{.}}{2020}]%
{ruppenhoferfine2020}
\bibfield{author}{\bibinfo{person}{Josef Ruppenhofer}, \bibinfo{person}{Ines
Rehbein}, {and} \bibinfo{person}{Carolina Flinz}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Fine-grained Named Entity Annotations for German
Biographic Interviews}.
\newblock (\bibinfo{year}{2020}).
\newblock
\bibitem[\protect\citeauthoryear{Salehi, Cook, and Baldwin}{Salehi
et~al\mbox{.}}{2014}]%
{salehi2014detecting}
\bibfield{author}{\bibinfo{person}{Bahar Salehi}, \bibinfo{person}{Paul Cook},
{and} \bibinfo{person}{Timothy Baldwin}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Detecting non-compositional mwe components using
wiktionary}. In \bibinfo{booktitle}{\emph{Proceedings of the 2014 Conference
on Empirical Methods in Natural Language Processing (EMNLP)}}.
\bibinfo{pages}{1792--1797}.
\newblock
\bibitem[\protect\citeauthoryear{Sang and De~Meulder}{Sang and
De~Meulder}{2003}]%
{sang2003introduction}
\bibfield{author}{\bibinfo{person}{Erik Tjong~Kim Sang} {and}
\bibinfo{person}{Fien De~Meulder}.} \bibinfo{year}{2003}\natexlab{}.
\newblock \showarticletitle{Introduction to the CoNLL-2003 Shared Task:
Language-Independent Named Entity Recognition}. In
\bibinfo{booktitle}{\emph{Proceedings of the Seventh Conference on Natural
Language Learning at HLT-NAACL 2003}}. \bibinfo{pages}{142--147}.
\newblock
\bibitem[\protect\citeauthoryear{Schiersch, Mironova, Schmitt, Thomas,
Gabryszak, and Hennig}{Schiersch et~al\mbox{.}}{2018}]%
{schiersch2018german}
\bibfield{author}{\bibinfo{person}{Martin Schiersch}, \bibinfo{person}{Veselina
Mironova}, \bibinfo{person}{Maximilian Schmitt}, \bibinfo{person}{Philippe
Thomas}, \bibinfo{person}{Aleksandra Gabryszak}, {and}
\bibinfo{person}{Leonhard Hennig}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{A German Corpus for Fine-Grained Named Entity
Recognition and Relation Extraction of Traffic and Industry Events}. In
\bibinfo{booktitle}{\emph{Proceedings of the Eleventh International
Conference on Language Resources and Evaluation (LREC 2018)}}.
\newblock
\bibitem[\protect\citeauthoryear{Shang, Liu, Gu, Ren, Ren, and Han}{Shang
et~al\mbox{.}}{2018}]%
{shang2018learning}
\bibfield{author}{\bibinfo{person}{Jingbo Shang}, \bibinfo{person}{Liyuan Liu},
\bibinfo{person}{Xiaotao Gu}, \bibinfo{person}{Xiang Ren},
\bibinfo{person}{Teng Ren}, {and} \bibinfo{person}{Jiawei Han}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Learning Named Entity Tagger using Domain-Specific
Dictionary}. In \bibinfo{booktitle}{\emph{Proceedings of the 2018 Conference
on Empirical Methods in Natural Language Processing}}.
\bibinfo{pages}{2054--2064}.
\newblock
\bibitem[\protect\citeauthoryear{Simon and Nemeskey}{Simon and
Nemeskey}{2012}]%
{simon2012automatically}
\bibfield{author}{\bibinfo{person}{Eszter Simon} {and}
\bibinfo{person}{D{\'a}vid~M{\'a}rk Nemeskey}.}
\bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Automatically generated NE tagged corpora for
English and Hungarian}. Association for Computational Linguistics.
\newblock
\bibitem[\protect\citeauthoryear{Tomori, Ninomiya, and Mori}{Tomori
et~al\mbox{.}}{2016}]%
{tomori2016domain}
\bibfield{author}{\bibinfo{person}{Suzushi Tomori}, \bibinfo{person}{Takashi
Ninomiya}, {and} \bibinfo{person}{Shinsuke Mori}.}
\bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Domain specific named entity recognition referring
to the real world by deep neural networks}. In
\bibinfo{booktitle}{\emph{Proceedings of the 54th Annual Meeting of the
Association for Computational Linguistics (Volume 2: Short Papers)}}.
\bibinfo{pages}{236--242}.
\newblock
\bibitem[\protect\citeauthoryear{Tuggener}{Tuggener}{2016}]%
{Tuggener2016}
\bibfield{author}{\bibinfo{person}{Don Tuggener}.}
\bibinfo{year}{2016}\natexlab{}.
\newblock \emph{\bibinfo{title}{Incremental Coreference Resolution for
German}}.
\newblock \bibinfo{thesistype}{Ph.\,D. Dissertation}.
\newblock
\bibitem[\protect\citeauthoryear{Weber and Buitelaar}{Weber and
Buitelaar}{2006}]%
{weber2006web}
\bibfield{author}{\bibinfo{person}{Nicolas Weber} {and} \bibinfo{person}{Paul
Buitelaar}.} \bibinfo{year}{2006}\natexlab{}.
\newblock \showarticletitle{Web-based ontology learning with isolde}. In
\bibinfo{booktitle}{\emph{Proc. of the ISWC Workshop on Web Content Mining
with Human Language Technologies}}. Citeseer.
\newblock
\bibitem[\protect\citeauthoryear{Yadav and Bethard}{Yadav and Bethard}{2018}]%
{yadav2018survey}
\bibfield{author}{\bibinfo{person}{Vikas Yadav} {and} \bibinfo{person}{Steven
Bethard}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{A Survey on Recent Advances in Named Entity
Recognition from Deep Learning models}. In
\bibinfo{booktitle}{\emph{Proceedings of the 27th International Conference on
Computational Linguistics}}. \bibinfo{pages}{2145--2158}.
\newblock
\bibitem[\protect\citeauthoryear{Zesch, M{\"u}ller, and Gurevych}{Zesch
et~al\mbox{.}}{2008}]%
{zesch2008extracting}
\bibfield{author}{\bibinfo{person}{Torsten Zesch}, \bibinfo{person}{Christof
M{\"u}ller}, {and} \bibinfo{person}{Iryna Gurevych}.}
\bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Extracting Lexical Semantic Knowledge from
Wikipedia and Wiktionary.}. In \bibinfo{booktitle}{\emph{LREC}},
Vol.~\bibinfo{volume}{8}. \bibinfo{pages}{1646--1652}.
\newblock
\bibitem[\protect\citeauthoryear{Zhang and Elhadad}{Zhang and Elhadad}{2013}]%
{zhang2013unsupervised}
\bibfield{author}{\bibinfo{person}{Shaodian Zhang} {and}
\bibinfo{person}{No{\'e}mie Elhadad}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{Unsupervised biomedical named entity recognition:
Experiments with clinical and biological texts}.
\newblock \bibinfo{journal}{\emph{Journal of biomedical informatics}}
\bibinfo{volume}{46}, \bibinfo{number}{6} (\bibinfo{year}{2013}),
\bibinfo{pages}{1088--1098}.
\newblock
\bibitem[\protect\citeauthoryear{Zhou, Li, and Chen}{Zhou
et~al\mbox{.}}{2015}]%
{zhou2015automatically}
\bibfield{author}{\bibinfo{person}{Jie Zhou}, \bibinfo{person}{Bi-cheng Li},
{and} \bibinfo{person}{Gang Chen}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{Automatically building large-scale named entity
recognition corpora from Chinese Wikipedia}.
\newblock \bibinfo{journal}{\emph{Frontiers of Information Technology \&
Electronic Engineering}} \bibinfo{volume}{16}, \bibinfo{number}{11}
(\bibinfo{year}{2015}), \bibinfo{pages}{940--956}.
\newblock
\end{thebibliography}