Where were my keys? -- Aggregating Spatial-Temporal Instances of Objects for Efficient Retrieval over Long Periods of Time
Extracted source text · arXiv 2110.13061
\def\year{2021}\relax
%File: formatting-instructions-latex-2021.tex
%release 2021.2
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai21} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%\nocopyright
%PDF Info Is REQUIRED.
% For /Author, add all authors within the parentheses, separated by commas. No accents or commands.
% For /Title, add Title in Mixed Case. No accents or commands. Retain the parentheses.
\pdfinfo{
/Title (AAAI Press Formatting Instructions for Authors Using LaTeX -- A Guide)
/Author (AAAI Press Staff, Pater Patel Schneider, Sunil Issar, J. Scott Penberthy, George Ferguson, Hans Guesgen, Francisco Cruz, Marc Pujol-Gonzalez)
/TemplateVersion (2021.2)
} %Leave this
% /Title ()
% Put your actual complete title (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case
% Leave the space between \Title and the beginning parenthesis alone
% /Author ()
% Put your actual complete list of authors (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case.
% Each author should be only by a comma. If the name contains accents, remove them. If there are any LaTeX commands,
% remove them.
% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
% The file aaai21.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%
% Title
% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Where were my keys? - Aggregating Spatial-Temporal Instances of Objects for Efficient Retrieval over Long Periods of Time}
\author{
%Authors
% All authors must be in the same font size and format.
% Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
% AAAI Style Contributions by Pater Patel Schneider,
% Sunil Issar, \\
% J. Scott Penberthy,
% George Ferguson,
% Hans Guesgen,
% Francisco Cruz,
% Marc Pujol-Gonzalez
% \\
Ifrah Idrees, Zahid Hasan, Steven P. Reiss, and Stefanie Tellex\\
}
\affiliations{
%Afiliations
Dept. of Computer Science, Brown University, Providence, RI\\
% \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
%If you have multiple authors and multiple affiliations
% use superscripts in text and roman font to identify them.
%For example,
% Sunil Issar, \textsuperscript{\rm 2}
% J. Scott Penberthy, \textsuperscript{\rm 3}
% George Ferguson,\textsuperscript{\rm 4}
% Hans Guesgen, \textsuperscript{\rm 5}.
% Note that the comma should be placed BEFORE the superscript for optimum readability
% 2275 East Bayshore Road, Suite 160\\
% Palo Alto, California 94303\\
% email address must be in roman text type, not monospace or sans serif
% publications21@aaai.org
ifrah
% See more examples next
}
\iffalse
%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Single Author}
\author {
% Author
Author Name \\
}
\affiliations{
Affiliation \\
Affiliation Line 2 \\
name@example.com
}
\fi
\iffalse
%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Multiple Authors}
\author {
% Authors
First Author Name,\textsuperscript{\rm 1}
Second Author Name, \textsuperscript{\rm 2}
Third Author Name \textsuperscript{\rm 1} \\
}
\affiliations {
% Affiliations
\textsuperscript{\rm 1} Affiliation 1 \\
\textsuperscript{\rm 2} Affiliation 2 \\
firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
}
\fi
\begin{document}
\maketitle
\begin{abstract}
AAAI creates proceedings, working notes, and technical reports directly from electronic source furnished by the authors. To ensure that all papers in the publication have a uniform appearance, authors must adhere to the following instructions.
\end{abstract}
\noindent Congratulations on having a paper selected for inclusion in an AAAI Press proceedings or technical report! This document details the requirements necessary to get your accepted paper published using PDF\LaTeX{}. If you are using Microsoft Word, instructions are provided in a different document. AAAI Press does not support any other formatting software.
The instructions herein are provided as a general guide for experienced \LaTeX{} users. If you do not know how to use \LaTeX{}, please obtain assistance locally. AAAI cannot provide you with support and the accompanying style files are \textbf{not} guaranteed to work. If the results you obtain are not in accordance with the specifications you received, you must correct your source file to achieve the correct result.
These instructions are generic. Consequently, they do not include specific dates, page charges, and so forth. Please consult your specific written conference instructions for details regarding your submission. Please review the entire document for specific instructions that might apply to your particular situation. All authors must comply with the following:
\begin{itemize}
\item You must use the 2021 AAAI Press \LaTeX{} style file and the aaai21.bst bibliography style files, which are located in the 2021 AAAI Author Kit (aaai21.sty, aaai21.bst).
\item You must complete, sign, and return by the deadline the AAAI copyright form (unless directed by AAAI Press to use the AAAI Distribution License instead).
\item You must read and format your paper source and PDF according to the formatting instructions for authors.
\item You must submit your electronic files and abstract using our electronic submission form \textbf{on time.}
\item You must pay any required page or formatting charges to AAAI Press so that they are received by the deadline.
\item You must check your paper before submitting it, ensuring that it compiles without error, and complies with the guidelines found in the AAAI Author Kit.
\end{itemize}
\section{Copyright}
All papers submitted for publication by AAAI Press must be accompanied by a valid signed copyright form. They must also contain the AAAI copyright notice at the bottom of the first page of the paper. There are no exceptions to these requirements. If you fail to provide us with a signed copyright form or disable the copyright notice, we will be unable to publish your paper. There are \textbf{no exceptions} to this policy. You will find a PDF version of the AAAI copyright form in the AAAI AuthorKit. Please see the specific instructions for your conference for submission details.
\section{Formatting Requirements in Brief}
We need source and PDF files that can be used in a variety of ways and can be output on a variety of devices. The design and appearance of the paper is strictly governed by the aaai style file (aaai21.sty).
\textbf{You must not make any changes to the aaai style file, nor use any commands, packages, style files, or macros within your own paper that alter that design, including, but not limited to spacing, floats, margins, fonts, font size, and appearance.} AAAI imposes requirements on your source and PDF files that must be followed. Most of these requirements are based on our efforts to standardize conference manuscript properties and layout. All papers submitted to AAAI for publication will be recompiled for standardization purposes. Consequently, every paper submission must comply with the following requirements:
\begin{quote}
\begin{itemize}
\item Your .tex file must compile in PDF\LaTeX{} --- (you may not include .ps or .eps figure files.)
\item All fonts must be embedded in the PDF file --- including includes your figures.
\item Modifications to the style file, whether directly or via commands in your document may not ever be made, most especially when made in an effort to avoid extra page charges or make your paper fit in a specific number of pages.
\item No type 3 fonts may be used (even in illustrations).
\item You may not alter the spacing above and below captions, figures, headings, and subheadings.
\item You may not alter the font sizes of text elements, footnotes, heading elements, captions, or title information (for references and mathematics, please see the limited exceptions provided herein).
\item You may not alter the line spacing of text.
\item Your title must follow Title Case capitalization rules (not sentence case).
\item Your .tex file must include completed metadata to pass-through to the PDF (see PDFINFO below).
\item \LaTeX{} documents must use the Times or Nimbus font package (you may not use Computer Modern for the text of your paper).
\item No \LaTeX{} 209 documents may be used or submitted.
\item Your source must not require use of fonts for non-Roman alphabets within the text itself. If your paper includes symbols in other languages (such as, but not limited to, Arabic, Chinese, Hebrew, Japanese, Thai, Russian and other Cyrillic languages), you must restrict their use to bit-mapped figures. Fonts that require non-English language support (CID and Identity-H) must be converted to outlines or 300 dpi bitmap or removed from the document (even if they are in a graphics file embedded in the document).
\item Two-column format in AAAI style is required for all papers.
\item The paper size for final submission must be US letter without exception.
\item The source file must exactly match the PDF.
\item The document margins may not be exceeded (no overfull boxes).
\item The number of pages and the file size must be as specified for your event.
\item No document may be password protected.
\item Neither the PDFs nor the source may contain any embedded links or bookmarks (no hyperref or navigator packages).
\item Your source and PDF must not have any page numbers, footers, or headers (no pagestyle commands).
\item Your PDF must be compatible with Acrobat 5 or higher.
\item Your \LaTeX{} source file (excluding references) must consist of a \textbf{single} file (use of the ``input" command is not allowed.
\item Your graphics must be sized appropriately outside of \LaTeX{} (do not use the ``clip" or ``trim'' command) .
\end{itemize}
\end{quote}
If you do not follow these requirements, your paper will be returned to you to correct the deficiencies.
\section{What Files to Submit}
You must submit the following items to ensure that your paper is published:
\begin{itemize}
\item A fully-compliant PDF file that includes PDF metadata.
\item Your \LaTeX{} source file submitted as a \textbf{single} .tex file (do not use the ``input" command to include sections of your paper --- every section must be in the single source file). (The only allowable exception is .bib file, which should be included separately).
\item The bibliography (.bib) file(s).
\item Your source must compile on our system, which includes only standard \LaTeX{} 2020 TeXLive support files.
\item Only the graphics files used in compiling paper.
\item The \LaTeX{}-generated files (e.g. .aux, .bbl file, PDF, etc.).
\end{itemize}
Your \LaTeX{} source will be reviewed and recompiled on our system (if it does not compile, your paper will be returned to you. \textbf{Do not submit your source in multiple text files.} Your single \LaTeX{} source file must include all your text, your bibliography (formatted using aaai21.bst), and any custom macros.
Your files should work without any supporting files (other than the program itself) on any computer with a standard \LaTeX{} distribution.
\textbf{Do not send files that are not actually used in the paper.} We don't want you to send us any files not needed for compiling your paper, including, for example, this instructions file, unused graphics files, style files, additional material sent for the purpose of the paper review, and so forth.
\textbf{Do not send supporting files that are not actually used in the paper.} We don't want you to send us any files not needed for compiling your paper, including, for example, this instructions file, unused graphics files, style files, additional material sent for the purpose of the paper review, and so forth.
\textbf{Obsolete style files.} The commands for some common packages (such as some used for algorithms), may have changed. Please be certain that you are not compiling your paper using old or obsolete style files.
\textbf{Final Archive.} Place your PDF and source files in a single archive which should be compressed using .zip. The final file size may not exceed 10 MB.
Name your source file with the last (family) name of the first author, even if that is not you.
\section{Using \LaTeX{} to Format Your Paper}
The latest version of the AAAI style file is available on AAAI's website. Download this file and place it in the \TeX\ search path. Placing it in the same directory as the paper should also work. You must download the latest version of the complete AAAI Author Kit so that you will have the latest instruction set and style file.
\subsection{Document Preamble}
In the \LaTeX{} source for your paper, you \textbf{must} place the following lines as shown in the example in this subsection. This command set-up is for three authors. Add or subtract author and address lines as necessary, and uncomment the portions that apply to you. In most instances, this is all you need to do to format your paper in the Times font. The helvet package will cause Helvetica to be used for sans serif. These files are part of the PSNFSS2e package, which is freely available from many Internet sites (and is often part of a standard installation).
Leave the setcounter for section number depth commented out and set at 0 unless you want to add section numbers to your paper. If you do add section numbers, you must uncomment this line and change the number to 1 (for section numbers), or 2 (for section and subsection numbers). The style file will not work properly with numbering of subsubsections, so do not use a number higher than 2.
\subsubsection{The Following Must Appear in Your Preamble}
\begin{quote}
\begin{scriptsize}\begin{verbatim}
\def\year{2021}\relax
% File: formatting-instruction.tex
\documentclass[letterpaper]{article}
% DO NOT CHANGE THIS
\usepackage{aaai21} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\usepackage{natbib}
% DO NOT CHANGE THIS OR ADD OPTIONS
\usepackage{caption}
% DO NOT CHANGE THIS OR ADD OPTIONS
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%
% PDF Info Is REQUIRED.
% For /Author, add all authors within the parentheses,
% separated by commas. No accents or commands.
% For /Title, add Title in Mixed Case.
% No accents or commands. Retain the parentheses.
\pdfinfo{
/Title (AAAI Press Formatting Instructions for Authors
Using LaTeX -- A Guide)
/Author (AAAI Press Staff, Pater Patel Schneider,
Sunil Issar, J. Scott Penberthy, George Ferguson,
Hans Guesgen, Francisco Cruz, Marc Pujol-Gonzalez)
/TemplateVersion (2021.1)
}
\end{verbatim}\end{scriptsize}
\end{quote}
\subsection{Preparing Your Paper}
After the preamble above, you should prepare your paper as follows:
\begin{quote}
\begin{scriptsize}\begin{verbatim}
%
\begin{document}
\maketitle
\begin{abstract}
%...
\end{abstract}\end{verbatim}\end{scriptsize}
\end{quote}
\noindent You should then continue with the body of your paper. Your paper must conclude with the references, which should be inserted as follows:
\begin{quote}
\begin{scriptsize}\begin{verbatim}
% References and End of Paper
% These lines must be placed at the end of your paper
\bibliography{Bibliography-File}
\end{document}
\end{verbatim}\end{scriptsize}
\end{quote}
\subsection{Inserting Document Metadata with \LaTeX{}}
PDF files contain document summary information that enables us to create an Acrobat index (pdx) file, and also allows search engines to locate and present your paper more accurately. \textit{Document metadata for author and title are REQUIRED.} You may not apply any script or macro to implementation of the title, author, and metadata information in your paper.
\textit{Important:} Do not include \textit{any} \LaTeX{} code or nonascii characters (including accented characters) in the metadata. The data in the metadata must be completely plain ascii. It may not include slashes, accents, linebreaks, unicode, or any \LaTeX{} commands. Type the title exactly as it appears on the paper (minus all formatting). Input the author names in the order in which they appear on the paper (minus all accents), separating each author by a comma. You may also include keywords in the optional Keywords field.
\begin{quote}
\begin{scriptsize}\begin{verbatim}
\begin{document}\\
\maketitle\\
...\\
\bibliography{Bibliography-File}\\
\end{document}\\
\end{verbatim}\end{scriptsize}
\end{quote}
\subsection{Commands and Packages That May Not Be Used}
\begin{table*}[t]
\centering
\begin{tabular}{l|l|l|l}
\textbackslash abovecaption &
\textbackslash abovedisplay &
\textbackslash addevensidemargin &
\textbackslash addsidemargin \\
\textbackslash addtolength &
\textbackslash baselinestretch &
\textbackslash belowcaption &
\textbackslash belowdisplay \\
\textbackslash break &
\textbackslash clearpage &
\textbackslash clip &
\textbackslash columnsep \\
\textbackslash float &
\textbackslash input &
\textbackslash input &
\textbackslash linespread \\
\textbackslash newpage &
\textbackslash pagebreak &
\textbackslash renewcommand &
\textbackslash setlength \\
\textbackslash text height &
\textbackslash tiny &
\textbackslash top margin &
\textbackslash trim \\
\textbackslash vskip\{- &
\textbackslash vspace\{- \\
\end{tabular}
%}
\caption{Commands that must not be used}
\label{table1}
\end{table*}
\begin{table}[t]
\centering
%\resizebox{.95\columnwidth}{!}{
\begin{tabular}{l|l|l|l}
authblk & babel & cjk & dvips \\
epsf & epsfig & euler & float \\
fullpage & geometry & graphics & hyperref \\
layout & linespread & lmodern & maltepaper \\
navigator & pdfcomment & pgfplots & psfig \\
pstricks & t1enc & titlesec & tocbind \\
ulem
\end{tabular}
\caption{LaTeX style packages that must not be used.}
\label{table2}
\end{table}
There are a number of packages, commands, scripts, and macros that are incompatable with aaai21.sty. The common ones are listed in tables \ref{table1} and \ref{table2}. Generally, if a command, package, script, or macro alters floats, margins, fonts, sizing, linespacing, or the presentation of the references and citations, it is unacceptable. Note that negative vskip and vspace may not be used except in certain rare occurances, and may never be used around tables, figures, captions, sections, subsections, subsections, or references.
\subsection{Page Breaks}
For your final camera ready copy, you must not use any page break commands. References must flow directly after the text without breaks. Note that some conferences require references to be on a separate page during the review process. AAAI Press, however, does not require this condition for the final paper.
\subsection{Paper Size, Margins, and Column Width}
Papers must be formatted to print in two-column format on 8.5 x 11 inch US letter-sized paper. The margins must be exactly as follows:
\begin{itemize}
\item Top margin: .75 inches
\item Left margin: .75 inches
\item Right margin: .75 inches
\item Bottom margin: 1.25 inches
\end{itemize}
The default paper size in most installations of \LaTeX{} is A4. However, because we require that your electronic paper be formatted in US letter size, the preamble we have provided includes commands that alter the default to US letter size. Please note that using any other package to alter page size (such as, but not limited to the Geometry package) will result in your final paper being returned to you for correction.
\subsubsection{Column Width and Margins.}
To ensure maximum readability, your paper must include two columns. Each column should be 3.3 inches wide (slightly more than 3.25 inches), with a .375 inch (.952 cm) gutter of white space between the two columns. The aaai21.sty file will automatically create these columns for you.
\subsection{Overlength Papers}
If your paper is too long and you resort to formatting tricks to make it fit, it is quite likely that it will be returned to you. The best way to retain readability if the paper is overlength is to cut text, figures, or tables. There are, a few acceptable ways to reduce paper size that don't affect readability. First, turn on \textbackslash frenchspacing, which will reduce the space after periods. Next, move all your figures and tables to the top of the page. Consider removing less important portions of a figure. If you use \textbackslash centering instead of \textbackslash begin\{center\} in your figure environment, you can also buy some space. For mathematical environments, you may reduce fontsize {\bf but not below 6.5 point}.
Commands that alter page layout are forbidden. These include \textbackslash columnsep, \textbackslash float, \textbackslash topmargin, \textbackslash topskip, \textbackslash textheight, \textbackslash textwidth, \textbackslash oddsidemargin, and \textbackslash evensizemargin (this list is not exhaustive). If you alter page layout, you will be required to pay the page fee. Other commands that are questionable and may cause your paper to be rejected include \textbackslash parindent, and \textbackslash parskip. Commands that alter the space between sections are forbidden. The title sec package is not allowed. Regardless of the above, if your paper is obviously ``squeezed" it is not going to to be accepted. Options for reducing the length of a paper include reducing the size of your graphics, cutting text, or paying the extra page charge (if it is offered).
\subsection{Type Font and Size}
Your paper must be formatted in Times Roman or Nimbus. We will not accept papers formatted using Computer Modern or Palatino or some other font as the text or heading typeface. Sans serif, when used, should be Courier. Use Symbol or Lucida or Computer Modern for \textit{mathematics only. }
Do not use type 3 fonts for any portion of your paper, including graphics. Type 3 bitmapped fonts are designed for fixed resolution printers. Most print at 300 dpi even if the printer resolution is 1200 dpi or higher. They also often cause high resolution imagesetter devices to crash. Consequently, AAAI will not accept electronic files containing obsolete type 3 fonts. Files containing those fonts (even in graphics) will be rejected. (Authors using blackboard symbols must be avoid those packages that use type 3 fonts.)
Fortunately, there are effective workarounds that will prevent your file from embedding type 3 bitmapped fonts. The easiest workaround is to use the required times, helvet, and courier packages with \LaTeX{}2e. (Note that papers formatted in this way will still use Computer Modern for the mathematics. To make the math look good, you'll either have to use Symbol or Lucida, or you will need to install type 1 Computer Modern fonts --- for more on these fonts, see the section ``Obtaining Type 1 Computer Modern.")
If you are unsure if your paper contains type 3 fonts, view the PDF in Acrobat Reader. The Properties/Fonts window will display the font name, font type, and encoding properties of all the fonts in the document. If you are unsure if your graphics contain type 3 fonts (and they are PostScript or encapsulated PostScript documents), create PDF versions of them, and consult the properties window in Acrobat Reader.
The default size for your type must be ten-point with twelve-point leading (line spacing). Start all pages (except the first) directly under the top margin. (See the next section for instructions on formatting the title page.) Indent ten points when beginning a new paragraph, unless the paragraph begins directly below a heading or subheading.
\subsubsection{Obtaining Type 1 Computer Modern for \LaTeX{}.}
If you use Computer Modern for the mathematics in your paper (you cannot use it for the text) you may need to download type 1 Computer fonts. They are available without charge from the American Mathematical Society:
http://www.ams.org/tex/type1-fonts.html.
\subsubsection{Nonroman Fonts.}
If your paper includes symbols in other languages (such as, but not limited to, Arabic, Chinese, Hebrew, Japanese, Thai, Russian and other Cyrillic languages), you must restrict their use to bit-mapped figures.
\subsection{Title and Authors}
Your title must appear in mixed case (nouns, pronouns, and verbs are capitalized) near the top of the first page, centered over both columns in sixteen-point bold type (twenty-four point leading). This style is called ``mixed case," which means that means all verbs (including short verbs like be, is, using, and go), nouns, adverbs, adjectives, and pronouns should be capitalized, (including both words in hyphenated terms), while articles, conjunctions, and prepositions are lower case unless they directly follow a colon or long dash. Author's names should appear below the title of the paper, centered in twelve-point type (with fifteen point leading), along with affiliation(s) and complete address(es) (including electronic mail address if available) in nine-point roman type (the twelve point leading). (If the title is long, or you have many authors, you may reduce the specified point sizes by up to two points.) You should begin the two-column format when you come to the abstract.
\subsubsection{Formatting Author Information.}
Author information has to be set according the following specification depending if you have one or more than one affiliation. You may not use a table nor may you employ the \textbackslash authorblk.sty package. For one or several authors from the same institution, please just separate with commas and write the affiliation directly below using the macros \textbackslash author and \textbackslash affiliations:
\begin{quote}\begin{scriptsize}\begin{verbatim}
\author{
Author 1, ..., Author n \\
}
\affiliations {
Address line \\
... \\
Address line
}
\end{verbatim}\end{scriptsize}\end{quote}
\noindent For authors from different institutions, use \textbackslash textsuperscript \{\textbackslash rm x \} to match authors and affiliations.
\begin{quote}\begin{scriptsize}\begin{verbatim}
\author{
AuthorOne,\textsuperscript{\rm 1}
AuthorTwo,\textsuperscript{\rm 2}
AuthorThree,\textsuperscript{\rm 3}
AuthorFour,\textsuperscript{\rm 4}
AuthorFive \textsuperscript{\rm 5}}\\
}
\affiliations {
\textsuperscript{\rm 1}AffiliationOne, \\
\textsuperscript{\rm 2}AffiliationTwo, \\
\textsuperscript{\rm 3}AffiliationThree, \\
\textsuperscript{\rm 4}AffiliationFour, \\
\textsuperscript{\rm 5}AffiliationFive \\
\{email, email\}@affiliation.com,
email@affiliation.com,
email@affiliation.com,
email@affiliation.com
}
\end{verbatim}\end{scriptsize}\end{quote}
Note that you may want to break the author list for better visualization. You can achieve this using a simple line break (\textbackslash \textbackslash).
\subsection{\LaTeX{} Copyright Notice}
The copyright notice automatically appears if you use aaai21.sty. It has been hardcoded and may not be disabled.
\subsection{Credits}
Any credits to a sponsoring agency should appear in the acknowledgments section, unless the agency requires different placement. If it is necessary to include this information on the front page, use
\textbackslash thanks in either the \textbackslash author or \textbackslash title commands.
For example:
\begin{quote}
\begin{small}
\textbackslash title\{Very Important Results in AI\textbackslash thanks\{This work is
supported by everybody.\}\}
\end{small}
\end{quote}
Multiple \textbackslash thanks commands can be given. Each will result in a separate footnote indication in the author or title with the corresponding text at the botton of the first column of the document. Note that the \textbackslash thanks command is fragile. You will need to use \textbackslash protect.
Please do not include \textbackslash pubnote commands in your document.
\subsection{Abstract}
Follow the example commands in this document for creation of your abstract. The command \textbackslash begin\{abstract\} will automatically indent the text block. Please do not indent it further. {Do not include references in your abstract!}
\subsection{Page Numbers}
Do not \textbf{ever} print any page numbers on your paper. The use of \textbackslash pagestyle is forbidden.
\subsection{Text }
The main body of the paper must be formatted in black, ten-point Times Roman with twelve-point leading (line spacing). You may not reduce font size or the linespacing. Commands that alter font size or line spacing (including, but not limited to baselinestretch, baselineshift, linespread, and others) are expressly forbidden. In addition, you may not use color in the text.
\subsection{Citations}
Citations within the text should include the author's last name and year, for example (Newell 1980). Append lower-case letters to the year in cases of ambiguity. Multiple authors should be treated as follows: (Feigenbaum and Engelmore 1988) or (Ford, Hayes, and Glymour 1992). In the case of four or more authors, list only the first author, followed by et al. (Ford et al. 1997).
\subsection{Extracts}
Long quotations and extracts should be indented ten points from the left and right margins.
\begin{quote}
This is an example of an extract or quotation. Note the indent on both sides. Quotation marks are not necessary if you offset the text in a block like this, and properly identify and cite the quotation in the text.
\end{quote}
\subsection{Footnotes}
Avoid footnotes as much as possible; they interrupt the reading of the text. When essential, they should be consecutively numbered throughout with superscript Arabic numbers. Footnotes should appear at the bottom of the page, separated from the text by a blank line space and a thin, half-point rule.
\subsection{Headings and Sections}
When necessary, headings should be used to separate major sections of your paper. Remember, you are writing a short paper, not a lengthy book! An overabundance of headings will tend to make your paper look more like an outline than a paper. The aaai21.sty package will create headings for you. Do not alter their size nor their spacing above or below.
\subsubsection{Section Numbers.}
The use of section numbers in AAAI Press papers is optional. To use section numbers in \LaTeX{}, uncomment the setcounter line in your document preamble and change the 0 to a 1 or 2. Section numbers should not be used in short poster papers.
\subsubsection{Section Headings.}
Sections should be arranged and headed as follows:
\subsubsection{Acknowledgments.}
The acknowledgments section, if included, appears after the main body of text and is headed ``Acknowledgments." This section includes acknowledgments of help from associates and colleagues, credits to sponsoring agencies, financial support, and permission to publish. Please acknowledge other contributors, grant support, and so forth, in this section. Do not put acknowledgments in a footnote on the first page. If your grant agency requires acknowledgment of the grant on page 1, limit the footnote to the required statement, and put the remaining acknowledgments at the back. Please try to limit acknowledgments to no more than three sentences.
\subsubsection{Appendices.}
Any appendices follow the acknowledgments, if included, or after the main body of text if no acknowledgments appear.
\subsubsection{References.}
The references section should be labeled ``References" and should appear at the very end of the paper (don't end the paper with references, and then put a figure by itself on the last page). A sample list of references is given later on in these instructions. Please use a consistent format for references. Poorly prepared or sloppy references reflect badly on the quality of your paper and your research. Please prepare complete and accurate citations.
\subsection{Illustrations and Figures}
\begin{figure}[t]
\centering
\includegraphics[width=0.9\columnwidth]{figure1} % Reduce the figure size so that it is slightly narrower than the column. Don't use precise values for figure width.This setup will avoid overfull boxes.
\caption{Using the trim and clip commands produces fragile layers that can result in disasters (like this one from an actual paper) when the color space is corrected or the PDF combined with others for the final proceedings. Crop your figures properly in a graphics program -- not in LaTeX}.
\label{fig1}
\end{figure}
\begin{figure*}[t]
\centering
\includegraphics[width=0.8\textwidth]{figure2} % Reduce the figure size so that it is slightly narrower than the column.
\caption{Adjusting the bounding box instead of actually removing the unwanted data resulted multiple layers in this paper. It also needlessly increased the PDF size. In this case, the size of the unwanted layer doubled the paper's size, and produced the following surprising results in final production. Crop your figures properly in a graphics program. Don't just alter the bounding box.}
\label{fig2}
\end{figure*}
% Using the \centering command instead of \begin{center} ... \end{center} will save space
% Positioning your figure at the top of the page will save space and make the paper more readable
% Using 0.95\columnwidth in conjunction with the
Your paper must compile in PDF\LaTeX{}. Consequently, all your figures must be .jpg, .png, or .pdf. You may not use the .gif (the resolution is too low), .ps, or .eps file format for your figures.
Figures, drawings, tables, and photographs should be placed throughout the paper on the page (or the subsequent page) where they are first discussed. Do not group them together at the end of the paper. If placed at the top of the paper, illustrations may run across both columns. Figures must not invade the top, bottom, or side margin areas. Figures must be inserted using the \textbackslash usepackage\{graphicx\}. Number figures sequentially, for example, figure 1, and so on. Do not use minipage to group figures.
If you normally create your figures using pgfplots, please create the figures first, and then import them as pdfs with proper bounding boxes, as the bounding and trim boxes created by pfgplots are fragile and not valid.
When you include your figures, you must crop them \textbf{outside} of \LaTeX{}. The command \textbackslash includegraphics*[clip=true, viewport 0 0 10 10]{...} might result in a PDF that looks great, but the image is \textbf{not really cropped.} The full image can reappear (and obscure whatever it is overlapping) when page numbers are applied or color space is standardized. Figures \ref{fig1}, and \ref{fig2} display some unwanted results that often occur.
If your paper includes illustrations that are not compatible with PDF\TeX{} (such as .eps or .ps documents), you will need to convert them. The epstopdf package will usually work for eps files. You will need to convert your ps files to PDF however.
\subsubsection {Figure Captions.}The illustration number and caption must appear \textit{under} the illustration. Labels, and other text with the actual illustration must be at least nine-point type. However, the font and size of figure captions must be 10 point roman. Do not make them smaller, bold, or italic. (Individual words may be italicized if the context requires differentiation.)
\subsection{Tables}
Tables should be presented in 10 point roman type. If necessary, they may be altered to 9 point type. You may not use any commands that further reduce point size below nine points. Tables that do not fit in a single column must be placed across double columns. If your table won't fit within the margins even when spanning both columns, you must split it. Do not use minipage to group tables.
\subsubsection {Table Captions.} The number and caption for your table must appear \textit{under} (not above) the table. Additionally, the font and size of table captions must be 10 point roman and must be placed beneath the figure. Do not make them smaller, bold, or italic. (Individual words may be italicized if the context requires differentiation.)
\subsubsection{Low-Resolution Bitmaps.}
You may not use low-resolution (such as 72 dpi) screen-dumps and GIF files---these files contain so few pixels that they are always blurry, and illegible when printed. If they are color, they will become an indecipherable mess when converted to black and white. This is always the case with gif files, which should never be used. The resolution of screen dumps can be increased by reducing the print size of the original file while retaining the same number of pixels. You can also enlarge files by manipulating them in software such as PhotoShop. Your figures should be 300 dpi when incorporated into your document.
\subsubsection{\LaTeX{} Overflow.}
\LaTeX{} users please beware: \LaTeX{} will sometimes put portions of the figure or table or an equation in the margin. If this happens, you need to make the figure or table span both columns. If absolutely necessary, you may reduce the figure, or reformat the equation, or reconfigure the table.{ \bf Check your log file!} You must fix any overflow into the margin (that means no overfull boxes in \LaTeX{}). \textbf{Nothing is permitted to intrude into the margin or gutter.}
\subsubsection{Using Color.}
Use of color is restricted to figures only. It must be WACG 2.0 compliant. (That is, the contrast ratio must be greater than 4.5:1 no matter the font size.) It must be CMYK, NOT RGB. It may never be used for any portion of the text of your paper. The archival version of your paper will be printed in black and white and grayscale. The web version must be readable by persons with disabilities. Consequently, because conversion to grayscale can cause undesirable effects (red changes to black, yellow can disappear, and so forth), we strongly suggest you avoid placing color figures in your document. If you do include color figures, you must (1) use the CMYK (not RGB) colorspace and (2) be mindful of readers who may happen to have trouble distinguishing colors. Your paper must be decipherable without using color for distinction.
\subsubsection{Drawings.}
We suggest you use computer drawing software (such as Adobe Illustrator or, (if unavoidable), the drawing tools in Microsoft Word) to create your illustrations. Do not use Microsoft Publisher. These illustrations will look best if all line widths are uniform (half- to two-point in size), and you do not create labels over shaded areas. Shading should be 133 lines per inch if possible. Use Times Roman or Helvetica for all figure call-outs. \textbf{Do not use hairline width lines} --- be sure that the stroke width of all lines is at least .5 pt. Zero point lines will print on a laser printer, but will completely disappear on the high-resolution devices used by our printers.
\subsubsection{Photographs and Images.}
Photographs and other images should be in grayscale (color photographs will not reproduce well; for example, red tones will reproduce as black, yellow may turn to white, and so forth) and set to a minimum of 300 dpi. Do not prescreen images.
\subsubsection{Resizing Graphics.}
Resize your graphics \textbf{before} you include them with LaTeX. You may \textbf{not} use trim or clip options as part of your \textbackslash includegraphics command. Resize the media box of your PDF using a graphics program instead.
\subsubsection{Fonts in Your Illustrations.}
You must embed all fonts in your graphics before including them in your LaTeX document.
\subsection{References}
The AAAI style includes a set of definitions for use in formatting references with BibTeX. These definitions make the bibliography style fairly close to the one specified below. To use these definitions, you also need the BibTeX style file ``aaai21.bst," available in the AAAI Author Kit on the AAAI web site. Then, at the end of your paper but before \textbackslash end{document}, you need to put the following lines:
\begin{quote}
\begin{small}
\textbackslash bibliography\{bibfile1,bibfile2,...\}
\end{small}
\end{quote}
Please note that the aaai21.sty class already sets the bibliographystyle for you, so you do not have to place any \textbackslash bibliographystyle command in the document yourselves. The aaai21.sty file is incompatible with the hyperref and navigator packages. If you use either, your references will be garbled and your paper will be returned to you.
References may be the same size as surrounding text. However, in this section (only), you may reduce the size to \textbackslash small if your paper exceeds the allowable number of pages. Making it any smaller than 9 point with 10 point linespacing, however, is not allowed. A more precise and exact method of reducing the size of your references minimally is by means of the following command: \begin{quote}
\textbackslash fontsize\{9.8pt\}\{10.8pt\}
\textbackslash selectfont\end{quote}
\noindent You must reduce the size equally for both font size and line spacing, and may not reduce the size beyond \{9.0pt\}\{10.0pt\}.
The list of files in the \textbackslash bibliography command should be the names of your BibTeX source files (that is, the .bib files referenced in your paper).
The following commands are available for your use in citing references:
\begin{quote}
{\em \textbackslash cite:} Cites the given reference(s) with a full citation. This appears as ``(Author Year)'' for one reference, or ``(Author Year; Author Year)'' for multiple references.\smallskip\\
{\em \textbackslash shortcite:} Cites the given reference(s) with just the year. This appears as ``(Year)'' for one reference, or ``(Year; Year)'' for multiple references.\smallskip\\
{\em \textbackslash citeauthor:} Cites the given reference(s) with just the author name(s) and no parentheses.\smallskip\\
{\em \textbackslash citeyear:} Cites the given reference(s) with just the date(s) and no parentheses.
\end{quote}
Formatted bibliographies should look like the following examples.
\smallskip \noindent \textit{Book with Multiple Authors}\\
Engelmore, R., and Morgan, A. eds. 1986. \textit{Blackboard Systems.} Reading, Mass.: Addison-Wesley.
\smallskip \noindent \textit{Journal Article}\\
Robinson, A. L. 1980a. New Ways to Make Microcircuits Smaller. \textit{Science} 208: 1019--1026.
\smallskip \noindent \textit{Magazine Article}\\
Hasling, D. W.; Clancey, W. J.; and Rennels, G. R. 1983. Strategic Explanations in Consultation. \textit{The International Journal of Man-Machine Studies} 20(1): 3--19.
\smallskip \noindent \textit{Proceedings Paper Published by a Society}\\
Clancey, W. J. 1983. Communication, Simulation, and Intelligent Agents: Implications of Personal Intelligent Machines for Medical Education. In \textit{Proceedings of the Eighth International Joint Conference on Artificial Intelligence,} 556--560. Menlo Park, Calif.: International Joint Conferences on Artificial Intelligence, Inc.
\smallskip \noindent \textit{Proceedings Paper Published by a Press or Publisher}\\
Clancey, W. J. 1984. Classification Problem Solving. In \textit{Proceedings of the Fourth National Conference on Artificial Intelligence,} 49--54. Menlo Park, Calif.: AAAI Press.
\smallskip \noindent \textit{University Technical Report}\\
Rice, J. 1986. Poligon: A System for Parallel Problem Solving, Technical Report, KSL-86-19, Dept. of Computer Science, Stanford Univ.
\smallskip \noindent \textit{Dissertation or Thesis}\\
Clancey, W. J. 1979. Transfer of Rule-Based Expertise through a Tutorial Dialogue. Ph.D. diss., Dept. of Computer Science, Stanford Univ., Stanford, Calif.
\smallskip \noindent \textit{Forthcoming Publication}\\
Clancey, W. J. 2021. The Engineering of Qualitative Models. Forthcoming.
For the most up to date version of the AAAI reference style, please consult the \textit{AI Magazine} Author Guidelines at \url{https://aaai.org/ojs/index.php/aimagazine/about/submissions#authorGuidelines}
\section{Proofreading Your PDF}
Please check all the pages of your PDF file. The most commonly forgotten element is the acknowledgements --- especially the correct grant number. Authors also commonly forget to add the metadata to the source, use the wrong reference style file, or don't follow the capitalization rules or comma placement for their author-title information properly. A final common problem is text (expecially equations) that runs into the margin. You will need to fix these common errors before submitting your file.
\section{Improperly Formatted Files }
In the past, AAAI has corrected improperly formatted files submitted by the authors. Unfortunately, this has become an increasingly burdensome expense that we can no longer absorb). Consequently, if your file is improperly formatted, it will be returned to you for correction.
\subsection{\LaTeX{} 209 Warning}
If you use \LaTeX{} 209 your paper will be returned to you unpublished. Convert your paper to \LaTeX{}2e.
\section{Naming Your Electronic File}
We require that you name your \LaTeX{} source file with the last name (family name) of the first author so that it can easily be differentiated from other submissions. Complete file-naming instructions will be provided to you in the submission instructions.
\section{Submitting Your Electronic Files to AAAI}
Instructions on paper submittal will be provided to you in your acceptance letter.
\section{Inquiries}
If you have any questions about the preparation or submission of your paper as instructed in this document, please contact AAAI Press at the address given below. If you have technical questions about implementation of the aaai style file, please contact an expert at your site. We do not provide technical support for \LaTeX{} or any other software package. To avoid problems, please keep your paper simple, and do not incorporate complicated macros and style files.
\begin{quote}
\noindent AAAI Press\\
2275 East Bayshore Road, Suite 160\\
Palo Alto, California 94303\\
\textit{Telephone:} (650) 328-3123\\
\textit{E-mail:} See the submission instructions for your particular conference or event.
\end{quote}
\section{Additional Resources}
\LaTeX{} is a difficult program to master. If you've used that software, and this document didn't help or some items were not explained clearly, we recommend you read Michael Shell's excellent document (testflow doc.txt V1.0a 2002/08/13) about obtaining correct PS/PDF output on \LaTeX{} systems. (It was written for another purpose, but it has general application as well). It is available at www.ctan.org in the tex-archive.
\section{ Acknowledgments}
AAAI is especially grateful to Peter Patel Schneider for his work in implementing the original aaai.sty file, liberally using the ideas of other style hackers, including Barbara Beeton. We also acknowledge with thanks the work of George Ferguson for his guide to using the style and BibTeX files --- which has been incorporated into this document --- and Hans Guesgen, who provided several timely modifications, as well as the many others who have, from time to time, sent in suggestions on improvements to the AAAI style. We are especially grateful to Francisco Cruz, Marc Pujol-Gonzalez, and Mico Loretan for the improvements to the Bib\TeX{} and \LaTeX{} files made in 2020.
The preparation of the \LaTeX{} and Bib\TeX{} files that implement these instructions was supported by Schlumberger Palo Alto Research, AT\&T Bell Laboratories, Morgan Kaufmann Publishers, The Live Oak Press, LLC, and AAAI Press. Bibliography style changes were added by Sunil Issar. \verb+\+pubnote was added by J. Scott Penberthy. George Ferguson added support for printing the AAAI copyright slug. Additional changes to aaai21.sty and aaai21.bst have been made by Francisco Cruz, Marc Pujol-Gonzalez, and Mico Loretan.
\bigskip
\noindent Thank you for reading these instructions carefully. We look forward to receiving your electronic files!
\end{document}
\def\year{2021}\relax
%File: formatting-instructions-latex-2021.tex
%release 2021.2
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai21} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%\nocopyright
%PDF Info Is REQUIRED.
% For /Author, add all authors within the parentheses, separated by commas. No accents or commands.
% For /Title, add Title in Mixed Case. No accents or commands. Retain the parentheses.
\pdfinfo{
/Title (AAAI Press Formatting Instructions for Authors Using LaTeX -- A Guide)
/Author (AAAI Press Staff, Pater Patel Schneider, Sunil Issar, J. Scott Penberthy, George Ferguson, Hans Guesgen, Francisco Cruz, Marc Pujol-Gonzalez)
/TemplateVersion (2021.2)
} %Leave this
% /Title ()
% Put your actual complete title (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case
% Leave the space between \Title and the beginning parenthesis alone
% /Author ()
% Put your actual complete list of authors (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case.
% Each author should be only by a comma. If the name contains accents, remove them. If there are any LaTeX commands,
% remove them.
% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
\newcommand{\MYhref}[3][blue]{\href{#2}{\color{#1}{#3}}}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{color, colortbl}
\definecolor{airforceblue}{rgb}{0.36, 0.54, 0.66}
\definecolor{LightCyan}{rgb}{0.88,1,1}
\definecolor{codegreen}{rgb}{0,0.6,0}
\usepackage{array,etoolbox}
% \AtBeginEnvironment{table}{\rmfamily \small}
\preto\tabular{\setcounter{magicrownumbers}{0}}
\newcounter{magicrownumbers}
\newcommand\rownumber{\stepcounter{magicrownumbers}\arabic{magicrownumbers}}
\newif\ifpaperfinal
\paperfinalfalse
% \paperfinaltrue
\ifpaperfinal
\newcommand{\tn}[1]{}
\newcommand{\iinote}[1]{}
\newcommand{\ernote}[1]{}
\newcommand{\stnote}[1]{}
\else
\newcommand{\iinote}[1]{\textcolor{codegreen}{\textbf{II: #1}}}
\newcommand{\stnote}[1]{\textcolor{blue}{\textbf{ST: #1}}}
\newcommand{\tn}[1]{\textcolor{airforceblue}{\textbf{TN: #1}}}
\newcommand{\ernote}[1]{\textcolor{red}{\textbf{ER: #1}}}
\fi
% The file aaai21.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%
% Title
% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Where were my keys? - Aggregating Spatial-Temporal Instances of Objects for Efficient Retrieval over Long Periods of Time}
\author{
%Authors
% All authors must be in the same font size and format.
% Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
% AAAI Style Contributions by Pater Patel Schneider,
% Sunil Issar, \\
% J. Scott Penberthy,
% George Ferguson,
% Hans Guesgen,
% Francisco Cruz,
% Marc Pujol-Gonzalez
% \\
Ifrah Idrees, Zahid Hasan, Steven P. Reiss, and Stefanie Tellex\\
}
\affiliations{
%Afiliations
Dept. of Computer Science, Brown University, Providence, RI\\
% \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
%If you have multiple authors and multiple affiliations
% use superscripts in text and roman font to identify them.
%For example,
% Sunil Issar, \textsuperscript{\rm 2}
% J. Scott Penberthy, \textsuperscript{\rm 3}
% George Ferguson,\textsuperscript{\rm 4}
% Hans Guesgen, \textsuperscript{\rm 5}.
% Note that the comma should be placed BEFORE the superscript for optimum readability
% 2275 East Bayshore Road, Suite 160\\
% Palo Alto, California 94303\\
% email address must be in roman text type, not monospace or sans serif
% publications21@aaai.org
% ifrah
% See more examples next
}
\iffalse
%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Single Author}
\author {
% Author
Author Name \\
}
\affiliations{
Affiliation \\
Affiliation Line 2 \\
name@example.com
}
\fi
\iffalse
%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Multiple Authors}
\author {
% Authors
First Author Name,\textsuperscript{\rm 1}
Second Author Name, \textsuperscript{\rm 2}
Third Author Name \textsuperscript{\rm 1} \\
}
\affiliations {
% Affiliations
\textsuperscript{\rm 1} Affiliation 1 \\
\textsuperscript{\rm 2} Affiliation 2 \\
firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
}
\fi
\begin{document}
\maketitle
\begin{abstract}
Robots equipped with situational awareness can help humans efficiently find their lost objects by leveraging spatial and temporal structure. Existing approaches to video and image retrieval do not take into account the unique constraints imposed by a moving camera with partial view of the environment. We present a \textbf{D}etection-based \textbf{3}-level hierarchical \textbf{A}ssociation approach, \textbf{D3A}, to create an efficient query-able spatial-temporal representation of unique object instances in an environment. D3A performs online incremental and hierarchical\iinote{hierarchical?} learning to identify keyframes that best represent the unique objects in the environment. These keyframes\iinote{are learned based on both spatial and temporal features and once identified cut add}
and their corresponding spatial-temporal information are then organized in a key-value database. D3A allows for a variety of query patterns such as querying for objects with/without the following: 1) specific attributes, 2) spatial relationships with other objects, and 3) time slices. For a given set of 150 queries, D3A returns a small set of candidate keyframes (which occupy only 0.17\% of the total sensory-data) with 81.98\% mean accuracy in 11.7 ms. This is 47x faster and 33\% more accurate than a baseline that naively stores the object matches (detections) in the database without \iinote{associating}the associated spatial-temporal information.
\end{abstract}
\section{INTRODUCTION}
\iinote{color package in table}
\iinote{reduce font size of all tables}
\iinote{fix table 2}
\iinote{remove vspace}
Home-service robots have great potential to assist human users by retrieving spatial-temporal information\footnote{Spatial-temporal information of object refers to the whereabouts of the object such as where the object has been identified in the physical environment of the robot and at what time} about objects from their long-term observations. For example, a person can ask a simple query such as \textit{``where did I leave my keys?''} \iinote{ Cut THIS NOTE and a situationally-aware robot can provide information about the whereabouts of the objects - such as identifying the object and where the object was and at what time} Service robots with such an ability will be well suited to help the elderly, especially those that have dementia.
There are two lines of works in image and video object retrieval: a) \textit{fixed-view cameras} \cite{object_retrieval, yadav2019vidcep, kang2019challenges} which assume full observability of the environment and do not deal with a moving camera (robot), and b) \textit{robotic object retrieval} \cite{ambrucs2014meta, bore2015retrieval} that either work on short-term time horizons (do not condense partial view detections of unique object instances for memory or speed efficiency) or perform long-term \iinote{robotic}object retrieval by specifically focusing on point cloud matching without incorporating spatial information\iinote{object's spatial informations in the physical environment, of where the objects are located in the environment}. These approaches for object retrieval will leave the robot searching over countless detections in visual sensor data from many different time slices, which will take up a lot of space and time. In addition, these detections will contain partial views of different object instances, not all of \iinote{these objects and thus partial views will be}which are relevant to the query. An algorithm that condenses the partial view detections of each unique object instance into a compact and query-able spatial-temporal representation will enable the robot to answer queries about unique object instances more efficiently.
\begin{figure}[]
\centering
% \includegraphics[width=0.8\linewidth]{images/story_2.png}
\includegraphics[width=0.75\linewidth,height=6cm]{images/story_2.png}
% \caption{Kuri Robot \citep{richardson_2018} \stnote{I would replace this with a storyboard of the system, showing it answering a question about where an object is. Also, make sure this is our picture of Kuri, use one you took that we have rights to.}}\label{fig:a}
\caption{The image shows the retrieved keyframes and associated temporal information for two sample queries answered by D3A while deployed on Kuri. The keyframes are overlaid on the map using the spatial information returned by our spatial-temporal representation. Also shown are the partial view detections that are aggregated into the keyframe cluster.}
\label{fig:Kuri}
\vspace{-3mm}
\end{figure}
To our knowledge, there has not been a single work that addresses all of the following challenges: 1) aggregating detections of object instances from different views in a map over hours of sensory information, 2) handling the object going out of the view, and 3) accounting for uncertainty in object poses due to noise from the robot pose. To mitigate these challenges, we introduce a new algorithm, D3A that extends the concept of keyframe extraction that has been previously used for video frame retrieval to condense
% \iinote{Add: that extends the concept of keyframe extraction that has been previously used for video frame(s) retrieval to condense}
multiview detections of unique objects over both physical space and time for long observation periods.
D3A extracts both spatial-temporal features of object instances in the environment as they get seen by the robot.
% Treating all these partial views of an object as unique and/or storing all of their spatial-temporal information will be memory and speed inefficient for object retrieval. To mitigate these challenges,
D3A associates these \iinote{partial view features instead of views} with what it has previously seen over the map to condense them.
% D3A enables scalable association and object retrieval by
It does so by performing three-tier online, incremental clustering and filtering of unique spatial-temporal instances. While performing association, D3A keeps track of the keyframe that best represents each unique spatial-temporal location of the object. The information about keyframe-centroid clusters are then spatial-temporally indexed in a key-value database, leading to a compact and query-able representation for efficient object retrieval.
This representation allows D3A to identify and return a small set of keyframes for the object(s) in a given query, which enables a person to quickly find the relevant information. D3A is storage efficient and is able to handle the case where the attributes of object instances (color, shape, patterns) asked for in the queries are not known in advance. In prior work all or subset of these attributes were provided at the time of query.
Our algorithm is deployed on a social robot Kuri from \citet{heykuri_2018} that patrolled a \iinote{robotics lab} environment and collected 22 hours of observations over four days. \footnote{A sample of the collected dataset can be found at this link: https://github.com/IfrahIdrees/D3A.git}. When tested on a set of 150 queries about different objects, D3A returns a small set of keyframes and their spatial-temporal information (which \iinote{on average} occupy only 0.17\% of the total \iinote{sensory data of object of interests}) and is 47x faster than the baseline model which treats all partial-view detections as independent instances. D3A is also 33\% more accurate than the baseline with an accuracy value of 81.98\%.
\section{Related Work}
% \ernote{I feel like you should discuss SLAM and semantic mapping literature. I know you're just using off-the-shelf algorithms for SLAM that comes with Kuri, but there's semantic mapping literature \cite{pronobis2012large} that track object attributes, though i dont think they ever focus on the memory side of things like D3A does.}
% \iinote{\citet{hsieh2018focus} proposed work similar to ours that can handle many hours of videos captured by a moving camera, except that it does not deal with the objects' spatial information within the global map and only tries to identify video frames with objects of certain classes. We perform an ablation study with this work and show that by using the objects' locations in the map, D3A improves 1) detection of unique object instances and 2) compactness and retrieval performance from our spatial-temporal database.}
% Video processing has advanced quite dramatically over the years in terms of fundamental algorithms for detection, and re-identification of objects.
% \iinote{How key frame detection works. different techniques for key frame extract like clustering methods, energy minimization algorithms or sequential techniques \cite{vazquez2013spatio}. Usually key frame extraction is used to identify a subset of a video frames that can represent the visual content of the sequence as close as possible, but with a limiting number of frame information And usually the spatial-temporal information used for clustering is based on the visual descriptors of the frame . Clustering techniques differ in how they define an appropriate similarity measure between visual descriptors of the frames. }
In the past, \textbf{Keyframe Extraction} techniques have been used to extract the summary key frames to represent video sequences \cite{6135507}.
% \cite{5201141, 6135507, lingam2019key, event, object_tracking}.
A lot of different techniques exists for keyframe extraction such as clustering \cite{vazquez2013spatio}, energy minimization \cite{essa2015modular}, or online techniques \cite{elahi2020online}.
% These differ in how they define the similarity measure between visual descriptors of the frames.
% \tn{the previous sentence feels out of place and should be cut. You can just say that D3A extendes the clustering keyframe extraction technique to ...}
D3A extends the clustering keyframe extraction technique in videos to incrementally find keyframes that best represent the different views of the unique objects in the environment over time, as the robot continues to capture new video frames.
% \iinote{Add: In the past \textbf{Key frame Extraction} techniques have been used to extract the summary key frames to represent video sequences \cite{5201141, 6135507, lingam2019key, event, object_tracking}. A lot of different techniques exists for keyframe extraction such as clustering, energy minimization or sequential techniques \cite{vazquez2013spatio}. These differ in how they define an appropriate similarity measure between visual descriptors of the frames. D3A extends keyframe extraction in videos to incrementally finding keyframes that best represent multi views of the unique objects in the environment over time, as the robot continues to capture new video frames.}
A lot of work has also been done in \textbf{Object Detection} \cite{MaskRCNN, RetinaNet, FastCNN, pose_estimation} for detecting and re-identifying objects. D3A is built upon the notion of object detection in a three-tier mechanism to match objects and condense their detections over long periods of observations. Additionally, there are \textbf{Object Tracking} algorithms that track object instances over time \cite{object_tracking, yilmaz2006object}. However, they usually assume input videos with fixed angles and cannot track/associate objects under occlusion or after the objects go out of view.
% \iinote{In past keyframes is a method used to extract the perform clustering of consecutive frames to find representative keyframes for all the consecutive frames video trying to find unique views of scene. we extend that to actually find keyframes to associate multi views of the object for object association over space and time.They differ in how they define an appropriate similarity measure between visual descriptors of the frames. }
% These algorithms don't scale well with the number of objects.
These algorithms are also not useful for retrieval since they do not focus on making the objects' trajectories query-able.
% \tn{can cut: We use an unsupervised clustering method in our implementation which can be replaced by any other techniques.}
% \ernote{What other techniques are likely? I feel like you mean that you chose a specific unsupervised clustering algorithm X, but any unsupervised clustering algorithm could be used?}
Although many \textbf{Image or Video Retrieval} systems have been introduced in the multimedia community, they focus on retrieval at the level of object classes and not instances. These works can be categorized into the following types:\\
\textit{1) Fixed-angle Video Input} \cite{object_retrieval, yadav2019vidcep, kang2019challenges}: These works do not account for the objects' spatial information in a global map. In contrast, D3A can perform object instance association in a partially observable environment monitored by a robot.\\
\textit{2) Query Optimized Object Retrieval} \cite{kang2019challenges}: This work uses information from the query to optimize the retrieval pipeline by searching over all the scaled down image frames in the original dataset. They do not reduce redundancy in the data,
% Their search space is over all the frames and they do not focus on reducing the redundancy in data \cite{kang2019challenges}.
% Our algorithm instead focuses on optimizing the spatial-temporal representation to be used for retrieval. We focus on condensing the detections to only store key frame information for retrieval.
whereas D3A focuses on only storing the keyframe information
% to condense the detections
and optimizing the spatial-temporal representation.
There are also works that can handle partial views of objects and perform view-invariant object detection at the category level \cite{hsieh2018focus, sivic2003video}.
% However, they do not record the objects' spatial information and cannot answer spatial queries like D3A.
\citet{hsieh2018focus} proposed work similar to ours that can handle many hours of videos captured by a moving camera (that is not a robot) but does not use object locations. We perform an ablation study with this work and show that by using the objects' locations in the map, D3A improves 1) detection of unique object instances and 2) compactness and retrieval performance from our spatial-temporal database. \citet{sivic2003video} assume to have seen a part of the scene captured by different static camera viewpoints to learn the visual vocabulary (which incorporates spatial information) to associate objects seen in later scenes. We perform online, incremental object matching, and do not pre-define the number of clusters/objects in the environment.
% Unlike both of these works, our clustering is based on both the object feature space and spatial-temporal information.
% There are other works that deal with robotic object retrieval but over short-time horizons and therefore they do not condense the partial views for memory and speed efficiency.
% Another relevant line of work is robotic object retrieval. Most works in this line operate over short-time horizons and do not condense partial view detections of unique object instances for memory or speed efficiency \cite{Temporal-Grounding} \tn{feels like you're missing citations}.
% \tn{what's the difference between these works and the ones in the previous paragraph?}
There are also works that specifically focus on long-term robotic object retrieval that handle partial observability in visual sensor data \citep{ambrucs2014meta, bore2015retrieval}. However, these works assume a point cloud of objects as input and introduce matching algorithms specifically in 3D space. D3A, in contrast, does matching using the RGB feature space and limited depth information (information provided by the depth sensors for obstacle detection) that can be derived from the 3D space. \tn{is this better bc it requires ``less" visual information or just different} These works also do not deal with the objects' spatial information in the environment. Furthermore, they do not focus on making a compact representation of the sensory data for retrieval, but instead store and operate over the original data and use the object attribute ``shape" provided in query to optimize retrieval.
% \tn{This also seems similar to the query optimized object retrieval works described above. Are these the works that require that the queried object attributes are known in advance? This bit is not explicitly stated anywhere}
% \stnote{use citep, not citet, only use cite for noun phrases. for multiple cites, put them in one citep command separated by commas }
% \begin{figure}[t]
% \centering
% \includegraphics[width=\linewidth]{images/high_level.png}
% \caption{Block Level diagram of D3A \tn{D3A}}
% % \roma{Trim the edges so they fit exactly and also make the figure width the line width so this fits the page nicely and the font is easier to read :-)}}
% \label{fig:a}
% \label{fig:block-diagram}
% \end{figure}
\begin{figure}[t]
\centering
\includegraphics[width=0.8\linewidth]{images/Processing_diagram.png}
\caption{Visualization of D3A Algorithm \iinote{Add Qi, si}}
% \label{fig:a}
\label{fig:Processing}
\vspace{-6mm}
\end{figure}
% \tn{this is super readable now! i recommend using a different color (darker green maybe) to label the tiers bc the current color is a little hard to see (same for Fig 3)}
\section{Problem Formulation}\label{Section:system_overview}
% \stnote{So you want to write technical approach in a more problem-driven way. Think of it as a tree of problems and solutions. You don't want to introduce a bit of technical material until you've first explained the problem you need to solve by that bit. The paragraph below is just saying what you built without saying why or situating it in what problems you are solving.}
% \iinote{how link speed and memory efficiency}
\iinote{fix time variable as t?}
A robot monitoring the environment over long periods is given a spatial-temporal query $Q$ to retrieve spatial temporal information of target objects seen in the environment. The query can be of any type as listed in Table-\ref{tab:example_queries}. The robot is equipped with the map of the environment and at any given time step $i$ can gather the following sensory information $s_i=<p_{robot,i}, d_i, f_i>$, where $p_{robot,i} \in P_{robot}$ is the associated robot's pose, and $d_i \in D$ is the corresponding depth information for image frame $f_i$. The robot collects large amounts of sensor data $S = \{s_1, s_2, ... , s_i\}$ and uses an object detector to extract the relevant object-centric information $a_i$ from every frame $f_i$. However, the robot has partial observability of the environment and the information required to answer the object retrieval query is going to be dispersed throughout $S$ in the form of multiple partial view detections. To answer object-retrieval queries efficiently, the robot needs to condense the multi-view detections of the objects into a memory efficient and query-able representation. This representation is then used by the robot to return a small set of keyframes, which enables a person to quickly find the relevant information about the target object.
% The sensor data $S$ will take up a lot of disc space and be inherently noisy. Searching all of this data for retrieving object's spatial-temporal information is going to be speed-inefficient.
% % The robot should also be able to answer queries where the attributes of object instances (color, shape, patterns) are not known in advance.
% For the robot to retrieve an object's spatial temporal information in a memory and speed efficient, it should be able to condense the multi-view detections of the objects over time and be able to organize them in queryable representation such that given a query $Q$, it can return a small set of keyframes, which enable a person to quickly find the relevant information about the target object.
\section{Approach}
% all this sensor data $S$ through which the object need to be retrieved takes up a lot of disc space and is inherently noisy.
% Equipped with $S$, the robot is given a spatial-temporal query $Q$ of any of the type listed in Table-\ref{tab:example_queries}. One example is ``Did you ever see an orange bowl" and is tasked to retrieve spatial temporal information of target objects seen in the environment. Ho
% A human given a robot collecting $S$ can ask from their personal assistant robot a spatial-temporal query - $Q$ about any object in the environment seen during any time of its patrol such as can be "Did you ever see an orange bowl".
% $=< p_{robot,i}, d_i, f_i>$ is the associated information for every captured image frame $f_i$. $p_{robot,i} \in P_{robot}$ is the associated robot's pose, and $d_i \in D$ is the corresponding depth information, for every image frame $f_i$.
% Note that the next time step $i+1$ is dependant on the frequency at which the robot is capturing the sensor data.
% All this sensor data $S$ takes up a lot of disc space and is inherently noisy.
For efficient spatial-temporal object retrieval in a partially observable environment, we need to compress this data $S$ into a compact and query-able spatial-temporal representation $R$. For this we first pre-process every $s_i$ to extract the relevant object-centric information $a_i$, then apply a three-tier online clustering algorithm to aggregate $a_{i}$'s. The output of D3A is a set of clusters of unique object instances that can be aggregated into the previous time step's compact representation $R_{i-1}$ to create a new representation $R_i$ of the environment for time step $i$. A diagram of our algorithm creating $R$ is shown in Fig-\ref{fig:Processing}.
% \stnote{problem formulation is only the problem, not the solution. So don't put anything about preprocessing etc in the problem formulation.}
% \vspace
\subsection{Pre-processing each frame:}
% For compactness and speed efficiency, D3A needs to reduce the dimensionality of the visual sensory data $f_i$ in $s_i=< p_{robot, i}, d_i, f_i>$, as well as aggregate multiple detections of the same object over long periods of time. D3A achieves this by extracting minimal amounts of relevant object-centric information $a_i = <C_i, Prob_i , B_i, P_{object,i}, Emb_i>$ for every frame $f_i$. There is a set $J_i$ for every $f_i$ that includes all the detected object instances in $f_i$. $C_i$ is the list of corresponding categories for all the detections in the set $J_{i}$,
% % for each detection $j \in J_i$ and $J_i$ represents the set of all detections in $f_i$,
% $Prob_i$ are the probabilities of detecting the object categories $C_i$, $B_i$ is the list of the object bounding boxes for $J_i$, $P_{object,i}$ is the list of object poses estimated for all the detections. $P_{object,i}$ is estimated from the robot's pose $p_{robot,i}$ and limited depth information $d_i$, and is a 3D world coordinate. $Emb_i$ is the list of embeddings corresponding to the detections in $J_i$, which is used to relate $j's$ over time and create the compact representation $Q$.
For compactness and speed efficiency, D3A needs to reduce the dimensionality of the visual sensory data $f_i$ in $s_i=< p_{robot, i}, d_i, f_i>$, as well as aggregate multiple detections of the same object over long periods of time. D3A achieves this by extracting minimal amounts of relevant object-centric information ${a_i}$:
\begin{center}
$a_i = [ <c_{i1},prob_{i1},b_{i1},p_{robot,i1},emb_{i1}>, <c_{i2},prob_{i2} ,b_{i2},p_{robot,i2},emb_{i2}>,..., <c_{ij}, prob_{ij},b_{ij},p_{robot,ij},emb_{ij}>]$
\end{center}
for every frame $f_i$, $c_{ij} \in C_i$, is the category detected for the $j^{th}$ object instance detection in $i^{th}$ image, $prob_{ij}$ is the probability of $j^{th}$ object being assigned the category $c_{ij}$, and $b_{ij}$ and $emb_{ij}$ is the bounding box and object embedding for the $j^{th}$ detection in $f_i$. $emb_{ij}$ is used to relate it with the other objects over time and create the compact representation $R$.
% There is a set $J_i$ for every $f_i$ that includes all the detected object instances in $f_i$. $C_i$ is the list of corresponding categories for all the detections in the set $J_{i}$,
% % for each detection $j \in J_i$ and $J_i$ represents the set of all detections in $f_i$,
% $Prob_i$ are the probabilities of detecting the object categories $C_i$, $B_i$ is the list of the object bounding boxes for $J_i$, $P_{object,i}$ is the list of object poses estimated for all the detections. $P_{object,i}$ is estimated from the robot's pose $p_{robot,i}$ and limited depth information $d_i$, and is a 3D world coordinate. $Emb_i$ is the list of embeddings corresponding to the detections in $J_i$, which is used to relate $j's$ over time and create the compact representation $Q$.
% % Overall, $a_{ij} = \{ c_{ij}, prob_{ij} , b_{ij}, p_{robot,ij}, emb_{ij} \}$ represents the corresponding tuple of object centric information for the $jth$ object instance detection (such as a red cup) in the $f_i$ image frame.
% \iinote{check if the ratio between height ad width is an attribute}
% $p_{object,ij}
% \tn{$l_{ij}$ is undefined} in the map.
% we $s_i$ by passing through an object detector to extract minimal amounts of relevant abstract information $a_i = <C >$. $a_i$ includes $C$
% the output of a \st{pretrained} object detector including probabilities of detecting the object categories, their bounding boxes, and object poses are estimated from the robot's pose $(x,y,z)$ and limited depth information.
% \iinote{Move to next section:We use a color histogram \cite{color_hist} to approximate an embedding of each object instance to relate them over time. We select the color histogram method due to its simplicity, and design the later processing levels to handle a noisy instance embedding generator. Notably, our approach is not constrained to any number of objects, particular object size or shape, or object dynamicness (\textit{i.e.,} the objects' locations can change over time). We are only constrained by the object categories the detector was trained to detect.}
\subsection{Tier 1- Clustering across adjacent frames}
% \tn{increase font size for text outside boxes}
% \stnote{This paragraph is a really nice example of introducing the problem first.}
The extracted meta-information for the $jth$ detection in the $ith$ frame -- $a_{ij}$ is noisy because of the uncertainty in the robot's pose estimate $p_{robot,i}$ and noise in the extracted object-instance centric information -- $c_{ij}$, $prob_{ij}$, $emb_{ij}$. To aggregate the different view detections of the same object and to identify the unique objects in the environment, we consider a sliding window $w$ over sensory data $\{s_{i - n}, ..., s_i \} $. Object-centric information $a_{ij}$ of all detections in all the frames of this sliding window are then clustered to create a set of clusters $G$, each identifying a unique object instance. The motivation behind this is that the object instances with similar embeddings and positions in space will be associated to the same cluster hence pruning the noisy detections. We assign a unique id $instance_{id} \in ID$ to each cluster $g \in G$ and perform an aggregation operation $\bigotimes$ on every cluster $g \in G$ to create a unified feature representation of each of the cluster. This results in an aggregated representation $a_{g,\bigotimes}$ which is then inserted into a collection $T$ indexed by the unique id $instance\_{id}$ of each cluster where $ID$ is the set of all $instance_{id}$'s \iinote{fix this}.
% all detected object insta $AllDetects = \{ J_i,J_{i+1}, ....J_{i+n}\}$. We concatenate $a_{ij}$ for every detection $j$ that is part of the sliding window $w$. We then perform spatial clustering over these concatenated feature vectors $fv_{ij}$ for every $Dt \tn{should be $J_i$?} \in AllDetects$. Specific implementation details of the clustering algorithm can be found in Sec-\ref{sec-Algo Implementation}.
% % \iinote{move to next section: all information that can be extracted from each detection, in real-time}
% This will create $G$ clusters and each cluster can be considered a unique object instance for this sliding window $w$. For the subset of detections $Detects \subseteq AllDetects $ assigned to one cluster $g \in G$, we perform an aggregation operation on the $fv_{ij}$'s associated with $Detects$ based on the $weight= number \ of \ detections$ in the cluster, leading to a $weightedfv_{ij}$ associated with each cluster $g$. This aggregation operation will account for noise in the individual $a_{ij}'s$ across multiple detections. Specific implementation details of which can be found in Sec-\ref{sec-Algo Implementation}. Tier 1 - Aggregation gives us as output a set of clusters $G$ and their associated $weightedfv_{ij}$. We now need to keep track of these clusters in $w$ to associate them with what the robot has seen in the past, hence we store then in a table $T$ which will be indexed by an $instance_{id} \in ID$ that is unique to each cluster.
% The set of ID gets populated with a new unique $instance_{id}$ every time a new cluster $g$ is formed.
% \iinote{aggregation operation}
% \iinote{add DBSCan clustering algorithm}
% \stnote{make title shorter, and let latex put the table where it wants; it looks better if it's at the top of the page.}
\subsection{Tier 2 - Filtering across multiple sliding windows}
% \begin{figure}[t]
% \centering
% \includegraphics[width=0.6\linewidth]{images/Database_design.png}
% \vspace{-2em}
% \caption{Schematic Design of Database . *2 -Set of time intervals tuples containing start and end time.
% *3.- Associated with each time interval is a bounding box and key frame \tn{the font's a little small, change the green color to be darker, and what is PK \& FK?}}\label{fig:a}
% \label{fig:Schema}
% \end{figure}
% \stnote{introduce problem first}
We also need to aggregate the detections of unique instances across the objects' multiple partial views in the past. This requires sharing information across multiple sliding windows via a second-tier filtering. To facilitate this, we maintain a fixed-length short-term memory (STM) indexed over $ID$ and aggregate information in it for all instance id's in $T$.
% The ouput of Tier 1 ($ID$ in Table $T$) still needs to pass through more filtering since the detections of unique object instances need to be aggregated with the previous objects' multiple partial views for further compactness and association. This requires sharing information across multiple sliding windows via a second-tier filtering. To facilitate this, we maintain a fixed-length short-term memory $(STM)$ that is indexed over $ID$.
For an object represented by $instance_{id}$ in collection $T$,
we add a new entry to STM only if the object is a new unique object. Otherwise, if the object has previously been seen by the robot, an entry for this object should already exist in STM and should be updated.
We do this by finding a set of $nearIDs$ in STM that are closest in embedding space in cosine similarity\iinote{tell here?} and measuring the euclidean distance between them and $instance_{id}$ in physical space. There are three cases to be considered here that symbolize whether the object is new or not and whether it has moved or not: \textbf{1)} If $nearIDs$ set is non-empty and euclidean distance is less than a threshold $d_{thresh}$ then we perform another aggregation operation $\bigodot$ on $instance_{id}$ and the closest $n_{id} \in nearIDs$ to create a unified feature representation for $n_{id}$. This results in an aggregated representation $a_{n_{id},\bigodot}$ that is updated in $STM$. \textbf{2)} If $nearIDs$ set is non-empty and euclidean distance is greater than a threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). \textbf{3)} If $nearIDs$ is empty for $instance_{id}$, we also add a new entry in the STM with instance id $instance_{id}$. This represents the scenario where the object has not been seen before.
%%% every cluster $g \in G$ to create a unified feature representation of each the cluster.
%%%We do this by finding the set of instance id's $\textbf{nearIDs}$ in the STM that are closest to $instance_{id}$ in the embedding space. After which we calculate the Euclidean distance between between the pose of $instance_{id}$ and every element of $\textbf{nearIDs}$. If the normalized Euclidean distance is higher than the threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). If the Euclidean distance is below the threshold $d_{thresh}$, we perform an aggregation operation over the metadata $weightedfv_{ij}$ of $instance_{id}$ and the closest $n_{id} \in \textbf{nearIDs}$. The old $weightedfv_{ij}$ of $n_{id}$ is updated with the new $weightedfv_{ij}$.
%%%We do this by finding the set of instance id's $\textbf{nearIDs}$ in the STM that are closest to $instance_{id}$ in the embedding space. After which we calculate the Euclidean distance between between the pose of $instance_{id}$ and every element of $\textbf{nearIDs}$. If the normalized Euclidean distance is higher than the threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). If the Euclidean distance is below the threshold $d_{thresh}$, we perform another aggregation operation $\bigodot$ on $instance_{id}$ and the closest $n_id \in \textbf{nearIDs}$ to create a a unified feature representation for $n_id$. This results in an aggregated representation $a_{n_{id},\bigodot}$. There is no constraint on two aggregation operation's $\bigotimes$ and $\bigodot$ to be the same. For our implementation they are same and described in sec-\ref{sec-Algo Implementation}.
% For a given query $Q$, D3A returns a small set of keyframes from its queryable representation $R$ to find relevant information about the target object.
% After the aggregation operation, the The keyframes also need to be associated with the updated $n_{id}$ or newly added $instance_{id}$ because for a given query $Q$, D3A returns a small set of keyframes from its queryable representation $R$ to find relevant information about the target object.
% The reason behind making $R$ compact is to allow for speed efficient object retrieval. The responses retrieved from $R$ will be a subset of relevant keyframes associated with unique objects in the environment. \tn{i'd cut the previous 2 sentences bc they seem out of place. also i know that you introduced Q in the intro paragraph for the section but by this point i've forgotten what it is and would need a reminder so it doesn't seem like it's coming from no where}
After the aggregation operation, the keyframe of $n_{id}$ is chosen to be the frame with the highest detection probability $prob_{ij}$ among the frames $f_i's$ associate with $instance_{id}$ and $n_{id}$. If a new instance is to be added to the STM when it has reached its maximum capacity, our algorithm evicts the last recently viewed (least-recently viewed entry), $lrv\_entry$, from the STM and moves it to the persistent storage key-value store $R$. This eviction strategy ensures that objects currently being viewed by the robot remain in the STM for further noise filtering and aggregation of meta-data from partial views.
% \iinote{we do it for all the $instance_{id}'s$ in the window.} \iinote{key-value database}
\subsection{Tier 3- Filtering across database entries} \label{subsubsection:p3}
The filtered entry $lrv\_entry$ from the STM could be directly added as the final aggregated cluster into our compact representation $R$. However, we want to aggregate detections of unique object instances over not just recent times but also long periods of time.
% However, we want a more compact representation that aggregate detections of unique object instances over long periods of time.
To do so, every time a $lrv\_entry$ is evicted from STM, our third-tier filtering level looks up and updates the aggregated clusters stored in $R$. We organize our aggregated spatial-temporal representation $R$ in a key-value database over two collections. The \textbf{Object Identification Collection} ($OIc$) is updated using the object information of the evicted clusters represented by the $lrv\_entry$, while the other store \textbf{Spatial-temporal Collection} ($STc$) stores the object entries indexed over time and position in physical space. Every object inserted in $OIc$ is associated with a unique identifier $ObjectID$ which is used to index $STc$ to get the object detections over space and time.
To update $R$, we find the set $Records \in OIc$ that are the closest in feature space with $lrv\_entry$ in cosine similarity. A new document is inserted into $STc$ if the normalized Euclidean distance between last evicted entry from STM $lrv\_entry$ and $Records \in STc$ is greater than $d_{thresh}$, otherwise the object information in the closest document $record \in Records$ is aggregated, via an operation $\oplus$, applied on the $lrv\_entry$ and the document $record$ hence updating the object information in $OIc$ using this aggregated output. For every new entry in $OIc$, an entry is added in the $STc$ against the $ObjectID$ containing the time and physical location of the new entry.
% The schematic design of our spatial-temporal represen
% The \textbf{Object Identification Collection} ($OIc$) stores object attributes of aggregated clusters in $STM$ such as the category $C_R$, embedding $Emb_R$ and weight $Weight_{R}$ \iinote{what to do about this weight?}. We also have a separate \textbf{Spatial-temporal Collection} ($STc$) that stores object entries indexed by the object's location in the environment $P_{object,R}$, time intervals $Time_R$, keyframe $K_R$, detection probability $Prob_R$, and the bounding box $B_R$ of the object in the keyframe. Such an inverted index schema for $R$ supports the flexible update and addition of multiple locations for the same object instance, with a one to many mapping from $OIc$ to $STc$ and a unique identifier $ObjectID$ in $OIc$ that can be used as a referential foreign key in $STc$.
% % Such an inverted index schema supports the flexible updating and adding of multiple locations for the same object instance, with a one to many mapping from $OIc$ to $STc$ with unique identifier $ObjectID$ in $OIc$ being a foreign key in $STc$.
% % \tn{Where did $e$ come from?}
% In our third-tier filtering, we find the set $Records \in OIc$ that are the closest in feature space with $lrv\_entry$ in cosine similarity. A new document is inserted into $STc$ if the normalized Euclidean distance between last evicted entry from STM $lrv\_entry$ and $Records \in STc$ is greater than $d_{thresh}$, otherwise the closest document $record \in Records$
% % \tn{is D the set of all documents? You might want to remove it bc it never got introduced}
% is updated with the result of an aggregation operation $\otimes$ applied to the spatial and the embedding vector of $lrv\_entry$ and the document $record$. The document in $OIc$ (with the same $ObjectID$) is updated with the aggregated embedding
% The schematic design of our spatial-temporal representation built in MongoDB is shown in Figure \ref{fig:Schema}.
% We organiz
% We use \textit{MongoDB}, which uses a special geospatial index to store the object instances. This makes the spatial-temporal lookup of objects very efficient. The schematic design of our database is shown in Figure-\ref{fig:Schema}. We choose not to directly insert $e$ into the database because we want to ensure the unique object instances are associated over long-time horizons hence reducing redundancy. In our third-tier filtering, we look up the object id's $obj_id$ from the \textbf{Object Identification Collection} that are closest in feature space with $e$ under cosine similarity. If the $obj_id$ is empty, we insert a new $obj_id'$ associated with $e$, symbolizing that this is potentially a new object instance not seen by the robot. Else we again perform weighted average of the feature of $obj_id$ and $e$ and update the \textbf{Object Identification Collection}. We then look-up set of documents $D$ in the \textbf{Spatial-temporal Collection} that have the same associated $obj_id$. A new document is inserted in the collection if the normalized Euclidean distance between $e$ and $d \in D$ is greater than the $d_thresh$, else the closest document $d' \in D$ to $e$ is updated with the weighted-average spatial information. The keyframe of this $d'$ is chosen to be the frame with the highest probability amongst the frame associate with $d'$ and $e$.
% \tn{The intro text at the beginning of the ``System Overview" section made it seem like storing the data in the database should be its own subsection, but it's already included in the 3-tier processing subsection right? If that's the case, you should edit the intro text to group processing and storing the data together.}
\subsection{Query Processing and Answering}
\begin{table}[]
\small
\caption{Query Types, Language Examples \& Formal Representations}
\centering
%\stnote{Can you add a column for the formal representation of the query?} \iinote{I had that previously but it was taking too much space so removed it? Will it be okay if we dont have it?}
\begin{tabular}{lp{3.5cm}p{3cm}}
\toprule
\bf{Type} & \bf{NL Example} & \bf{Formal Representation} \\
\midrule
Q1 & Did you ever see an orange bowl? & ${orange(x), bowl(y) |}$ ${ OIc.find(x,y) }$\\
\hline
% Q2 & { "Where was the $attr1$ pink cup"?} \\
% \hline
Q2 & When were the cup and bowl seen? together? & ${bowl(x), cup(y) |}$ ${STc.find(}$ ${OIc.find(x,y), together)}$\\
\hline
Q3 & Where did you see the pink cup between 1:00pm and 4:00pm on 1st Jan, 2020? & ${pink(x), cup(y), time(t) |}$ ${STc.find(OIc.find(x, y),}$ ${1600 >= t >= 1300)}$\\
% \hline
% Q4 & Where is the Sprite bottle \\
\bottomrule
% \label{example_queries}
\end{tabular}
\label{tab:example_queries}
\vspace{-6mm}
\end{table}
Natural language processing methods such as template-based \cite{li2017querying} or deep learning \cite{DBLP:journals/corr/abs-1911-04942} methods\tn{probably need citation} can be used to map a language query to a set $X$ of $ObjectID's$, time intervals, and location areas, which D3A takes as input for object retrieval. Table \ref{tab:example_queries} shows the types of queries, their natural language examples, and formal representations used in our evaluation. This formal representation is used to search over indexed $OIc$ and $STc$ collections of $R$. The spatial-temporal representation built through our method can handle a wide variety of queries, including: querying for objects with/without the following: 1) specific attributes such as ``orange bowl," 2) spatial relationships with other objects such as ``cup and bowl together," and 3) time slices which can be either a point in time or a time interval. Our representation can also efficiently handle negative queries where the answer does not exist in the dataset.
% objects with/without 1) specific attributes, 2) spatial relationship among objects, 3) the temporal relationship among objects, and 4) time slices.
% The query input to RoboMem is a combination of $ObjectID's$, time intervals and location area. This set can be generated from Natural Language Query and sample object image. However, our current system takes in a templatized query. The schematic design of the database can handle a variety of queries as follows: Q1, Q2: With specific attributes (for static objects and dynamic objects), Q3) and Q4) With time slices(for positive and negative query). A negative query is the one for which the answer in the database does not exists. Q5: with a query about latest time stamp, Q6, Q7: with spatial relationship among objects. Fig-\ref{fig:block-diagram} show examples of the query types in natural language and their corresponding DB query.
% 3) the temporal relationship among objects, and 4) time slices.
% \begin{enumerate}
% \item With specific attributes: "Where was the $attr1$ $catg$"? mapped to
% \begin{verbatim}[scale =0.99]
% Q1: object\_id=collection.find({Category: attr1}, feature vector: attr1)
% collection.find({obj\_id : object\_id})
% \end{verbatim}
% \item 2) spatial relationship among objects, 3) the temporal relationship among objects, and 4) time slices.
% \end{enumerate}
\section{Algorithmic Implementation} \label{sec-Algo Implementation}
% \iinote{talk about similarity measure}
% In this section we describe the algorithmic implementation for the real world deployment of D3A and what param
% \subsection{Algorithmic Implementation}
We deploy D3A on a mobile robot Kuri \cite{heykuri_2018} that patrolled a robotics lab environment for four days. Our system performs real-time object detection with Facebook's AI Research software system - Detectron \cite{Detectron2018} to extract $c_{ij}, prob_{ij}$ for every detection. We use a color histogram \cite{color_hist} to approximate an embedding $emb_{ij}$ of each object instance detection. We select the color histogram method due to its simplicity. However, our algorithm is not constraint to using the color histogram as the instance feature extractor. More sophisticated deep learning methods that use RGB or RGB-D images \cite{bras2018unsupervised, xu2014deep} can also be used. In Sec-\ref{sect:falsepositiveexp}, we show how varying the type of object detector (changing its false positive rate) affects the results of our algorithm. Notably, our approach is also not constrained to any particular number of objects, object size or shape, or object dynamicness (\textit{i.e.,} the objects' locations can change over time). We are only constrained by the object categories the detector was trained to detect. Processing of the visual data and pose estimates was done off-board on a stand-alone machine. The PC setup included an Intel(R) Core(TM) i7-6700 CPU (3.40GHz x 8), and a GeForce GTX 1070/PCIe/SSE2 GPU running Ubuntu 16.04. D3A was run online while the PC communicated with the Kuri robot over ROS \cite{quigley2009ros}. For Tier 1-clustering of detections across adjacent frames we use density-based spatial clustering (DB-Scan) \cite{khan2014dbscan}, a method known to be good at handling outliers/noise within the dataset. While other clustering methods can also be used, we defer the analysis of their impact on D3A's performance for future work. All the aggregation operations $\odot$, $\oplus$ and $\otimes$ that we applied are the weighted average functions. These aggregation functions can also be different. Weighted average was selected because of its simplicity and reasonable empirical performance for object retrieval on our smaller dataset of $<3$ hours of data. More sophisticated methods can also be used in place of the weighted average. However, as shown in Sec-\ref{sec:exp-des}, with just the weighted average, D3A still performs better than the baseline. We organized our aggregated spatial-temporal representation $R$ over two collections in \textit{MongoDB} \cite{banker2011mongodb}, a database that supports large volumes of both data and traffic. The two document Mongo-DB collections \textbf{Object Identification Collection} ($OIc$), and \textbf{Spatial-temporal Collection} ($STc$) are as described in Sec-\ref{subsubsection:p3}. $OIc$ in our implementation, stores the object information using the associate embedding of the cluster, the category assigned to the cluster and a weight which encodes the frequency of detections for this object and is used in the aggregation operation between $record$ in $OIc$ and $lrv\_entry$ evicted by the $STM$.
% \tn{the previous sentence doesn't make sense? Do you mean something like ``The two collections, $OIc$ and $STc$ are as described in section..."?}
For natural language processing of the queries, we annotate the queries (use the template-based method) to convert them into the logical form shown in Table \ref{tab:example_queries}. These logical forms are then used to query the collections for relevant spatial-temporal information. In \label{sec:exp3} we show how the performance of D3A changes with varying the uncertainty in the processing of the query but still it is able to perform better than the baselines.
% \iinote{We organize our aggregated spatial-temporal representation over two collections in \textit{MongoDB} \cite{banker2011mongodb}, a database that supports large volumes of both data and traffic. Our first collection in MongoDB is the \textbf{Object Identification Collection} ($OIc$) that stores object attributes of aggregated clusters such as the category, embedding and weight. We also have a separate \textbf{Spatial-temporal Collection} ($STc$) that stores documents containing a geo-spatial index for location, time intervals, the keyframe, and the detection probability and bounding box of the object in the keyframe.}
% \iinote{We annotate the queries to convert to the database query representation.}
\section{Evaluation}
% \stnote{The aim of our evaluation is to test the hypothesis that algorithm X improves the precision and recall of a database-backed system and answering queries.}
The aim of our evaluation is to test the hypothesis that a database-backed system with our algorithm D3A improves both 1) the compactness of the spatial-temporal representation of objects in the environment and 2) the object retrieval performance by returning a small subset of keyframes as measured by the mean reciprocal rank, miss rate and retrieval time as described in Experiment Design subsection.
% Sec-\ref{sec:exp-des}.
As a result, the user will have to search over just a few returned results to find the answer to their question.
To test our hypothesis, we perform a real-world evaluation, using both a range of synthetic detectors built from ground truth data to control the noise level and real detectors deployed on a mobile robot to assess the system's end-to-end accuracy.
% This is directly related to improved user experience.
%%%\stnote{So instead of telling that, can you like SHOW that? As a result, the user will have to search few just a few returned results to find the answer to their question.} \stnote{To test this hypothesis, we perform a real-world evaluation on a mobile robot using both simulated detectors built from ground truth data to test the noise and real detectors to assess the end-to-end accuracy. (PS, I am writing this fast, so feel free to reword/clean up any suggested sentence I put. But it's good to have something to say like, what the evaluation is going ot be and why it will answer our hypothesis.) }
% Our main results are:
% \beging{}
% and hence the recall\footnote{Defined as a function of true positives (tp: key frame containing the object of interest in the query), false negatives (fn: objects failed to be identified and retrieved). It is the fraction of relevant documents that are retrieved, in particular Recall = \frac{tp}{tp+fn}}
% measure for evaluatingreturned key frames to a sample of queries, ordetred by probability of correctness. The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. If none of the proposed results are correct, reciprocal rank is 0.}
% with varying range of sensor uncertainty for retrieved objects.
\subsection{Dataset Collection}
% \stnote{don't put colons after subsection names}
% We deployed D3A on a mobile robot Kuri \cite{heykuri_2018} that patrolled a robotics lab environment for four days.
The robotics lab environment in which our mobile robot Kuri \cite{heykuri_2018} patrolled for four days was uncontrolled and cluttered: people were allowed to use the space as is. For consistency, we kept the illumination same throughout the data collection(it was captured in a completely lit lab). The robotics lab area used for data collection and experimentation included the kitchen area and the general area with tables and chairs. A visualization recording of the initial state of the environment, along with its 2D map and sample raw sensory data can be found at \url{ https://github.com/IfrahIdrees/D3A.git}. This real-world dataset has 10,132 image frames over 22 hours and contains both static and dynamic objects such as various cup and bottles changing there location. The dataset details can be found in Table \ref{tab:my_label}.
% \iinote{for consistency keep the illumination same(captured during daytime.)}
% \stnote{cut: To our knowledge this is currently the longest dataset collected by a robot.} \stnote{No way. For example see the MIT SLAM dataset. And all the Waymo selfdriving data. }
\begin{table}[httb]
\small
\centering
\caption{Dataset Details}
\vspace{-3mm}
\begin{tabular}{l r r}
\toprule
Duration &22 hrs & 3hrs \\
\midrule
Total Number of Frames & $10,132$ & $991$\\
Frame Rate (per minute) & $7.67$ & $7.67$\\
Sensor Data Size (GB) & 10.53 & $1.43$\\
Total Number of Detections & $13,565$ & $2,558$ \\
Ground Truth Objects & N/A & $59$ \\
& & Static = 49, Dynamic = $10$\\
Number of Times & N/A & Mean = $2.1 \pm 1.3$, \\
Dynamic Objects move && Max = 5 \\
\bottomrule
\end{tabular}
\label{tab:my_label}
\vspace{-3mm}
\end{table}
\subsection{Dataset/Object Annotation}
% \stnote{No colons in section names.}
% \stnote{I take back the comment in the intro about 22 hours vs 3 hours, but we should make sure we use the 22 hours somehow in the evaluation. }
For evaluation purposes, the collected dataset was manually annotated by the authors. Every unique instance in the dataset was assigned a unique id and a ground truth location that was used to perform object association over time and space. Due to time and manual labour constraints, we were only able to annotate 3 hours of data.
\subsection{Parameter Selection}
% \stnote{No colons in section names.}
% \iinote{ We use a color histogram \cite{color_hist} to approximate an embedding of each object instance to relate them over time. % and reduce the set of potentially similar objects. We select the color histogram method due to its simplicity,}
The Euclidean distance threshold $d_{thresh}$ , sliding window length, normalized threshold for feature matching (cosine similarity), and size of short-term memory (STM) were set to 0.5m, 10, 0.4, and 400 (our RAM's maximum capacity), respectively. We performed a parameter sweep offline on the collected data and found this setting to be optimal. Increasing the feature matching and distance threshold increased the number of false positives during tier-1 processing. For the STM, the greater the buffer size, the more noise there will be in the object's pose and embedding to be filtered and processed.
% The selected parameter values and the justification for each are as follows:
% 1) The Euclidean distance threshold $d_{thresh}$ was set to 0.5 meters to account for the 0.5m standard deviation in the robot pose estimation. We performed a parameter sweep and found that setting the sliding window length = 10 and normalized threshold for feature matching = 0.4 to be optimal. Increasing the matching and distance threshold increased the number of false positives during tier-1 processing.
% 2) The short-term memory (STM) buffer was set to 400 objects, our RAM's maximum capacity. The greater the buffer size, the more noise there will be in the object's pose and embedding to be filtered and processed.
% We also performed a parameter sweep over different lengths of the buffer and found the results to be not impacted
\subsection{Experiment Designs}\label{sec:exp-des}
We conduct four experiments with two baselines. One experiment is a running example comparison.
% We conduct three \tn{four?} sets of quantitative experiments with a running example and two baselines (as described below).
% \stnote{Don't give results here until after you describe the algorithms. So move those two sentences later.}
\textbf{Baselines}: Our first baseline is a ``Naive" method that inserts every detection's spatial-temporal information directly into the database without aggregation. This baseline ablates the three-tier processing that is integral to our system for aggregating partial views of the objects, but can still index detections on space and time. We also conduct an ablation of the spatial information from the features to make a ``Non-spatial" baseline, similar to \citet{hsieh2018focus} which does not utilize spatial information during clustering, to compare the clusters resulting from both methods.
% We do another ablation study where we removing the spatial information used by D3A for three-tier clustering. We refer to this method as ``Non-spatial''. This is similar to \cite{hsieh2018focus} wich uses a similar clustering method but does not use the spatial information.
% For an ablation study with \citet{hsieh2018focus}, we implement a ``Non-spatial'' baseline, removing the spatial information used by D3A for three-tier clustering. We use a running example to compare D3A with this baseline and visualize the differences in their aggregated clusters.
%%%\stnote{I would rephrase that a bit, and say first, we do the abalation, and then, this is similar to hsieh whichuses a similar clusting method but does nto use spatial information}
% will show \tn{that clustering by spatial information improves...} how D3A improves the aggregation of detections.
% In the first, we show the changing trend in retrieval performance of our system relative to the uncertainity of the parameterized synthetic detector as compared to our baseline.
% We perform two separate ablation studies that extensively assess the improvements from our framework.
\textbf{Metric Definitions}:
We measure the quality of keyframes returned for a query with the MRR@50 evaluation metric, which is the multiplicative inverse of the rank of the correct keyframe in the top 50 returned frames ordered by their probability of correctness. We measure the miss rate (the number of objects missed by the algorithm during the clustering phase due to detection or classification error, hence cannot be found in the returned frames) to get an idea of our accuracy. We also report the speed efficiency of our system by measuring the mean number of frames and object id's returned for the queries, total time taken by a query divided into retrieval time and evaluation time to find the correct frame by matching against the ground truth.
% \begin{itemize}
% \item To measure how good the retrieved key frames are, we measure MRR@50 \footnote{A measure for evaluating the returned keyframes for a sample of queries, ordered by probability of correctness. The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. The first correct answer in our case is the first keyframe that contains the queried object instance. If none of the proposed results are correct, the reciprocal rank is 0. MRR values closer to 1 are preferred as they signify high retrieval precision where the set of returned keyframes contains the relevant response for the query}, Miss Rate which is equivalent to recall(the percentage of time our algorithm fails to recognize unique object instances). We use the miss rate to define the accuracy of the system as 1- average(Miss Rate).
% \item to measure the speed efficiency we measure, the the mean number of frames and object id's returned, 5) mean retrieval time from database, and 6) total evaluation time to check the returned keyframes against the ground truth annotation for all queries. Higher number of these metric mean longer times for the human user to search over the returned frames.
% % \item We define our system's accuracy as If the set of returned keyframes and their corresponding spatial-temporal information contains a frame with the answer to the query, we say that the query has been answered accurately.
% \end{itemize}
% We measure a number of metrics for the retrieval performance distributed over all experiments: 1) the mean reciprocal rank (MRR), and 2) the object miss rate --- the percentage of time our algorithm fails to recognize unique object instances \stnote{Is this recall? Or something different. I would explain how this si different from recall}. We also report 3) the mean number of frames returned, 4) mean number of object id's returned, 5) mean retrieval time (to retrieve keyframes and spatial-temporal data from the database), and 6) total evaluation time taken by the system (to check the returned keyframes against the ground truth annotation for all queries). \stnote{I think I wouldn't do this in a list, but rather would introduce WHY you need each metric, and then define it if necessary.} Our system's accuracy is defined as follows: since the query matching is uncertain, we return the top 50 frames for every query and rank them according to the \tn{their?} weighted average probability for the queried object stored in our database.
% which the object detector detects the category of the objects \tn{the last part is confusing, do you mean you rank them by the object detector's detection probability for the queried object?}.
% We then iterate over the frames to find the one that contains the answer to the query. If the set of returned keyframes and their corresponding spatial-temporal information contains a frame with the answer to the query, we say that the query has been answered accurately.
\begin{figure}[t]
\centering
\includegraphics[width=0.8\linewidth]{images/noise.png}
\caption{Exp1 - Retrieval performance with a synthetic embedding generator with increasing uncertainty}
% \label{fig:a}
\label{fig:graph_noise}
\vspace{-5mm}
\end{figure}
% \begin{table*}[!htbp]
\begin{table*}[httb]
\small
\centering
% \caption{\tn{Missing table caption. I'd suggest adding $\uparrow$ and $\downarrow$ next to the metric names to help the reader parse the numbers a bit easier ($\uparrow$ means higher is better and so on)}}
\caption{Exp 2 - Retrieval Performance for the three types of queries. $\uparrow$ means higher is better, $\downarrow$ means lower is better.}
\vspace{-2mm}
\label{table:queryperformance}
% $*^1$refers to the Naive baseline without D3A, $*^2$is D3A + synthetic Perfect embedding generator, $*^3$is D3A + Detectron (real world detector) with a color histogram as the embedding generator
\begin{tabular}{@{\makebox[2em][r]{\textcolor{airforceblue}\rownumber\space\space\space}} l p{1.2cm} p{1.5cm} | p{0.9cm} p{1.8cm} p{1.9cm} l p{1.9cm} p{2cm}}
\toprule
\multicolumn{1}{@{\makebox[3em][r]{}}} & \textbf{Type} & \textbf{Query Precision} & \textbf{Aggregation Method} & \textbf{Miss Rate}$\downarrow$ & \textbf{Mean Number of Frames} $\downarrow$ & \textbf{Mean Number of Object IDs} $\downarrow$ & \textbf{MRR @50} $\uparrow$ & \textbf{Mean Retrieval Time (ms)} $\downarrow$ & \textbf{Total Evaluation Time (ms) $\downarrow$}\\
\midrule
% \multirow{3}{*}{Q1} & perfect & Perfect & $0.00$ & $1.08$ & $1.00$ & $1.00$ & $0.03$ & $8.38$\\
% & perfect & Category & $0.24$ & $5.88$ & $3.75$ & $0.35$ & $0.89$ & $104.82$\\
% & perfect & Any object & $0.88$ & $38.00$ & $33.00$ & $0.07$ & $1.04$ & $2,031.02$\\
\multirow{9}{*}{Q1} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 10.30 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 3.93 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 10.50 & 1.00 & 1.00 & 0.02 & 6.63 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.33 & 792.73 & 339.00 & 0.19 & 22.24 & 2,567.62 \\
& & D3A+P & 0.20 & 6.29 & 4.50 & 0.33 & 0.96 & 101.31 \\
\rowcolor{LightCyan}
& & D3A+D & 0.20 & 12.43 & 1.50 & 0.29 & 0.94 & 104.95 \\
\cline{2-9}
& \multirow{3}{*}{Any object} & Naive & 0.93 & 2,558.00 & 2,558.00 & 0.01 & 119.74 & 926.67 \\
& & D3A+P & 0.20 & 38.00 & 33.00 & 0.24 & 0.82 & 116.15 \\
\rowcolor{LightCyan}
& & D3A+D & 0.13 & 75.00 & 25.00 & 0.20 & 1.12 & 89.19 \\
\hline
\multirow{9}{*}{Q2} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 2.55 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 6.94 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 10.20 & 1.00 & 1.00 & 0.03 & 23.23 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.88 & 861.00 & 861.00 & 0.04 & 25.99 & 2,108.60 \\
& & D3A+P & 0.31 & 7.50 & 7.00 & 0.31 & 1.63 & 55.81 \\
\rowcolor{LightCyan}
& & D3A+D & 0.38 & 18.00 & 3.50 & 0.18 & 1.54 & 73.26 \\
\cline{2-9}
& \multirow{3}{*}{Any object} & Naive & 0.94 & 2,558.00 & 2,558.00 & 0.01 & 117.96 & 996.27 \\
& & D3A+P & 0.25 & 38.00 & 33.00 & 0.13 & 0.85 & 173.65 \\
\rowcolor{LightCyan}
& & D3A+D & 0.38 & 75.00 & 25.00 & 0.10 & 1.14 & 153.19 \\
\hline
\multirow{9}{*}{Q3} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 5.05 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 4.15 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 5.50 & 1.00 & 1.00 & 0.02 & 3.53 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.67 & 96.00 & 96.00 & 0.25 & 336.60 & 110.58 \\
& & D3A+P & 0.27 & 1.50 & 1.50 & 0.36 & 1.60 & 9.20 \\
\rowcolor{LightCyan}
& & D3A+D & 0.47 & 1.00 & 1.00 & 0.33 & 1.35 & 5.68 \\
\cline{2-9}
&\multirow{3}{*}{Any object} & Naive & 0.86 & 684.00 & 684.00 & 0.08 & 2,723.35 & 160.27 \\
& & D3A+P & 0.14 & 14.00 & 14.00 & 0.21 & 13.25 & 31.45 \\
\rowcolor{LightCyan}
& & D3A+D & 0.07 & 12.00 & 12.00 & 0.26 & 11.51 & 28.93 \\
% \cline{2-9}
\bottomrule
\end{tabular}
\vspace{-3mm}
\end{table*}
\subsubsection{Exp 1 - Retrieval Performance with a Synthetic Embedding Generator with Increasing Uncertainty} \label{sect:falsepositiveexp}
The performance of D3A depends on the accuracy of the object detector and object pose estimates. In this experiment, we design a synthetic embedding generator that replaces the object detector and color histogram. Our synthetic embedding generator is parameterized with a false positive rate ($fpr$)\footnote{fpr denotes the probability of falsely labelling the detected instance} and false negative rate ($fnr$)\footnote{fnr denotes the detector's probability of not detecting the object}. We keep the $fnr$ low since we assume that the object is not in the environment if the detector fails to detects the object. We annotate ground truth object embeddings as one-hot encodings $ohe$ of the unique instances, where each index represents the unique id of a ground truth object. We then use the $fpr$ and $fnr$ to flip the bits of the $ohe$ and return the transformed encoding as the synthetic embedding generator's output.
%\stnote{You don't need to give pseudocode for this if you need more space.} \tn{+1}
% We measure the MRR and Miss Rate with varying the $fpr$ - level of uncertainty of our Embedding Generator and compare it with the ``Naive" baseline.
\textbf{Results}: We tested our system on Q1 type - \textit{Did you ever see an orange bowl?} for all ground truth object instances as mentioned in Table-\ref{tab:example_queries}. Results are shown in Fig-\ref{fig:graph_noise}. Our method's MRR@50 decreases as $fpr$ increases. Despite this trend, D3A's MRR@50 remains considerably higher than Naive's MRR@50. This indicates that even when the object detector used in D3A is not good, the user will always find the correct object's frame earlier with D3A than with the Naive baseline. D3A's Miss Rate increases as the synthetic detector becomes noisier, but is always lower than Naive's Miss rate.
The slight decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with a higher probability of flipping the bits of the $ohe$, the probability of different object instances being assigned a similar embedding also increases. This causes D3A to consider a higher proportion of object instances to be similar, leading to more cluster formations (because of different locations) and storage of their respective keyframes. At the time of retrieval for a given object, this causes more keyframes to be returned one of which includes the queried object.
% The slight decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with a higher probability of flipping the bits of the $ohe$, the probability of different object instances being assigned a similar embedding also increases. This causes D3A to consider a higher proportion of object instances to be similar even when they are not, leading to more detections being used for cluster formation, increasing the number of clusters formed and greater storage of their respective keyframes. At the time of retrieval this causes keyframes to be returned of more different objects and at least one of those frames includes the relevant object.
\tn{i don't get how this means a lower miss rate}\iinote{improve this?}
% \textbf{Results}: We tested our system on all the query types types in Table \ref{example queries} and measure the average MRR and Miss Rate for all three.
% The MRR@50 of our method shows a decreasing trend as $fpr$(noise level) increases and remains above the baseline. This indicates that even if the synthetic detector is not good the user always finds the correct key frame earlier than the Naive. The decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with higher probability of flipping the bits, the probablity of different object instances being assigned a similar embeddings also increases that is D3A considers more object id's to be similar, clustering and storing their keyframes more often.
% of the instances among clusters to become highly similar, recognizing causing it to detect objects which are not in the scene.
% We believe that the noise level of Naive remains constant as every independent detection even if noisy is added to the database and when searching for ground truth instances, some frame amongst the top 50 frames returned contains a correct detection.
% --------------------------------------------
% We tested our system on query type 1 in Table \ref{example queries}. \stnote{I would just name the query again here, e.g., Did you ever see an orange bowl?}
% As the $fpr$ of our synthetic object detector increases, the MRR@50 of the retrieved frames for every $gt$ decreases and the Miss rate increases until it hits 0.3 after which the Miss Rate drops, as shown in Figure \ref{fig:graph_noise}. This is because fliping of the bits in the ohe happens so much the probability of detections having $ohe$ with similar cosine similarity increases.
% is wrongly assigned some other detection
% there is no clustering going on and every mutated object represents detection of some other object. This causes a lot of wrong detections which are present in the database and end up being in the top 50 answers for queries.
% Except for the peak at around $fpr=0.3$, the Miss Rate of D3A is lower than the Miss Rate of `Naive', showing that D3A is able to accurately answer queries for more ground truth object instances than the Naive. The MRR@50 and Miss Rate for the `Naive' remains constant despite the changing noise level because of the intrinsic distribution of objects in our dataset. Flipping of the bits assigns the object instances ids
% As the bits are flipped more with increasing fpr, the
% Beyond a $fpr$ of $0.2$, D3A starts performing worse than the ``Naive" baseline. The noise in our deployed system with a real object detector is $0.18$. As the $fpr$ increases, \stnote{cut: the reason that} the Miss rate increases \stnote{cut:is} because the probability that the instance will be assigned the same $ohe$ decreases, and D3A starts to label these new instances as either noise (since it has only seen them in a frame of a sliding window) or instances of another category. \stnote{Can you compare this to the baseline? I'm confused because in Figure 4 it looks like the baseline's performance doesn't change with regard to noise level...}
% D3A starts to consider the objects with similar vector less often hence labelling it as noise or labelling it as some other category which is an intrinsic property of the increasing fpr of synthetic detector.
% \stnote{make this a float, with an algorithm block and a title}
% \begin{lstlisting}[language=Python, caption={Embedding Generator Implementation}, float, label=lst:sensor]
% for indices not groundtruth_index:
% if prob < self.false_positive_rate:
% one_hot_encoding[i] = 1
% else:
% one_hot_encoding[i] = 0
% for groundtruth_index:
% if prob < 1- self.false_negative_rate:
% one_hot_encoding[groundtruth_index] = 1
% else:
% one_hot_encoding[groundtruth_index] = 0
% \end{lstlisting}
\subsubsection{Exp 2 - Query Retrieval Performance}\label{sec:exp3}
Our second experiment compares the retrieval performance of
\iinote{two versions of D3A against the Naive Baseline. The first is $D3A+D$ - D3A with a real detector, Dectectron \cite{Detectron2018}, and color histogram and second is $D3A+P$ - D3A with a synthetic perfect sensor.}
D3A with a real detector -- Dectectron \cite{Detectron2018} and color histogram ($D3A+D$) against D3A with a synthetic perfect sensor ($D3A+P$) and the ``Naive" baseline.
We experimented over the 3 query types in Tablereal detector – Dectectron (Girshick et al. 2018) and color \ref{tab:example_queries} and the 59 ground truth object instances annotated in the data. The number of instances per query ranged from 15 -20 depending on the query, making the total number of queries to be 150.
% \tn{How does the math work here? 3*59 is not 426, and idk where the 15-20 fits in..}
% For the given \iinote{70} queries, we test the retrieval performance our \textit{deployed system} - on Kuri with real object detector.
\textbf{Query Matching:} Another factor affecting the retrieval performance in this experiment is the matching of the desired object in the natural language query to an $ObjectID$ in the database. To examine this axis, we discretize the query matching precision into three levels:
\begin{itemize}
\item \textit{Perfect}: The query matching component can map an object in the language query to an exact $ObjectID$ in the database, and removes any level of uncertainty.
\item \textit{Category}: The matching component is noisier and is unable to uniquely identify the queried object (\textit{e.g.,} orange bowl), and instead returns a set of $ObjectID's$ for all objects from the same category (\textit{e.g.,} bowl).
\item \textit{Any Object}: The matching component is the noisiest and cannot extract even the category-level information from the language query, and thus maps the query to the most generic set of all the $ObjectID's$ in the database.
\end{itemize}
% We report MRR for the top 50 returned frames for \textbf{U, C, A} labelled as MRR (N=50) in Table-\ref{table:results1}.
\textbf{Results:} The results are shown in Table \ref{table:queryperformance}.
With \textit{Perfect Query Precision} for all three types of queries Q1 [\textit{rows: 1-3}], Q2 [\textit{rows: 10-12}], and Q3 [\textit{rows: 19-21}], both the mean retrieval time and total evaluation time of $D3A+D$ are less than those of the ``Naive" baseline. On average, $D3A+D$'s total (retrieval + evaluation) time is \textbf{1.96x} slower than $D3A+P$ and \textbf{1.48x} faster than the baseline. This translates to improved user experience as the users' query will be answered more quickly and as expected the miss rate is 0 because of the perfect query matching precision.
With \textit{Category Query Precision} for Q1 [\textit{rows: 4-6}], Q2 [\textit{rows: 13-15}], and Q3 [rows:\textit{22-24}], the mean number of frames returned by $D3A+D$ is \textbf{55x} smaller than that of the ``Naive" baseline, and only \textbf{5 counts}
% \tn{is this correct? you should double-check your numbers just in case}
higher than $D3A+P$'s on average. The MRR@50 for $D3A+D$ is \textbf{0.26}, which signifies that if the queried object exists in the database, it will on average be found in the fourth frame. An empty set is returned for negative queries. While the baseline on average finds the object in the sixth frame, the probability of it not finding an object of interest in the first 50 frames (average miss rate) is \textbf{62.5\%}, which is really high compared to $D3A+D$'s \textbf{34.7\%}. On average, $D3A+D$ took \textbf{23ms} to retrieve and evaluate all the frames for all the object instances while the baseline took \textbf{500ms}. These evaluation times, however, are just for 3 hours of data. These times will increase linearly with the amount of sensor data for the ``Naive" and increase the human users' waiting time for their query to be answered.
The worst retrieval performance is with \textit{Any Object Query Precision} for Q1 [\textit{rows: 7-9}], Q2 [\textit{rows: 16-18}], and Q3 [\textit{25-27})]. Even then, $D3A+D$ returns \textbf{54} frames on average, compared to the \textbf{30} frames returned by $D3A+P$ and \textbf{1933} frames returned by the ``Naive" baseline. This difference is also reflected in their retrieval and evaluation times. The baseline on average finds the queried object in the 25th frame ($MRR=0.04$), while $D3A+D$ and $D3A+P$ finds the object around the fifth frame. Even with the worst query precision where the queried object is mapped to all the $ObjectID's$ in the database, D3A still achieves better retrieval performance by clustering detections and thus looking through fewer instances.
%%%\stnote{This is the one where your algorithm is really really good though. So highlight that even if the detector isn't good, this compression algorithm helps because it clusters things so you have to look through fewer results.}
% that is even when we can not figure out what the object is that we are talking about, D3A can return a small set of keyframes where the correct object of interests lies in the 5th frame.
D3A's overall accuracy is \textbf{81.98\%} with a retrieval time that is \textbf{47x} faster than the ``Naive" baseline. This can lead to improved user experience since with D3A, the user's query will be answered by searching over a smaller number of key frames (by \textbf{97\%}) than with the baseline.
% \tn{what is the 97\% trying to convey?}
\begin{figure}[]
\centering
\includegraphics[width=0.8\linewidth]{images/Figure_1.png}
\vspace{-3mm}
\caption{Exp 3 - Cumulative number of insertions to the database per hour}
% \label{fig:a}
% \tn{seems like there's extra white space around the figure, trim that to get back some space}
\label{fig:graphcumulative}
\end{figure}
\begin{table}[t]
\small
%\stnote{use booktabs, avoid vertical rules}
%\tn{i rounded up decimals to have max 4 decimal places.}
\caption{System Performance (On 3 Hours of Data)}
% \iinote{since they are so many number, I highlighted those that were a little more important and that I discuss even in the evaluation results section but maybe there is a better way to do that?} \tn{maybe add a footnote to say that ``the bolded numbers are highlighted results that are discussed in the text" or something?} }
\vspace{-3mm}
\centering
\begin{tabular}{p{3cm} l l l}
\toprule
{\bf{Performance Metric}} & & \bf{Naive} & \bf{D3A+D}\\
\midrule
Database Size (MB) & & $3312$ & $244$ \\
\hline
Total Processing Time & & $0$m $7.61$s & $2$m $26$s \\
\hline
Average Query Response Time (ms) & & $503.59$ & $10.63$ \\ %9.544548022
\hline
Total Number of Unique Object IDs & & $\bf{2558}$ & $\bf{25}$ \\
\hline
\multirow{2}{3cm}{Number of Duplicates per Object} & Mean & $ 44.10 \pm 75.66$ & {$1.19 \pm 0.46$} \\
& Max & $369$ & $3$ \\
\hline
Mean Accuracy (\%) & {} & $48.85$ & $81.98$ \\
\hline
\multirow{3}{3cm}{Average Number of Keyframes Returned} &
U & $1$ & $8.73$ \\
& C & $\textbf{583.24}$ & $\textbf{10.47}$ \\
& A & $\textbf{1933}$ & $\textbf{54}$ \\
\hline
\multirow{3}{3cm}{Mean Reciprocal Rank ($N=50$)} &
U & $1$ & $1$ \\ %0.9166666667
& C & $\textbf{0.16}$ & $\textbf{0.26}$ \\ %0.4476691691
& A & $0.032$ & $0.187$ \\ %0.3578010691 %0.4195961824
\bottomrule
\end{tabular}
\label{table:results1}
\footnotesize{* The bolded numbers are highlighted results that are discussed in the text}
\vspace{-6mm}
\end{table}
\subsubsection{Exp 3 - Compactness Comparison}
We measure the cumulative number of object id insertions in the database per hour over the complete 22 hours for both $D3A+D$ and the ``Naive" baseline. As seen in Fig-\ref{fig:graphcumulative}, the number of insertions per hour for $D3A+D$ is much less than that of the baseline and scales well as the amount of data increases, demonstrating D3A's success in aggregating partial views. D3A takes two orders of magnitude more time to process the raw data than the baseline, but outputs a representation \textbf{14.7x} more compact and efficient in answering questions, as shown in Table \ref{table:results1}.
\subsubsection{Exp 4 - Comparison with the ``Non-spatial'' Baseline}
\begin{figure}[t]
\centering
\includegraphics[width=0.6\linewidth]{images/final.png}
\caption{Exp4 - Visualization of the keyframe centroid and the aggregated detections by the ``Non-spatial'' baseline \cite{hsieh2018focus} for the pink cup. All the square images are the detections and the different bounding box colors show the respective clusters they were mapped to by D3A based onboth the embedding feature and spatial information. The dotted line shows if the two clusters were mapped to the same $ObjectID$.}
\label{fig:clusters}
\vspace{-7mm}
\end{figure}
In this experiment, we visually compare the aggregated clusters formed for the unique object instances by $D3A+D$ and the ``Non-spatial" baseline. Fig-\ref{fig:clusters} shows the baseline's aggregated detections for the pink cup asked for in Fig-\ref{fig:Kuri}. The baseline clusters all object instances that are similar in embedding space, irrespective of their location in the map. The images' different bounding box colors show the respective clusters they were mapped to by D3A based on both the embedding features and spatial information. The clusters formed by D3A most closely resemble ground truth. Furthermore, the ``Non-spatial'' baseline will fail to answer queries of type Q3 in Table \ref{tab:example_queries}.
\section{Discussion}
% \tn{cut: Our -deployed algorithm $D3A+D$, despite performing very well, is still unable to retrieve spatio-temporal information for all the ground truth objects.}
Our deployed algorithm $D3A+D$ shows memory and speed efficiency in clustering together partial views of the same object, even when the object has moved to different locations in the environment over time. In such cases, the same object in each of its new location is assigned to a different keyframe centroid, but is associated with the same $ObjectID$. This is shown in Fig-\ref{fig:clusters} where the pink cup is moved from the table (img3) to the kitchen counter (img4) and assigned to two separate keyframe centroid by D3A , but is assigned the same $ObjectID$, symbolized by same color of bounding box.
D3A can also disambiguate unique objects instances even if they have similar features. An example is shown in Fig-\ref{fig:clusters} where D3A assigns the two similar red cups in ``img1" and ``img2" to different clusters. Despite such results, our algorithm is unable to perfectly retrieve all of the locations a unique object was placed at every moments in time.
% \iinote{Add: Our deployed algorithm $D3A+D$ shows memory and speed efficiency in clustering together partial views of the same object even when moved to different locations in the environment over time. In such a case the same object in each of it's new location is assigned to a different key frame centroid but gets associated to the one object\_id. This is shown in Fig.\ref{fig:clusters} where the pink cup is moved from the table ``img3" to the kitchen counter ``img4" and is assigned two separate keyframe centroid by D3A since they are two locations but is assigned the same object\_id. D3A can also disambiguate unique objects instances even if they have similar feature. An example is shown in Fig.\ref{fig:clusters} where D3A assigns the two similar red cups in ``img1" and ``img2" different clusters. Despite such results, our algorithm is unable to perfectly retrieve all of the locations where a unique object was placed at every moment in time.}
% spatio-temporal information for every ground truth object.}
%%%Our deployed algorithm $D3A+D$, despite performing very well, is still unable to retrieve spatio-temporal information for all the ground truth objects.
% Overall, our deployed algorithm $D3A+D$ performs very well and correctly retrieves objects most of the time. However, it is still unable to retrieve the spatial-temporal information for all ground truth object instances.
This is because our system sometimes fails to distinguish objects when they have similar embedding features and are close together in Euclidean space (below $d_{thresh}$). In this case, D3A will assign same $ObjectID$ and keyframe centroid to two unique object instances.
% This will cause D3A to miss storing the spatial-temporal information of the wrongly assigned object and.
An example of this is shown in Fig-\ref{fig:clusters} where ``img1" of the red cup is assigned to the cluster for the pink cup since they have similar colors and are close together in the physical world.
This causes D3A to miss the associated information of the wrongly assigned red cup and it cannot retrieve this information in response to an object retrieval query related to the red cup. Furthermore, if an object instance is only seen in one frame of a sliding window, our algorithm will characterize it as noise to account for errors in the detection. This, however, causes D3A to sometimes miss unique object instances. Overall, D3A recognized 76 unique object locations in the map, but because of the above reasons assigns them to 25 object instances instead of the ground truth 59, while considering 18 of the objects as noise. As shown in Table \ref{table:results1}, the average number of duplicated $ObjectID's$ assigned to a ground truth object instance by D3A is $1.19 \pm 0.46$, which is 36X smaller than that of ``Naive".
% \iinote{Add: In case of Naive every partial view detection of the object is treated as a unique object leading to total number of unique detections to be greater than the total number of ground truth objects.}
In the case of the ``Naive" baseline, every partial view detection of the object is treated as a unique object, leading to a total number of unique detections greater than the number of ground truth objects. Fig-\ref{fig:graph_noise} and Table \ref{table:queryperformance} show that increasing the false positive rate of the visual detector and decreasing the query precision degrades the performance of object retrieval from the spatial-temporal representation created by D3A. Nevertheless, the deployed D3A+D algorithm on Kuri with the real-world detector performs better than the baseline in terms of memory and speed efficiency for object retrieval. D3A+D also performs better than ``Naive" despite the fact that they both suffer from drifting localization of the robot's pose over time. This is because D3A's three-tier clustering and filtering structure was able to filter out some of the localization drift as noise in the objects sensor data.
% In addition, D3A+D uses the object locations calculated based on the results of SLAM and thus suffers from drifting localization of the robot's pose over time as the Naive Nonetheless, D3A+D is still able to perform better than the ``Naive" because of the three-tier clustering and filtering structure that does noise filtering in the objects sensor data.
% \tn{doesn't the naive baseline suffer from the same problem?}
% \iinote{Add:
% % In Exp 2, we show the affect of varying query precision and table-\ref{table:queryperformance}. In the experiment runs for Exp 2 as shown in table-\ref{table:queryperformance} that we performeven with bad slam because it drift, our performs better than Naive. %
% Table-\ref{table:queryperformance} and Table-\ref{table:results1} show that increasing the false positive rate of the visual detector and decreasing the query precision degrades the performance of object retrieval from the spatial-temporal representaion created by D3A. Nevertheless, the deployed D3A+D algorithm on Kuri with the real-world detector performs better than the baseline in terms of memory and speed efficiency of object retrieval. Note that D3A+D uses the object locations calculated based on the results of SLAM and despite drifting localization results of the robot's pose over time, D3A+D is still able to perform better than the Naive Baseline.
% }
We take the first step in aggregating the detections of objects over space and time with our proposed algorithm - D3A, and defer the reduction of duplicates by using improved deep learned object embeddings instead of color histograms to future work.
% These reasons lead to duplicates in our system \textit{i.e.,} one object instance in the environment is mapped to multiple unique $object\_id$ s in the database.
% Further, because of the errors described above, the number of unique instances identified by our algorithm are 25 as compared to the actual 59. More probabilistic filtering approaches can be added to reduce the duplicates.
% \begin{figure}[!htbp]
% \centering
% \includegraphics[width=0.8\linewidth]{images/ablation_centroid.png}
% \caption{Exp4 - C}
% % \label{fig:a}
% \label{fig:graph_noise}
% \end{figure}
% \subsection{Results and Discussion}
% \subsubsection*{Exp 1:}
% \subsubsection*{Exp2:} \textbf{Compactness}:
% \textbf{Query Results}
% Sine most of the frames returned by the Naive do not contain the relevant object instance and many have duplicate views of the same object,
% our system takes takes {117x} faster than the baseline. \iinote{need to check results}.
% with a much higher miss rate\iinote{1.25x} since the first 50 frames do not contain the relevant object instance and many have duplicate views of the same object.
% This difference is also reflected in the retrieval and evaluation times.
% With \textit{category level query matching}, the mean frames returned by the deployed system are (50x) smaller than the mean frames returned by the baseline and only 2 counts higher than the perfect system. This difference is also reflected in the retrieval and evaluation times. On average our deployed system took \iinote{50ms} to retrieve and evaluate all the frames for all the object instances while the baseline took \iinote{2secs}. Keep in mind that these times are reported just for 3 hours of ground truth data. The MRR@50 for deployed system is on average \iinote{0.4} which signifies that on average the object of interest in the query is found in the third frame, while for the baseline it is \iinote{0.3} with a much higher miss rate\iinote{1.25x} since the first 50 frames do not contain the relevant object instance and many have duplicate views of the same object.
% There are still duplicates in our system \textit{i.e.} one object instance in the environment is mapped to multiple unique $object\_id$ s. The average number of $object\_id$s assigned to the same object instance by RoboMem is \iinote{23.6 $\pm$ 7.50}, which is \iinote{96\%} smaller than the number of duplicates in the baseline. Our algorithm takes two order magnitudes more time to process the data than the baseline however the representation is more compact and efficient in answering questions as shown in the results here.
% The overall accuracy of RoboMem is $81.98\%$ with $47x$ faster than the baseline. The retrieval performance of the query matching levels is shown to be in this order - Level 1, 2, 3 with level 3 performing the worst since more mean key frames need to be looked into.
% \iinote{talk about low frame rate?}
% \iinote{talk about reasons for missed frames}
% \iinote{The retrieval performance of the query matching levels is shown to be in this order - Level 1, 2, 3 with level 3 performing the worst since more mean key frames need to be looked into.}
\section{Conclusion}
We present a novel algorithm for robots to efficiently answer spatial-temporal queries about objects in the environment over long periods of time. Our algorithm aggregates partial view detections of unique instances to create a compact and query-able representation of the objects. By explicitly performing detection-based three-level association to identify the keyframes for unique object instances, our algorithm significantly outperformed baselines in query answering in terms of the retrieved frames' accuracy, mean reciprocal rank and retrieval time. A robot deployed with our algorithm was able to process 22 hours of
% \iinote{I evaluate on 3 hours? but do make compact for 22 hours so should it be 22 hours?} \tn{i'd say 22 bc it did process and compress that much data, we just don't know how well}
sensor data and develop a compact representation of objects in its environment, which is an encouraging step towards enhancing the sensory capabilities of home-service robots that can help the elderly find their lost, forgotten objects.
% \section{Copyright}
% All papers submitted for publication by AAAI Press must be accompanied by a valid signed copyright form. They must also contain the AAAI copyright notice at the bottom of the first page of the paper. There are no exceptions to these requirements. If you fail to provide us with a signed copyright form or disable the copyright notice, we will be unable to publish your paper. There are \textbf{no exceptions} to this policy. You will find a PDF version of the AAAI copyright form in the AAAI AuthorKit. Please see the specific instructions for your conference for submission details.
% \subsubsection{Acknowledgments.}
% The acknowledgments section, if included, appears after the main body of text and is headed ``Acknowledgments." This section includes acknowledgments of help from associates and colleagues, credits to sponsoring agencies, financial support, and permission to publish. Please acknowledge other contributors, grant support, and so forth, in this section. Do not put acknowledgments in a footnote on the first page. If your grant agency requires acknowledgment of the grant on page 1, limit the footnote to the required statement, and put the remaining acknowledgments at the back. Please try to limit acknowledgments to no more than three sentences.
% \subsubsection{Appendices.}
% Any appendices follow the acknowledgments, if included, or after the main body of text if no acknowledgments appear.
% The following commands are available for your use in citing references:
% \begin{quote}
% {\em \textbackslash cite:} Cites the given reference(s) with a full citation. This appears as ``(Author Year)'' for one reference, or ``(Author Year; Author Year)'' for multiple references.\smallskip\\
% {\em \textbackslash shortcite:} Cites the given reference(s) with just the year. This appears as ``(Year)'' for one reference, or ``(Year; Year)'' for multiple references.\smallskip\\
% {\em \textbackslash citeauthor:} Cites the given reference(s) with just the author name(s) and no parentheses.\smallskip\\
% {\em \textbackslash citeyear:} Cites the given reference(s) with just the date(s) and no parentheses.
% \end{quote}
\bibliography{references}
\end{document}
\def\year{2021}\relax
%File: formatting-instructions-latex-2021.tex
%release 2021.2
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai21} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
\usepackage[toc,page]{appendix}
%\nocopyright
%PDF Info Is REQUIRED.
% For /Author, add all authors within the parentheses, separated by commas. No accents or commands.
% For /Title, add Title in Mixed Case. No accents or commands. Retain the parentheses.
\pdfinfo{
/Title (AAAI Press Formatting Instructions for Authors Using LaTeX -- A Guide)
/Author (AAAI Press Staff, Pater Patel Schneider, Sunil Issar, J. Scott Penberthy, George Ferguson, Hans Guesgen, Francisco Cruz, Marc Pujol-Gonzalez)
/TemplateVersion (2021.2)
} %Leave this
% /Title ()
% Put your actual complete title (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case
% Leave the space between \Title and the beginning parenthesis alone
% /Author ()
% Put your actual complete list of authors (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case.
% Each author should be only by a comma. If the name contains accents, remove them. If there are any LaTeX commands,
% remove them.
% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
\newcommand{\MYhref}[3][blue]{\href{#2}{\color{#1}{#3}}}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{color, colortbl}
\definecolor{airforceblue}{rgb}{0.36, 0.54, 0.66}
\definecolor{LightCyan}{rgb}{0.88,1,1}
\definecolor{codegreen}{rgb}{0,0.6,0}
\usepackage{array,etoolbox}
% \AtBeginEnvironment{table}{\rmfamily \small}
\preto\tabular{\setcounter{magicrownumbers}{0}}
\newcounter{magicrownumbers}
\newcommand\rownumber{\stepcounter{magicrownumbers}\arabic{magicrownumbers}}
\newif\ifpaperfinal
% \paperfinalfalse
\paperfinaltrue
\ifpaperfinal
\newcommand{\tn}[1]{}
\newcommand{\iinote}[1]{}
\newcommand{\ernote}[1]{}
\newcommand{\stnote}[1]{}
\newcommand{\bknote}[1]{}
\else
\newcommand{\iinote}[1]{\textcolor{codegreen}{\textbf{II: #1}}}
\newcommand{\stnote}[1]{\textcolor{blue}{\textbf{ST: #1}}}
\newcommand{\tn}[1]{\textcolor{airforceblue}{\textbf{TN: #1}}}
\newcommand{\ernote}[1]{\textcolor{red}{\textbf{ER: #1}}}
\newcommand{\bknote}[1]{\textcolor{red}{\textbf{BK: #1}}}
\fi
% The file aaai21.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%
% Title
% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Where were my keys? - Aggregating Spatial-Temporal Instances of Objects for Efficient Retrieval over Long Periods of Time}
\author{
%Authors
% All authors must be in the same font size and format.
% Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
% AAAI Style Contributions by Pater Patel Schneider,
% Sunil Issar, \\
% J. Scott Penberthy,
% George Ferguson,
% Hans Guesgen,
% Francisco Cruz,
% Marc Pujol-Gonzalez
% \\
Ifrah Idrees, Zahid Hasan, Steven P. Reiss, and Stefanie Tellex\\
}
\affiliations{
%Afiliations
Dept. of Computer Science, Brown University, Providence, RI\\
% \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
%If you have multiple authors and multiple affiliations
% use superscripts in text and roman font to identify them.
%For example,
% Sunil Issar, \textsuperscript{\rm 2}
% J. Scott Penberthy, \textsuperscript{\rm 3}
% George Ferguson,\textsuperscript{\rm 4}
% Hans Guesgen, \textsuperscript{\rm 5}.
% Note that the comma should be placed BEFORE the superscript for optimum readability
% 2275 East Bayshore Road, Suite 160\\
% Palo Alto, California 94303\\
% email address must be in roman text type, not monospace or sans serif
% publications21@aaai.org
% ifrah
% See more examples next
}
\iffalse
%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Single Author}
\author {
% Author
Author Name \\
}
\affiliations{
Affiliation \\
Affiliation Line 2 \\
name@example.com
}
\fi
\iffalse
%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Multiple Authors}
\author {
% Authors
First Author Name,\textsuperscript{\rm 1}
Second Author Name, \textsuperscript{\rm 2}
Third Author Name \textsuperscript{\rm 1} \\
}
\affiliations {
% Affiliations
\textsuperscript{\rm 1} Affiliation 1 \\
\textsuperscript{\rm 2} Affiliation 2 \\
firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
}
\fi
\begin{document}
\maketitle
\begin{abstract}
Robots equipped with situational awareness can help humans efficiently find their lost objects by leveraging spatial and temporal structure. Existing approaches to video and image retrieval do not take into account the unique constraints imposed by a moving camera with partial view of the environment. We present a \textbf{D}etection-based \textbf{3}-level hierarchical \textbf{A}ssociation approach, \textbf{D3A}, to create an efficient query-able spatial-temporal representation of unique object instances in an environment. D3A performs online incremental and hierarchical learning to identify keyframes that best represent the unique objects in the environment. These keyframes are learned based on both spatial and temporal features and once identified their corresponding spatial-temporal information is organized in a key-value database. D3A allows for a variety of query patterns such as querying for objects with/without the following: 1) specific attributes, 2) spatial relationships with other objects, and 3) time slices. For a given set of 150 queries, D3A returns a small set of candidate keyframes (which occupy only 0.17\% of the total sensory data) with 81.98\% mean accuracy in 11.7 ms. This is 47x faster and 33\% more accurate than a baseline that naively stores the object matches (detections) in the database without associating spatial-temporal information.
\end{abstract}
\section{INTRODUCTION}
\iinote{color package in table}
\iinote{reduce font size of all tables}
\iinote{fix table 2}
\iinote{remove vspace}
Home-service robots have great potential to assist human users by retrieving spatial-temporal information\footnote{Spatial-temporal information of object refers to the whereabouts of the object such as where the object has been identified in the physical environment of the robot and at what times} about objects from their long-term observations. For example, a person can ask a simple query such as \textit{``where did I leave my keys?''} Service robots with such an ability will be well suited to help the elderly, especially those that have dementia.
There are two lines of works in image and video object retrieval: a) \textit{fixed-view cameras} \cite{object_retrieval, yadav2019vidcep, kang2019challenges} which assume full observability of the environment and do not deal with a moving camera (robot), and b) \textit{robotic object retrieval} \cite{ambrucs2014meta, bore2015retrieval} that either work on short-term time horizons (do not condense partial view detections of unique object instances for memory or speed efficiency) or perform long-term robotic object retrieval by specifically focusing on point cloud matching without incorporating object's spatial information in the physical environment. \iinote{object's spatial information in the physical environment, of where the objects are located in the environment}These approaches for object retrieval will leave the robot searching over countless detections in visual sensor data from many different time slices, which will take up a lot of space and time. In addition, these detections will contain partial views of different object instances, not all of these objects and their partial views will be\iinote{these objects and thus partial views will be} relevant to the query. An algorithm that condenses the partial view detections of each unique object instance into a compact and query-able spatial-temporal representation will enable the robot to answer queries about unique object instances more efficiently.
\begin{figure}
\centering
% \includegraphics[width=0.8\linewidth]{images/story_2.png}
% [width=0.8\linewidth,height=6cm
\includegraphics[width=0.8\linewidth]{images/story_2.png}
% \caption{Kuri Robot \citep{richardson_2018} \stnote{I would replace this with a storyboard of the system, showing it answering a question about where an object is. Also, make sure this is our picture of Kuri, use one you took that we have rights to.}}\label{fig:a}
\caption{The image shows the retrieved keyframes and associated temporal information for two sample queries answered by D3A while deployed on Kuri robot. The keyframes are overlaid on the map using the spatial information returned by our spatial-temporal representation. Also shown are the partial view detections that are aggregated into the keyframe cluster.}
\label{fig:Kuri}
\vspace{-4mm}
\end{figure}
To our knowledge, there has not been a single work that addresses all of the following challenges: 1) aggregating detections of object instances from different views in a map over hours of sensory information, 2) handling the object going out of the view, and 3) accounting for uncertainty in object poses due to noise from the robot pose. To mitigate these challenges, we introduce a new algorithm, D3A that extends the concept of keyframe extraction that has been previously used for video frame retrieval to condense
% \iinote{Add: that extends the concept of keyframe extraction that has been previously used for video frame(s) retrieval to condense}
multiview detections of unique objects over both physical space and time for long observation periods.
D3A extracts both spatial-temporal features of object instances in the environment as they get seen by the robot.
% Treating all these partial views of an object as unique and/or storing all of their spatial-temporal information will be memory and speed inefficient for object retrieval. To mitigate these challenges,
D3A associates these partial view features \iinote{partial view features instead of views} with what it has previously seen over the map to condense them.
% D3A enables scalable association and object retrieval by
It does so by performing three-tier online, incremental clustering and filtering of unique spatial-temporal instances. While performing association, D3A keeps track of the keyframe that best represents each unique spatial-temporal location of the object. The information about keyframe-centroid clusters are then spatial-temporally indexed in a key-value database, leading to a compact and query-able representation for efficient object retrieval.
This representation allows D3A to identify and return a small set of keyframes for the object(s) in a given query, which enables a person to quickly find the relevant information. D3A is storage efficient and is able to handle the case where the attributes of object instances (color, shape, patterns) asked for in the queries are not known in advance. In prior work all or subset of these attributes were provided at the time of query.
Our algorithm is deployed on a social robot Kuri from \citet{heykuri_2018} that patrolled a robotics lab\iinote{robotics lab} environment and collected 22 hours of observations over four days\footnote{\label{note1} A visualization recording of the initial state of the environment, along with its 2D map and sample of the collected dataset can be found at this link: https://github.com/IfrahIdrees/D3A.git}. When tested on a set of 150 queries about different objects, D3A returns a small set of keyframes and their spatial-temporal information (which \iinote{on average} occupy only 0.17\% of the total sensory data\iinote{sensory data of object of interests}) and is 47x faster than the baseline model which treats all partial-view detections as independent instances. D3A is also 33\% more accurate than the baseline with an accuracy value of 81.98\%.
\section{Related Work}
% \ernote{I feel like you should discuss SLAM and semantic mapping literature. I know you're just using off-the-shelf algorithms for SLAM that comes with Kuri, but there's semantic mapping literature \cite{pronobis2012large} that track object attributes, though i dont think they ever focus on the memory side of things like D3A does.}
% \iinote{\citet{hsieh2018focus} proposed work similar to ours that can handle many hours of videos captured by a moving camera, except that it does not deal with the objects' spatial information within the global map and only tries to identify video frames with objects of certain classes. We perform an ablation study with this work and show that by using the objects' locations in the map, D3A improves 1) detection of unique object instances and 2) compactness and retrieval performance from our spatial-temporal database.}
% Video processing has advanced quite dramatically over the years in terms of fundamental algorithms for detection, and re-identification of objects.
% \iinote{How key frame detection works. different techniques for key frame extract like clustering methods, energy minimization algorithms or sequential techniques \cite{vazquez2013spatio}. Usually key frame extraction is used to identify a subset of a video frames that can represent the visual content of the sequence as close as possible, but with a limiting number of frame information And usually the spatial-temporal information used for clustering is based on the visual descriptors of the frame . Clustering techniques differ in how they define an appropriate similarity measure between visual descriptors of the frames. }
In the past, \textbf{Keyframe Extraction} techniques have been used to extract the summary key frames to represent video sequences \cite{6135507}.
% \cite{5201141, 6135507, lingam2019key, event, object_tracking}.
A lot of different techniques exists for keyframe extraction such as clustering \cite{vazquez2013spatio}, energy minimization \cite{essa2015modular}, or online techniques \cite{elahi2020online}.
% These differ in how they define the similarity measure between visual descriptors of the frames.
% \tn{the previous sentence feels out of place and should be cut. You can just say that D3A extendes the clustering keyframe extraction technique to ...}
D3A extends the clustering keyframe extraction technique in videos to incrementally find keyframes that best represent the different views of the unique objects in the environment over time, as the robot continues to capture new video frames.
% \iinote{Add: In the past \textbf{Key frame Extraction} techniques have been used to extract the summary key frames to represent video sequences \cite{5201141, 6135507, lingam2019key, event, object_tracking}. A lot of different techniques exists for keyframe extraction such as clustering, energy minimization or sequential techniques \cite{vazquez2013spatio}. These differ in how they define an appropriate similarity measure between visual descriptors of the frames. D3A extends keyframe extraction in videos to incrementally finding keyframes that best represent multi views of the unique objects in the environment over time, as the robot continues to capture new video frames.}
A lot of work has also been done in \textbf{Object Detection} \cite{MaskRCNN, RetinaNet, FastCNN, pose_estimation} for detecting and re-identifying objects. D3A is built upon the notion of object detection in a three-tier mechanism to match objects and condense their detections over long periods of observations. Additionally, there are \textbf{Object Tracking} algorithms that track object instances over time \cite{object_tracking, yilmaz2006object}. However, they usually assume input videos with fixed angles and cannot track/associate objects under occlusion or after the objects go out of view.
% \iinote{In past keyframes is a method used to extract the perform clustering of consecutive frames to find representative keyframes for all the consecutive frames video trying to find unique views of scene. we extend that to actually find keyframes to associate multi views of the object for object association over space and time.They differ in how they define an appropriate similarity measure between visual descriptors of the frames. }
% These algorithms don't scale well with the number of objects.
These algorithms are also not useful for retrieval since they do not focus on making the objects' trajectories query-able.
% \tn{can cut: We use an unsupervised clustering method in our implementation which can be replaced by any other techniques.}
% \ernote{What other techniques are likely? I feel like you mean that you chose a specific unsupervised clustering algorithm X, but any unsupervised clustering algorithm could be used?}
Although many \textbf{Image or Video Retrieval} systems have been introduced in the multimedia community, they focus on retrieval at the level of object classes and not instances. These works can be categorized into the following types:\\
\textit{1) Fixed-angle Video Input} \cite{object_retrieval, yadav2019vidcep, kang2019challenges}: These works do not account for the objects' spatial information in a global map. In contrast, D3A can perform object instance association in a partially observable environment monitored by a robot.\\
\textit{2) Query Optimized Object Retrieval} \cite{kang2019challenges}: This work uses information from the query to optimize the retrieval pipeline by searching over all the scaled down image frames in the original dataset. They do not reduce redundancy in the data,
% Their search space is over all the frames and they do not focus on reducing the redundancy in data \cite{kang2019challenges}.
% Our algorithm instead focuses on optimizing the spatial-temporal representation to be used for retrieval. We focus on condensing the detections to only store key frame information for retrieval.
whereas D3A focuses on only storing the keyframe information
% to condense the detections
and optimizing the spatial-temporal representation.
There are also works that can handle partial views of objects and perform view-invariant object detection at the category level \cite{hsieh2018focus, sivic2003video}.
% However, they do not record the objects' spatial information and cannot answer spatial queries like D3A.
\citet{hsieh2018focus} proposed work similar to ours that can handle many hours of videos captured by a moving camera (that is not a robot) but does not use object locations. We perform an ablation study with this work and show that by using the objects' locations in the map, D3A improves 1) detection of unique object instances and 2) compactness and retrieval performance from our spatial-temporal database. \citet{sivic2003video} assume to have seen a part of the scene captured by different static camera viewpoints to learn the visual vocabulary (which incorporates spatial information) to associate objects seen in later scenes. We perform online, incremental object matching, and do not pre-define the number of clusters/objects in the environment.
% Unlike both of these works, our clustering is based on both the object feature space and spatial-temporal information.
% There are other works that deal with robotic object retrieval but over short-time horizons and therefore they do not condense the partial views for memory and speed efficiency.
% Another relevant line of work is robotic object retrieval. Most works in this line operate over short-time horizons and do not condense partial view detections of unique object instances for memory or speed efficiency \cite{Temporal-Grounding} \tn{feels like you're missing citations}.
% \tn{what's the difference between these works and the ones in the previous paragraph?}
There are also works that specifically focus on long-term robotic object retrieval that handle partial observability in visual sensor data \citep{ambrucs2014meta, bore2015retrieval}. However, these works assume a point cloud of objects as input and introduce matching algorithms specifically in 3D space. D3A, in contrast, does matching using the RGB feature space and limited depth information (information provided by the depth sensors for obstacle detection) that can be derived from the 3D space. \tn{is this better bc it requires ``less" visual information or just different} These works also do not deal with the objects' spatial information in the environment. Furthermore, they do not focus on making a compact representation of the sensory data for retrieval, but instead store and operate over the original data and use the object attribute ``shape" provided in query to optimize retrieval.
% \tn{This also seems similar to the query optimized object retrieval works described above. Are these the works that require that the queried object attributes are known in advance? This bit is not explicitly stated anywhere}
% \stnote{use citep, not citet, only use cite for noun phrases. for multiple cites, put them in one citep command separated by commas }
% \begin{figure}[t]
% \centering
% \includegraphics[width=\linewidth]{images/high_level.png}
% \caption{Block Level diagram of D3A \tn{D3A}}
% % \roma{Trim the edges so they fit exactly and also make the figure width the line width so this fits the page nicely and the font is easier to read :-)}}
% \label{fig:a}
% \label{fig:block-diagram}
% \end{figure}
% \begin{figure}[t]
% \centering
% \includegraphics[width=\linewidth]{images/1.jpg}
% \caption{\small Visualization of D3A Algorithm \iinote{Add Qi, si}}
% % \label{fig:a}
% \label{fig:Processing}
% \vspace{-6mm}
% \end{figure}
\begin{figure*}[t]
\centering
\includegraphics[width=0.75\textwidth]{images/Processing_diagram.jpg}
\caption{\small Visualization of D3A Algorithm \iinote{Add Qi, si}}
\label{fig:Processing}
\vspace{-3mm}
\end{figure*}
% \tn{this is super readable now! i recommend using a different color (darker green maybe) to label the tiers bc the current color is a little hard to see (same for Fig 3)}
\section{Problem Formulation}\label{Section:system_overview}
% \stnote{So you want to write technical approach in a more problem-driven way. Think of it as a tree of problems and solutions. You don't want to introduce a bit of technical material until you've first explained the problem you need to solve by that bit. The paragraph below is just saying what you built without saying why or situating it in what problems you are solving.}
% \iinote{how link speed and memory efficiency}
\iinote{fix time variable as t?}
A robot monitoring the environment over long periods is given a spatial-temporal query $Q$ to retrieve spatial temporal information of target objects seen in the environment. The query can be of any type as listed in Table-\ref{tab:example_queries}. The robot is equipped with the map of the environment and at any given time step $i$ can gather the following sensory information $s_i=<p_{robot,i}, d_i, f_i>$, where $p_{robot,i} \in P_{robot}$ is the associated robot's pose, and $d_i \in D$ is the corresponding depth information for image frame $f_i$. The robot collects large amounts of sensor data $S = \{s_1, s_2, ... , s_i\}$ and uses an object detector to extract the relevant object-centric information $a_i$ from every frame $f_i$. However, the robot has partial observability of the environment and the information required to answer the object retrieval query is going to be dispersed throughout $S$ in the form of multiple partial view detections. To answer object-retrieval queries efficiently, the robot needs to condense the multi-view detections of the objects into a memory efficient and query-able representation. This representation is then used by the robot to return a small set of keyframes, which enables a person to quickly find the relevant information about the target object.
% The sensor data $S$ will take up a lot of disc space and be inherently noisy. Searching all of this data for retrieving object's spatial-temporal information is going to be speed-inefficient.
% % The robot should also be able to answer queries where the attributes of object instances (color, shape, patterns) are not known in advance.
% For the robot to retrieve an object's spatial temporal information in a memory and speed efficient, it should be able to condense the multi-view detections of the objects over time and be able to organize them in queryable representation such that given a query $Q$, it can return a small set of keyframes, which enable a person to quickly find the relevant information about the target object.
\section{Approach}
% all this sensor data $S$ through which the object need to be retrieved takes up a lot of disc space and is inherently noisy.
% Equipped with $S$, the robot is given a spatial-temporal query $Q$ of any of the type listed in Table-\ref{tab:example_queries}. One example is ``Did you ever see an orange bowl" and is tasked to retrieve spatial temporal information of target objects seen in the environment. Ho
% A human given a robot collecting $S$ can ask from their personal assistant robot a spatial-temporal query - $Q$ about any object in the environment seen during any time of its patrol such as can be "Did you ever see an orange bowl".
% $=< p_{robot,i}, d_i, f_i>$ is the associated information for every captured image frame $f_i$. $p_{robot,i} \in P_{robot}$ is the associated robot's pose, and $d_i \in D$ is the corresponding depth information, for every image frame $f_i$.
% Note that the next time step $i+1$ is dependant on the frequency at which the robot is capturing the sensor data.
% All this sensor data $S$ takes up a lot of disc space and is inherently noisy.
For efficient spatial-temporal object retrieval in a partially observable environment, we need to compress this data $S$ into a compact and query-able spatial-temporal representation $R$. For this we first pre-process every $s_i$ to extract the relevant object-centric information $a_i$, then apply a three-tier online clustering algorithm to aggregate $a_{i}$'s. The output of D3A is a set of clusters of unique object instances that can be aggregated into the previous time step's compact representation $R_{i-1}$ to create a new representation $R_i$ of the environment for time step $i$. A diagram of our algorithm creating $R$ is shown in Fig-\ref{fig:Processing}. Details of algorithmic implementation can be found in the Appendix.
% \stnote{problem formulation is only the problem, not the solution. So don't put anything about preprocessing etc in the problem formulation.}
% \vspace
\subsection{Pre-processing each frame:}
% For compactness and speed efficiency, D3A needs to reduce the dimensionality of the visual sensory data $f_i$ in $s_i=< p_{robot, i}, d_i, f_i>$, as well as aggregate multiple detections of the same object over long periods of time. D3A achieves this by extracting minimal amounts of relevant object-centric information $a_i = <C_i, Prob_i , B_i, P_{object,i}, Emb_i>$ for every frame $f_i$. There is a set $J_i$ for every $f_i$ that includes all the detected object instances in $f_i$. $C_i$ is the list of corresponding categories for all the detections in the set $J_{i}$,
% % for each detection $j \in J_i$ and $J_i$ represents the set of all detections in $f_i$,
% $Prob_i$ are the probabilities of detecting the object categories $C_i$, $B_i$ is the list of the object bounding boxes for $J_i$, $P_{object,i}$ is the list of object poses estimated for all the detections. $P_{object,i}$ is estimated from the robot's pose $p_{robot,i}$ and limited depth information $d_i$, and is a 3D world coordinate. $Emb_i$ is the list of embeddings corresponding to the detections in $J_i$, which is used to relate $j's$ over time and create the compact representation $Q$.
For compactness and speed efficiency, D3A needs to reduce the dimensionality of the visual sensory data $f_i$ in $s_i=< p_{robot, i}, d_i, f_i>$, as well as aggregate multiple detections of the same object over long periods of time. D3A achieves this by extracting minimal amounts of relevant object-centric information ${a_i}$:
\begin{center}
$a_i = [ <c_{i1},prob_{i1},b_{i1},p_{robot,i1},emb_{i1}>, <c_{i2},prob_{i2} ,b_{i2},p_{robot,i2},emb_{i2}>,..., <c_{ij}, prob_{ij},b_{ij},p_{robot,ij},emb_{ij}>]$
\end{center}
for every frame $f_i$, $c_{ij} \in C_i$, is the category detected for the $j^{th}$ object instance detection in $i^{th}$ image, $prob_{ij}$ is the probability of $j^{th}$ object being assigned the category $c_{ij}$, and $b_{ij}$ and $emb_{ij}$ is the bounding box and object embedding for the $j^{th}$ detection in $f_i$. $emb_{ij}$ is used to relate it with the other objects over time and create the compact representation $R$.
% There is a set $J_i$ for every $f_i$ that includes all the detected object instances in $f_i$. $C_i$ is the list of corresponding categories for all the detections in the set $J_{i}$,
% % for each detection $j \in J_i$ and $J_i$ represents the set of all detections in $f_i$,
% $Prob_i$ are the probabilities of detecting the object categories $C_i$, $B_i$ is the list of the object bounding boxes for $J_i$, $P_{object,i}$ is the list of object poses estimated for all the detections. $P_{object,i}$ is estimated from the robot's pose $p_{robot,i}$ and limited depth information $d_i$, and is a 3D world coordinate. $Emb_i$ is the list of embeddings corresponding to the detections in $J_i$, which is used to relate $j's$ over time and create the compact representation $Q$.
% % Overall, $a_{ij} = \{ c_{ij}, prob_{ij} , b_{ij}, p_{robot,ij}, emb_{ij} \}$ represents the corresponding tuple of object centric information for the $jth$ object instance detection (such as a red cup) in the $f_i$ image frame.
% \iinote{check if the ratio between height ad width is an attribute}
% $p_{object,ij}
% \tn{$l_{ij}$ is undefined} in the map.
% we $s_i$ by passing through an object detector to extract minimal amounts of relevant abstract information $a_i = <C >$. $a_i$ includes $C$
% the output of a \st{pretrained} object detector including probabilities of detecting the object categories, their bounding boxes, and object poses are estimated from the robot's pose $(x,y,z)$ and limited depth information.
% \iinote{Move to next section:We use a color histogram \cite{color_hist} to approximate an embedding of each object instance to relate them over time. We select the color histogram method due to its simplicity, and design the later processing levels to handle a noisy instance embedding generator. Notably, our approach is not constrained to any number of objects, particular object size or shape, or object dynamicness (\textit{i.e.,} the objects' locations can change over time). We are only constrained by the object categories the detector was trained to detect.}
\subsection{Tier 1- Clustering across adjacent frames}
% \tn{increase font size for text outside boxes}
% \stnote{This paragraph is a really nice example of introducing the problem first.}
The extracted meta-information for the $jth$ detection in the $ith$ frame -- $a_{ij}$ is noisy because of the uncertainty in the robot's pose estimate $p_{robot,i}$ and noise in the extracted object-instance centric information -- $c_{ij}$, $prob_{ij}$, $emb_{ij}$. To aggregate the different view detections of the same object and to identify the unique objects in the environment, we consider a sliding window $w$ over sensory data $\{s_{i - n}, ..., s_i \} $. Object-centric information $a_{ij}$ of all detections in all the frames of this sliding window are then clustered to create a set of clusters $G$, each identifying a unique object instance. The motivation behind this is that the object instances of same category with similar embeddings and positions in space will be associated to the same cluster hence pruning the noisy detections. We assign a unique id $instance_{id} \in ID$ to each cluster $g \in G$ and perform an aggregation operation $\bigotimes$ on every cluster $g \in G$ to create a unified feature representation of each of the cluster. This results in an aggregated representation $a_{g,\bigotimes}$ which is then inserted into a collection $T$ indexed by the unique id $instance_{id}$ of each cluster where $ID$ is the set of all $instance_{id}$'s \iinote{fix this}.
% all detected object insta $AllDetects = \{ J_i,J_{i+1}, ....J_{i+n}\}$. We concatenate $a_{ij}$ for every detection $j$ that is part of the sliding window $w$. We then perform spatial clustering over these concatenated feature vectors $fv_{ij}$ for every $Dt \tn{should be $J_i$?} \in AllDetects$. Specific implementation details of the clustering algorithm can be found in Sec-\ref{sec-Algo Implementation}.
% % \iinote{move to next section: all information that can be extracted from each detection, in real-time}
% This will create $G$ clusters and each cluster can be considered a unique object instance for this sliding window $w$. For the subset of detections $Detects \subseteq AllDetects $ assigned to one cluster $g \in G$, we perform an aggregation operation on the $fv_{ij}$'s associated with $Detects$ based on the $weight= number \ of \ detections$ in the cluster, leading to a $weightedfv_{ij}$ associated with each cluster $g$. This aggregation operation will account for noise in the individual $a_{ij}'s$ across multiple detections. Specific implementation details of which can be found in Sec-\ref{sec-Algo Implementation}. Tier 1 - Aggregation gives us as output a set of clusters $G$ and their associated $weightedfv_{ij}$. We now need to keep track of these clusters in $w$ to associate them with what the robot has seen in the past, hence we store then in a table $T$ which will be indexed by an $instance_{id} \in ID$ that is unique to each cluster.
% The set of ID gets populated with a new unique $instance_{id}$ every time a new cluster $g$ is formed.
% \iinote{aggregation operation}
% \iinote{add DBSCan clustering algorithm}
% \stnote{make title shorter, and let latex put the table where it wants; it looks better if it's at the top of the page.}
\subsection{Tier 2 - Filtering across multiple sliding windows}
% \begin{figure}[t]
% \centering
% \includegraphics[width=0.6\linewidth]{images/Database_design.png}
% \vspace{-2em}
% \caption{Schematic Design of Database . *2 -Set of time intervals tuples containing start and end time.
% *3.- Associated with each time interval is a bounding box and key frame \tn{the font's a little small, change the green color to be darker, and what is PK \& FK?}}\label{fig:a}
% \label{fig:Schema}
% \end{figure}
% \stnote{introduce problem first}
We also need to aggregate the detections of unique instances across the objects' multiple partial views in the past. This requires sharing information across multiple sliding windows via a second-tier filtering. To facilitate this, we maintain a fixed-length short-term memory (STM) indexed over $ID$ \bknote{read this?}and aggregate information in it for all instance id's in $T$.
% The ouput of Tier 1 ($ID$ in Table $T$) still needs to pass through more filtering since the detections of unique object instances need to be aggregated with the previous objects' multiple partial views for further compactness and association. This requires sharing information across multiple sliding windows via a second-tier filtering. To facilitate this, we maintain a fixed-length short-term memory $(STM)$ that is indexed over $ID$.
For an object represented by $instance_{id}$ in collection $T$,
we add a new entry to STM only if the object is a new unique object. Otherwise, if the object has previously been seen by the robot, an entry for this object should already exist in STM and should be updated.
We do this by finding a set of $nearIDs$ in STM that are closest in embedding space in cosine similarity\iinote{tell here?} and measuring the euclidean distance between them and $instance_{id}$ in physical space. There are three cases to be considered here that symbolize whether the object is new or not and whether it has moved or not: \textbf{1)} If $nearIDs$ set is non-empty and euclidean distance is less than a threshold $d_{thresh}$ then we perform another aggregation operation $\bigodot$ on $instance_{id}$ and the closest $n_{id} \in nearIDs$ to create a unified feature representation for $n_{id}$. This results in an aggregated representation $a_{n_{id},\bigodot}$ that is updated in $STM$. \textbf{2)} If $nearIDs$ set is non-empty and euclidean distance is greater than a threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). \textbf{3)} If $nearIDs$ is empty for $instance_{id}$, we also add a new entry in the STM with instance id $instance_{id}$. This represents the scenario where the object has not been seen before.
%%% every cluster $g \in G$ to create a unified feature representation of each the cluster.
%%%We do this by finding the set of instance id's $\textbf{nearIDs}$ in the STM that are closest to $instance_{id}$ in the embedding space. After which we calculate the Euclidean distance between between the pose of $instance_{id}$ and every element of $\textbf{nearIDs}$. If the normalized Euclidean distance is higher than the threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). If the Euclidean distance is below the threshold $d_{thresh}$, we perform an aggregation operation over the metadata $weightedfv_{ij}$ of $instance_{id}$ and the closest $n_{id} \in \textbf{nearIDs}$. The old $weightedfv_{ij}$ of $n_{id}$ is updated with the new $weightedfv_{ij}$.
%%%We do this by finding the set of instance id's $\textbf{nearIDs}$ in the STM that are closest to $instance_{id}$ in the embedding space. After which we calculate the Euclidean distance between between the pose of $instance_{id}$ and every element of $\textbf{nearIDs}$. If the normalized Euclidean distance is higher than the threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). If the Euclidean distance is below the threshold $d_{thresh}$, we perform another aggregation operation $\bigodot$ on $instance_{id}$ and the closest $n_id \in \textbf{nearIDs}$ to create a a unified feature representation for $n_id$. This results in an aggregated representation $a_{n_{id},\bigodot}$. There is no constraint on two aggregation operation's $\bigotimes$ and $\bigodot$ to be the same. For our implementation they are same and described in sec-\ref{sec-Algo Implementation}.
% For a given query $Q$, D3A returns a small set of keyframes from its queryable representation $R$ to find relevant information about the target object.
% After the aggregation operation, the The keyframes also need to be associated with the updated $n_{id}$ or newly added $instance_{id}$ because for a given query $Q$, D3A returns a small set of keyframes from its queryable representation $R$ to find relevant information about the target object.
% The reason behind making $R$ compact is to allow for speed efficient object retrieval. The responses retrieved from $R$ will be a subset of relevant keyframes associated with unique objects in the environment. \tn{i'd cut the previous 2 sentences bc they seem out of place. also i know that you introduced Q in the intro paragraph for the section but by this point i've forgotten what it is and would need a reminder so it doesn't seem like it's coming from no where}
After the aggregation operation, the keyframe of $n_{id}$ is chosen to be the frame with the highest detection probability $prob_{ij}$ among the frames $f_i's$ associate with $instance_{id}$ and $n_{id}$. If a new instance is to be added to the STM when it has reached its maximum capacity, our algorithm evicts the last recently viewed (least-recently viewed entry), $lrv\_entry$, from the STM and moves it to the persistent storage key-value store $R$. This eviction strategy ensures that objects currently being viewed by the robot remain in the STM for further noise filtering and aggregation of meta-data from partial views.
% \iinote{we do it for all the $instance_{id}'s$ in the window.} \iinote{key-value database}
\subsection{Tier 3- Filtering across Database Entries} \label{subsubsection:p3}
The filtered entry $lrv\_entry$ from the STM could be directly added as the final aggregated cluster into our compact representation $R$. However, we want to aggregate detections of unique object instances over not just recent times but also long periods of time.
% However, we want a more compact representation that aggregate detections of unique object instances over long periods of time.
To do so, every time a $lrv\_entry$ is evicted from STM, our third-tier filtering level looks up and updates the aggregated clusters stored in $R$. We organize our aggregated spatial-temporal representation $R$ in a key-value database over two collections. The \textbf{Object Identification Collection} ($OIc$) is updated using the object information of the evicted clusters represented by the $lrv\_entry$, while the other store \textbf{Spatial-temporal Collection} ($STc$) stores the object entries indexed over time and position in physical space. Every object inserted in $OIc$ is associated with a unique identifier $ObjectID$ which is used to index $STc$ to get the object detections over space and time.
To update $R$, we find the set $Records \in OIc$ that are the closest in feature space with $lrv\_entry$ in cosine similarity. A new document is inserted into $STc$ if the normalized Euclidean distance between last evicted entry from STM $lrv\_entry$ and $Records \in STc$ is greater than $d_{thresh}$, otherwise the object information in the closest document $record \in Records$ is aggregated, via an operation $\oplus$, applied on the $lrv\_entry$ and the document $record$ hence updating the object information in $OIc$ using this aggregated output. This new document insertion in $STc$ symbolizes a moving object while the update symbolizes operation for a static object. For every new entry in $OIc$, an entry is added in the $STc$ against the $ObjectID$ containing the time and physical location of the new entry.
% The schematic design of our spatial-temporal represen
% The \textbf{Object Identification Collection} ($OIc$) stores object attributes of aggregated clusters in $STM$ such as the category $C_R$, embedding $Emb_R$ and weight $Weight_{R}$ \iinote{what to do about this weight?}. We also have a separate \textbf{Spatial-temporal Collection} ($STc$) that stores object entries indexed by the object's location in the environment $P_{object,R}$, time intervals $Time_R$, keyframe $K_R$, detection probability $Prob_R$, and the bounding box $B_R$ of the object in the keyframe. Such an inverted index schema for $R$ supports the flexible update and addition of multiple locations for the same object instance, with a one to many mapping from $OIc$ to $STc$ and a unique identifier $ObjectID$ in $OIc$ that can be used as a referential foreign key in $STc$.
% % Such an inverted index schema supports the flexible updating and adding of multiple locations for the same object instance, with a one to many mapping from $OIc$ to $STc$ with unique identifier $ObjectID$ in $OIc$ being a foreign key in $STc$.
% % \tn{Where did $e$ come from?}
% In our third-tier filtering, we find the set $Records \in OIc$ that are the closest in feature space with $lrv\_entry$ in cosine similarity. A new document is inserted into $STc$ if the normalized Euclidean distance between last evicted entry from STM $lrv\_entry$ and $Records \in STc$ is greater than $d_{thresh}$, otherwise the closest document $record \in Records$
% % \tn{is D the set of all documents? You might want to remove it bc it never got introduced}
% is updated with the result of an aggregation operation $\otimes$ applied to the spatial and the embedding vector of $lrv\_entry$ and the document $record$. The document in $OIc$ (with the same $ObjectID$) is updated with the aggregated embedding
% The schematic design of our spatial-temporal representation built in MongoDB is shown in Figure \ref{fig:Schema}.
% We organiz
% We use \textit{MongoDB}, which uses a special geospatial index to store the object instances. This makes the spatial-temporal lookup of objects very efficient. The schematic design of our database is shown in Figure-\ref{fig:Schema}. We choose not to directly insert $e$ into the database because we want to ensure the unique object instances are associated over long-time horizons hence reducing redundancy. In our third-tier filtering, we look up the object id's $obj_id$ from the \textbf{Object Identification Collection} that are closest in feature space with $e$ under cosine similarity. If the $obj_id$ is empty, we insert a new $obj_id'$ associated with $e$, symbolizing that this is potentially a new object instance not seen by the robot. Else we again perform weighted average of the feature of $obj_id$ and $e$ and update the \textbf{Object Identification Collection}. We then look-up set of documents $D$ in the \textbf{Spatial-temporal Collection} that have the same associated $obj_id$. A new document is inserted in the collection if the normalized Euclidean distance between $e$ and $d \in D$ is greater than the $d_thresh$, else the closest document $d' \in D$ to $e$ is updated with the weighted-average spatial information. The keyframe of this $d'$ is chosen to be the frame with the highest probability amongst the frame associate with $d'$ and $e$.
% \tn{The intro text at the beginning of the ``System Overview" section made it seem like storing the data in the database should be its own subsection, but it's already included in the 3-tier processing subsection right? If that's the case, you should edit the intro text to group processing and storing the data together.}
\subsection{Query Processing and Answering}
\begin{table}
\small
\centering
%\stnote{Can you add a column for the formal representation of the query?} \iinote{I had that previously but it was taking too much space so removed it? Will it be okay if we dont have it?}
\begin{tabular}{p{0.4cm}p{3cm}p{3.4cm}}
\toprule
\bf{Type} & \bf{Language Example} & \bf{Formal Representation} \\
\midrule
Q1 & Did you ever see an orange bowl? & ${orange(x), bowl(y) |}$ ${ OIc.find(x,y) }$\\
\hline
% Q2 & { "Where was the $attr1$ pink cup"?} \\
% \hline
Q2 & When were the cup and bowl seen together? & ${bowl(x), cup(y) |}$ ${STc.find(}$ ${OIc.find(x,y), together)}$\\
\hline
Q3 & Where did you see the pink cup between 1:00pm and 4:00pm on 1st Jan, 2020? & ${pink(x), cup(y), time(t) |}$ ${STc.find(OIc.find(x, y),}$ ${1600 >= t >= 1300)}$\\
% \hline
% Q4 & Where is the Sprite bottle \\
\bottomrule
% \label{example_queries}
\end{tabular}
\vspace{-2mm}
\caption{Query Types, Language Examples \& Formal Representations}
\label{tab:example_queries}
\vspace{-6mm}
\end{table}
Template-based NLP methods \cite{li2017querying} or deep learning \cite{DBLP:journals/corr/abs-1911-04942} methods\tn{probably need citation} can be used to map a language query to a set $X$ of $ObjectID's$, time intervals, and location areas, which D3A takes as input for object retrieval. Table \ref{tab:example_queries} shows the types of queries, their natural language examples, and formal representations used in our evaluation. This formal representation is used to search over indexed $OIc$ and $STc$ collections of $R$. The representation $R$ built through our method can handle a wide variety of queries, including: querying for objects with/without the following: 1) specific attributes such as ``orange bowl," 2) spatial relationships with other objects such as ``cup and bowl together," and 3) time slices which can be either a point in time or a time interval. Our representation can also efficiently handle negative queries where the answer does not exist in the dataset.
% objects with/without 1) specific attributes, 2) spatial relationship among objects, 3) the temporal relationship among objects, and 4) time slices.
% The query input to RoboMem is a combination of $ObjectID's$, time intervals and location area. This set can be generated from Natural Language Query and sample object image. However, our current system takes in a templatized query. The schematic design of the database can handle a variety of queries as follows: Q1, Q2: With specific attributes (for static objects and dynamic objects), Q3) and Q4) With time slices(for positive and negative query). A negative query is the one for which the answer in the database does not exists. Q5: with a query about latest time stamp, Q6, Q7: with spatial relationship among objects. Fig-\ref{fig:block-diagram} show examples of the query types in natural language and their corresponding DB query.
% 3) the temporal relationship among objects, and 4) time slices.
% \begin{enumerate}
% \item With specific attributes: "Where was the $attr1$ $catg$"? mapped to
% \begin{verbatim}[scale =0.99]
% Q1: object\_id=collection.find({Category: attr1}, feature vector: attr1)
% collection.find({obj\_id : object\_id})
% \end{verbatim}
% \item 2) spatial relationship among objects, 3) the temporal relationship among objects, and 4) time slices.
% \end{enumerate}
\section{Evaluation}
% \stnote{The aim of our evaluation is to test the hypothesis that algorithm X improves the precision and recall of a database-backed system and answering queries.}
\begin{table}[t]
\small
\centering
\begin{tabular}{l p{0.5cm} r}
\toprule
Duration &22hrs & 3hrs \\
\midrule
Total Number of Frames & $10,132$ & $991$\\
Frame Rate (per minute) & $7.67$ & $7.67$\\
Sensor Data Size (GB) & 10.53 & $1.43$\\
Total Number of Detections & $13,565$ & $2,558$ \\
Ground Truth Objects & N/A & $59$ \\
& & Static = 49, Dynamic = $10$\\
Number of Times & N/A & Mean = $2.1 \pm 1.3$, \\
Dynamic Objects move && Max = 5.0 \\
\bottomrule
\end{tabular}
\vspace{-3mm}
\caption{Dataset Details}
\vspace{-3mm}
\label{tab:my_label}
%\vspace{-3mm}
\end{table}
The aim of our evaluation is to test the hypothesis that a database-backed system with our algorithm D3A improves both 1) the compactness of the spatial-temporal representation of objects in the environment and 2) the object retrieval performance by returning a small subset of keyframes as measured by the mean reciprocal rank, miss rate and retrieval time as described in Experiment Design subsection.
% Sec-\ref{sec:exp-des}.
As a result, the user will have to search over just a few returned results to find the answer to their question.
To test our hypothesis, we perform a real-world evaluation, using both a range of synthetic detectors built from ground truth data to control the noise level and real detectors deployed on a mobile robot to assess the system's end-to-end accuracy.
% This is directly related to improved user experience.
%%%\stnote{So instead of telling that, can you like SHOW that? As a result, the user will have to search few just a few returned results to find the answer to their question.} \stnote{To test this hypothesis, we perform a real-world evaluation on a mobile robot using both simulated detectors built from ground truth data to test the noise and real detectors to assess the end-to-end accuracy. (PS, I am writing this fast, so feel free to reword/clean up any suggested sentence I put. But it's good to have something to say like, what the evaluation is going ot be and why it will answer our hypothesis.) }
% Our main results are:
% \beging{}
% and hence the recall\footnote{Defined as a function of true positives (tp: key frame containing the object of interest in the query), false negatives (fn: objects failed to be identified and retrieved). It is the fraction of relevant documents that are retrieved, in particular Recall = \frac{tp}{tp+fn}}
% measure for evaluatingreturned key frames to a sample of queries, ordetred by probability of correctness. The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. If none of the proposed results are correct, reciprocal rank is 0.}
% with varying range of sensor uncertainty for retrieved objects.
\subsection{Dataset Collection}
% \stnote{don't put colons after subsection names}
% We deployed D3A on a mobile robot Kuri \cite{heykuri_2018} that patrolled a robotics lab environment for four days.
The robotics lab environment in which our mobile robot Kuri \cite{heykuri_2018} patrolled for four days was uncontrolled and cluttered: people were allowed to use the space as is. For consistency, we kept the illumination same throughout the data collection (it was captured in a completely lit lab). The robotics lab area used for data collection and experimentation included the kitchen area and the general area with tables and chairs. This real-world dataset\footnote{ A visualization recording of the initial state of the environment, along with its 2D map and sample raw sensory data can be found at \url{ https://github.com/IfrahIdrees/D3A.git}.}
has 10,132 image frames over 22 hours and contains both static and dynamic objects such as various cup and bottles changing there location. The dataset details can be found in Table \ref{tab:my_label}.
% \iinote{for consistency keep the illumination same(captured during daytime.)}
% \stnote{cut: To our knowledge this is currently the longest dataset collected by a robot.} \stnote{No way. For example see the MIT SLAM dataset. And all the Waymo selfdriving data. }
\subsection{Dataset/Object Annotation}
\begin{figure}[t]
\centering
\includegraphics[width=0.9\linewidth]{images/noise.png}
\caption{Exp 1 - Retrieval performance with a synthetic embedding generator with increasing uncertainty}
% \label{fig:a}
\label{fig:graph_noise}
\vspace{-4mm}
\end{figure}
% \stnote{No colons in section names.}
% \stnote{I take back the comment in the intro about 22 hours vs 3 hours, but we should make sure we use the 22 hours somehow in the evaluation. }
For evaluation purposes, the collected dataset was manually annotated by the authors. Every unique instance in the dataset was assigned a unique id and a ground truth location that was used to perform object association over time and space. Due to time and manual labour constraints, we were only able to annotate 3 hours of data.
\subsection{Parameter Selection}
% \stnote{No colons in section names.}
% \iinote{ We use a color histogram \cite{color_hist} to approximate an embedding of each object instance to relate them over time. % and reduce the set of potentially similar objects. We select the color histogram method due to its simplicity,}
The Euclidean distance threshold $d_{thresh}$ , sliding window length, normalized threshold for feature matching (cosine similarity), and size of short-term memory (STM) were set to 0.5m, 10, 0.4, and 400 (our RAM's maximum capacity), respectively. We performed a parameter sweep offline on the collected data and found this setting to be optimal. Increasing the feature matching and distance threshold increased the number of false positives during tier-1 processing. For the STM, the greater the buffer size, the more noise there will be in the object's pose and embedding to be filtered and processed.
% The selected parameter values and the justification for each are as follows:
% 1) The Euclidean distance threshold $d_{thresh}$ was set to 0.5 meters to account for the 0.5m standard deviation in the robot pose estimation. We performed a parameter sweep and found that setting the sliding window length = 10 and normalized threshold for feature matching = 0.4 to be optimal. Increasing the matching and distance threshold increased the number of false positives during tier-1 processing.
% 2) The short-term memory (STM) buffer was set to 400 objects, our RAM's maximum capacity. The greater the buffer size, the more noise there will be in the object's pose and embedding to be filtered and processed.
% We also performed a parameter sweep over different lengths of the buffer and found the results to be not impacted
\subsection{Experiment Designs}\label{sec:exp-des}
We conduct four experiments with two baselines. One experiment is a running example comparison.
% We conduct three \tn{four?} sets of quantitative experiments with a running example and two baselines (as described below).
% \stnote{Don't give results here until after you describe the algorithms. So move those two sentences later.}
\textbf{Baselines}: Our first baseline is a ``Naive" method that inserts every detection's spatial-temporal information directly into the database without aggregation. This baseline ablates the three-tier processing that is integral to our system for aggregating partial views of the objects, but can still index detections on space and time. We also conduct an ablation of the spatial information from the features to make a ``Non-spatial" baseline, similar to \citet{hsieh2018focus} which does not utilize spatial information during clustering, to compare the clusters resulting from both methods.
% We do another ablation study where we removing the spatial information used by D3A for three-tier clustering. We refer to this method as ``Non-spatial''. This is similar to \cite{hsieh2018focus} wich uses a similar clustering method but does not use the spatial information.
% For an ablation study with \citet{hsieh2018focus}, we implement a ``Non-spatial'' baseline, removing the spatial information used by D3A for three-tier clustering. We use a running example to compare D3A with this baseline and visualize the differences in their aggregated clusters.
%%%\stnote{I would rephrase that a bit, and say first, we do the abalation, and then, this is similar to hsieh whichuses a similar clusting method but does nto use spatial information}
% will show \tn{that clustering by spatial information improves...} how D3A improves the aggregation of detections.
% In the first, we show the changing trend in retrieval performance of our system relative to the uncertainity of the parameterized synthetic detector as compared to our baseline.
% We perform two separate ablation studies that extensively assess the improvements from our framework.
\textbf{Metric Definitions}:
We measure the quality of keyframes returned for a query with the MRR@50 evaluation metric, which is the multiplicative inverse of the rank of the correct keyframe in the top 50 returned frames ordered by their probability of correctness. We want the rank of the correct frame to be as close as the start of the list of returned frames and hence higher MRR closer to 1 is better. We measure the miss rate (the number of objects missed by the algorithm during the clustering phase due to detection or classification error, hence cannot be found in the returned frames) to get an idea of our accuracy. We also report the speed efficiency of our system by measuring the mean number of frames and object id's returned for the queries, total time taken by a query divided into retrieval time and evaluation time to find the correct frame by matching against the ground truth.
% \begin{itemize}
% \item To measure how good the retrieved key frames are, we measure MRR@50 \footnote{A measure for evaluating the returned keyframes for a sample of queries, ordered by probability of correctness. The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. The first correct answer in our case is the first keyframe that contains the queried object instance. If none of the proposed results are correct, the reciprocal rank is 0. MRR values closer to 1 are preferred as they signify high retrieval precision where the set of returned keyframes contains the relevant response for the query}, Miss Rate which is equivalent to recall(the percentage of time our algorithm fails to recognize unique object instances). We use the miss rate to define the accuracy of the system as 1- average(Miss Rate).
% \item to measure the speed efficiency we measure, the the mean number of frames and object id's returned, 5) mean retrieval time from database, and 6) total evaluation time to check the returned keyframes against the ground truth annotation for all queries. Higher number of these metric mean longer times for the human user to search over the returned frames.
% % \item We define our system's accuracy as If the set of returned keyframes and their corresponding spatial-temporal information contains a frame with the answer to the query, we say that the query has been answered accurately.
% \end{itemize}
% We measure a number of metrics for the retrieval performance distributed over all experiments: 1) the mean reciprocal rank (MRR), and 2) the object miss rate --- the percentage of time our algorithm fails to recognize unique object instances \stnote{Is this recall? Or something different. I would explain how this si different from recall}. We also report 3) the mean number of frames returned, 4) mean number of object id's returned, 5) mean retrieval time (to retrieve keyframes and spatial-temporal data from the database), and 6) total evaluation time taken by the system (to check the returned keyframes against the ground truth annotation for all queries). \stnote{I think I wouldn't do this in a list, but rather would introduce WHY you need each metric, and then define it if necessary.} Our system's accuracy is defined as follows: since the query matching is uncertain, we return the top 50 frames for every query and rank them according to the \tn{their?} weighted average probability for the queried object stored in our database.
% which the object detector detects the category of the objects \tn{the last part is confusing, do you mean you rank them by the object detector's detection probability for the queried object?}.
% We then iterate over the frames to find the one that contains the answer to the query. If the set of returned keyframes and their corresponding spatial-temporal information contains a frame with the answer to the query, we say that the query has been answered accurately.
\begin{table*}[httb]
\small
\centering
% \caption{\tn{Missing table caption. I'd suggest adding $\uparrow$ and $\downarrow$ next to the metric names to help the reader parse the numbers a bit easier ($\uparrow$ means higher is better and so on)}}
% $*^1$refers to the Naive baseline without D3A, $*^2$is D3A + synthetic Perfect embedding generator, $*^3$is D3A + Detectron (real world detector) with a color histogram as the embedding generator
\begin{tabular}{@{\makebox[2em][r]{\textcolor{airforceblue}\rownumber\space\space\space}} l p{1.2cm} p{1.5cm} | p{0.9cm} p{1.8cm} p{1.9cm} l p{1.9cm} p{2cm}}
\toprule
\multicolumn{1}{@{\makebox[3em][r]{}}} & \textbf{Type} & \textbf{Query Precision} & \textbf{Aggregation Method} & \textbf{Miss Rate}$\downarrow$ & \textbf{Mean \newline Number of Frames} $\downarrow$ & \textbf{Mean \newline Number of Object IDs} $\downarrow$ & \textbf{MRR @50} $\uparrow$ & \textbf{Mean \newline Retrieval Time (ms)} $\downarrow$ & \textbf{Total \newline Evaluation Time (ms) $\downarrow$}\\
\midrule
% \multirow{3}{*}{Q1} & perfect & Perfect & $0.00$ & $1.08$ & $1.00$ & $1.00$ & $0.03$ & $8.38$\\
% & perfect & Category & $0.24$ & $5.88$ & $3.75$ & $0.35$ & $0.89$ & $104.82$\\
% & perfect & Any object & $0.88$ & $38.00$ & $33.00$ & $0.07$ & $1.04$ & $2,031.02$\\
\multirow{9}{*}{Q1} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 10.30 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 3.93 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 10.50 & 1.00 & 1.00 & 0.02 & 6.63 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.33 & 792.73 & 339.00 & 0.19 & 22.24 & 2,567.62 \\
& & D3A+P & 0.20 & 6.29 & 4.50 & 0.33 & 0.96 & 101.31 \\
\rowcolor{LightCyan}
& & D3A+D & 0.20 & 12.43 & 1.50 & 0.29 & 0.94 & 104.95 \\
\cline{2-9}
& \multirow{3}{*}{Any object} & Naive & 0.93 & 2,558.00 & 2,558.00 & 0.01 & 119.74 & 926.67 \\
& & D3A+P & 0.20 & 38.00 & 33.00 & 0.24 & 0.82 & 116.15 \\
\rowcolor{LightCyan}
& & D3A+D & 0.13 & 75.00 & 25.00 & 0.20 & 1.12 & 89.19 \\
\hline
\multirow{9}{*}{Q2} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 2.55 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 6.94 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 10.20 & 1.00 & 1.00 & 0.03 & 23.23 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.88 & 861.00 & 861.00 & 0.04 & 25.99 & 2,108.60 \\
& & D3A+P & 0.31 & 7.50 & 7.00 & 0.31 & 1.63 & 55.81 \\
\rowcolor{LightCyan}
& & D3A+D & 0.38 & 18.00 & 3.50 & 0.18 & 1.54 & 73.26 \\
\cline{2-9}
& \multirow{3}{*}{Any object} & Naive & 0.94 & 2,558.00 & 2,558.00 & 0.01 & 117.96 & 996.27 \\
& & D3A+P & 0.25 & 38.00 & 33.00 & 0.13 & 0.85 & 173.65 \\
\rowcolor{LightCyan}
& & D3A+D & 0.38 & 75.00 & 25.00 & 0.10 & 1.14 & 153.19 \\
\hline
\multirow{9}{*}{Q3} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 5.05 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 4.15 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 5.50 & 1.00 & 1.00 & 0.02 & 3.53 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.67 & 96.00 & 96.00 & 0.25 & 336.60 & 110.58 \\
& & D3A+P & 0.27 & 1.50 & 1.50 & 0.36 & 1.60 & 9.20 \\
\rowcolor{LightCyan}
& & D3A+D & 0.47 & 1.00 & 1.00 & 0.33 & 1.35 & 5.68 \\
\cline{2-9}
&\multirow{3}{*}{Any object} & Naive & 0.86 & 684.00 & 684.00 & 0.08 & 2,723.35 & 160.27 \\
& & D3A+P & 0.14 & 14.00 & 14.00 & 0.21 & 13.25 & 31.45 \\
\rowcolor{LightCyan}
& & D3A+D & 0.07 & 12.00 & 12.00 & 0.26 & 11.51 & 28.93 \\
% \cline{2-9}
\bottomrule
\end{tabular}
\vspace{-2mm}
\caption{\small Exp 2 - Retrieval Performance for the three types of queries. The highlighted row corresponding to D3A+D shows performance of our deployed D3A with real detector. D3A+D's performance is better than the Naive Baseline Aggregation method and sightly worse than D3A+P (D3A's implementation with a perfect object detector) in all the query types. $\uparrow$ means higher is better, $\downarrow$ means lower is better.}
\vspace{-3mm}
\label{table:queryperformance}
\end{table*}
% \begin{table*}[!htbp]
\subsubsection{Exp 1 - Retrieval Performance with a Synthetic Embedding Generator with Increasing Uncertainty} \label{sect:falsepositiveexp}
The performance of D3A depends on the accuracy of the object detector and object pose estimates. In this experiment, we design a synthetic embedding generator that replaces the object detector and color histogram. Our synthetic embedding generator is parameterized with a false positive rate ($fpr$)\footnote{ \label{note2} $fpr$ denotes the probability of falsely labelling the detected instance} and false negative rate ($fnr$)\footnote{$fnr$ denotes the detector's probability of not detecting the object}. We keep the $fnr$ low since we assume that the object is not in the environment if the detector fails to detects the object. We annotate ground truth object embeddings as one-hot encodings $ohe$ of the unique instances, where each index represents the unique id of a ground truth object. We then use the $fpr$ and $fnr$ to flip the bits of the $ohe$ and return the transformed encoding as the synthetic embedding generator's output.
%\stnote{You don't need to give pseudocode for this if you need more space.} \tn{+1}
% We measure the MRR and Miss Rate with varying the $fpr$ - level of uncertainty of our Embedding Generator and compare it with the ``Naive" baseline.
\textbf{Results}: We tested our system on Q1 type - \textit{Did you ever see an orange bowl?} for all ground truth object instances as mentioned in Table \ref{tab:example_queries}. Results are shown in Fig-\ref{fig:graph_noise}. Our method's MRR@50 decreases as $fpr$ increases. Despite this trend, D3A's MRR@50 remains considerably higher than Naive's MRR@50. This indicates that even when the object detector used in D3A is not good, the user will always find the correct object's frame earlier with D3A than with the Naive baseline. D3A's Miss Rate increases as the synthetic detector becomes noisier, but is always lower than Naive's Miss rate.
The slight decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with a higher probability of flipping the bits of the $ohe$, the probability of different object instances being assigned a similar embedding also increases. This causes D3A to consider a higher proportion of object instances to be similar, leading to more cluster formations (because of different locations) and storage of their respective keyframes. At the time of retrieval for a given object, this causes more keyframes to be returned one of which includes the queried object.
% The slight decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with a higher probability of flipping the bits of the $ohe$, the probability of different object instances being assigned a similar embedding also increases. This causes D3A to consider a higher proportion of object instances to be similar even when they are not, leading to more detections being used for cluster formation, increasing the number of clusters formed and greater storage of their respective keyframes. At the time of retrieval this causes keyframes to be returned of more different objects and at least one of those frames includes the relevant object.
\tn{i don't get how this means a lower miss rate}\iinote{improve this?}
% \textbf{Results}: We tested our system on all the query types types in Table \ref{example queries} and measure the average MRR and Miss Rate for all three.
% The MRR@50 of our method shows a decreasing trend as $fpr$(noise level) increases and remains above the baseline. This indicates that even if the synthetic detector is not good the user always finds the correct key frame earlier than the Naive. The decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with higher probability of flipping the bits, the probablity of different object instances being assigned a similar embeddings also increases that is D3A considers more object id's to be similar, clustering and storing their keyframes more often.
% of the instances among clusters to become highly similar, recognizing causing it to detect objects which are not in the scene.
% We believe that the noise level of Naive remains constant as every independent detection even if noisy is added to the database and when searching for ground truth instances, some frame amongst the top 50 frames returned contains a correct detection.
% --------------------------------------------
% We tested our system on query type 1 in Table \ref{example queries}. \stnote{I would just name the query again here, e.g., Did you ever see an orange bowl?}
% As the $fpr$ of our synthetic object detector increases, the MRR@50 of the retrieved frames for every $gt$ decreases and the Miss rate increases until it hits 0.3 after which the Miss Rate drops, as shown in Figure \ref{fig:graph_noise}. This is because fliping of the bits in the ohe happens so much the probability of detections having $ohe$ with similar cosine similarity increases.
% is wrongly assigned some other detection
% there is no clustering going on and every mutated object represents detection of some other object. This causes a lot of wrong detections which are present in the database and end up being in the top 50 answers for queries.
% Except for the peak at around $fpr=0.3$, the Miss Rate of D3A is lower than the Miss Rate of `Naive', showing that D3A is able to accurately answer queries for more ground truth object instances than the Naive. The MRR@50 and Miss Rate for the `Naive' remains constant despite the changing noise level because of the intrinsic distribution of objects in our dataset. Flipping of the bits assigns the object instances ids
% As the bits are flipped more with increasing fpr, the
% Beyond a $fpr$ of $0.2$, D3A starts performing worse than the ``Naive" baseline. The noise in our deployed system with a real object detector is $0.18$. As the $fpr$ increases, \stnote{cut: the reason that} the Miss rate increases \stnote{cut:is} because the probability that the instance will be assigned the same $ohe$ decreases, and D3A starts to label these new instances as either noise (since it has only seen them in a frame of a sliding window) or instances of another category. \stnote{Can you compare this to the baseline? I'm confused because in Figure 4 it looks like the baseline's performance doesn't change with regard to noise level...}
% D3A starts to consider the objects with similar vector less often hence labelling it as noise or labelling it as some other category which is an intrinsic property of the increasing fpr of synthetic detector.
% \stnote{make this a float, with an algorithm block and a title}
% \begin{lstlisting}[language=Python, caption={Embedding Generator Implementation}, float, label=lst:sensor]
% for indices not groundtruth_index:
% if prob < self.false_positive_rate:
% one_hot_encoding[i] = 1
% else:
% one_hot_encoding[i] = 0
% for groundtruth_index:
% if prob < 1- self.false_negative_rate:
% one_hot_encoding[groundtruth_index] = 1
% else:
% one_hot_encoding[groundtruth_index] = 0
% \end{lstlisting}
\subsubsection{Exp 2 - Query Retrieval Performance}\label{sec:exp3}
Our second experiment compares the retrieval performance of two versions of D3A against the Naive Baseline. The first is $D3A+D$ - D3A with a real detector, Dectectron \cite{Detectron2018}, and color histogram and second is $D3A+P$ - D3A with a synthetic perfect sensor.
\iinote{two versions of D3A against the Naive Baseline. The first is $D3A+D$ - D3A with a real detector, Dectectron \cite{Detectron2018}, and color histogram and second is $D3A+P$ - D3A with a synthetic perfect sensor.}
% D3A with a real detector -- Dectectron \cite{Detectron2018} and color histogram ($D3A+D$) against D3A with a synthetic perfect sensor ($D3A+P$) and the ``Naive" baseline.
We experimented over the 3 query types in Table \ref{tab:example_queries} and the 59 ground truth object instances annotated in the data. The number of instances per query ranged from 15 -20 depending on the query, making the total number of queries to be 150.
% \tn{How does the math work here? 3*59 is not 426, and idk where the 15-20 fits in..}
% For the given \iinote{70} queries, we test the retrieval performance our \textit{deployed system} - on Kuri with real object detector.
\begin{figure}
\centering
\includegraphics[width=0.85\linewidth]{images/Figure_1.png}
\caption{Exp 3 - Cumulative number of insertions to the database per hour}
% \label{fig:a}
% \tn{seems like there's extra white space around the figure, trim that to get back some space}
\label{fig:graphcumulative}
\vspace{-4mm}
\end{figure}
\textbf{Query Matching:} Another factor affecting the retrieval performance in this experiment is the matching of the desired object in the natural language query to an $ObjectID$ in the database. To examine this axis, we discretize the query matching precision into three levels:
\begin{itemize}
\item \textit{Perfect}: The query matching component can map an object in the language query to an exact $ObjectID$ in the database, and removes any level of uncertainty.
\item \textit{Category}: The matching component is noisier and is unable to uniquely identify the queried object (\textit{e.g.,} orange bowl), and instead returns a set of $ObjectID's$ for all objects from the same category (\textit{e.g.,} bowl).
\item \textit{Any Object}: The matching component is the noisiest and cannot extract even the category-level information from the language query, and thus maps the query to the most generic set of all the $ObjectID's$ in the database.
\end{itemize}
% We report MRR for the top 50 returned frames for \textbf{U, C, A} labelled as MRR (N=50) in Table-\ref{table:results1}.
\textbf{Results:} The results are shown in Table \ref{table:queryperformance}.
With \textit{Perfect Query Precision} for all three types of queries Q1 [\textit{rows: 1-3}], Q2 [\textit{rows: 10-12}], and Q3 [\textit{rows: 19-21}], both the mean retrieval time and total evaluation time of $D3A+D$ are less than those of the ``Naive" baseline. On average, $D3A+D$'s total (retrieval + evaluation) time is \textbf{1.96x} slower than $D3A+P$ and \textbf{1.48x} faster than the baseline. This translates to improved user experience as the users' query will be answered more quickly and as expected the miss rate is 0 because of the perfect query matching precision.
With \textit{Category Query Precision} for Q1 [\textit{rows: 4-6}], Q2 [\textit{rows: 13-15}], and Q3 [rows:\textit{22-24}], the mean number of frames returned by $D3A+D$ is \textbf{55x} smaller than that of the ``Naive" baseline, and only \textbf{5 counts}
% \tn{is this correct? you should double-check your numbers just in case}
higher than $D3A+P$'s on average. The MRR@50 for $D3A+D$ is \textbf{0.26}, which signifies that if the queried object exists in the database, it will on average be found in the fourth frame. An empty set is returned for negative queries. While the baseline on average finds the object in the sixth frame, the probability of it not finding an object of interest in the first 50 frames (average miss rate) is \textbf{62.5\%}, which is really high compared to $D3A+D$'s \textbf{34.7\%}. On average, $D3A+D$ took \textbf{23ms} to retrieve and evaluate all the frames for all the object instances while the baseline took \textbf{500ms}. These evaluation times, however, are just for 3 hours of data. These times will increase linearly with the amount of sensor data for the ``Naive" and increase the human users' waiting time for their query to be answered.
\begin{table}[t]
\small
%\stnote{use booktabs, avoid vertical rules}
%\tn{i rounded up decimals to have max 4 decimal places.}
% \iinote{since they are so many number, I highlighted those that were a little more important and that I discuss even in the evaluation results section but maybe there is a better way to do that?} \tn{maybe add a footnote to say that ``the bolded numbers are highlighted results that are discussed in the text" or something?} }
% \vspace{-3mm}
\centering
\begin{tabular}{p{3cm} l l l}
\toprule
{\bf{Performance Metric}} & & \bf{Naive} & \bf{D3A+D}\\
\midrule
Database Size (MB) & & $3312$ & $244$ \\
\hline
Total Processing Time & & $0$m $8$s & $2$m $26$s \\
\hline
Average Query Response Time (ms) & & $503.59$ & $10.63$ \\ %9.544548022
\hline
Total Number of Unique Object IDs & & $\bf{2558}$ & $\bf{25}$ \\
\hline
\multirow{2}{3cm}{Number of Duplicates per Object} & Mean & $ 44.10 \pm 75.66$ & {$1.19 \pm 0.46$} \\
& Max & $369.00$ & $3.00$ \\
\hline
Mean Accuracy (\%) & {} & $48.85$ & $81.98$ \\
\hline
\multirow{3}{3cm}{Average Number of Keyframes Returned} &
U & $1.00$ & $8.73$ \\
& C & $\textbf{583.24}$ & $\textbf{10.47}$ \\
& A & $\textbf{1933}$ & $\textbf{54.00}$ \\
\hline
\multirow{3}{3cm}{Mean Reciprocal Rank ($N=50$)} &
U & $1.00$ & $1.00$ \\ %0.9166666667
& C & $\textbf{0.16}$ & $\textbf{0.26}$ \\ %0.4476691691
& A & $0.03$ & $0.19$ \\ %0.3578010691 %0.4195961824
\bottomrule
\end{tabular}
\vspace{-2mm}
\caption{System Performance (On 3 Hours of Data)}
\footnotesize{* The bolded numbers are highlighted results discussed in the text}
\label{table:results1}
\vspace{-3mm}
\end{table}
The worst retrieval performance is with \textit{Any Object Query Precision} for Q1 [\textit{rows: 7-9}], Q2 [\textit{rows: 16-18}], and Q3 [\textit{25-27})]. Even then, $D3A+D$ returns \textbf{54} frames on average, compared to the \textbf{30} frames returned by $D3A+P$ and \textbf{1933} frames returned by the ``Naive" baseline. This difference is also reflected in their retrieval and evaluation times. The baseline on average finds the queried object in the 25th frame ($MRR=0.04$), while $D3A+D$ and $D3A+P$ finds the object around the fifth frame. Even with the worst query precision where the queried object is mapped to all the $ObjectID's$ in the database, D3A still achieves better retrieval performance by clustering detections and thus looking through fewer instances.
%%%\stnote{This is the one where your algorithm is really really good though. So highlight that even if the detector isn't good, this compression algorithm helps because it clusters things so you have to look through fewer results.}
% that is even when we can not figure out what the object is that we are talking about, D3A can return a small set of keyframes where the correct object of interests lies in the 5th frame.
D3A's overall accuracy is \textbf{81.98\%} with a retrieval time that is \textbf{47x} faster than the ``Naive" baseline. This can lead to improved user experience since with D3A, the user's query will be answered by searching over a smaller number of key frames (by \textbf{97\%}) than with the baseline.
% \tn{what is the 97\% trying to convey?}
\subsubsection{Exp 3 - Compactness Comparison}
We measure the cumulative number of object id insertions in the database per hour over the complete 22 hours for both $D3A+D$ and the ``Naive" baseline. As seen in Fig-\ref{fig:graphcumulative}, the number of insertions per hour for $D3A+D$ is much less than that of the baseline and scales well as the amount of data increases, demonstrating D3A's success in aggregating partial views. D3A takes two orders of magnitude more time to process the raw data than the baseline, but outputs a representation \textbf{14.7x} more compact and efficient in answering questions, as shown in Table \ref{table:results1}. This demonstrates that D3A allows for scalable spatial-temporal representation of objects.
\subsection{Exp 4 - Comparison with the ``Non-spatial'' Baseline}
In this experiment, we visually compare the aggregated clusters formed for the unique object instances by $D3A+D$ and the ``Non-spatial" baseline. Fig-\ref{fig:clusters} shows the baseline's aggregated detections for the pink cup asked in Fig-\ref{fig:Kuri}. The baseline clusters all object instances that are similar in embedding space, irrespective of their location in the map. The images' different bounding box colors show the respective clusters they were mapped to by D3A based on both the embedding features and spatial information. The clusters formed by D3A most closely resemble ground truth. Furthermore, the ``Non-spatial'' baseline will fail to answer queries of type Q3 in Table \ref{tab:example_queries}.
\section{Discussion}
% \tn{cut: Our -deployed algorithm $D3A+D$, despite performing very well, is still unable to retrieve spatio-temporal information for all the ground truth objects.}
Our deployed algorithm $D3A+D$ shows memory and speed efficiency in clustering together partial views of the same object, even when the object has moved to different locations in the environment over time. In such cases, the same object in each of its new location is assigned to a different keyframe centroid, but is associated with the same $ObjectID$. This is shown in Fig-\ref{fig:clusters} where the pink cup is moved from the table in img3 to the kitchen counter in img4 and assigned to two separate keyframe centroid by D3A, but is assigned the same $ObjectID$, symbolized by the same bounding box color.
D3A can also disambiguate unique object instances with similar features. An example is shown in Fig-\ref{fig:clusters} where D3A assigns the two red cups in img1 and img2 to different clusters. However, our algorithm cannot perfectly retrieve all the locations a unique object was at every moment in time.
% \iinote{Add: Our deployed algorithm $D3A+D$ shows memory and speed efficiency in clustering together partial views of the same object even when moved to different locations in the environment over time. In such a case the same object in each of it's new location is assigned to a different key frame centroid but gets associated to the one object\_id. This is shown in Fig.\ref{fig:clusters} where the pink cup is moved from the table ``img3" to the kitchen counter ``img4" and is assigned two separate keyframe centroid by D3A since they are two locations but is assigned the same object\_id. D3A can also disambiguate unique objects instances even if they have similar feature. An example is shown in Fig.\ref{fig:clusters} where D3A assigns the two similar red cups in ``img1" and ``img2" different clusters. Despite such results, our algorithm is unable to perfectly retrieve all of the locations where a unique object was placed at every moment in time.}
% spatio-temporal information for every ground truth object.}
%%%Our deployed algorithm $D3A+D$, despite performing very well, is still unable to retrieve spatio-temporal information for all the ground truth objects.
% Overall, our deployed algorithm $D3A+D$ performs very well and correctly retrieves objects most of the time. However, it is still unable to retrieve the spatial-temporal information for all ground truth object instances.
This is because our system sometimes fails to distinguish objects when they have similar embedding features and are close together in Euclidean space (below $d_{thresh}$). In this case, D3A will assign the same $ObjectID$ and keyframe centroid to two unique object instances.
% This will cause D3A to miss storing the spatial-temporal information of the wrongly assigned object and.
An example of this is shown in Fig-\ref{fig:clusters} where img1 of the red cup is assigned to the cluster for the pink cup since they have similar colors and are close together in the physical world.
This causes D3A to miss the associated information of the wrongly assigned red cup and not be able to retrieve this information in response to an object retrieval query related to the red cup.
Furthermore, if an object is only seen in one frame of a sliding window, our algorithm will characterize it as noise to account for errors in the detection. This, however, causes D3A to sometimes miss unique object instances. Overall, D3A recognized 76 unique object locations in the map, but because of the above reasons assigns them to 25 object instances instead of the ground truth 59, while considering 18 of the objects as noise. As shown in Table \ref{table:results1}, the average number of duplicated $ObjectID's$ assigned to a ground truth object instance by D3A is $1.19 \pm 0.46$, which is 36X smaller than that of ``Naive."
% \iinote{Add: In case of Naive every partial view detection of the object is treated as a unique object leading to total number of unique detections to be greater than the total number of ground truth objects.}
In the case of the ``Naive" baseline, every partial view detection of the object is treated as a unique object, leading to a total number of unique detections much greater than the number of ground truth objects.
Fig-\ref{fig:graph_noise} and Table \ref{table:queryperformance} show that increasing the false positive rate of the visual detector and decreasing the query precision degrades object retrieval performance from the spatial-temporal representation created by D3A. Nevertheless, the deployed D3A+D algorithm on Kuri with the real-world detector performs better than the baseline in terms of memory and speed efficiency for object retrieval. D3A+D also performs better than ``Naive" despite the fact that they both suffer from drifting localization of the robot's pose over time. This is because D3A's three-tier clustering and filtering structure was able to filter out some of the localization drift as noise in the objects' sensor data.
We took the first step in aggregating the detections of objects over space and time with our proposed algorithm - D3A, and defer the reduction of duplicates by using improved deep-learned object embeddings instead of color histograms to future work. D3A can enable service robots to find misplaced objects for the elderly over long periods of time.
%The lost objects could be misplaced while remaining in robot's line of sight.
Our research also opens venues for object retrieval in other surveillance related applications. Our work has only been evaluated in an environment where the pre-built map was provided to us. It will be interesting to explore how our system can be adopted to work in unknown open domain environments. We plan to conduct a human user study in the future to investigate their experience when querying D3A and searching over the small subset of returned keyframes to find answers to their object retrieval queries.
% the improvement in comfort brought to the user by searching over fewer keyframes when querying spatial-temporal information of the objects.
% These reasons lead to duplicates in our system \textit{i.e.,} one object instance in the environment is mapped to multiple unique $object\_id$ s in the database.
% Further, because of the errors described above, the number of unique instances identified by our algorithm are 25 as compared to the actual 59. More probabilistic filtering approaches can be added to reduce the duplicates.
% In addition, D3A+D uses the object locations calculated based on the results of SLAM and thus suffers from drifting localization of the robot's pose over time as the Naive Nonetheless, D3A+D is still able to perform better than the ``Naive" because of the three-tier clustering and filtering structure that does noise filtering in the objects sensor data.
% \tn{doesn't the naive baseline suffer from the same problem?}
% \iinote{Add:
% % In Exp 2, we show the affect of varying query precision and table-\ref{table:queryperformance}. In the experiment runs for Exp 2 as shown in table-\ref{table:queryperformance} that we performeven with bad slam because it drift, our performs better than Naive. %
% Table-\ref{table:queryperformance} and Table-\ref{table:results1} show that increasing the false positive rate of the visual detector and decreasing the query precision degrades the performance of object retrieval from the spatial-temporal representaion created by D3A. Nevertheless, the deployed D3A+D algorithm on Kuri with the real-world detector performs better than the baseline in terms of memory and speed efficiency of object retrieval. Note that D3A+D uses the object locations calculated based on the results of SLAM and despite drifting localization results of the robot's pose over time, D3A+D is still able to perform better than the Naive Baseline.
% }
% \begin{figure}[!htbp]
% \centering
% \includegraphics[width=0.8\linewidth]{images/ablation_centroid.png}
% \caption{Exp4 - C}
% % \label{fig:a}
% \label{fig:graph_noise}
% \end{figure}
% \subsection{Results and Discussion}
% \subsubsection*{Exp 1:}
% \subsubsection*{Exp2:} \textbf{Compactness}:
% \textbf{Query Results}
% Sine most of the frames returned by the Naive do not contain the relevant object instance and many have duplicate views of the same object,
% our system takes takes {117x} faster than the baseline. \iinote{need to check results}.
% with a much higher miss rate\iinote{1.25x} since the first 50 frames do not contain the relevant object instance and many have duplicate views of the same object.
% This difference is also reflected in the retrieval and evaluation times.
% With \textit{category level query matching}, the mean frames returned by the deployed system are (50x) smaller than the mean frames returned by the baseline and only 2 counts higher than the perfect system. This difference is also reflected in the retrieval and evaluation times. On average our deployed system took \iinote{50ms} to retrieve and evaluate all the frames for all the object instances while the baseline took \iinote{2secs}. Keep in mind that these times are reported just for 3 hours of ground truth data. The MRR@50 for deployed system is on average \iinote{0.4} which signifies that on average the object of interest in the query is found in the third frame, while for the baseline it is \iinote{0.3} with a much higher miss rate\iinote{1.25x} since the first 50 frames do not contain the relevant object instance and many have duplicate views of the same object.
% There are still duplicates in our system \textit{i.e.} one object instance in the environment is mapped to multiple unique $object\_id$ s. The average number of $object\_id$s assigned to the same object instance by RoboMem is \iinote{23.6 $\pm$ 7.50}, which is \iinote{96\%} smaller than the number of duplicates in the baseline. Our algorithm takes two order magnitudes more time to process the data than the baseline however the representation is more compact and efficient in answering questions as shown in the results here.
% The overall accuracy of RoboMem is $81.98\%$ with $47x$ faster than the baseline. The retrieval performance of the query matching levels is shown to be in this order - Level 1, 2, 3 with level 3 performing the worst since more mean key frames need to be looked into.
% \iinote{talk about low frame rate?}
% \iinote{talk about reasons for missed frames}
% \iinote{The retrieval performance of the query matching levels is shown to be in this order - Level 1, 2, 3 with level 3 performing the worst since more mean key frames need to be looked into.}
\vspace{-3mm}
\section{Conclusion}
\begin{figure}
\centering
\includegraphics[width=0.7\linewidth]{images/final.png}
\caption{Exp 4 - Visualization of the keyframe centroid and the aggregated detections by the ``Non-spatial'' baseline \cite{hsieh2018focus} for the pink cup. All the square images are the detections and the different bounding box colors show the respective clusters they were mapped to by D3A based on both the embedding feature and spatial information. The dotted line connects clusters that were mapped to the same \textit{ObjectID}.}
\label{fig:clusters}
\vspace{-4mm}
\end{figure}
We present a novel algorithm for robots to efficiently answer spatial-temporal queries about objects in the environment over long periods of time. Our algorithm aggregates partial view detections of unique instances to create a compact and query-able representation of the objects. By explicitly performing detection-based three-level association to identify the keyframes for unique object instances, our algorithm significantly outperforms baselines in answering queries in terms of the retrieved frames' accuracy, mean reciprocal rank and retrieval time. A robot deployed with our algorithm was able to process 22 hours of
% \iinote{I evaluate on 3 hours? but do make compact for 22 hours so should it be 22 hours?} \tn{i'd say 22 bc it did process and compress that much data, we just don't know how well}
sensor data and develop a compact representation of objects in its environment, which is an encouraging step towards enhancing the sensory capabilities of home-service robots that can help the elderly find their lost, forgotten objects.
% \section{Copyright}
% All papers submitted for publication by AAAI Press must be accompanied by a valid signed copyright form. They must also contain the AAAI copyright notice at the bottom of the first page of the paper. There are no exceptions to these requirements. If you fail to provide us with a signed copyright form or disable the copyright notice, we will be unable to publish your paper. There are \textbf{no exceptions} to this policy. You will find a PDF version of the AAAI copyright form in the AAAI AuthorKit. Please see the specific instructions for your conference for submission details.
% \subsubsection{Acknowledgments.}
% The acknowledgments section, if included, appears after the main body of text and is headed ``Acknowledgments." This section includes acknowledgments of help from associates and colleagues, credits to sponsoring agencies, financial support, and permission to publish. Please acknowledge other contributors, grant support, and so forth, in this section. Do not put acknowledgments in a footnote on the first page. If your grant agency requires acknowledgment of the grant on page 1, limit the footnote to the required statement, and put the remaining acknowledgments at the back. Please try to limit acknowledgments to no more than three sentences.
% \subsubsection{Appendices.}
% Any appendices follow the acknowledgments, if included, or after the main body of text if no acknowledgments appear.
% The following commands are available for your use in citing references:
% \begin{quote}
% {\em \textbackslash cite:} Cites the given reference(s) with a full citation. This appears as ``(Author Year)'' for one reference, or ``(Author Year; Author Year)'' for multiple references.\smallskip\\
% {\em \textbackslash shortcite:} Cites the given reference(s) with just the year. This appears as ``(Year)'' for one reference, or ``(Year; Year)'' for multiple references.\smallskip\\
% {\em \textbackslash citeauthor:} Cites the given reference(s) with just the author name(s) and no parentheses.\smallskip\\
% {\em \textbackslash citeyear:} Cites the given reference(s) with just the date(s) and no parentheses.
% \end{quote}
\section{Acknowledgments}
This work was supported by NSF under grant award IIS-1652561 and an award from Echo Labs. The authors would like to thank Baber Khalid, Thao Nguyen, and Eric Rosen for their valuable feedback and insights on the research and Baber Khalid for being a supportive husband.
\bibliography{references}
\newpage
\section{Appendix}
% \section{Graph Appendix} \label{appendix:graph}
% \appendix
\subsection{Algorithmic Implementation} \label{sec-Algo Implementation}
We deploy D3A on a mobile robot Kuri \cite{heykuri_2018} that patrolled a robotics lab environment for four days. Our system performs real-time object detection with Facebook's AI Research software system - Detectron \cite{Detectron2018} to extract $c_{ij}, prob_{ij}$ for every detection. We use a color histogram \cite{color_hist} to approximate an embedding $emb_{ij}$ of each object instance detection. We select the color histogram method due to its simplicity. However, our algorithm is not constraint to using the color histogram as the instance feature extractor. More sophisticated deep learning methods that use RGB or RGB-D images \cite{bras2018unsupervised, xu2014deep} can also be used.
In the subsection Experiment 1 of section Evaluation, we show how varying the type of object detector (changing its false positive rate) affects the results of our algorithm. Notably, our approach is also not constrained to any particular number of objects, object size or shape, or object dynamicness (\textit{i.e.,} the objects' locations can change over time). We are only constrained by the object categories the detector was trained to detect. Processing of the visual data and pose estimates was done off-board on a stand-alone machine. The PC setup included an Intel(R) Core(TM) i7-6700 CPU (3.40GHz x 8), and a GeForce GTX 1070/PCIe/SSE2 GPU running Ubuntu 16.04. D3A was run online while the PC communicated with the Kuri robot over ROS \cite{quigley2009ros}. For Tier 1-clustering of detections across adjacent frames we use density-based spatial clustering (DB-Scan) \cite{khan2014dbscan}, a method known to be good at handling outliers/noise within the dataset. While other clustering methods can also be used, we defer the analysis of their impact on D3A's performance for future work.
All the aggregation operations $\odot$, $\oplus$ and $\otimes$ that we applied are the weighted average functions. These aggregation functions can also be different. Weighted average was selected because of its simplicity and reasonable empirical performance for object retrieval on our smaller dataset of $<3$ hours of data. More sophisticated methods can also be used in place of the weighted average. However, as shown in section -- Experiment Designs of Evaluation,
% -\ref{sec:exp-des},
with just the weighted average, D3A still performs better than the baseline.
We organized our aggregated spatial-temporal representation $R$ over two collections in \textit{MongoDB} \cite{banker2011mongodb}, a database that supports large volumes of both data and traffic. The two document Mongo-DB collections \textbf{Object Identification Collection} ($OIc$), and \textbf{Spatial-temporal Collection} ($STc$) are as described in Section -- Tier 3- Filtering across database entries of Approach.
% Sec-\ref{subsubsection:p3}.
$OIc$ in our implementation, stores the object information using the associate embedding of the cluster, the category assigned to the cluster and a weight which encodes the frequency of detections for this object and is used in the aggregation operation between $record$ in $OIc$ and $lrv\_entry$ evicted by the $STM$.
% \tn{the previous sentence doesn't make sense? Do you mean something like ``The two collections, $OIc$ and $STc$ are as described in section..."?}
For natural language processing of the queries, we annotate the queries (use the template-based method) to convert them into the logical form shown in Table \ref{tab:example_queries}. These logical forms are then used to query the collections for relevant spatial-temporal information. In the subsection Experiment 2 of section Evaluation we show how the performance of D3A changes with varying the uncertainty in the processing of the query but still it is able to perform better than the baselines.
% \iinote{We organize our aggregated spatial-temporal representation over two collections in \textit{MongoDB} \cite{banker2011mongodb}, a database that supports large volumes of both data and traffic. Our first collection in MongoDB is the \textbf{Object Identification Collection} ($OIc$) that stores object attributes of aggregated clusters such as the category, embedding and weight. We also have a separate \textbf{Spatial-temporal Collection} ($STc$) that stores documents containing a geo-spatial index for location, time intervals, the keyframe, and the detection probability and bounding box of the object in the keyframe.}
% \iinote{We annotate the queries to convert to the database query representation.}
% \section{Additional Results}
% \end{appendix}
% \appendix
\end{document}
\def\year{2021}\relax
%File: formatting-instructions-latex-2021.tex
%release 2021.2
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai21} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%\nocopyright
%PDF Info Is REQUIRED.
% For /Author, add all authors within the parentheses, separated by commas. No accents or commands.
% For /Title, add Title in Mixed Case. No accents or commands. Retain the parentheses.
\pdfinfo{
/Title (AAAI Press Formatting Instructions for Authors Using LaTeX -- A Guide)
/Author (AAAI Press Staff, Pater Patel Schneider, Sunil Issar, J. Scott Penberthy, George Ferguson, Hans Guesgen, Francisco Cruz, Marc Pujol-Gonzalez)
/TemplateVersion (2021.2)
} %Leave this
% /Title ()
% Put your actual complete title (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case
% Leave the space between \Title and the beginning parenthesis alone
% /Author ()
% Put your actual complete list of authors (no codes, scripts, shortcuts, or LaTeX commands) within the parentheses in mixed case.
% Each author should be only by a comma. If the name contains accents, remove them. If there are any LaTeX commands,
% remove them.
% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
\newcommand{\MYhref}[3][blue]{\href{#2}{\color{#1}{#3}}}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{color, colortbl}
\definecolor{airforceblue}{rgb}{0.36, 0.54, 0.66}
\definecolor{LightCyan}{rgb}{0.88,1,1}
\definecolor{codegreen}{rgb}{0,0.6,0}
\usepackage{array,etoolbox}
% \AtBeginEnvironment{table}{\rmfamily \small}
\preto\tabular{\setcounter{magicrownumbers}{0}}
\newcounter{magicrownumbers}
\newcommand\rownumber{\stepcounter{magicrownumbers}\arabic{magicrownumbers}}
\newif\ifpaperfinal
% \paperfinalfalse
\paperfinaltrue
\ifpaperfinal
\newcommand{\tn}[1]{}
\newcommand{\iinote}[1]{}
\newcommand{\ernote}[1]{}
\newcommand{\stnote}[1]{}
\else
\newcommand{\iinote}[1]{\textcolor{codegreen}{\textbf{II: #1}}}
\newcommand{\stnote}[1]{\textcolor{blue}{\textbf{ST: #1}}}
\newcommand{\tn}[1]{\textcolor{airforceblue}{\textbf{TN: #1}}}
\newcommand{\ernote}[1]{\textcolor{red}{\textbf{ER: #1}}}
\fi
% The file aaai21.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%
% Title
% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Where were my keys? - Aggregating Spatial-Temporal Instances of Objects for Efficient Retrieval over Long Periods of Time}
\author{
%Authors
% All authors must be in the same font size and format.
% Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
% AAAI Style Contributions by Pater Patel Schneider,
% Sunil Issar, \\
% J. Scott Penberthy,
% George Ferguson,
% Hans Guesgen,
% Francisco Cruz,
% Marc Pujol-Gonzalez
% \\
Ifrah Idrees, Zahid Hasan, Steven P. Reiss, and Stefanie Tellex\\
}
\affiliations{
%Afiliations
Dept. of Computer Science, Brown University, Providence, RI\\
% \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
%If you have multiple authors and multiple affiliations
% use superscripts in text and roman font to identify them.
%For example,
% Sunil Issar, \textsuperscript{\rm 2}
% J. Scott Penberthy, \textsuperscript{\rm 3}
% George Ferguson,\textsuperscript{\rm 4}
% Hans Guesgen, \textsuperscript{\rm 5}.
% Note that the comma should be placed BEFORE the superscript for optimum readability
% 2275 East Bayshore Road, Suite 160\\
% Palo Alto, California 94303\\
% email address must be in roman text type, not monospace or sans serif
% publications21@aaai.org
% ifrah
% See more examples next
}
\iffalse
%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Single Author}
\author {
% Author
Author Name \\
}
\affiliations{
Affiliation \\
Affiliation Line 2 \\
name@example.com
}
\fi
\iffalse
%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Multiple Authors}
\author {
% Authors
First Author Name,\textsuperscript{\rm 1}
Second Author Name, \textsuperscript{\rm 2}
Third Author Name \textsuperscript{\rm 1} \\
}
\affiliations {
% Affiliations
\textsuperscript{\rm 1} Affiliation 1 \\
\textsuperscript{\rm 2} Affiliation 2 \\
firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
}
\fi
\begin{document}
\maketitle
\begin{abstract}
Robots equipped with situational awareness can help humans efficiently find their lost objects by leveraging spatial and temporal structure. Existing approaches to video and image retrieval do not take into account the unique constraints imposed by a moving camera with partial view of the environment. We present a \textbf{D}etection-based \textbf{3}-level hierarchical \textbf{A}ssociation approach, \textbf{D3A}, to create an efficient query-able spatial-temporal representation of unique object instances in an environment. D3A performs online incremental and hierarchical learning to identify keyframes that best represent the unique objects in the environment. These keyframes are learned based on both spatial and temporal features and once identified their corresponding spatial-temporal information is organized in a key-value database. D3A allows for a variety of query patterns such as querying for objects with/without the following: 1) specific attributes, 2) spatial relationships with other objects, and 3) time slices. For a given set of 150 queries, D3A returns a small set of candidate keyframes (which occupy only 0.17\% of the total sensory data) with 81.98\% mean accuracy in 11.7 ms. This is 47x faster and 33\% more accurate than a baseline that naively stores the object matches (detections) in the database without associating spatial-temporal information.
\end{abstract}
\section{INTRODUCTION}
\iinote{color package in table}
\iinote{reduce font size of all tables}
\iinote{fix table 2}
\iinote{remove vspace}
Home-service robots have great potential to assist human users by retrieving spatial-temporal information\footnote{Spatial-temporal information of object refers to the whereabouts of the object such as where the object has been identified in the physical environment of the robot and at what times} about objects from their long-term observations. For example, a person can ask a simple query such as \textit{``where did I leave my keys?''} Service robots with such an ability will be well suited to help the elderly, especially those that have dementia.
There are two lines of works in image and video object retrieval: a) \textit{fixed-view cameras} \cite{object_retrieval, yadav2019vidcep, kang2019challenges} which assume full observability of the environment and do not deal with a moving camera (robot), and b) \textit{robotic object retrieval} \cite{ambrucs2014meta, bore2015retrieval} that either work on short-term time horizons (do not condense partial view detections of unique object instances for memory or speed efficiency) or perform long-term robotic object retrieval by specifically focusing on point cloud matching without incorporating object's spatial information in the physical environment. \iinote{object's spatial information in the physical environment, of where the objects are located in the environment}These approaches for object retrieval will leave the robot searching over countless detections in visual sensor data from many different time slices, which will take up a lot of space and time. In addition, these detections will contain partial views of different object instances, not all of these objects and their partial views will be\iinote{these objects and thus partial views will be} relevant to the query. An algorithm that condenses the partial view detections of each unique object instance into a compact and query-able spatial-temporal representation will enable the robot to answer queries about unique object instances more efficiently.
\begin{figure}[]
\centering
% \includegraphics[width=0.8\linewidth]{images/story_2.png}
\includegraphics[width=0.75\linewidth,height=6cm]{images/story_2.png}
% \caption{Kuri Robot \citep{richardson_2018} \stnote{I would replace this with a storyboard of the system, showing it answering a question about where an object is. Also, make sure this is our picture of Kuri, use one you took that we have rights to.}}\label{fig:a}
\caption{The image shows the retrieved keyframes and associated temporal information for two sample queries answered by D3A while deployed on Kuri robot. The keyframes are overlaid on the map using the spatial information returned by our spatial-temporal representation. Also shown are the partial view detections that are aggregated into the keyframe cluster.}
\label{fig:Kuri}
\vspace{-3mm}
\end{figure}
To our knowledge, there has not been a single work that addresses all of the following challenges: 1) aggregating detections of object instances from different views in a map over hours of sensory information, 2) handling the object going out of the view, and 3) accounting for uncertainty in object poses due to noise from the robot pose. To mitigate these challenges, we introduce a new algorithm, D3A that extends the concept of keyframe extraction that has been previously used for video frame retrieval to condense
% \iinote{Add: that extends the concept of keyframe extraction that has been previously used for video frame(s) retrieval to condense}
multiview detections of unique objects over both physical space and time for long observation periods.
D3A extracts both spatial-temporal features of object instances in the environment as they get seen by the robot.
% Treating all these partial views of an object as unique and/or storing all of their spatial-temporal information will be memory and speed inefficient for object retrieval. To mitigate these challenges,
D3A associates these partial view features \iinote{partial view features instead of views} with what it has previously seen over the map to condense them.
% D3A enables scalable association and object retrieval by
It does so by performing three-tier online, incremental clustering and filtering of unique spatial-temporal instances. While performing association, D3A keeps track of the keyframe that best represents each unique spatial-temporal location of the object. The information about keyframe-centroid clusters are then spatial-temporally indexed in a key-value database, leading to a compact and query-able representation for efficient object retrieval.
This representation allows D3A to identify and return a small set of keyframes for the object(s) in a given query, which enables a person to quickly find the relevant information. D3A is storage efficient and is able to handle the case where the attributes of object instances (color, shape, patterns) asked for in the queries are not known in advance. In prior work all or subset of these attributes were provided at the time of query.
Our algorithm is deployed on a social robot Kuri from \citet{heykuri_2018} that patrolled a robotics lab\iinote{robotics lab} environment and collected 22 hours of observations over four days. \footnote{\label{note1} A visualization recording of the initial state of the environment, along with its 2D map and sample of the collected dataset can be found at this link: https://github.com/IfrahIdrees/D3A.git}. When tested on a set of 150 queries about different objects, D3A returns a small set of keyframes and their spatial-temporal information (which \iinote{on average} occupy only 0.17\% of the total sensory data\iinote{sensory data of object of interests}) and is 47x faster than the baseline model which treats all partial-view detections as independent instances. D3A is also 33\% more accurate than the baseline with an accuracy value of 81.98\%.
\section{Related Work}
% \ernote{I feel like you should discuss SLAM and semantic mapping literature. I know you're just using off-the-shelf algorithms for SLAM that comes with Kuri, but there's semantic mapping literature \cite{pronobis2012large} that track object attributes, though i dont think they ever focus on the memory side of things like D3A does.}
% \iinote{\citet{hsieh2018focus} proposed work similar to ours that can handle many hours of videos captured by a moving camera, except that it does not deal with the objects' spatial information within the global map and only tries to identify video frames with objects of certain classes. We perform an ablation study with this work and show that by using the objects' locations in the map, D3A improves 1) detection of unique object instances and 2) compactness and retrieval performance from our spatial-temporal database.}
% Video processing has advanced quite dramatically over the years in terms of fundamental algorithms for detection, and re-identification of objects.
% \iinote{How key frame detection works. different techniques for key frame extract like clustering methods, energy minimization algorithms or sequential techniques \cite{vazquez2013spatio}. Usually key frame extraction is used to identify a subset of a video frames that can represent the visual content of the sequence as close as possible, but with a limiting number of frame information And usually the spatial-temporal information used for clustering is based on the visual descriptors of the frame . Clustering techniques differ in how they define an appropriate similarity measure between visual descriptors of the frames. }
In the past, \textbf{Keyframe Extraction} techniques have been used to extract the summary key frames to represent video sequences \cite{6135507}.
% \cite{5201141, 6135507, lingam2019key, event, object_tracking}.
A lot of different techniques exists for keyframe extraction such as clustering \cite{vazquez2013spatio}, energy minimization \cite{essa2015modular}, or online techniques \cite{elahi2020online}.
% These differ in how they define the similarity measure between visual descriptors of the frames.
% \tn{the previous sentence feels out of place and should be cut. You can just say that D3A extendes the clustering keyframe extraction technique to ...}
D3A extends the clustering keyframe extraction technique in videos to incrementally find keyframes that best represent the different views of the unique objects in the environment over time, as the robot continues to capture new video frames.
% \iinote{Add: In the past \textbf{Key frame Extraction} techniques have been used to extract the summary key frames to represent video sequences \cite{5201141, 6135507, lingam2019key, event, object_tracking}. A lot of different techniques exists for keyframe extraction such as clustering, energy minimization or sequential techniques \cite{vazquez2013spatio}. These differ in how they define an appropriate similarity measure between visual descriptors of the frames. D3A extends keyframe extraction in videos to incrementally finding keyframes that best represent multi views of the unique objects in the environment over time, as the robot continues to capture new video frames.}
A lot of work has also been done in \textbf{Object Detection} \cite{MaskRCNN, RetinaNet, FastCNN, pose_estimation} for detecting and re-identifying objects. D3A is built upon the notion of object detection in a three-tier mechanism to match objects and condense their detections over long periods of observations. Additionally, there are \textbf{Object Tracking} algorithms that track object instances over time \cite{object_tracking, yilmaz2006object}. However, they usually assume input videos with fixed angles and cannot track/associate objects under occlusion or after the objects go out of view.
% \iinote{In past keyframes is a method used to extract the perform clustering of consecutive frames to find representative keyframes for all the consecutive frames video trying to find unique views of scene. we extend that to actually find keyframes to associate multi views of the object for object association over space and time.They differ in how they define an appropriate similarity measure between visual descriptors of the frames. }
% These algorithms don't scale well with the number of objects.
These algorithms are also not useful for retrieval since they do not focus on making the objects' trajectories query-able.
% \tn{can cut: We use an unsupervised clustering method in our implementation which can be replaced by any other techniques.}
% \ernote{What other techniques are likely? I feel like you mean that you chose a specific unsupervised clustering algorithm X, but any unsupervised clustering algorithm could be used?}
Although many \textbf{Image or Video Retrieval} systems have been introduced in the multimedia community, they focus on retrieval at the level of object classes and not instances. These works can be categorized into the following types:\\
\textit{1) Fixed-angle Video Input} \cite{object_retrieval, yadav2019vidcep, kang2019challenges}: These works do not account for the objects' spatial information in a global map. In contrast, D3A can perform object instance association in a partially observable environment monitored by a robot.\\
\textit{2) Query Optimized Object Retrieval} \cite{kang2019challenges}: This work uses information from the query to optimize the retrieval pipeline by searching over all the scaled down image frames in the original dataset. They do not reduce redundancy in the data,
% Their search space is over all the frames and they do not focus on reducing the redundancy in data \cite{kang2019challenges}.
% Our algorithm instead focuses on optimizing the spatial-temporal representation to be used for retrieval. We focus on condensing the detections to only store key frame information for retrieval.
whereas D3A focuses on only storing the keyframe information
% to condense the detections
and optimizing the spatial-temporal representation.
There are also works that can handle partial views of objects and perform view-invariant object detection at the category level \cite{hsieh2018focus, sivic2003video}.
% However, they do not record the objects' spatial information and cannot answer spatial queries like D3A.
\citet{hsieh2018focus} proposed work similar to ours that can handle many hours of videos captured by a moving camera (that is not a robot) but does not use object locations. We perform an ablation study with this work and show that by using the objects' locations in the map, D3A improves 1) detection of unique object instances and 2) compactness and retrieval performance from our spatial-temporal database. \citet{sivic2003video} assume to have seen a part of the scene captured by different static camera viewpoints to learn the visual vocabulary (which incorporates spatial information) to associate objects seen in later scenes. We perform online, incremental object matching, and do not pre-define the number of clusters/objects in the environment.
% Unlike both of these works, our clustering is based on both the object feature space and spatial-temporal information.
% There are other works that deal with robotic object retrieval but over short-time horizons and therefore they do not condense the partial views for memory and speed efficiency.
% Another relevant line of work is robotic object retrieval. Most works in this line operate over short-time horizons and do not condense partial view detections of unique object instances for memory or speed efficiency \cite{Temporal-Grounding} \tn{feels like you're missing citations}.
% \tn{what's the difference between these works and the ones in the previous paragraph?}
There are also works that specifically focus on long-term robotic object retrieval that handle partial observability in visual sensor data \citep{ambrucs2014meta, bore2015retrieval}. However, these works assume a point cloud of objects as input and introduce matching algorithms specifically in 3D space. D3A, in contrast, does matching using the RGB feature space and limited depth information (information provided by the depth sensors for obstacle detection) that can be derived from the 3D space. \tn{is this better bc it requires ``less" visual information or just different} These works also do not deal with the objects' spatial information in the environment. Furthermore, they do not focus on making a compact representation of the sensory data for retrieval, but instead store and operate over the original data and use the object attribute ``shape" provided in query to optimize retrieval.
% \tn{This also seems similar to the query optimized object retrieval works described above. Are these the works that require that the queried object attributes are known in advance? This bit is not explicitly stated anywhere}
% \stnote{use citep, not citet, only use cite for noun phrases. for multiple cites, put them in one citep command separated by commas }
% \begin{figure}[t]
% \centering
% \includegraphics[width=\linewidth]{images/high_level.png}
% \caption{Block Level diagram of D3A \tn{D3A}}
% % \roma{Trim the edges so they fit exactly and also make the figure width the line width so this fits the page nicely and the font is easier to read :-)}}
% \label{fig:a}
% \label{fig:block-diagram}
% \end{figure}
\begin{figure}[t]
\centering
\includegraphics[width=0.8\linewidth]{images/Processing_diagram.png}
\caption{\small Visualization of D3A Algorithm \iinote{Add Qi, si}}
% \label{fig:a}
\label{fig:Processing}
\vspace{-6mm}
\end{figure}
% \tn{this is super readable now! i recommend using a different color (darker green maybe) to label the tiers bc the current color is a little hard to see (same for Fig 3)}
\section{Problem Formulation}\label{Section:system_overview}
% \stnote{So you want to write technical approach in a more problem-driven way. Think of it as a tree of problems and solutions. You don't want to introduce a bit of technical material until you've first explained the problem you need to solve by that bit. The paragraph below is just saying what you built without saying why or situating it in what problems you are solving.}
% \iinote{how link speed and memory efficiency}
\iinote{fix time variable as t?}
A robot monitoring the environment over long periods is given a spatial-temporal query $Q$ to retrieve spatial temporal information of target objects seen in the environment. The query can be of any type as listed in Table-\ref{tab:example_queries}. The robot is equipped with the map of the environment and at any given time step $i$ can gather the following sensory information $s_i=<p_{robot,i}, d_i, f_i>$, where $p_{robot,i} \in P_{robot}$ is the associated robot's pose, and $d_i \in D$ is the corresponding depth information for image frame $f_i$. The robot collects large amounts of sensor data $S = \{s_1, s_2, ... , s_i\}$ and uses an object detector to extract the relevant object-centric information $a_i$ from every frame $f_i$. However, the robot has partial observability of the environment and the information required to answer the object retrieval query is going to be dispersed throughout $S$ in the form of multiple partial view detections. To answer object-retrieval queries efficiently, the robot needs to condense the multi-view detections of the objects into a memory efficient and query-able representation. This representation is then used by the robot to return a small set of keyframes, which enables a person to quickly find the relevant information about the target object.
% The sensor data $S$ will take up a lot of disc space and be inherently noisy. Searching all of this data for retrieving object's spatial-temporal information is going to be speed-inefficient.
% % The robot should also be able to answer queries where the attributes of object instances (color, shape, patterns) are not known in advance.
% For the robot to retrieve an object's spatial temporal information in a memory and speed efficient, it should be able to condense the multi-view detections of the objects over time and be able to organize them in queryable representation such that given a query $Q$, it can return a small set of keyframes, which enable a person to quickly find the relevant information about the target object.
\section{Approach}
% all this sensor data $S$ through which the object need to be retrieved takes up a lot of disc space and is inherently noisy.
% Equipped with $S$, the robot is given a spatial-temporal query $Q$ of any of the type listed in Table-\ref{tab:example_queries}. One example is ``Did you ever see an orange bowl" and is tasked to retrieve spatial temporal information of target objects seen in the environment. Ho
% A human given a robot collecting $S$ can ask from their personal assistant robot a spatial-temporal query - $Q$ about any object in the environment seen during any time of its patrol such as can be "Did you ever see an orange bowl".
% $=< p_{robot,i}, d_i, f_i>$ is the associated information for every captured image frame $f_i$. $p_{robot,i} \in P_{robot}$ is the associated robot's pose, and $d_i \in D$ is the corresponding depth information, for every image frame $f_i$.
% Note that the next time step $i+1$ is dependant on the frequency at which the robot is capturing the sensor data.
% All this sensor data $S$ takes up a lot of disc space and is inherently noisy.
For efficient spatial-temporal object retrieval in a partially observable environment, we need to compress this data $S$ into a compact and query-able spatial-temporal representation $R$. For this we first pre-process every $s_i$ to extract the relevant object-centric information $a_i$, then apply a three-tier online clustering algorithm to aggregate $a_{i}$'s. The output of D3A is a set of clusters of unique object instances that can be aggregated into the previous time step's compact representation $R_{i-1}$ to create a new representation $R_i$ of the environment for time step $i$. A diagram of our algorithm creating $R$ is shown in Fig-\ref{fig:Processing}.
% \stnote{problem formulation is only the problem, not the solution. So don't put anything about preprocessing etc in the problem formulation.}
% \vspace
\subsection{Pre-processing each frame:}
% For compactness and speed efficiency, D3A needs to reduce the dimensionality of the visual sensory data $f_i$ in $s_i=< p_{robot, i}, d_i, f_i>$, as well as aggregate multiple detections of the same object over long periods of time. D3A achieves this by extracting minimal amounts of relevant object-centric information $a_i = <C_i, Prob_i , B_i, P_{object,i}, Emb_i>$ for every frame $f_i$. There is a set $J_i$ for every $f_i$ that includes all the detected object instances in $f_i$. $C_i$ is the list of corresponding categories for all the detections in the set $J_{i}$,
% % for each detection $j \in J_i$ and $J_i$ represents the set of all detections in $f_i$,
% $Prob_i$ are the probabilities of detecting the object categories $C_i$, $B_i$ is the list of the object bounding boxes for $J_i$, $P_{object,i}$ is the list of object poses estimated for all the detections. $P_{object,i}$ is estimated from the robot's pose $p_{robot,i}$ and limited depth information $d_i$, and is a 3D world coordinate. $Emb_i$ is the list of embeddings corresponding to the detections in $J_i$, which is used to relate $j's$ over time and create the compact representation $Q$.
For compactness and speed efficiency, D3A needs to reduce the dimensionality of the visual sensory data $f_i$ in $s_i=< p_{robot, i}, d_i, f_i>$, as well as aggregate multiple detections of the same object over long periods of time. D3A achieves this by extracting minimal amounts of relevant object-centric information ${a_i}$:
\begin{center}
$a_i = [ <c_{i1},prob_{i1},b_{i1},p_{robot,i1},emb_{i1}>, <c_{i2},prob_{i2} ,b_{i2},p_{robot,i2},emb_{i2}>,..., <c_{ij}, prob_{ij},b_{ij},p_{robot,ij},emb_{ij}>]$
\end{center}
for every frame $f_i$, $c_{ij} \in C_i$, is the category detected for the $j^{th}$ object instance detection in $i^{th}$ image, $prob_{ij}$ is the probability of $j^{th}$ object being assigned the category $c_{ij}$, and $b_{ij}$ and $emb_{ij}$ is the bounding box and object embedding for the $j^{th}$ detection in $f_i$. $emb_{ij}$ is used to relate it with the other objects over time and create the compact representation $R$.
% There is a set $J_i$ for every $f_i$ that includes all the detected object instances in $f_i$. $C_i$ is the list of corresponding categories for all the detections in the set $J_{i}$,
% % for each detection $j \in J_i$ and $J_i$ represents the set of all detections in $f_i$,
% $Prob_i$ are the probabilities of detecting the object categories $C_i$, $B_i$ is the list of the object bounding boxes for $J_i$, $P_{object,i}$ is the list of object poses estimated for all the detections. $P_{object,i}$ is estimated from the robot's pose $p_{robot,i}$ and limited depth information $d_i$, and is a 3D world coordinate. $Emb_i$ is the list of embeddings corresponding to the detections in $J_i$, which is used to relate $j's$ over time and create the compact representation $Q$.
% % Overall, $a_{ij} = \{ c_{ij}, prob_{ij} , b_{ij}, p_{robot,ij}, emb_{ij} \}$ represents the corresponding tuple of object centric information for the $jth$ object instance detection (such as a red cup) in the $f_i$ image frame.
% \iinote{check if the ratio between height ad width is an attribute}
% $p_{object,ij}
% \tn{$l_{ij}$ is undefined} in the map.
% we $s_i$ by passing through an object detector to extract minimal amounts of relevant abstract information $a_i = <C >$. $a_i$ includes $C$
% the output of a \st{pretrained} object detector including probabilities of detecting the object categories, their bounding boxes, and object poses are estimated from the robot's pose $(x,y,z)$ and limited depth information.
% \iinote{Move to next section:We use a color histogram \cite{color_hist} to approximate an embedding of each object instance to relate them over time. We select the color histogram method due to its simplicity, and design the later processing levels to handle a noisy instance embedding generator. Notably, our approach is not constrained to any number of objects, particular object size or shape, or object dynamicness (\textit{i.e.,} the objects' locations can change over time). We are only constrained by the object categories the detector was trained to detect.}
\subsection{Tier 1- Clustering across adjacent frames}
% \tn{increase font size for text outside boxes}
% \stnote{This paragraph is a really nice example of introducing the problem first.}
The extracted meta-information for the $jth$ detection in the $ith$ frame -- $a_{ij}$ is noisy because of the uncertainty in the robot's pose estimate $p_{robot,i}$ and noise in the extracted object-instance centric information -- $c_{ij}$, $prob_{ij}$, $emb_{ij}$. To aggregate the different view detections of the same object and to identify the unique objects in the environment, we consider a sliding window $w$ over sensory data $\{s_{i - n}, ..., s_i \} $. Object-centric information $a_{ij}$ of all detections in all the frames of this sliding window are then clustered to create a set of clusters $G$, each identifying a unique object instance. The motivation behind this is that the object instances of same category with similar embeddings and positions in space will be associated to the same cluster hence pruning the noisy detections. We assign a unique id $instance_{id} \in ID$ to each cluster $g \in G$ and perform an aggregation operation $\bigotimes$ on every cluster $g \in G$ to create a unified feature representation of each of the cluster. This results in an aggregated representation $a_{g,\bigotimes}$ which is then inserted into a collection $T$ indexed by the unique id $instance_{id}$ of each cluster where $ID$ is the set of all $instance_{id}$'s \iinote{fix this}.
% all detected object insta $AllDetects = \{ J_i,J_{i+1}, ....J_{i+n}\}$. We concatenate $a_{ij}$ for every detection $j$ that is part of the sliding window $w$. We then perform spatial clustering over these concatenated feature vectors $fv_{ij}$ for every $Dt \tn{should be $J_i$?} \in AllDetects$. Specific implementation details of the clustering algorithm can be found in Sec-\ref{sec-Algo Implementation}.
% % \iinote{move to next section: all information that can be extracted from each detection, in real-time}
% This will create $G$ clusters and each cluster can be considered a unique object instance for this sliding window $w$. For the subset of detections $Detects \subseteq AllDetects $ assigned to one cluster $g \in G$, we perform an aggregation operation on the $fv_{ij}$'s associated with $Detects$ based on the $weight= number \ of \ detections$ in the cluster, leading to a $weightedfv_{ij}$ associated with each cluster $g$. This aggregation operation will account for noise in the individual $a_{ij}'s$ across multiple detections. Specific implementation details of which can be found in Sec-\ref{sec-Algo Implementation}. Tier 1 - Aggregation gives us as output a set of clusters $G$ and their associated $weightedfv_{ij}$. We now need to keep track of these clusters in $w$ to associate them with what the robot has seen in the past, hence we store then in a table $T$ which will be indexed by an $instance_{id} \in ID$ that is unique to each cluster.
% The set of ID gets populated with a new unique $instance_{id}$ every time a new cluster $g$ is formed.
% \iinote{aggregation operation}
% \iinote{add DBSCan clustering algorithm}
% \stnote{make title shorter, and let latex put the table where it wants; it looks better if it's at the top of the page.}
\subsection{Tier 2 - Filtering across multiple sliding windows}
% \begin{figure}[t]
% \centering
% \includegraphics[width=0.6\linewidth]{images/Database_design.png}
% \vspace{-2em}
% \caption{Schematic Design of Database . *2 -Set of time intervals tuples containing start and end time.
% *3.- Associated with each time interval is a bounding box and key frame \tn{the font's a little small, change the green color to be darker, and what is PK \& FK?}}\label{fig:a}
% \label{fig:Schema}
% \end{figure}
% \stnote{introduce problem first}
We also need to aggregate the detections of unique instances across the objects' multiple partial views in the past. This requires sharing information across multiple sliding windows via a second-tier filtering. To facilitate this, we maintain a fixed-length short-term memory (STM) indexed over $ID$ and aggregate information in it for all instance id's in $T$.
% The ouput of Tier 1 ($ID$ in Table $T$) still needs to pass through more filtering since the detections of unique object instances need to be aggregated with the previous objects' multiple partial views for further compactness and association. This requires sharing information across multiple sliding windows via a second-tier filtering. To facilitate this, we maintain a fixed-length short-term memory $(STM)$ that is indexed over $ID$.
For an object represented by $instance_{id}$ in collection $T$,
we add a new entry to STM only if the object is a new unique object. Otherwise, if the object has previously been seen by the robot, an entry for this object should already exist in STM and should be updated.
We do this by finding a set of $nearIDs$ in STM that are closest in embedding space in cosine similarity\iinote{tell here?} and measuring the euclidean distance between them and $instance_{id}$ in physical space. There are three cases to be considered here that symbolize whether the object is new or not and whether it has moved or not: \textbf{1)} If $nearIDs$ set is non-empty and euclidean distance is less than a threshold $d_{thresh}$ then we perform another aggregation operation $\bigodot$ on $instance_{id}$ and the closest $n_{id} \in nearIDs$ to create a unified feature representation for $n_{id}$. This results in an aggregated representation $a_{n_{id},\bigodot}$ that is updated in $STM$. \textbf{2)} If $nearIDs$ set is non-empty and euclidean distance is greater than a threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). \textbf{3)} If $nearIDs$ is empty for $instance_{id}$, we also add a new entry in the STM with instance id $instance_{id}$. This represents the scenario where the object has not been seen before.
%%% every cluster $g \in G$ to create a unified feature representation of each the cluster.
%%%We do this by finding the set of instance id's $\textbf{nearIDs}$ in the STM that are closest to $instance_{id}$ in the embedding space. After which we calculate the Euclidean distance between between the pose of $instance_{id}$ and every element of $\textbf{nearIDs}$. If the normalized Euclidean distance is higher than the threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). If the Euclidean distance is below the threshold $d_{thresh}$, we perform an aggregation operation over the metadata $weightedfv_{ij}$ of $instance_{id}$ and the closest $n_{id} \in \textbf{nearIDs}$. The old $weightedfv_{ij}$ of $n_{id}$ is updated with the new $weightedfv_{ij}$.
%%%We do this by finding the set of instance id's $\textbf{nearIDs}$ in the STM that are closest to $instance_{id}$ in the embedding space. After which we calculate the Euclidean distance between between the pose of $instance_{id}$ and every element of $\textbf{nearIDs}$. If the normalized Euclidean distance is higher than the threshold $d_{thresh}$, we make a new entry in the STM with instance id $instance_{id}$. This new entry with the same $instance_{id}$ symbolizes dynamic objects (where the same object has moved in the environment). If the Euclidean distance is below the threshold $d_{thresh}$, we perform another aggregation operation $\bigodot$ on $instance_{id}$ and the closest $n_id \in \textbf{nearIDs}$ to create a a unified feature representation for $n_id$. This results in an aggregated representation $a_{n_{id},\bigodot}$. There is no constraint on two aggregation operation's $\bigotimes$ and $\bigodot$ to be the same. For our implementation they are same and described in sec-\ref{sec-Algo Implementation}.
% For a given query $Q$, D3A returns a small set of keyframes from its queryable representation $R$ to find relevant information about the target object.
% After the aggregation operation, the The keyframes also need to be associated with the updated $n_{id}$ or newly added $instance_{id}$ because for a given query $Q$, D3A returns a small set of keyframes from its queryable representation $R$ to find relevant information about the target object.
% The reason behind making $R$ compact is to allow for speed efficient object retrieval. The responses retrieved from $R$ will be a subset of relevant keyframes associated with unique objects in the environment. \tn{i'd cut the previous 2 sentences bc they seem out of place. also i know that you introduced Q in the intro paragraph for the section but by this point i've forgotten what it is and would need a reminder so it doesn't seem like it's coming from no where}
After the aggregation operation, the keyframe of $n_{id}$ is chosen to be the frame with the highest detection probability $prob_{ij}$ among the frames $f_i's$ associate with $instance_{id}$ and $n_{id}$. If a new instance is to be added to the STM when it has reached its maximum capacity, our algorithm evicts the last recently viewed (least-recently viewed entry), $lrv\_entry$, from the STM and moves it to the persistent storage key-value store $R$. This eviction strategy ensures that objects currently being viewed by the robot remain in the STM for further noise filtering and aggregation of meta-data from partial views.
% \iinote{we do it for all the $instance_{id}'s$ in the window.} \iinote{key-value database}
\subsection{Tier 3- Filtering across Database Entries} \label{subsubsection:p3}
The filtered entry $lrv\_entry$ from the STM could be directly added as the final aggregated cluster into our compact representation $R$. However, we want to aggregate detections of unique object instances over not just recent times but also long periods of time.
% However, we want a more compact representation that aggregate detections of unique object instances over long periods of time.
To do so, every time a $lrv\_entry$ is evicted from STM, our third-tier filtering level looks up and updates the aggregated clusters stored in $R$. We organize our aggregated spatial-temporal representation $R$ in a key-value database over two collections. The \textbf{Object Identification Collection} ($OIc$) is updated using the object information of the evicted clusters represented by the $lrv\_entry$, while the other store \textbf{Spatial-temporal Collection} ($STc$) stores the object entries indexed over time and position in physical space. Every object inserted in $OIc$ is associated with a unique identifier $ObjectID$ which is used to index $STc$ to get the object detections over space and time.
To update $R$, we find the set $Records \in OIc$ that are the closest in feature space with $lrv\_entry$ in cosine similarity. A new document is inserted into $STc$ if the normalized Euclidean distance between last evicted entry from STM $lrv\_entry$ and $Records \in STc$ is greater than $d_{thresh}$, otherwise the object information in the closest document $record \in Records$ is aggregated, via an operation $\oplus$, applied on the $lrv\_entry$ and the document $record$ hence updating the object information in $OIc$ using this aggregated output. This new document insertion in $STc$ symbolizes a moving object while the update symbolizes operation for a static object. For every new entry in $OIc$, an entry is added in the $STc$ against the $ObjectID$ containing the time and physical location of the new entry.
% The schematic design of our spatial-temporal represen
% The \textbf{Object Identification Collection} ($OIc$) stores object attributes of aggregated clusters in $STM$ such as the category $C_R$, embedding $Emb_R$ and weight $Weight_{R}$ \iinote{what to do about this weight?}. We also have a separate \textbf{Spatial-temporal Collection} ($STc$) that stores object entries indexed by the object's location in the environment $P_{object,R}$, time intervals $Time_R$, keyframe $K_R$, detection probability $Prob_R$, and the bounding box $B_R$ of the object in the keyframe. Such an inverted index schema for $R$ supports the flexible update and addition of multiple locations for the same object instance, with a one to many mapping from $OIc$ to $STc$ and a unique identifier $ObjectID$ in $OIc$ that can be used as a referential foreign key in $STc$.
% % Such an inverted index schema supports the flexible updating and adding of multiple locations for the same object instance, with a one to many mapping from $OIc$ to $STc$ with unique identifier $ObjectID$ in $OIc$ being a foreign key in $STc$.
% % \tn{Where did $e$ come from?}
% In our third-tier filtering, we find the set $Records \in OIc$ that are the closest in feature space with $lrv\_entry$ in cosine similarity. A new document is inserted into $STc$ if the normalized Euclidean distance between last evicted entry from STM $lrv\_entry$ and $Records \in STc$ is greater than $d_{thresh}$, otherwise the closest document $record \in Records$
% % \tn{is D the set of all documents? You might want to remove it bc it never got introduced}
% is updated with the result of an aggregation operation $\otimes$ applied to the spatial and the embedding vector of $lrv\_entry$ and the document $record$. The document in $OIc$ (with the same $ObjectID$) is updated with the aggregated embedding
% The schematic design of our spatial-temporal representation built in MongoDB is shown in Figure \ref{fig:Schema}.
% We organiz
% We use \textit{MongoDB}, which uses a special geospatial index to store the object instances. This makes the spatial-temporal lookup of objects very efficient. The schematic design of our database is shown in Figure-\ref{fig:Schema}. We choose not to directly insert $e$ into the database because we want to ensure the unique object instances are associated over long-time horizons hence reducing redundancy. In our third-tier filtering, we look up the object id's $obj_id$ from the \textbf{Object Identification Collection} that are closest in feature space with $e$ under cosine similarity. If the $obj_id$ is empty, we insert a new $obj_id'$ associated with $e$, symbolizing that this is potentially a new object instance not seen by the robot. Else we again perform weighted average of the feature of $obj_id$ and $e$ and update the \textbf{Object Identification Collection}. We then look-up set of documents $D$ in the \textbf{Spatial-temporal Collection} that have the same associated $obj_id$. A new document is inserted in the collection if the normalized Euclidean distance between $e$ and $d \in D$ is greater than the $d_thresh$, else the closest document $d' \in D$ to $e$ is updated with the weighted-average spatial information. The keyframe of this $d'$ is chosen to be the frame with the highest probability amongst the frame associate with $d'$ and $e$.
% \tn{The intro text at the beginning of the ``System Overview" section made it seem like storing the data in the database should be its own subsection, but it's already included in the 3-tier processing subsection right? If that's the case, you should edit the intro text to group processing and storing the data together.}
\subsection{Query Processing and Answering}
\begin{table}[]
\small
\centering
%\stnote{Can you add a column for the formal representation of the query?} \iinote{I had that previously but it was taking too much space so removed it? Will it be okay if we dont have it?}
\begin{tabular}{p{0.4cm}p{3.4cm}p{3cm}}
\toprule
\bf{Type} & \bf{NL Example} & \bf{Formal Representation} \\
\midrule
Q1 & Did you ever see an orange bowl? & ${orange(x), bowl(y) |}$ ${ OIc.find(x,y) }$\\
\hline
% Q2 & { "Where was the $attr1$ pink cup"?} \\
% \hline
Q2 & When were the cup and bowl seen? together? & ${bowl(x), cup(y) |}$ ${STc.find(}$ ${OIc.find(x,y), together)}$\\
\hline
Q3 & Where did you see the pink cup between 1:00pm and 4:00pm on 1st Jan, 2020? & ${pink(x), cup(y), time(t) |}$ ${STc.find(OIc.find(x, y),}$ ${1600 >= t >= 1300)}$\\
% \hline
% Q4 & Where is the Sprite bottle \\
\bottomrule
% \label{example_queries}
\end{tabular}
\vspace{-2mm}
\caption{Query Types, Language Examples \& Formal Representations}
\label{tab:example_queries}
\vspace{-6mm}
\end{table}
Template-based NLP methods \cite{li2017querying} or deep learning \cite{DBLP:journals/corr/abs-1911-04942} methods\tn{probably need citation} can be used to map a language query to a set $X$ of $ObjectID's$, time intervals, and location areas, which D3A takes as input for object retrieval. Table \ref{tab:example_queries} shows the types of queries, their natural language examples, and formal representations used in our evaluation. This formal representation is used to search over indexed $OIc$ and $STc$ collections of $R$. The representation $R$ built through our method can handle a wide variety of queries, including: querying for objects with/without the following: 1) specific attributes such as ``orange bowl," 2) spatial relationships with other objects such as ``cup and bowl together," and 3) time slices which can be either a point in time or a time interval. Our representation can also efficiently handle negative queries where the answer does not exist in the dataset.
% objects with/without 1) specific attributes, 2) spatial relationship among objects, 3) the temporal relationship among objects, and 4) time slices.
% The query input to RoboMem is a combination of $ObjectID's$, time intervals and location area. This set can be generated from Natural Language Query and sample object image. However, our current system takes in a templatized query. The schematic design of the database can handle a variety of queries as follows: Q1, Q2: With specific attributes (for static objects and dynamic objects), Q3) and Q4) With time slices(for positive and negative query). A negative query is the one for which the answer in the database does not exists. Q5: with a query about latest time stamp, Q6, Q7: with spatial relationship among objects. Fig-\ref{fig:block-diagram} show examples of the query types in natural language and their corresponding DB query.
% 3) the temporal relationship among objects, and 4) time slices.
% \begin{enumerate}
% \item With specific attributes: "Where was the $attr1$ $catg$"? mapped to
% \begin{verbatim}[scale =0.99]
% Q1: object\_id=collection.find({Category: attr1}, feature vector: attr1)
% collection.find({obj\_id : object\_id})
% \end{verbatim}
% \item 2) spatial relationship among objects, 3) the temporal relationship among objects, and 4) time slices.
% \end{enumerate}
\section{Algorithmic Implementation} \label{sec-Algo Implementation}
% \iinote{talk about similarity measure}
% In this section we describe the algorithmic implementation for the real world deployment of D3A and what param
% \subsection{Algorithmic Implementation}
We deploy D3A on a mobile robot Kuri \cite{heykuri_2018} that patrolled a robotics lab environment for four days. Our system performs real-time object detection with Facebook's AI Research software system - Detectron \cite{Detectron2018} to extract $c_{ij}, prob_{ij}$ for every detection. We use a color histogram \cite{color_hist} to approximate an embedding $emb_{ij}$ of each object instance detection. We select the color histogram method due to its simplicity. However, our algorithm is not constraint to using the color histogram as the instance feature extractor. More sophisticated deep learning methods that use RGB or RGB-D images \cite{bras2018unsupervised, xu2014deep} can also be used. In subsection Experiment 1 of section Evaluation, we show how varying the type of object detector (changing its false positive rate) affects the results of our algorithm. Notably, our approach is also not constrained to any particular number of objects, object size or shape, or object dynamicness (\textit{i.e.,} the objects' locations can change over time). We are only constrained by the object categories the detector was trained to detect. Processing of the visual data and pose estimates was done off-board on a stand-alone machine. The PC setup included an Intel(R) Core(TM) i7-6700 CPU (3.40GHz x 8), and a GeForce GTX 1070/PCIe/SSE2 GPU running Ubuntu 16.04. D3A was run online while the PC communicated with the Kuri robot over ROS \cite{quigley2009ros}. For Tier 1-clustering of detections across adjacent frames we use density-based spatial clustering (DB-Scan) \cite{khan2014dbscan}, a method known to be good at handling outliers/noise within the dataset. While other clustering methods can also be used, we defer the analysis of their impact on D3A's performance for future work. All the aggregation operations $\odot$, $\oplus$ and $\otimes$ that we applied are the weighted average functions. These aggregation functions can also be different. Weighted average was selected because of its simplicity and reasonable empirical performance for object retrieval on our smaller dataset of $<3$ hours of data. More sophisticated methods can also be used in place of the weighted average. However, as shown in section -- Experiment Designs of Evaluation,
% -\ref{sec:exp-des},
with just the weighted average, D3A still performs better than the baseline. We organized our aggregated spatial-temporal representation $R$ over two collections in \textit{MongoDB} \cite{banker2011mongodb}, a database that supports large volumes of both data and traffic. The two document Mongo-DB collections \textbf{Object Identification Collection} ($OIc$), and \textbf{Spatial-temporal Collection} ($STc$) are as described in Section -- Tier 3- Filtering across database entries of Approach.
% Sec-\ref{subsubsection:p3}.
$OIc$ in our implementation, stores the object information using the associate embedding of the cluster, the category assigned to the cluster and a weight which encodes the frequency of detections for this object and is used in the aggregation operation between $record$ in $OIc$ and $lrv\_entry$ evicted by the $STM$.
% \tn{the previous sentence doesn't make sense? Do you mean something like ``The two collections, $OIc$ and $STc$ are as described in section..."?}
For natural language processing of the queries, we annotate the queries (use the template-based method) to convert them into the logical form shown in Table \ref{tab:example_queries}. These logical forms are then used to query the collections for relevant spatial-temporal information. In \label{sec:exp3} we show how the performance of D3A changes with varying the uncertainty in the processing of the query but still it is able to perform better than the baselines.
% \iinote{We organize our aggregated spatial-temporal representation over two collections in \textit{MongoDB} \cite{banker2011mongodb}, a database that supports large volumes of both data and traffic. Our first collection in MongoDB is the \textbf{Object Identification Collection} ($OIc$) that stores object attributes of aggregated clusters such as the category, embedding and weight. We also have a separate \textbf{Spatial-temporal Collection} ($STc$) that stores documents containing a geo-spatial index for location, time intervals, the keyframe, and the detection probability and bounding box of the object in the keyframe.}
% \iinote{We annotate the queries to convert to the database query representation.}
\section{Evaluation}
% \stnote{The aim of our evaluation is to test the hypothesis that algorithm X improves the precision and recall of a database-backed system and answering queries.}
\begin{table}[t]
\small
\centering
\begin{tabular}{l p{0.5cm} r}
\toprule
Duration &22 hrs & 3hrs \\
\midrule
Total Number of Frames & $10,132$ & $991$\\
Frame Rate (per minute) & $7.67$ & $7.67$\\
Sensor Data Size (GB) & 10.53 & $1.43$\\
Total Number of Detections & $13,565$ & $2,558$ \\
Ground Truth Objects & N/A & $59$ \\
& & Static = 49, Dynamic = $10$\\
Number of Times & N/A & Mean = $2.1 \pm 1.3$, \\
Dynamic Objects move && Max = 5 \\
\bottomrule
\end{tabular}
\vspace{-3mm}
\caption{Dataset Details}
\vspace{-3mm}
\label{tab:my_label}
\vspace{-3mm}
\end{table}
The aim of our evaluation is to test the hypothesis that a database-backed system with our algorithm D3A improves both 1) the compactness of the spatial-temporal representation of objects in the environment and 2) the object retrieval performance by returning a small subset of keyframes as measured by the mean reciprocal rank, miss rate and retrieval time as described in Experiment Design subsection.
% Sec-\ref{sec:exp-des}.
As a result, the user will have to search over just a few returned results to find the answer to their question.
To test our hypothesis, we perform a real-world evaluation, using both a range of synthetic detectors built from ground truth data to control the noise level and real detectors deployed on a mobile robot to assess the system's end-to-end accuracy.
% This is directly related to improved user experience.
%%%\stnote{So instead of telling that, can you like SHOW that? As a result, the user will have to search few just a few returned results to find the answer to their question.} \stnote{To test this hypothesis, we perform a real-world evaluation on a mobile robot using both simulated detectors built from ground truth data to test the noise and real detectors to assess the end-to-end accuracy. (PS, I am writing this fast, so feel free to reword/clean up any suggested sentence I put. But it's good to have something to say like, what the evaluation is going ot be and why it will answer our hypothesis.) }
% Our main results are:
% \beging{}
% and hence the recall\footnote{Defined as a function of true positives (tp: key frame containing the object of interest in the query), false negatives (fn: objects failed to be identified and retrieved). It is the fraction of relevant documents that are retrieved, in particular Recall = \frac{tp}{tp+fn}}
% measure for evaluatingreturned key frames to a sample of queries, ordetred by probability of correctness. The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. If none of the proposed results are correct, reciprocal rank is 0.}
% with varying range of sensor uncertainty for retrieved objects.
\subsection{Dataset Collection}
% \stnote{don't put colons after subsection names}
% We deployed D3A on a mobile robot Kuri \cite{heykuri_2018} that patrolled a robotics lab environment for four days.
The robotics lab environment in which our mobile robot Kuri \cite{heykuri_2018} patrolled for four days was uncontrolled and cluttered: people were allowed to use the space as is. For consistency, we kept the illumination same throughout the data collection(it was captured in a completely lit lab). The robotics lab area used for data collection and experimentation included the kitchen area and the general area with tables and chairs. This real-world dataset\footnote{ A visualization recording of the initial state of the environment, along with its 2D map and sample raw sensory data can be found at \url{ https://github.com/IfrahIdrees/D3A.git}.}
has 10,132 image frames over 22 hours and contains both static and dynamic objects such as various cup and bottles changing there location. The dataset details can be found in Table \ref{tab:my_label}.
% \iinote{for consistency keep the illumination same(captured during daytime.)}
% \stnote{cut: To our knowledge this is currently the longest dataset collected by a robot.} \stnote{No way. For example see the MIT SLAM dataset. And all the Waymo selfdriving data. }
\subsection{Dataset/Object Annotation}
\begin{figure}[]
\centering
\includegraphics[width=0.8\linewidth]{images/Figure_1.png}
\vspace{-3mm}
\caption{Exp 3 - Cumulative number of insertions to the database per hour}
% \label{fig:a}
% \tn{seems like there's extra white space around the figure, trim that to get back some space}
\label{fig:graphcumulative}
\end{figure}
% \stnote{No colons in section names.}
% \stnote{I take back the comment in the intro about 22 hours vs 3 hours, but we should make sure we use the 22 hours somehow in the evaluation. }
For evaluation purposes, the collected dataset was manually annotated by the authors. Every unique instance in the dataset was assigned a unique id and a ground truth location that was used to perform object association over time and space. Due to time and manual labour constraints, we were only able to annotate 3 hours of data.
\subsection{Parameter Selection}
% \stnote{No colons in section names.}
% \iinote{ We use a color histogram \cite{color_hist} to approximate an embedding of each object instance to relate them over time. % and reduce the set of potentially similar objects. We select the color histogram method due to its simplicity,}
The Euclidean distance threshold $d_{thresh}$ , sliding window length, normalized threshold for feature matching (cosine similarity), and size of short-term memory (STM) were set to 0.5m, 10, 0.4, and 400 (our RAM's maximum capacity), respectively. We performed a parameter sweep offline on the collected data and found this setting to be optimal. Increasing the feature matching and distance threshold increased the number of false positives during tier-1 processing. For the STM, the greater the buffer size, the more noise there will be in the object's pose and embedding to be filtered and processed.
% The selected parameter values and the justification for each are as follows:
% 1) The Euclidean distance threshold $d_{thresh}$ was set to 0.5 meters to account for the 0.5m standard deviation in the robot pose estimation. We performed a parameter sweep and found that setting the sliding window length = 10 and normalized threshold for feature matching = 0.4 to be optimal. Increasing the matching and distance threshold increased the number of false positives during tier-1 processing.
% 2) The short-term memory (STM) buffer was set to 400 objects, our RAM's maximum capacity. The greater the buffer size, the more noise there will be in the object's pose and embedding to be filtered and processed.
% We also performed a parameter sweep over different lengths of the buffer and found the results to be not impacted
\subsection{Experiment Designs}\label{sec:exp-des}
We conduct four experiments with two baselines. One experiment is a running example comparison.
% We conduct three \tn{four?} sets of quantitative experiments with a running example and two baselines (as described below).
% \stnote{Don't give results here until after you describe the algorithms. So move those two sentences later.}
\textbf{Baselines}: Our first baseline is a ``Naive" method that inserts every detection's spatial-temporal information directly into the database without aggregation. This baseline ablates the three-tier processing that is integral to our system for aggregating partial views of the objects, but can still index detections on space and time. We also conduct an ablation of the spatial information from the features to make a ``Non-spatial" baseline, similar to \citet{hsieh2018focus} which does not utilize spatial information during clustering, to compare the clusters resulting from both methods.
% We do another ablation study where we removing the spatial information used by D3A for three-tier clustering. We refer to this method as ``Non-spatial''. This is similar to \cite{hsieh2018focus} wich uses a similar clustering method but does not use the spatial information.
% For an ablation study with \citet{hsieh2018focus}, we implement a ``Non-spatial'' baseline, removing the spatial information used by D3A for three-tier clustering. We use a running example to compare D3A with this baseline and visualize the differences in their aggregated clusters.
%%%\stnote{I would rephrase that a bit, and say first, we do the abalation, and then, this is similar to hsieh whichuses a similar clusting method but does nto use spatial information}
% will show \tn{that clustering by spatial information improves...} how D3A improves the aggregation of detections.
% In the first, we show the changing trend in retrieval performance of our system relative to the uncertainity of the parameterized synthetic detector as compared to our baseline.
% We perform two separate ablation studies that extensively assess the improvements from our framework.
\textbf{Metric Definitions}:
We measure the quality of keyframes returned for a query with the MRR@50 evaluation metric, which is the multiplicative inverse of the rank of the correct keyframe in the top 50 returned frames ordered by their probability of correctness. We measure the miss rate (the number of objects missed by the algorithm during the clustering phase due to detection or classification error, hence cannot be found in the returned frames) to get an idea of our accuracy. We also report the speed efficiency of our system by measuring the mean number of frames and object id's returned for the queries, total time taken by a query divided into retrieval time and evaluation time to find the correct frame by matching against the ground truth.
% \begin{itemize}
% \item To measure how good the retrieved key frames are, we measure MRR@50 \footnote{A measure for evaluating the returned keyframes for a sample of queries, ordered by probability of correctness. The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer. The first correct answer in our case is the first keyframe that contains the queried object instance. If none of the proposed results are correct, the reciprocal rank is 0. MRR values closer to 1 are preferred as they signify high retrieval precision where the set of returned keyframes contains the relevant response for the query}, Miss Rate which is equivalent to recall(the percentage of time our algorithm fails to recognize unique object instances). We use the miss rate to define the accuracy of the system as 1- average(Miss Rate).
% \item to measure the speed efficiency we measure, the the mean number of frames and object id's returned, 5) mean retrieval time from database, and 6) total evaluation time to check the returned keyframes against the ground truth annotation for all queries. Higher number of these metric mean longer times for the human user to search over the returned frames.
% % \item We define our system's accuracy as If the set of returned keyframes and their corresponding spatial-temporal information contains a frame with the answer to the query, we say that the query has been answered accurately.
% \end{itemize}
% We measure a number of metrics for the retrieval performance distributed over all experiments: 1) the mean reciprocal rank (MRR), and 2) the object miss rate --- the percentage of time our algorithm fails to recognize unique object instances \stnote{Is this recall? Or something different. I would explain how this si different from recall}. We also report 3) the mean number of frames returned, 4) mean number of object id's returned, 5) mean retrieval time (to retrieve keyframes and spatial-temporal data from the database), and 6) total evaluation time taken by the system (to check the returned keyframes against the ground truth annotation for all queries). \stnote{I think I wouldn't do this in a list, but rather would introduce WHY you need each metric, and then define it if necessary.} Our system's accuracy is defined as follows: since the query matching is uncertain, we return the top 50 frames for every query and rank them according to the \tn{their?} weighted average probability for the queried object stored in our database.
% which the object detector detects the category of the objects \tn{the last part is confusing, do you mean you rank them by the object detector's detection probability for the queried object?}.
% We then iterate over the frames to find the one that contains the answer to the query. If the set of returned keyframes and their corresponding spatial-temporal information contains a frame with the answer to the query, we say that the query has been answered accurately.
\begin{figure}[t]
\centering
\includegraphics[width=0.8\linewidth]{images/noise.png}
\caption{Exp1 - Retrieval performance with a synthetic embedding generator with increasing uncertainty}
% \label{fig:a}
\label{fig:graph_noise}
\vspace{-5mm}
\end{figure}
% \begin{table*}[!htbp]
\begin{table*}[httb]
\small
\centering
% \caption{\tn{Missing table caption. I'd suggest adding $\uparrow$ and $\downarrow$ next to the metric names to help the reader parse the numbers a bit easier ($\uparrow$ means higher is better and so on)}}
% $*^1$refers to the Naive baseline without D3A, $*^2$is D3A + synthetic Perfect embedding generator, $*^3$is D3A + Detectron (real world detector) with a color histogram as the embedding generator
\begin{tabular}{@{\makebox[2em][r]{\textcolor{airforceblue}\rownumber\space\space\space}} l p{1.2cm} p{1.5cm} | p{0.9cm} p{1.8cm} p{1.9cm} l p{1.9cm} p{2cm}}
\toprule
\multicolumn{1}{@{\makebox[3em][r]{}}} & \textbf{Type} & \textbf{Query Precision} & \textbf{Aggregation Method} & \textbf{Miss Rate}$\downarrow$ & \textbf{Mean Number of Frames} $\downarrow$ & \textbf{Mean Number of Object IDs} $\downarrow$ & \textbf{MRR @50} $\uparrow$ & \textbf{Mean Retrieval Time (ms)} $\downarrow$ & \textbf{Total Evaluation Time (ms) $\downarrow$}\\
\midrule
% \multirow{3}{*}{Q1} & perfect & Perfect & $0.00$ & $1.08$ & $1.00$ & $1.00$ & $0.03$ & $8.38$\\
% & perfect & Category & $0.24$ & $5.88$ & $3.75$ & $0.35$ & $0.89$ & $104.82$\\
% & perfect & Any object & $0.88$ & $38.00$ & $33.00$ & $0.07$ & $1.04$ & $2,031.02$\\
\multirow{9}{*}{Q1} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 10.30 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 3.93 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 10.50 & 1.00 & 1.00 & 0.02 & 6.63 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.33 & 792.73 & 339.00 & 0.19 & 22.24 & 2,567.62 \\
& & D3A+P & 0.20 & 6.29 & 4.50 & 0.33 & 0.96 & 101.31 \\
\rowcolor{LightCyan}
& & D3A+D & 0.20 & 12.43 & 1.50 & 0.29 & 0.94 & 104.95 \\
\cline{2-9}
& \multirow{3}{*}{Any object} & Naive & 0.93 & 2,558.00 & 2,558.00 & 0.01 & 119.74 & 926.67 \\
& & D3A+P & 0.20 & 38.00 & 33.00 & 0.24 & 0.82 & 116.15 \\
\rowcolor{LightCyan}
& & D3A+D & 0.13 & 75.00 & 25.00 & 0.20 & 1.12 & 89.19 \\
\hline
\multirow{9}{*}{Q2} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 2.55 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 6.94 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 10.20 & 1.00 & 1.00 & 0.03 & 23.23 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.88 & 861.00 & 861.00 & 0.04 & 25.99 & 2,108.60 \\
& & D3A+P & 0.31 & 7.50 & 7.00 & 0.31 & 1.63 & 55.81 \\
\rowcolor{LightCyan}
& & D3A+D & 0.38 & 18.00 & 3.50 & 0.18 & 1.54 & 73.26 \\
\cline{2-9}
& \multirow{3}{*}{Any object} & Naive & 0.94 & 2,558.00 & 2,558.00 & 0.01 & 117.96 & 996.27 \\
& & D3A+P & 0.25 & 38.00 & 33.00 & 0.13 & 0.85 & 173.65 \\
\rowcolor{LightCyan}
& & D3A+D & 0.38 & 75.00 & 25.00 & 0.10 & 1.14 & 153.19 \\
\hline
\multirow{9}{*}{Q3} & \multirow{3}{*}{Perfect} & Naive & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 5.05 \\
& & D3A+P & 0.00 & 1.00 & 1.00 & 1.00 & 0.02 & 4.15 \\
\rowcolor{LightCyan}
& & D3A+D & 0.00 & 5.50 & 1.00 & 1.00 & 0.02 & 3.53 \\
\cline{2-9}
& \multirow{3}{*}{Category} & Naive & 0.67 & 96.00 & 96.00 & 0.25 & 336.60 & 110.58 \\
& & D3A+P & 0.27 & 1.50 & 1.50 & 0.36 & 1.60 & 9.20 \\
\rowcolor{LightCyan}
& & D3A+D & 0.47 & 1.00 & 1.00 & 0.33 & 1.35 & 5.68 \\
\cline{2-9}
&\multirow{3}{*}{Any object} & Naive & 0.86 & 684.00 & 684.00 & 0.08 & 2,723.35 & 160.27 \\
& & D3A+P & 0.14 & 14.00 & 14.00 & 0.21 & 13.25 & 31.45 \\
\rowcolor{LightCyan}
& & D3A+D & 0.07 & 12.00 & 12.00 & 0.26 & 11.51 & 28.93 \\
% \cline{2-9}
\bottomrule
\end{tabular}
\vspace{-2mm}
\caption{\small Exp 2 - Retrieval Performance for the three types of queries. $\uparrow$ means higher is better, $\downarrow$ means lower is better.}
\vspace{-3mm}
% \vspace{-3mm}
\label{table:queryperformance}
\end{table*}
\subsubsection{Exp 1 - Retrieval Performance with a Synthetic Embedding Generator with Increasing Uncertainty} \label{sect:falsepositiveexp}
The performance of D3A depends on the accuracy of the object detector and object pose estimates. In this experiment, we design a synthetic embedding generator that replaces the object detector and color histogram. Our synthetic embedding generator is parameterized with a false positive rate ($fpr$)\footnote{ \label{note2} $fpr$ denotes the probability of falsely labelling the detected instance} and false negative rate ($fnr$)\footnote{$fnr$ denotes the detector's probability of not detecting the object}. We keep the $fnr$ low since we assume that the object is not in the environment if the detector fails to detects the object. We annotate ground truth object embeddings as one-hot encodings $ohe$ of the unique instances, where each index represents the unique id of a ground truth object. We then use the $fpr$ and $fnr$ to flip the bits of the $ohe$ and return the transformed encoding as the synthetic embedding generator's output.
%\stnote{You don't need to give pseudocode for this if you need more space.} \tn{+1}
% We measure the MRR and Miss Rate with varying the $fpr$ - level of uncertainty of our Embedding Generator and compare it with the ``Naive" baseline.
\textbf{Results}: We tested our system on Q1 type - \textit{Did you ever see an orange bowl?} for all ground truth object instances as mentioned in Table-\ref{tab:example_queries}. Results are shown in Fig-\ref{fig:graph_noise}. Our method's MRR@50 decreases as $fpr$ increases. Despite this trend, D3A's MRR@50 remains considerably higher than Naive's MRR@50. This indicates that even when the object detector used in D3A is not good, the user will always find the correct object's frame earlier with D3A than with the Naive baseline. D3A's Miss Rate increases as the synthetic detector becomes noisier, but is always lower than Naive's Miss rate.
The slight decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with a higher probability of flipping the bits of the $ohe$, the probability of different object instances being assigned a similar embedding also increases. This causes D3A to consider a higher proportion of object instances to be similar, leading to more cluster formations (because of different locations) and storage of their respective keyframes. At the time of retrieval for a given object, this causes more keyframes to be returned one of which includes the queried object.
% The slight decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with a higher probability of flipping the bits of the $ohe$, the probability of different object instances being assigned a similar embedding also increases. This causes D3A to consider a higher proportion of object instances to be similar even when they are not, leading to more detections being used for cluster formation, increasing the number of clusters formed and greater storage of their respective keyframes. At the time of retrieval this causes keyframes to be returned of more different objects and at least one of those frames includes the relevant object.
\tn{i don't get how this means a lower miss rate}\iinote{improve this?}
% \textbf{Results}: We tested our system on all the query types types in Table \ref{example queries} and measure the average MRR and Miss Rate for all three.
% The MRR@50 of our method shows a decreasing trend as $fpr$(noise level) increases and remains above the baseline. This indicates that even if the synthetic detector is not good the user always finds the correct key frame earlier than the Naive. The decreasing trend of D3A's Miss Rate for $fpr >0.3$ is because with higher probability of flipping the bits, the probablity of different object instances being assigned a similar embeddings also increases that is D3A considers more object id's to be similar, clustering and storing their keyframes more often.
% of the instances among clusters to become highly similar, recognizing causing it to detect objects which are not in the scene.
% We believe that the noise level of Naive remains constant as every independent detection even if noisy is added to the database and when searching for ground truth instances, some frame amongst the top 50 frames returned contains a correct detection.
% --------------------------------------------
% We tested our system on query type 1 in Table \ref{example queries}. \stnote{I would just name the query again here, e.g., Did you ever see an orange bowl?}
% As the $fpr$ of our synthetic object detector increases, the MRR@50 of the retrieved frames for every $gt$ decreases and the Miss rate increases until it hits 0.3 after which the Miss Rate drops, as shown in Figure \ref{fig:graph_noise}. This is because fliping of the bits in the ohe happens so much the probability of detections having $ohe$ with similar cosine similarity increases.
% is wrongly assigned some other detection
% there is no clustering going on and every mutated object represents detection of some other object. This causes a lot of wrong detections which are present in the database and end up being in the top 50 answers for queries.
% Except for the peak at around $fpr=0.3$, the Miss Rate of D3A is lower than the Miss Rate of `Naive', showing that D3A is able to accurately answer queries for more ground truth object instances than the Naive. The MRR@50 and Miss Rate for the `Naive' remains constant despite the changing noise level because of the intrinsic distribution of objects in our dataset. Flipping of the bits assigns the object instances ids
% As the bits are flipped more with increasing fpr, the
% Beyond a $fpr$ of $0.2$, D3A starts performing worse than the ``Naive" baseline. The noise in our deployed system with a real object detector is $0.18$. As the $fpr$ increases, \stnote{cut: the reason that} the Miss rate increases \stnote{cut:is} because the probability that the instance will be assigned the same $ohe$ decreases, and D3A starts to label these new instances as either noise (since it has only seen them in a frame of a sliding window) or instances of another category. \stnote{Can you compare this to the baseline? I'm confused because in Figure 4 it looks like the baseline's performance doesn't change with regard to noise level...}
% D3A starts to consider the objects with similar vector less often hence labelling it as noise or labelling it as some other category which is an intrinsic property of the increasing fpr of synthetic detector.
% \stnote{make this a float, with an algorithm block and a title}
% \begin{lstlisting}[language=Python, caption={Embedding Generator Implementation}, float, label=lst:sensor]
% for indices not groundtruth_index:
% if prob < self.false_positive_rate:
% one_hot_encoding[i] = 1
% else:
% one_hot_encoding[i] = 0
% for groundtruth_index:
% if prob < 1- self.false_negative_rate:
% one_hot_encoding[groundtruth_index] = 1
% else:
% one_hot_encoding[groundtruth_index] = 0
% \end{lstlisting}
\subsubsection{Exp 2 - Query Retrieval Performance}\label{sec:exp3}
Our second experiment compares the retrieval performance of two versions of D3A against the Naive Baseline. The first is $D3A+D$ - D3A with a real detector, Dectectron \cite{Detectron2018}, and color histogram and second is $D3A+P$ - D3A with a synthetic perfect sensor.
\iinote{two versions of D3A against the Naive Baseline. The first is $D3A+D$ - D3A with a real detector, Dectectron \cite{Detectron2018}, and color histogram and second is $D3A+P$ - D3A with a synthetic perfect sensor.}
% D3A with a real detector -- Dectectron \cite{Detectron2018} and color histogram ($D3A+D$) against D3A with a synthetic perfect sensor ($D3A+P$) and the ``Naive" baseline.
We experimented over the 3 query types in Table \ref{tab:example_queries} and the 59 ground truth object instances annotated in the data. The number of instances per query ranged from 15 -20 depending on the query, making the total number of queries to be 150.
% \tn{How does the math work here? 3*59 is not 426, and idk where the 15-20 fits in..}
% For the given \iinote{70} queries, we test the retrieval performance our \textit{deployed system} - on Kuri with real object detector.
\textbf{Query Matching:} Another factor affecting the retrieval performance in this experiment is the matching of the desired object in the natural language query to an $ObjectID$ in the database. To examine this axis, we discretize the query matching precision into three levels:
\begin{itemize}
\item \textit{Perfect}: The query matching component can map an object in the language query to an exact $ObjectID$ in the database, and removes any level of uncertainty.
\item \textit{Category}: The matching component is noisier and is unable to uniquely identify the queried object (\textit{e.g.,} orange bowl), and instead returns a set of $ObjectID's$ for all objects from the same category (\textit{e.g.,} bowl).
\item \textit{Any Object}: The matching component is the noisiest and cannot extract even the category-level information from the language query, and thus maps the query to the most generic set of all the $ObjectID's$ in the database.
\end{itemize}
% We report MRR for the top 50 returned frames for \textbf{U, C, A} labelled as MRR (N=50) in Table-\ref{table:results1}.
\textbf{Results:} The results are shown in Table \ref{table:queryperformance}.
With \textit{Perfect Query Precision} for all three types of queries Q1 [\textit{rows: 1-3}], Q2 [\textit{rows: 10-12}], and Q3 [\textit{rows: 19-21}], both the mean retrieval time and total evaluation time of $D3A+D$ are less than those of the ``Naive" baseline. On average, $D3A+D$'s total (retrieval + evaluation) time is \textbf{1.96x} slower than $D3A+P$ and \textbf{1.48x} faster than the baseline. This translates to improved user experience as the users' query will be answered more quickly and as expected the miss rate is 0 because of the perfect query matching precision.
With \textit{Category Query Precision} for Q1 [\textit{rows: 4-6}], Q2 [\textit{rows: 13-15}], and Q3 [rows:\textit{22-24}], the mean number of frames returned by $D3A+D$ is \textbf{55x} smaller than that of the ``Naive" baseline, and only \textbf{5 counts}
% \tn{is this correct? you should double-check your numbers just in case}
higher than $D3A+P$'s on average. The MRR@50 for $D3A+D$ is \textbf{0.26}, which signifies that if the queried object exists in the database, it will on average be found in the fourth frame. An empty set is returned for negative queries. While the baseline on average finds the object in the sixth frame, the probability of it not finding an object of interest in the first 50 frames (average miss rate) is \textbf{62.5\%}, which is really high compared to $D3A+D$'s \textbf{34.7\%}. On average, $D3A+D$ took \textbf{23ms} to retrieve and evaluate all the frames for all the object instances while the baseline took \textbf{500ms}. These evaluation times, however, are just for 3 hours of data. These times will increase linearly with the amount of sensor data for the ``Naive" and increase the human users' waiting time for their query to be answered.
The worst retrieval performance is with \textit{Any Object Query Precision} for Q1 [\textit{rows: 7-9}], Q2 [\textit{rows: 16-18}], and Q3 [\textit{25-27})]. Even then, $D3A+D$ returns \textbf{54} frames on average, compared to the \textbf{30} frames returned by $D3A+P$ and \textbf{1933} frames returned by the ``Naive" baseline. This difference is also reflected in their retrieval and evaluation times. The baseline on average finds the queried object in the 25th frame ($MRR=0.04$), while $D3A+D$ and $D3A+P$ finds the object around the fifth frame. Even with the worst query precision where the queried object is mapped to all the $ObjectID's$ in the database, D3A still achieves better retrieval performance by clustering detections and thus looking through fewer instances.
%%%\stnote{This is the one where your algorithm is really really good though. So highlight that even if the detector isn't good, this compression algorithm helps because it clusters things so you have to look through fewer results.}
% that is even when we can not figure out what the object is that we are talking about, D3A can return a small set of keyframes where the correct object of interests lies in the 5th frame.
D3A's overall accuracy is \textbf{81.98\%} with a retrieval time that is \textbf{47x} faster than the ``Naive" baseline. This can lead to improved user experience since with D3A, the user's query will be answered by searching over a smaller number of key frames (by \textbf{97\%}) than with the baseline.
% \tn{what is the 97\% trying to convey?}
\subsubsection{Exp 3 - Compactness Comparison}
We measure the cumulative number of object id insertions in the database per hour over the complete 22 hours for both $D3A+D$ and the ``Naive" baseline. As seen in Fig-\ref{fig:graphcumulative}, the number of insertions per hour for $D3A+D$ is much less than that of the baseline and scales well as the amount of data increases, demonstrating D3A's success in aggregating partial views. D3A takes two orders of magnitude more time to process the raw data than the baseline, but outputs a representation \textbf{14.7x} more compact and efficient in answering questions, as shown in Table \ref{table:results1}.
\begin{table}[t]
\small
%\stnote{use booktabs, avoid vertical rules}
%\tn{i rounded up decimals to have max 4 decimal places.}
% \iinote{since they are so many number, I highlighted those that were a little more important and that I discuss even in the evaluation results section but maybe there is a better way to do that?} \tn{maybe add a footnote to say that ``the bolded numbers are highlighted results that are discussed in the text" or something?} }
\vspace{-3mm}
\centering
\begin{tabular}{p{3cm} l l l}
\toprule
{\bf{Performance Metric}} & & \bf{Naive} & \bf{D3A+D}\\
\midrule
Database Size (MB) & & $3312$ & $244$ \\
\hline
Total Processing Time & & $0$m $7.61$s & $2$m $26$s \\
\hline
Average Query Response Time (ms) & & $503.59$ & $10.63$ \\ %9.544548022
\hline
Total Number of Unique Object IDs & & $\bf{2558}$ & $\bf{25}$ \\
\hline
\multirow{2}{3cm}{Number of Duplicates per Object} & Mean & $ 44.10 \pm 75.66$ & {$1.19 \pm 0.46$} \\
& Max & $369$ & $3$ \\
\hline
Mean Accuracy (\%) & {} & $48.85$ & $81.98$ \\
\hline
\multirow{3}{3cm}{Average Number of Keyframes Returned} &
U & $1$ & $8.73$ \\
& C & $\textbf{583.24}$ & $\textbf{10.47}$ \\
& A & $\textbf{1933}$ & $\textbf{54}$ \\
\hline
\multirow{3}{3cm}{Mean Reciprocal Rank ($N=50$)} &
U & $1$ & $1$ \\ %0.9166666667
& C & $\textbf{0.16}$ & $\textbf{0.26}$ \\ %0.4476691691
& A & $0.032$ & $0.187$ \\ %0.3578010691 %0.4195961824
\bottomrule
\end{tabular}
\vspace{-2mm}
\caption{System Performance (On 3 Hours of Data)}
\footnotesize{* The bolded numbers are highlighted results discussed in the text}
\label{table:results1}
\vspace{-5mm}
\end{table}
\subsubsection{Exp 4 - Comparison with the ``Non-spatial'' Baseline}
In this experiment, we visually compare the aggregated clusters formed for the unique object instances by $D3A+D$ and the ``Non-spatial" baseline. Fig-\ref{fig:clusters} shows the baseline's aggregated detections for the pink cup asked for in Fig-\ref{fig:Kuri}. The baseline clusters all object instances that are similar in embedding space, irrespective of their location in the map. The images' different bounding box colors show the respective clusters they were mapped to by D3A based on both the embedding features and spatial information. The clusters formed by D3A most closely resemble ground truth. Furthermore, the ``Non-spatial'' baseline will fail to answer queries of type Q3 in Table \ref{tab:example_queries}.
\section{Discussion}
% \tn{cut: Our -deployed algorithm $D3A+D$, despite performing very well, is still unable to retrieve spatio-temporal information for all the ground truth objects.}
Our deployed algorithm $D3A+D$ shows memory and speed efficiency in clustering together partial views of the same object, even when the object has moved to different locations in the environment over time. In such cases, the same object in each of its new location is assigned to a different keyframe centroid, but is associated with the same $ObjectID$. This is shown in Fig-\ref{fig:clusters} where the pink cup is moved from the table (img3) to the kitchen counter (img4) and assigned to two separate keyframe centroid by D3A, but is assigned the same $ObjectID$, symbolized by same color of bounding box.
D3A can also disambiguate unique objects instances even if they have similar features. An example is shown in Fig-\ref{fig:clusters} where D3A assigns the two similar red cups in ``img1" and ``img2" to different clusters. Despite such results, our algorithm is unable to perfectly retrieve all of the locations a unique object was placed at every moments in time.
% \iinote{Add: Our deployed algorithm $D3A+D$ shows memory and speed efficiency in clustering together partial views of the same object even when moved to different locations in the environment over time. In such a case the same object in each of it's new location is assigned to a different key frame centroid but gets associated to the one object\_id. This is shown in Fig.\ref{fig:clusters} where the pink cup is moved from the table ``img3" to the kitchen counter ``img4" and is assigned two separate keyframe centroid by D3A since they are two locations but is assigned the same object\_id. D3A can also disambiguate unique objects instances even if they have similar feature. An example is shown in Fig.\ref{fig:clusters} where D3A assigns the two similar red cups in ``img1" and ``img2" different clusters. Despite such results, our algorithm is unable to perfectly retrieve all of the locations where a unique object was placed at every moment in time.}
% spatio-temporal information for every ground truth object.}
%%%Our deployed algorithm $D3A+D$, despite performing very well, is still unable to retrieve spatio-temporal information for all the ground truth objects.
% Overall, our deployed algorithm $D3A+D$ performs very well and correctly retrieves objects most of the time. However, it is still unable to retrieve the spatial-temporal information for all ground truth object instances.
This is because our system sometimes fails to distinguish objects when they have similar embedding features and are close together in Euclidean space (below $d_{thresh}$). In this case, D3A will assign same $ObjectID$ and keyframe centroid to two unique object instances.
% This will cause D3A to miss storing the spatial-temporal information of the wrongly assigned object and.
An example of this is shown in Fig-\ref{fig:clusters} where ``img1" of the red cup is assigned to the cluster for the pink cup since they have similar colors and are close together in the physical world.
This causes D3A to miss the associated information of the wrongly assigned red cup and not be able to retrieve this information in response to an object retrieval query related to the red cup.
Furthermore, if an object instance is only seen in one frame of a sliding window, our algorithm will characterize it as noise to account for errors in the detection. This, however, causes D3A to sometimes miss unique object instances. Overall, D3A recognized 76 unique object locations in the map, but because of the above reasons assigns them to 25 object instances instead of the ground truth 59, while considering 18 of the objects as noise. As shown in Table \ref{table:results1}, the average number of duplicated $ObjectID's$ assigned to a ground truth object instance by D3A is $1.19 \pm 0.46$, which is 36X smaller than that of ``Naive".
% \iinote{Add: In case of Naive every partial view detection of the object is treated as a unique object leading to total number of unique detections to be greater than the total number of ground truth objects.}
In the case of the ``Naive" baseline, every partial view detection of the object is treated as a unique object, leading to a total number of unique detections greater than the number of ground truth objects.
Fig-\ref{fig:graph_noise} and Table \ref{table:queryperformance} show that increasing the false positive rate of the visual detector and decreasing the query precision degrades the performance of object retrieval from the spatial-temporal representation created by D3A. Nevertheless, the deployed D3A+D algorithm on Kuri with the real-world detector performs better than the baseline in terms of memory and speed efficiency for object retrieval. D3A+D also performs better than ``Naive" despite the fact that they both suffer from drifting localization of the robot's pose over time. This is because D3A's three-tier clustering and filtering structure was able to filter out some of the localization drift as noise in the objects sensor data.
% In addition, D3A+D uses the object locations calculated based on the results of SLAM and thus suffers from drifting localization of the robot's pose over time as the Naive Nonetheless, D3A+D is still able to perform better than the ``Naive" because of the three-tier clustering and filtering structure that does noise filtering in the objects sensor data.
% \tn{doesn't the naive baseline suffer from the same problem?}
% \iinote{Add:
% % In Exp 2, we show the affect of varying query precision and table-\ref{table:queryperformance}. In the experiment runs for Exp 2 as shown in table-\ref{table:queryperformance} that we performeven with bad slam because it drift, our performs better than Naive. %
% Table-\ref{table:queryperformance} and Table-\ref{table:results1} show that increasing the false positive rate of the visual detector and decreasing the query precision degrades the performance of object retrieval from the spatial-temporal representaion created by D3A. Nevertheless, the deployed D3A+D algorithm on Kuri with the real-world detector performs better than the baseline in terms of memory and speed efficiency of object retrieval. Note that D3A+D uses the object locations calculated based on the results of SLAM and despite drifting localization results of the robot's pose over time, D3A+D is still able to perform better than the Naive Baseline.
% }
We take the first step in aggregating the detections of objects over space and time with our proposed algorithm - D3A, and defer the reduction of duplicates by using improved deep learned object embeddings instead of color histograms to future work. We also plan to conduct human user study in the future to investigate their user experience when querying D3A.
% the improvement in comfort brought to the user by searching over fewer keyframes when querying spatial-temporal information of the objects.
% These reasons lead to duplicates in our system \textit{i.e.,} one object instance in the environment is mapped to multiple unique $object\_id$ s in the database.
% Further, because of the errors described above, the number of unique instances identified by our algorithm are 25 as compared to the actual 59. More probabilistic filtering approaches can be added to reduce the duplicates.
% \begin{figure}[!htbp]
% \centering
% \includegraphics[width=0.8\linewidth]{images/ablation_centroid.png}
% \caption{Exp4 - C}
% % \label{fig:a}
% \label{fig:graph_noise}
% \end{figure}
% \subsection{Results and Discussion}
% \subsubsection*{Exp 1:}
% \subsubsection*{Exp2:} \textbf{Compactness}:
% \textbf{Query Results}
% Sine most of the frames returned by the Naive do not contain the relevant object instance and many have duplicate views of the same object,
% our system takes takes {117x} faster than the baseline. \iinote{need to check results}.
% with a much higher miss rate\iinote{1.25x} since the first 50 frames do not contain the relevant object instance and many have duplicate views of the same object.
% This difference is also reflected in the retrieval and evaluation times.
% With \textit{category level query matching}, the mean frames returned by the deployed system are (50x) smaller than the mean frames returned by the baseline and only 2 counts higher than the perfect system. This difference is also reflected in the retrieval and evaluation times. On average our deployed system took \iinote{50ms} to retrieve and evaluate all the frames for all the object instances while the baseline took \iinote{2secs}. Keep in mind that these times are reported just for 3 hours of ground truth data. The MRR@50 for deployed system is on average \iinote{0.4} which signifies that on average the object of interest in the query is found in the third frame, while for the baseline it is \iinote{0.3} with a much higher miss rate\iinote{1.25x} since the first 50 frames do not contain the relevant object instance and many have duplicate views of the same object.
% There are still duplicates in our system \textit{i.e.} one object instance in the environment is mapped to multiple unique $object\_id$ s. The average number of $object\_id$s assigned to the same object instance by RoboMem is \iinote{23.6 $\pm$ 7.50}, which is \iinote{96\%} smaller than the number of duplicates in the baseline. Our algorithm takes two order magnitudes more time to process the data than the baseline however the representation is more compact and efficient in answering questions as shown in the results here.
% The overall accuracy of RoboMem is $81.98\%$ with $47x$ faster than the baseline. The retrieval performance of the query matching levels is shown to be in this order - Level 1, 2, 3 with level 3 performing the worst since more mean key frames need to be looked into.
% \iinote{talk about low frame rate?}
% \iinote{talk about reasons for missed frames}
% \iinote{The retrieval performance of the query matching levels is shown to be in this order - Level 1, 2, 3 with level 3 performing the worst since more mean key frames need to be looked into.}
\section{Conclusion}
\begin{figure}[]
\centering
\includegraphics[width=0.6\linewidth]{images/final.png}
\caption{Exp4 - Visualization of the keyframe centroid and the aggregated detections by the ``Non-spatial'' baseline \cite{hsieh2018focus} for the pink cup. All the square images are the detections and the different bounding box colors show the respective clusters they were mapped to by D3A based onboth the embedding feature and spatial information. The dotted line shows if the two clusters were mapped to the same \textit{ObjectID}.}
\label{fig:clusters}
\vspace{-7mm}
\end{figure}
We present a novel algorithm for robots to efficiently answer spatial-temporal queries about objects in the environment over long periods of time. Our algorithm aggregates partial view detections of unique instances to create a compact and query-able representation of the objects. By explicitly performing detection-based three-level association to identify the keyframes for unique object instances, our algorithm significantly outperforms baselines in answering queries in terms of the retrieved frames' accuracy, mean reciprocal rank and retrieval time. A robot deployed with our algorithm was able to process 22 hours of
% \iinote{I evaluate on 3 hours? but do make compact for 22 hours so should it be 22 hours?} \tn{i'd say 22 bc it did process and compress that much data, we just don't know how well}
sensor data and develop a compact representation of objects in its environment, which is an encouraging step towards enhancing the sensory capabilities of home-service robots that can help the elderly find their lost, forgotten objects.
% \section{Copyright}
% All papers submitted for publication by AAAI Press must be accompanied by a valid signed copyright form. They must also contain the AAAI copyright notice at the bottom of the first page of the paper. There are no exceptions to these requirements. If you fail to provide us with a signed copyright form or disable the copyright notice, we will be unable to publish your paper. There are \textbf{no exceptions} to this policy. You will find a PDF version of the AAAI copyright form in the AAAI AuthorKit. Please see the specific instructions for your conference for submission details.
% \subsubsection{Acknowledgments.}
% The acknowledgments section, if included, appears after the main body of text and is headed ``Acknowledgments." This section includes acknowledgments of help from associates and colleagues, credits to sponsoring agencies, financial support, and permission to publish. Please acknowledge other contributors, grant support, and so forth, in this section. Do not put acknowledgments in a footnote on the first page. If your grant agency requires acknowledgment of the grant on page 1, limit the footnote to the required statement, and put the remaining acknowledgments at the back. Please try to limit acknowledgments to no more than three sentences.
% \subsubsection{Appendices.}
% Any appendices follow the acknowledgments, if included, or after the main body of text if no acknowledgments appear.
% The following commands are available for your use in citing references:
% \begin{quote}
% {\em \textbackslash cite:} Cites the given reference(s) with a full citation. This appears as ``(Author Year)'' for one reference, or ``(Author Year; Author Year)'' for multiple references.\smallskip\\
% {\em \textbackslash shortcite:} Cites the given reference(s) with just the year. This appears as ``(Year)'' for one reference, or ``(Year; Year)'' for multiple references.\smallskip\\
% {\em \textbackslash citeauthor:} Cites the given reference(s) with just the author name(s) and no parentheses.\smallskip\\
% {\em \textbackslash citeyear:} Cites the given reference(s) with just the date(s) and no parentheses.
% \end{quote}
\bibliography{references}
\end{document}
\begin{thebibliography}{27}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\providecommand{\urlprefix}{URL }
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi:\discretionary{}{}{}#1}\else
\providecommand{\doi}{doi:\discretionary{}{}{}\begingroup
\urlstyle{rm}\Url}\fi
\bibitem[{Ambru{\c{s}} et~al.(2014)Ambru{\c{s}}, Bore, Folkesson, and
Jensfelt}]{ambrucs2014meta}
Ambru{\c{s}}, R.; Bore, N.; Folkesson, J.; and Jensfelt, P. 2014.
\newblock Meta-rooms: Building and maintaining long term spatial models in a
dynamic world.
\newblock In \emph{2014 IEEE/RSJ IROS}, 1854--1861. IEEE.
\bibitem[{Banker(2011)}]{banker2011mongodb}
Banker, K. 2011.
\newblock \emph{MongoDB in action}.
\newblock Manning Publications Co.
\bibitem[{Bore, Jensfelt, and Folkesson(2015)}]{bore2015retrieval}
Bore, N.; Jensfelt, P.; and Folkesson, J. 2015.
\newblock Retrieval of arbitrary 3D objects from robot observations.
\newblock In \emph{2015 European Conference on Mobile Robots (ECMR)}, 1--8.
IEEE.
\bibitem[{Br{\'a}s and Neto(2018)}]{bras2018unsupervised}
Br{\'a}s, A.; and Neto, P. 2018.
\newblock Unsupervised Feature Extraction from RGB-D Data for Object
Classification: a Case Study on the YCB Object and Model Set.
\newblock In \emph{IECON 2018-44th Annual Conference of the IEEE Industrial
Electronics Society}, 3673--3678. IEEE.
\bibitem[{Elahi and Yang(2020)}]{elahi2020online}
Elahi, G.; and Yang, Y.-H. 2020.
\newblock Online Learnable Keyframe Extraction in Videos and its Application
with Semantic Word Vector in Action Recognition.
\newblock \emph{arXiv preprint arXiv:2009.12434} .
\bibitem[{Essa, Sidike, and Asari(2015)}]{essa2015modular}
Essa, A.; Sidike, P.; and Asari, V. 2015.
\newblock A modular approach for key-frame selection in wide area surveillance
video analysis.
\newblock In \emph{2015 National Aerospace and Electronics Conference
(NAECON)}, 41--44. IEEE.
\bibitem[{Girshick et~al.(2018)Girshick, Radosavovic, Gkioxari, Doll\'{a}r, and
He}]{Detectron2018}
Girshick, R.; Radosavovic, I.; Gkioxari, G.; Doll\'{a}r, P.; and He, K. 2018.
\newblock Detectron.
\newblock https://github.com/facebookresearch/detectron.
\bibitem[{Girshick(2015)}]{FastCNN}
Girshick, R.~B. 2015.
\newblock Fast {R-CNN}.
\newblock \emph{CoRR} abs/1504.08083.
\newblock \urlprefix\url{http://arxiv.org/abs/1504.08083}.
\bibitem[{He et~al.(2018)He, Luo, Tian, and Zeng}]{object_tracking}
He, A.; Luo, C.; Tian, X.; and Zeng, W. 2018.
\newblock A Twofold Siamese Network for Real-Time Object Tracking.
\newblock \emph{CoRR} abs/1802.08817.
\newblock \urlprefix\url{http://arxiv.org/abs/1802.08817}.
\bibitem[{He et~al.(2017)He, Gkioxari, Doll{\'{a}}r, and Girshick}]{MaskRCNN}
He, K.; Gkioxari, G.; Doll{\'{a}}r, P.; and Girshick, R.~B. 2017.
\newblock Mask {R-CNN}.
\newblock \emph{CoRR} abs/1703.06870.
\newblock \urlprefix\url{http://arxiv.org/abs/1703.06870}.
\bibitem[{Hsieh et~al.(2018)Hsieh, Ananthanarayanan, Bodik, Venkataraman, Bahl,
Philipose, Gibbons, and Mutlu}]{hsieh2018focus}
Hsieh, K.; Ananthanarayanan, G.; Bodik, P.; Venkataraman, S.; Bahl, P.;
Philipose, M.; Gibbons, P.~B.; and Mutlu, O. 2018.
\newblock Focus: Querying large video datasets with low latency and low cost.
\newblock In \emph{13th $\{$USENIX$\}$ Symposium on Operating Systems Design
and Implementation ($\{$OSDI$\}$ 18)}, 269--286.
\bibitem[{Kang, Bailis, and Zaharia(2019)}]{kang2019challenges}
Kang, D.; Bailis, P.; and Zaharia, M. 2019.
\newblock Challenges and Opportunities in DNN-Based Video Analytics: A
Demonstration of the BlazeIt Video Query Engine.
\newblock In \emph{CIDR}.
\bibitem[{Khan et~al.(2014)Khan, Rehman, Aziz, Fong, and
Sarasvady}]{khan2014dbscan}
Khan, K.; Rehman, S.~U.; Aziz, K.; Fong, S.; and Sarasvady, S. 2014.
\newblock DBSCAN: Past, present and future.
\newblock In \emph{The fifth (ICADIWT 2014)}, 232--238. IEEE.
\bibitem[{Li(2017)}]{li2017querying}
Li, F. 2017.
\newblock \emph{Querying RDBMS Using Natural Language}.
\newblock Ph.D. thesis, University of Michigan.
\bibitem[{Lin et~al.(2017)Lin, Goyal, Girshick, He, and
Doll{\'{a}}r}]{RetinaNet}
Lin, T.; Goyal, P.; Girshick, R.~B.; He, K.; and Doll{\'{a}}r, P. 2017.
\newblock Focal Loss for Dense Object Detection.
\newblock \emph{CoRR} abs/1708.02002.
\newblock \urlprefix\url{http://arxiv.org/abs/1708.02002}.
\bibitem[{Mayfield(2018)}]{heykuri_2018}
Mayfield, R. 2018.
\newblock Meet Kuri! The Adorable Home Robot.
\newblock \urlprefix\url{https://www.heykuri.com/}.
\bibitem[{{Philbin} et~al.(2007){Philbin}, {Chum}, {Isard}, {Sivic}, and
{Zisserman}}]{object_retrieval}
{Philbin}, J.; {Chum}, O.; {Isard}, M.; {Sivic}, J.; and {Zisserman}, A. 2007.
\newblock Object retrieval with large vocabularies and fast spatial matching.
\newblock In \emph{2007 IEEE CVPR}, 1--8.
\newblock ISSN 1063-6919.
\newblock \doi{10.1109/CVPR.2007.383172}.
\bibitem[{Quigley et~al.(2009)Quigley, Conley, Gerkey, Faust, Foote, Leibs,
Wheeler, and Ng}]{quigley2009ros}
Quigley, M.; Conley, K.; Gerkey, B.; Faust, J.; Foote, T.; Leibs, J.; Wheeler,
R.; and Ng, A.~Y. 2009.
\newblock ROS: an open-source Robot Operating System.
\newblock In \emph{ICRA workshop on open source software}, volume~3, 5. Kobe,
Japan.
\bibitem[{Sivic and Zisserman(2003)}]{sivic2003video}
Sivic, J.; and Zisserman, A. 2003.
\newblock Video Google: A text retrieval approach to object matching in videos.
\newblock In \emph{null}, 1470. IEEE.
\bibitem[{Spannbauer(2018)}]{color_hist}
Spannbauer, A. 2018.
\newblock Finding and Using Images' Dominant Colors using Python and OpenCV.
\newblock
\urlprefix\url{https://adamspannbauer.github.io/2018/03/02/app-icon-dominant-colors/}.
\bibitem[{V{\'a}zquez-Mart{\'\i}n and Bandera(2013)}]{vazquez2013spatio}
V{\'a}zquez-Mart{\'\i}n, R.; and Bandera, A. 2013.
\newblock Spatio-temporal feature-based keyframe detection from video shots
using spectral clustering.
\newblock \emph{Pattern Recognition Letters} 34(7): 770--779.
\bibitem[{Wang et~al.(2019)Wang, Shin, Liu, Polozov, and
Richardson}]{DBLP:journals/corr/abs-1911-04942}
Wang, B.; Shin, R.; Liu, X.; Polozov, O.; and Richardson, M. 2019.
\newblock {RAT-SQL:} Relation-Aware Schema Encoding and Linking for Text-to-SQL
Parsers.
\newblock \emph{CoRR} abs/1911.04942.
\newblock \urlprefix\url{http://arxiv.org/abs/1911.04942}.
\bibitem[{{Wang} et~al.(2012){Wang}, {Hong}, {Li}, {Zha}, {Yan}, and
{Chua}}]{6135507}
{Wang}, M.; {Hong}, R.; {Li}, G.; {Zha}, Z.; {Yan}, S.; and {Chua}, T. 2012.
\newblock Event Driven Web Video Summarization by Tag Localization and Key-Shot
Identification.
\newblock \emph{IEEE Transactions on Multimedia} 14(4): 975--985.
\newblock ISSN 1520-9210.
\newblock \doi{10.1109/TMM.2012.2185041}.
\bibitem[{Xu et~al.(2014)Xu, Mo, Feng, Zhong, Lai, Eric, and
Chang}]{xu2014deep}
Xu, Y.; Mo, T.; Feng, Q.; Zhong, P.; Lai, M.; Eric, I.; and Chang, C. 2014.
\newblock Deep learning of feature representation with multiple instance
learning for medical image analysis.
\newblock In \emph{2014 IEEE (ICASSP)}, 1626--1630. IEEE.
\bibitem[{Yadav and Curry(2019)}]{yadav2019vidcep}
Yadav, P.; and Curry, E. 2019.
\newblock VidCEP: Complex Event Processing Framework to Detect Spatiotemporal
Patterns in Video Streams.
\newblock In \emph{2019 IEEE International Conference on Big Data (Big Data)},
2513--2522. IEEE.
\bibitem[{Yilmaz, Javed, and Shah(2006)}]{yilmaz2006object}
Yilmaz, A.; Javed, O.; and Shah, M. 2006.
\newblock Object tracking: A survey.
\newblock \emph{Acm computing surveys (CSUR)} 38(4): 13--es.
\bibitem[{Zakharov, Shugurov, and Ilic(2019)}]{pose_estimation}
Zakharov, S.; Shugurov, I.; and Ilic, S. 2019.
\newblock {DPOD:} Dense 6D Pose Object Detector in {RGB} images.
\newblock \emph{CoRR} abs/1902.11020.
\newblock \urlprefix\url{http://arxiv.org/abs/1902.11020}.
\end{thebibliography}