Cyclist Trajectory Forecasts by Incorporation of Multi-View Video Information
Extracted source text ยท arXiv 2106.15991
%% bare_conf.tex
%% V1.4b
%% 2015/08/26
%% by Michael Shell
%% See:
%% http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.8b or later) with an IEEE
%% conference paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/pkg/ieeetran
%% and
%% http://www.ieee.org/
%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE!
%% User assumes all risk.
%% In no event shall the IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including **
%% ** renaming them and changing author support contact information. **
%%*************************************************************************
% *** Authors should verify (and, if needed, correct) their LaTeX system ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. The IEEE's font choices and paper sizes can ***
% *** trigger bugs that do not appear when using other class files. *** ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/
\documentclass[letterpaper, 10pt, conference]{ieeeconf}
% Use this line for a4 paper
\IEEEoverridecommandlockouts
% Needed if you want to use the \thanks command
\overrideIEEEmargins
% Some Computer Society conferences also require the compsoc mode option,
% but others use the standard conference format.
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[conference]{../sty/IEEEtran}
% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)
% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
% % pdf code
% \else
% % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/pkg/ifpdf
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.
% *** CITATION PACKAGES ***
%
%\usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of the IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off
% such as if a citation ever needs to be enclosed in parenthesis.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 5.0 (2009-03-20) and later if using hyperref.sty.
% The latest version can be obtained at:
% http://www.ctan.org/pkg/cite
% The documentation is contained in the cite.sty file itself.
%\ifCLASSINFOpdf
\usepackage[pdftex]{graphicx}
% declare the path(s) where your graphic files are
\graphicspath{{../pdf/}{../jpeg/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
\DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage[noend]{algpseudocode}
\usepackage{dblfloatfix}
\usepackage{siunitx}
\usepackage{arydshln}
\usepackage{graphicx}
\sisetup{detect-all=true}
\DeclareSIUnit\pixel{px}
%\else
% or other class option (dvipsone, dvipdf, if not using dvips). graphicx
% will default to the driver specified in the system graphics.cfg if no
% driver is specified.
%\usepackage[dvips]{graphicx}
% declare the path(s) where your graphic files are
% \graphicspath{{../eps/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.eps}
%\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/graphics/
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found as epslatex.ps or
% epslatex.pdf at: http://www.ctan.org/tex-archive/info/
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex
\usepackage[natural]{xcolor}
% *** MATH PACKAGES ***
%
\usepackage[cmex10]{amsmath}
% *** SPECIALIZED LIST PACKAGES ***
%
%\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as the IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/pkg/algorithms
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/pkg/algorithmicx
% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/array
% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.
% *** SUBFIGURE PACKAGES ***
%\ifCLASSOPTIONcompsoc
% \usepackage[caption=false,font=normalsize,labelfont=sf,textfont=sf]{subfig}
%\else
% \usepackage[caption=false,font=footnotesize]{subfig}
%\fi
% subfig.sty, written by Steven Douglas Cochran, is the modern replacement
% for subfigure.sty, the latter of which is no longer maintained and is
% incompatible with some LaTeX packages including fixltx2e. However,
% subfig.sty requires and automatically loads Axel Sommerfeldt's caption.sty
% which will override IEEEtran.cls' handling of captions and this will result
% in non-IEEE style figure/table captions. To prevent this problem, be sure
% and invoke subfig.sty's "caption=false" package option (available since
% subfig.sty version 1.3, 2005/06/28) as this is will preserve IEEEtran.cls
% handling of captions.
% Note that the Computer Society format requires a larger sans serif font
% than the serif footnote size font used in traditional IEEE formatting
% and thus the need to invoke different subfig.sty package options depending
% on whether compsoc mode has been enabled.
%
% The latest version and documentation of subfig.sty can be obtained at:
% http://www.ctan.org/pkg/subfig
% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure.
% Be aware that LaTeX2e kernels dated 2015 and later have fixltx2e.sty's
% corrections already built into the system in which case a warning will
% be issued if an attempt is made to load fixltx2e.sty as it is no longer
% needed.
% The latest version and documentation can be found at:
% http://www.ctan.org/pkg/fixltx2e
%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/stfloats
% Do not use the stfloats baselinefloat ability as the IEEE does not allow
% \baselineskip to stretch. Authors submitting work to the IEEE should note
% that the IEEE rarely uses double column equations and that authors should try
% to avoid such use. Do not be tempted to use the cuted.sty or midfloat.sty
% packages (also by Sigitas Tolusis) as the IEEE does not format its papers in
% such ways.
% Do not attempt to use stfloats with fixltx2e as they are incompatible.
% Instead, use Morten Hogholm'a dblfloatfix which combines the features
% of both fixltx2e and stfloats:
%
% \usepackage{dblfloatfix}
% The latest version can be found at:
% http://www.ctan.org/pkg/dblfloatfix
% *** PDF, URL AND HYPERLINK PACKAGES ***
%
%\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/url
% Basically, \url{my_url_here}.
% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex). ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )
% correct bad hyphenation here
\usepackage[pdftex]{graphicx}
\usepackage{url}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage[noend]{algpseudocode}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage[natural]{xcolor}
\usepackage{changes}
\usepackage[cmex10]{amsmath}
\usepackage{mathtools}
\usepackage{url}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage[noend]{algpseudocode}
\usepackage{siunitx}
\usepackage{gensymb}
\usepackage{multirow}
%
%\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
%
% paper title
% Titles are generally capitalized except for words such as a, an, and, as,
% at, but, by, for, in, nor, of, on, or, the, to and up, which are usually
% not capitalized unless they are the first or last word of the title.
% Linebreaks \\ can be used within to get better formatting as desired.
% Do not put math or special symbols in the title.
\title{Cyclist Trajectory Forecasts by Incorporation of Multi-View Video Information}
% author names and affiliations
% use a multiple column layout for up to three different
% affiliations
\author{Stefan Zernetsch, Oliver Trupp, Viktor Kress, Konrad Doll, and Bernhard Sick
% <-this % stops a space
%\thanks{This work was not supported by any organization}% <-this % stops a space
\thanks{S. Zernetsch, O. Trupp, V. Kress, and K. Doll are with the Faculty of Engineering,
University of Applied Sciences Aschaffenburg, Aschaffenburg, Germany
{\tt\footnotesize stefan.zernetsch, oliver.trupp, viktor.kress, konrad.doll\{@th-ab.de\}}}
\thanks{B. Sick is with the Intelligent Embedded Systems Lab, University of Kassel,
Kassel, Germany
{\tt\footnotesize bsick@uni-kassel.de}}%
}
%\author{\IEEEauthorblockN{Stefan Zernetsch, Oliver Trupp, Viktor Kress, and Konrad Doll}
%\IEEEauthorblockA{University of Applied Sciences Aschaffenburg\\
%Aschaffenburg, Germany\\
%\{stefan.zernetsch, oliver.trupp, viktor.kress, konrad.doll\}@th-ab.de}
%\and
%\IEEEauthorblockN{Bernhard Sick}
%\IEEEauthorblockA{University of Kassel\\
%Kassel, Germany\\
%bsick@uni-kassel.de}}
% conference papers do not typically use \thanks and this command
% is locked out in conference mode. If really needed, such as for
% the acknowledgment of grants, issue a \IEEEoverridecommandlockouts
% after \documentclass
% for over three affiliations, or if they all won't fit within the width
% of the page, use this alternative format:
%
%\author{\IEEEauthorblockN{Michael Shell\IEEEauthorrefmark{1},
%Homer Simpson\IEEEauthorrefmark{2},
%James Kirk\IEEEauthorrefmark{3},
%Montgomery Scott\IEEEauthorrefmark{3} and
%Eldon Tyrell\IEEEauthorrefmark{4}}
%\IEEEauthorblockA{\IEEEauthorrefmark{1}School of Electrical and Computer Engineering\\
%Georgia Institute of Technology,
%Atlanta, Georgia 30332--0250\\ Email: see http://www.michaelshell.org/contact.html}
%\IEEEauthorblockA{\IEEEauthorrefmark{2}Twentieth Century Fox, Springfield, USA\\
%Email: homer@thesimpsons.com}
%\IEEEauthorblockA{\IEEEauthorrefmark{3}Starfleet Academy, San Francisco, California 96678-2391\\
%Telephone: (800) 555--1212, Fax: (888) 555--1212}
%\IEEEauthorblockA{\IEEEauthorrefmark{4}Tyrell Inc., 123 Replicant Street, Los Angeles, California 90210--4321}}
% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}
% make the title area
\maketitle
% As a general rule, do not put math, special symbols or citations
% in the abstract
\input{sections/abstract}
% no keywords
% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle
\input{sections/introduction}
\input{sections/method_overview}
\input{sections/dataset}
\input{sections/method_basic_movement_detection}
\input{sections/results_outline}
\input{sections/conclusion}
\input{sections/acknowledgment}
% conference papers do not normally have an appendix
% use section* for acknowledgment
%\section*{Acknowledgment}
%The authors would like to thank...
% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}
% references section
% can use a bibliography generated by BibTeX as a .bbl file
% BibTeX documentation can be easily obtained at:
% http://mirror.ctan.org/biblio/bibtex/contrib/doc/
% The IEEEtran BibTeX style support page is at:
% http://www.michaelshell.org/tex/ieeetran/bibtex/
\bibliographystyle{IEEEtran}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{IEEEabrv,../bib/paper}
%
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
{\small
\bibliography{IEEEabrv,sz}
}
% that's all folks
\end{document}
\begin{abstract}
This article presents a novel approach to incorporate visual cues from video-data from a wide-angle stereo camera system mounted at an urban intersection into the forecast of cyclist trajectories. We extract features from image and optical flow (OF) sequences using 3D convolutional neural networks (3D-ConvNet) and combine them with features extracted from the cyclist's past trajectory to forecast future cyclist positions. By the use of additional information, we are able to improve positional accuracy by about 7.5\,\% for our test dataset and by up to 22\,\% for specific motion types compared to a method solely based on past trajectories. Furthermore, we compare the use of image sequences to the use of OF sequences as additional information, showing that OF alone leads to significant improvements in positional accuracy. By training and testing our methods using a real-world dataset recorded at a heavily frequented public intersection and evaluating the methods' runtimes, we demonstrate the applicability in real traffic scenarios. Our code and parts of our dataset are made publicly available.
%In this article, we present an approach to detect basic movements of cyclists in real world traffic situations based on image sequences, optical flow (OF) sequences, and past positions using a multi-stream 3D convolutional neural network (3D-ConvNet) architecture. To resolve occlusions of cyclists by other traffic participants or road structures, we use a wide angle stereo camera system mounted at a heavily frequented public intersection. We created a large dataset consisting of 1,639 video sequences containing cyclists, recorded in real world traffic, resulting in over 1.1 million samples. Through modeling the cyclists' behavior by a state machine of basic cyclist movements, our approach takes every situation into account and is not limited to certain scenarios. We compare our method to an approach solely based on position sequences. Both methods are evaluated taking into account frame wise and scene wise classification results of basic movements, and detection times of basic movement transitions, where our approach outperforms the position based approach by producing more reliable detections with shorter detection times. Our code and parts of our dataset are made publicly available.
\end{abstract}
\section{\large Introduction}
\label{sec_introduction}
\subsection{Motivation}
In future traffic, vehicles will become more and more automated. In this scenario, automated vehicles will share the road with non-automated vehicles and vulnerable road users (VRU) such as pedestrians and cyclists. Therefore, every automated vehicle must be aware of its surroundings at all times to ensure safe interaction with other traffic participants. Automated vehicles not only have to know the current positions of other traffic participants but anticipate their movements to plan a safe trajectory. Especially the trajectories of VRU are hard to anticipate since they can change their direction quickly and are not limited to roads. Additionally, they are prone to occlusion by other road users or structures on the roadside and, therefore, hard to track by a single vehicle's sensors in complex traffic scenarios. Sensors mounted at the roadside can be used to resolve this problem. Infrastructural sensors are less susceptible to occlusions and can deliver higher positional accuracy compared to vehicle sensors.
To anticipate the future behavior of VRU, methods that use the past VRU trajectories extracted from image data have been proposed. While these methods effectively reduce the original images' input dimensions, visual cues that can help to generate more reliable forecasts, like gaze direction, are lost. In this article, we propose a method to incorporate video information from a wide-angle stereo camera system to improve forecasts of VRU, showing that the additional information can lead to significant improvements in positional accuracy. Our method is trained and tested for cyclists, but can also be applied to pedestrians. The focus of this article is on the incorporation of multi-view video information into the forecast process. Therefore, we do not consider additional information like interaction between different road users or environmental information like maps in our proposed method or in our baseline method.
%In future traffic, automated vehicles will share the road with non automated vehicles and vulnerable road users (VRU), such as pedestrians and cyclists. To ensure safe interaction in this mixed traffic scenario, it is crucial that automated vehicles know their surroundings at all time. To plan a safe trajectory, automated vehicles not only have to know the positions of other traffic participants, but must also anticipate their future trajectories. While automated vehicles can share their planned paths, trajectories of other road users have to be estimated. Anticipating the behavior of VRUs is particularly difficult. Compared to vehicles, they are more agile and not limited to roads. Additionally, due to occlusions by other road users or structures on the roadside, it is not possible to detect VRU intentions from a single vehicle. To resolve this problem, sensors mounted at the roadside can be used. Infrastructural sensors are less susceptible to occlusions and can deliver higher positional accuracy compared to vehicle sensors.
%Activity in the field of pedestrian intention detection has grown recently. However, when it comes to cyclist intention detection, there is still little research. Therefore, our proposed method is dedicated to detect basic movements of cyclists in real traffic situations in the form of probability outputs with high accuracy. As shown in \cite{ownBieshaarTrans2018}, these probabilities can serve as weights for specialized trajectory forecasting models and therefore, can act as a basis for planing algorithms in automated vehicles in future mixed traffic scenarios.
\subsection{Related Work}
Over the past years, the field of VRU trajectory forecast has become more active. Most of the proposed methods use information about the past position to forecast future positions. Goldhammer et al. extract polynomial coefficients from trajectories as input for a neural network to forecast future pedestrian trajectories~\cite{goldhammer2016phd}. The method is adapted to cyclists in~\cite{ownZernetsch_iv2016}. Compared to physical models, the neural network produces forecasts with larger positional accuracies.
To improve forecast accuracies, additional information can be added to the past trajectory. Alahi et al. propose a long short-term memory (LSTM) combined with a social pooling to add additional information about interactions with other pedestrians~\cite{Alahi_2016_CVPR}. Their method outperforms an LSTM, which uses solely past positions as input. Pool et al. use cyclists' past trajectory and add additional information about the road topology to mix specialized filters to forecast the future cyclist trajectory~\cite{Pool2017Context}. The authors are able to improve forecast accuracy by up to 20\,\% on sharp turns compared to a single model approach. Instead of a pedestrian's single past trajectory, Quintero et al. use the past trajectories of eleven body joints to provide additional information about movement and body language~\cite{quintero2018}. The joint trajectories are used in combination with balanced Gaussian process dynamical models (GPDM) to detect basic movements and forecast trajectories of pedestrians. Kress et al. use joint trajectories in combination with gated recurrent units (GRU) to forecast VRU trajectories~\cite{Kress2020}. Using the additional information from body joints, they are able to forecast more accurate positions with shorter observation periods than a solely trajectory based method. Keller and Gavrila compare the use of features derived from dense optical flow (OF) in combination with GPDM and hierarchical trajectory matching to two solely trajectory-based methods~\cite{willpedcross}. The methods based on OF improve the forecast errors by 10-50 cm at time horizons of 0-0.77 s. Sagedian et al. incorporate physical and social constraints into their forecast of pedestrian trajectories~\cite{Sadeghian2019}. They pass features extracted from a single scene image using a convolutional neural network (CNN) and hidden states of LSTM for multiple agents to an attention module. Their model outperforms baseline models like LSTM or social LSTM from~\cite{Alahi_2016_CVPR}.
Aside from intention detection in traffic scenarios, video-based human action recognition has become an active research area in recent years, with the emergence of several network architectures that can be applied to the field of VRU intention detection. In \cite{Carreira}, Carreira and Zisserman present a novel approach to video-based action recognition and compare it to the most commonly used architectures. The authors propose a two-stream approach with two 3D-ConvNets with inception architecture using an image sequence and an OF sequence as input data. The network, referred to as I3D (Inflated 3D), outperforms the other architectures on the Kinetics human action detection video dataset~\cite{Kay2017TheKH} and achieves 98.0\,\% accuracy on the UCF101 Human Action Classes dataset~\cite{Khurram}. The I3D architecture is extended to cyclist action in~\cite{ownZernetsch_icpr2020} and is used as a feature extractor in this article for cyclist trajectory forecasts. While we chose I3D due to its great performance regarding cyclist action recognition, the feature extractor used in the method is interchangeable.
% By adding information from video sequences, we aim to incorporate different features, like body language or interaction with close-by traffic participants into our forecast.
%Over the past years, the field of VRU intention detection has become more active. The developed methods can roughly be grouped into position based methods, where current or past positions of the VRUs are used to detect their current movement state, and image based methods, where either raw images, image sequences or extracted image features are used to detect the current movement state.
%In \cite{willpedcross}, Keller and Gavrila compared four methods to detect pedestrians' intentions to stop at a curbside or continue walking. Two methods are based on Gaussian process dynamical models (GPDM) and hierarchical trajectory matching which use features derived from dense OF and two are based on Kalman Filters that use positional information only, where the OF based methods showed more accurate predictions for stopping movements. K\"ohler et al. \cite{Koehler.2013}\cite{koehler2015} used a motion history image (MHI) based histogram feature vector (MCHOG) extracted from stereo image sequences in combination with a support vector machine to detect motion transitions of pedestrians, such as \textit{start} or \textit{stop}. In \cite{goldhammer2019}, the approach was compared to a method using a polynomial approximation of the pedestrians' past head velocities in combination with a multilayer preceptron (MLP). The comparison showed a faster and more accurate state detection of the MCHOG approach. In \cite{Quintero.2017} and \cite{Quintero.2019}, Quintero et al. used GPDM in combination with human poses extracted from image sequences to detect start intentions of pedestrians. Kress et al. \cite{kress.itsc.2019} used 3D poses of cyclists to detect start intentions from a moving vehicle. Compared to a solely head trajectory based approach, the pose based approach achieved higher scores for shorter input trajectories.
%In \cite{ownZernetsch_iv2018}, the authors used MHI of cyclists in combination with a residual network architecture (ResNet) to detect start intentions and compared the results to the MCHOG based approach from \cite{Koehler.2013}, where the ResNet approach lead to faster and more accurate detections. In \cite{ownBieshaarTrans2018}, the authors used a spatio temporal 3D ConvNet with ResNet architecture to detect start intentions using image sequences of starting cyclists as input. The predicted starting probabilities were used to weigh between two specialized trajectory forecast models, leading to higher positional accuracy compared to a single forecast model. Most of these methods were developed for pedestrian intention detection. When it comes to cyclists intention detection, there are still few publications available. Furthermore, most of these methods focus on specific scenarios, e.\,g., detection of starting or crossing intentions. In this work, we created a comprehensive dataset of cyclist movements at a public intersection and modeled the cyclists' behavior by a state machine containing all possible types of movements of the cyclists, which are then detected using our proposed method.
%Aside from intention detection in traffic scenarios, video based action recognition has become an active field of research over the past years, where different network architectures emerged, which have the potential to be applied to the domain of VRU intention detection. In \cite{Carreira}, Carreira and Zisserman introduce a novel approach for video based action recognition and compare it to the most commonly used architectures, such as combinations of ConvNets with long short-term memories (LSTM), where every image of a sequence is fed to a ConvNet and the output is passed to an LSTM \cite{donahue}\cite{HeiNg}, 3D-ConvNets, where spatio temporal convolution is performed on image sequences \cite{Ji}\cite{ownBieshaarTrans2018}, and two-stream approaches, where an image and an OF sequence are fused using two ConvNets \cite{Simonyan}\cite{Feichtenhofer}. The authors extend the two-stream approach by inflating two streams of 2D Inception ConvNets \cite{Szegedy} to 3D ConvNets (I3D) using an image sequence, as well as an OF sequence as input. Their two-stream I3D network outperformed the other architectures on the Kinetics Human Action Detection Video Dataset \cite{Kay2017TheKH} and achieved 98.0\% accuracy on the UCF101 Human Action Classes dataset \cite{Khurram}. Therefore, the architecture we created in this article, is based on the two-stream I3D network proposed in \cite{Carreira}.
\subsection{Main Contributions and Outline of this Paper}
The main contribution of this article is a method to incorporate multi-view video information from cameras mounted at a research intersection into the trajectory forecast of cyclists. By adding information from video sequences, we aim to incorporate different features, like body language or gestures into our forecast. The use of a wide-angle stereo camera system allows us to extract features of the cyclists from different view angles and resolve occlusions. We are able to improve the positional accuracy of forecasts by about 10\,\% compared to a baseline model using positional information only. For specific motion types, we are able to reduce the forecast error by up to 22\,\%. Our method is trained and tested using a dataset recorded at a public intersection showing its real-world application. Evaluation of runtimes shows that our method can be incorporated into a real-time system. The dataset and code used to train and test our final method are made publicly available \cite{zernetsch_cyclistactionrec}\cite{zernetsch_github_iv2021}.
The article's remainder is structured as follows: In Sec.~2, we describe the intersection, where the dataset is created, and the dataset itself. Sec.~3 describes the architectures of the baseline model and the video-based network, followed by the evaluation method we use. In Sec.~4, the results of both methods are visualized and compared, followed by a conclusion and a short outlook in Sec.~5.
%The main contribution of this article is an approach to detect basic movements of cyclists using image sequences in combination with a 3D-ConvNet architecture. The method is trained and tested using real world data recorded at a public intersection and covers all occurring situations, i.\,e., it is not limited to certain scenarios. In a comprehensive evaluation, taking into account the $\text{F}_1$-scores of frame wise classifications, as well as scene based metrics such as segment scores and detection times, we show that our approach outperforms a solely position based method. Additionally, we investigated the use of different input sequences from different cameras, showing that using inputs from different camera angles can have a positive influence on the detection quality. Our code and parts of our dataset used to create the results in this paper are made publicly available \cite{zernetsch_cyclistactionrec} \cite{zernetsch_github_icpr}.
%The remainder of this paper is structured as follows: In Sec. \ref{subsec:dataset}, we describe the intersection, where the dataset was created, and the dataset itself. Sec. \ref{subsec:basic_movements} describes the state machine defined by us, followed by network architectures of the baseline model and the motion sequence (MS) based network in Sec. \ref{subsec:network_arch}. The frame wise, as well as the segment based evaluation methods are described in Sec. \ref{subsec:evaluation_methods}. In Sec. \ref{sec_ResultsOutline}, the results of both methods are visualized and compared, followed by a conclusion and a short outlook in Sec. \ref{sec_conclusion}.
\section{\large Test Site and Dataset}
\label{subsec:dataset}
\begin{figure}
\begin{center}
\vskip 0mm
\includegraphics[width = 0.99\columnwidth, trim={0 0 0 30mm},clip]{images/intersection/intersection_map.pdf}
\vskip 0mm
\caption{Map of intersection with sidewalk (green), bike lane (blue), and pedestrian crossings (yellow) and example images of camera 1 and 2.}
\label{fig:intersection}
\end{center}
\vskip 0mm
\end{figure}
In this section, we describe the test site where the dataset
used for trajectory forecast is created, and provide a description
of the dataset itself. Except for the ground truth data, the dataset is the same as the dataset used in~\cite{ownZernetsch_icpr2020}.
The dataset was collected at an intersection close to the University of Applied Sciences Aschaffenburg using a wide-angle stereo camera system (Fig. \ref{fig:intersection})~\cite{Goldhammer}. The university's research intersection consists of four arms, two arms with five vehicle lanes each, one arm with three lanes, and one with two lanes. With up to 24,000 vehicles a day, the intersection is highly frequented. A stereo camera system, operating at 50 fps, was mounted to capture a corner of the intersection with a sidewalk (Fig. \ref{fig:intersection} green), two pedestrian crossings (Fig. \ref{fig:intersection} yellow), and a bike lane (Fig. \ref{fig:intersection} blue). The full HD grayscale cameras 1 and 2 are mounted approx. 5 m above the ground at an angle of approx. 90\degree\,relative to each other. Most occlusions by vehicles or other VRU can be avoided by using this setup. The dataset used in this work consists of samples of image sequences of cameras 1 and 2 $I_{1/2}(u, v, t)$, with $u$ and $v$ being the pixel position and $t$ being the temporal dimension and OF sequences of camera 1 and 2 $O_{1/2}(u, v, t)$. Furthermore, the set contains the past and ground truth trajectories of the cyclists $^e\mathcal{T}=\{[^ex_{t-(1s)}, ^ey_{t-(1s)}, ^ez_{t-(1s)}]...[^ex_{t}, ^ey_{t}, ^ez_{t}]\}$ and $^e\mathcal{T}_{gt}=\{[^ex_{gt;t+(0.02s)}, ^ey_{gt;t+(0.02s)}]...[^ex_{gt;t+(2.5s)}, ^ey_{gt;t+(2.5s)}]\}$ as ordered sets of positions transformed to the cyclist's ego coordinate system (indicated by $^e$) in $x$, $y$, and $z$ directions from time step $t-(1s)$ to the current time step $t$. Since we only aim to forecast the future positions in $x$ and $y$ direction, the $z$ coordinate is removed from the ground truth trajectory.
Object detection is performed on each camera image using a pre-trained object detection network from \cite{Huang}, trained on the COCO dataset \cite{Lin2014} (Fig. \ref{fig:ms_creation} left), to generate image sequences. The output of the network consists of object boxes of different classes. We use detected boxes for the classes \textit{person} and \textit{bike} to create a cyclist bounding box. The detected \textit{person} boxes are associated with the \textit{bike} boxes using a simple nearest neighbor-strategy. They are combined to create a bounding box enclosing the cyclist and the bicycle. The resulting bounding box's size is enlarged by a factor $f_b$ to create a region of interest (ROI) to account for the cyclist movements from previous time steps (Fig. \ref{fig:ms_creation} left, blue box). The ROI created at time $t$ is used to extract the sequence of camera images over the last second. Instead of using all images to create the sequence, only the ones from time steps \{$t-$\SI{0.00}{\second}, $t-$\SI{0.02}{\second}, $t-$\SI{0.04}{\second}, $t-$\SI{0.06}{\second}, $t-$\SI{0.08}{\second}, $t-$\SI{0.20}{\second}, $t-$\SI{0.40}{\second}, $t-$\SI{0.60}{\second}, $t-$\SI{0.80}{\second}, $t-$\SI{1.00}{\second}\} are used. We use the five most recent images ($t-$\SI{0.00}{\second} to $t-$\SI{0.08}{\second}). Since older images become less relevant for the current movement, we chose larger gaps between the oldest five images ($t-$\SI{0.20}{\second} to $t-$\SI{1.00}{\second}) to reduce our input data.
The length of the input horizon of \SI{1}{\second} is chosen based on previous investigations. We found that while larger input horizons lead to smaller forecast errors, the differences are not significant. An input horizon of \SI{1}{\second} has also proven to work well for pedestrian action recognition \cite{goldhammer2016phd}. The extracted images are resized to $192\times192$ px and stacked to a three-dimensional array resulting in an image sequence $I_{1/2}(u, v, t)$ with dimensions $192\times192\times10$ px, the third dimension being the temporal dimension. This size is chosen because it corresponds to approximately the size of an ROI in the region where most cyclists occur.
The OF sequence is created from the image sequence using the pre-trained OF network from \cite{Sun}. For each of the nine consecutive image pairs in $I_{1/2}(u, v, t)$, an OF image with two channels (OF in $u$ and $v$ direction) is created and stacked along the channel axis resulting in an optical flow sequence $O_{1/2}(u, v, t)$ with dimension $192\times192\times18$ px.
The trajectories are created by tracking the head position of the cyclist in each camera image. To ensure that the correct cyclist in every scene is tracked, this process is performed semi-automatically and corrected manually if needed. Occlusions of a cyclist in camera images are handled by tracking the position using a Kalman Filter with a constant velocity model. The 3D head position in world coordinates $^w\vec{p}_t=[^wx_{t}, ^wy_{t}, ^wz_{t}]$ for every time step $t$ is calculated using triangulation. As network input, we use the trajectory of the past second in ego coordinates, which we achieve by transformation in a way that the origin of the coordinate system is the head position of the cyclist at the current time and the $y$ axis parallel to the cyclist's longitudinal movement direction. The OF sequences and trajectories are made publicly available \cite{zernetsch_cyclistactionrec}. Due to German privacy laws, we are not permitted to publish image sequences.
%We convert the positions of the past $t-1s$ time steps by:
%\begin{equation}
%^e\vec{y}_{t-i} = \begin{bmatrix}
%\cos(\varphi_t) & \sin(\varphi_t) & 0\\
%-\sin(\varphi_t) & \cos(\varphi_t) & 0\\
%0 & 0 & 1
%\end{bmatrix}\cdot (^w\vec{y}_{t-i}-^w\vec{y}_{t}),
%\label{eq:ego_transformation}
%\end{equation} with $\varphi_t$ being the movement direction of the cyclist at time step $t$ and $i\in\{1s, 0.98s, ..., 0.02s, 0.0s\}$.
\begin{figure}
\begin{center}
\vskip 0mm
\includegraphics[width = 0.99\columnwidth]{images/intersection/sequence_creation_hor.pdf}
\vskip 0mm
\caption{Exemplary image and OF sequence of cyclist detected on image of camera 1, with generated ROI marked in blue.}
\label{fig:ms_creation}
\end{center}
\vskip 0mm
\end{figure}
\section{\large Method}
\label{subsec:basic_movement_detection}
This section describes our overall method. Sec. \ref{subsec:baseline} shows our baseline method, which uses GRU to forecast trajectories based on solely past positions. In Sec. \ref{subsec:trajectory_forecast}, we describe how we incorporate video information into the forecast. The evaluation method is described in Sec. \ref{subsec:evaluation_methods}.
%In this section, we describe the overall basic movement detection process. In Sec. \ref{subsec:basic_movements}, we specify the term basic movements, Sec. \ref{subsec:network_arch_baseline} and Sec. \ref{subsec:network_arch_3d} describe the architectures of the baseline network and the MS based network (MS-Net) used to create the results. In Sec. \ref{subsec:evaluation_methods}, the methods to evaluate the results and compare the MS-Net to the baseline network are explained.
\subsection{Baseline Method}
\label{subsec:baseline}
We use a GRU, which uses solely positional information as input data, as a baseline method to compare against our proposed algorithm. In preliminary tests, we compared the use of MLP, LSTM, and GRU against each other, where the smallest positional errors were achieved by GRU. Fig. \ref{fig:gru_arch} depicts the architecture of our baseline model. The positions of the last second are fed to GRU cells. The number of hidden GRU cell layers and hidden output states are part of a hyperparameter optimization. The GRU cell's output in the final layer is passed to an MLP architecture with linear output. The number of hidden layers and neurons of the MLP are also part of the hyperparameter optimization. As loss function, we use
\begin{equation}
\begin{split}
L_{t}(^e\mathcal{T}_{gt;t}, &^e\hat{\mathcal{T}}_{t})=\\
\frac{1}{|\mathcal{H}|}\sum_{h\in\mathcal{H}}&\frac{(^ex_{gt;t+h}-^e\hat{x}_{t+h})^2 + (^ey_{gt;t+h}-^e\hat{y}_{t+h})^2}{h},
\end{split}
\label{eq:loss_traj}
\end{equation}
with the ground truth and forecasted trajectory $^e\mathcal{T}_{gt;t}$ and $^e\hat{\mathcal{T}}_{t}$ in ego coordinates and the set of all forecasted time horizons $\mathcal{H}=\{\SI{0.02}{\second}, \SI{0.04}{\second}, ..., \SI{2.5}{\second}\}$. Through division by $h$, we achieve a normalization of the quadratic forecast error to its individual forecast horizon. The model is implemented in TensorFlow \cite{tensorflow2015} and trained using the adaptive moment estimation (Adam) optimizer.
\begin{figure}
\begin{center}
\includegraphics[width = 0.8\columnwidth]{images/trajectory/baseline_gru.pdf}
\caption{Architecture of the GRU baseline network.}
\label{fig:gru_arch}
\end{center}
\vskip 0mm
\end{figure}
\subsection{Video-Based Trajectory Forecast}
\label{subsec:trajectory_forecast}
To incorporate video information into our trajectory forecast process, we implement the I3D architecture from \cite{Carreira} to extract features from our image and OF sequences. We extended the architecture to the wide-angle stereo camera system described in Sec.~\ref{subsec:dataset}. The extracted video features are fused with features extracted from the input trajectory using a feature union. Our architecture, which we call MS-Net (motion sequence), is depicted in Fig.~\ref{fig:bmd_arch}, with one Two-Stream 3D-ConvNet per camera (Fig.~\ref{fig:bmd_arch}a-d) using the respective image sequence $I_{1/2}(u, v, t)$ and OF sequence $O_{1/2}(u, v, t)$ as input. We pass the trajectory of the past second $^e\mathcal{T}_{t}$ to an MLP architecture (Fig.~\ref{fig:bmd_arch}e). The outputs of the Two-Stream 3D-ConvNets and the MLP are concatenated and passed to an MLP architecture with linear output, which outputs the trajectory forecast (Fig.~\ref{fig:bmd_arch}f).
\begin{figure}
\begin{center}
\includegraphics[width = 0.9\columnwidth]{images/motion_sequence/pipeline_ms.pdf}
\vskip 0mm
\caption{Motion sequence network architecture.}
\label{fig:bmd_arch}
\end{center}
\vskip 0mm
\end{figure}
\begin{figure}
\begin{center}
\vskip 0mm
\includegraphics[width = 0.6\columnwidth]{images/motion_sequence/conv3d_module.pdf}
\vskip 0mm
\rule{0.51\columnwidth}{1pt}
\vskip 1mm
\includegraphics[width = 0.6\columnwidth]{images/motion_sequence/inception_module.pdf}
\vskip -1mm
\caption{3D-ConvNet architecture (top) and detailed inception module (Inc., bottom) with 3D convolution and max-pool layers with dimension width$\times$height$\times$temporal length.}
\label{fig:i3d}
\end{center}
\vskip 0mm
\end{figure}
The architecture of the 3D-ConvNets is depicted in Fig.~\ref{fig:i3d}. The top figure shows the overall architecture, and the bottom figure shows a detailed description of the I3D inception modules. Compared to the original implementation \cite{Carreira}, we used similar strides in pooling and convolution layers. However, the authors used 64 RGB and 64 OF images with a resolution of 224$\times$224 px in an input sequence, covering a temporal window of \SI{2.56}{\second}. In contrast, we only use ten grayscale frames in the input image sequence and nine two-channel OF frames with a resolution of 192$\times$192 px, covering a temporal window of \SI{1}{\second}.
To train the network, we use the same loss as for the baseline network (Eq.~\ref{eq:loss_traj}) in combination with the Adam optimizer. The weights of the 3D-ConvNets are shared between the networks of the image sequences (Fig.~\ref{fig:bmd_arch}a/b) and OF sequences (Fig.~\ref{fig:bmd_arch}c/d) of cameras 1 and 2, since these networks have similar inputs and need to extract similar features. The network was implemented using TensorFlow~\cite{tensorflow2015}.
\subsection{Evaluation Methods}
\label{subsec:evaluation_methods}
We evaluate the benefit of incorporating video information into the forecast by comparing the baseline method's positional error with MS-Net's positional error. We first calculate the average Euclidean errors (AEE) for every forecast horizon $h\in\mathcal{H}$ by
\begin{equation}
\begin{split}
AEE_{h}=\frac{1}{N_s}\sum_{i=1}^{N_s}||^e\vec{p}_{gt;h;i}-^e\hat{p}_{h;i}||_2
\end{split}
\label{eq:aee}
\end{equation} with the number of samples $N_s$ and $^e\vec{p}_{gt;h;i}$ and $^e\hat{p}_{h;i}$ as ground truth and forecasted positions of the $i$-th sample at forecast horizon $h$. Using Eq. \ref{eq:aee}, we get the average distance between forecasted and ground truth position for every forecast horizon, which we combine into the average specific AEE (ASAEE) from \cite{goldhammer2016phd}. Eq. \ref{eq:asaee} shows how the ASAEE is calculated.
\begin{equation}
ASAEE=\frac{1}{|\mathcal{H}|}\sum_{h\in\mathcal{H}}\frac{AEE_h}{h},
\label{eq:asaee}
\end{equation}
We receive a single score consisting of the average AEE normalized according to their respective forecast horizon, which we use to compare our methods' overall performances.
In addition to using all video input sources combined, we investigate the use of image and optical flow sequences separately. We call the models MS$_{I}$, MS$_{OF}$, and MS$_{I;OF}$, where the index describes which inputs ($I$ image sequences, $OF$ optical flow sequences) are used in addition to the trajectory. Models where only image or OF sequences from camera 1 or 2 are used are referred to as MS$_{I;1}$, MS$_{I;2}$, MS$_{OF;1}$, and MS$_{OF;2}$. For the best models, we also create the results for different movement types, to evaluate if there are certain scenarios where incorporation of video information is especially useful. We identify the motion states \textit{wait}, \textit{start}, \textit{stop}, \textit{move straight}, \textit{turn left}, and \textit{turn right}. Due to traffic lights, about half of our samples are \textit{wait} samples. Since \textit{wait} forecasts show much lower forecast errors than the other motion states, we also create the results for all samples except \textit{wait} (denoted by $\overline{\text{wait}}$).
To evaluate whether any differences between the methods' results are significant or merely exist due to chance, we perform a statistical test. Therefore, we divide our dataset into sub-datasets, which we achieve by grouping all samples of one VRU to a subset. We create the ASAEE for every subset and every model and determine ranks for every subset. The ranks are used to perform the Friedman-test to determine whether significant differences in the ranks exist~\cite{friedman}. If the differences prove significant, we perform the Nemenyi-test to determine which of the models show significant differences~\cite{nemenyi}.
\section{Experimental results}
\label{sec_ResultsOutline}
\subsection{Training the networks}
\begin{figure*}
\begin{center}
\includegraphics[width = 1.85\columnwidth]{images/eval/aee.pdf}
\caption{Comparison of AEE over forecast horizon of GRU (green), MS$_{I}$ (red), MS$_{OF}$ (orange), and MS$_{I;OF}$ (blue) for different motion states.}
\label{fig:aee}
\end{center}
\vskip 0mm
\end{figure*}
\begin{figure}
\begin{center}
\includegraphics[width = 0.99\columnwidth]{images/eval/all_nemenyi_095.pdf}
\vskip 0mm
\caption{Performance comparison of different models by mean ranks. Groups of models that show no significant differences are connected ($p=0.05$).}
\label{fig:nem}
\end{center}
\end{figure}
\subsubsection*{Baseline}
In the first step, we perform a hyperparameter search for the GRU model. The network parameters we optimize are the number of hidden cells, the size of the hidden state of the GRU cells, and the number of hidden layers and neurons of the MLP. Additionally, we train the network using different batch sizes and learning rates. All possible combinations are trained and evaluated using the training and validation set. The model with the smallest validation error is used to create the test results. Table~\ref{tab:param_sweep_rnn} shows the parameters we investigated and the best configuration. The number of hidden states per layer are represented as vectors, where the elements represent the number of hidden states in consecutive layers.
\subsubsection*{MS-Net}
Since the MS-Net training takes considerably longer (about two weeks vs. about 2 hours for the GRU), we did not perform an extensive parameter sweep. By variation of the number of CNN filters in the I3D ConvNets, we found that we can achieve similar results with only 20\,\% of feature maps produced by the conv layers compared to the original implementation, which leads to shorter training and inference times. Aside from the number of filters, we use the network configuration from the original implementation. The MLP for feature extraction from the trajectory and the output MLP both consist of three hidden layers, with 100 neurons each. The networks are trained until the best validation score is reached and used to create the individual test results.
\subsection{Test Results}
\begin{table} [b]
\begin{center}
\caption{Evaluated combinations of different inputs, with image ($I$) and optical flow ($OF$) sequences of cameras $1$, $2$, or both, and the baseline model GRU with only trajectory ($T$) input.}
\label{tab:overall_asaee_results}
\begin{tabular}{|c | c | c | c | c | c | c |}
\hline
Modellname & $I1$ & $I2$ & $OF1$ & $OF2$ & $T$ & ASAEE/(m/s)\\ \hline
GRU & & & & & x & 0,211 \\ \hline
MS$_{I;1}$ & x & & & & x & 0,209 \\ \hline
MS$_{I;2}$ & & x & & & x & 0,212 \\ \hline
MS$_{OF;1}$ & & & x & & x & 0,198 \\ \hline
MS$_{OF;2}$ & & & & x & x & 0,197 \\ \hline
MS$_{I}$ & x & x & & & x & 0,196 \\ \hline
MS$_{OF}$ & & & x & x & x & 0,189 \\ \hline
MS$_{I;OF}$ & x & x & x & x & x & 0,194 \\ \hline
\end{tabular}
\end{center}
\end{table}
\begin{table}
\caption{GRU hyperparameter sweep. The combination with the best score is marked in blue.}
\vskip -8mm
\begin{center}
\begin{tabular}{|c | l|}
\hline
GRU layers& [50], {\color{blue}[50 50]}, [100], [100 100], \\ \hline
MLP layers& [10 10], {\color{blue}[100 100]}, [100 100 100], [1000 1000], \\ \hline
Batch size&1000, {\color{blue}5000}, 10000 \\ \hline
learning rate &$5\cdot10^{-4}$, {\color{blue}$10^{-4}$}, $10^{-5}$ \\ \hline
\end{tabular}
\end{center}
\label{tab:param_sweep_rnn}
\vskip 0mm
\end{table}
\begin{table} [b]
\begin{center}
\caption{Mean ASAEE values of VRU sets of all models for different motion types.}
\vskip 0mm
\scalebox{0.8}{
\begin{tabular}{| l | c | c | c | c | c | c | c | c |}
\hline
&All& $\overline{\text{wait}}$ & wait & straight & start & stop & left & right \\ \hline
GRU& 0.290 & 0.395 & 0.110 & 0.414 & 0.383 & 0.292 & 0.456 & 0.478 \\ \hline
MS$_{I}$& 0.277 & 0.377 & 0.117 & 0.378 & 0.372 & 0.344 & 0.423 & 0.379 \\ \hline
MS$_{OF}$& 0.252 & 0.342 & 0.109 & 0.359 & 0.333 & 0.277 & 0.357 & 0.400 \\ \hline
MS$_{I;OF}$& 0.268 & 0.357 & 0.121 & 0.360 & 0.347 & 0.310 & 0.378 & 0.408 \\ \hline
\end{tabular}}
\label{tab:asaee_results}
\end{center}
\vskip 0mm
\end{table}
We first compare the overall ASAEE of every model in Table~\ref{tab:overall_asaee_results}. We can see, that the addition of image sequences of only one camera (MS$_{I;1}$ and MS$_{I;2}$) leads to similar errors compared to the baseline model. By combining the two camera inputs (MS$_{I}$), we are able to improve the forecast error by 7\,\%. By using a single optical flow sequences as input (MS$_{OF;1}$ and MS$_{OF;2}$), we already achieve overall ASAEE which are 6\,\% lower than the baseline model's. By combining the two inputs (MS$_{OF}$), we achieve a 10\,\% smaller ASAEE than the baseline model. In both cases, the combination of both camera sources lead to an improved result. We see two main reasons for this. First, the use of both cameras helps to resolve occlusions that can occur within one camera. Second, depending on the current motion state, a certain view of the cyclist can help to gain better understanding of the cyclist's intention. E.\,g., the starting intention of a waiting cyclist can be forecasted more reliable using an image sequence that show the cyclist from the side, while the view from the front or the back is better used for the forecast of turning motions. The combination of image and OF sequences produces only slightly better results compared to image sequences only and worse results compared to OF sequences only.
\begin{figure}
\begin{center}
\includegraphics[width = 0.99\columnwidth]{images/eval/examples/example.pdf}
\caption{Examples of \textit{turn left}, \textit{move straight}, and \textit{turn right} forecasts, with input trajectories (gray), ground truth trajectories (black), and trajectory forecasts by MS$_{OF}$ (blue) and GRU (orange).}
\label{fig:example_forecast}
\end{center}
\vskip 0mm
\end{figure}
For each best model regarding their input source (MS$_I$, MS$_{OF}$, MS$_{I;OF}$, and GRU), we create the results for all VRU subsets to perform statistical tests. We also evaluate the models regarding specific motion types as described in Sec.~\ref{subsec:evaluation_methods}. Fig~\ref{fig:aee} and Table~\ref{tab:asaee_results} show the overal AEE over the forecasted horizons and the mean ASAEE values of the subset split by VRU. Fig.~\ref{fig:nem} shows the results of the Nemenyi-Test for all motion types based on the VRU specific ASAEE, where models connected by a black line do not show significant differences. The model with OF sequences as additional information generates the lowest mean ASAEE, followed by the model with the combination of image and OF sequences. The model with image sequences produces lower ASAEE compared to the baseline model. However, the test shows no significant difference between MS$_{I}$ and GRU. MS$_{OF}$ shows significantly lower ASAEE compared to MS$_{I;OF}$. As for the results of different movements, MS$_{I;OF}$ only performs significantly worse than MS$_{OF}$ for waiting samples. For the rest of the movement types, there is no significant difference. Aside from \textit{turn right} and \textit{move straight}, there is no significant difference between the performance of MS$_{I}$ and GRU. One reason for the OF sequences to lead to better performance than image sequences might be because they already encode the magnitude and direction of the cyclists' motions, which is arguably an important feature to forecast movements. If we look at the motion state \textit{wait}, where almost no cyclist movements occur, we see no significant improvement by MS$_{OF}$ compared to GRU. Another reason might be that the image sequences contain a lot of information that is not considered useful for the task, like background objects.
When we look at the different motion types, we see that the most considerable improvements are achieved for the motion states \textit{turn left}, \textit{turn right}, and \textit{move straight}. By looking at the samples with the most considerable improvements compared to the baseline models, we find that, in the case of turns, the baseline model often forecasts a straight movement at the beginning of a turn, while the MS-Net forecasts a turn much closer to the real trajectory. In the case of \textit{move straight}, the baseline model often forecasts a turn due to lateral oscillation in the past cyclist trajectory. If we look at the video sequences of these cases, we can see that the cyclists start turning their head early into the direction they will turn to before noticeable bends in the cyclists' past trajectories occur. In some cases, the cyclists stop pedaling before turning. Fig.~\ref{fig:example_forecast} shows three examples of forecasts by MS$_{OF}$ and GRU compared to the ground truth. Another motion type where we see a large improvement by adding video information is \textit{start}. While the baseline model often over or underestimates the distance the cyclist will travel, MS-Net achieve much more accurate starting forecasts.
To assess whether we can use our developed methods in a real-time system, we measure the different methods' runtimes. The runtime of the baseline method is below 1 ms. The runtimes of MS$_{I}$, MS$_{OF}$, and MS$_{I;OF}$ are measured at 10~ms, 13~ms, and 20~ms, using an NVIDIA Titan V GPU. With further optimization and current hardware developments, we are confident that our method can be implemented into a real-time system.
\section{\large Conclusions and Future Work}
\label{sec_conclusion}
In this article, we presented an approach to incorporate video information of a wide-angle stereo camera system at an urban intersection into the forecast of cyclist trajectories. By combining the sequences from the two cameras, we were able to resolve occlusions and extract more appropriate features for the forecast of certain movement types by utilizing different view angles. Our evaluations showed that especially the use of OF sequences leads to a significant improvement regarding forecast errors. The overall forecast error was reduced by 7.5\,\%. The most considerable improvements were achieved for the motion states \textit{turn left}, \textit{turn right}, and \textit{move straight}, with the largest gain of 22\,\% for \textit{turn left}. We attribute the improvements to the MS-Net being able to identify early movement indicators like turning of the head towards the movement direction. Our runtime evaluations show that our method can be implemented into a real-time system. Compared to the OF based approach, the model using image sequences generated no significant improvements in most cases. We attribute this to the large amount of information within the image sequences. Therefore, in our future work, we plan to incorporate attention mechanisms into our approach to reduce the information to relevant parts of the sequence. Since the goal of this article was to demonstrate the gain of incorporating multi-view image information into the forecast process, we chose rather simplistic forecast methods. In our future work, we plan to combine our approach with more sophisticated methods that incorporate social and physical constraints~\cite{Sadeghian2019}. Furthermore, we will compare our method to models based on body joints, especially whether our approach can reduce the needed input length~\cite{Kress2020}. We also plan to extend our approach from a deterministic forecast to a probabilistic forecast, to evaluate whether our approach can create more reliable forecasts than an approach solely based on trajectories~\cite{ownZernetsch_iv2019}.
\section{\large Acknowledgment}
This work results from the project DeCoInt$^2$, supported by the German Research Foundation (DFG) within the priority program SPP 1835: ``Kooperativ interagierende Automobile'', grant numbers DO 1186/1-2 and SI 674/11-2. Additionally, the work is supported by ``Zentrum Digitalisierung Bayern''.
% Generated by IEEEtran.bst, version: 1.12 (2007/01/11)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{goldhammer2016phd}
M.~Goldhammer, ``{Selbstlernende {A}lgorithmen zur videobasierten
{A}bsichtserkennung von {F}u{\ss}g\"angern},'' Ph.D. dissertation, University
of Kassel, 2016.
\bibitem{ownZernetsch_iv2016}
S.~{Zernetsch}, S.~{Kohnen}, M.~{Goldhammer}, K.~{Doll}, and B.~{Sick},
``Trajectory prediction of cyclists using a physical model and an artificial
neural network,'' in \emph{2016 IEEE Intelligent Vehicles Symposium (IV)},
Gothenburg, Sweden, June 2016, pp. 833--838.
\bibitem{Alahi_2016_CVPR}
A.~Alahi, K.~Goel, V.~Ramanathan, A.~Robicquet, L.~Fei-Fei, and S.~Savarese,
``{Social LSTM: Human Trajectory Prediction in Crowded Spaces},'' in
\emph{The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
Las vegas, NV, 2016.
\bibitem{Pool2017Context}
E.~A.~I. {Pool}, J.~F.~P. {Kooij}, and D.~M. {Gavrila}, ``Using road topology
to improve cyclist path prediction,'' in \emph{2017 IEEE Intelligent Vehicles
Symposium (IV)}, 2017, pp. 289--296.
\bibitem{quintero2018}
R.~{Quintero Minguez}, I.~{Parra Alonso}, D.~{Fernandez-Llorca}, and M.~A.
{Sotelo}, ``Pedestrian path, pose, and intention prediction through gaussian
process dynamical models and pedestrian activity recognition,'' \emph{IEEE
Transactions on Intelligent Transportation Systems}, vol.~20, no.~5, pp.
1803--1814, 2019.
\bibitem{Kress2020}
V.~{Kress}, S.~{Schreck}, S.~{Zernetsch}, K.~{Doll}, and B.~{Sick}, ``Pose
based action recognition of vulnerable road users using recurrent neural
networks,'' in \emph{2020 IEEE Symposium Series on Computational Intelligence
(SSCI)}, 2020, pp. 2723--2730.
\bibitem{willpedcross}
C.~G. {Keller} and D.~M. {Gavrila}, ``{Will the Pedestrian Cross? A Study on
Pedestrian Path Prediction},'' \emph{IEEE Transactions on Intelligent
Transportation Systems}, vol.~15, no.~2, pp. 494--506, April 2014.
\bibitem{Sadeghian2019}
A.~{Sadeghian}, V.~{Kosaraju}, A.~{Sadeghian}, N.~{Hirose}, H.~{Rezatofighi},
and S.~{Savarese}, ``Sophie: An attentive gan for predicting paths compliant
to social and physical constraints,'' in \emph{2019 IEEE/CVF Conference on
Computer Vision and Pattern Recognition (CVPR)}, 2019, pp. 1349--1358.
\bibitem{Carreira}
J.~{Carreira} and A.~{Zisserman}, ``{Quo Vadis, Action Recognition? A New Model
and the Kinetics Dataset},'' in \emph{2017 IEEE Conference on Computer Vision
and Pattern Recognition (CVPR)}, July 2017, pp. 4724--4733.
\bibitem{Kay2017TheKH}
W.~Kay, J.~Carreira, K.~Simonyan, B.~Zhang, C.~Hillier, S.~Vijayanarasimhan,
F.~Viola, T.~Green, T.~Back, A.~Natsev, M.~Suleyman, and A.~Zisserman, ``{The
Kinetics Human Action Video Dataset},'' \emph{ArXiv}, vol. abs/1705.06950,
2017.
\bibitem{Khurram}
\BIBentryALTinterwordspacing
K.~Soomro, A.~R. Zamir, and M.~Shah, ``{UCF101: A Dataset of 101 Human Actions
Classes From Videos in The Wild},'' \emph{CoRR}, vol. abs/1212.0402, 2012.
[Online]. Available:
\url{http://dblp.uni-trier.de/db/journals/corr/corr1212.html#abs-1212-0402}
\BIBentrySTDinterwordspacing
\bibitem{ownZernetsch_icpr2020}
S.~{Zernetsch}, S.~{Schreck}, V.~{Kress}, K.~{Doll}, and B.~{Sick}, ``Image
sequence based cyclist action recognition using multi-stream 3d
convolution,'' in \emph{2020 International Conference on Pattern Recognition
(ICPR) (accepted)}, Milan, Italy, 2020.
\bibitem{zernetsch_cyclistactionrec}
\BIBentryALTinterwordspacing
S.~Zernetsch, V.~Kress, M.~Bieshaar, G.~Reitberger, E.~Fuchs, K.~Doll, and
B.~Sick, ``{Cyclist Actions: Optical Flow Sequences and Trajectories},'' Mar.
2020. [Online]. Available: \url{https://doi.org/10.5281/zenodo.3734038}
\BIBentrySTDinterwordspacing
\bibitem{zernetsch_github_iv2021}
S.~Zernetsch, O.~Trupp, T.~Dwertmann, H.~Reichert, V.~Kress, and J.~Jung,
\url{https://github.com/CooperativeAutomatedTrafficSystemsLab/VideoBasedCyclistTrajectoryForecast},
2020.
\bibitem{Goldhammer}
M.~{Goldhammer}, E.~{Strigel}, D.~{Meissner}, U.~{Brunsmann}, K.~{Doll}, and
K.~{Dietmayer}, ``{Cooperative multi sensor network for traffic safety
applications at intersections},'' in \emph{2012 15th International IEEE
Conference on Intelligent Transportation Systems}, Anchorage, AK, Sep. 2012,
pp. 1178--1183.
\bibitem{Huang}
\BIBentryALTinterwordspacing
J.~Huang \emph{et~al.}, ``{Speed/accuracy trade-offs for modern convolutional
object detectors},'' \emph{CoRR}, vol. abs/1611.10012, 2016. [Online].
Available: \url{http://arxiv.org/abs/1611.10012}
\BIBentrySTDinterwordspacing
\bibitem{Lin2014}
T.-Y. Lin, M.~Maire, S.~Belongie, J.~Hays, P.~Perona, D.~Ramanan,
P.~Doll{\'a}r, and C.~L. Zitnick, \emph{{"Microsoft COCO: Common Objects in
Context}}.\hskip 1em plus 0.5em minus 0.4em\relax Cham: Springer
International Publishing, 2014, pp. 740--755.
\bibitem{Sun}
D.~{Sun}, X.~{Yang}, M.~{Liu}, and J.~{Kautz}, ``{PWC-Net: CNNs for Optical
Flow Using Pyramid, Warping, and Cost Volume},'' in \emph{2018 IEEE/CVF
Conference on Computer Vision and Pattern Recognition (CVPR)}, Salt Lake
City, UT, June 2018, pp. 8934--8943.
\bibitem{tensorflow2015}
\BIBentryALTinterwordspacing
M.~Abadi \emph{et~al.}, ``{{TensorFlow}: Large-Scale Machine Learning on
Heterogeneous Systems},'' 2015, software available from tensorflow.org.
[Online]. Available: \url{https://www.tensorflow.org/}
\BIBentrySTDinterwordspacing
\bibitem{friedman}
M.~{Hollander}, D.~A. {Wolfe}, and E.~Chicken, \emph{Nonparametric Statistical
Methods. Wiley Series in Probability and Statistics.}\hskip 1em plus 0.5em
minus 0.4em\relax Wiley, 2013.
\bibitem{nemenyi}
J.~Dem\v{s}ar, ``Statistical comparisons of classifiers over multiple data
sets,'' \emph{J. Mach. Learn. Res.}, vol.~7, pp. 1--30, Dec. 2006.
\bibitem{ownZernetsch_iv2019}
S.~{Zernetsch}, H.~{Reichert}, V.~{Kress}, K.~{Doll}, and B.~{Sick},
``Trajectory forecasts with uncertainties of vulnerable road users by means
of neural networks,'' in \emph{2019 IEEE Intelligent Vehicles Symposium
(IV)}, Paris, France, 2019, pp. 810--815.
\end{thebibliography}