On the Use of Abundant Road Speed Data for Travel Demand Calibration of Urban Traffic Simulators
Extracted source text · arXiv 2412.14089
%%
%% This is file `sample-sigconf.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigconf')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% Commands for TeXCount
%TC:macro \cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf]{acmart}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{dblfloatfix}
\setlength{\belowcaptionskip}{-12pt}
\usepackage[utf8]{inputenc}
%% NOTE that a single column version is required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmcopyright}
\copyrightyear{2023}
\acmYear{2023}
\acmDOI{XXXXXXX.XXXXXXX}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[SIGSPATIAL '23]{the 31st ACM SIGSPATIAL
International Conference on Advances
in Geographic Information Systems}{November 13--16,
2023}{Hamburg, Germany}
%
% Uncomment \acmBooktitle if th title of the proceedings is different
% from ``Proceedings of ...''!
%
%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
% June 03--05, 2018, Woodstock, NY}
\acmPrice{}
\acmISBN{978-1-4503-XXXX-X/18/06}
\usepackage{psboxit}
\newcommand{\caro}[1]{%}
\begin{boxitpara}{}
\textbf{[Carolina]} #1
\end{boxitpara}
}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
\pagenumbering{arabic}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{On the Use of Abundant Road Speed Data for Travel Demand Calibration of Urban Traffic Simulators}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Suyash Vishnoi}
\authornote{Work done while at Google Research}
\email{scvishnoi@google.com}
\orcid{0000-0002-6592-818X}
\affiliation{%
\institution{Google Research/ UT Austin}
\streetaddress{}
\city{Mountain View}
\state{CA}
\country{USA/ Austin, TX, USA}
\postcode{}
}
\author{Akhil Shetty}
\authornotemark[1]
\email{shetty.akhil@berkeley.edu}
% \orcid{}
\affiliation{%
\institution{UC Berkeley}
\streetaddress{}
\city{Berkeley}
\state{CA}
\country{USA}
\postcode{}
}
\author{Iveel Tsogsuren}
\email{iveel@google.com}
% \orcid{}
\affiliation{%
\institution{Google Research}
\streetaddress{}
\city{Mountain View}
\state{CA}
\country{USA}
\postcode{}
}
\author{Neha Arora}
\email{nehaarora@google.com}
% \orcid{}
\affiliation{%
\institution{Google Research}
\streetaddress{}
\city{Mountain View}
\state{CA}
\country{USA}
\postcode{}
}
\author{Carolina Osorio}
\authornotemark[1]
\email{osorioc@google.com}
% \orcid{}
\affiliation{%
\institution{Google Research/ HEC Montreal}
\streetaddress{}
\city{Mountain View}
\state{CA}
\country{USA/Montreal, QC, Canada}
\postcode{}
}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Vishnoi, Shetty, Tsogsuren, Arora and Osorio}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
This work develops a compute-efficient algorithm to tackle a fundamental problem in transportation: that of urban travel demand estimation. It focuses on the calibration of origin-destination travel demand input parameters for high-resolution traffic simulation models. It considers the use of abundant traffic road speed data. The travel demand calibration problem is formulated as a continuous, high-dimensional, simulation-based optimization (SO) problem with bound constraints. There is a lack of compute efficient algorithms to tackle this problem. We propose the use of an SO algorithm that relies on an efficient, analytical, differentiable, physics-based traffic model, known as a metamodel or surrogate model. We formulate a metamodel that enables the use of road speed data. Tests are performed on a Salt Lake City network. We study how the amount of data, as well as the congestion levels, impact both in-sample and out-of-sample performance.
%We compare the approach to the traditional, commonly used, calibration based on road vehicular count (or volume) data.
The proposed method outperforms the benchmark for both in-sample and out-of-sample performance by $84.4\%$ and $72.2\%$ in terms of speeds and counts, respectively. Most importantly, the proposed method yields the highest compute efficiency, identifying solutions with good performance within few simulation function evaluations (i.e., with small samples).
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10010520.10010553.10010562</concept_id>
<concept_desc>Computer systems organization~Embedded systems</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010575.10010755</concept_id>
<concept_desc>Computer systems organization~Redundancy</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>10010520.10010553.10010554</concept_id>
<concept_desc>Computer systems organization~Robotics</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>10003033.10003083.10003095</concept_id>
<concept_desc>Networks~Network reliability</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
% TODO(scvishnoi): Update the CSS concepts
\ccsdesc[500]{Applied computing~Transportation}
\ccsdesc[300]{Computing Methodologies~Modeling and simulation}
% \ccsdesc{Computer systems organization~Robotics}
% \ccsdesc[100]{Networks~Network reliability}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{urban travel demand calibration, metamodel-based optimization, speeds-based calibration}
\received{}
\received[revised]{}
\received[accepted]{}
\maketitle
\section{Introduction}
Traffic microsimulation software is a widely used tool to plan changes in the city by simulating the impact of those changes before implementing them in the real-world. To assess the impact of large-scale changes it is important that the simulator be well calibrated to the traffic conditions of the actual network.
These settings include the Origin-Destination (OD) demands which determines the amount of traffic flows on the network. %\cite{paz2014calibration,osorio2019high, osorio2019efficient}.
Most often, the ground truth (GT) data available for OD calibration is spatially sparse vehicular count data on segments \cite{osorio2019high}. Work that includes GT
speed data include \cite{cipriani2011gradient,huang2010accelerated}.
%However, generally this data is sparse because it requires the installation of underground inductive loop detectors or cameras \cite{xing2023urban}. %On the other hand, vehicular probe data can be less spatially sparse meaning that it can monitor a larger set of segments \cite{amirjamshidi2019multi, jie2011calibration, morita2019microsimulation}.
In this paper, we focus on the use of non-spatially sparse segment (i.e., a road/link) speed data for travel demand calibration of traffic microsimulation models. An efficient method to solve the proposed optimization problem is presented and compared with an existing algorithm for speeds-based calibration from the literature. %This is followed by its comparison to the use of counts data for calibration and an analysis of the value derived from the available speed data.
Demand calibration problems of microsimulation models are simulation-based, stochastic, nonlinear and high dimensional problems. A sample efficient algorithm for OD demand calibration using vehicle count data on large-scale networks is presented in \cite{osorio2019high} which exploits a physics-based metamodel to perform efficient calibration. This approach uses a low-resolution analytical traffic model to convert the OD demands which are the features of the problem to segment counts and further uses a statistical model as simple as a linear regression model to learn the difference between the counts generated by the analytical traffic model and the simulation. This results in a loss function that is differentiable making it compatible with efficient gradient-based solvers. %algorithm using available solvers thus making it more efficient than a random search based algorithm.
%
%Besides, none of the aforementioned studies analyze the added benefit of the increasing amounts of speed data which is becoming available with the use of navigation applications and compare the performance against calibration using count data which is mostly sparse. Considering these gaps,
The present work extends the approach from \cite{osorio2019high} to speeds-based OD demand calibration for urban road networks, while making the following contributions.
We propose a novel metamodel formulation that is scalable and computationally efficient. It is based on a novel relationship between segment demands and segment speeds \cite{lia2022fundamental}, known as the fundamental diagram (FD), in calibration which is appropriate for arterial/urban segments compared to existing studies \cite{osorio2019high}.
The proposed approach is tested under different levels of sparsity of segments with GT speed data to assess the value of the amount of speed data in calibration.
%By using abundant speed data, the calibration performance across the network improves by $42.0\%$ and $48.8\%$ in terms of speeds and counts, respectively as compared to the use of a smaller amount of speed data.
% The proposed approach is benchmarked against SPSA, it shows an improvement of $84.4\%$ and $72.2\%$ in terms of speeds and counts, respectively on in-sample segments as compared to the benchmark. %The proposed approach with abundant speed data also outperforms counts-based calibration with sparse data by $17.4\%$ and $33.9\%$ in terms of speeds and counts, respectively when evaluated over both in-sample and out-of-sample segments.
\section{Methodology}
\label{s:methodology}
%\subsection{Optimization Problem}
An urban area is discretized into traffic analysis zones (TAZs), which can serve as origins or destinations for trips. Pairs of TAZs are referred to as origin-destination (OD) pairs. In this work, we focus on the offline calibration of demands for a subset of all OD pairs such that the simulated traffic metrics are close to the ground truth (GT) metrics. The problem considered in this work can be defined as follows:
\textbf{Problem 1.} \textit{Given the GT speeds for a set of segments on the network, what are the demands that can be loaded onto the OD pairs in a microsimulation model?}
The above problem can be written as an SO problem with the OD pair demands as features and the error between the simulated speeds and the GT speeds as the loss function. A detailed description of the optimization problem is given below.
To formulate the calibration problem, we introduce the following notation:
\begin{tabular}{l}
$x$: feature vector (i.e., vector of hourly OD demands),\\
$\mathcal{I}$: set of segments with GT speeds,\\
$f(x)$: simulation-based loss function,\\
$u_1$: vector of endogenous simulation variables,\\
$u_2$: vector of exogenous simulation parameters,\\
$v_i^{\text{GT}}$: GT measurement of average (space mean) speed of segment $i$,\\
$E[v_i(x,u_1;u_2)]$: expected (space mean) speed of segment $i$,\\
$x_U$: vector of upper bounds on the hourly OD demands,\\
$v_i^{\text{max}}$: speed limit of segment $i$,\\
$w_i$: loss function weight corresponding to segment $i$.\\
\end{tabular}
The optimization problem is formulated as follows:
\begin{align}
\min_{x} \quad & f(x)=\frac{1}{|\mathcal{I}|}\sum_{i\in \mathcal{I}}w_i(v_i^{\text{GT}} - E[v_i(x,u_1;u_2)])^2\label{e:so_optimization_problem_loss} \\
\textrm{s.t.} \quad & 0 \le x \le x_U\label{e:so_optimization_problem_constraints}
\end{align}
Here, $|\mathcal{I}|$ denotes the cardinality of the set $\mathcal{I}$. The loss function of the problem is the minimization of the weighted mean squared error between the GT measurements of speeds and the corresponding expected speeds $E[v_i(x,u_1;u_2)]$ from the simulator. The corresponding segment weight $w_i$ in this work is defined as:
\begin{equation*}
w_i = \min\biggl\{\frac{v_i^{\text{GT}}}{v_i^{\text{max}}}, 1-\frac{v_i^{\text{GT}}}{v_i^{\text{max}}}\biggl\}.
\end{equation*}
This weight term represents the importance of a segment for calibration which in this case implies that we put a lower weight on segments for which the GT speeds are either very close to the speed limit or are very close to 0 m/s. This is because traffic in these regimes is known to have a many-to-one relationship between counts and speeds (and hence demands and speeds) \cite{lia2022fundamental} which makes them less valuable than other segments which have a unique relationship between demands and speeds and thus drive the solver to a unique solution. Besides the feature vector, the expected space-mean speed of segments depend upon the endogenous variables $u_1$ such as queue lengths and travel time, and exogenous parameters $u_2$ such as segment attributes, car-following parameters, network topology, and route-choice probabilities among other parameters. Unlike \cite{osorio2019high}, here we do not consider a regularization term in the formulation.
% \vspace{-3mm}
\subsection*{Physics-informed Metamodel Algorithm}
\label{s:metamodel_based_solution}
To solve the optimization problem presented in \eqref{e:so_optimization_problem_loss}-\eqref{e:so_optimization_problem_constraints}, we propose a physics-informed metamodel algorithm inspired by \cite{osorio2019high} and \cite{osorio2019efficient}. This approach approximates the simulation-based and non-differentiable optimization problem loss function \eqref{e:so_optimization_problem_loss} with an analytical and differentiable metamodel, or surrogate model. The chosen metamodel is a combination of an analytical component relating the features with the loss function through a known physics-based or traffic-based relationship, and a general-purpose, traffic agnostic, functional component which is a statistical model (in this case a linear model) that aims to learn the difference between the analytical physics-based model and the simulator through the feature vector. In simpler terms, the functional component is trained on the difference between the loss function calculated using the analytical physics-based model and that calculated using the simulator. The resulting problem is differentiable and can be solved with the help of efficient gradient-based algorithms. The metamodel algorithm is an iterative approach: at every epoch of the SO algorithm, new simulation function evaluations become available, the metamodel parameters are then trained or updated, so as to bring the metamodel closer to the actual simulation-based loss function. The newly trained metamodel is then used to solve an analytical and differentiable optimization problem, the solution of which is evaluated via simulation, leading to new simulation function evaluations.
\begin{table*}[!b]
\centering
\vspace*{3mm}
\caption{Average nRMSE over 5 runs of speeds-based calibration using the metamodel-based approach and SPSA along with that obtained using the initial demands (marked with `-' in the algorithm column).}
\begin{tabular}{p{0.1\textwidth}p{0.12\textwidth}ccccc}
\toprule
\#in-sample segments & \#out-of-sample segments & Algorithm & \multicolumn{2}{c}{In-sample nRMSE, evaluated over:} & \multicolumn{2}{c}{Out-of-sample nRMSE, evaluated over:}\\
& & & speeds & counts & speeds & counts\\
\midrule
331 & 1,951 & metamodel-based & 0.346 & 0.603 & 0.212& 0.884 \\
& & SPSA & 0.593 & 1.118 & 0.396 & 1.534 \\
& & - & 0.718 & 1.231 & 0.438 & 1.72\\
456 & 1,826 & metamodel-based & 0.171 & 0.372 & 0.048 & 0.488 \\
& & SPSA & 0.542 & 1.028 & 0.380 & 1.356\\
& & - & 0.628 & 1.304 & 0.438 & 1.735\\
2,282 & 0 & metamodel-based & 0.049 & 0.387 & - & - \\
& & SPSA & 0.315 & 1.394 & - & -\\
& & - & 0.461 & 1.624 & - & -\\
\bottomrule
\end{tabular}
\label{t:nrmse_insample_speeds}
\end{table*}
The optimization problem solved at every epoch is known as the metamodel optimization problem. To formulate it, we define the following notation.
\begin{tabular}{l}
$k$: algorithm epoch,\\
$m_k$: metamodel function at epoch $k$,\\
$\beta_k$: parameter vector of metamodel $m_k$,\\
$\beta_{k,j}$: scalar element $j$ of the parameter vector,\\
$f_A(x)$: physics-based analytical approximation of the \\optimization problem loss function \eqref{e:so_optimization_problem_loss} provided by an \\analytical traffic model,\\
$\phi(x;\beta_k)$: functional component of the metamodel $m_k$,\\
$x_z$: expected hourly demand for OD pair $z$,\\
$\mathcal{Z}$: set of indices of OD pairs $\mathcal{Z}=\{1,2,\dots,z_0\}$,\\
\textbf{Endogenous variables of the analytical traffic model}\\
$v_i^{\text{a}}$: physics-based analytical speed on segment $i$,\\
$q_i$: expected hourly demand for segment $i$,\\
$q$: vector of expected hourly demands,\\
\textbf{Exogenous parameters of the analytical traffic model}\\
$A$: matrix to map OD demands to segment demands,\\
$\alpha_{i}^1, \alpha_{i}^2$: FD parameters for segment $i$,\\
$q^{\text{max}}_i$: maximum demand for segment $i$,\\
$v^{\text{min}}_i$: minimum speed on segment $i$.\\
\end{tabular}
The metamodel optimization problem at any epoch $k$ is written as follows:
\begin{align}
\min_{x} \quad & m_k(x;\beta_k)=\beta_{k,0}f_A(x) + \phi(x;
\beta_k)\label{e:metamodel}\\
\textrm{s.t.} \quad & f_A(x) = \frac{1}{|\mathcal{I}|}\sum_{i\in\mathcal{I}}{w_i(v_i^{\text{GT}}-v_i^{\text{a}})^2}\label{e:analytical}\\
\quad & \phi(x;\beta_k)=\beta_{k,1}+\sum_{z\in\mathcal{Z}}{\beta_{k,z+1}x_z}\label{e:functional}\\
\quad & v_i^{\text{a}} = v^{\text{min}}_i + (v^{\text{max}}_i - v^{\text{min}}_i)\left(1 - \left(\frac{q_i}{q_i^{\text{max}}}\right)^{\alpha^1_i}\right)^{\alpha^2_i}\forall i \in \mathcal{I}\label{e:urbanfd}\\
\quad & q = Ax\label{e:assignment}\\
\quad & 0 \le x \le x_U
\end{align}
%Here the physics-based analytical component \eqref{e:analytical} of the metamodel is defined corresponding to the first expression in the loss function \eqref{e:metamodel}. It is written as the weighted mean squared error between the GT speeds and the
%expected speeds which in this case are obtained using an
%physics-based analytical speeds. The physics-based analytical model is defined using a recently found FD for urban segments \eqref{e:urbanfd} proposed in \cite{lia2022fundamental} which relates the segment speeds to segment demands and \eqref{e:assignment} which relates the segment demands to OD demands.
The physics-based analytical component \eqref{e:analytical} of the metamodel is defined as the weighted mean squared error between the ground truth (GT) speeds and the physics-based analytical speeds, corresponding to the first expression in the loss function \eqref{e:metamodel}. The physics-based analytical model is defined using a recently proposed FD for urban segments \eqref{e:urbanfd} in \cite{lia2022fundamental} , which relates the segment speeds to segment demands, and \eqref{e:assignment}, which relates the segment demands to OD demands. Therefore, the physics-based analytical component gives an approximate analytical relationship between the feature vector which consists of the OD demands and the loss function of the SO problem \eqref{e:so_optimization_problem_loss}. In this work, we assume that the route choices are exogenous and independent of the actual flow of traffic at any epoch. Therefore, $A$ which is also known as the assignment matrix is fixed. Out of the parameters of the FD model, $v^{\text{max}}_i$, $v^{\text{min}}_i$, and $q^{\text{max}}_i$ are considered fixed for each segment and set based on observed GT data. On the other hand $\alpha_{i}^1$ and $\alpha_{i}^2$ are either fitted based on simulated traffic data for the network or assigned according to observed values from the literature \cite{lia2022fundamental}. %Appendix \ref{a:cf_fitting} presents more details on the approach used to obtain these parameter values in the current study.
The functional component $\phi(x,\beta_k)$ is a linear model defined using the OD demands.
%Prior to any epoch $k$, the coefficients of the functional component of the metamodel as well as the coefficient of the analytical model $\beta_{k,0}$ are trained against the error between the first expression of the loss function \eqref{e:metamodel} obtained using speeds from the actual simulator and the value of the analytical component both obtained using various sets of OD demands.
More details on metamodel training can be found in \cite{osorio2019high}.
%The given optimization problem is solved at every epoch $k$ with a fixed metamodel $m_k(\cdot;\beta_k)$ to obtain a calibrated vector of OD demands which minimizes the metamodel-based loss function for that epoch. In general, these OD demands obtained at every epoch $k$ of the algorithm are simulated and used to train the metamodel coefficients for the next epoch and the optimization is solved again to optimality with a metamodel function with the new parameter values. Additional OD demands can also be simulated before starting the algorithm run or in between epochs to increase the rate of learning for the metamodel coefficients.
The bound constraints are the same as in the original problem \eqref{e:so_optimization_problem_constraints}.
%Note that the metamodel-based optimization formulation in \cite{osorio2019high} also uses a FD relating segment speeds to densities which considers the same set of FD parameters for all the segments with GT data. This was possible in that work since highway FD parameters are quite consistent across different highway segments, as they depend only on demand conditions and not on any supply conditions. However, the FDs for arterials are more susceptible to change with the change in supply conditions such as traffic light settings, among others. Choosing common FD parameters for all the segments regardless of supply conditions is not ideal when performing calibration on arterial segments which depends on the FD. Therefore, in this work, we fit segment specific FD parameters to simulated data before the calibration run
%See Appendix \ref{a:cf_fitting} for details on FD parameter fitting. % Here, we ensure that only plausible FD shapes are considered for calibration and for all segments where a plausible FD shape is not obtained using the simulated data due to limitations of the simulation setup, a common set of FD parameters is assigned to all such segments.
\section{Salt Lake City Case Study}
\label{s:numerical_study}
%In this section we present a detailed analysis of the performance of the metamodel-based optimization approach for calibration of OD demands using speeds data. The following section presents a description of the network and simulation used for this purpose.
%\subsection{Simulation Setup}
\label{s:network_description}
%The current study uses a Salt Lake City, Utah, network modeled with SUMO \cite{sumo} that has a total of 15,254 segments,
%with 2,282 major arterial segments on which traffic is observed in this study and with 2,818 intersections out of which 94 are signalized intersections. We consider 62 OD pairs in the network. Each of the 62 ODs has three possible routes resulting in a total of 186 routes across the network.
%We use a synthetic OD demand set as GT demands. Demands are generated for the 62 OD pairs similar to \cite{mladenov2022adversarial}. GT field data is obtained from 10 simulation runs using the GT OD demands. We assume that there is no available historical data about the GT demands and therefore we initiate the algorithm with a uniformly sampled random set of OD demands.
%The primary evaluation metric used to judge the quality of the calibrated demands is the normalized-\textit{root mean squared error} (nRMSE) \cite{arora2021efficient}.
This study uses a network model of Salt Lake City, Utah, created with SUMO \cite{sumo}. The network has 15,254 segments, including 2,282 major arterial segments on which traffic is observed in this study. It also has 2,818 intersections, of which 94 are signalized. We consider 62 origin-destination (OD) pairs in the network. Each of the 62 ODs has three possible routes, resulting in a total of 186 routes across the network. We use a synthetic OD demand set as ground truth (GT) demands. Demands are generated for the 62 OD pairs similar to \cite{mladenov2022adversarial}. GT field data is obtained from 10 simulation runs using the GT OD demands. We assume that no historical data is available about the GT demands. Therefore, we initiate the algorithm with a uniformly sampled random set of OD demands. The primary evaluation metric used to judge the quality of the calibrated demands is the normalized-\textit{root mean squared error} (nRMSE) \cite{arora2021efficient}.
%\subsection{Speeds-based Calibration}
We benchmark our speeds-based OD demand calibration approach against SPSA algorithm~\cite{spall2005introduction} on 3 sets of GT segments grouped by congestion level in a synthetic GT simulation:
\begin{itemize}
\item 331 segments with speed/speed-limit $\leq$ 0.8
\item 456 segments with speed/speed-limit $\leq$ 0.9
\item 2,282 segments with speed/speed-limit $\leq$ 1.0
\end{itemize}
For each set of GT segments, we run each algorithm 5 times. An algorithm run is terminated once a total of 250 ODs have been simulated. Final nRMSE estimated are obtained by averaging over 5 simulation replications.
%We perform speeds-based calibration of OD demands considering different numbers of segments with GT speed data. We benchmark the approach with the SPSA algorithm~\cite{spall2005introduction}.
%The implementation details of the calibration algorithms namely the metamodel solution algorithm using both speeds and counts and SPSA for speeds-based calibration are discussed below followed by the analysis and results.
%
%
%, in particular about the values assigned to the various algorithmic parameters and other choices made in the implementation of the algorithm
%
%\subsection{Results and Analysis}
%In this section, we discuss the results obtained from applying the aforementioned three algorithms on the OD demand calibration problem for the network and synthetic demand scenario described in Section \ref{s:network_description} .
%\subsubsection{Results}
%\label{s:speed_calibration_metamodel}
%The plots of the simulated speeds from the calibrated demands and the simulated speeds from the initial demand against the GT speeds for each of the segment sets used for calibration are presented in Figure \ref{f:insample_nrmse_speeds}. Each plotted point represents a segment from the segment-set, with the y-axis value denoting the average simulated speed over the 5 calibration runs, and the x-axis value denoting the GT speed. The error bars have a half-width of one standard deviation in the simulated speeds over the 5 runs.
Plots of simulated speeds from calibrated demand against GT speeds for each segment set used for calibration are shown in Figure \ref{f:insample_nrmse_speeds}. Each point represents a segment, with the y-axis denoting average simulated speed over 5 calibration runs and the x-axis denoting GT speed. Error bars have a half-width of one standard deviation in simulated speeds over the 5 runs. Ideally, after calibration, simulated speeds should equal GT speeds. The top histogram shows the number of segments around the GT speeds on the x-axis, and the right histogram shows the number of segments with simulated speeds around the values on the y-axis. In, Figure \ref{f:insample_nrmse_speeds}, we compare the plots of columns 1 (SPSA plots) and 2 (metamodel plots) and it shows that the proposed approach outperforms SPSA in terms of fit to GT data. This holds for all three sets of segments (i.e., all three rows of plots).
% \begin{figure*}
% \centering
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/initial_seed_331.png}
% % \caption{}
% \end{subfigure}
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/spsa_calib_331.png}
% % \caption{}
% \end{subfigure}
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/calib_331.png}
% % \caption{}
% \end{subfigure}
% \hfill
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/initial_seed_456.png}
% % \caption{}
% \end{subfigure}
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/spsa_calib_456.png}
% % \caption{}
% \end{subfigure}
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/calib_456.png}
% % \caption{}
% \end{subfigure}
% \hfill
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/initial_seed_2282.png}
% % \caption{}
% \end{subfigure}
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/spsa_calib_2282.png}
% % \caption{}
% \end{subfigure}
% \begin{subfigure}[b]{0.3\textwidth}
% \centering
% \includegraphics[width=\textwidth]{figures/calib_2282.png}
% % \caption{}
% \end{subfigure}
% \caption{Simulated versus GT speeds for [left column] initial demand and demand obtained from speeds-based calibration using [middle column] SPSA and [right column] metamodel algorithm. Plots in the top row are for calibration with 331 segments, similarly middle row for calibration with 456 segments, and bottom row for calibration with 2,282 segments. The $x=y$ line is presented in each plot for reference.}
% \label{f:insample_nrmse_speeds}
% \end{figure*}
Table \ref{t:nrmse_insample_speeds} shows the in-sample and out-of-sample average nRMSE in speeds and counts for each segment set used for calibration, as well as the average nRMSE for initial demands. The in-sample nRMSE in speeds is the primary measure of convergence, as it is directly related to the metric minimized in the optimization problem. The average out-of-sample nRMSE in speeds and in-sample and out-of-sample nRMSE in counts are secondary measures of calibration performance. Over the three sets of segments used for calibration, the proposed approach improves the in-sample nRMSE in speeds by $41.6\%$, $68.4\%$, and $84.4\%$ compared to SPSA, and the in-sample nRMSE in counts by $46.1\%$, $63.8\%$, and $72.2\%$, respectively.
% \begin{table*}[]
% \centering
% \begin{tabular}{p{0.1\textwidth}p{0.12\textwidth}ccccc}
% \toprule
% \#in-sample segments & \#out-of-sample segments & Algorithm & \multicolumn{2}{c}{In-sample nRMSE, evaluated over:} & \multicolumn{2}{c}{Out-of-sample nRMSE, evaluated over:}\\
% & & & speeds & counts & speeds & counts\\
% \midrule
% 331 & 1,951 & metamodel-based & 0.346 (0.028) & 0.603 (0.064) & 0.212 (0.020) & 0.884 (0.083) \\
% & & SPSA & 0.593 (0.011) & 1.118 (0.112) & 0.396 (0.008) & 1.534 (0.138)\\
% & & - & 0.718 & 1.231 & 0.438 & 1.72\\
% 456 & 1,826 & metamodel-based & 0.171 (0.002) & 0.372 (0.020) & 0.048 (0.009) & 0.488 (0.026)\\
% & & SPSA & 0.542 (0.014) & 1.028 (0.157) & 0.380 (0.038) & 1.356 (0.189)\\
% & & - & 0.628 & 1.304 & 0.438 & 1.735\\
% 2,282 & 0 & metamodel-based & 0.049 (0.004) & 0.387 (0.022) & - & - \\
% & & SPSA & 0.315 (0.028) & 1.394 (0.174) & - & -\\
% & & - & 0.461 & 1.624 & - & -\\
% \bottomrule
% \end{tabular}
% \caption{Average nRMSE over 5 runs of speeds-based calibration using the metamodel-based approach and SPSA along with standard deviation in nRMSE (in parenthesis). nRMSE obtained with initial demand is presented for reference and marked with `-' in the algorithm column.}
% \label{t:nrmse_insample_speeds}
% \end{table*}
\begin{figure}
\begin{subfigure}{0.45\columnwidth}
\includegraphics[width=\textwidth]{figures/spsa_calib_331.png}
\end{subfigure}
\hfill
\begin{subfigure}{0.45\columnwidth}
\includegraphics[width=\textwidth]{figures/calib_331.png}
\end{subfigure}
%
\begin{subfigure}{0.45\columnwidth}
\includegraphics[width=\textwidth]{figures/spsa_calib_456.png}
\end{subfigure}
\hfill
\begin{subfigure}{0.45\columnwidth}
\includegraphics[width=\textwidth]{figures/calib_456.png}
\end{subfigure}
%
\begin{subfigure}{0.45\columnwidth}
\includegraphics[width=\textwidth]{figures/spsa_calib_2282.png}
\end{subfigure}
\hfill
\begin{subfigure}{0.45\columnwidth}
\includegraphics[width=\textwidth]{figures/calib_2282.png}
\end{subfigure}
\caption{Simulated vs. GT speeds for SPSA (left) and metamodel (right) calibrated demands. Top, middle, and bottom rows show results for calibration with 331, 456, and 2,282 segments, respectively. The $x=y$ line is shown for reference.}
%\caption{Simulated versus GT speeds for [left column] demand obtained from speeds-based calibration using SPSA and [right column] metamodel algorithm. Plots in the top row are for calibration with 331 segments, similarly middle row for calibration with 456 segments, and bottom row for calibration with 2,282 segments. The $x=y$ line is presented in each plot for reference.}
\label{f:insample_nrmse_speeds}
\end{figure}
The performance of the proposed approach improves with the number of in-sample segments. The in-sample nRMSE in speeds decreases by $51.7\%$, $72.7\%$, and $89.2\%$, respectively, and the in-sample nRMSE in counts decreases by $51.0\%$, $71.5\%$, and $76.1\%$, respectively, compared to the speeds simulated using the initial demand. The out-of-sample nRMSE in speeds decreases by $51.6\%$ and $89.0\%$ over the 331 and 456 segment cases, and the out-of-sample nRMSE in counts decreases by $48.6\%$ and $71.8\%$, respectively. The percentage improvement over initial statistics increases with the number of segments used for calibration.
%Let us now consider how the performance of the proposed approach varies with the number of in-sample segments.
%It is observed that with increasing number of in-sample segments nRMSE in speeds decreases by $51.7\%$, $72.7\%$, and $89.2\%$ respectively as compared to the speeds simulated using the initial demand, while the in-sample nRMSE in counts reduce by $51.0\%$, $71.5\%$, and $76.1\%$ respectively. Similarly, the out-of-sample nRMSE in speeds decreases by $51.6\%$ and $89.0\%$ respectively over the 331 and 456 segment cases. The case with $2,282$ segments does not have any out-of-sample segments and hence no out-of-sample statistics. The out-of-sample nRMSE in counts decreases by $48.6\%$ and $71.8\%$ respectively. For the average in-sample and out-of-sample nRMSE in both speeds and counts, it is observed that the percentage improvement over initial statistics increases with an increase in the number of segments used for calibration.% While an improvement in the nRMSE in simulated traffic statistics with an increase in in-sample segments in general is expected as there is more data to fit the demands on, in case of the nRMSE in speeds, this improvement can also be attributed to an increase in the number of free-flowing segments in the sample set. Free-flow speeds usually occur on segments away from intersections due to lack of supply restrictions. On such segments it is easier to replicate free-flow speeds in simulation unless there is severe congestion in the network (as in the case of the initial demand which is seen to lead to a lot of low speeds indicating severe congestion across the network). On the other hand, higher improvement in the nRMSE in counts with an increase in in-sample segments more directly indicates that a larger number of segments with GT speed data can lead to improved calibration performance.
%Between speeds and counts, the absolute nRMSE is higher for counts which is expected since the calibration inherently only tries to minimize the error in speeds while the reduction in the error in counts is only a byproduct. Based on the observed FD shapes \cite{lia2022fundamental}, segment speeds and segment demands have a one-to-many mapping close to free-flowing and jammed states so several different counts (from different segment demands) which are not necessarily the GT counts can lead to speeds that are close to GT speeds.
Out-of-sample nRMSE in speeds is smaller than in-sample nRMSE, which makes sense since out-of-sample segments mostly contain free-flow speeds that are easier to calibrate. The nRMSE in counts for the 331-segment case is large compared to acceptable levels observed in past work \cite{zhang2017efficient}, but it reduces to closer to acceptable levels with more segments, especially in-sample. Note that the nRMSE values in Table \ref{t:nrmse_insample_speeds} cannot be directly compared between segment sets because they contain different in-sample segments.
%Between in-sample and out-of-sample nRMSE in speeds, the latter is observed to be smaller which makes sense since out-of-sample segments mostly contain segments with free-flow speeds, and these tend to be easier to calibrate. The nRMSE in counts for the case with 331 in-sample segments over both in-sample and out-of-sample segments is quite large compared to acceptable calibrated metrics observed in past work \cite{zhang2017efficient}. The nRMSE reduces to closer to acceptable levels with an increase in the number of segments in particular when measured over the in-sample segments.
% The standard deviation in all cases is quite small compared to the values of the average nRMSE indicating that the nRMSE values are consistent across calibration runs.
%Note that it is not possible to directly compare the nRMSE values presented in Table \ref{t:nrmse_insample_speeds} between the different sets of segments for calibration since they contain different in-sample segments. A fairer comparison can be made by evaluating the performance over the same set of segments in all three cases. This is carried out in Section \ref{s:category_comparison} along with a comparison with calibration using GT counts data.
% Table \ref{t:nrmse_insample_speeds} presents the values of the in-sample and out-of-sample average and standard deviation in nRMSE over the 5 runs in both speeds and counts with the calibrated demand.
%Table \ref{t:nrmse_insample_speeds} presents the values of the in-sample and out-of-sample average nRMSE over the 5 runs in both speeds and counts with the calibrated demand.
%The percentage improvement in the in-sample nRMSE in speeds over the initial speeds is $17.4\%$, $13.7\%$, and $31.7\%$ respectively with an increase in the calibration segments sample size. Similarly, in-sample nRMSE in counts improves by $8.8\%$, $20.3\%$, and $13.8\%$, respectively. For out-of-sample segments, the percentage improvements in counts and speeds are $10.0\%$ and $11.9\%$, and $10.5\%$ and $21.2\%$, respectively.
\section{Conclusion}
\label{s:conclusion}
This paper proposes an efficient metamodel-based approach for calibrating OD travel demands for urban road networks using GT speed data. We test the method under different data sparsity levels. Our numerical study shows that speeds-based calibration improves the fit to GT speeds by $89.2\%$ and the fit to GT counts by $76.1\%$ compared to the initial demands, with abundant data. It also shows the value of speed data, with increasing the number of segments with GT speed data improving the fit to both speeds and counts by $42.0\%$ and $48.8\%$, respectively. The proposed approach outperforms SPSA, a popular calibration algorithm, by $84.4\%$ and $72.2\%$ in terms of the fit to GT speeds and counts, respectively, under the same computational budget. Future work will investigate how abundant speed data can help mitigate the underdetermination (ill-posedness) of calibration problems, which is critical for using traffic models to perform counterfactually robust transportation analysis.
%Present work proposes an efficient metamodel-based approach for the calibration of OD travel demands for urban road networks using GT speed data. Tests are carried out on the performance of the method under different levels of sparsity of data. Numerical study shows that speeds-based calibration of demands results in an improvement of $89.2\%$ in the fit to GT speeds compared to the initial demand as well as an improvement of $76.1\%$ in the fit to GT counts with an abundant amount of data. Additionally, the results show the value of abundant speed data. Increasing the number of segments with GT speed data in calibration results in an improvement of $42.0\%$ and $48.8\%$ in terms of both speeds and counts, respectively. %The proposed approach is further compared to counts-based calibration using sparse count data, which represents the state-of-the-practice. %Evaluation is carried out on segments across the network including both in-sample and out-of-sample segments.
%Comparison shows that the proposed approach using abundant speed data outperforms counts in terms of the error in both speeds and counts by $17.4\%$ and $33.9\%$, respectively. This is a promising outcome since count-data is extremely sparse and installing and maintaining additional sensors %only for the purpose of calibration % can be costly and infeasible. %On the other hand, speed data can be obtained from navigational applications on a majority of the network.
%The proposed approach is also benchmarked against SPSA which is a popular algorithm used for calibration using GT speed data. Results indicate that under the same computational budget, the metamodel-based approach outperforms SPSA by $84.4\%$ and $72.2\%$ in terms of the fit to GT speeds and counts, respectively.
%Of interest for future work is the study of how the abundant speed data can contribute to mitigate the level of underdetermination (i.e., ill-posedness) of calibration problems. This is critical to enable traffic models to be used to perform counterfactually robust transportation analysis. %Moreover, exploiting spatial-temporal dependency information from the segment data can lead to novel problem formulations and to novel algorithms with enhanced sample efficiency.
%With the above preliminary results indicating the benefits of using speed data for calibration, further studies can be conducted on the joint use of counts and speeds for calibration. Studies can also be conducted into further improving the efficiency of the algorithm by trying various functional formulations for the metamodel.
%%
%% The acknowledgments section is defined using the "acks" environment
%% (and NOT an unnumbered section). This ensures the proper
%% identification of the section in the article metadata, and the
%% consistent spelling of the heading.
%\begin{acks}
%\end{acks}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-sigconf}
\end{document}
\endinput
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{10}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Arora et~al\mbox{.}(2021)]%
{arora2021efficient}
\bibfield{author}{\bibinfo{person}{Neha Arora}, \bibinfo{person}{Yi-fan Chen},
\bibinfo{person}{Sanjay Ganapathy}, \bibinfo{person}{Yechen Li},
\bibinfo{person}{Ziheng Lin}, \bibinfo{person}{Carolina Osorio},
\bibinfo{person}{Andrew Tomkins}, {and} \bibinfo{person}{Iveel Tsogsuren}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{An efficient simulation-based travel demand
calibration algorithm for large-scale metropolitan traffic models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2109.11392}}
(\bibinfo{year}{2021}).
\newblock
\bibitem[Cipriani et~al\mbox{.}(2011)]%
{cipriani2011gradient}
\bibfield{author}{\bibinfo{person}{Ernesto Cipriani}, \bibinfo{person}{Michael
Florian}, \bibinfo{person}{Michael Mahut}, {and} \bibinfo{person}{Marialisa
Nigro}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{A gradient approximation approach for adjusting
temporal origin--destination matrices}.
\newblock \bibinfo{journal}{\emph{Transportation Research Part C: Emerging
Technologies}} \bibinfo{volume}{19}, \bibinfo{number}{2}
(\bibinfo{year}{2011}), \bibinfo{pages}{270--282}.
\newblock
\bibitem[Huang et~al\mbox{.}(2010)]%
{huang2010accelerated}
\bibfield{author}{\bibinfo{person}{Enyang Huang}, \bibinfo{person}{Constantinos
Antoniou}, \bibinfo{person}{Jorge Lopes}, \bibinfo{person}{Yang Wen}, {and}
\bibinfo{person}{Moshe Ben-Akiva}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{Accelerated on-line calibration of dynamic traffic
assignment using distributed stochastic gradient approximation}. In
\bibinfo{booktitle}{\emph{13th International IEEE Conference on Intelligent
Transportation Systems}}. IEEE, \bibinfo{pages}{1166--1171}.
\newblock
\bibitem[Li et~al\mbox{.}(2022)]%
{lia2022fundamental}
\bibfield{author}{\bibinfo{person}{Y Li}, \bibinfo{person}{N Arora}, {and}
\bibinfo{person}{C Osorio}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{On the fundamental diagram of signal controlled
urban roads}. In \bibinfo{booktitle}{\emph{Proceedings of the 11th Triennial
Symposium on Transportation Analysis}}.
\newblock
\bibitem[Lopez et~al\mbox{.}(2018)]%
{sumo}
\bibfield{author}{\bibinfo{person}{Pablo~Alvarez Lopez},
\bibinfo{person}{Michael Behrisch}, \bibinfo{person}{Laura Bieker-Walz},
\bibinfo{person}{Jakob Erdmann}, \bibinfo{person}{Yun-Pang Flötteröd},
\bibinfo{person}{Robert Hilbrich}, \bibinfo{person}{Leonhard Lücken},
\bibinfo{person}{Johannes Rummel}, \bibinfo{person}{Peter Wagner}, {and}
\bibinfo{person}{Evamarie Wiessner}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Microscopic Traffic Simulation using SUMO}. In
\bibinfo{booktitle}{\emph{2018 21st International Conference on Intelligent
Transportation Systems (ITSC)}}. \bibinfo{pages}{2575--2582}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.1109/ITSC.2018.8569938}
\showDOI{\tempurl}
\bibitem[Mladenov et~al\mbox{.}(2022)]%
{mladenov2022adversarial}
\bibfield{author}{\bibinfo{person}{Martin Mladenov},
\bibinfo{person}{Sanjay~Ganapathy Subramaniam}, \bibinfo{person}{Chih-wei
Hsu}, \bibinfo{person}{Neha Arora}, \bibinfo{person}{Andrew Tomkins},
\bibinfo{person}{Craig Boutilier}, {and} \bibinfo{person}{Carolina Osorio}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{An adversarial variational inference approach for
travel demand calibration of urban traffic simulators}. In
\bibinfo{booktitle}{\emph{Proceedings of the 30th International Conference on
Advances in Geographic Information Systems}}. \bibinfo{pages}{1--4}.
\newblock
\bibitem[Osorio(2019)]%
{osorio2019high}
\bibfield{author}{\bibinfo{person}{Carolina Osorio}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{High-dimensional offline origin-destination (OD)
demand calibration for stochastic traffic simulators of large-scale road
networks}.
\newblock \bibinfo{journal}{\emph{Transportation Research Part B:
Methodological}} \bibinfo{volume}{124} (\bibinfo{year}{2019}),
\bibinfo{pages}{18--43}.
\newblock
\bibitem[Osorio and Punzo(2019)]%
{osorio2019efficient}
\bibfield{author}{\bibinfo{person}{Carolina Osorio} {and}
\bibinfo{person}{Vincenzo Punzo}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Efficient calibration of microscopic car-following
models for large-scale stochastic network simulators}.
\newblock \bibinfo{journal}{\emph{Transportation Research Part B:
Methodological}} \bibinfo{volume}{119} (\bibinfo{year}{2019}),
\bibinfo{pages}{156--173}.
\newblock
\bibitem[Spall(2005)]%
{spall2005introduction}
\bibfield{author}{\bibinfo{person}{James~C Spall}.}
\bibinfo{year}{2005}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Introduction to stochastic search and
optimization: estimation, simulation, and control}}.
\newblock \bibinfo{publisher}{John Wiley \& Sons}.
\newblock
\bibitem[Zhang and Osorio(2017)]%
{zhang2017efficient}
\bibfield{author}{\bibinfo{person}{Chao Zhang} {and} \bibinfo{person}{Carolina
Osorio}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Efficient offline calibration of origin-destination
(demand) for large-scale stochastic traffic models}.
\newblock In \bibinfo{booktitle}{\emph{Technical Report}}.
\bibinfo{publisher}{Massachusetts Institute of Technology}.
\newblock
\end{thebibliography}