CALM: Cognitive Assessment using Light-insensitive Model
Extracted source text · arXiv 2409.03888
%%
%% Copyright 2019-2024 Elsevier Ltd
%%
%% This file is part of the 'CAS Bundle'.
%% --------------------------------------
%%
%% It may be distributed under the conditions of the LaTeX Project Public
%% License, either version 1.3c of this license or (at your option) any
%% later version. The latest version of this license is in
%% http://www.latex-project.org/lppl.txt
%% and version 1.3c or later is part of all distributions of LaTeX
%% version 1999/12/01 or later.
%%
%% The list of all files belonging to the 'CAS Bundle' is
%% given in the file `manifest.txt'.
%%
%% Template article for cas-dc documentclass for
%% double column output.
\documentclass[a4paper,fleqn]{cas-dc}
% If the frontmatter runs over more than one page
% use the longmktitle option.
%\documentclass[a4paper,fleqn,longmktitle]{cas-dc}
%\usepackage[numbers]{natbib}
%\usepackage[authoryear]{natbib}
\usepackage[authoryear,longnamesfirst]{natbib}
%%%Author macros
\def\tsc#1{\csdef{#1}{\textsc{\lowercase{#1}}\xspace}}
\tsc{WGM}
\tsc{QE}
%%%
% Uncomment and use as if needed
%\newtheorem{theorem}{Theorem}
%\newtheorem{lemma}[theorem]{Lemma}
%\newdefinition{rmk}{Remark}
%\newproof{pf}{Proof}
%\newproof{pot}{Proof of Theorem \ref{thm}}
\begin{document}
\let\WriteBookmarks\relax
\def\floatpagepagefraction{1}
\def\textpagefraction{.001}
% Short title
\shorttitle{}
% Short author
\shortauthors{}
% Main title of the paper
\title [mode = title]{CALM: Cognitive Assessment using Light-insensitive Model}
% Title footnote mark
% eg: \tnotemark[1]
%\tnotemark[1]
% Title footnote 1.
% eg: \tnotetext[1]{Title footnote text}
%\tnotetext[1]{}
% First author
%
% Options: Use if required
% eg: \author[1,3]{Author Name}[type=editor,
% style=chinese,
% auid=000,
% bioid=1,
% prefix=Sir,
% orcid=0000-0000-0000-0000,
% facebook=<facebook id>,
% twitter=<twitter id>,
% linkedin=<linkedin id>,
% gplus=<gplus id>]
\author[1]{Akhil Meethal}%[<options>]
% Corresponding author indication
\cormark[1]
% Footnote of the first author
%\fnmark[1]
% Email id of the first author
\ead{akhilpm135@gmail.com}
% URL of the first author
%\ead[url]{}
\author[1,2]{Anita Paas}
\author[1]{Nerea Urrestilla Anguiozar}
\author[1]{David St-Onge}
% Credit authorship
% eg: \credit{Conceptualization of this study, Methodology, Software}
%\credit{}
% Address/affiliation
\affiliation[1]{organization={ETS Montreal},
addressline={ University of Quebec},
% citysep={}, % Uncomment if no comma needed between city and postcode
country={Canada}}
%\author[2]{}%[]
% Footnote of the second author
%\fnmark[2]
% Email id of the second author
%\ead{}
% URL of the second author
%\ead[url]{}
% Credit authorship
%\credit{}
% Address/affiliation
\affiliation[2]{organization={Department of Psychology, Concordia University},
country={Canada}}
% Corresponding author text
\cortext[1]{Corresponding author}
% Footnote text
%\fntext[1]{}
% For a title note without a number/mark
%\nonumnote{}
% Here goes the abstract
\input{Chapters/0-Abstract}
\nocite{*}%% Remove this line from your manuscript.
% Use if graphical abstract is present
%\begin{graphicalabstract}
%\includegraphics{}
%\end{graphicalabstract}
% Research highlights
\begin{comment}
\begin{highlights}
\item Pupillometry-based cognitive load estimation is sensitive to light.
\item Combining pupillometry with Heart Rate Variability reduces its light sensitivity.
\item Multimodal data collection improves cognitive load estimation accuracy and reliability.
\item Fitness grade devices are sufficient for HRV data in multimodal cognitive load estimation.
\item Transformer-based signal classification architecture with learnable features is competitive to the popular Random Forest classifier with hand-crafted features used in cognitive load classification.
\end{highlights}
\end{comment}
%\nocite{*}
% Keywords
% Each keyword is seperated by \sep
\begin{keywords}
Multimodal data \sep Cognitive load estimation \sep \sep
\end{keywords}
\maketitle
% Main text
\section{Introduction}
\label{sec:intro}
\input{Chapters/1-Introduction}
% ------------------ RELATED WORKS ------------------
%\section{Related Works}
%\label{sec:related_works}
%\input{Chapters/2-RelatedWorks}
% ------------------ METHODS ------------------
\section{Methods and Materials}
\label{sec:method}
\input{Chapters/3-Methods}
% ------------------ EXPERIMENTS ------------------
\section{Results}
\label{sec:experiments}
\input{Chapters/4-Results}
% ------------------ RESULTS ------------------
\section{Discussion}
\label{sec:discussion}
\input{Chapters/5-Discussion}
% ------------------ RESULTS ------------------
%\label{sec:dl}
%\input{Chapters/6-DL}
% ------------------ CONCLUSION ------------------
\section{Conclusion}
\label{sec:conclusion}
\input{Chapters/7-Conclusion}
\noindent \textbf{Acknowledgments}: This work was supported by the Natural Sciences and Engineering Research Council (NSERC) of Canada Discovery grant RGPIN-2020-06121.
%\section{Appendix}
%\input{Chapters/7-appendix}
% Numbered list
% Use the style of numbering in square brackets.
% If nothing is used, default style will be taken.
%\begin{enumerate}[a)]
%\item
%\item
%\item
%\end{enumerate}
% Unnumbered list
%\begin{itemize}
%\item
%\item
%\item
%\end{itemize}
% Description list
%\begin{description}
%\item[]
%\item[]
%\item[]
%\end{description}
%\clearpage %%Remove this from your manuscript
% Uncomment and use as the case may be
%\begin{theorem}
%\end{theorem}
% Uncomment and use as the case may be
%\begin{lemma}
%\end{lemma}
%% The Appendices part is started with the command \appendix;
%% appendix sections are then done as normal sections
%% \appendix
% To print the credit authorship contribution details
%\printcredits
%% Loading bibliography style file
%\bibliographystyle{model1-num-names}
\bibliographystyle{cas-model2-names}
% Loading bibliography database
\bibliography{cas-refs}
% Biography
%\bio{}
% Here goes the biography details.
%\endbio
%\bio{pic1}
% Here goes the biography details.
%\endbio
\end{document}
\begin{abstract}
The demand for cognitive load assessment with low-cost easy-to-use equipment is increasing, with applications ranging from safety-critical industries to entertainment. Though pupillometry is an attractive solution for cognitive load estimation in such applications, its sensitivity to light makes it less robust under varying lighting conditions. Multimodal data acquisition provides a viable alternative, where pupillometry is combined with electrocardiography (ECG) or electroencephalography (EEG). In this work, we study the sensitivity of pupillometry-based cognitive load estimation to light. By collecting heart rate variability (HRV) data during the same experimental sessions, we analyze how the multimodal data reduces this sensitivity and increases robustness to light conditions. In addition to this, we compared the performance in multimodal settings using the HRV data obtained from low-cost fitness-grade equipment to that from clinical-grade equipment by synchronously collecting data from both devices for all task conditions. Our results indicate that multimodal data improves the robustness of cognitive load estimation under changes in light conditions and improves the accuracy by more than 20\% points over assessment based on pupillometry alone. In addition to that, the fitness grade device is observed to be a potential alternative to the clinical grade one, even in controlled laboratory settings.
\end{abstract}
The level of automation in task performance is increasing across various domains, including manufacturing, transportation, and healthcare \cite{automation-Coombs-2019}. Concurrently, human involvement has shifted towards the verification of activities rather than direct task completion \cite{work_design-Parker-2022}. In most applications, humans still bear the pressure of success or failure, even if they are not directly controlling the automation. They must verify and ensure the correct functioning of automated tasks indirectly. Given this trend, understanding Cognitive Load (CL) during task performance is receiving increased attention. This must be performed in real-time in uncontrolled environments where the human operator is performing the task. Thus, reliable Cognitive Load Estimation (CLE) using devices that won't obstruct the task performance is paramount. %Adaptive automation, which adjusts automation levels based on the operator's cognitive state—such as stress, fatigue, and workload—is gaining popularity \cite{adaptive_automation-Margherita-2024}. %, adding further requirements for processing speed. Moving forward, understanding the cognitive load of human users will be a critical requirement in human-centered design. %Recent studies have emphasized the need for human-centered work design as automation continues to advance \cite{human-centred-Ngoc-2021}. Focused studies have investigated how poor human-centered design choices can lead to information overload, lack of intuitiveness, inconsistencies, and complex task demands \cite{interface_design-Sharon-2006}. These factors contribute to higher cognitive load for users, negatively impacting their experience and performance.
The increasing availability of low-cost, non-invasive devices makes it easier to measure cognitive response in real time during activities. Currently, several commercial devices measure pupillometry, heart rate variability (HRV), skin conductance, electroencephalography, and body motion \cite{mobile_measure-Caldiroli-2023, low_cost_cogload-Rajat-2014}. Among these, pupillometry provides an attractive option due to its low cost, unobstructive during the task and ease of setup and use, especially in field applications. But, Pupillary Response (PR) can be influenced by changes in light conditions, which should not be misattributed to cognitive load \cite{pupilary_response-Arthur-1991}. This is particularly important when measuring cognitive load in uncontrolled environments where light conditions can vary significantly. Research indicates that combining pupillometry with other physiological and behavioral measures offers a more comprehensive assessment of cognitive load \cite{multimodal_cogload-Vanneste-2020}.
%Nevertheless, pupil measurement is a versatile modality for many applications, especially when combining behavioral data (gaze) with direct cognition-related measurements (pupil diameter)~\cite{duval2023}. %Some of these devices are even designed for use in uncontrolled environments, equipped with wireless communication, and providing measurements comparable to those made in controlled settings. Many studies, such as those described in \cite{Souchet2022} and \cite{Rai2022}, show that features extracted from these devices can provide detailed information about the operator's state related to task performance and cognitive load. %Being wireless devices broadens the field of application, as they can be taken out of the experiment laboratory.
While the existing studies employed multimodal data for cognitive load estimation, focused studies on understanding the reliability and robustness of the pupillometry-based multimodal CLE under varying light conditions are lacking. Also, the studies in controlled environments (simulation-based laboratory studies) mainly used clinical-grade devices \cite{multimodal_cle_biopac-Oppelt-2023} whereas the uncontrolled field studies used fitness-grade devices \cite{multimodal_filed-David-2024} for the HRV. A comparison of both types of devices in the same settings to understand their characteristics is also absent in the current literature. The primary objective of this research is to understand the impact of light conditions on pupillometry-based CLE and quantify how a multimodal approach, combined with HRV, is improving the reliability and robustness. We used HRV because it is invariant to light conditions. In addition to the primary objective, our study includes a secondary objective aimed at comparing the low-cost, easy-to-use fitness-grade devices for HRV data in multimodal settings over clinical-grade HRV devices. %This is to obtain reliable cognitive load estimation in field applications, without obstructing the task performance.
%Pupillometry-based cognitive load estimation is sensitive to external light, and several works have attempted to develop features invariant to light conditions \cite{Duchowski2018}. However, recent research indicates that the light sensitivity issue remains unresolved \cite{light_compare-Nickolas-2024}. Particularly, we investigate how a multimodal approach combining pupillometry with HRV can provide a more comprehensive and reliable assessment of cognitive load under varying light conditions.
\subsection{Pupillometry and the light condition}
The variation of the pupil diameter when performing a task is widely used as a cognitive load estimator. Comparison of the cognitive load estimation results from gaze tracking and pupillary response identified the benefits of both eye movement-based measures \cite{pd_gaze_cogload-Krejtz-2018}. Even though it is among the most recent quantitative cognitive metrics, pupillometry accounts for a significant amount of research \cite{pupil_cogload-Weber-2021, PupillometryPP-Matht-2018}. The utility of pupillometry as a mental load estimator stems from the pupil's sensitivity to physiological factors related to the autonomic nervous system (ANS). Pupillometry is non-invasive and barely affects the task performance, making it an attractive choice for field applications.
One of the main drawbacks of pupillometry is that pupil diameter changes according to external light conditions. This means that features derived from pupil diameter are influenced not only by cognitive load but also by ambient light. To mitigate this sensitivity, Duchowski et al. developed the Index of Pupillary Activity (IPA) and the Low/High Index of Pupillary Activity (LHIPA) features \cite{Duchowski2018, Duchowski2020}. The Index of Cognitive Activity (ICA) is also widely recognized as less sensitive to light \cite{ica-Marshall-2002}. However, many recent studies have observed that these features still suffer from variations in light conditions \cite{light_compare-Nickolas-2024}. Our experiments also confirmed the sensitivity of pupillometry-derived features to ambient light, leading us to adopt a multimodal approach to reduce this sensitivity.
\subsection{Multimodal CLE with Pupillometry and HRV}
HRV is a well-studied metric in cognitive load estimation \cite{Solhjoo2019}. HRV is relatively robust to most external influences and is easy to measure and post-process. This measure is favored because of its direct relation to the ANS. The interaction between the sympathetic and parasympathetic nervous systems, the two main components of the ANS, is identified in heart activity as low-frequency (LF) and high-frequency (HF) bands. The balance between these two frequencies, known as the sympathovagal balance, is directly related to cognitive load \cite{hrv_sympathovogal-Kathrin-2021}.
HRV is sensitive to variations in the electromagnetic field and breathing cycle \cite{Shaffer2017}. HRV signal acquisition can also be affected when the operator is moving, due to touch intensity variations leading to fluctuations in the recorded signal. Using a multimodal approach, we seek to extract complementary information from electrocardiography (ECG) and pupillometry for robust cognitive load estimation.
%In order to ensure ease of use and scalability to more subjects, the sensors should fulfill the requirements of being non-invasive, wearable, reliable, and inexpensive.
To evaluate the feasibility of using fitness grade HRV devices over clinical grade ones, we chose to collect HRV from two distinct devices at the same time: the Biopac MP35 and the Polar H10. The Biopac MP35 is a sophisticated HRV data-collection tool predominantly used in clinical research contexts. This device requires participants to remain immobile throughout data collection, thereby restricting participant movement. Conversely, the Polar H10 serves as a heart rate monitoring device designed specifically for use during physical activity and exercise \cite{wearable_ECG-Alugubelli-2022}. The Polar H10 communicates wirelessly with the data acquisition system, facilitating seamless data collection even during activities. To compare the Polar and Biopac devices fairly under the same experimental conditions, we combined the pupillometry data with HRV data from each device separately and assessed the cognitive load estimation performance. The research objectives of this study can be summarized as follows:
\begin{itemize}
\item How does the change in light conditions affect the performance of pupillometry-based CLE? Combined with HRV, how can the multimodal data improves the reliability and robustness against change in light conditions?
\item Is there a significant difference in CLE performance when low-cost fitness-grade devices are used for HRV data acquisition over clinical-grade ones in a multimodal setting?
\end{itemize}
To validate our objectives, we collected participants' HRV and pupillometry data under light and dark conditions. Participants were asked to perform the n-back task to vary the level of cognitive load \cite{n-back_main-Kirchner-1958, n-back_modern-Kane-2007}. We conducted extensive experiments varying the cognitive load and light conditions. Multimodal data is acquired from all settings using both HRV devices. %Our user study and data processing pipeline feed on the previous work mentioned above and validate our classification pipeline with two experiments with human participants. %This section contains detailed information on the participants recruited, the methodology followed for the data acquisition, the sensors used, and the post-processing of collected data both for the pupillometry and the HRV measures.
\subsection{Participants}
\noindent We recruited 10 participants (5 female, 5 male) from the research group. All participants were between 20 to 45 years old. This research was approved by the ÉTS (Ecole de Technology Supérieure) Research Ethics Board in accordance with the Canadian Tri-Council Policy Statement regarding the Ethical Conduct for Research Involving Humans (TCPS2; Approval number H20210302). % and they were researchers and students from Ecole de technology supérieure in Montréal, Canada
\subsection{Experimental Design}
\subsubsection{Sensors}
\noindent The sensors used for recording the data were Tobii glasses along with two ECG devices: the BioPac MP35 and the Polar H10 chest band. The Polar H10 (see Fig. \ref{fig:hrvdev}; Polar, Electro, Oy, Finland) is a wireless chest band equipped with two electrodes positioned one inch apart on the chest to measure ECG. Although it is expected to be less accurate compared to clinical-grade devices, it offers the advantages of portability and ease of use. The BioPac MP35, on the other hand, is a three-electrode device that provides high-fidelity data collection for measuring HRV. We used a single analog channel with three electrodes placed on the body as illustrated in Fig. \ref{fig:pupildev}: right arm (white), right leg (black), and left calf (red). This configuration allows for triangulation to extract a clean ECG signal. The main drawback of the BioPac MP35 is that it is wired, which limits its field of application, but it provides a more detailed dataset compared to the Polar H10.
%In the first experiment, we used Pupil Labs eye-tracking glasses (Pupil Labs, Berlin, Germany) and the BioPac MP35 ECG unit (SS2L, BioPac Systems Inc, Goleta, California), while in the second experiment, we employed
\begin{figure}[!h]
\centering
\includegraphics[width=0.6\columnwidth]{Images/BiopacSetup.jpg}
\includegraphics[width=0.35\columnwidth]{Images/Polar.jpg}
\caption{HRV devices: Left, BioPac M35 with the electrodes position, Right, Polar H10 chest band used only in the second experiment.}
\label{fig:hrvdev}
\end{figure}
For measuring and recording pupillometry, Tobii Pro 3 glasses were used (see Fig. \ref{fig:pupildev}). It is a wireless eye-tracking platform that contains cameras pointing at the pupil mounted on a glass frame with illuminators. It is able to record and perform the calibration automatically. The data was streamed over WiFi to the researcher's machine, on which all the data was stored and synchronized. The glasses are wired to the streaming and battery-containing device which is attached to the waist of the participant.
\begin{figure}[!h]
\centering
\includegraphics[width=0.57\columnwidth]{Images/TobiiGlasses.jpeg}
\caption{Pupillometry device: Tobii Pro 3 glasses}
\label{fig:pupildev}
\end{figure}
\subsubsection{Procedure}
\noindent Experiments took place in a laboratory environment, where the lighting conditions could be controlled. Participants came to the testing room and the task and equipment were explained to them. After consenting to continue with the experiment, participants were instructed how and where to place the Polar H10 and then shown to the bathroom where they could dampen the electrodes and place the chest band underneath their shirt across their chest with the electrodes at their sternum. Next, the BioPac MP35 was put on. Three 3M Red Dot electrodes with a built-in abrader (3M, ID: 7000128699) were placed on the recently cleaned right wrist and both ankles. Finally, the Tobii Pro 3 glasses were put on. The glasses were then calibrated according to the procedures provided by Tobii to ensure accurate measuring and recording of eye data. After all the devices were set up, the experimenter started recording the data, synchronized across devices using ROS. Once the experimenter ensured that all the devices were recorded properly, the n-back task (described below) began. Participants were comfortably seated in a chair throughout each experiment.
%This experiment focused on the impact of light and how multimodal data can mitigate this sensitivity
There were three levels of mental workload (rest, low, and high) and two levels of lighting (light [210 Lux] and dark [1 lux]). In the rest condition, participants looked at a point on the wall and relaxed while their pupil and heart rate data were recorded. During the low workload condition, participants performed the 1-back task, at which they heard a sequence of stimuli (numbers between 0 and 26) and pressed a key on the keyboard whenever they heard a match between the current stimulus and the stimulus presented previously. In the high workload condition, participants performed a 2-back task \cite{n-back_main-Kirchner-1958}. In the 2-back task, participants heard a sequence of stimuli - numbers between 0 and 26. Each time participants heard a number that matched the number presented 2 stimuli previously in the sequence, they pressed a key on the keyboard to indicate they heard a match. This task induces a high mental workload because it requires participants to maintain and continuously update two digits in their working memory. The duration of the task was 3 minutes. Between each condition, there was a break of at least 2 minutes (longer if the participant needed more time). The order of conditions (both workload and lighting) was counterbalanced across participants. Figure \ref{fig:setup} shows the setup for this experiment, with the light condition on the left and the dark condition on the right. The whole experiment duration was approximately one hour. %During this experiment, pupil data was recorded with the Pupil Labs glasses, and heart rate data was recorded with the BioPac M35.
%The first experiment had two mental workload levels, Rest and High Load states. Therefore this is a binary classification problem of the cognitive load. This experiment focuses on the study of the impact of light and how multimodal data can mitigate this sensitivity. In the Rest state, the participant was asked to look to a point and relax while the acquisition took place. For the High Load state, the participant was asked to perform a 2-back task \cite{n-back_main-Kirchner-1958}. The task consisted of an audio cue with numbers from 0 to 26 and each time the participant noticed the cue, they needed to press a button to register their response on a keyboard. For the High Load state with 2-back task, for each stimulus, the participants must indicate whether it matches the stimulus presented 2 trials/steps back in the sequence. This task induces a high mental workload because it requires the participants to maintain a longer sequence in their working memory and to make more complex comparisons between current and past stimuli. The duration of the task was 3 minutes. Between each of the states, a minimum of 2-minute break was given. In case the participant needed more time, the break was increased until he or she was ready to start the next condition. The order in which the tasks were given to the participants was counterbalanced during the data collection.
%The second experiment had three mental workload levels: Rest, low CL (Cognitive Load), and high CL. The tasks participants needed to perform for the Rest and high CL states were the same as above. For the low CL state, they performed a 1-back task in which for each new stimulus, they indicated whether it matched the stimulus presented one step/trial before. The light conditions were varied in each experiment for this set as well along with multimodal data collection. HRV data was acquired in this case using both Biopac and Polar. Thus in addition to assessing the improved cognitive load estimation with multimodal data, this experiment is also used to compare the cognitive load estimation performance of Biopac and Polar.
%Each experiment was divided into two conditions, the first one was performed with regular ambient light (Light condition; 210 lux) and the second one in complete darkness (Dark condition; 1 lux). This permitted us to effectively study the impact of external light on pupillometry-based cognitive load understanding. The order of the conditions in each experiment was counterbalanced. The full experiment duration was approximately 1 hour. As shown in Fig. \ref{fig:setup} left, for the Light condition, the participant was looking at the well-lit area of the room; For the dark condition, they were facing the wall at the low-lit area of the room (Fig. \ref{fig:setup} right). Each of the experimental conditions lasted three minutes, during which participants completed all tasks with different levels of mental workload. The lighting conditions order was counterbalanced between participants.
\begin{figure}[!h]
\centering
\includegraphics[scale=0.38]{Images/ExperimentSetup.png}
\caption{Left: Experiment setup for the Light condition. Right: Experiment setup for the Dark condition.}
\label{fig:setup}
\end{figure}
%To differentiate the cognitively loaded vs rest state, another set of binary cognitive load classification data was collected. In the cognitively loaded state, the participants performed a 2-back task. This binary dataset was used to study the impact of cognitive load estimation as lighting conditions change. Each experiment in this case had two conditions of 2-minute duration; in the first condition, participants completed a rest state (without any mental workload) and the second condition was the cognitively loaded state where participants performed a 2-back task. %\textcolor{red}{What is the purpose of the binary dataset exactly? What is the mental task for high in the binary dataset?}
\subsection{Data Pre-processing and Feature Extraction}
\subsubsection{Pupillometry}
Pupil diameter data are usually noisy due to eye movements and blinking, which results in missing data blocks. Packet loss during recording also results in missing data. The missing data were handled by re-sampling the signal, keeping a fixed sampling frequency (set to 100Hz). As a first step towards cleaning the data, outliers were removed, keeping the diameter range between 1.5 and 9 mm (the general minimum and maximum for pupil diameter). The noise was handled using filtering and smoothing. A low-pass butter filter of order 5 and a cutoff of 4 Hz was used in this process \cite{butter_filter-Knapen-2016}. We used a sliding window of 60 seconds and the pupil diameter values were averaged over each window. %The initial hypothesis was to have no difference in the values between both of them. This was considered following the work by Duchowsky et al. %\cite{Duchowski2018} which stated that the cognitive load measure of the pupil suffers no external light effect on its value while using their own developed parameter named the Index of Pupillary Activity (IPA). %For the noise, 100ms long average sliding windows can be used, where an average of each window is performed to smooth the data; another solution is the application of low-pass filters such as the butter filter. For the missing data, the main solution is to resample the data to a higher frequency, in this case to 1000Hz, so that the missing data points are interpolated. Due to the aforementioned problems with the pupil data, post-processing is required. With the objective of obtaining the best performing methodology on the classifier, four different processing techniques were used, combining the solutions for noise and missing data samples. The first case was using sliding windows without resampling, the second one sliding windows with resampling, the third one low-pass filter without resampling and the last one low-pass filter with resampling. Besides the actions mentioned above, there are other processings that helped cleaning the data. First, outliers were removed, and for that, the range that was selected to keep the pupil diameter was between 2 and 9mm. Next the median of the rest condition was calculated for the light and no light conditions, and removed from the data, to facilitate comparisons between participants. The next step followed the previous processing steps, having the resampling to 1000Hz, sliding windows of 100ms and low-pass filtering for each corresponding case. The butter low-pass filter was of order 3, and two cutoff frequencies were compared, 4 and 10Hz.
Within each window, the recorded pupil diameter values were used to compute the required features. After careful consideration of the significance of the features in predicting cognitive load, the following features were selected: pupil diameter, standard deviation of the pupil, Index of Pupillary Activity (IPA), PDAuC (Pupil Diameter Area under the Curve), and PDRoC (Pupil Diameter Rate of Change). PDRoC finds the slope of the regression line fit on pupil size data. Both PDAuC and PDRoC were obtained using the PyTrack library \cite{pytrack-Tortora-2022}. IPA is computed following the implementation from \cite{ipa-Duchowski-2018}
\subsubsection{Heart Rate Variability}
Heart-related data were recorded by two different devices during the experiments - the BioPac MP35 and the Polar H10. The HRV features were extracted from the raw signals of both devices, over a window of 60 seconds. Sliding windows with an overlap of 10 seconds between windows were used for feature extraction. For Polar H10, the sampling rate was 120 Hz and the signal was resampled to account for the loss of data. For BioPac MP35, the sampling rate was 1000 Hz. The signal values outside the range [300-1000] mV were considered outliers and they were removed. Finally, resampling was performed to make the session length consistent to a fixed 3-minute duration.
The feature extraction was carried out using the neurokit2 toolbox \cite{neurokit2-Makowski-2021}. For this, RR peaks (the time between successive R waves) were first computed from the raw ECG data after pre-processing. Once this was completed, the next step was to compute the time domain and the frequency domain features. From the time domain, the features extracted were RMSSD (root mean square of successive differences), SDNN (standard deviation of normal-to-normal intervals), pNN50 (percentage of successive normal RR intervals that differ by more than 50ms), mean RR interval, median RR interval, and respiration rate. From the frequency domain, the features extracted were HF (high frequency) and LFHF Ratio (low-frequency power to high-frequency power). %Feature importance was studied to understand the contribution of individual features in the overall prediction results. Though more features were considered for the analysis, based on the importance calculated, features with low importance were discarded.
\subsection{Classifiers}\label{met:class}
The classifier used in our experiments was primarily the random forest (RF) classifier, but we also compared the results with neural network-based classifiers. The RF classifier was chosen due to its robustness, the feasibility of doing feature importance analysis, and low risk of over-fitting, especially with smaller datasets. In the RF classifier, 100 estimators were used. We used classification accuracy on the test set as the main metric for comparison as it provides a direct measure of how well the model generalizes to unseen data. High accuracy on a test set indicates good generalization assuming the test set is representative of the real-world data the model will encounter (commonly used assumption in machine learning classifiers). A 70-10-20 split was used for training, validation, and test set respectively.
We selected two neural network-based classifiers for the comparative study, with and without using hand-crafted features. The first was a multi-layer network (called MLP or multi-layer perceptron) trained on the same features used with the RF classifier (with hand-crafted features). The second was a transformer-based architecture \cite{transformer-Vaswani-2017} which learns the features from the raw input signals (without hand-crafted features). The learnable features eliminate the need for deep domain expertise required to design and verify the hand-crafted features \cite{feat_learning-Bengio-2013}. The MLP is a 4-layer network with ReLU activation and batch normalization \cite{batch_norm-Sergey-2015}. The hidden layer dimensions are 256, 128, and 64, respectively, from layers 1 to 3. The Transformer classifier is a custom-designed architecture with 4-layer attention-based network \cite{transformer-Vaswani-2017} with GeLU activation and LayerNorm \cite{layernorm-Xu-S2019}. To manage the large sequence length at the input of the Transformer, a 7-layer 1D convolutional network is employed initially to obtain compact embeddings from the input, inspired from \cite{wave2vec-Baveski-2020}.
\begin{comment}
\begin{figure}[!h]
\centering
\includegraphics[scale=0.55]{Images/feat_imp_new.png}
\caption{Feature influence in the multimodal training of an RF model. Features from HRV are shown in dark green and features from pupillometry are shown in light green.}
\label{fig:feature_importance}
\end{figure}
\end{comment}
%However, the features obtained trade the interpretability of the classification model. Also, in limited data regimes, using handcrafted features can still be competitive over deep learnable features \cite{hand_crafted_feats-Nanni-2017}. Detailed information on the network architectures and comparison to the RF classifier is provided in section \ref{sec:dl}.
%The random forest algorithm from the scikit-learn (\url{https://scikit-learn.org/stable/}) python package was used. As for the parameters selected for it, we were based on the parameters used in the work from Albuquerque et al. \cite{Albuquerque2020}. We utilized 15 estimators, K-fold data divider with 5 splits and we tested with the maximum features set to auto, sqrt, log2. Inside each split, the model was trained 50 times and each time the best maximum features parameters was selected, and then tested on the test split. Once all the iterations were done, the average of the accuracy of the model from all of them was used as the main output parameter.
%There was no subjective assessment at the end of the experiment.
The extent of our experiments allows us to address several topics from the analysis of the data. We start by analyzing the overall performance difference due to change in light conditions and how the multimodal data is improving it. Then we go for a more fine-grained analysis at the feature level when light conditions vary. All models in this section are trained with an RF classifier using 100 estimators. %most meaningful features and then dive into our core interest: the light sensitivity of pupillometry and the multimodal data for addressing it.
\subsection{Pupillometry and change in light conditions}
To highlight the sensitivity of pupillometry to light, we trained classifiers under different lighting conditions and tested them separately in light and dark conditions. Specifically, five settings were considered for training and evaluation. The training set includes both light-only conditions (labeled as "Light") and mixed light and dark conditions (labeled as "All"). The test set includes light-only, dark-only (labeled as "Dark"), and mixed light and dark conditions. This split aims to understand the impact of light on pupillometry-based cognitive load estimation.
\begin{table}
\caption{Comparison of the accuracy between Pupillometry only and multimodal (Pupillometry+HRV) data on all classifiers (RF, MLP and Transformers).}
\centering\footnotesize
\begin{tabular}{ l c c c c c}
\toprule
\textbf{Sensors} & \textbf{Train} & \textbf{Test} & \textbf{RF} & \textbf{MLP} & \textbf{Trans.} \\
\midrule
\midrule
Pupil & Light & Light & 79.46 & 51.16 & 56.94 \\
Pupil & Light & Dark & 46.56 & 34.38 & 37.50 \\
Pupil & All & Light & 80.30 & 55.53 & 58.33 \\
Pupil & All & Dark & 73.67 & 38.75 & 52.78 \\
Pupil & All & All & 77.05 & 53.35 & 55.14 \\
\midrule
HRV(B)+Pupil & Light & Light & 92.24 & 82.81 & 86.11 \\
HRV(B)+Pupil & Light & Dark & 47.71 & 38.88 & 41.67 \\
HRV(B)+Pupil & All & Light & 91.53 & 81.25 & 84.72 \\
HRV(B)+Pupil & All & Dark & 83.59 & 62.50 & 79.17 \\
HRV(B)+Pupil & All & All & 89.42 & 78.95 & 84.66 \\
\midrule
HRV(P)+Pupil & Light & Light & 96.29 & 84.38 & 92.19 \\
HRV(P)+Pupil & Light & Dark & 50.77 & 39.25 & 38.19 \\
HRV(P)+Pupil & All & Light & 95.46 & 83.59 & 91.41 \\
HRV(P)+Pupil & All & Dark & 91.98 & 65.63 & 79.16 \\
HRV(P)+Pupil & All & All & 92.26 & 81.86 & 90.23 \\
\bottomrule
\end{tabular}
\label{tab:three_load_results}
\end{table}
\begin{comment}
The results from the binary cognitive load classification task (experiment 1) are reported in table \ref{tab:binary_load_results}. The performance of pupillometry-only models significantly decreased (by more than 12 percentage points) when trained on light conditions and tested on dark conditions. Despite using features like IPA to reduce sensitivity to light, its impact remains, as noted in \cite{light_compare-Nickolas-2024}. As expected, training on a mix of light and dark conditions yields better performance because it mitigates the distribution shift during testing. %\textcolor{red}{here the drop is only 12\% compared to 33\% in the 3-class. why?}.
\end{comment}
The results of the cognitive load classification task are reported in table \ref{tab:three_load_results}. In the table, HRV(B) and HRV(P) denote the HRV features computed from Biopac and Polar devices respectively. First, we will analyze the results of the RF classifier. In all cases, we see a significant drop in performance (33-45\% points) when trained on light conditions and tested on dark conditions, clearly showing how the distribution shift due to light is impacting the cognitive load estimation. The performance of the pupillometry only model decreases despite using features like IPA to reduce the sensitivity to light, as noted in \cite{light_compare-Nickolas-2024}. When the training set has data from both light and dark conditions, the accuracy drop is less (in the range of 4-9\% points). Training and testing in the light condition has the best performance in all cases. %\textcolor{red}{possible reasons?}
\subsection{Multimodal models}
The core strategy envisioned to reduce the lighting influence over pupillometry is to train on multiple modalities together. We thus used multimodal input to train a classifier combining HRV features with pupillometry features under the same conditions as previously described. The results are shown in the lower parts of Table \ref{tab:three_load_results}. It is evident that using multimodal inputs significantly enhances the classifier's performance. An overall improvement of 12.4\% and 15.2\% is observed with Biopac and Polar, respectively, using the multimodal features.
Training in light and testing in dark conditions incurs significant performance loss. With pupillometry alone, this loss is around 33\% points. Although the multimodal settings show slight improvement compared to the pupillometry-only case, the performance remains lower than the results when the training set contains data from all light conditions. This implies that the distribution shift due to light conditions still impacts performance as the pupillometry features show significant variance. Thus, we can conclude that even though multimodality improves the accuracy of CLE, the impact of distribution shift due to light cannot be neglected. Therefore, it is crucial to include data from diverse lighting conditions in the training set to achieve robust cognitive load estimation.
In figure \ref{fig:cm_comparison}, we compared the confusion matrices of the predictions from pupillometry and the two multimodal settings. When pupillometry alone is used, the classifier struggles to distinguish between the mental workload levels. Additionally, the Rest state shows significant confusion with other states. However, with multimodal data, the mental workload levels are more clearly separated by both Biopac and Polar sensors. Polar reduces the confusion of all states more than the Biopac.
\begin{figure}[!ht]
\centering
\begin{tabular}{c @{\hspace{0.1cm}} c @{\hspace{0.1cm}} c @{\hspace{0.1cm}} c}
\includegraphics[width=0.28\linewidth, height=2.4cm]{Images/CM/cm_pupil.png} &
\includegraphics[width=0.28\linewidth, height=2.4cm]{Images/CM/cm_T_B.png} &
\includegraphics[width=0.28\linewidth, height=2.4cm]{Images/CM/cm_T_PO.png} &
\includegraphics[width=0.08\linewidth, height=2.4cm]{Images/CM/hmap.png} \\
Pup. & HRV(B)+Pup. & HRV(P)+Pup. & \\
\end{tabular}
\caption{Comparison of confusion matrices between pupillometry only and multimodal models.}
\label{fig:cm_comparison}
\vspace{-0.4cm}
\end{figure}
%But in the multi-modal settings, the same settings report a drop of 2.4\% points only. Thus in this case, we can conclude that the light-insensitive HRV features were able to retain the performance well under the distribution shift between the train and test set. \textcolor{red}{more explanations?}. Specifically, the accuracy is improved by 19.5\% points when HRV features are combined with pupillometry features. Training on both Light and Dark conditions always gives better results when the light conditions are varying at test time.
\subsection{Impact of HRV device type}
When comparing the Biopac MP35 and the Polar H10 under our experimental conditions, the Polar H10 yielded the best results. The overall accuracy difference between the two was not substantial, around 2.8 percentage points. Although Biopac offers high-fidelity data collection suitable for clinical research applications, its wired setup is sensitive to participant movements, resulting in noisy data. In our experiments, participants were seated in a chair and exhibited movement while performing the n-back task, affecting the data quality. Conversely, Polar, designed for activity tracking, facilitated easier data collection. Based on the observations in Table \ref{tab:three_load_results}, we conclude that Polar is as good as Biopac or even better in experimental conditions where the participants are comfortably seated during data collection but may still exhibit movement. In adaptive automation contexts, workers are expected to be mobile, moving between stations and adjusting their working positions to prevent injuries or illnesses from prolonged sitting.
An overall comparison of both individual HRV device performances and their combination with pupillometry is reported in Table \ref{tab:overall_biopac_pupil}. The training and test sets included data from both light and dark conditions. The results show only a small difference of 1.2\% in cognitive load estimation accuracy between Biopac and Polar. When combined with pupillometry features, the accuracy improved by approximately 10\% points compared to using Biopac and Polar alone. The multimodal settings also showed increased performance compared to pupillometry-only results, with an accuracy improvement of over 12\% points for Biopac and 15\% for Polar. Additional material with more experiments using PupilLabs and Biopac sensors on a binary cognitive load classification problem (Rest vs Load) is provided here: \href{https://initrobots.ca/calm/}{https://initrobots.ca/calm/}
\begin{table}
\caption{Accuracy comparison of all models with full data under single-modal and multimodal settings.}
\centering\footnotesize
\begin{tabular}{ l c c c}
\hline
\textbf{Sensors} & \textbf{RF} & \textbf{MLP} & \textbf{Transformer} \\
\hline
\hline
Pupil & 77.05$\pm$0.40 & 53.35$\pm$0.18 & 55.14$\pm$p0.34 \\
HRV(B) & 80.96$\pm$0.41 & 74.61$\pm$0.21 & 79.04$\pm$p0.32 \\
HRV(P) & 82.15$\pm$0.28 & 75.20$\pm$0.25 & 83.98$\pm$0.29 \\
HRV(B)+Pupil & 89.42$\pm$0.28 & 78.95$\pm$0.22 & 85.66$\pm$0.38 \\
HRV(P)+Pupil & 92.26$\pm$0.19 & 81.86$\pm$0.17 & 90.23$\pm$0.28\\
\hline
\end{tabular}
\label{tab:overall_biopac_pupil}
\end{table}
\subsection{Comparison with Neural Net models}
Table \ref{tab:overall_biopac_pupil} also presents the comparison of neural network classifiers with the RF classifier. The advantages of multimodal signals over a single modality are evident from these results as well, wherein all cases, using multimodal signals yielded the best results. Although the light-insensitive HRV sensors are superior to pupillometry alone, pupillometry still provides valuable information that further enhances the CLE. This is due to the sensitivity of HRV measurements to participant movement. The Transformer-based classifier with learned features shows better results than the MLP in all cases. Except for pupillometry, the results of the Transformer are comparable to the RF classifier. It is widely known that neural networks are sensitive to distribution shifts \cite{domain_shift_prot-Meethal-2024, cross_domain_shift-Li-2022}. Given that we explicitly captured shifted data distributions for pupillometry, the Transformer models are expected to perform less effectively compared to RF. In table \ref{tab:three_load_results}, a detailed analysis of the performance of all classifiers under varying lighting conditions is provided.
%Training on light conditions and testing on dark conditions results in a significant degradation of accuracy in pupillometry-based cognitive load estimation. The MLP classifier performs the least among all classifiers. Although the multimodal input improves performance in this scenario, the classifiers still struggle to fully overcome the impact of the distribution shift. However, when the training set includes data from dark conditions as well, the classifiers perform much better.
\subsection{Feature-level Analysis}
%When using HRV and pupillometry measurements to classify cognitive states (or workload), the literature shows that each study selects its most appropriate set of features. To properly compare their influence, we quantify the importance of the available features. As random forest (RF) is still the most common model in this field \cite{light_compare-Nickolas-2024, multi_modal_CogLoad-Kim-2021, Dengbo2022}, we computed the feature importance from an RF classifier. Figure \ref{fig:feature_importance} illustrates the relative importance of all the features used in our study. Overall, we can see that the features derived from HRV strike as more influential, with frequency-domain features from HRV (HF, LFHF) showing more importance than time-domain features.
\begin{comment}
\begin{figure}[!h]
\centering
\includegraphics[scale=0.55]{Images/feat_imp_new.png}
\caption{Feature influence in the multimodal training of an RF model. Features from HRV are shown in dark green and features from pupillometry are shown in light green.}
\label{fig:feature_importance}
\end{figure}
\end{comment}
\begin{figure*}[!ht]
\centering
\begin{tabular}{c @{\hspace{-0.3cm}} c @{\hspace{0.2cm}} | c}
\includegraphics[width=0.33\linewidth, height=3.5cm]{Images/features/pd_mean_3load.png} &
\includegraphics[width=0.33\linewidth, height=3.5cm]{Images/features/pd_variance_3load.png} &
\includegraphics[width=0.32\linewidth, height=3.5cm]{Images/diameter_tobii.png} \\
(a) Pupil Diam. Mean & (b) Pupil Diam. Var & Pupil Diam. Mean \\
\includegraphics[width=0.33\linewidth, height=3.5cm]{Images/features/rmssd_3load.png} &
\includegraphics[width=0.32\linewidth, height=3.5cm]{Images/features/pnn50_3load.png} &
\includegraphics[width=0.32\linewidth, height=3.5cm]{Images/rmssd_polar.png} \\
(a) RMSSD & (b) pNN50 & RMSSD \\
\end{tabular}
\caption{\textbf{Left}: Density plots of pupillometry and HRV features under light and dark conditions. The top row shows pupillometry features (mean pupil diameter, and pupil diameter variance), while the bottom row shows HRV features (RSMMD, and pNN50). Higher sensitivity to light of pupillometry features can be observed from their complete distributions' change. The HRV features are relatively less sensitive to light. \textbf{Right}: Box plot of the Pupil mean and RMSSD values for all cognitive load conditions using the Polar device. CL1 and CL2 stand for cognitive load for the 1-back (low) and 2-back (high) tasks respectively.}
\label{fig:feat_violin_plot}
\vspace{-0.6cm}
\end{figure*}
\begin{comment}
\begin{figure}[!h]
\centering
\includegraphics[scale=0.4]{Images/diameter_tobii.png}
\caption{Box plot of the mean pupil diameter values for all cognitive load conditions using the Tobii device. CL1 and CL2 stand for cognitive load for the 1-back (low) and 2-back (high) tasks respectively.}
\label{fig:pupilboxplot}
\vspace{-0.6cm}
\end{figure}
\begin{figure}[!h]
\centering
\includegraphics[scale=0.4]{Images/rmssd_polar.png}
\caption{Box plot of the RMSSD values for all cognitive load conditions using the Biopac device. CL1 and CL2 stand for cognitive load for the 1-back (low) and 2-back (high) tasks respectively.}
\label{fig:ecgboxplot}
\vspace{-0.6cm}
\end{figure}
\end{comment}
To understand the feature level differences from varying light conditions, we selected the most common features from both pupillometry and HRV and compared their distribution using violin plots under light and dark conditions. Figure \ref{fig:feat_violin_plot} left shows the distribution under light and dark conditions. The pupillometry features exhibit a significant distribution shift between these conditions. In contrast, the distribution shift is significantly less for the HRV features, suggesting that combining both types of features can enhance robustness to light variations.
Figure \ref{fig:feat_violin_plot} right shows box plots of the most commonly used features mean pupil diameter and RMSSD. It reveals distinctly different distributions for light and dark conditions, with this difference being consistent across rest, 1-back, and 2-back conditions. This again confirms that pupil diameter is highly sensitive to lighting changes. For HRV, the difference between light and dark conditions is less pronounced, indicating that RMSSD is less sensitive to variations in lighting conditions.
\subsubsection{Statistical test results}
We conducted t-tests to understand if the features analyzed above shows significant difference between light and dark conditions. The Mean Pupil Diameter shows a p-value of 7.164e-26, thus it has a significant difference between light and dark conditions. However, RMSSD obtains a p-value of 0.488, thus there is no statistically significant difference between light and dark conditions. More detailed t-tests comparing the effectiveness of Mean Pupil Diameter and RMSSD in distinguishing different cognitive load states are presented in table \ref{tab:ttest}. Here we observed that the Mean Pupil Diameter is unable to differentiate between CL1 and CL2 states whereas RMSSD is unable to differentiate between Rest and CL1 states. This again highlights the importance of having multimodal data to accurately differentiate the cognitive load levels.
\begin{table}
\caption{Statistical test results comparing Pupil Diameter and RMSSD across all states (p-values are shown).}
\centering\footnotesize
\begin{tabular}{ l c c c}
\hline
\textbf{Feature} & \textbf{Rest vs CL1} & \textbf{Rest vs CL2} & \textbf{CL1 vs CL2} \\
\hline
\hline
Pupil Diam & 0.0106 & 0.0052 & 0.1517 \\
RMSSD & 0.3218 & 0.0010 & 0.0003\\
\hline
\end{tabular}
\label{tab:ttest}
\end{table}
\subsection{Reliability over light conditions}
Another way to understand the challenge of brightness impact is to analyze the reliability of sensors in estimating cognitive load under varying light conditions. For this purpose, we computed the Expected Calibration Error (ECE) for individual sensors and the multimodal settings. ECE represents the expected absolute difference between the predicted confidence and the model accuracy \cite{ECE-Hauskrecht-2015}. Let $y$ be the true label, $\hat{y}$ be the predicted label, and $\hat{p}$ be the confidence of the predicted label. ECE is computed as $\textrm{E}_p\big[|\mathcal{P}(\hat{y}=y|\hat{p})-\hat{p}|\big]$. The results are shown in Table \ref{tab:ece_comparison}. It can be observed that HRV features report better ECE in estimating cognitive load under varying light conditions. The ECE of pupillometry-based cognitive load prediction is high, indicating unreliability under changing light conditions. In the multimodal settings, where HRV and pupillometry are used together, ECE shows significant improvement over the pupillometry-only setting.
\begin{table}
\caption{Expected Calibration Error(ECE) when using individual sensors and their combination (multimodal setting). The lower the ECE, the higher the reliability.}
\centering
\resizebox{.3\textwidth}{!}{%
\begin{tabular}{ c c c c }
\hline
\textbf{Metric} & \textbf{HRV} & \textbf{Pupil} & \textbf{HRV+Pupil}\\
\hline
\hline
\textbf{ECE} & 0.197 & 0.307 & 0.221 \\
\hline
\end{tabular}
}
\label{tab:ece_comparison}
\end{table}
\begin{comment}
We also created the reliability plots for the same settings as shown in Figure \ref{fig:reliability_plot}. In practice, ECE is estimated approximately, by grouping the predictions into M-equispaced bins and computing the mean accuracy and confidence in each bin. The reliability plot shows the mismatch between predictive confidence and accuracy in each bin. We can find the classifier is overestimating the confidence for values above 0.8 for both pupillometry and HRV+pupillometry. When the model shows confidence values above 0.8 and the reliability is not good, it means the high-confidence predictions from the model are not reliable. We can also observe that even though the multimodal setting is improving the accuracy of the cognitive load estimation, due to the sensitivity of the pupillometry to light, the reliability is still lower than the HRV-only setting.
\begin{figure*}[!ht]
\centering
\begin{tabular}{c @{\hspace{-0.3cm}} c @{\hspace{-0.3cm}} c}
%\includegraphics[width=0.35\linewidth, height=3.4cm]{Images/calibration/ece_eye_rest.png} &
%\includegraphics[width=0.33\linewidth, height=3.4cm]{Images/calibration/ece_ecg_rest.png} &
%\includegraphics[width=0.33\linewidth, height=3.4cm]{Images/calibration/ece_all_rest.png} \\
% (a) Eye & (b) ECG & (c) Eye+ECG \\
\includegraphics[width=0.33\linewidth, height=3.4cm]{Images/calibration/ece_ecg_load.png} &
\includegraphics[width=0.33\linewidth, height=3.4cm]{Images/calibration/ece_eye_load.png} &
\includegraphics[width=0.33\linewidth, height=3.4cm]{Images/calibration/ece_all_load.png} \\
(a) HRV & (b) Pupil & (c) HRV+Pupil \\
\end{tabular}
\caption{Reliability plots. HRV features are more reliable over pupillometry when light is changed. Pupillometry features combined with HRV features improve the reliability of the cognitive load estimate.}
\label{fig:reliability_plot}
\vspace{-0.4cm}
\end{figure*}
\end{comment}
%\subsection{Ablation Studies}
Several aspects of the data processing pipeline are often assumed in the literature and in the available libraries for HRV and pupil data. Here, we discuss the impact of some key processing steps on classifier design and performance. To this end, we conducted a handful of ablation studies. Polar H10 is used as the HRV device for the studies discussed in this section.
\subsection{Impact of window size for feature estimation}
The recommended window length for HRV features is 60 seconds \cite{60sec_window-Bernardes-2022}. When the window length is shorter, the computed features are typically less meaningful, especially the frequency domain. We compared the results from a 30-second window to a 60-second window, as shown in Table \ref{tab:30_60_sec_window}. The accuracy drops significantly for HRV features when the window size is 30 seconds (by 20.6 percentage points). As our feature importance analysis from the RF classifier highlights the significance of frequency-domain features, this drop in performance is expected without a precise estimation of those features. The pupillometry features show a smaller drop of 4.4\%, indicating they remain relatively reliable with a 30-second window. The overall performance drop in the multimodal settings is also more than 20\%. These results confirm the recommended window size of 60 seconds.
\begin{table}
\caption{Performance comparison between 30 second and 60 second window size for feature estimation.}
\centering\footnotesize
\begin{tabular}{ l c c}
\hline
\textbf{Sensors} & \textbf{30 Sec} & \textbf{60 Sec} \\
\hline
\hline
Pupil & 66.67$\pm$0.22 & 77.07$\pm$0.40 \\
HRV & 69.05$\pm$0.24 & 82.15$\pm$0.28\\
HRV+Pupil & 71.42$\pm$0.18 & 92.26$\pm$0.19\\
\hline
\end{tabular}
\label{tab:30_60_sec_window}
\end{table}
\subsection{Impact of noise filtering on pupillometry data}
To cope with the noise in the pupil diameter data, we used filtering and smoothing with a butter low-pass filter of order 5 and cut-off 4Hz. Butter low-pass filters are commonly used in the pre-processing step for noise removal from pupillometry \cite{butter_filter-Knapen-2016}. Filtering with a low-pass filter helps us to remove high-frequency noise in the data. Smoothing helps to reduce the impact of short-term fluctuations. Table \ref{tab:filter_impact} shows the importance of filtering in our pre-processing pipeline. Without this filtering, the accuracy drops by more than 13.3\% points. Thus a significant fraction of the noise is filtered by the butter low-pass filter.
\begin{table}
\caption{Low-pass filter impact on RF classification accuracy on Experiment 1 (binary case) using all light conditions.}
\centering\footnotesize
\begin{tabular}{ l c}
\hline
\textbf{Settings} & \textbf{Accuracy} \\ %\textbf{AUROC} \\
\hline
\hline
Without Butter Low-pass Filter & 57.81 \\ %& 59.38\\
With Butter Low-pass Filter & 77.05 \\ %65.62 \\
\hline
\end{tabular}
\label{tab:filter_impact}
\end{table}
\begin{comment}
\subsection{Impact of feature selection}
To reduce the number of feature extraction steps and avoid over-fitting our dataset, we evaluated the impact of removing several features from the model. Table \ref{tab:80_percent_results} shows the results of using feature selection for the RF classifier. A model trained only with the top 80\% of the most important features is used in the comparison. Based on this importance criterion, most of the eliminated features are from pupillometry. The results indicate significant performance degradation when both the training and test sets contain data only from light conditions. Thus, when no distribution shift occurs, pupillometry features are crucial for the RF classifier. Conversely, when trained on the light condition and tested on the dark condition, the loss in performance is negligible. This indicates that pupillometry features do not significantly contribute to overall performance when lighting conditions change. When the training set includes data from both light and dark conditions, performance increases in both test sets. However, the loss remains significant for the light condition.
\begin{table}
\caption{Comparison of the accuracy when using the top 80\% features and full features in the multimodal settings. The feature importance is provided by the RF classifier.}
\centering\footnotesize
\begin{tabular}{c c c c}
\hline
\textbf{Train} & \textbf{Test} & \textbf{80\% Feats.} & \textbf{Full Feats.} \\
\hline
\hline
Light & Light & 62.01 & 96.29 \\
Light & Dark & 77.68 & 50.77 \\
All & Light & 71.24 & 95.46 \\
All & Dark & 86.46 & 91.98 \\
All & All & 89.47 & 92.26 \\
\hline
\end{tabular}
\label{tab:80_percent_results}
\end{table}
\end{comment}
The aim of this research is to understand the influence of ambient light on pupillometry based CLE and how multimodal data acquisition helps to mitigate it. The multimodal in our case is realized using HRV alongwith pupillometry. We observed the robustness of pupillometry based CLE is low when light conditions are changed in the experimental settings. However, our study shows that by incorporating diverse lighting conditions in the training, the robustness to change in light conditions is improved. Combined with HRV, the multimodal data showed improved accuracy and robustness. HRV alone also has limitations due to its sensitivity to variations in the electromagnetic field and breathing cycle. The multimodal settings complements the limitations of any single modality, giving the best accuracy in all cases.
The HRV devices commonly used in cognitive load studies were chosen according to the experimental conditions. When controlled laboratory studies used high-fidelity HRV devices such as Biopac MP35, field application used fitness grade devices like Polar H10 or smartwatches. We compared both devices under the same task conditions by synchronously collecting data from Biopac MP35 and Polar H10. We then compared the CLE performance combining pupillometry with each device separately. The results show that using fitness grade Polar H10 is as good or sometimes better than clinical grade Biopac MP35, especially when the subjects can move during the data acquisition.
Our findings also reveal that neural network-based classifiers, particularly those using transformers, can achieve performance comparable to conventional random forest techniques. This is promising for future work, suggesting that advanced neural network architectures may offer robust solutions for cognitive load estimation, even in the presence of varying lighting conditions. The transformer-based classifier's ability to learn directly from raw signals without extensive feature engineering marks a significant advancement in the field.
%However, our study underscores the necessity of incorporating diverse lighting conditions in the training set to ensure robust cognitive load estimation. Despite the improvements achieved through multimodal approaches, the distribution shift due to changing lighting conditions remains a challenge. Therefore, future research should focus on developing more sophisticated methods to mitigate this issue, potentially exploring advanced machine learning techniques that can better handle such variations.
%In summary, our research demonstrates the efficacy of a multimodal approach in enhancing cognitive load estimation and provides valuable insights into the practical application of HRV and pupillometry in diverse environments. These findings pave the way for more reliable and accessible cognitive load assessment tools, with significant implications for various fields, including ergonomics, human-computer interaction, and adaptive automation systems. Future work will aim to refine these methods further, exploring additional modalities and more complex real-world scenarios to continue improving the accuracy and applicability of cognitive load measurement technologies.
\begin{thebibliography}{62}
\expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi
\providecommand{\url}[1]{\texttt{#1}}
\providecommand{\href}[2]{#2}
\providecommand{\path}[1]{#1}
\providecommand{\DOIprefix}{doi:}
\providecommand{\ArXivprefix}{arXiv:}
\providecommand{\URLprefix}{URL: }
\providecommand{\Pubmedprefix}{pmid:}
\providecommand{\doi}[1]{\href{http://dx.doi.org/#1}{\path{#1}}}
\providecommand{\Pubmed}[1]{\href{pmid:#1}{\path{#1}}}
\providecommand{\bibinfo}[2]{#2}
\ifx\xfnm\relax \def\xfnm[#1]{\unskip,\space#1}\fi
%Type = Incollection
\bibitem[{nas(1988)}]{nasa_tlx}
, \bibinfo{year}{1988}.
\newblock \bibinfo{title}{Development of nasa-tlx (task load index): Results of empirical and theoretical research}, in: \bibinfo{booktitle}{Human Mental Workload}. Advances in Psychology, pp. \bibinfo{pages}{139--183}.
\newblock \DOIprefix\doi{https://doi.org/10.1016/S0166-4115(08)62386-9}.
%Type = Article
\bibitem[{Carneiro~de Albuquerque et~al.(2020)Carneiro~de Albuquerque, Tiwari, Parent, Cassani, Gagnon, Lafond, Tremblay and Falk}]{Albuquerque2020}
\bibinfo{author}{Carneiro~de Albuquerque, I.M.}, \bibinfo{author}{Tiwari, A.}, \bibinfo{author}{Parent, M.}, \bibinfo{author}{Cassani, R.}, \bibinfo{author}{Gagnon, J.F.}, \bibinfo{author}{Lafond, D.}, \bibinfo{author}{Tremblay, S.}, \bibinfo{author}{Falk, T.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{Wauc: A multi-modal database for mental workload assessment under physical activity}.
\newblock \bibinfo{journal}{Frontiers in Neuroscience} \bibinfo{volume}{14}.
\newblock \DOIprefix\doi{10.3389/fnins.2020.549524}.
%Type = Article
\bibitem[{Alugubelli~N(2022)}]{wearable_ECG-Alugubelli-2022}
\bibinfo{author}{Alugubelli~N, Abuissa~H, R.A.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Wearable devices for remote monitoring of heart rate and heart rate variability-what we know and what is coming}.
\newblock \bibinfo{journal}{IEEE Sensors} \DOIprefix\doi{10.3390/s22228903}.
%Type = Article
\bibitem[{Aristizabal et~al.(2020)Aristizabal, Navegantes, Melo and Jr}]{hrv_covid-Aristizabal-2020}
\bibinfo{author}{Aristizabal, J.P.}, \bibinfo{author}{Navegantes, R.}, \bibinfo{author}{Melo, E.}, \bibinfo{author}{Jr, A.P.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{Use of heart rate variability biofeedback to reduce the psychological burden of frontline healthcare professionals against covid-19}.
\newblock \bibinfo{journal}{Frontiers in psychology} \DOIprefix\doi{10.3389/fpsyg.2020.572191}.
%Type = Inproceedings
\bibitem[{Bacula and LaViers(2018)}]{Bacula2018}
\bibinfo{author}{Bacula, A.}, \bibinfo{author}{LaViers, A.}, \bibinfo{year}{2018}.
\newblock \bibinfo{title}{Character recognition on a humanoid robotic platform via a laban movement analysis}, in: \bibinfo{booktitle}{Proceedings of the 5th International Conference on Movement and Computing}, \bibinfo{publisher}{Association for Computing Machinery}.
\newblock \DOIprefix\doi{10.1145/3212721.3212836}.
%Type = Inproceedings
\bibitem[{Baevski et~al.(2020)Baevski, Zhou, Mohamed and Auli}]{wave2vec-Baveski-2020}
\bibinfo{author}{Baevski, A.}, \bibinfo{author}{Zhou, H.}, \bibinfo{author}{Mohamed, A.}, \bibinfo{author}{Auli, M.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{wav2vec 2.0: a framework for self-supervised learning of speech representations}, in: \bibinfo{booktitle}{Proceedings of the 34th International Conference on Neural Information Processing Systems}.
%Type = Inproceedings
\bibitem[{Beh et~al.(2021)Beh, Wu and Wu}]{MAUS_dataset-Beh-2021}
\bibinfo{author}{Beh, W.K.}, \bibinfo{author}{Wu, Y.H.}, \bibinfo{author}{Wu, A.Y.}, \bibinfo{year}{2021}.
\newblock \bibinfo{title}{Maus: A dataset for mental workload assessmenton n-back task using wearable sensor}.
%Type = Inproceedings
\bibitem[{Belal et~al.(2024)Belal, Meethal, Romero, Pedersoli and Granger}]{domain_shift_prot-Meethal-2024}
\bibinfo{author}{Belal, A.}, \bibinfo{author}{Meethal, A.}, \bibinfo{author}{Romero, F.}, \bibinfo{author}{Pedersoli, M.}, \bibinfo{author}{Granger, E.}, \bibinfo{year}{2024}.
\newblock \bibinfo{title}{Multi-source domain adaptation for object detection with prototype-based mean teacher}, in: \bibinfo{booktitle}{2024 IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)}, pp. \bibinfo{pages}{1266--1275}.
\newblock \DOIprefix\doi{10.1109/WACV57701.2024.00131}.
%Type = Article
\bibitem[{Bengio et~al.(2013)Bengio, Courville and Vincent}]{feat_learning-Bengio-2013}
\bibinfo{author}{Bengio, Y.}, \bibinfo{author}{Courville, A.}, \bibinfo{author}{Vincent, P.}, \bibinfo{year}{2013}.
\newblock \bibinfo{title}{Representation learning: A review and new perspectives}.
\newblock \bibinfo{journal}{IEEE Trans. Pattern Anal. Mach. Intell.} , \bibinfo{pages}{1798–1828}\URLprefix \url{https://doi.org/10.1109/TPAMI.2013.50}, \DOIprefix\doi{10.1109/TPAMI.2013.50}.
%Type = Article
\bibitem[{Bernabei and Costantino(2024)}]{adaptive_automation-Margherita-2024}
\bibinfo{author}{Bernabei, M.}, \bibinfo{author}{Costantino, F.}, \bibinfo{year}{2024}.
\newblock \bibinfo{title}{Adaptive automation: Status of research and future challenges}.
\newblock \bibinfo{journal}{Robotics and Computer-Integrated Manufacturing} \DOIprefix\doi{https://doi.org/10.1016/j.rcim.2024.102724}.
%Type = Article
\bibitem[{Bernardes et~al.(2022)Bernardes, Couceiro, Medeiros, Henriques, Teixeira, Simões, Durães, Barbosa, Madeira and Carvalho}]{60sec_window-Bernardes-2022}
\bibinfo{author}{Bernardes, A.}, \bibinfo{author}{Couceiro, R.}, \bibinfo{author}{Medeiros, J.}, \bibinfo{author}{Henriques, J.}, \bibinfo{author}{Teixeira, C.}, \bibinfo{author}{Simões, M.}, \bibinfo{author}{Durães, J.}, \bibinfo{author}{Barbosa, R.}, \bibinfo{author}{Madeira, H.}, \bibinfo{author}{Carvalho, P.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{How reliable are ultra-short-term hrv measurements during cognitively demanding tasks?}
\newblock \bibinfo{journal}{Sensors} \bibinfo{volume}{22}.
\newblock \DOIprefix\doi{10.3390/s22176528}.
%Type = Article
\bibitem[{Caldiroli et~al.(2023)Caldiroli, Gasparini, Corchs, Mangiatordi, Garbo, Antonietti and Mantovani}]{mobile_measure-Caldiroli-2023}
\bibinfo{author}{Caldiroli, C.L.}, \bibinfo{author}{Gasparini, F.}, \bibinfo{author}{Corchs, S.}, \bibinfo{author}{Mangiatordi, A.}, \bibinfo{author}{Garbo, R.}, \bibinfo{author}{Antonietti, A.}, \bibinfo{author}{Mantovani, F.}, \bibinfo{year}{2023}.
\newblock \bibinfo{title}{Comparing online cognitive load on mobile versus pc-based devices}.
\newblock \bibinfo{journal}{Personal and ubiquitous computing} \DOIprefix\doi{https://doi.org/10.1007/s00779-022-01707-8}.
%Type = Article
\bibitem[{Coombs et~al.(2020)Coombs, Hislop, Taneva and Barnard}]{automation-Coombs-2019}
\bibinfo{author}{Coombs, C.}, \bibinfo{author}{Hislop, D.}, \bibinfo{author}{Taneva, S.K.}, \bibinfo{author}{Barnard, S.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{The strategic impacts of intelligent automation for knowledge and service work: An interdisciplinary review}.
\newblock \bibinfo{journal}{The Journal of Strategic Information Systems} \bibinfo{volume}{29}.
\newblock \DOIprefix\doi{https://doi.org/10.1016/j.jsis.2020.101600}.
%Type = Inproceedings
\bibitem[{Das et~al.(2014)Das, Chatterjee, Das, Sinharay and Sinha}]{low_cost_cogload-Rajat-2014}
\bibinfo{author}{Das, R.}, \bibinfo{author}{Chatterjee, D.}, \bibinfo{author}{Das, D.}, \bibinfo{author}{Sinharay, A.}, \bibinfo{author}{Sinha, A.}, \bibinfo{year}{2014}.
\newblock \bibinfo{title}{Cognitive load measurement - a methodology to compare low cost commercial eeg devices}, in: \bibinfo{booktitle}{International Conference on Advances in Computing, Communications and Informatics (ICACCI)}.
%Type = Article
\bibitem[{Duchowski(2018)}]{Duchowski2018}
\bibinfo{author}{Duchowski, A.T.}, \bibinfo{year}{2018}.
\newblock \bibinfo{title}{{The Index of Pupillary Activity. Measuring Cognitive Load vis-à-vis Task Difficulty with Pupil Oscillation}}.
\newblock \bibinfo{journal}{CHI '18: Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems} \DOIprefix\doi{10.1145/3173574.3173856}.
%Type = Inproceedings
\bibitem[{Duchowski et~al.(2020)Duchowski, Krejtz, Gehrer, Bafna and B\ae{}kgaard}]{Duchowski2020}
\bibinfo{author}{Duchowski, A.T.}, \bibinfo{author}{Krejtz, K.}, \bibinfo{author}{Gehrer, N.A.}, \bibinfo{author}{Bafna, T.}, \bibinfo{author}{B\ae{}kgaard, P.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{The low/high index of pupillary activity}, in: \bibinfo{booktitle}{Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems}, \bibinfo{publisher}{Association for Computing Machinery}. p. \bibinfo{pages}{1–12}.
\newblock \DOIprefix\doi{10.1145/3313831.3376394}.
%Type = Inproceedings
\bibitem[{Duchowski et~al.(2018)Duchowski, Krejtz, Krejtz, Biele, Niedzielska, Kiefer, Raubal and Giannopoulos}]{ipa-Duchowski-2018}
\bibinfo{author}{Duchowski, A.T.}, \bibinfo{author}{Krejtz, K.}, \bibinfo{author}{Krejtz, I.}, \bibinfo{author}{Biele, C.}, \bibinfo{author}{Niedzielska, A.}, \bibinfo{author}{Kiefer, P.}, \bibinfo{author}{Raubal, M.}, \bibinfo{author}{Giannopoulos, I.}, \bibinfo{year}{2018}.
\newblock \bibinfo{title}{The index of pupillary activity: Measuring cognitive load vis-\`{a}-vis task difficulty with pupil oscillation}, in: \bibinfo{booktitle}{Proceedings of the 2018 CHI Conference on Human Factors in Computing Systems}, p. \bibinfo{pages}{1–13}.
\newblock \DOIprefix\doi{10.1145/3173574.3173856}.
%Type = Inproceedings
\bibitem[{Duval et~al.(2023)Duval, Paas, Abdalwhab and St-Onge}]{duval2023}
\bibinfo{author}{Duval, A.}, \bibinfo{author}{Paas, A.}, \bibinfo{author}{Abdalwhab, A.}, \bibinfo{author}{St-Onge, D.}, \bibinfo{year}{2023}.
\newblock \bibinfo{title}{The eyes and hearts of uav pilots: observations of physiological responses in real-life scenarios}, in: \bibinfo{booktitle}{2023 32nd IEEE International Conference on Robot and Human Interactive Communication (RO-MAN)}, pp. \bibinfo{pages}{2352--2358}.
\newblock \DOIprefix\doi{10.1109/RO-MAN57019.2023.10309366}.
%Type = Article
\bibitem[{Eckert et~al.(2022)Eckert, Robotham, Habets and Rummukainen}]{Eckert2022}
\bibinfo{author}{Eckert, M.}, \bibinfo{author}{Robotham, T.}, \bibinfo{author}{Habets, E.A.P.}, \bibinfo{author}{Rummukainen, O.S.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Pupillary light reflex correction for robust pupillometry in virtual reality}.
\newblock \bibinfo{journal}{Proc. ACM Comput. Graph. Interact. Tech.} \DOIprefix\doi{10.1145/3530798}.
%Type = Article
\bibitem[{Fred~Shaffer(2017)}]{Shaffer2017}
\bibinfo{author}{Fred~Shaffer, J.P.G.}, \bibinfo{year}{2017}.
\newblock \bibinfo{title}{{An Overview of Heart Rate Variability Metrics and Norms}}.
\newblock \bibinfo{journal}{Front Public Health. 2017 Sep 28} \bibinfo{volume}{5:258}.
\newblock \DOIprefix\doi{10.3389/fpubh.2017.00258}.
%Type = Inproceedings
\bibitem[{Fridman et~al.(2018)Fridman, Reimer, Mehler and Freeman}]{cogload_wild-Fridman-2018}
\bibinfo{author}{Fridman, A.}, \bibinfo{author}{Reimer, B.}, \bibinfo{author}{Mehler, B.}, \bibinfo{author}{Freeman, W.T.}, \bibinfo{year}{2018}.
\newblock \bibinfo{title}{Cognitive load estimation in the wild}, in: \bibinfo{booktitle}{Proceedings of the CHI Conference on Human Factors in Computing Systems}.
%Type = Article
\bibitem[{H et~al.(2022)H, J, S and C.}]{Mitre2022}
\bibinfo{author}{H, M.H.}, \bibinfo{author}{J, S.R.}, \bibinfo{author}{S, N.M.}, \bibinfo{author}{C., L.A.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Classifying the difficulty levels of working memory tasks by using pupillary response} \DOIprefix\doi{https://doi.org/10.7717/peerj.12864}.
%Type = Article
\bibitem[{Haj et~al.(2022)Haj, Kapogiannis and Boutoleau-Bretonnière}]{Mohamad2022}
\bibinfo{author}{Haj, M.E.}, \bibinfo{author}{Kapogiannis, D.}, \bibinfo{author}{Boutoleau-Bretonnière, C.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Pupillometry as an index for cognitive processing in behavioral variant frontotemporal dementia: a series of case studies}.
\newblock \bibinfo{journal}{Neurocase} , \bibinfo{pages}{1--6}\DOIprefix\doi{10.1080/13554794.2022.2094809}.
%Type = Article
\bibitem[{He et~al.(2022)He, Wang, Khalil, Donmez, Qiao and Kumar}]{Dengbo2022}
\bibinfo{author}{He, D.}, \bibinfo{author}{Wang, Z.}, \bibinfo{author}{Khalil, E.B.}, \bibinfo{author}{Donmez, B.}, \bibinfo{author}{Qiao, G.}, \bibinfo{author}{Kumar, S.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Classification of driver cognitive load: Exploring the benefits of fusing eye-tracking and physiological measures}.
\newblock \bibinfo{journal}{Transportation Research Record} \DOIprefix\doi{10.1177/03611981221090937}.
%Type = Article
\bibitem[{Hilgarter et~al.(2021)Hilgarter, Schmid-Zalaudek, Csanády-Leitner, Mörtl, Rössler and Lackner}]{hrv_sympathovogal-Kathrin-2021}
\bibinfo{author}{Hilgarter, K.}, \bibinfo{author}{Schmid-Zalaudek, K.}, \bibinfo{author}{Csanády-Leitner, R.}, \bibinfo{author}{Mörtl, M.}, \bibinfo{author}{Rössler, A.}, \bibinfo{author}{Lackner, H.K.}, \bibinfo{year}{2021}.
\newblock \bibinfo{title}{Phasic heart rate variability and the association with cognitive performance: A cross-sectional study in a healthy population setting}.
\newblock \bibinfo{journal}{PLOS ONE} \bibinfo{volume}{16}, \bibinfo{pages}{1--19}.
\newblock \URLprefix \url{https://doi.org/10.1371/journal.pone.0246968}, \DOIprefix\doi{10.1371/journal.pone.0246968}.
%Type = Inproceedings
\bibitem[{Ioffe and Szegedy(2015)}]{batch_norm-Sergey-2015}
\bibinfo{author}{Ioffe, S.}, \bibinfo{author}{Szegedy, C.}, \bibinfo{year}{2015}.
\newblock \bibinfo{title}{Batch normalization: accelerating deep network training by reducing internal covariate shift}, p. \bibinfo{pages}{448–456}.
%Type = Article
\bibitem[{Kane et~al.(2007)Kane, Conway, Miura and Colflesh}]{n-back_modern-Kane-2007}
\bibinfo{author}{Kane, M.J.}, \bibinfo{author}{Conway, A.R.}, \bibinfo{author}{Miura, T.K.}, \bibinfo{author}{Colflesh, G.J.}, \bibinfo{year}{2007}.
\newblock \bibinfo{title}{Working memory, attention control, and the n-back task: a question of construct validity.}
\newblock \bibinfo{journal}{Journal of Experimental psychology: learning, memory, and cognition} , \bibinfo{pages}{615}.
%Type = Article
\bibitem[{Kavousi(2022)}]{Kavousi2022}
\bibinfo{author}{Kavousi, S.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{{Cognitive Load Estimation Using Heart Rate Variability Measures for Driver Monitoring Systems}}.
\newblock \bibinfo{journal}{Electronic Theses and Dissertations} .
%Type = Article
\bibitem[{Kirchner(1958)}]{n-back_main-Kirchner-1958}
\bibinfo{author}{Kirchner, W.K.}, \bibinfo{year}{1958}.
\newblock \bibinfo{title}{Age differences in short-term retention of rapidly changing information.}
\newblock \bibinfo{journal}{Journal of experimental psychology} , \bibinfo{pages}{352}.
%Type = Article
\bibitem[{Knapen et~al.(2016)Knapen, de~Gee, Brascamp, Nuiten, Hoppenbrouwers and Theeuwes}]{butter_filter-Knapen-2016}
\bibinfo{author}{Knapen, T.}, \bibinfo{author}{de~Gee, J.W.}, \bibinfo{author}{Brascamp, J.}, \bibinfo{author}{Nuiten, S.}, \bibinfo{author}{Hoppenbrouwers, S.}, \bibinfo{author}{Theeuwes, J.}, \bibinfo{year}{2016}.
\newblock \bibinfo{title}{Cognitive and ocular factors jointly determine pupil responses under equiluminance}.
\newblock \bibinfo{journal}{PLOS ONE} \bibinfo{volume}{11}, \bibinfo{pages}{1--13}.
\newblock \URLprefix \url{https://doi.org/10.1371/journal.pone.0155574}, \DOIprefix\doi{10.1371/journal.pone.0155574}.
%Type = Article
\bibitem[{Kramer(1991)}]{pupilary_response-Arthur-1991}
\bibinfo{author}{Kramer, A.F.}, \bibinfo{year}{1991}.
\newblock \bibinfo{title}{Physiological metrics of mental workload: A review of recent progress}.
\newblock \bibinfo{journal}{CRC Press} .
%Type = Article
\bibitem[{Krejtz et~al.(2018)Krejtz, Duchowski, Niedzielska, Biele and Krejtz}]{pd_gaze_cogload-Krejtz-2018}
\bibinfo{author}{Krejtz, K.}, \bibinfo{author}{Duchowski, A.T.}, \bibinfo{author}{Niedzielska, A.}, \bibinfo{author}{Biele, C.}, \bibinfo{author}{Krejtz, I.}, \bibinfo{year}{2018}.
\newblock \bibinfo{title}{Eye tracking cognitive load using pupil diameter and microsaccades with fixed gaze}.
\newblock \bibinfo{journal}{PLOS ONE} \bibinfo{volume}{13}, \bibinfo{pages}{1--23}.
\newblock \URLprefix \url{https://doi.org/10.1371/journal.pone.0203629}, \DOIprefix\doi{10.1371/journal.pone.0203629}.
%Type = Article
\bibitem[{Lenis et~al.(2017)Lenis, Kircher, L{\'{a}}zaro, Bail{\'{o}}n, Gil and Doessel}]{Lenis2017}
\bibinfo{author}{Lenis, G.}, \bibinfo{author}{Kircher, M.}, \bibinfo{author}{L{\'{a}}zaro, J.}, \bibinfo{author}{Bail{\'{o}}n, R.}, \bibinfo{author}{Gil, E.}, \bibinfo{author}{Doessel, O.}, \bibinfo{year}{2017}.
\newblock \bibinfo{title}{{Separating the effect of respiration on the heart rate variability using Granger's causality and linear filtering}}.
\newblock \bibinfo{journal}{Biomedical Signal Processing and Control} , \bibinfo{pages}{272--287}\DOIprefix\doi{10.1016/j.bspc.2016.07.014}.
%Type = Inproceedings
\bibitem[{Li et~al.(2022)Li, Dai, Ma, Liu, Chen, Wu, He, Kitani and Vajda}]{cross_domain_shift-Li-2022}
\bibinfo{author}{Li, Y.J.}, \bibinfo{author}{Dai, X.}, \bibinfo{author}{Ma, C.Y.}, \bibinfo{author}{Liu, Y.C.}, \bibinfo{author}{Chen, K.}, \bibinfo{author}{Wu, B.}, \bibinfo{author}{He, Z.}, \bibinfo{author}{Kitani, K.}, \bibinfo{author}{Vajda, P.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Cross-domain adaptive teacher for object detection}, in: \bibinfo{booktitle}{IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}.
%Type = Article
\bibitem[{Makowski et~al.(2021)Makowski, Pham, Lau, Brammer, Lespinasse, Pham, Schölzel and Chen}]{neurokit2-Makowski-2021}
\bibinfo{author}{Makowski, D.}, \bibinfo{author}{Pham, T.}, \bibinfo{author}{Lau, Z.J.}, \bibinfo{author}{Brammer, J.C.}, \bibinfo{author}{Lespinasse, F.}, \bibinfo{author}{Pham, H.}, \bibinfo{author}{Schölzel, C.}, \bibinfo{author}{Chen, S.H.A.}, \bibinfo{year}{2021}.
\newblock \bibinfo{title}{{NeuroKit}2: A python toolbox for neurophysiological signal processing}.
\newblock \bibinfo{journal}{Behavior Research Methods} \bibinfo{volume}{53}, \bibinfo{pages}{1689--1696}.
\newblock \DOIprefix\doi{10.3758/s13428-020-01516-y}.
%Type = Inproceedings
\bibitem[{Marshall(2002)}]{ica-Marshall-2002}
\bibinfo{author}{Marshall, S.}, \bibinfo{year}{2002}.
\newblock \bibinfo{title}{The index of cognitive activity: measuring cognitive workload}, in: \bibinfo{booktitle}{Proceedings of the IEEE 7th Conference on Human Factors and Power Plants}.
\newblock \DOIprefix\doi{10.1109/HFPP.2002.1042860}.
%Type = Article
\bibitem[{Math{\^o}t(2018)}]{PupillometryPP-Matht-2018}
\bibinfo{author}{Math{\^o}t, S.}, \bibinfo{year}{2018}.
\newblock \bibinfo{title}{Pupillometry: Psychology, physiology, and function}.
\newblock \bibinfo{journal}{Journal of Cognition} \bibinfo{volume}{1}.
%Type = Inproceedings
\bibitem[{Meethal et~al.(2022)Meethal, Pedersoli, Zhu, Romero and Granger}]{nn_sampling-Meethal-2022}
\bibinfo{author}{Meethal, A.}, \bibinfo{author}{Pedersoli, M.}, \bibinfo{author}{Zhu, Z.}, \bibinfo{author}{Romero, F.P.}, \bibinfo{author}{Granger, E.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Semi-weakly supervised object detection by sampling pseudo ground-truth boxes}, in: \bibinfo{booktitle}{International Joint Conference on Neural Networks (IJCNN)}.
%Type = Incollection
\bibitem[{Meshkati(1988)}]{hrv_old-MESHKATI-1988}
\bibinfo{author}{Meshkati, N.}, \bibinfo{year}{1988}.
\newblock \bibinfo{title}{Heart rate variability and mental workload assessment}, in: \bibinfo{booktitle}{Human Mental Workload}. \bibinfo{publisher}{North-Holland}. volume~\bibinfo{volume}{52} of \textit{\bibinfo{series}{Advances in Psychology}}, pp. \bibinfo{pages}{101--115}.
\newblock \DOIprefix\doi{https://doi.org/10.1016/S0166-4115(08)62384-5}.
%Type = Article
\bibitem[{Mishra et~al.(2020)Mishra, Pope, Lord, Lewia, Lowens, Caine, Sen, Halter and Kotz}]{polar_biopac_compare-Mishra-2020}
\bibinfo{author}{Mishra, V.}, \bibinfo{author}{Pope, G.}, \bibinfo{author}{Lord, S.}, \bibinfo{author}{Lewia, S.}, \bibinfo{author}{Lowens, B.}, \bibinfo{author}{Caine, K.}, \bibinfo{author}{Sen, S.}, \bibinfo{author}{Halter, R.}, \bibinfo{author}{Kotz, D.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{Continuous detection of physiological stress with commodity hardware}.
\newblock \bibinfo{journal}{ACM Trans. Comput. Healthcare} \bibinfo{volume}{1}.
\newblock \URLprefix \url{https://doi.org/10.1145/3361562}, \DOIprefix\doi{10.1145/3361562}.
%Type = Article
\bibitem[{Murata(1998)}]{Murata1998}
\bibinfo{author}{Murata, A.}, \bibinfo{year}{1998}.
\newblock \bibinfo{title}{{Evaluation of Mental Workload by Fluctuation Analysis of Pupil Area}}.
\newblock \bibinfo{journal}{Proceedings of the 20th Annual International Conference of the IEEE Engineering in Medicine and Biology Society} \bibinfo{volume}{20}.
\newblock \DOIprefix\doi{10.1109/IEMBS.1998.746146}.
%Type = Article
\bibitem[{Nanni et~al.(2017)Nanni, Ghidoni and Brahnam}]{hand_crafted_feats-Nanni-2017}
\bibinfo{author}{Nanni, L.}, \bibinfo{author}{Ghidoni, S.}, \bibinfo{author}{Brahnam, S.}, \bibinfo{year}{2017}.
\newblock \bibinfo{title}{Handcrafted vs. non-handcrafted features for computer vision classification}.
\newblock \bibinfo{journal}{Pattern Recognition} \bibinfo{volume}{71}, \bibinfo{pages}{158--172}.
\newblock \URLprefix \url{https://www.sciencedirect.com/science/article/pii/S0031320317302224}, \DOIprefix\doi{https://doi.org/10.1016/j.patcog.2017.05.025}.
%Type = Article
\bibitem[{Ngoc et~al.(2021)Ngoc, Lasa and Iriarte}]{human-centred-Ngoc-2021}
\bibinfo{author}{Ngoc, H.N.}, \bibinfo{author}{Lasa, G.}, \bibinfo{author}{Iriarte, I.}, \bibinfo{year}{2021}.
\newblock \bibinfo{title}{Human-centred design in industry 4.0: case study review and opportunities for future research}.
\newblock \bibinfo{journal}{Journal of Intelligent Manufacturing} , \bibinfo{pages}{35–76}\DOIprefix\doi{https://doi.org/10.1007/s10845-021-01796-x}.
%Type = Article
\bibitem[{Oppelt et~al.(2023)Oppelt, Foltyn, Deuschel, Lang, Holzer, Eskofier and Yang}]{multimodal_cle_biopac-Oppelt-2023}
\bibinfo{author}{Oppelt, M.P.}, \bibinfo{author}{Foltyn, A.}, \bibinfo{author}{Deuschel, J.}, \bibinfo{author}{Lang, N.R.}, \bibinfo{author}{Holzer, N.}, \bibinfo{author}{Eskofier, B.M.}, \bibinfo{author}{Yang, S.H.}, \bibinfo{year}{2023}.
\newblock \bibinfo{title}{Adabase: A multimodal dataset for cognitive load estimation}.
\newblock \bibinfo{journal}{Sensors} \bibinfo{volume}{23}.
\newblock \URLprefix \url{https://www.mdpi.com/1424-8220/23/1/340}, \DOIprefix\doi{10.3390/s23010340}.
%Type = Article
\bibitem[{Ouwehand et~al.(2021)Ouwehand, Kroef, Wong and Paas}]{multi_modal_CogLoad-Kim-2021}
\bibinfo{author}{Ouwehand, K.}, \bibinfo{author}{Kroef, A.v.d.}, \bibinfo{author}{Wong, J.}, \bibinfo{author}{Paas, F.}, \bibinfo{year}{2021}.
\newblock \bibinfo{title}{Measuring cognitive load: Are there more valid alternatives to likert rating scales?}
\newblock \bibinfo{journal}{Frontiers in Education} \bibinfo{volume}{6}.
\newblock \DOIprefix\doi{10.3389/feduc.2021.702616}.
%Type = Inproceedings
\bibitem[{Oviatt(2006)}]{interface_design-Sharon-2006}
\bibinfo{author}{Oviatt, S.}, \bibinfo{year}{2006}.
\newblock \bibinfo{title}{Human-centered design meets cognitive load theory: designing interfaces that help people think}, in: \bibinfo{booktitle}{Proceedings of the 14th ACM International Conference on Multimedia}, \bibinfo{publisher}{Association for Computing Machinery}. p. \bibinfo{pages}{871–880}.
\newblock \DOIprefix\doi{10.1145/1180639.1180831}.
%Type = Article
\bibitem[{Pakdaman~Naeini et~al.(2015)Pakdaman~Naeini, Cooper and Hauskrecht}]{ECE-Hauskrecht-2015}
\bibinfo{author}{Pakdaman~Naeini, M.}, \bibinfo{author}{Cooper, G.}, \bibinfo{author}{Hauskrecht, M.}, \bibinfo{year}{2015}.
\newblock \bibinfo{title}{Obtaining well calibrated probabilities using bayesian binning}.
\newblock \bibinfo{journal}{Proceedings of the AAAI Conference on Artificial Intelligence} \DOIprefix\doi{10.1609/aaai.v29i1.9602}.
%Type = Article
\bibitem[{Park et~al.(2022)Park, Goldberg, Al-Ballaa, Tayeb, Basurrah, Abahuje, Miccile, Pozner, Yule and Dias}]{hrv_edu-Park-2022}
\bibinfo{author}{Park, S.}, \bibinfo{author}{Goldberg, S.A.}, \bibinfo{author}{Al-Ballaa, A.}, \bibinfo{author}{Tayeb, B.}, \bibinfo{author}{Basurrah, M.}, \bibinfo{author}{Abahuje, E.}, \bibinfo{author}{Miccile, C.}, \bibinfo{author}{Pozner, C.N.}, \bibinfo{author}{Yule, S.}, \bibinfo{author}{Dias, R.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Objective measurement of learners’ cognitive load during simulation-based trauma team training: A pilot study}.
\newblock \bibinfo{journal}{Journal of Surgical Research} \bibinfo{volume}{279}, \bibinfo{pages}{361--367}.
\newblock \DOIprefix\doi{https://doi.org/10.1016/j.jss.2022.06.023}.
%Type = Article
\bibitem[{Parker and Grote(2022)}]{work_design-Parker-2022}
\bibinfo{author}{Parker, S.K.}, \bibinfo{author}{Grote, G.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Automation, algorithms, and beyond: Why work design matters more than ever in a digital world}.
\newblock \bibinfo{journal}{Applied Psychology} \bibinfo{volume}{71}, \bibinfo{pages}{1171--1204}.
\newblock \DOIprefix\doi{https://doi.org/10.1111/apps.12241}.
%Type = Inproceedings
\bibitem[{Rai(2022)}]{Rai2022}
\bibinfo{author}{Rai, Aman Anandand~Ahirwal, M.K.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Electroencephalogram-based cognitive load classification during mental arithmetic task}, in: \bibinfo{booktitle}{Edge Analytics}, \bibinfo{publisher}{Springer Singapore}. pp. \bibinfo{pages}{479--487}.
\newblock \DOIprefix\doi{https://doi.org/10.1007/978-981-19-0019-8_36}.
%Type = Article
\bibitem[{S et~al.(2016)S, J-B, van~der Linden and der Stigchel}]{hci_pupil-mathot-2016}
\bibinfo{author}{S, M.}, \bibinfo{author}{J-B, M.}, \bibinfo{author}{van~der Linden}, \bibinfo{author}{der Stigchel, V.}, \bibinfo{year}{2016}.
\newblock \bibinfo{title}{The mind-writing pupil: A human-computer interface based on decoding of covert attention through pupillometry}.
\newblock \bibinfo{journal}{PLoS ONE 11(2)} \DOIprefix\doi{10.1371/journal.pone.0148805}.
%Type = Article
\bibitem[{Schierhorst et~al.(2024)Schierhorst, Rick, Czerniak, Brandl, Mertens and Nitsch}]{light_compare-Nickolas-2024}
\bibinfo{author}{Schierhorst, N.J.}, \bibinfo{author}{Rick, V.B.}, \bibinfo{author}{Czerniak, J.N.}, \bibinfo{author}{Brandl, C.}, \bibinfo{author}{Mertens, A.}, \bibinfo{author}{Nitsch, V.}, \bibinfo{year}{2024}.
\newblock \bibinfo{title}{Comparison of two pupillometric measures as indicators of cognitive strain and under the influence of screen luminance}.
\newblock \bibinfo{journal}{Applied Ergonomics} \bibinfo{volume}{117}.
\newblock \DOIprefix\doi{https://doi.org/10.1016/j.apergo.2024.104242}.
%Type = Article
\bibitem[{Shaffer and Ginsberg(2017a)}]{hrv_metrics-Shaffer-2017}
\bibinfo{author}{Shaffer, F.}, \bibinfo{author}{Ginsberg, J.P.}, \bibinfo{year}{2017}a.
\newblock \bibinfo{title}{An overview of heart rate variability metrics and norms}.
\newblock \bibinfo{journal}{Frontiers in Public Health} \bibinfo{volume}{5}.
%Type = Article
\bibitem[{Shaffer and Ginsberg(2017b)}]{hrv_features-Fred-2017}
\bibinfo{author}{Shaffer, F.}, \bibinfo{author}{Ginsberg, J.P.}, \bibinfo{year}{2017}b.
\newblock \bibinfo{title}{An overview of heart rate variability metrics and norms}.
\newblock \bibinfo{journal}{Frontiers in Public Health} \bibinfo{volume}{5}.
\newblock \URLprefix \url{https://www.frontiersin.org/journals/public-health/articles/10.3389/fpubh.2017.00258}, \DOIprefix\doi{10.3389/fpubh.2017.00258}.
%Type = Article
\bibitem[{Solhjoo et~al.(2019)Solhjoo, Haigney, McBee, van Merrienboer, Schuwirth, Artino, Battista, Ratcliffe, Lee and Durning}]{Solhjoo2019}
\bibinfo{author}{Solhjoo, S.}, \bibinfo{author}{Haigney, M.C.}, \bibinfo{author}{McBee, E.}, \bibinfo{author}{van Merrienboer, J.J.}, \bibinfo{author}{Schuwirth, L.}, \bibinfo{author}{Artino, A.R.}, \bibinfo{author}{Battista, A.}, \bibinfo{author}{Ratcliffe, T.A.}, \bibinfo{author}{Lee, H.D.}, \bibinfo{author}{Durning, S.J.}, \bibinfo{year}{2019}.
\newblock \bibinfo{title}{{Heart Rate and Heart Rate Variability Correlate with Clinical Reasoning Performance and Self-Reported Measures of Cognitive Load}}.
\newblock \bibinfo{journal}{Scientific Reports} \bibinfo{volume}{9}, \bibinfo{pages}{1--9}.
\newblock \URLprefix \url{http://dx.doi.org/10.1038/s41598-019-50280-3}, \DOIprefix\doi{10.1038/s41598-019-50280-3}.
%Type = Article
\bibitem[{Souchet et~al.(2022)Souchet, Philippe, Lourdeaux and Leroy}]{Souchet2022}
\bibinfo{author}{Souchet, A.D.}, \bibinfo{author}{Philippe, S.}, \bibinfo{author}{Lourdeaux, D.}, \bibinfo{author}{Leroy, L.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Measuring visual fatigue and cognitive load via eye tracking while learning with virtual reality head-mounted displays: A review}.
\newblock \bibinfo{journal}{International Journal of Human–Computer Interaction} \bibinfo{volume}{38}, \bibinfo{pages}{801--824}.
\newblock \DOIprefix\doi{10.1080/10447318.2021.1976509}.
%Type = Article
\bibitem[{Tortora et~al.(2022)Tortora, Cordelli and Soda}]{pytrack-Tortora-2022}
\bibinfo{author}{Tortora, M.}, \bibinfo{author}{Cordelli, E.}, \bibinfo{author}{Soda, P.}, \bibinfo{year}{2022}.
\newblock \bibinfo{title}{Pytrack: A map-matching-based python toolbox for vehicle trajectory reconstruction}.
\newblock \bibinfo{journal}{IEEE Access} \bibinfo{volume}{10}.
\newblock \DOIprefix\doi{10.1109/ACCESS.2022.3216565}.
%Type = Article
\bibitem[{Vanneste et~al.(2020)Vanneste, Raes, Morton, Bombeke, Acker, Larmuseau, Depaepe and den Noortgate}]{multimodal_cogload-Vanneste-2020}
\bibinfo{author}{Vanneste, P.}, \bibinfo{author}{Raes, A.}, \bibinfo{author}{Morton, J.}, \bibinfo{author}{Bombeke, K.}, \bibinfo{author}{Acker, B.B.V.}, \bibinfo{author}{Larmuseau, C.}, \bibinfo{author}{Depaepe, F.}, \bibinfo{author}{den Noortgate, W.V.}, \bibinfo{year}{2020}.
\newblock \bibinfo{title}{Towards measuring cognitive load through multimodal physiological data}.
\newblock \bibinfo{journal}{Cognition, Technology \& Work} \bibinfo{volume}{23}, \bibinfo{pages}{567--585}.
%Type = Inproceedings
\bibitem[{Vaswani et~al.(2017)Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez, Kaiser and Polosukhin}]{transformer-Vaswani-2017}
\bibinfo{author}{Vaswani, A.}, \bibinfo{author}{Shazeer, N.}, \bibinfo{author}{Parmar, N.}, \bibinfo{author}{Uszkoreit, J.}, \bibinfo{author}{Jones, L.}, \bibinfo{author}{Gomez, A.N.}, \bibinfo{author}{Kaiser, L.u.}, \bibinfo{author}{Polosukhin, I.}, \bibinfo{year}{2017}.
\newblock \bibinfo{title}{Attention is all you need}, in: \bibinfo{booktitle}{Advances in Neural Information Processing Systems}, \bibinfo{publisher}{Curran Associates, Inc.}
%Type = Inproceedings
\bibitem[{Weber et~al.(2021)Weber, Rupprecht, Wiesen, Hamann and Ebert}]{pupil_cogload-Weber-2021}
\bibinfo{author}{Weber, P.}, \bibinfo{author}{Rupprecht, F.}, \bibinfo{author}{Wiesen, S.}, \bibinfo{author}{Hamann, B.}, \bibinfo{author}{Ebert, A.}, \bibinfo{year}{2021}.
\newblock \bibinfo{title}{Assessing cognitive load via pupillometry}, in: \bibinfo{booktitle}{Advances in Artificial Intelligence and Applied Cognitive Computing}, \bibinfo{publisher}{Springer International Publishing}. pp. \bibinfo{pages}{1087--1096}.
%Type = Article
\bibitem[{Wozniak and Zahabi(2024)}]{multimodal_filed-David-2024}
\bibinfo{author}{Wozniak, D.}, \bibinfo{author}{Zahabi, M.}, \bibinfo{year}{2024}.
\newblock \bibinfo{title}{Cognitive workload classification of law enforcement officers using physiological responses}.
\newblock \bibinfo{journal}{Applied Ergonomics} \bibinfo{volume}{119}, \bibinfo{pages}{104305}.
\newblock \URLprefix \url{https://www.sciencedirect.com/science/article/pii/S0003687024000826}, \DOIprefix\doi{https://doi.org/10.1016/j.apergo.2024.104305}.
%Type = Inproceedings
\bibitem[{Xu et~al.(2019)Xu, Sun, Zhang, Zhao and Lin}]{layernorm-Xu-S2019}
\bibinfo{author}{Xu, J.}, \bibinfo{author}{Sun, X.}, \bibinfo{author}{Zhang, Z.}, \bibinfo{author}{Zhao, G.}, \bibinfo{author}{Lin, J.}, \bibinfo{year}{2019}.
\newblock \bibinfo{title}{Understanding and improving layer normalization}, in: \bibinfo{booktitle}{Advances in Neural Information Processing Systems}.
\end{thebibliography}