LeanTTA: A Backpropagation-Free and Stateless Approach to Quantized Test-Time Adaptation on Edge Devices
Extracted source text · arXiv 2503.15889
\documentclass{article}
% Recommended, but optional, packages for figures and better typesetting:
\usepackage{microtype}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{booktabs} % for professional tables
\usepackage{multirow}
\usepackage{xspace}
\usepackage{wrapfig}
\usepackage{xcolor,soul}
\usepackage[linesnumbered,ruled,vlined]{algorithm2e}
\newcommand\mycommfont[1]{\footnotesize\ttfamily\textcolor{blue}{#1}}
\SetCommentSty{mycommfont}
\SetKwInput{KwInput}{Input} % Set the Input
\SetKwInput{KwOutput}{Output} % set the Output
\SetKwInput{KwRequire}{Require} % set the Output
% hyperref makes hyperlinks in the resulting PDF.
% If your build breaks (sometimes temporarily if a hyperlink spans a page)
% please comment out the following usepackage line and replace
% \usepackage{icml2025} with \usepackage[nohyperref]{icml2025} above.
\usepackage{hyperref}
\usepackage{minitoc}
% Attempt to make hyperref and algorithmic work together better:
\newcommand{\theHalgorithm}{\arabic{algorithm}}
\newcommand{\systemName}{LeanTTA\xspace}
% Use the following line for the initial blind version submitted for review:
% \usepackage{icml2025}
% If accepted, instead use the following line for the camera-ready submission:
\usepackage[accepted]{icml2025}
% For theorems and such
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
% if you use cleveref..
\usepackage[capitalize,noabbrev]{cleveref}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% THEOREMS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
% Todonotes is useful during development; simply uncomment the next line
% and comment out the line below the next line to turn off comments
%\usepackage[disable,textsize=tiny]{todonotes}
\usepackage[textsize=tiny]{todonotes}
% The \icmltitle you define below is probably too long as a header.
% Therefore, a short form for the running title is supplied here:
\icmltitlerunning{\systemName: A Backpropagation-Free and Stateless Approach to Quantized Test-Time Adaptation on Edge Devices}
\begin{document}
\twocolumn[
\icmltitle{\systemName: A Backpropagation-Free and Stateless Approach to Quantized Test-Time Adaptation on Edge Devices}
% It is OKAY to include author information, even for blind
% submissions: the style file will automatically remove it for you
% unless you've provided the [accepted] option to the icml2025
% package.
% List of affiliations: The first argument should be a (short)
% identifier you will use later to specify author affiliations
% Academic affiliations should list Department, University, City, Region, Country
% Industry affiliations should list Company, City, Region, Country
% You can specify symbols, otherwise they are numbered in order.
% Ideally, you should not use this facility. Affiliations will be numbered
% in order of appearance and this is the preferred way.
\begin{icmlauthorlist}
\icmlauthor{Cynthia Dong}{uw, cam}
\icmlauthor{Hong Jia}{mel, cam}
\icmlauthor{Young D. Kwon}{sam, cam}
\icmlauthor{Georgios Rizos}{cam}
\icmlauthor{Cecilia Mascolo}{cam}
\end{icmlauthorlist}
\icmlaffiliation{uw}{Paul G. Allen School of Computer Science and Engineering, University of Washington, Seattle, United States of America}
\icmlaffiliation{cam}{Department of Computer Science and Technology, University of Cambridge, Cambridge, United Kingdom}
\icmlaffiliation{mel}{School of Computing and Information Systems, University of Melbourne, Melbourne, Australia}
\icmlaffiliation{sam}{Samsung AI Center, Cambridge, United Kingdom}
\icmlcorrespondingauthor{Cecilia Mascolo}{cm54@cam.ac.uk}
% You may provide any keywords that you
% find helpful for describing your paper; these are used to populate
% the "keywords" metadata in the PDF but will not be shown in the document
\icmlkeywords{Machine Learning, ICML}
\vskip 0.3in
]
% this must go after the closing bracket ] following \twocolumn[ ...
% This command actually creates the footnote in the first column
% listing the affiliations and the copyright notice.
% The command takes one argument, which is text to display at the start of the footnote.
% The \icmlEqualContribution command is standard text for equal contribution.
% Remove it (just {}) if you do not need this facility.
%\printAffiliationsAndNotice{} % leave blank if no need to mention equal contribution
% \printAffiliationsAndNotice{\icmlEqualContribution} % otherwise use the standard text.
\input{sections/0abstract}
\input{sections/1intro}
% \input{sections/2background}
\input{sections/3relatedworks}
\input{sections/4methods}
% \input{sections/5experiments}
\input{sections/6results}
\input{sections/7conclusion}
% \section*{Impact Statement}
% Authors are \textbf{required} to include a statement of the potential
% broader impact of their work, including its ethical aspects and future
% societal consequences. This statement should be in an unnumbered
% section at the end of the paper (co-located with Acknowledgements --
% the two may appear in either order, but both must be before References),
% and does not count toward the paper page limit. In many cases, where
% the ethical impacts and expected societal implications are those that
% are well established when advancing the field of Machine Learning,
% substantial discussion is not required, and a simple statement such
% as the following will suffice:
% ``This paper presents work whose goal is to advance the field of
% Machine Learning. There are many potential societal consequences
% of our work, none which we feel must be specifically highlighted here.''
% The above statement can be used verbatim in such cases, but we
% encourage authors to think about whether there is content which does
% warrant further discussion, as this statement will be apparent if the
% paper is later flagged for ethics review.
\bibliography{main}
\bibliographystyle{icml2025}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\appendix
\onecolumn
\input{sections/8supplemental}
\end{document}
% This document was modified from the file originally made available by
% Pat Langley and Andrea Danyluk for ICML-2K. This version was created
% by Iain Murray in 2018, and modified by Alexandre Bouchard in
% 2019 and 2021 and by Csaba Szepesvari, Gang Niu and Sivan Sabato in 2022.
% Modified again in 2023 and 2024 by Sivan Sabato and Jonathan Scarlett.
% Previous contributors include Dan Roy, Lise Getoor and Tobias
% Scheffer, which was slightly modified from the 2010 version by
% Thorsten Joachims & Johannes Fuernkranz, slightly modified from the
% 2009 version by Kiri Wagstaff and Sam Roweis's 2008 version, which is
% slightly modified from Prasad Tadepalli's 2007 version which is a
% lightly changed version of the previous year's version by Andrew
% Moore, which was in turn edited from those of Kristian Kersting and
% Codrina Lauth. Alex Smola contributed to the algorithmic style files.
\begin{abstract}
While there are many advantages to deploying machine learning models on edge devices, the resource constraints of mobile platforms, the dynamic nature of the environment, and differences between the distribution of training versus in-the-wild data make such deployments challenging. Current test-time adaptation methods are often \textit{memory-intensive} and not designed to be quantization-compatible or deployed on low-resource devices. To address these challenges, we present \systemName, a novel backpropagation-free and stateless framework for quantized test-time adaptation tailored to edge devices. Our approach minimizes computational costs by dynamically updating normalization statistics without backpropagation, which frees \systemName from the common pitfall of relying on large batches and historical data, making our method robust to realistic deployment scenarios. Our approach is the first to enable further computational gains by combining partial adaptation with quantized module fusion. We validate our framework across sensor modalities, demonstrating significant improvements over state-of-the-art TTA methods, including a 15.7\% error reduction, peak memory usage of only 11.2MB for ResNet18, and fast adaptation within an order-of-magnitude of normal inference speeds on-device. \systemName provides a robust solution for achieving the right trade offs between accuracy and system efficiency in edge deployments, addressing the unique challenges posed by limited data and varied operational conditions.
\end{abstract}
\section{Introduction}
Performing inference directly on edge devices instead of relaying information to server-hosted models brings security, reliability, and accessibility benefits~\cite{qayyum20_mlsec,mollah17_mobsec, guo18_rural, dai19_skin}. However, deploying machine learning models at the edge is challenging: mobile devices are resource-poor in memory and energy, and often lack the scale and power of cloud-hosted GPU acceleration~\cite{dhar21_ondevice}.
% To address these challenges, edge models are trained in full-precision, then compressed for deployment — often using quantization~\cite{gholami21_quantrev}.
By nature, on-device models operate with varied sensors in a range of environments, processing data from distributions not seen during training~\cite{zhou22_device}. A potential mobile deployment of deep learning is bioacoustics~\cite{zualkernan2021aiot}, specifically citizen science, where people capture images or audio of flora and fauna with their devices~\cite{stowell18_birds}. Another is on-device diagnostics of cough or heart sounds collected using mobile phones ~\cite{han2022sounds, shariat22_phoneheart}. In such scenarios, \textit{only a single, critical data point is available} — one rare bird video in a certain forest, or one measurement from a specific patient; distributions may shift \textit{abruptly}. Models trained on data gathered in a lab or clinic may not anticipate varied conditions in the wild, potentially resulting in high error rates~\cite{koh20_wilds}. Even large deep learning models can significantly drop in accuracy under previously unseen distributions~\cite{liang23_litrev};
edge-deployed models, compressed to save memory and compute, are even less capable of generalization~\cite{zhou22_device}.
Test-time adaptation (TTA), as illustrated in Figure~\ref{fig:tta}, seeks to improve model accuracy under distribution shift \textit{without access to the original source or labeled target data}~\cite{liang23_litrev,koh20_wilds}. Changes in distribution can be broad and unpredictable ~\cite{liang23_litrev,koh20_wilds}.
Significant challenges stand in the way of on-device deployment of current TTA methods. \textit{Backpropagation-based optimization} of normalization layers~\cite{wang21_tent,niu23_sar,wang22_cotta,niu22_eata} severely strains resource-constrained devices, requiring a nontrivial increase in memory and power over backpropagation-free methods~\cite{xu_mandheling_2022,lin24_tt,huang_elastictrainer_mobisys23,kwon2024tinytrain}.
Data availability at the edge further complicates on-device TTA: many TTA methods perform very poorly under temporal correlation~\cite{wang22_cotta,gong23_note} or limited data~\cite{benz21_rstats}. Not only does memory constrain batch size, but in the case of low-frequency inference where data is captured intermittently (such as citizen science apps or health signals analysis)~\cite{stowell18_birds, dong24_intent}, \textit{there may only be a single data point per distribution shift, and few data points overall}. Most TTA methods will \textit{induce model collapse} with the small batch sizes and abruptly changing domains characteristic to mobile deployments, and cannot adapt to data instances collected from different domains individually.
% TTA literature also rarely considers usage on modalities other than images: edge deployments record and analyze audio, or other signals — different modalities may be susceptible to different types of shift, and require model architectures not studied in existing TTA literature.
% Leveraging both TTA and the acceleration provided by combining module fusion with quantization is underexplored. Niu et al. developed FOA~\cite{niu24_foa}, a vision transformer-specific TTA method evaluated on quantized models. However, FOA does not generalize across model architectures and requires either large batch sizes or additional storage and processing of historical data. In addition, due to their high memory and compute requirements, ViTs are less suited for highly constrained edge deployments than convolutional neural networks (CNNs) or ViT-CNN hybrids like MobileViT~\cite{mehta22_mobilevit}.
To address these gaps, we propose a novel method of backpropagation-free, stateless, and quantized TTA for edge devices: \systemName, which adapts by dynamically updating the quantized model's normalization statistics. Specifically, (1) we propose a backpropogation-free TTA module capable of adapting to each unlabeled data point regardless of prior data availability or continuity. (2) So that the module adjusts to different distribution shifts without over-reliance on incoming information, we propose a stateless statistics update mechanism based on the per-sample divergence between train-time statistics and stabilized incoming statistics. (3) Lastly, we propose layer-wise updates with adaptive fusion, a novel combination which enables rapid adaptation with quantized models and provides novel insights into how layer depth impacts the efficacy of adaptation.
By working with the ubiquitous batch normalization layer~\cite{ioffe15_bn}, present in most edge models, such as MobileViT~\cite{mehta22_mobilevit},EfficientNet~\cite{tan20_efficientnet}, and ShuffleNet~\cite{zhang17_shufflenet}, our work remains generalizable in the edge model space.
We evaluated \systemName on a typical edge device, over different sensor modalities and datasets, and conducted extensive experiments. Figure~\ref{fig:resnet_scatter} shows that of the assessed systems, \systemName alone \textit{consistently} reduces distribution shift error with low latency and memory consumption on par with inference — for both quantized and unquantized models. \systemName provides a robust and efficient solution, which achieves high system efficiency on edge devices, addressing the unique challenges posed by limited system resources, scarce data, and diverse operational conditions.
% \cm{the fact that it is a short subsection does not convince me that it is not worthwhile: can we change this into a more convicing argument of why it's not enough?}.
\begin{figure}
\centering
\includegraphics[width=0.99\linewidth]{figs/resnet_scatter.pdf}
\vspace{-1em}
\caption{\systemName comparison against SOTA TTA methods under abruptly changing distribution shifts. Circle size is scaled to the maximum memory required for adaptation. Labels indicate batch size. For methods where models collapsed at batch size one, results were omitted. Results below the baseline indicate that adaptation worsened accuracy. The data are comprised of shuffled images, sampled across all distributions — the method is introduced in ~\ref{app:subsec:datasets}.}
\label{fig:resnet_scatter}
% \vspace{-2em}
\end{figure}
\section{Related Work}
\subsection{Conventional TTA}
The restrictive data assumptions inherent in Tent undermine its robustness in practical applications. Later research efforts consider several of these limitations. CoTTA~\cite{wang22_cotta} tackles catastrophic forgetting and continuous shifts, while NOTE~\cite{gong23_note} and RoTTA~\cite{yuan23_rotta} both reserve batches of data to simulate independent and identically distributed (i.i.d.) data even under non-i.i.d. conditions. While these methods improve on Tent's robustness, they rely on entropy-loss backpropagation, meaning their accuracy improvements may be dependent on the optimizer and hyperparameters used~\cite{zhao23_pitfalls}; entropy loss may also lead to model collapse~\cite{niu24_foa}. Being backpropagation-dependent means that they are also, like Tent, much more memory-intensive than standard inference.
\subsection{Memory-efficient TTA}
Some memory-efficient TTA methods attempt to reduce the depth or frequency of backpropagation. EATA~\cite{niu22_eata} and SAR~\cite{niu23_sar} backpropagate selectively to reduce the computation requirements and avoid noisy gradients. MECTA~\cite{hong23_mecta} takes a different approach to improving memory efficiency, by combining pruning and selective training. However, reliance on backpropagation means that, for all aforementioned methods, their worst-case computational cost is still on-par with Tent, and may involve complex implementations if quantized and moved on-device.
% SAR, in particular, lowers its peak memory footprint by relying on different normalization layers, such as Group Norm and Layer Norm, that operate well with fewer batch sizes.
% \vspace{-2em}
\subsection{Backpropagation-free TTA}
% Inference-only adaptation methods comprise a divergent branch of TTA methods. Being backpropagation-free, their memory and energy consumption lie on the same order of magnitude as inference. Li et al. propose adapting by directly updating the frozen source statistics in the normalization layers of a model with statistics calculated over the entirety of the target data~\cite{li16_adabn}; more realistically, Benz et al. use only mini-batch target statistics~\cite{benz21_rstats}, though their method fails as batch sizes approach one. Schneider et al. present a variation on Li et al.'s method, reducing the target data needed by instead using a weighted average of source and target statistics, based on a tuned hyperparameter~\cite{schneider20_navg}. Further reducing the target data needed down to a single image, SITA~\cite{khurana21_sita} and InTEnt~\cite{dong24_intent}, respectively, use feature augmentation and entropy-weighted integration across a hyperparameter space. Both require multiple passes through the entire model per-adaptation for hyperparameter selection. SITA also relies on vision-based augmentations, leaving its applicability to multiple modalities unclear~\cite{khurana21_sita}. FOA~\cite{niu24_foa} achieves forward-only adaptation on transformer architectures while processing single datapoints, and tests with quantization, but may be better suited to static shifts, as FOA requires at least 32 datapoints of warm-up per distribution shift.
Inference-only adaptation methods comprise a divergent branch of TTA methods. Being backpropagation-free, their memory and energy consumption lie on the same order of magnitude as inference.
\citet{li16_adabn} propose directly updating the frozen source statistics in normalization layers of a model with statistics calculated over the entirety of the target data~\cite{li16_adabn}. A more practical approach utilizes only mini-batch target statistics~\cite{benz21_rstats}, though this method fails as batch sizes approach one. An alternative method extends the original approach by reducing the target data required through the use of a weighted average of source and target statistics~\cite{schneider20_navg}. Further reducing the target data needed down to a single image, SITA~\cite{khurana21_sita} and InTEnt~\cite{dong24_intent}, respectively, use feature augmentation and entropy-weighted integration across a hyperparameter space, though both these mothods require multiple passes through the entire model per-adaptation. FOA~\cite{niu24_foa} enables forward-only adaptation, but is ViT-specific and still requires large batch sizes or cached historical data. RealisticTTA takes a different approach involving two forward passes, trading off the hyperparameter search for a long warm-up period of roughly 1,000 test images for batch size one~\cite{su24_tema}.
% SITA also relies on vision-based augmentations, leaving its applicability to multiple modalities unclear~\cite{khurana21_sita}.
% In comparison, our statistics stabilization strategy eliminates the need for a lengthy warm-up period and prevents model collapse, and changes dynamically to adjust to different distribution shifts.
% In short, the trade-offs between these methods highlight the challenge of small batch adaptation — statistics from a single datapoint can be noisy and unrepresentative.
% \subsection{Novelty}
Compared with existing baselines, our proposed method has several advantages. Like other methods, \systemName is backpropagation-free; yet, it eschews ensembles or augmentations, and adapts dynamically per data point, even if points are sampled from vastly different domains — independent of batch size or previous data. Our statistics stabilization strategy eliminates the need for a long warmup period, and our per-data-point reset prevents any chance of long-term model collapse. Finally, \systemName improves accuracy alongside computational efficiency by proposing adaptation on a subset of the model and fusing the non-adaptive layers.
\section{The Method}
\subsection{Problem Description}
We formulate the test-time adaptation problem as follows: a given model $f_{\theta}(\textbf{x})$ is trained over a source dataset with inputs and labels $x_s, y_s$, sampled from the joint probability distribution $P_{source}(x_s, y_s)$. At test-time, the model $f_{\theta}(\textbf{x})$ with parameters $\theta$ encounters inputs $x_t$, with unknown labels $y_t$, where $x_t, y_t$ are distributed over $P_{target}(x_t, y_t)$, and $P_{target}(x_t, y_t)$ may have diverged from $P_{source}(x_s, y_s)$. The model $f_{\theta}(\textbf{x})$ must adapt to any differences between $P_{source}$ and $P_{target}$ with access only to $x_t$. For realistic on-device deployment on edge devices, our method must also function on models that have been compressed from floating point (32-bit) to integer precision (8-bit).
Ideally, TTA for edge devices should adapt to scarce data, require limited hyperparameter tuning, and function well across different modalities. In addition, given that data encountered in the wild are unlikely to perfectly match the distribution of finite training data, TTA should be widely applied~\cite{koh20_wilds}. As such, the method must be lightweight, quantization-compatible, low-latency, and memory-efficient, and should consume little power relative to normal inference.
\subsection{\systemName}\label{subsec:method}
\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{figs/method.pdf}
% \vspace{-1em}
\caption{Proposed updated normalization layer. For each intermediate activation, target statistics are recorded and stabilized using source statistics. The Mahalanobis distance, $d$, estimates how far out-of-distribution the stabilized statistics have moved at each layer. The source and target statistics are then recombined according to the scaled distance, and used for normalization.}
% \chj{why pi why lambda...replotting it ing}/chj{fixed}
\label{fig:method}
\vspace{-1em}
\end{figure}
The proposed method \systemName is illustrated in Figure~\ref{fig:method}. Given a pretrained model, the system updates and adapts normalization layers to target domain statistics while maintaining its performance on the source domain. This process consists of four key steps: (1) extract and stabilize incoming statistics, (2) calculate sample divergence, (3) balance source and target statistics, and (4) normalize and \textit{reset} the model to adapt to the next sample without any chance of model collapse.
\subsubsection{\textbf{Extract Statistical Information}}
Our method is designed to adapt to single data instances under unknown distribution shifts, enhancing performance in both continuous and sharply-changing domains. The process begins by extracting statistical information from the input data \( x_t^l \). We calculate the mean (\(\mu_t\)) and variance (\(\sigma_t^2\)) per-feature as follows:
% \chj{make it dense}
% \vspace{-0.5em}
\begin{equation}
\mu_t \leftarrow \frac{1}{H \times W} \sum x_t^l
\end{equation}
% \vspace{-1em}
\begin{equation}
\sigma_t^2 \leftarrow Var(x_t^l)
\end{equation}
\vspace{-1em}
Here, $H$ and $W$ represent the height and width of the input data, respectively. These target statistics are combined with source statistics $(\mu_s, \sigma_s^2)$ using a weighted average, parameterized by $\tau$, to stabilize the target statistics:
\vspace{-1em}
\begin{equation}
\mu_b \leftarrow \tau \mu_s + (1 - \tau) \mu_t
\end{equation}
\vspace{-1em}
\begin{equation}
\sigma_b^2 \leftarrow \tau \sigma_s^2 + (1 - \tau) \sigma_t^2
\end{equation}
\vspace{-1em}
Directly replacing source statistics with target statistics from a single data point — highly variable, as shown by the solid blue line in Figure~\ref{fig:run_v_inst} — lowers accuracy precipitously. Here, we first stabilize incoming statistics with the original training statistics, simulating momentum using $\tau$ without relying on prior data (Figure~\ref{fig:run_v_inst}, pink dashed line). $\tau$ is a parameter ranging between 0 and 1 which determines the weight assigned to the source ($\tau$) and the target statistics ($1-\tau$). This balance allows for a nuanced stabilization depending on $\tau$.
% Additionally, the equation includes an adjustment term, $\tau(1 - \tau)(\mu_s - \mu_t)^2$, which accounts for the discrepancy between the source mean $\mu_s$ and the target mean $\mu_t$. This adjustment is crucial as it modifies the variance to reflect any shifts in the means of the source and target distributions. The factor $\tau(1 - \tau)$ scales this adjustment, ensuring the influence of the mean difference is appropriately weighted according to the relative emphasis on the source and target statistics.
% We set $\tau$ to 0.9, which is a conservative setting. Generally, a
We found that, across datasets and models, $\tau=0.9$ stabilizes the statistics such that, if $\lambda$ is high enough, accuracy not only increases, but will not decrease under distribution shift relative to when $\tau = 1$ (no dependence on target statistics). Other selections may yield higher accuracy on different levels of distribution shift, but require foresight into the nature of the shift. See Figure \ref{fig:hyperparams} in the Appendix for further analysis.
\subsubsection{\textbf{Calculate Sample Divergence}}
To effectively adapt to single data points, we calculate the Mahalanobis distance $d$~\cite{de2000mahalanobis}, which has recently been applied in many SOTA out-of-distribution detection tasks~\cite{podolskiy2021revisiting,colombo2022beyond}, to measure the divergence between source and our now-stabilized target distributions:
\vspace{-1em}
\label{eqn:mahalanobis}
\begin{equation}
d \leftarrow 1 - e^{-(\mu_b - \mu_s)^T(\Sigma_s^2)^{-1}(\mu_b - \mu_s)}
\end{equation}
\vspace{-1.5em}
% The distance $d$ serves as a dynamic indicator of distribution shift severity, allowing the method to adjust adaptively rather than relying on static hyperparameters.
Here, $(\mu_b - \mu_s)^T(\Sigma_s^2)^{-1}(\mu_b - \mu_s)$ calculates the squared Mahalanobis distance, taking into account the variance and mean of the source distribution. The exponential transforms this distance into a metric ranging from 0 to 1, and subtracting it from 1 makes $d$ a measure of divergence, where larger values indicate greater distribution shifts. $\Sigma_s^2$ is the diagonal variance matrix, and can be efficiently inverted.
% This dynamic measure is crucial for adapting to changing data characteristics across different environments.
\subsubsection{\textbf{Balance Source and Target}}
The Mahalanobis distance indicates distribution shift severity, thus informing whether we give more weight to the source or stabilized incoming statistics. For severe shifts, more emphasis is placed on source statistics; less-severe shifts place more emphasis on stabilized incoming statistics.
\vspace{-1em}
\begin{equation}
\mu_{\text{new}} \leftarrow d\lambda\mu_s + (1 - d\lambda)\mu_b
\end{equation}
\vspace{-1em}
\begin{equation}
\sigma_{\text{new}}^2 \leftarrow d\lambda\sigma_s^2 + (1 - d\lambda)\sigma_b^2
\end{equation}
\vspace{-1em}
Here, we set $\lambda$ as a balancing parameter to ensure that even when the Mahalanobis distance $d$ is 1, some of the target statistic will still be incorporated to improve upon the accuracy to ensures a smooth transition between source and target statistics.
% This approach ensures a smooth transition between source and target statistics, enhancing accuracy without requiring additional data.
\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{figs/running_v_instance.pdf}
\vspace{-1.5em}
\caption{Mean (top) and variance (bottom) recorded from a single layer of ResNet18 on the the shift-free CIFAR10 dataset, for a running average with momentum=0.9 (orange), with statistics calculated at each image (blue), and with stabilized statistics (red).}
\label{fig:run_v_inst}
\vspace{-1em}
\end{figure}
\subsubsection{\textbf{Normalize and Reset}}
The final step involves normalizing the input data $x_t^l$ using the updated statistics, resulting in the output $x_t^{l+1}$:
\vspace{-1.5em}
\begin{equation}
x_t^{l+1} = \gamma \left(\frac{x_t^l - \mu_{\text{new}}}{\sigma_{\text{new}}}\right) + \beta
\end{equation}
\vspace{-1.5em}
We reset the model at each data instance, preventing catastrophic forgetting and ensuring robust performance across diverse data shifts.
As such, our method is designed to be forward-only and quantization-compatible, making it well-suited for resource-limited devices. It requires only a single forward pass with a batch size of one to improve TTA accuracy. As each data instance progresses through the model's normalization layers, statistics are updated sequentially. Once the intermediate representation is computed at a layer, the statistics are immediately reset, preparing the system for the next data point.
% \begin{algorithm}[!t]
% \caption{Normalization Layer Update}
% \label{alg:bn}
% \SetAlgoNoLine
% \DontPrintSemicolon
% \KwRequire{Parameters $\mu_{s}, \sigma^{2}_{s}$, $\tau=0.9$, $\lambda=0.9$}
% \KwInput{$x^{l}_{t}$}
% \KwOutput{$x^{l+1}_t$}
% \SetAlgoVlined
% $\mu_{t} \gets \frac{1}{H*W} \sum x^{l}_{t}$\;
% $\sigma^{2}_{t} \gets Var(x^{l}_{t})$\;
% $\mu_{b} \gets \tau\mu_{s} + (1-\tau)\mu_{t}$\;
% $\sigma^{2}_{b} \gets \tau\sigma^{2}_{s} + (1-\tau)\sigma^{2}_{t} + \tau(1-\tau)(\mu_{s}-\mu_{t})^{2}$\;
% $d \gets 1-e^{-(\mu_{b}-\mu_{s})^{T} (\Sigma^{2}_{s})^{-1} (\mu_{b}-\mu_{s})}$\;
% $\mu_{new} \gets d*\lambda*\mu_{s} + (1-d*\lambda)*\mu_{b}$\;
% $\sigma^{2}_{new} \gets d*\lambda*\sigma^{2}_{s} + (1-d*\lambda)*\sigma^{2}_{b}$\;
% $x^{l+1}_{t} = \gamma\frac{(x^{l}_{t} - \mu_{new})}{\sigma^{2}_{new}} + \beta$\;
% \end{algorithm}
% \vspace{-2em}
\subsection{Partial Fusion Strategy}\label{subsec:partial_fusion}
% \chj{back to here}
% Since no backpropagation is required, our method is naturally compatible with quantization, significantly improving TTA efficiency. In comparison, SOTA TTA methods always
% \cm{usually? or always?}\hj{fixed}
% rely on backpropagation; once quantized, they would need to perform real quantized training on-device, which is usually impossible as specially designed kernels are required for such a process~\cite{gholami21_quantrev, lin24_tt}.
Our method's inherent efficiency and compatibility with quantization is because it is backpropagation-free. When these methods are quantized, they necessitate real, quantized training on-device, which is typically infeasible without specially designed kernels to facilitate stable training, as discussed in previous studies~\cite{gholami21_quantrev,lin24_tt}.
% The necessity for real quantized training on-device is typically infeasible due to the requirement for specially designed kernels to facilitate the process, as discussed in previous studies~\cite{gholami21_quantrev, lin24_tt}.
% Although recent studies show that quantized training reduces the quantization error associated with scaling mismatches in calculated gradients~\cite{lin24_tt}, combining this system with methods such as EATA, which use pre-defined hyperparameters for gradient selection~\cite{niu22_eata}, is difficult to implement and adjust to adapt the model to distribution shifts. batch normalization and convolutional layers are often fused during quantization. Layer fusion serves to decrease the number of parameters in a model by folding batch normalization layers into neighboring convolutional layers.
% As such, we also propose a partial fusion quantized strategy for any given pretrained models for TTA, as shown in Algorithm~\ref{alg:full} (1-7). Specifically, our method leaving batch normalization and convolutional layers separate during quantization, while fuses other layers before update nomalization parameters, significantly decreasing the number of parameters in a model. Such design Given that batch normalization layers have relatively few parameters, and that we do not need to worry about storing activations for backpropagation. Depending on the architecture and training of the model, we also propose partial fusion of the network, leaving the shallow half of the network free for adaptation.
%%% original version
% In this study, we introduce a partial fusion quantized strategy applicable to any pretrained models for Test-Time Adaptation (TTA), as delineated in Algorithm~\ref{alg:full} (steps 1-6). Our approach maintains the separation of batch normalization and convolutional layers during the quantization process, while fusing other layers prior to updating the normalization parameters. This strategy effectively reduces the number of parameters within the model. The rationale for this design is based on the observation that batch normalization layers typically contain a minimal number of parameters, and there is no necessity to store activations for backpropagation. Furthermore, contingent upon the model's architecture and training regimen, we advocate for a partial fusion approach that preserves the adaptability of the network's shallow layers.
%%% Young's version (Taylor's version? jk haha — edited this a bit)
\begin{algorithm}[!t]
\caption{Backpropagation-Free Quantized Test-Time Adaptation}
\label{alg:full}
\SetAlgoNoLine
\DontPrintSemicolon
\KwRequire{Model $f_{\theta}$; Data $X_{s}, X_{t}$, Unfused Layers~$L$}
\SetAlgoVlined
Begin quantization calibration\;
\For{$x_{s}$ in $X_{s}$}
{
$f_{\theta}(x_{s})$\;
}
\For{$l$ in $L$}
{
Separate BN and Conv layer, $l$, in $f_{\theta}$\;
}
\For{$l$ not in $L$}
{
Fuse a layer, $l$, in $f_{\theta}$\;
}
Quantize the fused model $\hat{f_{\theta}}$\;
\For{normalization layer $l$}
{
Replace $l$ with \systemName layer\;
}
Save original $\theta$ for normalization layers\;
\For{$x_{t}$ in $X_{t}$}
{
$\hat{y_{t}} \gets f_{\theta}(x_{t})$\;
Reset all normalization layers\;
}
\end{algorithm}
% \ydk{}\hj{fixed}
In this study, we introduce a partial-fusion quantized TTA strategy applicable to any pretrained models. We not only circumvent reliance on specially-designed kernels, but also further improve the efficiency of \systemName. We were inspired by results from Figure~\ref{fig:layer_ablation}: updating the first half of the model achieves similar or higher accuracy compared to updating the entire model across different architectures and datasets. From Algorithm~\ref{alg:full} (steps 1-8), given the set of unfused layers for architecture, our proposed partial fusion quantized strategy fuses only deeper layers. Then, it updates only the unfused layers based on our TTA method (Section~\ref{subsec:method}).
This strategy further improves the computational efficiency of the update step of \systemName, as the unfused layers are slower than fused layers with QNNPACK, the quantization engine which is currently supported on ARM architectures. Moreover, fused layers do not require storing BN layer parameters and activations for the update step.
% by folding batch normalization layers into neighboring convolutional layers.
% For TTA methods reliant on adjustment of batch normalization parameters, fusion makes an already challenging task difficult without recalculating the parameters at each iteration. In comparison, backpropagation-dependent methods would have to optimize the entire network with small quantities of unlabeled data.
% Given that batch normalization layers have relatively few parameters, and that we do not need to worry about storing activations for backpropagation, the benefits of layer fusion are modest. Instead of designing additional methods to enable fused TTA, we propose simply
% leaving batch normalization and convolutional layers separate during quantization. Depending on the architecture and training of the model, we also propose partial fusion of the network, leaving the shallow half of the network free for adaptation.
% Thus, \systemName sidesteps the challenges of integrating TTA with quantized training and layer fusion, instead presenting a quantization-ready system that can directly quantize the model, calibrate parameters, and conduct quantized inference.
% Here, we chose to quantize to int8; quantization to int4 is a possible path for future investigation.
\section{Results \& Discussion}
\subsection{Experimental Setup}
\label{sec:exp_setup}
We briefly explain experimental setups in this section. For more details, refer to Appendix~\ref{app:detailed_exp_setup}.
\textbf{Datasets.}
We employ commonly used TTA datasets such as CIFAR10-C and CIFAR100-C~\cite{hendrycks19_c10,hendrycks19_c100} and realistic, on-device applications with different modalities, specifically the audio datasets, such as BirdVox-DCASE-20k~\cite{stowell18_birds} and Warblr~\cite{stowell18_birds} (bird song detection task). We use BirdVox to pretrain our model for audio modality and test it using Warblr for the abrupt domain shift.
% MobilenetV2 is noteworthy for its design tailored to mobile computing, offering an efficient balance of performance and resource usage~\cite{yun21_mobinet}. To investigate the impact of scaling the number of classes, we extend our experimentation by training on the more complex CIFAR100 dataset.
\textbf{Models.}
For image modality, we employ two compact architectures, namely, MobileNetV2~\cite{sandler18_mbnet} and ResNet18~\cite{he16_resnet}.
Then, for audio modality, we adopt VGGish~\cite{vggish}.
We highlight the adaptability of our method across varying model architectures and underscore its robustness in handling datasets of increasing complexity.
\textbf{Abrupt Domain Shift.}
% Non-continuous distribution shifts — such as those captured by photography, or recordings spaced out across time or distance, especially of rare events or subjects — are, from the perspective of the model conducting adaptation, abruptly changing domains.
To simulate the real-world scenario of abruptly changing domains (e.g., rare events like photographs or intermittent recordings), we used two datasets: (1) CIFAR10-C and CIFAR100-C shuffled randomly across shift types and severity levels, following prior works~\cite{niu23_sar,su24_tema} and (2) the Warblr dataset collected under diverse conditions and sensors intermittently.
% The Warblr dataset is an exemplar of the challenges of TTA: training on data gathered from a contained spatiotemporal area, using uniform hardware, then testing on data gathered under diverse conditions and sensors. Warblr is also challenging, as unlike the training BirdVox dataset, it is imbalanced — 75.6\% of the audio samples collected by citizen scientists contain birdsong, while the other 24.4\% contain other sounds. We simulate a similar setting for the image modality with CIFAR10/100-C, following the example of SAR~\cite{niu23_sar} and RealisticTTA~\cite{su24_tema} to sample and shuffle together 100 images from each of the 19 corruption categories and 5 severity levels. There is some label shift present in the CIFAR10/100-C datasets — compared to the training set, the sampling process unbalances the class datasets.
\textbf{Gradual Domain Shift.}
As opposed to the abruptly changing datasets above, gradually changing datasets illustrate a different use case. Here, the data changes domain slowly, from severity level 1, to 5, then from 5 back to 1, then to the next domain. We use CIFAR10/100-C datasets to simulate this. Label shift is still present, as we otherwise used the same sampling technique as above.
\textbf{Baselines.}
We compare \systemName with four SOTA TTA benchmarks: Tent~\cite{wang21_tent}, CoTTA~\cite{wang22_cotta}, EATA~\cite{niu22_eata}, and RealisticTTA~\cite{su24_tema}. In implementing the TTA methods, we followed the suggestions from their respective papers.
\textbf{Device.}
For system measurements, we employ Raspberry Pi Zero 2W, with 4 GB of swap memory available. We measure end-to-end latency. Then, we approximate the energy using a power draw recording device, whose measurements are sampled throughout and averaged, then multiplied by time to obtain power in watt-hours.
\textbf{Quantization.}
Quantized models were generated using the QNNPACK backend using PyTorch; post-static quantization scales and zero-points were recorded over 100 batches of size 64 from the CIFAR10/CIFAR100 training datasets. When necessary, layer fusion was done on adjacent convolutional and batch normalization layers.
\subsection{\systemName Accuracy Improvement}
\begin{table*}[t]
\centering
\caption{\% Accuracy on abruptly and gradually changing datasets for batch size = 1. N=5 trials. \textbf{Bold}: best, \underline{underline}: 2nd best, \textit{italics}: worse than baseline.}
\label{tab:cifar10_results}
\resizebox{\textwidth}{!}{
\begin{tabular}{l l c c c c c c c c}
\toprule
\multirow{2}{*}{\textbf{Method}} & \multirow{2}{*}{\textbf{Variant}} & \multicolumn{2}{c}{\textbf{CIFAR10-C Abrupt}} & \multicolumn{2}{c}{\textbf{CIFAR10-C Gradual}} & \multicolumn{2}{c}{\textbf{CIFAR100-C Abrupt}} & \multicolumn{2}{c}{\textbf{CIFAR100-C Gradual}} \\
\cmidrule(lr){3-4} \cmidrule(lr){5-6} \cmidrule(lr){7-8} \cmidrule(lr){9-10}
& & \textbf{ResNet18} & \textbf{MobilenetV2} & \textbf{ResNet18} & \textbf{MobilenetV2} & \textbf{ResNet18} & \textbf{MobilenetV2} & \textbf{ResNet18} & \textbf{MobilenetV2} \\
\midrule
\multirow{2}{*}{None} & Batch 1 & 76.3$\pm$0.0 & 67.0$\pm$0.0 & 76.3$\pm$0.0 & 67.0$\pm$0.0 & 51.4$\pm$0.0 & 40.7$\pm$0.0 & 51.4$\pm$0.0 & 40.7$\pm$0.0 \\
& int8 & 75.9$\pm$0.1 & 66.4$\pm$0.0 & 75.9$\pm$0.0 & 66.4$\pm$0.0 & 51.6$\pm$0.0 & 40.4$\pm$0.0 & 51.6$\pm$0.0 & 40.4$\pm$0.0 \\
\midrule
RealisticTTA & Batch 1 & \textit{63.4$\pm$0.1} & \textit{46.2$\pm$0.0} &
78.1$\pm$1.8 &
68.4$\pm$1.5 &
\textit{42.1$\pm$0.3} & \textit{28.0$\pm$0.4} & \textit{51.2$\pm$0.3} &
\textit{32.7$\pm$0.3}\\
\midrule
Tent & Batch 1 & \textit{9.8$\pm$1.6} & \textit{12.3$\pm$2.3} & \textit{9.0$\pm$0.002} & \textit{14.0$\pm$0.01} & \textit{0.0$\pm$0.0} & \textit{0.002$\pm$0.004} & \textit{0.0$\pm$0.0} &
\textit{0.0$\pm$0.0} \\
\midrule
EATA & Batch 1 & \textit{9.0$\pm$0.0} & \textit{13.5$\pm$0.1} & \textit{8.96$\pm$0.0} & \textit{13.4$\pm$0.1} & \textit{0.18$\pm$0.0} & 0.44$\pm$0.04 & \textit{0.18$\pm$0.0} &
\textit{0.49$\pm$0.0}\\
\midrule
CoTTA & Batch 1 & \textit{9.0$\pm$0.0} & \textit{9.6$\pm$0.1} & \textit{9.0$\pm$0.0} & \textit{9.78$\pm$0.18} & \textit{0.77$\pm$0.03} & \textit{0.0$\pm$0.0} & \textit{0.75$\pm$0.02} &
\textit{0.0$\pm$0.0}\\
\midrule
\multirow{2}{*}{\systemName (Ours)} & Batch 1 & \textbf{79.1$\pm$0.0} & \textbf{73.5$\pm$0.0} & \underline{79.1$\pm$0.0} & \textbf{73.5$\pm$0.0} &
\underline{54.1$\pm$0.0} &
\textbf{41.2$\pm$0.0} &
\underline{54.1$\pm$0.0} &
\textbf{41.2$\pm$0.0} \\
& int8 & \underline{78.8}$\pm$0.0 & \underline{72.3$\pm$0.0} & \underline{78.8$\pm$0.0} &
\underline{72.3$\pm$0.0} &
\textbf{54.4$\pm$0.0} & \underline{\textit{40.0$\pm$0.0}} & \textbf{54.4$\pm$0.0} &
\underline{\textit{40.0$\pm$0.0}} \\
\bottomrule
\end{tabular}
}
\vspace{-1em}
\end{table*}
\subsubsection{Abruptly Changing Domains}
Under abrupt shifts, applying state-of-the-art TTA methods, even at large batch sizes, can lower accuracy instead of increasing it. As shown in Table~\ref{tab:cifar10_results}, across all distribution shifts in the CIFAR10/100-C datasets, and across the natural heterogeneity of the Warblr dataset, \systemName more consistently improves accuracy relative to model without adaptation under abruptly changing domains. For example, it reduces shift error by 2.8\% and 6.5\% for ResNet18 and MobileNetV2 on CIFAR10-C, respectively.
% \textit{increases} error by 10.9\%, for example.
Compared to TTA-based methods, \systemName outperformed the best baseline, RealisticTTA, by 15.7\% and 12\% on CIFAR10-C and CIFAR100-C, respectively, when using ResNet18 under abruptly changing domains. When using MobileNetV2, \systemName achieved improvements of 27.3\% on CIFAR10-C and 13.2\% on CIFAR100-C.
In history-free, single-batch datasets, shown in Table~\ref{tab:cifar10_results}, most methods collapse, with accuracies comparable to random guessing. Even when other methods are tested with a batch size of 64—pushing the limits of many mobile devices—under abrupt domain shifts, as shown in Table~\ref{tab:cifar10_results_b}, most fail to improve upon the baseline adaptation-free accuracy; In comparison, \systemName improves accuracy gains \textit{consistently}. Methods reliant on entropy-loss backpropagation or warmup periods struggle to extract stable, representative statistics from mini-batches composed of data points collected under different distribution shifts, especially with small batches. Practically, such methods could not adapt on the first collected data point without waiting to accrue further data, while our method is history- and batch-size agnostic.
% \cm{why not use the word abrupt instead of mixed so we do not need to explain?}
% Contrary to expectation, benchmark TTA methods generally fail to improve accuracy under abruptly changing domain shifts, instead decreasing accuracy relative to the baseline. This is a result of the data scenario, which is realistic but infrequently tested: when data points within a batch come from different distribution shifts, adaptation methods which assume that the batch is comprised of data from similar distributions will perform poorly. For methods which rely on backpropagation with representative test mini-batch, combining statistics from unrelated domains often results in decreased accuracy.
% Even RealisticTTA, which does not rely on either backpropagation or batches, assumes that past data points will come from a stable or continuous distribution shift. While this may hold under scenarios where data are collected frequently, when data are sparse, as in many real-world situations, the method is unable to significantly improve accuracy.
\systemName's success with abrupt distribution shifts and single-instance batches extends across architectures and datasets. (See Appendix \ref{app:mobivit} for experiments on MobileViT). Even on the label-shifted and challenging real-world scenario presented by Warblr, our method was able to improve the weighted F1 score of the model, shown in Table~\ref{tab:birds}. (Weighted F1 was used due to class imbalance). Methods with batch sizes of 16 and 64 achieved a similar improvement in performance, but no backpropagation-based methods could run with these batches on our edge device, due to memory constraints.
\begin{table}[t]
\centering
\caption{Weighted F1 scores for TTA methods applied to the Warblr dataset. \textbf{Bold} indicates best performance, \underline{underline} indicates second-best.}
\label{tab:birds}
\begin{tabular}{l l c}
\toprule
\multirow{2}{*}{\textbf{Method}} & \multirow{2}{*}{\textbf{Variant}} & \textbf{Birds} \\
& & \textbf{(Weighted F1)}\\
\midrule
\multirow{2}{*}{None} & Batch 1 & 71.1$\pm$0.0 \\
& int8 & 70.1$\pm$0.0 \\
\midrule
RealisticTTA & $m=0$ & 66.2$\pm$0.1 \\
\midrule
\multirow{3}{*}{Tent} & Batch 1 & 47.1$\pm$26.1 \\
& Batch 16 & 71.0$\pm$1.4 \\
& Batch 64 & \textbf{73.2$\pm$0.16} \\
\midrule
\multirow{3}{*}{EATA} & Batch 1 & 44.1$\pm$3.7 \\
& Batch 16 & 69.0$\pm$0.2 \\
& Batch 64 & 69.1$\pm$0.0 \\
\midrule
\multirow{3}{*}{CoTTA} & Batch 1 & 45.4$\pm$0.0 \\
& Batch 16 & 51.9$\pm$0.9 \\
& Batch 64 & 59.1$\pm$0.9 \\
\midrule
\multirow{2}{*}{\systemName (Ours)} & Batch 1 & \underline{72.5$\pm$0.0} \\
& int8 & 71.7$\pm$0.0 \\
\bottomrule
\end{tabular}
\vspace{-1em}
\end{table}
\subsubsection{Gradually Changing Domains}
As expected, while \systemName never induces precipitous drops in accuracy, we do see that other methods outperform our method with larger batch sizes under gradually changing domains. As shown in the column marked ``Gradual'' in Table~\ref{tab:cifar10_results}, our method improves accuracy, though not as much as backpropagation-driven methods Tent and EATA — but only when these methods are equipped with large batch sizes, which comes at a significant memory cost. For example, we were unable to realistically use Tent, EATA, and CoTTA on the Warblr dataset on our memory-constrained device. Thus, slightly lower accuracy gains may be the cost of high-efficiency adaptation. RealisticTTA also performs well, but requires a long warmup period — accuracy begins at a random guess and does not stabilize until about 1000 data points of analysis (see Table~\ref{tab:sev5_resnet}); it is also high-latency.
In mobile and edge applications, the continuity of distribution shifts is often unknown, or expected to change drastically between individual data instances. From Tables~\ref{tab:cifar10_results} and~\ref{tab:cifar10_results_b}, we suggest that \systemName is better-suited for such environments, and for deployments where efficiency is a concern.
% For less memory-constrained environments where data are sampled with enough frequency to capture continuous distribution shift, methods such as EATA and Tent may be better-suited.\yk{I think it's unnecessary to cheer other method here. we can discuss such things in discussion if needed.} Removed!
% \cm{table 1 caption still refers to mixed and gradual.i think these can be deleted}
% \cm{is that intended that our method is not in table2? => can put it in, did not for space reasons}
\begin{table*}[t]
\centering
\caption{\% Accuracy on abruptly and gradually changing CIFAR10-C datasets for backpropagation-based methods with batch sizes 16 and 64. These results may result in out-of-memory errors on highly memory-constrained devices. N=5 trials. \textbf{Bold}: best, \underline{underline}: 2nd best, \textit{italics}: worse than baseline. CIFAR100-C may be more imbalanced due to sampling. Note that Tent requires at least 16 batches to perform better than \systemName on the CIFAR100-C dataset, while \systemName requires only a single batch during TTA.}
\label{tab:cifar10_results_b}
\resizebox{\textwidth}{!}{
\begin{tabular}{l l c c c c c c c c}
\toprule
\multirow{2}{*}{\textbf{Method}} & \multirow{2}{*}{\textbf{Variant}} & \multicolumn{2}{c}{\textbf{CIFAR10 Abrupt}} & \multicolumn{2}{c}{\textbf{CIFAR10 Gradual}} & \multicolumn{2}{c}{\textbf{CIFAR100 Abrupt}} & \multicolumn{2}{c}{\textbf{CIFAR100 Gradual}} \\
\cmidrule(lr){3-4} \cmidrule(lr){5-6} \cmidrule(lr){7-8} \cmidrule(lr){9-10}
& & \textbf{ResNet18} & \textbf{MobileNetV2} & \textbf{ResNet18} & \textbf{MobileNetV2} & \textbf{ResNet18} & \textbf{MobileNetV2} & \textbf{ResNet18} & \textbf{MobileNetV2} \\
\midrule
\multirow{2}{*}{None} & Batch 1 & 76.3$\pm$0.0 & 67.0$\pm$0.0 & 76.3$\pm$0.0 & 67.0$\pm$0.0 & 51.4$\pm$0.0 & 40.7$\pm$0.0 & 51.4$\pm$0.0 & 40.7$\pm$0.0 \\
& int8 & 75.9$\pm$0.1 & 66.4$\pm$0.0 & 75.9$\pm$0.0 & 66.4$\pm$0.0 & 51.6$\pm$0.0 & 40.4$\pm$0.0 & 51.6$\pm$0.0 & 40.4$\pm$0.0 \\
\midrule
\multirow{2}{*}{Tent} & Batch 16 & \textit{43.7$\pm$5.3} & \textit{19.9$\pm$2.8} & \textit{49.3$\pm$3.5} & \textit{35.2$\pm$3.1} & \textit{51.0 $\pm$1.1} & \underline{42.77$\pm$0.79} & \underline{64.3$\pm$0.4} &
\textit{51.2$\pm$4.0}\\
& Batch 64 &
\textit{69.6$\pm$1.7} & \textit{52.6$\pm$2.8} &
78.4$\pm$3.73 & \textit{66.6$\pm$1.6} & 53.9$\pm$0.47 & \textbf{44.83$\pm$0.69} & \textbf{68.5$\pm$0.6} &
\textbf{58.2$\pm$1.3} \\
\midrule
\multirow{2}{*}{EATA} & Batch 16 & \textit{54.2$\pm$4.6} & \textit{62.8$\pm$0.4} & \textit{68.4$\pm$3.1} & \textit{61.6$\pm$3.2} & \textit{50.4$\pm$0.25} &
39.5$\pm$0.19 &
63.2$\pm$0.38 &
51.2$\pm$0.67 \\
& Batch 64 &
\textit{75.3$\pm$0.74} & \textit{65.8$\pm$0.1} & \textbf{83.0$\pm$1.1} & \textbf{79.6$\pm$0.9} & \textit{49.8$\pm$0.19} &
39.76$\pm$0.11 &
62.9$\pm$1.2 &
\underline{52.5$\pm$0.1} \\
\midrule
\multirow{2}{*}{CoTTA} & Batch 16 & \textit{56.8$\pm$0.2} & \textit{50.4$\pm$0.3} & \textit{58.0$\pm$0.7} & \textit{51.9$\pm$0.4} & \textit{29.0$\pm$0.33} & \textit{23.17$\pm$0.28} & \textit{31.8$\pm$0.2} &
\textit{22.8$\pm$0.04} \\
& Batch 64 & \textit{58.8$\pm$0.2} & \textit{51.9$\pm$0.3} & \textit{64.7$\pm$0.37} & \textit{56.5$\pm$0.13} & \textit{32.2$\pm$0.12} & \textit{27.8$\pm$0.27} & \textit{36.1$\pm$0.06} &
\textit{30.4$\pm$0.13} \\
\midrule
\multirow{2}{*}{\systemName (Ours)} & Batch 1 & \textbf{79.1$\pm$0.0} & \textbf{73.5$\pm$0.0} & \underline{79.1$\pm$0.0} & \underline{73.5$\pm$0.0} & \underline{54.1$\pm$0.0} &
41.2$\pm$0.0 &
54.1$\pm$0.0 &
41.2$\pm$0.0 \\
& int8 & \underline{78.8$\pm$0.0} & \underline{72.3$\pm$0.0} &
78.8$\pm$0.0 &
72.3$\pm$0.0 & \textbf{54.4$\pm$0.0} & \textit{40.0$\pm$0.0} &
54.4$\pm$0.0 &
\textit{40.0$\pm$0.0} \\
\bottomrule
\bottomrule
\end{tabular}
}
\end{table*}
\begin{table*}[t]
\centering
\caption{\% System measurements for memory (MB), time (seconds-per-instance), and energy (joules-per-instance) for the three evaluated models on the Raspberry Pi Zero 2W. ``-'' indicates that the Pi was unable to run adaptation due to resource constraints. Bold text indicates best performance, \underline{underlined} indicates second-best (with the exception of the adaptation-free baseline).}
\label{tab:system}
\resizebox{\textwidth}{!}{
\begin{tabular}{l l c c c c c c c c c}
\toprule
\multirow{2}{*}{\textbf{Method}} & \multirow{2}{*}{\textbf{Variant}} & \multicolumn{3}{c}{\textbf{VGGish}} & \multicolumn{3}{c}{\textbf{ResNet18}} & \multicolumn{3}{c}{\textbf{MobileNetV2}} \\
\cmidrule(lr){3-5} \cmidrule(lr){6-8} \cmidrule(lr){9-11}
& & Memory & Time & Energy & Memory & Time & Energy & Memory & Time & Energy \\
\midrule
\multirow{2}{*}{None} & Batch 1 & 18.7 & 1.46$\pm$1.40 & 4 & 44.7 & 0.113$\pm$0.04 & 0.3 & 8.95 & 0.18$\pm$0.11 & 0.5 \\
& int8 & 4.77 & 0.48$\pm$0.10 & 1 & 11.2 & 0.159$\pm$0.0 & 0.4 & 2.24 & 0.50$\pm$0.06 & 1 \\
\midrule
RealisticTTA & Batch 1 & 18.7 & 4.13$\pm$0.60 & 10 & 44.7 & 3.89$\pm$0.11 & 8.6 & 8.95 & 19.6$\pm$0.16 & 46 \\
\midrule
\multirow{3}{*}{Tent} & Batch 1 & 78.2 & 2.59$\pm$1.49 & 6 & 48.0 & 0.319$\pm$0.19 & 0.8 & 36.0 & \underline{0.45$\pm$0.30} & \underline{1} \\
& Batch 16 & 306.8 & - & - & 60.7 & 0.402$\pm$0.12 & 0.9 & 138.4 & 4.4$\pm$0.50 & 5 \\
& Batch 64 & 1038.2 & - & - & 101.1 & 0.830$\pm$0.20 & 1 & 466.1 & - & - \\
\midrule
\multirow{3}{*}{EATA} & Batch 1 & 63.5 & \underline{1.52$\pm$1.17} & 3 & 47.2 & \textbf{0.147$\pm$0.22} & \underline{0.4} & 29.3 & \textbf{0.219$\pm$0.013} & \textbf{0.6} \\
& Batch 16 & 292.0 & - & - & 59.8 & 0.088$\pm$0.16 & \textbf{0.2} & 131.7 & 3.44$\pm$0.44 & 4 \\
& Batch 64 & 1023.4 & - & - & 100.3 & 0.442$\pm$0.18 & 0.8 & 459.4 & - & - \\
\midrule
\multirow{3}{*}{CoTTA} & Batch 1 & 263.8 & 55.31$\pm$5.39 & 74 & 279.8 & 53.9$\pm$4.5 & 70 & 152.4 & 14.8$\pm$1.16 & 25 \\
& Batch 16 & 1098.8 & - & - & 340.6 & 7.31$\pm$0.39 & 12 & 652.7 & 14.78$\pm$1.16 & 27 \\
& Batch 64 & 3770.9 & - & - & 535.2 & 5.78$\pm$1.84 & 11 & 2253.5 & - & - \\
\midrule
\multirow{2}{*}{\systemName (Ours)} & Batch 1 & \underline{18.7} & 1.54$\pm$0.34 & \underline{2} & \underline{44.7} & 0.205$\pm$0.01 & 0.6 & \underline{8.95} & 0.531$\pm$0.030 & \underline{1} \\
& int8 & \textbf{4.77} & \textbf{0.85}$\pm$\textbf{0.20} & \textbf{1} & \textbf{11.2} & \underline{0.185$\pm$0.021} & 0.5 & \textbf{2.24} & 0.947$\pm$0.01 & 2 \\
\bottomrule
\end{tabular}
}
\end{table*}
\subsection{System Evaluation}
When assessed for latency and memory usage, especially on-device, \systemName outperforms existing TTA methods capable of correcting distribution shift error (Table~\ref{tab:system}). For example, EATA is faster than \systemName at a batch size of 1, and sometimes 16. However, EATA does not consistently improve accuracy at these batch sizes, as shown in Table~\ref{tab:cifar10_results_b}. Backpropagation-based methods running on memory-constrained devices must trade off improved accuracy from large batch sizes and the corresponding increased latency and memory consumption. For MobilenetV2, in fact, we found that, on the Raspberry Pi Zero 2W, the backpropagation required for batch 64 overwhelms the available memory, crashing the device (see Table~\ref{tab:system}); the same for batch sizes of 16 and 64 on the VGGish model with the Warblr data. RealisticTTA, while memory-efficient, requires a significant amount of compute time even with only a forward pass, and fails to produce improvements in accuracy under certain data conditions. Thus, \systemName strikes the most suitable and consistent tradeoff between efficiency and accuracy improvement for edge computing environments. Refer to Appendix~\ref{app:fbgemm} for further latency analysis depending on system architectures.
% On platforms that are even more resource-constrained than the Raspberry Pi Zero 2W, we hypothesize that some state-of-the-art TTA methods would not be able to backpropagate even with batch size one; for forward-only systems like ours, as long as the platform supports inference, adaptation is also possible.\cm{will reviewers ask to prove this? do we need to say this?}
% \cm{anything to say about our chances?}
% \cm{are the numbers in table 4 final for mobile nets as they are still quite high and i do not think we explain it?}
\subsection{Impact of Layers on Adaptation}
\begin{figure}[t]
\centering
\includegraphics[width=0.84\linewidth]{figs/layer_ablation.pdf}
\vspace{-1em}
\caption{Layerwise ablation analysis in two directions: (1) progressively removing adaptation from shallow layers (blue solid line) and (2) progressively adding adaptation to deeper layers (orange dotted line). Results are from full-precision (fp32) Resnet18 and MobileNetV2 evaluated on the Abrupt CIFAR10 dataset.}
\label{fig:layer_ablation}
\end{figure}
Restricting adaptation to certain layers can further improve system efficiency — even more so, in the case of quantized models, where only adapting certain layers allows us to fuse the remaining layers as part of the quantization process. Figure~\ref{fig:layer_ablation} shows the effects of ablating shallow layers closer to the input (blue, solid line) or deep layers closer to the output (orange, dashed line). Removing adaptation from deep layers does not meaningfully lower accuracy — sometimes even increasing it. Removing adaptation from shallow layers resulted in low accuracy improvements, indicating that shallow layers are more important to adaptation. We hypothesize that this is because deep layers focus on larger structures while shallow layers learn general patterns and components. Such ``stylistic'' information is related to distribution shift~\cite{benz21_rstats}.
\begin{figure}[t]
\centering
\includegraphics[width=0.84\linewidth]{figs/fused_speeds.pdf}
\vspace{-1em}
\caption{Fused, half-fused, and unfused quantized time-per-iteration for \systemName applied to ResNet18 (left) and MobileNetV2 (right) with a 500-image subset of CIFAR10-C and CIFAR100-C. For ``Half'', batch normalization and convolutional layers are fused in the half of the model closest to the outputs.}
\label{fig:fused_speeds}
\vspace{-1em}
\end{figure}
% \begin{table*}[t]
% \centering
% \resizebox{\textwidth}{!}{
% \begin{tabular}{llcccccc}
% \toprule
% \multirow{2}{*}{Architecture} & \multirow{2}{*}{Dataset} & \multicolumn{2}{c}{Fused} & \multicolumn{2}{c}{Half-Fused} & \multicolumn{2}{c}{Unfused} \\
% & & Acc. & Time & Acc. & Time & Acc. & Time \\
% \midrule
% Resnet18 & CIFAR10 & 59.8 & 0.064$\pm$0.008 & 65.4 & 0.118$\pm$0.014 & 65.4 & 0.184$\pm$0.020 \\
% MobileNetV2 & CIFAR10 & 65.5 & 285.56 Hz & 72.5 & 75.73 Hz & 69.9 & 35.43 Hz \\
% Resnet18 & CIFAR100 & 52.5 & 345.39 Hz & 54.6 & 178.23 Hz & 54.4 & 121.30 Hz \\
% MobileNetV2 & CIFAR100 & 40.7 & 302.25 Hz & 43.3 & 80.72 Hz & 40.0 & 36.51 Hz \\
% \bottomrule
% \end{tabular}}
% \caption{Accuracy and timing impact of fusing (and thus skipping adaptation) over all layers, the deepest half of the layers, and none of the layers in quantized architectures. Fusing all layers is equivalent to not conducting any adaptation. A subset of the abruptly shifting dataset was used, with 500 images per test.}
% \label{tab:fused_timings}
% \end{table*}
% Figures~\ref{fig:layer_ablation} and~\ref{fig:fused_speeds} show that, for CIFAR10 and CIFAR100 datasets, across both Resnet18 and MobileNetV2 architectures, by only conducting our method of adaptation on the shallow half the model still yields accuracy gains, while fusing the other half yields a much more efficient runtime, inspiring our proposed method, partial fusion strategy, described in Section~\ref{subsec:partial_fusion}.\cm{please simplify this very convoluted paragraph}\hj{fixed}
Figures~\ref{fig:layer_ablation} and~\ref{fig:fused_speeds} demonstrate that for both the CIFAR10 and CIFAR100 datasets, and across Resnet18 and MobileNetV2 architectures, applying our adaptation method to only the shallow half of the model still results in accuracy improvements. (See Appendix \ref{app:mobivit} for half-adaptation on MobileViT). Meanwhile, fusing the other half significantly enhances runtime efficiency. This observation led to our proposed partial fusion strategy, detailed in Section~\ref{subsec:partial_fusion}.
\subsection{Hyperparameters}
Guided by the TTA literature~\cite{niu24_foa, benz21_rstats}, we suggest setting two hyperparameters to conservative static values.
%We present two hyperparameters that, following TTA literature~\cite{niu24_foa, benz21_rstats}, we suggest setting to a conservative static values.
Even under different distribution shifts that might have an alternative optimal set of hyperparameters, our hyperparameter search demonstrates that by staying conservatively close to the source statistic weight, our method will likely decrease shift error, or at the least, not worsen accuracy. In Figure~\ref{fig:hyperparams}, we can see that staying at 0.9 for both $\tau$ and $\lambda$ will generally keep the accuracy higher than when $\tau=1.0$ and $\lambda=1.0$ — the far lower-right square, representative of inference without adaptation.
% \subsection{Quantization}
% \cm{to do?} => folded into the 'impact of layers'
\section{Conclusions}
Inspired by the ineffectiveness of existing TTA methods for quantized models deployed on low-resource devices, and their poor performance under diverse, realistic scarce-data scenarios, we presented \systemName, a novel advancement in on-device TTA. \systemName consistently improves accuracy under the challenging low-data, low-memory, low-power environment of edge deployments. \systemName dynamically analyzes each incoming data point independently based on per-sample shift severity measured with Mahalanobis distance, without having to maintain historical data or large batches. Given its layerwise nature, \systemName also enables adaptation alongside quantized module fusion, further optimizing the system efficiency of our proposed quantized TTA while also providing insights into the importance of layer depth to adaptation. Under data-scarce, rapidly shifting distributions, \systemName avoids model collapse, outperforms the state-of-the-art while also reducing latency and memory consumption.
%We propose our method as the best choice for deployments on edge devices under the .\cm{dandling}
\clearpage
\section*{Impact Statement}\label{sec:limits_future}
\systemName looks to advance the accessibility of machine learning by enabling robust, adaptive methods on low-resource, often inexpensive devices. The system's ability to adapt with limited data to domains not present during training broader the applications to which machine learning models can be applied, and generally improve accuracy.
\textbf{Generalizability.}
The main criteria for the application of our method to a given model is the existence of normalization layers. An area of future research is the interaction of our method with layerwise and group normalization layers, or significantly different architectures, like ViT. Experiments with MobileViT can be found in the Appendix.
\textbf{Further accuracy improvements.}
Our suggested method relies on the selection of two hyperparameters, $\tau$ and $\lambda$. We suggest a conservative choice. One might be able to make a more optimal selection to reach even higher accuracies, but in real TTA settings, there may not be sufficient data available to tune hyperparameters. Future work could investigate the possibility of more noise-resistant metrics for calculating how far out-of-distribution an individual data point lies.
\textbf{Applicability to other data scenarios.}
In this work, we focus on the data scenario where inputs to the model are widely distributed across space and time. However, our system performs well under both continual and static distribution shift, as it operates independent of prior data. For situations where distribution shift does not change, further efficiency gains are possible: investigating whether our method could adapt on a single representative image, then freeze the updated statistics until the distribution shifts again, is a promising area of future research. Problematically, this assumes foresight into the nature of test data, which is often impossible to obtain.
% If the distribution shift is static or changes very slowly, it could be possible be employed at a minimal computational cost: instead of adapting at each input, one could adapt a single time at the beginning of data processing, without resetting.
% The model can then be frozen, and normal inference conducted for all subsequent samples until a change in distribution shift is anticipated. Notably, applying our method in this manner assumes foresight into the nature of test data, which is often impossible to obtain.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\appendix
\onecolumn
% \section{You \emph{can} have an appendix here.}
% You can have as much text here as you want. The main body must be at most $8$ pages long.
% For the final version, one more page can be added.
% If you want, you can use an appendix like this one, even using the one-column format.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \section*{Appendix}\label{app:1}
\vbox{
\hsize\textwidth
\linewidth\hsize
\vskip 0.1in
\hrule height 4pt
\vskip 0.25in%
\vskip -\parskip%
\centering \LARGE\bf Supplementary Material \\
\Large \systemName: A Backpropagation-Free and Stateless Approach to Quantized Test-Time Adaptation on Edge Devices \par
\vskip 0.29in
\vskip -\parskip
\hrule height 1pt
\vskip 0.09in%
}
% Make the "Part I" text invisible
\renewcommand \thepart{}
\renewcommand \partname{}
\doparttoc % Tell to minitoc to generate a toc for the parts
\faketableofcontents % Run a fake tableofcontents command for the partocs
\addcontentsline{toc}{section}{Appendix}
\part{}
\parttoc
% \clearpage
% \section*{\centering \LARGE\bf Supplementary Material\\
% \sysname: Deep Neural Network Training at the Extreme Edge}\label{app:1}
% \tableofcontents
% \clearpage
\section{Background}\label{app:background}
\subsection{Test-time Adaptation}
Test-time adaptation (TTA), different from the related approaches of domain adaptation and generalization, is restricted to the realistic scenario where the model cannot access source data, nor the labels and potential distribution shifts of test data~\cite{liang23_litrev}.
Many TTA methods employ normalization layers, frequently present in deep learning models, to correct distribution shifts. During training, these layers record sample statistics — the expected value or mean of the input data (\(\mu_{B}\)) and the variance of the input data (\(\sigma^{2}_{B}\)) for each mini-batch \(B\). A momentum term estimates the moving average across mini-batches. An affine transformation of the input is learned and applied after normalization with parameters \(\gamma\) and \(\beta\) \cite{ioffe15_bn}:
\begin{equation}
y_{i} = \gamma\frac{x_{i} - \mu_{B}}{\sigma^{2}_{B} + \epsilon} + \beta
\end{equation}
Here, $x_{i}$ represents the input, $\epsilon$ is a constant added for numerical stability, and $\gamma$ and $\beta$ are learned scaling and shifting parameters, respectively.
Standard inference assumes that the recorded mean and variance will not change between training and test data. However, when statistics do shift, test-time adaptation methods like Tent~\cite{wang21_tent} unfreeze and update normalization layer parameters, allowing adaptation to different distributions. While swapping sample statistics $\mu_{s}$ and $\sigma^{2}_{s}$ for per-batch running statistics is straightforward, updating learned parameters $\gamma$ and $\beta$ requires backpropagation. As there no labeled data is available, Tent proposes entropy minimization~\cite{wang21_tent}, which has become a common foundation for subsequent works.
\subsection{Quantization}
Increasingly, many TTA methods strive to be more efficient than baselines such as Tent \cite{niu24_foa,hong23_mecta,khurana21_sita}, but few have been designed to be both backpropagation-free and quantization-compatible.
Quantization is the compression of model parameters to low-precision, which speeds up computation and reduces memory usage during inference and training. On accelerators designed for high performance, lower memory costs lead to compounding improvements in latency and energy consumption, as energy costs may increase exponentially with each memory hierarchy level traversed \cite{sze17_effdnn}. Layer fusion reduces adjacent convolution and batch normalization layers (as well as activation functions) to a single algebraically equivalent convolutional layer, which further reduces memory savings and decreases runtime.
One could use a system for low-memory quantized training \cite{lin24_tt} for either continual learning or TTA to adapt models to distribution shifts, but such methods may still consume more time and memory than quantized, inference-only adaptation, while achieving similar accuracies; additionally, these methods may not work for rare, discontinuous data.
\begin{figure}
\centering
\includegraphics[width=0.75\linewidth]{figs/mphil_dis_tta.png}
\caption{Illustration of an example TTA problem. Here, the data distribution learned by the model during training over $x_{s}$, in this example, gathered by 3 different cameras in different conditions, differs from the distribution of $x_{t}$ encountered at test-time from a different camera in a different location.}
\label{fig:tta}
\vspace{-1em}
\end{figure}
\section{Detailed Experimental Setup}\label{app:detailed_exp_setup}
\subsection{Datasets}\label{app:subsec:datasets}
% We first establish the performance of our method along both accuracy and efficiency axes on image datasets to enable comparison with state-of-the-art works, then expand to an audio test case in order to show how our approach would work on more realistic, on-device applications and different modalities.
% \hj{Check this:}
This section describes the \systemName benchmark tasks: CIFAR10-C, CIFAR100-C, and realistic, on-device applications with different modalities, specifically the audio datasets BirdVox-DCASE-20k and Warblr. These datasets were selected to represent common TTA application domains across various conditions.
For image classification, we use the CIFAR10-C and CIFAR100-C datasets \cite{hendrycks19_c10, hendrycks19_c100}, widely-used benchmarks for test-time adaptation. The CIFAR10/100-C datasets contain 19 distinct domains, each with 5 levels of severity. The images are 3-channel, with 32$\times$32 pixels each. The corrupted datasets are based on the 10- and 100-class CIFAR10 and CIFAR100 \cite{krizhevsky09_cifar} datasets.
For a real-world test case without artificially induced domain shift, in an audio setting, we use the BirdVox-DCASE-20k~\cite{stowell18_birds} and the Warblr~\cite{stowell18_birds} datasets. BirdVox-DCASE-20k is an acoustic birdsong dataset collected from 6 locations around Ithaca, NY over the span of a single night in fall 2015; 50.09\% of the 20,000 10-second samples contain birdsong. Warblr is a collection of 8,000 10-second samples collected by users around the United Kingdom using their personal smartphones, with 76\% of the samples containing birdsong. The other 24\% contain miscellaneous sounds, such as weather, human chatter, and traffic noise~\cite{stowell18_birds}. We simulate an experiment where a model is trained on BirdVox, then deployed on mobile devices to analyze Warblr. Warblr represents a distribution shift consisting of sounds not included in the training set, different audio sensors, and different locations. The two birdsong datasets were provided as part of the DCASE 2018 challenge~\cite{stowell18_birds}.
\subsection{Models}
\subsubsection{CIFAR models}
% To assess our method on the CIFAR10 dataset, we use two model architectures: MobilenetV2 and ResNet18. Both are relatively small and compact, with MobilenetV2 being designed specifically for mobile computing~\cite{yun21_mobinet}. To explore the effect of increasing the number of classes, we also trained the ResNet18 model on CIFAR100.
To rigorously evaluate our method on the CIFAR10 dataset, we employ two compact model architectures: MobilenetV2 and ResNet18. MobilenetV2 is noteworthy for its design tailored to mobile computing, offering an efficient balance of performance and resource usage~\cite{yun21_mobinet}. To investigate the impact of scaling the number of classes, we extend our experimentation by training on the more complex CIFAR100 dataset. We highlight the adaptability of our method across varying model architectures and underscore its robustness in handling datasets of increasing complexity.
% To rigorously evaluate our method on the CIFAR10 dataset, we employ two cutting-edge model architectures: MobilenetV2 and ResNet18. This strategic choice allows us to showcase the versatility and effectiveness of our approach across diverse neural network designs, highlighting its potential to drive advancements in the field.
\subsubsection{Bird Call Detection Models}
For the bird call detection task, we used a VGGish architecture model based on the baseline architecture provided as part of the DCASE challenge~\cite{kong18_dcase}. We trained the model on 16,000 samples from the BirdVox dataset with 4,000 samples held out for validation and quantization calibration. Training was carried out for 7,000 iterations, past the stabilization of the validation loss, and the best-performing model was selected. The inference task was to detect whether the sound recorded in a 10-second sample contained a real bird call, or other noises.
% \subsection{Datasets}
\subsection{Domain Shift Cases}
% \subsubsection{Abruptly Changing Datasets}
\subsubsection{Abrupt Domain Shift}
Non-continuous distribution shifts — such as those captured by photography, or recordings spaced out across time or distance, especially of rare events or subjects — are, from the perspective of the model conducting adaptation, abruptly changing domains. To simulate the real-world scenario of intermittent data collection throughout variable domains, we used two datasets: first, a benchmark dataset of CIFAR10-C and CIFAR100-C shuffled randomly across shift types and severity levels, and secondly, the Warblr dataset.
The Warblr dataset is an exemplar of the challenges of TTA: training on data gathered from a contained spatiotemporal area, using uniform hardware, then testing on data gathered under diverse conditions and sensors. Warblr is also challenging, as unlike the training BirdVox dataset, it is imbalanced — 75.6\% of the audio samples collected by citizen scientists contain birdsong, while the other 24.4\% contain other sounds. We simulate a similar setting for the image modality with CIFAR10/100-C, following the example of SAR~\cite{niu23_sar} and RealisticTTA~\cite{su24_tema} to sample and shuffle together 100 images from each of the 19 corruption categories and 5 severity levels. There is some label shift present in the CIFAR10/100-C datasets — compared to the training set, the sampling process unbalances the class datasets.
Quantized models were generated using the QNNPACK backend using PyTorch; post-static quantization scales and zero-points were recorded over 100 batches of size 64 from the CIFAR10/CIFAR100 training datasets. When necessary, layer fusion was done on adjacent convolutional and batch normalization layers.
%\subsubsection{Gradually Changing Datasets}
\subsubsection{Gradual Domain Shift}
As opposed to the abruptly changing
datasets above, gradually changing datasets illustrate a different use case. Here, data changes domain slowly, from severity level 1, to 5, then from 5 back down to 1, then to the next domain. We used the CIFAR10/100-C datasets to simulate this. Label shift is still present, as we otherwise used the same sampling technique as above.
\subsection{TTA Baselines}
We compared our method to 4 state-of-the-art TTA benchmarks: Tent~\cite{wang21_tent}, CoTTA~\cite{wang22_cotta}, EATA~\cite{niu22_eata}, and RealisticTTA~\cite{su24_tema}. In implementing the TTA methods, we followed the suggestions from their respective papers. For Tent, we used the Adam optimizer with a learning rate of 0.001, no weight decay, and betas of 0.9 and 0.999. For CoTTA, we used SGD with momentum at 0.9 as the optimizer and a learning rate of 0.001; in addition, we only optimized the batch normalization layers. The fisher values for EATA were calculated using 300 batch 64 data samples; TTA was conducted with SGD + momentum at 0.9. The e\_margin value was set to $0.4 * ln(num classes)$. RealisticTTA was implemented with a batch normalization momentum of 1.0; we selected a momentum value of 0 for abrupt and 0.99 for gradual shifts.
% For the birds task, Tent, CoTTA, and EATA had to be modified to suit a binary classification problem by converting the single-logit output to two logits, and adding a small noise constant to mitigate undefined natural logs. Any hyperparameters dependent on the number of classes were also changed.
\subsection{Systems}
Systems analysis was done on a Raspberry Pi Zero 2W, with 4 GB of swap memory available. All timings were conducted on the Raspberry Pi. Energy was approximated using a power draw recording device, whose measurements were sampled throughout and averaged, then multiplied by time to obtain power in watt-hours.
% \subsection{Hyperparameter Search}
% To conduct the hyperparameter search, we changed $\tau$ and $\lambda$ in increments of 0.1, between 0 and 1 (inclusive) and recorded the accuracy for ResNet18 on various datasets.
\begin{table*}[t]
\centering
\caption{Accuracy from various adaptation methods on the CIFAR10-C abruptly shifting dataset using only data from the highest try severity level, with the ResNet18 model. N=5. \textbf{Bold text} indicates highest performance per-column.}
\label{tab:sev5_resnet}
\resizebox{\textwidth}{!}{
\begin{tabular}{l|cccccccccc}
\toprule
Method & Gaussian Noise & Shot Noise & Impulse Noise & Defocus Blur & Frosted Glass Blur & Motion Blur & Zoom Blur & Snow & Frost & Fog \\
\midrule
None & 13.91$\pm$1.08 & 17.34$\pm$1.44 & 17.02$\pm$0.54 & 53.04$\pm$1.84 & 42.8$\pm$1.16 & 62.02$\pm$0.91 & 59.33$\pm$1.1 & 72.66$\pm$1.74 & 58.39$\pm$1.68 & 67.48$\pm$1.33 \\
None (int8) & 13.61$\pm$0.36 & 16.77$\pm$0.51 & 17.88$\pm$0.56 & 52.44$\pm$0.76 & 43.49$\pm$1.13 & 58.59$\pm$0.67 & 59.76$\pm$1.61 & 72.1$\pm$1.55 & 58.89$\pm$0.85 & 63.71$\pm$1.67 \\
RealisticTTA (1st 1k) & 39.92$\pm$1.66 & 42.3$\pm$0.49 & 35.92$\pm$0.72 & 66.10$\pm$1.21 & 44.58$\pm$1.05 & 64.76$\pm$1.06 & 63.76$\pm$0.94 & 59.32$\pm$1.17 & 59.42$\pm$0.64 & 65.70$\pm$1.35 \\
RealisticTTA (2nd 1k) & 43.3$\pm$0.42 & 44.86$\pm$1.09 & 40.1$\pm$1.06 & 73.97$\pm$1.0 & 49.97$\pm$1.15 & 70.11$\pm$1.2 & 69.63$\pm$1.48 & 65.37$\pm$1.1 & 65.17$\pm$1.46 & 72.49$\pm$0.99 \\
Tent (1) & 9.97$\pm$0.61 & 11.13$\pm$0.44 & 10.07$\pm$0.85 & 9.75$\pm$0.89 & 9.97$\pm$0.64 & 9.71$\pm$1.16 & 9.97$\pm$0.42 & 10.25$\pm$0.7 & 9.51$\pm$0.77 & 9.91$\pm$0.82 \\
Tent (16) & 43.25$\pm$0.69 & 46.17$\pm$4.76 & 42.58$\pm$2.18 & 64.19$\pm$2.46 & 43.97$\pm$5.04 & 59.27$\pm$2.54 & 63.06$\pm$2.49 & 56.71$\pm$4.1 & 56.67$\pm$2.17 & 61.11$\pm$3.76 \\
Tent (64) & \textbf{59.24$\pm$1.81} & 58.73$\pm$1.37 & 53.55$\pm$1.7 & 80.12$\pm$1.46 & 56.95$\pm$2.27 & 75.53$\pm$1.47 & 79.26$\pm$1.98 & 74.86$\pm$1.4 & 72.01$\pm$1.9 & 78.87$\pm$0.73 \\
EATA (1) & 10.67$\pm$0.51 & 10.29$\pm$0.83 & 9.97$\pm$0.54 & 12.69$\pm$0.73 & 10.89$\pm$0.49 & 12.51$\pm$0.89 & 11.97$\pm$0.35 & 11.75$\pm$0.66 & 12.11$\pm$0.93 & 12.57$\pm$0.94 \\
EATA (16) & 47.38$\pm$6.0 & 46.33$\pm$0.85 & 41.63$\pm$4.02 & 72.66$\pm$2.08 & 47.5$\pm$3.92 & 69.86$\pm$1.8 & 72.38$\pm$1.87 & 66.13$\pm$1.61 & 62.1$\pm$0.6 & 70.69$\pm$2.88 \\
EATA (64) & 57.44$\pm$1.92 & \textbf{60.06$\pm$0.77} & \textbf{54.37$\pm$1.02} & \textbf{84.49$\pm$1.37} & \textbf{62.46$\pm$1.11} & \textbf{81.05$\pm$0.44} & \textbf{82.71$\pm$0.35} & 75.23$\pm$0.77 & 74.53$\pm$0.87 & \textbf{82.85$\pm$1.06} \\
CoTTA (1) & 9.31$\pm$0.88 & 9.97$\pm$0.76 & 10.37$\pm$0.86 & 12.15$\pm$1.27 & 10.61$\pm$0.28 & 10.77$\pm$0.88 & 10.95$\pm$0.96 & 10.89$\pm$0.55 & 10.27$\pm$0.15 & 11.35$\pm$0.63 \\
CoTTA (16) & 49.86$\pm$1.45 & 46.29$\pm$0.85 & 53.61$\pm$1.18 & 53.55$\pm$1.4 & 43.35$\pm$1.56 & 53.17$\pm$1.46 & 54.74$\pm$1.19 & 58.29$\pm$0.41 & 62.34$\pm$1.2 & 47.52$\pm$0.93 \\
\systemName (Ours) & 32.59$\pm$2.21 & 38.52$\pm$2.21 & 43.5$\pm$1.42 & 72.19$\pm$0.97 & 52.27$\pm$1.62 & 74.73$\pm$1.52 & 72.29$\pm$0.96 & \textbf{77.94$\pm$0.98} & \textbf{74.73$\pm$1.65} & 77.86$\pm$1.21 \\
\systemName int8 (Ours) & 30.61$\pm$1.11 & 37.42$\pm$1.66 & 42.24$\pm$0.61 & 68.67$\pm$1.16 & 50.11$\pm$1.18 & 71.33$\pm$0.85 & 70.21$\pm$1.32 & 77.8$\pm$1.16 & 72.43$\pm$1.64 & 76.82$\pm$0.63 \\
\midrule
Method & Brightness & Contrast & Elastic & Pixelate & JPEG Compression & Gaussian Blur & Saturate & Spatter & Speckle Noise & - \\
\midrule
None & 88.31$\pm$0.57 & 19.74$\pm$1.26 & 70.5$\pm$1.25 & 46.33$\pm$1.35 & 69.03$\pm$1.54 & 36.71$\pm$0.97 & 84.01$\pm$0.94 & \textbf{79.46$\pm$0.77} & 22.16$\pm$1.8 & - \\
None (int8) & 88.35$\pm$1.11 & 17.16$\pm$0.9 & 68.99$\pm$1.13 & 46.83$\pm$0.95 & \textbf{70.44$\pm$1.29} & 36.83$\pm$2.07 & 83.29$\pm$1.01 & 78.57$\pm$0.94 & 21.69$\pm$1.06 & - \\
RealisticTTA (1st 1k) & 68.38$\pm$0.56 & 62.90$\pm$0.98 & 54.08$\pm$1.66 & 56.96$\pm$1.21 & 46.46$\pm$1.68 & 65.06$\pm$1.27 & 68.90$\pm$1.40 & 59.74$\pm$1.13 & 42.42$\pm$1.06 & - \\
RealisticTTA (2nd 1k) & 76.36$\pm$1.38 & 71.39$\pm$1.2 & 59.02$\pm$0.93 & 62.28$\pm$1.04 & 51.77$\pm$0.65 & 72.53$\pm$1.01 & 75.96$\pm$1.13 & 64.9$\pm$2.31 & 46.89$\pm$1.5 & - \\
Tent (1) & 9.65$\pm$0.89 & 9.65$\pm$0.92 & 9.99$\pm$0.93 & 9.75$\pm$0.93 & 10.15$\pm$0.6 & 10.51$\pm$0.99 & 9.03$\pm$0.72 & 10.43$\pm$0.37 & 9.97$\pm$1.22 & - \\
Tent (16) & 66.69$\pm$3.21 & 59.54$\pm$3.27 & 52.26$\pm$2.27 & 56.79$\pm$3.64 & 50.44$\pm$1.76 & 62.56$\pm$5.3 & 69.98$\pm$2.22 & 62.66$\pm$2.91 & 46.85$\pm$3.82 & - \\
Tent (64) & 82.71$\pm$1.74 & 78.24$\pm$1.08 & 67.4$\pm$1.27 & 72.77$\pm$1.1 & 63.01$\pm$1.92 & 81.11$\pm$1.57 & 83.85$\pm$1.3 & 75.86$\pm$1.38 & 58.44$\pm$3.59 & - \\
EATA (1) & 12.07$\pm$0.83 & 12.51$\pm$0.23 & 11.79$\pm$0.55 & 12.39$\pm$1.52 & 11.19$\pm$0.52 & 12.19$\pm$0.93 & 12.85$\pm$0.14 & 12.35$\pm$0.82 & 9.79$\pm$0.48 & - \\
EATA (16) & 76.01$\pm$1.81 & 70.28$\pm$1.99 & 59.05$\pm$1.64 & 64.58$\pm$2.36 & 57.58$\pm$0.56 & 72.02$\pm$1.74 & 79.8$\pm$1.29 & 65.34$\pm$1.29 & 47.64$\pm$2.95 & - \\
EATA (64) & 87.07$\pm$1.94 & \textbf{81.64$\pm$1.03} & \textbf{71.02$\pm$1.26} & \textbf{74.18$\pm$1.74} & 66.8$\pm$0.86 & \textbf{83.71$\pm$1.91} & 86.29$\pm$0.76 & 76.86$\pm$0.55 & \textbf{60.0$\pm$1.57} & - \\
CoTTA (1) & 11.97$\pm$0.22 & 11.53$\pm$1.05 & 11.35$\pm$0.46 & 10.71$\pm$0.66 & 11.07$\pm$0.89 & 11.31$\pm$0.64 & 12.03$\pm$0.46 & 11.75$\pm$0.5 & 9.83$\pm$0.72 & - \\
CoTTA (16) & 68.91$\pm$1.44 & 50.95$\pm$0.98 & 48.21$\pm$1.44 & 52.76$\pm$1.34 & 54.05$\pm$0.97 & 53.37$\pm$1.07 & 62.66$\pm$1.67 & 58.97$\pm$2.18 & 43.13$\pm$1.01 & - \\
\systemName (Ours) & 88.75$\pm$0.73 & 47.45$\pm$1.89 & 68.47$\pm$1.86 & 55.12$\pm$1.35 & 64.56$\pm$0.93 & 60.02$\pm$0.82 & \textbf{86.81$\pm$0.87} & 78.68$\pm$1.34 & 45.13$\pm$0.67 & - \\
\systemName int8 (Ours) & \textbf{89.39$\pm$1.02} & 36.66$\pm$0.92 & 66.25$\pm$1.66 & 54.93$\pm$1.21 & 65.23$\pm$1.14 & 58.46$\pm$1.14 & 86.27$\pm$0.98 & 77.1$\pm$0.84 & 44.36$\pm$1.07 & - \\
\bottomrule
\end{tabular}}
\end{table*}
\begin{table}[t]
\centering
\resizebox{0.5\linewidth}{!}{
\begin{tabular}{l|c|c}
\toprule
Model Precision & Resnet18 & MobileNetV2 \\
\midrule
FP32 (no adaptation) & 10.09$\pm$0.14 & 7.02$\pm$0.29 \\
INT8 (no adaptation) & 2.26$\pm$0.50 & 4.77$\pm$ 0.09 \\
\systemName (none fused) & 8.81$\pm$0.17 & 29.70$\pm$0.41 \\
\systemName (half fused) & 4.84$\pm$0.07 & 13.06$\pm$0.16 \\
\systemName (all fused) & 1.82$\pm$1.10 & 3.55$\pm$ 0.08 \\
% \midrule
% FP32 (no adaptation) & 7.02$\pm$0.29 \\
% INT8 (no adaptation) & 4.77$\pm$ 0.09 \\
% Ours (none fused) & 29.70$\pm$0.41 \\
% Ours (half fused) & 13.06$\pm$0.16 \\
% Ours (all fused) & 3.55$\pm$ 0.08 \\
\bottomrule
\end{tabular}}
\caption{Time-per-iteration in ms for 500 frames from the abruptly-changing dataset. FP32 and INT8 have no attached adaptation; convolutional and batch norm layers were fused for our method. }
\label{tab:fbgemm}
\end{table}
\section{Hyperparameter Exploration}
\begin{figure*}[t]
\centering
\includegraphics[width=0.85\linewidth]{figs/hyperparams.png}
\vspace{-1.5em}
\caption{Hyperparameter space search for Source Weight $\tau$ and Distance Scaler $\lambda$ with ResNet18 applied to CIFAR10-C. From left to right, ``All'' shows accuracies across hyperparameter choices from all 19 categories and 5 severity levels, whereas ``Contrast 5'' and ``Shot Noise 5'' show accuracies from the corresponding noise type and severity. Lighter shaded squares indicate higher accuracy.}
\label{fig:hyperparams}
\vspace{-1em}
\end{figure*}
An exploration of the effect of hyperparameter tuning indicated to us that, as shown in Figure \ref{fig:hyperparams}, hyperparameter choice can impact the \textit{degree to which adaptation improves accuracy} across different domain shifts and severity levels; however, we found that in keeping the hyperparameters set to 0.9, indicating a conservative balance between the source (90\%) and target (10\%) data, we achieved consistent improvement on baseline accuracies. Consistency is key, as it is difficult to anticipate the severity of the domain shift the model may encounter in the wild, when operating under limited data and computational resources.
\section{High-Severity Domain Shifts}
The results presented in Table \ref{tab:sev5_resnet} illustrate the performance of various adaptation methods on the CIFAR10-C dataset, focusing on data from the highest severity level, using a ResNet18 model. The table highlights the effectiveness of different approaches in handling a range of corruptions, including Gaussian Noise, Shot Noise, and Impulse Noise, among others. Notably, our method, even with only one sample, demonstrates robust performance across several corruption types, achieving the highest accuracy in categories such as Snow, Frost, Brightness, and Saturate, with scores of 77.94\%, 74.73\%, 89.39\%, and 86.81\%, respectively. While some methods, such as EATA (which requires batch sizes of 64) and Tent (which also requires batch sizes of 64), achieve higher accuracies in certain noise categories, our method remains competitive in others. A significant advantage of our approach is its efficiency, as it only requires a batch size of 1, making it highly adaptable and resource-efficient compared to other methods that necessitate larger batch sizes for optimal performance. This efficiency does not come at the cost of accuracy, as evidenced by our method's strong performance across various corruption types. The results suggest that our method strikes an effective balance between computational efficiency and accuracy, making it a viable option for real-time applications where resources may be limited. Additionally, the int8 variant of our method maintains a commendable performance, further underscoring the adaptability and robustness of our approach in different computational settings. Overall, the results indicate that our method is a promising solution for adapting to severe dataset corruptions, providing a balance between high performance and low computational requirements.
\section{FBGEMM Timings}\label{app:fbgemm}
For the Raspberry Pi, which uses the ARM architecture, a QNNPACK backend is required; however, for other computing platforms, (FaceBook GEneral Matrix Multiplication) FBGEMM or x86 can be used as alternative quantization engines. In Table~\ref{tab:fbgemm}, we profiled the timings of our method with FBGEMM on a AMD Ryzen 7 3700X 8-Core Processor to demonstrate the terrific efficiency scaling of our system on other platforms.
We observed that quantization with the QNNPACK backend engine can slow down quantized inference with and without adaptation, especially for MobileNetV2, (Table~\ref{tab:system}). With the FBGEMM engine, we observed a significant speedup post-quantization; however, FBGEMM is incompatible with ARM architectures, like that of the Raspberry Pi line. We anticipate that future improvements to the QNNPACK implementation will lead to greater acceleration after quantization. The VGGish architecture did not suffer the same post-QNNPACK quantization slowdown, likely because it is a shallow model with very few layers relative to the deeper Resnet18 and MobileNetV2 architectures, which allow inefficiencies to compound.
Table~\ref{tab:fbgemm} presents a comparative analysis of time-per-iteration in milliseconds for 500 frames from the abruptly changing dataset, evaluating the performance of ResNet18 and MobileNetV2 across different model precision and adaptation strategies. Our method of partial layer fusion alongside partial adaptation, provides significant improvements in TTA speed. Specifically, for ResNet18, our ``none fused'' approach reduces the time-per-iteration to 8.81 ms compared to the FP32 baseline of 10.09 ms, marking an improvement of approximately 12.7\%. In the MobileNetV2 model, which has many more normalization layers (52) than ResNet18 (18), the ``none fused'' configuration achieves 29.70 ms; even with GPU acceleration, Tent with a batch size of one takes 139.7$\pm$8.5 ms to analyze one frame.
With partial fusion (``half fused''), ResNet18's iteration time further drops to 4.84 ms, improving by 52.8\% from the ``none fused'' setup. For MobileNetV2, the ``half fused'' configuration reduces the iteration time to 13.06 ms, enhancing speed by over 56\%. The ``all fused'' configuration offers the most significant improvements, with ResNet18 achieving 1.82 ms, a 79.3\% improvement over the ``none fused'' time, and MobileNetV2 reaching 3.55 ms, a 88\% reduction from the ``none fused'' time. These enhancements underscore the substantial TTA efficiency gains achieved through our proposed partial layer fusion strategy.
\section{Layerwise Distances}
To illustrate the way that the Mahalanobis distance $d$ between source and target statistics changes across layers, we show $d$ recorded at each layer during adaptation on subsets of CIFAR10/100-C, disaggregated by the severity of the distribution shift (1 being the least shifted, 5 being the most). From Figure~\ref{fig:layer_dist}, we can see that there are subtle changes between $d$ at the different severities, indicating that $d$ is able to detect changes in shift severity and dynamically adjust the balance between source and target statistics in our adaptation module.
The pattern of these curves are specific to their model architectures, but not the dataset. We see two distinct patterns. For the Resnet18 distances, at layers after 12 or 13, low-severity shifts begin to produce greater divergences between source and target statistics, and the distances begin to explode. For MobileNetV2, we see oscillatory behavior of $d$, which is a possible area of further investigation but is likely specific to the architecture of the model. There are two peaks of high distance in the MobileNetV2 architecture For future works, areas of high distance (and thus, implied high variance) may allow us to further optimize which layers we fuse, versus which layers we add adaptation to, potentially leading to accuracy gains.
\begin{figure}[t]
\centering
\includegraphics[width=0.8\linewidth]{figs/layerwise_distance.pdf}
\caption{Layerwise Mahalnobis distance $d$ calculated after stabilization in Equation~\ref{eqn:mahalanobis}, separated by severity of distribution shift in the CIFAR10/100-C datasets (light being the least severe, dark being the most). Shading indicates one standard deviation on either side of the average over N=5 trials.}
\label{fig:layer_dist}
\end{figure}
\section{MobileViT Experiments} \label{app:mobivit}
MobileViT~\cite{mehta22_mobilevit} is a transformer-convolution hybrid, which uses ``transformers as convolutions''. It has been adopted as a state-of-the-art model which combines the representational power of local spatial inductive biases (from convolutions) and the global structural understanding of transformers. The hybrid architecture makes it a good candidate for on-device deployments, as the inclusion of inductive biases improves accuracy over similarly sized transformer models~\cite{mehta22_mobilevit}. We applied \systemName to the batch normalization layers of a full-precision MobileViT-small in order to understand its performance in interaction with transformers and layerwise normalization. The model was trained on ImageNet and fine-tuned with CIFAR10's training set, then tested on an abruptly shifting dataset sampled from CIFAR10-C, as described in our experimental methods.
As shown in Table \ref{tab:mobivit_results}, our method, especially when using partial adaptation over the shallowest half of the model (1st 8, 1st 16), outperforms all other methods, even at large batch sizes. We did not include batch 1 analyses because the nature of the normalization layers used in the MobileViT-s did not allow for training with batch size set to 1. Table \ref{tab:mobivit_results} demonstrates three things: the generalizability of our method, even to architectures with transformer and layer normalization layers; the potential importance of partial layerwise adaptation, supporting our findings from Figure \ref{fig:layer_ablation}; and the breakdown of other methods under certain data conditions.
\begin{table}[t]
\centering
\caption{\% Accuracy on abruptly changing CIFAR10-C datasets using the MobileViT-s architecture (32 BN layers) fine-tuned to the CIFAR10 dataset. N=5 trials. \textbf{Bold}: best, \underline{underline}: 2nd best, \textit{italics}: worse than baseline.}
\label{tab:mobivit_results}
\resizebox{0.4\linewidth}{!}{
\begin{tabular}{l | l | c}
\toprule
\textbf{Method} & \textbf{Variant} &
\textbf{Accuracy} \\
\midrule
None & Batch 1 & 72.0 $\pm$ 0.4 \\
\midrule
Ours & Full Adap. & 73.8 $\pm$ 0.7 \\
Ours & 1st 8 & \underline{75.3 $\pm$ 0.3} \\
Ours & 1st 16 & \textbf{75.6 $\pm$ 0.3} \\
Ours & 1st 24 & 74.8 $\pm$ 0.5 \\
\midrule
Tent & Batch 64 & 75.2 $\pm$ 0.4 \\
Tent & Batch 16 & \textit{69.8 $\pm$ 1.0} \\
\midrule
EATA & Batch 64 & 74.2 $\pm$ 0.2 \\
EATA & Batch 16 & \textit{71.1 $\pm$ 0.7} \\
\midrule
CoTTA & Batch 64 & \textit{60.9 $\pm$ 1.0} \\
CoTTA & Batch 16 & \textit{57.7 $\pm$ 0.5} \\
\midrule
RealisticTTA & Batch 2 & \textit{15.5 $\pm$ 0.4} \\
\bottomrule
\end{tabular}}
\vspace{-1em}
\end{table}
\begin{thebibliography}{48}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Benz et~al.(2021)Benz, Zhang, Karjauv, and Kweon]{benz21_rstats}
Benz, P., Zhang, C., Karjauv, A., and Kweon, I.~S.
\newblock Revisiting batch normalization for improving corruption robustness.
\newblock In \emph{Proceedings of the IEEE/CVF winter conference on applications of computer vision}, pp.\ 494--503, 2021.
\bibitem[Colombo et~al.(2022)Colombo, Dadalto, Staerman, Noiry, and Piantanida]{colombo2022beyond}
Colombo, P., Dadalto, E., Staerman, G., Noiry, N., and Piantanida, P.
\newblock Beyond mahalanobis distance for textual ood detection.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 17744--17759, 2022.
\bibitem[Dai et~al.(2019)Dai, Spasi{\'c}, Meyer, Chapman, and Andres]{dai19_skin}
Dai, X., Spasi{\'c}, I., Meyer, B., Chapman, S., and Andres, F.
\newblock Machine learning on mobile: An on-device inference app for skin cancer detection.
\newblock In \emph{2019 fourth international conference on fog and mobile edge computing (FMEC)}, pp.\ 301--305. IEEE, 2019.
\bibitem[De~Maesschalck et~al.(2000)De~Maesschalck, Jouan-Rimbaud, and Massart]{de2000mahalanobis}
De~Maesschalck, R., Jouan-Rimbaud, D., and Massart, D.~L.
\newblock The mahalanobis distance.
\newblock \emph{Chemometrics and intelligent laboratory systems}, 50\penalty0 (1):\penalty0 1--18, 2000.
\bibitem[Dhar et~al.(2021)Dhar, Guo, Liu, Tripathi, Kurup, and Shah]{dhar21_ondevice}
Dhar, S., Guo, J., Liu, J., Tripathi, S., Kurup, U., and Shah, M.
\newblock A survey of on-device machine learning: An algorithms and learning theory perspective.
\newblock \emph{ACM Transactions on Internet of Things}, 2\penalty0 (3):\penalty0 1--49, 2021.
\bibitem[Dong et~al.(2024)Dong, Konz, Gu, and Mazurowski]{dong24_intent}
Dong, H., Konz, N., Gu, H., and Mazurowski, M.~A.
\newblock Medical image segmentation with intent: Integrated entropy weighting for single image test-time adaptation, 2024.
\newblock URL \url{https://arxiv.org/abs/2402.09604}.
\bibitem[Gholami et~al.(2021)Gholami, Kim, Dong, Yao, Mahoney, and Keutzer]{gholami21_quantrev}
Gholami, A., Kim, S., Dong, Z., Yao, Z., Mahoney, M.~W., and Keutzer, K.
\newblock A survey of quantization methods for efficient neural network inference.
\newblock \emph{CoRR}, abs/2103.13630, 2021.
\newblock URL \url{https://arxiv.org/abs/2103.13630}.
\bibitem[Gong et~al.(2023)Gong, Jeong, Kim, Kim, Shin, and Lee]{gong23_note}
Gong, T., Jeong, J., Kim, T., Kim, Y., Shin, J., and Lee, S.-J.
\newblock Note: Robust continual test-time adaptation against temporal correlation, 2023.
\bibitem[Guo \& Li(2018)Guo and Li]{guo18_rural}
Guo, J. and Li, B.
\newblock The application of medical artificial intelligence technology in rural areas of developing countries.
\newblock \emph{Health Equity}, 2\penalty0 (1):\penalty0 174--181, 2018.
\newblock \doi{10.1089/heq.2018.0037}.
\newblock URL \url{https://doi.org/10.1089/heq.2018.0037}.
\newblock PMID: 30283865.
\bibitem[Han et~al.(2022)Han, Xia, Spathis, Bondareva, Brown, Chauhan, Dang, Grammenos, Hasthanasombat, Floto, et~al.]{han2022sounds}
Han, J., Xia, T., Spathis, D., Bondareva, E., Brown, C., Chauhan, J., Dang, T., Grammenos, A., Hasthanasombat, A., Floto, A., et~al.
\newblock Sounds of covid-19: exploring realistic performance of audio-based digital testing.
\newblock \emph{NPJ digital medicine}, 5\penalty0 (1):\penalty0 16, 2022.
\bibitem[He et~al.(2016)He, Zhang, Ren, and Sun]{he16_resnet}
He, K., Zhang, X., Ren, S., and Sun, J.
\newblock Deep residual learning for image recognition.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pp.\ 770--778, 2016.
\bibitem[Hendrycks(2019{\natexlab{a}})]{hendrycks19_c10}
Hendrycks, D.
\newblock Cifar-10-c and cifar-10-p, January 2019{\natexlab{a}}.
\newblock URL \url{https://doi.org/10.5281/zenodo.2535967}.
\bibitem[Hendrycks(2019{\natexlab{b}})]{hendrycks19_c100}
Hendrycks, D.
\newblock Cifar-100-c, December 2019{\natexlab{b}}.
\newblock URL \url{https://doi.org/10.5281/zenodo.3555552}.
\bibitem[Hershey et~al.(2017)Hershey, Chaudhuri, Ellis, Gemmeke, Jansen, Moore, Plakal, Platt, Saurous, Seybold, Slaney, Weiss, and Wilson]{vggish}
Hershey, S., Chaudhuri, S., Ellis, D. P.~W., Gemmeke, J.~F., Jansen, A., Moore, R.~C., Plakal, M., Platt, D., Saurous, R.~A., Seybold, B., Slaney, M., Weiss, R.~J., and Wilson, K.
\newblock Cnn architectures for large-scale audio classification.
\newblock In \emph{2017 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, pp.\ 131--135, 2017.
\newblock \doi{10.1109/ICASSP.2017.7952132}.
\bibitem[Hong et~al.(2023)Hong, Lyu, Zhou, and Spranger]{hong23_mecta}
Hong, J., Lyu, L., Zhou, J., and Spranger, M.
\newblock {MECTA}: Memory-economic continual test-time model adaptation.
\newblock In \emph{The Eleventh International Conference on Learning Representations}, 2023.
\newblock URL \url{https://openreview.net/forum?id=N92hjSf5NNh}.
\bibitem[Huang et~al.(2023)Huang, Yang, and Gao]{huang_elastictrainer_mobisys23}
Huang, K., Yang, B., and Gao, W.
\newblock Elastictrainer: Speeding up on-device training with runtime elastic tensor selection.
\newblock In \emph{Proceedings of the 21st Annual International Conference on Mobile Systems, Applications and Services}, MobiSys '23, pp.\ 56–69, New York, NY, USA, 2023. Association for Computing Machinery.
\newblock ISBN 9798400701108.
\newblock \doi{10.1145/3581791.3596852}.
\newblock URL \url{https://doi.org/10.1145/3581791.3596852}.
\bibitem[Ioffe \& Szegedy(2015)Ioffe and Szegedy]{ioffe15_bn}
Ioffe, S. and Szegedy, C.
\newblock Batch normalization: Accelerating deep network training by reducing internal covariate shift.
\newblock \emph{CoRR}, abs/1502.03167, 2015.
\newblock URL \url{http://arxiv.org/abs/1502.03167}.
\bibitem[Khurana et~al.(2021)Khurana, Paul, Rai, Biswas, and Aggarwal]{khurana21_sita}
Khurana, A., Paul, S., Rai, P., Biswas, S., and Aggarwal, G.
\newblock {SITA:} single image test-time adaptation.
\newblock \emph{CoRR}, abs/2112.02355, 2021.
\newblock URL \url{https://arxiv.org/abs/2112.02355}.
\bibitem[Koh et~al.(2020)Koh, Sagawa, Marklund, Xie, Zhang, Balsubramani, Hu, Yasunaga, Phillips, Beery, Leskovec, Kundaje, Pierson, Levine, Finn, and Liang]{koh20_wilds}
Koh, P.~W., Sagawa, S., Marklund, H., Xie, S.~M., Zhang, M., Balsubramani, A., Hu, W., Yasunaga, M., Phillips, R.~L., Beery, S., Leskovec, J., Kundaje, A., Pierson, E., Levine, S., Finn, C., and Liang, P.
\newblock {WILDS:} {A} benchmark of in-the-wild distribution shifts.
\newblock \emph{CoRR}, abs/2012.07421, 2020.
\newblock URL \url{https://arxiv.org/abs/2012.07421}.
\bibitem[Kong et~al.(2018)Kong, Iqbal, Xu, Wang, and Plumbley]{kong18_dcase}
Kong, Q., Iqbal, T., Xu, Y., Wang, W., and Plumbley, M.~D.
\newblock {DCASE} 2018 challenge baseline with convolutional neural networks.
\newblock \emph{CoRR}, abs/1808.00773, 2018.
\newblock URL \url{http://arxiv.org/abs/1808.00773}.
\bibitem[Krizhevsky et~al.(2009)Krizhevsky, Hinton, et~al.]{krizhevsky09_cifar}
Krizhevsky, A., Hinton, G., et~al.
\newblock Learning multiple layers of features from tiny images, 2009.
\bibitem[Kwon et~al.(2024)Kwon, Li, Venieris, Chauhan, Lane, and Mascolo]{kwon2024tinytrain}
Kwon, Y.~D., Li, R., Venieris, S.~I., Chauhan, J., Lane, N.~D., and Mascolo, C.
\newblock Tinytrain: Resource-aware task-adaptive sparse training of dnns at the data-scarce edge.
\newblock In \emph{International Conference on Machine Learning}, 2024.
\bibitem[Li et~al.(2016)Li, Wang, Shi, Liu, and Hou]{li16_adabn}
Li, Y., Wang, N., Shi, J., Liu, J., and Hou, X.
\newblock Revisiting batch normalization for practical domain adaptation, 2016.
\newblock URL \url{https://arxiv.org/abs/1603.04779}.
\bibitem[Liang et~al.(2023)Liang, He, and Tan]{liang23_litrev}
Liang, J., He, R., and Tan, T.
\newblock A comprehensive survey on test-time adaptation under distribution shifts, 2023.
\bibitem[Lin et~al.(2024)Lin, Zhu, Chen, Wang, Gan, and Han]{lin24_tt}
Lin, J., Zhu, L., Chen, W.-M., Wang, W.-C., Gan, C., and Han, S.
\newblock On-device training under 256kb memory, 2024.
\bibitem[Mehta \& Rastegari(2022)Mehta and Rastegari]{mehta22_mobilevit}
Mehta, S. and Rastegari, M.
\newblock Mobilevit: Light-weight, general-purpose, and mobile-friendly vision transformer, 2022.
\newblock URL \url{https://arxiv.org/abs/2110.02178}.
\bibitem[Mollah et~al.(2017)Mollah, Azad, and Vasilakos]{mollah17_mobsec}
Mollah, M.~B., Azad, M. A.~K., and Vasilakos, A.
\newblock Security and privacy challenges in mobile cloud computing: Survey and way ahead.
\newblock \emph{Journal of Network and Computer Applications}, 84:\penalty0 38--54, 2017.
\bibitem[Niu et~al.(2022)Niu, Wu, Zhang, Chen, Zheng, Zhao, and Tan]{niu22_eata}
Niu, S., Wu, J., Zhang, Y., Chen, Y., Zheng, S., Zhao, P., and Tan, M.
\newblock Efficient test-time model adaptation without forgetting.
\newblock In Chaudhuri, K., Jegelka, S., Song, L., Szepesvari, C., Niu, G., and Sabato, S. (eds.), \emph{Proceedings of the 39th International Conference on Machine Learning}, volume 162 of \emph{Proceedings of Machine Learning Research}, pp.\ 16888--16905. PMLR, 17--23 Jul 2022.
\newblock URL \url{https://proceedings.mlr.press/v162/niu22a.html}.
\bibitem[Niu et~al.(2023)Niu, Wu, Zhang, Wen, Chen, Zhao, and Tan]{niu23_sar}
Niu, S., Wu, J., Zhang, Y., Wen, Z., Chen, Y., Zhao, P., and Tan, M.
\newblock Towards stable test-time adaptation in dynamic wild world, 2023.
\bibitem[Niu et~al.(2024)Niu, Miao, Chen, Wu, and Zhao]{niu24_foa}
Niu, S., Miao, C., Chen, G., Wu, P., and Zhao, P.
\newblock Test-time model adaptation with only forward passes, 2024.
\bibitem[Podolskiy et~al.(2021)Podolskiy, Lipin, Bout, Artemova, and Piontkovskaya]{podolskiy2021revisiting}
Podolskiy, A., Lipin, D., Bout, A., Artemova, E., and Piontkovskaya, I.
\newblock Revisiting mahalanobis distance for transformer-based out-of-domain detection.
\newblock \emph{Proceedings of the AAAI conference on artificial intelligence}, 35\penalty0 (15):\penalty0 13675--13682, 2021.
\bibitem[Qayyum et~al.(2020)Qayyum, Ijaz, Usama, Iqbal, Qadir, Elkhatib, and Al-Fuqaha]{qayyum20_mlsec}
Qayyum, A., Ijaz, A., Usama, M., Iqbal, W., Qadir, J., Elkhatib, Y., and Al-Fuqaha, A.
\newblock Securing machine learning in the cloud: A systematic review of cloud machine learning security.
\newblock \emph{Front. Big Data}, 3:\penalty0 587139, November 2020.
\bibitem[Sandler et~al.(2018)Sandler, Howard, Zhu, Zhmoginov, and Chen]{sandler18_mbnet}
Sandler, M., Howard, A., Zhu, M., Zhmoginov, A., and Chen, L.-C.
\newblock Mobilenetv2: Inverted residuals and linear bottlenecks.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pp.\ 4510--4520, 2018.
\bibitem[Schneider et~al.(2020)Schneider, Rusak, Eck, Bringmann, Brendel, and Bethge]{schneider20_navg}
Schneider, S., Rusak, E., Eck, L., Bringmann, O., Brendel, W., and Bethge, M.
\newblock Improving robustness against common corruptions by covariate shift adaptation, 2020.
\newblock URL \url{https://arxiv.org/abs/2006.16971}.
\bibitem[{Shariat Panah} et~al.(2022){Shariat Panah}, Hines, McKeever, and McKeever]{shariat22_phoneheart}
{Shariat Panah}, D., Hines, A., McKeever, J.~A., and McKeever, S.
\newblock An audio processing pipeline for acquiring diagnostic quality heart sounds via mobile phone.
\newblock \emph{Computers in Biology and Medicine}, 145:\penalty0 105415, 2022.
\newblock ISSN 0010-4825.
\newblock \doi{https://doi.org/10.1016/j.compbiomed.2022.105415}.
\newblock URL \url{https://www.sciencedirect.com/science/article/pii/S0010482522002074}.
\bibitem[Stowell et~al.(2018)Stowell, Stylianou, Wood, Pamu{\l}a, and Glotin]{stowell18_birds}
Stowell, D., Stylianou, Y., Wood, M., Pamu{\l}a, H., and Glotin, H.
\newblock Automatic acoustic detection of birds through deep learning: the first bird audio detection challenge.
\newblock \emph{Methods in Ecology and Evolution}, 2018.
\newblock URL \url{https://arxiv.org/abs/1807.05812]}.
\bibitem[Su et~al.(2024)Su, Guo, Yao, Yang, Wang, and Huang]{su24_tema}
Su, Z., Guo, J., Yao, K., Yang, X., Wang, Q., and Huang, K.
\newblock Unraveling batch normalization for realistic test-time adaptation, 2024.
\bibitem[Sze et~al.(2017)Sze, Chen, Yang, and Emer]{sze17_effdnn}
Sze, V., Chen, Y.-H., Yang, T.-J., and Emer, J.~S.
\newblock Efficient processing of deep neural networks: A tutorial and survey.
\newblock \emph{Proceedings of the IEEE}, 105\penalty0 (12):\penalty0 2295--2329, 2017.
\bibitem[Tan \& Le(2020)Tan and Le]{tan20_efficientnet}
Tan, M. and Le, Q.~V.
\newblock Efficientnet: Rethinking model scaling for convolutional neural networks, 2020.
\newblock URL \url{https://arxiv.org/abs/1905.11946}.
\bibitem[Wang et~al.(2021)Wang, Shelhamer, Liu, Olshausen, and Darrell]{wang21_tent}
Wang, D., Shelhamer, E., Liu, S., Olshausen, B., and Darrell, T.
\newblock Tent: Fully test-time adaptation by entropy minimization, 2021.
\bibitem[Wang et~al.(2022)Wang, Fink, Gool, and Dai]{wang22_cotta}
Wang, Q., Fink, O., Gool, L.~V., and Dai, D.
\newblock Continual test-time domain adaptation, 2022.
\bibitem[Xu et~al.(2022)Xu, Xu, Wang, Wang, Ma, Huang, Huang, Jin, and Liu]{xu_mandheling_2022}
Xu, D., Xu, M., Wang, Q., Wang, S., Ma, Y., Huang, K., Huang, G., Jin, X., and Liu, X.
\newblock {Mandheling: Mixed-Precision On-Device {DNN} Training with {DSP} Offloading}.
\newblock In \emph{{Annual} {International} {Conference} on {Mobile} {Computing} {And} {Networking} (MobiCom)}, 2022.
\bibitem[Yuan et~al.(2023)Yuan, Xie, and Li]{yuan23_rotta}
Yuan, L., Xie, B., and Li, S.
\newblock Robust test-time adaptation in dynamic scenarios, 2023.
\newblock URL \url{https://arxiv.org/abs/2303.13899}.
\bibitem[Yun \& Wong(2021)Yun and Wong]{yun21_mobinet}
Yun, S. and Wong, A.
\newblock Do all mobilenets quantize poorly? gaining insights into the effect of quantization on depthwise separable convolutional networks through the eyes of multi-scale distributional dynamics.
\newblock \emph{CoRR}, abs/2104.11849, 2021.
\newblock URL \url{https://arxiv.org/abs/2104.11849}.
\bibitem[Zhang et~al.(2017)Zhang, Zhou, Lin, and Sun]{zhang17_shufflenet}
Zhang, X., Zhou, X., Lin, M., and Sun, J.
\newblock Shufflenet: An extremely efficient convolutional neural network for mobile devices, 2017.
\newblock URL \url{https://arxiv.org/abs/1707.01083}.
\bibitem[Zhao et~al.(2023)Zhao, Liu, Alahi, and Lin]{zhao23_pitfalls}
Zhao, H., Liu, Y., Alahi, A., and Lin, T.
\newblock On pitfalls of test-time adaptation, 2023.
\newblock URL \url{https://arxiv.org/abs/2306.03536}.
\bibitem[Zhou et~al.(2022)Zhou, Zhang, Zang, Yang, Loy, and Liu]{zhou22_device}
Zhou, K., Zhang, Y., Zang, Y., Yang, J., Loy, C.~C., and Liu, Z.
\newblock On-device domain generalization.
\newblock \emph{arXiv preprint arXiv:2209.07521}, 2022.
\bibitem[Zualkernan et~al.(2021)Zualkernan, Judas, Mahbub, Bhagwagar, and Chand]{zualkernan2021aiot}
Zualkernan, I., Judas, J., Mahbub, T., Bhagwagar, A., and Chand, P.
\newblock An aiot system for bat species classification.
\newblock In \emph{2020 IEEE international conference on Internet of Things and Intelligence System (IoTaIS)}, pp.\ 155--160. IEEE, 2021.
\end{thebibliography}