SPAR: Self-supervised Placement-Aware Representation Learning for Distributed Sensing
Extracted source text · arXiv 2505.16936
%%%%%%%% ICML 2026 EXAMPLE LATEX SUBMISSION FILE %%%%%%%%%%%%%%%%%
\documentclass{article}
\input{math_commands.tex}
% Recommended, but optional, packages for figures and better typesetting:
\usepackage{microtype}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs} % for professional tables
% hyperref makes hyperlinks in the resulting PDF.
% If your build breaks (sometimes temporarily if a hyperlink spans a page)
% please comment out the following usepackage line and replace
% \usepackage{icml2026} with \usepackage[nohyperref]{icml2026} above.
\usepackage{hyperref}
% Attempt to make hyperref and algorithmic work together better:
\newcommand{\theHalgorithm}{\arabic{algorithm}}
% Use the following line for the initial blind version submitted for review:
% \usepackage{icml2026}
% For preprint, use
\usepackage[preprint]{icml2026}
% If accepted, instead use the following line for the camera-ready submission:
% \usepackage[accepted]{icml2026}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
% if you use cleveref..
\usepackage[capitalize,noabbrev]{cleveref}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% THEOREMS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
% Todonotes is useful during development; simply uncomment the next line
% and comment out the line below the next line to turn off comments
%\usepackage[disable,textsize=tiny]{todonotes}
\usepackage[textsize=tiny]{todonotes}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{url} % simple URL typesetting
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage[table]{xcolor} % colors
\usepackage{wasysym}
\usepackage{wrapfig}
\usepackage{mwe}
\usepackage{multirow}
\usepackage{adjustbox}
\usepackage{physics}
\usepackage{sansmath}
\newcommand{\MODEL}{{SPAR}}
\newcommand{\etal}{\textit{et al.}\xspace}
\newcommand{\mask}{\text{\texttt{mask}}}
\newcommand{\concat}{\text{\texttt{cat}}}
\newcommand{\fspembed}{\gF_\mathrm{sp\_embed}^{(k)}}
\newcommand{\fspembednk}{\gF_\mathrm{sp\_embed}}
\newcommand{\fstembed}{\gF_\mathrm{st\_embed}^{(k)}}
\newcommand{\fstembednk}{\gF_\mathrm{st\_embed}}
\newcommand{\fsigembed}{\gF_\mathrm{sig\_embed}^{(k)}}
\newcommand{\fsigembednk}{\gF_\mathrm{sig\_embed}}
\newcommand{\fenc}{\gF_\mathrm{enc}^{(k)}}
\newcommand{\fencnk}{\gF_\mathrm{enc}}
\newcommand{\fembencnk}{\gF_\mathrm{embed\_enc}}
\newcommand{\fencnktilde}{\widetilde{\gF}_\mathrm{enc}}
\newcommand{\fjointenc}{\gF_\mathrm{joint\_enc}}
\newcommand{\fsigdec}{\gF_\mathrm{sig\_dec}^{(k)}}
\newcommand{\fsigdecnk}{\gF_\mathrm{sig\_dec}}
\newcommand{\fspdec}{\gF_\mathrm{sp\_dec}^{(k)}}
\newcommand{\fdecnk}{\gF_\mathrm{dec}}
\newcommand{\fsigdecnktilde}{\widetilde{\gF}_\mathrm{sig\_dec}}
\newcommand{\fspdecnktilde}{\widetilde{\gF}_\mathrm{sp\_dec}}
\newcommand{\fdecnktilde}{\widetilde{\gF}_\mathrm{dec}}
% The \icmltitle you define below is probably too long as a header.
% Therefore, a short form for the running title is supplied here:
\icmltitlerunning{SPAR: Self-supervised Placement-Aware Representation Learning for Distributed Sensing}
\begin{document}
\twocolumn[
\icmltitle{SPAR: Self-supervised Placement-Aware Representation Learning for Distributed Sensing}
% It is OKAY to include author information, even for blind submissions: the
% style file will automatically remove it for you unless you've provided
% the [accepted] option to the icml2026 package.
% List of affiliations: The first argument should be a (short) identifier you
% will use later to specify author affiliations Academic affiliations
% should list Department, University, City, Region, Country Industry
% affiliations should list Company, City, Region, Country
% You can specify symbols, otherwise they are numbered in order. Ideally, you
% should not use this facility. Affiliations will be numbered in order of
% appearance and this is the preferred way.
\icmlsetsymbol{equal}{*}
\begin{icmlauthorlist}
\icmlauthor{Yizhuo Chen}{uiuc}
\icmlauthor{Tianchen Wang}{uiuc}
\icmlauthor{You Lyu}{uiuc}
\icmlauthor{Yanlan Hu}{stanford}
\icmlauthor{Jinyang Li}{uiuc}
\icmlauthor{Tomoyoshi Kimura}{uiuc}
\icmlauthor{Hongjue Zhao}{uiuc}
\icmlauthor{Yigong Hu}{uiuc}
\icmlauthor{Denizhan Kara}{uiuc}
\icmlauthor{Tarek Abdelzaher}{uiuc}
\end{icmlauthorlist}
\icmlaffiliation{uiuc}{University of Illinois Urbana-Champaign, Urbana, IL, USA}
\icmlaffiliation{stanford}{Stanford University, Stanford, CA, USA}
\icmlcorrespondingauthor{Yizhuo Chen}{yizhuoc@illinois.edu}
\icmlcorrespondingauthor{Jinyang Li}{jinyang7@illinois.edu}
% You may provide any keywords that you find helpful for describing your
% paper; these are used to populate the "keywords" metadata in the PDF but
% will not be shown in the document
\icmlkeywords{Machine Learning, ICML}
\vskip 0.3in
]
% this must go after the closing bracket ] following \twocolumn[ ...
% This command actually creates the footnote in the first column listing the
% affiliations and the copyright notice. The command takes one argument, which
% is text to display at the start of the footnote. The \icmlEqualContribution
% command is standard text for equal contribution. Remove it (just {}) if you
% do not need this facility.
% Use ONE of the following lines. DO NOT remove the command.
% If you have no special notice, KEEP empty braces:
\printAffiliationsAndNotice{} % no special notice (required even if empty)
% Or, if applicable, use the standard equal contribution text:
% \printAffiliationsAndNotice{\icmlEqualContribution}
\begin{abstract}
We present SPAR, a framework for self-supervised placement-aware representation learning in distributed sensing. Distributed sensing spans applications where multiple spatially distributed and multimodal sensors jointly observe an environment, from vehicle monitoring to human activity recognition and earthquake localization. A central challenge shared by this wide spectrum of applications is that observed signals are inseparably shaped by sensor placements, including their spatial locations and structural characteristics. However, existing pretraining methods remain largely placement-agnostic. SPAR addresses this gap through a unifying principle: the duality between signals and positions. Guided by this principle, SPAR introduces spatial and structural positional embeddings together with dual reconstruction objectives, explicitly modeling how observing positions and observed signals shape each other. Placement is thus treated not as auxiliary metadata but as intrinsic to representation learning. SPAR is theoretically supported by analyses from information theory and occlusion-invariant learning. Extensive experiments on three real-world datasets show that SPAR achieves superior robustness and generalization across various modalities, placements, and downstream tasks.
\end{abstract}
\section{Introduction}
\label{sec:intro}
\begin{figure*}[!t] \centering
\includegraphics[width=0.9\linewidth]{fig/problem.png} \vspace{-0.1in} \caption{\small An overview of the \MODEL{} workflow applied to a multi-modal multi-node distributed sensing application. Each node from each modality collects its own signal and is associated with a spatial position, as well as unique characteristics that influence its signal patterns. During pretraining, \MODEL{} encodes information from all these aspects to generate latent embeddings, optimized via self-supervised objectives on unlabeled data. In the fine-tuning stage, the encoder is frozen and used to extract representations, which are then fed into task-specific heads trained with labeled data for downstream tasks.} \vspace{-0.1in} \label{fig:motivation} \end{figure*}
This paper advances self-supervised \textbf{placement-aware representation learning}, motivated by the broad class of applications we term \textbf{distributed sensing}. By distributed sensing, we refer to systems where multiple spatially distributed sensing points, potentially spanning diverse modalities, jointly observe an environment. This definition unifies a wide spectrum of domains, including seismic and acoustic monitoring for security~\citep{li2025restoreml, caltech1926socal}, human activity recognition with body-worn sensors~\citep{gu2021survey, sztyler2016body}, vehicle monitoring in urban spaces~\citep{bathla2022autonomous}, environmental monitoring~\citep{ullo2020advances}, and smart cities~\citep{syed2021iot}. These scenarios, though superficially distinct, share the common challenge of reconstructing or representing an environment from heterogeneous, distributed vantage points.
\textbf{Sensor placement} lies at the core of distributed sensing. A sensor’s vantage point is determined by both its \textbf{spatial location} (e.g., GPS coordinates of roadside microphones monitoring passing vehicles, governing the propagation paths of acoustic waves) and its \textbf{structural characteristic} (e.g., differences in sensor mounting, orientation, or coupling to the ground, which shape frequency response and noise patterns). Robust representation learning in this setting requires models that not only capture signal content but also interpret how those signals are mediated by sensor placement.
Despite rapid progress in pretraining, current approaches, whether contrastive~\citep{ouyang2024mmbind}, generative reconstruction~\citep{kara2024freqmae}, or language-model-based~\citep{ouyang2024llmsense}, remain largely placement-agnostic, overlooking the fact that distributed sensing signals are inseparably shaped by sensor placement. This omission limits generalization across layouts and tasks.
To address this gap, we introduce \MODEL{} (\textbf{S}elf-supervised \textbf{P}lacement-\textbf{A}ware \textbf{R}epresentation learning), a general-purpose pretraining framework that explicitly incorporates placement into representation learning for distributed sensing. Our design is guided by a core principle: the \textbf{duality between positions and signals}. That is, spatial and structural configurations are not auxiliary metadata to the signals, but stand in an equal and mutually-determining relationship with signals. Together, they define how observations are generated, propagated, and interpreted. This principle is both general and intrinsic, applying across the full spectrum of distributed sensing applications.
Building on this principle, \MODEL{} introduces explicit spatial and structural positional embeddings, and introduces \textbf{dual reconstruction objectives} that enforce the mutual recoverability of placements and signals with contextual awareness. Together, these elements yield a cohesive, placement-aware pretraining strategy that is broadly applicable across sensing modalities and layouts. An overview of the \MODEL{} workflow is illustrated in Figure~\ref{fig:motivation}. To our knowledge, this is the first work to treat placement as a universal inductive bias for distributed sensing systems as a whole, rather than as an application-specific add-on.
We further provide theoretical analyses grounded in information theory and occlusion-invariant representation learning~\citep{kong2023understanding} to elucidate the rationale behind our design. Experiments on three real-world datasets, covering vehicle monitoring~\citep{li2025restoreml}, human activity recognition~\citep{sztyler2016body}, and earthquake localization~\citep{caltech1926socal}, demonstrate that \MODEL{} consistently outperforms existing approaches across diverse sensing modalities, spatial configurations, and downstream tasks.
In summary, this paper makes the following contributions: \textbf{(1)} We introduce \MODEL{}, a novel, general pretraining framework for distributed sensing that explicitly models spatial layouts and node-specific characteristics, guided by the duality between positions and signals. \textbf{(2)} We provide theoretical analyses from information-theoretic and occlusion-invariant perspectives that explain the effectiveness of our design. \textbf{(3)} We validate \MODEL{} through extensive experiments on three real-world distributed sensing datasets, demonstrating superior generalizability and robustness compared to prior methods. We release our implementation and scripts at \url{https://anonymous.4open.science/r/SPAR-F6F3/}.
\section{Related Work}\label{sec:rw}
\noindent\textbf{Pretraining and Foundation Models for Sensing.}
Pretraining for sensing seeks transferable representations from unlabeled data and broadly follows three paradigms: contrastive learning, masked reconstruction, and LLM-based frameworks. Contrastive methods align multi-modal embeddings, ranging from intra-sample approaches such as Cosmo~\citep{ouyang2022cosmo}, Cocoa~\citep{deldari2022cocoa}, and FOCAL~\citep{liu2023focal}, to large-scale or loosely paired models like ImageBind~\citep{girdhar2023imagebind} and MMBind~\citep{ouyang2024mmbind}. Generative approaches are dominated by masked autoencoding~\citep{woo2024unified, das2024decoder}, including time-series adaptations such as Ti-MAE~\citep{li2023ti}, MOMENT~\citep{goswami2024moment}, TS-MAE~\citep{liu2025ts}, and frequency-aware variants like FreqMAE~\citep{kara2024freqmae} and PhyMask~\citep{kara2024phymask}. LLM-based frameworks integrate sensing into language-centric models~\citep{gruver2023large, garza2023timegpt}, including LIMU-BERT~\citep{xu2021limu}, Penetrative AI~\citep{xu2024penetrative}, LLMSense~\citep{ouyang2024llmsense}, and IoT-LM~\citep{mo2024iot}. Despite their success, these methods remain largely placement-agnostic, overlooking the spatial layout and node-specific characteristics intrinsic to distributed sensing; in contrast, \MODEL{} explicitly incorporates spatial and structural information during pretraining to improve contextual grounding and robustness.
\noindent\textbf{Pretraining with Different Notions of "Spatial" Context.}
Several works incorporate spatial context, though definitions of “spatial” differ. In vision, it typically refers to grids of pixels or patches, as in video~\citep{feichtenhofer2022masked, wu2023dropmae}, remote sensing~\citep{lin2023ss, reed2023scale, irvin2023usat}, and 3D medical imaging~\citep{gu2024self}. Beyond vision, "spatial" often denotes discrete symbolic entities, e.g., joints in SkeletonMAE~\citep{wu2023skeletonmae}, EEG channels in MV-SSTMA~\citep{li2022multi} and MMM~\citep{yi2023learning}, or sensor identities in Gao~\etal~\citep{gao2023spatial} and Miao~\etal~\citep{miao2024spatial}. By contrast, our method integrates continuous node coordinates into pretraining, enabling modeling of arbitrary sensor layouts. Another related line is scene reconstruction and novel view synthesis~\citep{mildenhall2021nerf,kerbl20233d,wu20244d}, which also exploits spatial layouts but targets synthesis quality, rather than transferable representations for sensing.
\noindent\textbf{Pretraining via Positional Reconstruction Objectives.}
A third line of work, often without using the term “spatial,” incorporates positional reconstruction objectives. In vision, Doersch~\etal~\citep{doersch2015unsupervised} proposed predicting relative patch positions, extended by jigsaw~\citep{noroozi2016unsupervised} and content restoration~\citep{kim2018learning}. DeepPermNet~\citep{santa2017deeppermnet} learns permutation structures, MP3~\citep{zhai2022position} predicts absolute patch locations, and LOCA~\citep{caron2024location} predicts relative positions of clustered patches. In NLP, StructBERT~\citep{wang2019structbert}, ALBERT~\citep{lan2019albert}, and SLM~\citep{lee2020slm} use sentence order prediction and sequence restoration, while Nandy~\etal~\citep{nandy2024order} extend to permutation-based objectives. Beyond vision and language, GeoMAE~\citep{tian2023geomae} reconstructs geometric features of masked point clouds, and LEGO~\citep{sun20243d} recovers perturbed molecular geometries. While conceptually related, these methods operate on discrete, domain-specific positional targets (e.g., patch indices, sentence order). By contrast, our approach reconstructs continuous physical positions of nodes in distributed sensing applications.
\section{Method}\label{sec:method}
\begin{figure*}[!tbp]
\centering
% \vspace{-0.1in}
\includegraphics[width=\linewidth]{fig/framework.png}
\vspace{-0.2in}
\caption{\small Architectural overview of \MODEL{}. Each node is assigned a continuous learnable structural position to capture its unique characteristics. The signals, spatial positions, and structural positions of all nodes are projected into a shared embedding space, combined, and encoded into latent embeddings. The latent embeddings are optimized with dual reconstruction objectives, encouraging the model to effectively utilize and retain both signal and positional information in a self-supervised and context-aware manner. }
\label{fig:framework}
\vspace{-0.1in}
\end{figure*}
% high-level goal and framework summary
To develop a pretraining method that effectively utilizes the unique placement characteristics of sensing nodes, we propose \MODEL{}, which explicitly leverages \textbf{the duality between observer placement and signals} in the distributed sensing data. Specifically, we extend the traditional MAE framework by introducing explicit spatial and structural positional embeddings, to represent device locations and to encapsulate effects of other placement characteristics (such as orientation). Furthermore, we propose to optimize our model with \textbf{dual reconstruction objectives} to enhance its ability to retain both signal and spatial information in its learned representations. The overall architecture of \MODEL{} is shown in Figure~\ref{fig:framework}, with each component detailed below. Importantly, \MODEL{} is grounded in solid theoretical foundations from both information theory and the study of occlusion-invariant representations, offering deep insights into our design.
% notation
For clarity, we adopt the following notation convention throughout the paper: scalars are denoted by lowercase or uppercase letters (e.g., $k, K$), matrices by bold uppercase letters (e.g., $\mR, \mS$), tensors by bold calligraphic letters (e.g., $\tR, \tS$), and random tensor variables by sans-serif uppercase letters (e.g., $\mathsf{R}, \mathsf{S}$). We use $\gF$ with appropriate subscripts to denote the forward operations of various transformer-based modules. A summary of notations is provided in Table~\ref{tb:notation} in Appendix~\ref{app:notation}.
\subsection{Embedding for Signals, Spatial Positions, and Structural Positions}\label{sec:method_embed}
% basic system assumptions and notations and signal embeddings
We consider a multi-modality distributed sensing system with $K$ modalities, where the $k$-th modality ($k \in \{1,\dots, K\}$) consists of $n^{(k)}$ sensing nodes. The signals collected from these nodes are first tokenized to be compatible with transformer encoders. The tokenization strategy is task-specific and flexible. For example, IMU time-series data can be divided into temporal segments, while acoustic spectrograms can be split into patches. We denote the tokenized signals as $\tX^{(k)} \in \sR^{n^{(k)} \times m^{(k)} \times d_\tX^{(k)}}$, where $m^{(k)}$ is the number of tokens and $d_\tX^{(k)}$ is the token dimension. We then project these tokens into the transformer embedding space using a learnable linear layer, as $\widetilde{\tX}^{(k)}_{i,j,:} = \fsigembed(\tX^{(k)}_{i,j,:})$, yielding signal embeddings $\widetilde{\tX}^{(k)} \in \sR^{n^{(k)} \times m^{(k)} \times d}$, where $d$ is the transformer model dimension.
% spatial positional embedding
A distinguishing aspect of distributed sensing data is the availability of \textbf{spatial positions} of the nodes, reflecting their physical layout in the field, which can be denoted as $\mS^{(k)} \in \sR^{n^{(k)} \times d_\mS}$. For instance, in Figure~\ref{fig:framework}, the spatial positions are two-dimensional, indicating longitudinal and lateral node locations. Unlike the discrete ordinal indices typically used in NLP~\citep{vaswani2017attention} or CV~\citep{dosovitskiy2020image}, spatial positions in distributed sensing data are continuous vectors, making classical positional embedding strategies unsuitable~\citep{vaswani2017attention, dosovitskiy2020image, su2024roformer, press2021train}. To address this, we propose to continuously project the spatial positions into the embedding space as $\widetilde{\tS}^{(k)}_{i,j,:} = \fspembed(\tS_{i,j,:}^{(k)})$, where $\tS_{:,j,:}^{(k)} = \mS^{(k)}$ is the spatial positions broadcasted to match the dimension of the tokenized signals. The spatial positional embeddings $\widetilde{\tS}^{(k)}$ are then added to the signal embeddings to incorporate spatial context into the model.
However, two challenges arise in practice. First, spatial positions may vary widely in absolute locations and scales. For the example in Figure~\ref{fig:framework}, data may be collected in different cities, with some layouts covering small parking lots and others spanning large open areas. To ensure consistency, we \textbf{normalize} the spatial positions of each sample to have zero mean and unit variance. Second, existing datasets often contain only a limited number of distinct spatial layouts for which data were collected, leading the spatial embeddings (and the model) to overfit in pretraining, reducing generalizability to potentially unseen spatial arrangements during fine-tuning or testing. To mitigate this, we apply \textbf{geometric augmentation} during pretraining by randomly rotating and translating the normalized spatial positions, improving robustness to unseen layouts.
% structural positional embedding
While spatial positions capture physical layout, they do not fully represent structural placement conditions, such as the body part a sensor is attached to, or the orientation used for a directional measurement device (e.g., front-facing versus rear-facing camera on an autonomous car). Manually labeling these characteristics for all nodes is often costly and non-scalable. To address this, we assign each node a continuous learnable vector, called \textbf{structural position}. The structural positions for all nodes are denoted as $\mR^{(k)} \in \sR^{n^{(k)} \times d_\mR}$, where we typically choose the dimension of structural position $d_\mR \ll d$ to ensure training efficiency and scalability to large-scale sensing applications. As with spatial positions, we broadcast $\mR^{(k)}$ to form $\tR^{(k)} \in \sR^{n^{(k)} \times m^{(k)} \times d_\mR^{(k)}}$, project it into the embedding space via $\widetilde{\tR}^{(k)}_{i,j,:} = \fstembed(\tR^{(k)}_{i,j,:})$, and add it to the signal embeddings. These learnable structural positions are trained jointly with the rest of the model in the pretraining stage, enabling it to automatically capture node-specific information.
\subsection{Masked Autoencoding with Dual Reconstruction Objectives}
After combining the signal embeddings $\widetilde{\tX}^{(k)}$, spatial positional embeddings $\widetilde{\tS}^{(k)}$, and structural positional embeddings $\widetilde{\tR}^{(k)}$ (as well as any additional task-specific positional embeddings, such as 2D patch positions in a spectrogram, which we omit in the rest of this paper for clarity), we apply a binary mask $\mM^{(k)} \in \{0,1\}^{n^{(k)} \times m^{(k)}}$ over the combined embeddings to randomly mask out a fraction of tokens. The unmasked tokens are then fed into a per-modality transformer encoder to produce latent embeddings $\mZ^{(k)}$:
\[
\mZ^{(k)} = \fenc(\mask(\widetilde{\tX}^{(k)}+\widetilde{\tS}^{(k)}+\widetilde{\tR}^{(k)}; \mM^{(k)})),
\]
where $\mask(\cdot; \cdot)$ denotes the masking operation. To enable cross-modal interactions, we then apply a joint transformer encoder over the concatenated latent embeddings from all modalities:
\[
(\widetilde{\mZ}^{(1)},\dots, \widetilde{\mZ}^{(K)}) = \fjointenc(\concat(\mZ^{(1)},\dots, \mZ^{(K)})),
\]
where $\concat(\cdot)$ denotes contatenation, and $\widetilde{\mZ}^{(k)}$ denotes the post-fusion latent embeddings for the $k$-th modality. In the fine-tuning stage, the encoders are frozen, and the post-fusion latent embeddings are extracted and passed into a task-specific prediction head, which is trained using appropriate supervised objectives.
During the pretraining stage, however, the post-fusion latent embeddings are decoded, enabling the encoders to be optimized with self-supervised objectives. In the standard MAE framework, a single decoder is typically used to reconstruct the masked signals, which overlooks the rich spatial and structural context inherent in distributed sensing data. To address this, we introduce two decoders with \textbf{dual reconstruction objectives}, explicitly exploiting the duality between positions and signals. Specifically, the \textbf{signal decoder} is tasked with reconstructing the masked signals, using both the latent embeddings and the masked spatial and structural positional embeddings:
\[
\widehat{\tX}^{(k)} = \fsigdec(\concat(\widetilde{\mZ}^{(k)}, \mask(\widetilde{\tS}^{(k)}+\widetilde{\tR}^{(k)} ; \overline{\mM}^{(k)}))),
\]
where $\overline{\mM}^{(k)} = 1 - \mM^{(k)}$ is the complement mask, and $\widehat{\tX}^{(k)}$ denotes the reconstructed signals. In parallel, the \textbf{spatial decoder} is responsible for reconstructing the masked spatial positions, conditioned on the latent embeddings and the masked signal and structural positional embeddings:
\[
\widehat{\tS}^{(k)} = \fspdec(\concat(\widetilde{\mZ}^{(k)}, \mask(\widetilde{\tX}^{(k)} +\widetilde{\tR}^{(k)}; \overline{\mM}^{(k)}))),
\]
where $\widehat{\tS}^{(k)}$ denotes the reconstructed spatial positions. The loss $L$ used to train our model combines the Mean Squared Error (MSE) reconstruction losses over both decoders:
\[
\begin{aligned}
L = \sum_{k=1}^K & \| \mask(\tX^{(k)}-\widehat{\tX}^{(k)};\overline{\mM}^{(k)})\|_2^2 \\
+ & \| \mask(\tS^{(k)}-\widehat{\tS}^{(k)};\overline{\mM}^{(k)})\|_2^2.
\end{aligned}
\]
Our dual reconstruction objectives compel the model to extract, utilize, and preserve the full spectrum of signal, spatial, and structural information.
A practical challenge in multi-modal, multi-node distributed sensing systems is the frequent occurrence of missing data due to hardware failures or unreliable communication links. To mitigate their impact, we pad missing entries with zeros and exclude them from the reconstruction loss by setting their corresponding loss terms to zero.
\subsection{Theoretical Analyses}
In this subsection, we provide theoretical support for the design of \MODEL{}, drawing from principles in both information theory and occlusion-invariant representation learning. These insights help illuminate the rationale behind \MODEL{}'s design.
\noindent\textbf{Analysis from the Perspective of Information Theory.}
We first analyze \MODEL{} in comparison to classical MAE through the lens of information theory, as formalized in the following proposition:
\begin{proposition} \label{pp:information}
Let $\mathsf{X}^{(k)}, \widetilde{\mathsf{Z}}^{(k)}, \mathsf{S}^{(k)}, \mathsf{R}^{(k)}$ denote the random variables corresponding to the signals, the post-fusion latent embeddings, the spatial positions, and the structural positions, for $k \in \{1, \dots, K\}$, respectively.
% whose realizations are $\tX^{(k)}, \widetilde{\tZ}^{(k)}, \tS^{(k)}, \tR^{(k)}$, respectively.
Let $\E[L']$ and $\E[L]$ denote the expected losses of classical MAE and \MODEL{} over the data distribution, respectively, and let $C'$ and $C$ be constants independent of model parameters. Then, under certain assumptions, for classical MAE, we can have the following bound:
\[
-\E[L'] + C' \leq \sum_{k=1}^K I(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)}),
\]
where $I(\cdot;\cdot)$ denotes mutual information. In contrast, for \MODEL{}, we can have
\[
\begin{aligned}
-\E[L] + C \leq \sum_{k=1}^K & I(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)} | \mathsf{S}^{(k)}, \mathsf{R}^{(k)}) \\
+ & I(\mathsf{S}^{(k)}; \widetilde{\mathsf{Z}}^{(k)} | \mathsf{X}^{(k)}, \mathsf{R}^{(k)}).
\end{aligned}
\]
where $I(\cdot;\cdot|\cdot)$ denotes conditional mutual information.
\end{proposition}
The proof is detailed in Appendix~\ref{pf:information}. This result highlights a key distinction between classical MAE and \MODEL{}. In classical MAE, minimizing the expected loss encourages latent embeddings to retain information about the input signals, but without explicitly incorporating spatial or structural context. In contrast, \MODEL{} is designed to promote embeddings that capture signal information beyond what is explained by structural and spatial cues, and similarly, to retain spatial information conditioned on the signal and structural characteristics. This encourages the embeddings to be context-aware and jointly informative of both signals and spatial layout, while avoiding memorizing redundant information.
\noindent\textbf{Analysis from the Perspective of Occlusion-invariant Representation.}
We next analyze \MODEL{} through the lens of occlusion-invariant representation learning. For clarity and readability, we present the analysis for a single modality by omitting the superscript $(k)$; the generalization to the multi-modality case is straightforward. The core result is formalized in the following proposition:
\begin{proposition}\label{pp:occlusion}
As shown by Kong~\etal~\citep{kong2023understanding}, classical MAE can be viewed as a form of contrastive learning, where the positive pair consists of two complementary masked views of the signals:
\[
\left[\textnormal{\mask}({\tX};\mM), \quad \textnormal{\mask}({\tX};\overline{\mM})\right].
\]
In contrast, \MODEL{} can be interpreted as performing contrastive learning over two types of enriched positive pairs: 1) complementary masked views of signals with shared spatial and structural context:
\[
\left[\left(\textnormal{\mask}({\tX};\mM),{\tS}, {\tR}\right) , \quad \left(\textnormal{\mask}({\tX};\overline{\mM}),{\tS}, {\tR}\right)\right],
\]
and 2) complementary masked views of spatial positions with shared signal and structural context:
\[
\left[\left({\tX},\textnormal{\mask}({\tS};\mM), {\tR}\right) , \quad \left({\tX},\textnormal{\mask}({\tS};\overline{\mM}), {\tR}\right)\right].
\]
\end{proposition}
The proof is detailed in Appendix~\ref{pf:occlusion}. This formulation highlights another key distinction: by treating masked views of the signal embeddings as positive pairs, classical MAE promotes occlusion-invariant representations solely within the signal domain, without accounting for spatial or structural positions. In contrast, \MODEL{} encourages representations to be invariant to occlusion in both the signal and spatial domains, while preserving the presence of each other and the structural characteristics, leading to more robust and context-aware learned representations.
\section{Evaluation}\label{sec:experiment}
In this section, we present our experimental evaluation of \MODEL{} on three multiple multi-modal, multi-node distributed sensing datasets spanning diverse sensing modalities and spatial scales. To ensure a fair comparison, all baseline methods and our model use the same ViT backbone architecture~\citep{dosovitskiy2020image} and identical task-specific prediction heads. Pretraining and fine-tuning are conducted for the same number of epochs across all methods. All reported results are aggregated over three random seeds. The prediction heads are designed to be lightweight and straightforward, tailored to the needs of each downstream task. Detailed descriptions of each task setup can be found in Appendix~\ref{app:experiment}.
\textbf{Datasets.} We conducted experiments on three real-world distributed sensing datasets:
(1) the M3N-VC dataset\citep{li2025restoreml}, which includes acoustic and seismic signals from moving vehicles, collected across six distinct outdoor scenes;
(2) the Ridgecrest Seismicity Dataset\citep{caltech1926socal}, containing multi-modal seismic waveform recordings of earthquake events in the Ridgecrest region of California; and
(3) the RealWorld-HAR dataset\citep{sztyler2016body}, comprising accelerometer, gyroscope, and magnetometer readings for human activity recognition.
Further dataset details are available in Appendix~\ref{app:experiment}.
\textbf{Baselines.} We compare \MODEL{} against eight baseline methods: CMC~\citep{tian2020contrastive}, Cosmo~\citep{ouyang2022cosmo}, SimCLR~\citep{chen2020simple}, AudioMAE~\citep{huang2022masked}, CAV-MAE~\citep{gong2022contrastive}, FOCAL~\citep{liu2023focal}, FreqMAE~\citep{kara2024freqmae}, and PhyMask~\citep{kara2024phymask}. These baselines are chosen to provide (i) \emph{paradigm coverage} across contrastive learning (CMC, Cosmo, SimCLR, FOCAL) and masked autoencoding (AudioMAE, CAV-MAE, FreqMAE), and (ii) \emph{domain coverage} spanning widely used, strong general-purpose pretraining methods for time-series/multimodal representation learning (e.g., AudioMAE, CAV-MAE, CMC, SimCLR) as well as recent sensing-tailored approaches (e.g., Cosmo, FOCAL, FreqMAE, PhyMask). Please see Appendix~\ref{app:baselines} for detailed descriptions.
\subsection{Evaluation on M3N-VC Dataset}\label{exp:m3n-vc}
\definecolor{lightblue}{HTML}{E6F0FF}
\newcommand{\mygray}[1]{\textcolor{gray}{\footnotesize #1}}
\begin{table*}[!tb]
% \vspace{-0.1in}
\caption{\small Comparison of the MSE and averaged Distance Error between \MODEL{} and baselines on M3N-VC single-vehicle localization task. The label ratio during fine-tuning varies from 1.0 to 0.2.}
\vspace{-0.1in}
\label{tb:m3n-vc1}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{lcccccc}
\toprule
\multirow{3}{*}{Method} & \multicolumn{6}{c}{M3N-VC Single-vehicle Localization} \\ \cmidrule(lr){2-7}
& \multicolumn{2}{c}{Label Ratio 1.0} & \multicolumn{2}{c}{Label Ratio 0.5} & \multicolumn{2}{c}{Label Ratio 0.2} \\ \cmidrule(lr){2-3} \cmidrule(lr){4-5} \cmidrule(lr){6-7}
& \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} & \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} & \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} \\
\midrule
CMC & \multicolumn{1}{c}{51.11 \mygray{$\pm$ 14.67}} & \multicolumn{1}{c}{6.76 \mygray{$\pm$ 0.75}} & \multicolumn{1}{c}{71.81 \mygray{$\pm$ 15.32}} & \multicolumn{1}{c}{7.99 \mygray{$\pm$ 0.64}} & \multicolumn{1}{c}{111.37 \mygray{$\pm$ 8.02}} & \multicolumn{1}{c}{11.05 \mygray{$\pm$ 0.57}} \\
Cosmo & \multicolumn{1}{c}{38.40 \mygray{$\pm$ 4.14}} & \multicolumn{1}{c}{6.03 \mygray{$\pm$ 0.21}} & \multicolumn{1}{c}{53.12 \mygray{$\pm$ 9.75}} & \multicolumn{1}{c}{7.19 \mygray{$\pm$ 0.40}} & \multicolumn{1}{c}{97.08 \mygray{$\pm$ 9.49}} & \multicolumn{1}{c}{10.95 \mygray{$\pm$ 0.57}} \\
SimCLR & \multicolumn{1}{c}{34.40 \mygray{$\pm$ 4.47}} & \multicolumn{1}{c}{5.64 \mygray{$\pm$ 0.25}} & \multicolumn{1}{c}{45.14 \mygray{$\pm$ 7.34}} & \multicolumn{1}{c}{6.57 \mygray{$\pm$ 0.08}} & \multicolumn{1}{c}{74.53 \mygray{$\pm$ 3.13}} & \multicolumn{1}{c}{9.48 \mygray{$\pm$ 0.17}} \\
AudioMAE & \multicolumn{1}{c}{22.36 \mygray{$\pm$ 0.49}} & \multicolumn{1}{c}{5.40 \mygray{$\pm$ 0.11}} & \multicolumn{1}{c}{30.12 \mygray{$\pm$ 2.97}} & \multicolumn{1}{c}{6.33 \mygray{$\pm$ 0.28}} & \multicolumn{1}{c}{41.75 \mygray{$\pm$ 3.30}} & \multicolumn{1}{c}{7.47 \mygray{$\pm$ 0.28}} \\
CAV-MAE & \multicolumn{1}{c}{18.85 \mygray{$\pm$ 0.41}} & \multicolumn{1}{c}{5.06 \mygray{$\pm$ 0.04}} & \multicolumn{1}{c}{22.90 \mygray{$\pm$ 0.82}} & \multicolumn{1}{c}{5.58 \mygray{$\pm$ 0.12}} & \multicolumn{1}{c}{24.84 \mygray{$\pm$ 0.33}} & \multicolumn{1}{c}{5.78 \mygray{$\pm$ 0.10}} \\
FOCAL & \multicolumn{1}{c}{32.43 \mygray{$\pm$ 4.68}} & \multicolumn{1}{c}{5.37 \mygray{$\pm$ 0.22}} & \multicolumn{1}{c}{40.84 \mygray{$\pm$ 2.82}} & \multicolumn{1}{c}{6.20 \mygray{$\pm$ 0.19}} & \multicolumn{1}{c}{69.62 \mygray{$\pm$ 5.62}} & \multicolumn{1}{c}{8.50 \mygray{$\pm$ 0.35}} \\
FreqMAE & \multicolumn{1}{c}{29.61 \mygray{$\pm$ 2.85}} & \multicolumn{1}{c}{5.36 \mygray{$\pm$ 0.16}} & \multicolumn{1}{c}{42.06 \mygray{$\pm$ 14.44}} & \multicolumn{1}{c}{6.25 \mygray{$\pm$ 0.70}} & \multicolumn{1}{c}{91.40 \mygray{$\pm$ 35.32}} & \multicolumn{1}{c}{9.15 \mygray{$\pm$ 1.27}} \\
PhyMask & \multicolumn{1}{c}{28.02 \mygray{$\pm$ 5.91}} & \multicolumn{1}{c}{5.29 \mygray{$\pm$ 0.33}} & \multicolumn{1}{c}{33.74 \mygray{$\pm$ 2.18}} & \multicolumn{1}{c}{5.85 \mygray{$\pm$ 0.12}} & \multicolumn{1}{c}{64.36 \mygray{$\pm$ 4.70}} & \multicolumn{1}{c}{8.44 \mygray{$\pm$ 0.36}} \\
\rowcolor{lightblue}
\MODEL & \multicolumn{1}{c}{\textbf{12.98 \mygray{$\pm$ 0.11}}} & \multicolumn{1}{c}{\textbf{4.20 \mygray{$\pm$ 0.07}}} & \multicolumn{1}{c}{\textbf{15.07 \mygray{$\pm$ 1.03}}} & \multicolumn{1}{c}{\textbf{4.51 \mygray{$\pm$ 0.09}}} & \multicolumn{1}{c}{\textbf{21.36 \mygray{$\pm$ 0.62}}} & \multicolumn{1}{c}{\textbf{5.40 \mygray{$\pm$ 0.04}}} \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
\vspace{-0.1in}
\end{table*}
\begin{table}[!tb]
% \vspace{-0.1in}
\caption{\small Comparison of the mAP@\emph{r} metric (\%) ($\uparrow$) (\emph{r} is the distance threshold varying across \{2,4,6,8\} meters) between \MODEL{} and baselines on M3N-VC multi-vehicle joint classification and localization task.}
\vspace{-0.1in}
\label{tb:m3n-vc2}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{lcccc}
\toprule
\multirow{2}{*}{Method} & \multicolumn{4}{c}{M3N-VC Multi-vehicle Joint Classification and Localization} \\ \cmidrule(lr){2-5}
& \multicolumn{1}{c}{mAP@4m } & \multicolumn{1}{c}{mAP@6m } & \multicolumn{1}{c}{mAP@8m } & \multicolumn{1}{c}{mAP@10m } \\
\midrule
CMC & \multicolumn{1}{c}{0.06 \mygray{$\pm$ 0.05}} & \multicolumn{1}{c}{0.48 \mygray{$\pm$ 0.36}} & \multicolumn{1}{c}{1.61 \mygray{$\pm$ 1.10}} & \multicolumn{1}{c}{3.62 \mygray{$\pm$ 2.19}} \\
Cosmo & \multicolumn{1}{c}{0.16 \mygray{$\pm$ 0.05}} & \multicolumn{1}{c}{1.66 \mygray{$\pm$ 0.23}} & \multicolumn{1}{c}{4.77 \mygray{$\pm$ 0.72}} & \multicolumn{1}{c}{9.52 \mygray{$\pm$ 1.20}} \\
SimCLR & \multicolumn{1}{c}{0.31 \mygray{$\pm$ 0.14}} & \multicolumn{1}{c}{2.22 \mygray{$\pm$ 0.58}} & \multicolumn{1}{c}{6.53 \mygray{$\pm$ 1.24}} & \multicolumn{1}{c}{13.07 \mygray{$\pm$ 2.08}} \\
AudioMAE & \multicolumn{1}{c}{1.39 \mygray{$\pm$ 0.48}} & \multicolumn{1}{c}{6.96 \mygray{$\pm$ 1.42}} & \multicolumn{1}{c}{17.11 \mygray{$\pm$ 3.24}} & \multicolumn{1}{c}{28.98 \mygray{$\pm$ 4.01}} \\
CAV-MAE & \multicolumn{1}{c}{22.12 \mygray{$\pm$ 2.94}} & \multicolumn{1}{c}{52.08 \mygray{$\pm$ 4.16}} & \multicolumn{1}{c}{73.41 \mygray{$\pm$ 3.24}} & \multicolumn{1}{c}{85.36 \mygray{$\pm$ 1.78}} \\
FOCAL & \multicolumn{1}{c}{0.08 \mygray{$\pm$ 0.05}} & \multicolumn{1}{c}{0.82 \mygray{$\pm$ 0.40}} & \multicolumn{1}{c}{2.94 \mygray{$\pm$ 1.04}} & \multicolumn{1}{c}{6.82 \mygray{$\pm$ 1.99}} \\
FreqMAE & \multicolumn{1}{c}{0.24 \mygray{$\pm$ 0.01}} & \multicolumn{1}{c}{1.67 \mygray{$\pm$ 0.32}} & \multicolumn{1}{c}{5.34 \mygray{$\pm$ 0.99}} & \multicolumn{1}{c}{11.31 \mygray{$\pm$ 1.49}} \\
PhyMask & \multicolumn{1}{c}{0.08 \mygray{$\pm$ 0.03}} & \multicolumn{1}{c}{0.88 \mygray{$\pm$ 0.24}} & \multicolumn{1}{c}{3.04 \mygray{$\pm$ 0.74}} & \multicolumn{1}{c}{6.64 \mygray{$\pm$ 1.46}} \\ \rowcolor{lightblue}
\MODEL & \multicolumn{1}{c}{\textbf{41.57 \mygray{$\pm$ 2.69}}} & \multicolumn{1}{c}{\textbf{71.82 \mygray{$\pm$ 3.69}}} & \multicolumn{1}{c}{\textbf{86.28 \mygray{$\pm$ 1.77}}} & \multicolumn{1}{c}{\textbf{92.99 \mygray{$\pm$ 0.79}}} \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
\vspace{-0.1in}
\end{table}
\begin{table}[!htb]
% \vspace{-0.1in}
\caption{\small Comparison of the MSE and Distance Error between \MODEL{} and baselines on M3N-VC single-vehicle localization task. \MODEL{} and baselines are finetuned and evaluated on a placement unseen in the pretraining. }
\vspace{-0.1in}
\label{tb:m3n-vc5}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{lcc}
\toprule
\multirow{3}{*}{Method} & \multicolumn{2}{c}{M3N-VC Single-vehicle Localization} \\
& \multicolumn{2}{c}{(Finetuned and Evaluated on Unseen Placement)} \\ \cmidrule(lr){2-3}
& \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} \\
\midrule
CMC & \multicolumn{1}{c}{61.78 \mygray{$\pm$ 17.68}} & \multicolumn{1}{c}{7.23 \mygray{$\pm$ 0.78}} \\
Cosmo & \multicolumn{1}{c}{63.43 \mygray{$\pm$ 12.85}} & \multicolumn{1}{c}{7.22 \mygray{$\pm$ 0.63}} \\
SimCLR & \multicolumn{1}{c}{35.82 \mygray{$\pm$ 7.57}} & \multicolumn{1}{c}{5.92 \mygray{$\pm$ 0.39}} \\
AudioMAE & \multicolumn{1}{c}{41.25 \mygray{$\pm$ 4.60}} & \multicolumn{1}{c}{6.64 \mygray{$\pm$ 0.31}} \\
CAV-MAE & \multicolumn{1}{c}{37.01 \mygray{$\pm$ 0.68}} & \multicolumn{1}{c}{6.27 \mygray{$\pm$ 0.04}} \\
FOCAL & \multicolumn{1}{c}{41.79 \mygray{$\pm$ 11.04}} & \multicolumn{1}{c}{5.91 \mygray{$\pm$ 0.41}} \\
FreqMAE & \multicolumn{1}{c}{30.65 \mygray{$\pm$ 1.14}} & \multicolumn{1}{c}{5.51 \mygray{$\pm$ 0.19}} \\
PhyMask & \multicolumn{1}{c}{34.83 \mygray{$\pm$ 8.81}} & \multicolumn{1}{c}{5.60 \mygray{$\pm$ 0.38}} \\ \rowcolor{lightblue}
\MODEL & \multicolumn{1}{c}{\textbf{21.76 \mygray{$\pm$ 1.00}}} & \multicolumn{1}{c}{\textbf{5.09 \mygray{$\pm$ 0.10}}} \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
\vspace{-0.1in}
\end{table}
\begin{table}[!htb]
% \vspace{-0.1in}
\caption{\small Comparison of the MSE and Distance Error between \MODEL{} and ablations on M3N-VC single-vehicle localization task.}
\vspace{-0.1in}
\label{tb:m3n-vc6}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{lcc}
\toprule
\multirow{2}{*}{Method} & \multicolumn{2}{c}{Single-vehicle Localization} \\ \cmidrule(lr){2-3}
& \multicolumn{1}{c}{MSE ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($\downarrow$)} \\
\midrule
\MODEL & \multicolumn{1}{c}{12.98 \mygray{$\pm$ 0.11}} & \multicolumn{1}{c}{4.20 \mygray{$\pm$ 0.07}} \\ \midrule
\textit{w/o} Geometric Augmentation in Pretrain & \multicolumn{1}{c}{15.59 \mygray{$\pm$ 0.56}} & \multicolumn{1}{c}{4.67 \mygray{$\pm$ 0.04}} \\
\textit{w/o} Reconstructing Spatial Positions & \multicolumn{1}{c}{14.73 \mygray{$\pm$ 0.35}} & \multicolumn{1}{c}{4.62 \mygray{$\pm$ 0.03}} \\
\textit{w/o} Spatial Positional Embedding & \multicolumn{1}{c}{15.12 \mygray{$\pm$ 0.58}} & \multicolumn{1}{c}{4.67 \mygray{$\pm$ 0.07}} \\
\textit{w/o} Structural Positional Embedding & \multicolumn{1}{c}{22.55 \mygray{$\pm$ 2.98}} & \multicolumn{1}{c}{5.08 \mygray{$\pm$ 0.13}} \\
+ Node-Drop Masking & \multicolumn{1}{c}{17.71 \mygray{$\pm$ 3.17}} & \multicolumn{1}{c}{4.82 \mygray{$\pm$ 0.33}} \\
+ Node-Balanced Masking & \multicolumn{1}{c}{\textbf{12.54 \mygray{$\pm$ 1.63}}} & \multicolumn{1}{c}{\textbf{4.14 \mygray{$\pm$ 0.21}}} \\
+ Mask Ratio 0.85 & \multicolumn{1}{c}{13.89 \mygray{$\pm$ 2.17}} & \multicolumn{1}{c}{4.35 \mygray{$\pm$ 0.24}} \\
+ Mask Ratio 0.5 & \multicolumn{1}{c}{14.81 \mygray{$\pm$ 2.57}} & \multicolumn{1}{c}{4.45 \mygray{$\pm$ 0.31}} \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
\vspace{-0.1in}
\end{table}
\begin{table*}[!t]
\caption{\small Comparison between \MODEL{} and baselines across three tasks: (1) M3N-VC single-vehicle classification, (2) Ridgecrest Seismicity Dataset earthquake localization, and (3) RealWorld-HAR activity recognition. Each block reports task-specific metrics.}
\vspace{-0.1in}
\label{tb:all}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{lcccccc}
\toprule
\multirow{2}{*}{Method}
& \multicolumn{2}{c}{M3N-VC Classification}
& \multicolumn{2}{c}{Ridgecrest Earthquake Localization}
& \multicolumn{2}{c}{RealWorld-HAR Recognition} \\
\cmidrule(lr){2-3} \cmidrule(lr){4-5} \cmidrule(lr){6-7}
& Accuracy (\%) ($\uparrow$) & F1 (\%) ($\uparrow$)
& MSE ($km^2$) ($\downarrow$) & Dist. Err. ($km$) ($\downarrow$)
& Accuracy (\%) ($\uparrow$) & F1 (\%) ($\uparrow$) \\
\midrule
CMC & 89.53 \mygray{$\pm$ 7.62} & 89.33 \mygray{$\pm$ 7.78} & 94.25 \mygray{$\pm$ 6.67} & 10.38 \mygray{$\pm$ 0.63} & 74.97 \mygray{$\pm$ 1.23} & 74.82 \mygray{$\pm$ 2.18} \\
Cosmo & 94.21 \mygray{$\pm$ 0.50} & 94.04 \mygray{$\pm$ 0.54} & 98.24 \mygray{$\pm$ 13.77} & 10.44 \mygray{$\pm$ 0.83} & 84.37 \mygray{$\pm$ 0.33} & 85.30 \mygray{$\pm$ 0.43} \\
SimCLR & 95.53 \mygray{$\pm$ 0.73} & 95.41 \mygray{$\pm$ 0.74} & 99.87 \mygray{$\pm$ 11.31} & 10.29 \mygray{$\pm$ 0.52} & 84.36 \mygray{$\pm$ 0.47} & 85.49 \mygray{$\pm$ 0.36} \\
AudioMAE & 99.06 \mygray{$\pm$ 0.23} & 99.03 \mygray{$\pm$ 0.24} & 33.65 \mygray{$\pm$ 3.51} & 5.65 \mygray{$\pm$ 0.29} & 89.18 \mygray{$\pm$ 0.32} & 90.11 \mygray{$\pm$ 0.53} \\
CAV-MAE & 98.97 \mygray{$\pm$ 0.04} & 98.94 \mygray{$\pm$ 0.04} & 31.58 \mygray{$\pm$ 3.57} & 5.48 \mygray{$\pm$ 0.37} & 88.12 \mygray{$\pm$ 0.24} & 89.05 \mygray{$\pm$ 0.35} \\
FOCAL & 93.62 \mygray{$\pm$ 0.75} & 93.46 \mygray{$\pm$ 0.76} & 131.50 \mygray{$\pm$ 1.48} & 12.53 \mygray{$\pm$ 0.09} & 84.98 \mygray{$\pm$ 0.73} & 86.24 \mygray{$\pm$ 0.77} \\
FreqMAE & 92.72 \mygray{$\pm$ 0.75} & 92.55 \mygray{$\pm$ 0.79} & 54.08 \mygray{$\pm$ 5.44} & 7.14 \mygray{$\pm$ 0.25} & 83.43 \mygray{$\pm$ 0.56} & 84.07 \mygray{$\pm$ 0.50} \\
PhyMask & 83.38 \mygray{$\pm$ 2.33} & 82.68 \mygray{$\pm$ 2.27} & 56.39 \mygray{$\pm$ 3.27} & 7.67 \mygray{$\pm$ 0.39} & 84.79 \mygray{$\pm$ 3.23} & 82.15 \mygray{$\pm$ 9.13} \\
\rowcolor{lightblue}
\MODEL & \textbf{99.27 \mygray{$\pm$ 0.07}} & \textbf{99.26 \mygray{$\pm$ 0.07}}
& \textbf{23.46 \mygray{$\pm$ 2.77}} & \textbf{5.37 \mygray{$\pm$ 0.24}}
& \textbf{89.63 \mygray{$\pm$ 0.57}} & \textbf{90.45 \mygray{$\pm$ 0.63}} \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
\vspace{-0.1in}
\end{table*}
\begin{table*}[!tb] \caption{\small Impact of data compression on \MODEL{} across three tasks: (1) M3N-VC single-vehicle classification, (2) Ridgecrest Seismicity Dataset earthquake localization, and (3) RealWorld-HAR activity recognition. } \vspace{-0.1in} \label{tb:compression_all} \begin{center} \begin{adjustbox}{max width=\linewidth} \begin{tabular}{lccccccccc} \toprule \multirow{2}{*}{Method} & \multicolumn{3}{c}{M3N-VC Single-vehicle Localization} & \multicolumn{3}{c}{Ridgecrest Earthquake Localization} & \multicolumn{3}{c}{RealWorld-HAR Recognition} \\ \cmidrule(lr){2-4} \cmidrule(lr){5-7} \cmidrule(lr){8-10} & Traffic (\%) $\downarrow$ & MSE $\downarrow$ & Dist. Err. $\downarrow$ & Traffic (\%) $\downarrow$ & MSE $\downarrow$ & Dist. Err. $\downarrow$ & Traffic (\%) $\downarrow$ & Acc. $\uparrow$ & F1 $\uparrow$ \\ \midrule \MODEL{} & 100.00 & 12.12 & 4.12 & 100.00 & 22.17 & 5.10 & 100.00 & 90.23 & 91.11 \\ \MODEL{} w. Compression & \textbf{10.70} & 12.26 & 4.13 & \textbf{6.30} & 22.18 & 5.10 & \textbf{24.03} & 90.00 & 90.91 \\ \bottomrule \end{tabular} \end{adjustbox} \end{center} \vspace{-0.1in} \end{table*}
\noindent\textbf{Single-vehicle Localization.} We begin with the M3N-VC dataset, focusing on the task of \textbf{single-vehicle localization}, where the goal is to predict the position of a vehicle within the monitored area. We pretrain on the full dataset and finetune only the prediction head (a single transformer layer) on scene "H24," which contains a single moving vehicle. To test robustness under limited supervision, we vary the ratio of labeled data from 100\% to 20\%. As shown in Table~\ref{tb:m3n-vc1}, \MODEL{} consistently achieves the lowest MSE and Distance Error across all label ratios, demonstrating resilience to scarce supervision. Example localization visualizations are provided in Figure~\ref{fig:loc1} (Appendix~\ref{app:vis}).
\noindent\textbf{Single-vehicle Classification.} In this task, we aim to distinguish among four vehicle types and a background class. The setup mirrors that of localization. As reported in Table~\ref{tb:all}, \MODEL{} attains the highest accuracy and F1 score among all methods. The confusion matrix (Figure~\ref{fig:cls2}) and T-SNE plot (Figure~\ref{fig:tsne}) confirm that the learned latent embeddings cleanly separate all five classes, underscoring their discriminative power.
\noindent\textbf{Multi-vehicle Detection.} We next consider a more challenging task, where we jointly localize and classify multiple vehicles moving simultaneously, which is similar to the object detection task in CV. We pretrain on the full dataset and finetune on the scene "I22," which includes multiple vehicles. A 2-layer transformer head with a DETR-style loss~\citep{carion2020end} is used as the detection head. To evaluate performance, we adopt mAP@\emph{r} from object detection~\citep{lin2014microsoft}, where predictions are correct only if both class and location are accurate within radius \emph{r}. As shown in Table~\ref{tb:m3n-vc2}, \MODEL{} significantly outperforms all baselines across thresholds, including strict ones (e.g., 4m), despite noisy 1Hz smartphone GPS labels. This highlights its strong spatial reasoning under complex conditions. Additional visualizations of predictions are included in Figure~\ref{fig:multi} (Appendix~\ref{app:vis}).
\noindent\textbf{Unseen Sensor Placements}. Next, we assess generalization to unseen sensor placements. All models are pretrained on the full dataset, excluding scenes H08 and H24 (which share similar configurations), then finetuned and evaluated on H24. To simulate transfer, nodes in H24 are assigned structural position vectors randomly drawn from those learned during pretraining. As reported in Table~\ref{tb:m3n-vc5}, \MODEL{} continues to outperform baselines, underscoring its placement-aware generalization ability.
\noindent\textbf{Ablation Study}. Finally, we conduct ablations to quantify the contribution of each design choice (Table~\ref{tb:m3n-vc6}). Removing geometric augmentation, the spatial reconstruction objective, or spatial embeddings all lead to comparable performance drops, highlighting their complementary roles. Excluding structural embeddings results in the most severe degradation, underscoring their critical role in modeling node-specific characteristics. We also test alternative masking strategies: Node-Drop Masking (masking entire nodes) reduces performance, while Node-Balanced Masking (ensuring a minimum number of unmasked tokens per node) offers slight gains over random masking. We further vary the mask ratio (0.85 and 0.5) and observe only minor performance changes relative to the default 0.75, indicating the robustness of the framework.
Finally, we conduct another three complementary evaluations that are detailed in Appendix~\ref{app:results}:
(1) \textbf{Robustness to Lossy Communication}: \MODEL{} remains robust under random node-level data dropout, outperforming baselines (Table~\ref{tb:m3n-vc4}).
(2) \textbf{Preservation of Spatial Information in Learned Representations}: We provide evidence that \MODEL{} preserves spatial information in its representations via a spatial reconstruction ablation study (Figure~\ref{fig:noise_reconstruction}).
(3) \textbf{Robustness to Noise in Spatial Positions}: \MODEL{}'s learned representation is robust to noise in nodes' spatial positions.
\subsection{Evaluation on Ridgecrest Seismicity Dataset}
We next evaluate \MODEL{} on the Ridgecrest Seismicity Dataset for \textbf{earthquake event localization}. Here, the goal is to predict 3D earthquake coordinates from multi-modal seismic waveforms collected across 16 monitoring stations. Compared to vehicle monitoring, this task involves a much larger spatial scale (tens of kilometers) and a 20.38\% inherent missing-data rate, as weak seismic waves often fail to reach distant stations. Despite these challenges, \MODEL{} achieves the lowest MSE and Distance Error among all methods (Table~\ref{tb:all}), demonstrating strong spatial reasoning in large-scale, partially observed environments. Visualizations of predictions are provided in Figure~\ref{fig:loc2} (Appendix~\ref{app:vis}).
\subsection{Evaluation on RealWorld-HAR Dataset}
Finally, we evaluate \MODEL{} on the RealWorld-HAR dataset for \textbf{human activity recognition} using IMU signals. This task differs from the above in operating at a smaller spatial scale but involving highly diverse placements: sensors mounted on the wrist, ankle, chest, etc., yield signals with distinct characteristics. Despite this heterogeneity, \MODEL{} achieves the best accuracy and F1 among all baselines (Table~\ref{tb:all}), underscoring its robustness to placement diversity. The confusion matrix (Figure~\ref{fig:cls2}) and t-SNE visualizations (Figure~\ref{fig:tsne}) show that predicted activity patterns align well with the conceptual separability of classes.
\subsection{Robustness under Communication Constraints}
Distributed sensing often operates under limited bandwidth, restricting the transmission of raw sensor data. To test \MODEL{} in such settings, we compress raw inputs using standard formats, such as JPEG for M3N-VC and Ridgecrest, and WebP for RealWorld-HAR. As shown in Table~\ref{tb:compression_all}, compression reduces communication traffic by up to 90\% with negligible performance loss across all tasks. This demonstrates that \MODEL{} is robust to severe bandwidth constraints, making it practical for real-world deployments.
\section{Conclusion}
This paper presents \MODEL{}, a self-supervised pretraining framework designed for the full spectrum of multi-modal, multi-node distributed sensing. By introducing spatial and structural positional embeddings alongside dual reconstruction objectives, \MODEL{} leverages the inherent duality between observer positions and observed signals to enable placement-aware representation learning. Theoretical analyses grounded in information theory and occlusion-invariant learning offer principled support for the framework. Extensive experiments across three real-world datasets, spanning diverse sensing modalities, placement configurations, and task types, demonstrate the superior generalizability and robustness of \MODEL{}. We hope \MODEL{} inspires broader efforts toward integrating spatial and structural context into foundational representation learning paradigms for the wide range of distributed sensing applications.
\newpage
\section*{Impact Statement}
This work advances self-supervised representation learning for distributed sensing systems by explicitly incorporating sensor placement into pretraining. By enabling more robust, generalizable, and data-efficient learning across diverse sensing modalities and layouts, the proposed framework has the potential to improve real-world applications such as environmental monitoring, infrastructure safety, transportation systems, and human activity analysis. More accurate and resilient sensing models can support better decision-making, resource allocation, and situational awareness in both civilian and scientific contexts. This research focuses on methodological and foundational advances in representation learning, and we do not anticipate direct negative societal impacts arising from the proposed approach.
\bibliography{example_paper}
\bibliographystyle{icml2026}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\appendix
\onecolumn
\section*{Appendix}
\section{Additional Experiments Results}\label{app:results}
\begin{table}[!htb]
% \vspace{-0.1in}
\caption{\small Comparison of the MSE and Distance Error between \MODEL{} and baselines on M3N-VC single-vehicle localization task, under various message drop rates.}
\vspace{-0.1in}
\label{tb:m3n-vc4}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{lcccccc}
\toprule
\multirow{3}{*}{Method} & \multicolumn{6}{c}{M3N-VC Single-vehicle Localization} \\ \cmidrule(lr){2-7}
& \multicolumn{2}{c}{Message Drop Rate 0.05} & \multicolumn{2}{c}{Message Drop Rate 0.1} & \multicolumn{2}{c}{Message Drop Rate 0.2} \\ \cmidrule(lr){2-3} \cmidrule(lr){4-5} \cmidrule(lr){6-7}
& \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} & \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} & \multicolumn{1}{c}{MSE ($m^2$) ($\downarrow$)} & \multicolumn{1}{c}{Dist. Err. ($m$) ($\downarrow$)} \\
\midrule
CMC & \multicolumn{1}{c}{50.13 \mygray{$\pm$ 17.69}} & \multicolumn{1}{c}{6.48 \mygray{$\pm$ 0.91}} & \multicolumn{1}{c}{50.27 \mygray{$\pm$ 18.03}} & \multicolumn{1}{c}{6.60 \mygray{$\pm$ 1.17}} & \multicolumn{1}{c}{55.38 \mygray{$\pm$ 22.01}} & \multicolumn{1}{c}{6.91 \mygray{$\pm$ 1.29}} \\
Cosmo & \multicolumn{1}{c}{29.97 \mygray{$\pm$ 2.30}} & \multicolumn{1}{c}{5.44 \mygray{$\pm$ 0.12}} & \multicolumn{1}{c}{29.37 \mygray{$\pm$ 2.43}} & \multicolumn{1}{c}{5.44 \mygray{$\pm$ 0.10}} & \multicolumn{1}{c}{33.19 \mygray{$\pm$ 2.67}} & \multicolumn{1}{c}{5.61 \mygray{$\pm$ 0.12}} \\
SimCLR & \multicolumn{1}{c}{26.77 \mygray{$\pm$ 2.55}} & \multicolumn{1}{c}{5.16 \mygray{$\pm$ 0.07}} & \multicolumn{1}{c}{25.65 \mygray{$\pm$ 1.28}} & \multicolumn{1}{c}{5.15 \mygray{$\pm$ 0.07}} & \multicolumn{1}{c}{28.31 \mygray{$\pm$ 3.94}} & \multicolumn{1}{c}{5.4 \mygray{$\pm$ 0.18}} \\
AudioMAE & \multicolumn{1}{c}{19.29 \mygray{$\pm$ 1.42}} & \multicolumn{1}{c}{4.91 \mygray{$\pm$ 0.17}} & \multicolumn{1}{c}{18.25 \mygray{$\pm$ 1.23}} & \multicolumn{1}{c}{4.77 \mygray{$\pm$ 0.10}} & \multicolumn{1}{c}{19.03 \mygray{$\pm$ 1.14}} & \multicolumn{1}{c}{4.77 \mygray{$\pm$ 0.05}} \\
CAV-MAE & \multicolumn{1}{c}{16.28 \mygray{$\pm$ 0.17}} & \multicolumn{1}{c}{4.68 \mygray{$\pm$ 0.03}} & \multicolumn{1}{c}{15.85 \mygray{$\pm$ 0.81}} & \multicolumn{1}{c}{4.57 \mygray{$\pm$ 0.10}} & \multicolumn{1}{c}{15.98 \mygray{$\pm$ 1.67}} & \multicolumn{1}{c}{4.44 \mygray{$\pm$ 0.12}} \\
FOCAL & \multicolumn{1}{c}{26.62 \mygray{$\pm$ 1.02}} & \multicolumn{1}{c}{5.21 \mygray{$\pm$ 0.17}} & \multicolumn{1}{c}{27.42 \mygray{$\pm$ 2.33}} & \multicolumn{1}{c}{5.32 \mygray{$\pm$ 0.20}} & \multicolumn{1}{c}{33.03 \mygray{$\pm$ 2.28}} & \multicolumn{1}{c}{5.70 \mygray{$\pm$ 0.15}} \\
FreqMAE & \multicolumn{1}{c}{27.48 \mygray{$\pm$ 1.43}} & \multicolumn{1}{c}{5.14 \mygray{$\pm$ 0.11}} & \multicolumn{1}{c}{28.32 \mygray{$\pm$ 1.22}} & \multicolumn{1}{c}{5.17 \mygray{$\pm$ 0.14}} & \multicolumn{1}{c}{28.68 \mygray{$\pm$ 6.96}} & \multicolumn{1}{c}{5.32 \mygray{$\pm$ 0.24}} \\
PhyMask & \multicolumn{1}{c}{23.18 \mygray{$\pm$ 4.96}} & \multicolumn{1}{c}{4.98 \mygray{$\pm$ 0.32}} & \multicolumn{1}{c}{24.09 \mygray{$\pm$ 3.31}} & \multicolumn{1}{c}{5.03 \mygray{$\pm$ 0.27}} & \multicolumn{1}{c}{27.37 \mygray{$\pm$ 2.04}} & \multicolumn{1}{c}{5.31 \mygray{$\pm$ 0.21}} \\ \rowcolor{lightblue}
\MODEL & \multicolumn{1}{c}{\textbf{12.65 \mygray{$\pm$ 0.61}}} & \multicolumn{1}{c}{\textbf{4.09 \mygray{$\pm$ 0.11}}} & \multicolumn{1}{c}{\textbf{12.48 \mygray{$\pm$ 0.68}}} & \multicolumn{1}{c}{\textbf{4.07 \mygray{$\pm$ 0.02}}} & \multicolumn{1}{c}{\textbf{14.56 \mygray{$\pm$ 2.47}}} & \multicolumn{1}{c}{\textbf{4.27 \mygray{$\pm$ 0.07}}} \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
% \vspace{-0.1in}
\end{table}
\begin{figure}[!htbp]
\centering
\begin{subfigure}[t]{0.48\linewidth}
\centering
\includegraphics[width=\linewidth]{fig/noise_reconstruction.png}
\caption{\small Average spatial position reconstruction error versus the average magnitude of spatial noise added to all nodes.}
\label{fig:noise_reconstruction}
\end{subfigure}
\hfill
\begin{subfigure}[t]{0.48\linewidth}
\centering
\includegraphics[width=\linewidth]{fig/noise_vehicle_localization.png}
\caption{\small Average vehicle localization error versus the range of uniformly distributed spatial noise added to a random node.}
\label{fig:noise_vehicle_localization}
\end{subfigure}
% \vspace{-0.15in}
\caption{\small Ablation studies on spatial information preservation and robustness}
\end{figure}
Beyond the primary experiments in Section~\ref{exp:m3n-vc}, we conduct additional studies to further evaluate the robustness and generalization of \MODEL{}.
\noindent\textbf{Lossy Communication.} We first examine \MODEL{}’s resilience to missing data caused by message drops, a common challenge in real-world sensor networks. Each node’s data is independently dropped with probabilities of 5\%, 10\%, or 20\%. As shown in Table~\ref{tb:m3n-vc4}, \MODEL{} consistently achieves the lowest mean squared error and distance error across all settings, demonstrating strong localization performance even under substantial input loss.
\noindent\textbf{Preservation of Spatial Information in Learned Representations.} We conduct an additional ablation study to directly examine whether \MODEL{} preserves spatial information in its learned representations. Using the M3N-VC dataset under the single-vehicle localization setting, we randomly perturb each node’s spatial position by adding i.i.d. uniformly distributed noise with varying magnitudes. The pretrained encoder is frozen, and a lightweight spatial decoder is trained to reconstruct the perturbed spatial positions from the learned representations. We compare \MODEL{} with an ablated variant that does not include the spatial reconstruction objective during pretraining. As shown in Figure~\ref{fig:noise_reconstruction}, the reconstruction error for the ablated model increases approximately linearly with the magnitude of the injected noise, while \MODEL{} maintains consistently low reconstruction error across all noise levels, providing direct evidence that \MODEL{}'s representations robustly encode spatial position information.
\noindent\textbf{Robustness to Noise in Spatial Positions.} We further investigate the robustness of \MODEL{} to noise in sensor spatial positions by perturbing the position of a single node with uniformly distributed random noise in the M3N-VC single-vehicle localization task. We consider two settings: injecting noise only at test time, and injecting noise consistently during both fine-tuning and testing. Figure~\ref{fig:noise_vehicle_localization} shows that localization error increases as the magnitude of spatial perturbation grows, indicating that the downstream localization head actively relies on spatial information encoded in the representations. At the same time, performance remains stable under small perturbations, deteriorates smoothly when the noise increases, and stays reasonable even when the maximal perturbation magnitude exceeds the average distance to the nearest neighboring node. These observations suggest that \MODEL{}’s representations capture spatial context without depending on precise coordinates of individual nodes, demonstrating the robustness to realistic spatial noises.
\section{Qualitative Visualizations}\label{app:vis}
To complement the quantitative results, we provide qualitative visualizations that illustrate \MODEL{}’s spatial reasoning and representation quality across tasks. These examples highlight its ability to accurately localize targets, distinguish between classes, and learn well-structured embeddings compared to baselines.
\begin{figure}[!htbp]
\centering
\begin{subfigure}[t]{0.48\linewidth}
\centering
\includegraphics[width=\linewidth]{fig/vehicle_loc_vis.jpg}
\caption{\small Single-vehicle localization in the M3N-VC dataset, overlaid on an OpenStreetMap basemap (© contributors, ODbL).}
\label{fig:loc1}
\end{subfigure}
\hfill
\begin{subfigure}[t]{0.48\linewidth}
\centering
\includegraphics[width=\linewidth]{fig/earthquake_loc_vis.jpg}
\caption{\small Earthquake localization in the Ridgecrest region of California, overlaid on a topographic basemap © Esri, HERE, Garmin, FAO, NOAA, USGS, EPA, NPS. }
\label{fig:loc2}
\end{subfigure}
% \vspace{-0.15in}
\caption{\small Visualization of localization results. Blue dots denote ground truth locations (vehicle or earthquake epicenter), red dots are predictions by \MODEL{}, and yellow triangles represent the spatial positions of deployed sensor nodes/stations. \MODEL{} produces predictions that closely align with ground truth locations, demonstrating its robust spatial reasoning capability.}
\label{fig:loc_combined}
\vspace{-0.2in}
\end{figure}
\begin{figure}[!htbp]
\centering
\includegraphics[width=0.8\linewidth]{fig/confusion_matrix.pdf}
\vspace{-0.1in}
\caption{\small Confusion matrix of \MODEL{} for the M3N-VC single-vehicle classification task (left) and the RealWorld-HAR activity recognition task (right). The classes are mostly separated by \MODEL{}, and the confusion patterns generally align with the conceptual closeness of the classes.}
\label{fig:cls2}
\vspace{-0.2in}
\end{figure}
\begin{figure}[!htbp]
\centering
\vspace{-0.1in}
\includegraphics[width=0.8\linewidth]{fig/t-SNE.pdf}
\vspace{-0.1in}
\caption{\small t-SNE plot of \MODEL{} and FreqMAE on the M3N-VC Single-vehicle classification task and on the Realworld-HAR activity recognition task. \MODEL{} produces clearly structured clusters: each vehicle class is distinct and separable from the background, and most activity classes (e.g., Walking, ClimbingUp, ClimbingDown) are well differentiated, with only minor overlap between semantically similar classes like Standing and Sitting. In contrast, FreqMAE yields less structured embeddings, where vehicle classes mix more heavily, and activity classes such as Walking, ClimbingUp, and ClimbingDown collapse into broad clusters, indicating weaker fine-grained semantic alignment.}
\label{fig:tsne}
\vspace{-0.2in}
\end{figure}
\begin{figure}[!htbp]
\centering
\vspace{-0.2in}
\includegraphics[width=\linewidth]{fig/multi_loc_2x2.pdf}
\vspace{-0.1in}
\caption{\small Representative examples from the multi-vehicle localization task. Each subplot displays the ground truth vehicle classes and locations, the predicted classes and locations, and the spatial positions of sensor nodes. A 4-meter radius is drawn around each ground truth vehicle to represent the spatial threshold used for metric mAP@4m during evaluation. Predictions that correctly match both the class and fall within this radius are labeled as true positives (TP). Predictions with incorrect class labels or those that fall outside the threshold are labeled as false positives (FP), while ground truth vehicles with no matching predictions are considered false negatives (FN). The top row shows scenarios where predictions are accurate in both class and location. The bottom row illustrates challenging cases where mismatches in class or location lead to evaluation errors. These illustrations demonstrate \MODEL{}’s ability to produce accurate predictions under strict matching criteria. }
\label{fig:multi}
\vspace{-0.1in}
\end{figure}
\newpage
\section{Notation Table}\label{app:notation}
For the reader’s convenience, we provide a summary of the notations used throughout the paper, along with their corresponding dimensions and definitions, in Table~\ref{tb:notation}.
\begin{table}[!htb]
% \vspace{-0.1in}
\caption{Summary of the notations and their corresponding dimensions and definitions.}
\label{tb:notation}
\begin{center}
\begin{adjustbox}{max width=\linewidth}
\begin{tabular}{c|c|c}
\toprule
Notation & Dimension(s) & Definition \\\midrule
$K$ & $\sN$ & Number of modalities \\
$n^{(k)}, m^{(k)}, m_\mM^{(k)}$ & $\sN$ & Number of nodes, tokens, and masked tokens \\
$d, d_\tX^{(k)}$ & $\sN$ & Model dimension, tokenized signal dimension \\
$d_\mS, d_\mR$ & $\sN$ & Spatial and structural position dimensions \\
$L, L'$ & $\sN$ & Loss function of \MODEL{} and classical MAE \\
$\tX^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\tX^{(k)}}$ & Signals \\
$\widehat{\tX}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\tX^{(k)}}$ & Reconstructed signals \\
$\mathsf{X}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\tX^{(k)}}$ & Signals random variable \\
$\widetilde{\tX}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d}$ & Signal embeddings \\
$\mS^{(k)}$ & $\sR^{n^{(k)} \times d_\mS}$ & Spatial positions \\
$\tS^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\mS}$ & Spatial positions (broadcasted) \\
$\widehat{\tS}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\mS}$ & Reconstructed spatial positions\\
$\mathsf{S}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\mS}$ & Spatial positions random variable \\
$\widetilde{\tS}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d}$ & Spatial positional embeddings \\
$\mR^{(k)}$ & $\sR^{n^{(k)} \times d_\mR}$ & Structural positions \\
$\tR^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\mR}$ & Structural positions (broadcasted) \\
$\mathsf{R}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d_\mR}$ & Structural positions random variable \\
$\widetilde{\tR}^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d}$ & Structural positional embeddings \\
% $\tE^{(k)}$ & $\sR^{n^{(k)} \times m^{(k)} \times d}$ & Task-specific positional embeddings\\
$\mM^{(k)}$ & $\{0, 1\}^{n^{(k)}\times m^{(k)}}$ & Mask \\
$\overline{\mM}^{(k)}$ & $\{0, 1\}^{n^{(k)}\times m^{(k)}}$ & Complement mask \\
$\mZ^{(k)}$ & $\sR^{m_\mM^{(k)} \times d}$ & Pre-fusion latent embeddings\\
$\widetilde{\mZ}^{(k)}$ & $\sR^{m_\mM^{(k)} \times d}$ & Post-fusion latent embeddings \\
$\widetilde{\mathsf{Z}}^{(k)}$ & $\sR^{m_\mM^{(k)} \times d}$ & Post-fusion latent embeddings random variable \\
\bottomrule
\end{tabular}
\end{adjustbox}
\end{center}
% \vspace{-0.2in}
\end{table}
\section{Proofs}\label{app:proofs}
\subsection{Proof of Proposition \ref{pp:information}}
\begin{proof}\label{pf:information}
In this proof, we use $C$ and $C'$ to denote generic constants independent of model parameters, whose specific values may change from Equation to Equation.
\noindent\textbf{Classical MAE.} We begin with the case of classical MAE. We assume, following prior works~\citep{li2022exploring,kong2023understanding}, that due to the high dimension of the latent embeddings relative to the original signal, the latent embeddings $\widetilde{\mZ}^{(k)}$ may contain the full information about the unmasked part of the signals $\mask({\tX^{(k)};\mM^{(k)}})$, which can be reconstructed by the decoder from the latent embeddings with negligible loss. As a result, we consider the reconstruction loss calculated on the masked signals to be equivalent to the reconstruction loss calculated on the full signals:
\begin{equation}
\begin{aligned}
L' &= \sum_{k=1}^K \| \mask(\tX^{(k)}-\widehat{\tX}^{(k)};\overline{\mM}^{(k)})\|_2^2 \\
&= \sum_{k=1}^K \| \tX^{(k)}-\widehat{\tX}^{(k)}\|_2^2 \\
& = \sum_{k=1}^K L'^{(k)},
\end{aligned}
\end{equation}
where $L'^{(k)}$ is the reconstruction loss calculated for the $k$-th modality.
Like in the analysis for general regression tasks, the likelihood $P_\mathrm{dec}(\mathsf{X}^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)})$ implicitly modeled by the decoder is defined as a fully factorized Gaussian distribution with mean $\widehat{\tX}^{(k)}$:
\begin{equation}
P_\mathrm{dec}(\mathsf{X}^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)}) \overset{\underset{\mathrm{def}}{}}{=} \gN(\widehat{\tX}^{(k)}, \frac{1}{\sqrt{2}}\mathrm{I}).
\end{equation}
Then, we can interpret the MSE loss $L'^{(k)}$ as proportional to the negative log-likelihood:
\begin{equation}\label{eq:nll}
\begin{aligned}
-\log P_\mathrm{dec}(\mathsf{X}^{(k)}=\tX^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)}) & = \| \tX^{(k)}-\widehat{\tX}^{(k)}\|_2^2 + C' \\
&= L'^{(k)} + C'.
\end{aligned}
\end{equation}
Since the prior probability $P(\mathsf{X}^{(k)}=\tX^{(k)})$ is also a constant independent of the model parameters (only determined by the dataset distribution), we can further have
\begin{equation}
\begin{aligned}
L'^{(k)} + C' = -\log \frac{ P_\mathrm{dec}(\mathsf{X}^{(k)} =\tX^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)})}{P(\mathsf{X}^{(k)} =\tX^{(k)})}.
\end{aligned}
\end{equation}
Taking expectation over the data distribution and applying the standard mutual information decomposition, we can have:
\begin{equation}\label{eq:bound}
\begin{aligned}
\E[L'^{(k)}] + C' & = \E[-\log \frac{ P_\mathrm{dec}(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)})}{P(\mathsf{X}^{(k)} )}] \\
& = \E[-\log \frac{ P(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)})}{P(\mathsf{X}^{(k)} )} + \log \frac{ P(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)})}{P_\mathrm{dec}(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)})}] \\
& = - I(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)}) + KL(P(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)}) || P_\mathrm{dec}(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)})) \\
& \geq - I(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)}),
\end{aligned}
\end{equation}
where $P(\mathsf{X}^{(k)} |\widetilde{\mathsf{Z}}^{(k)})$ denotes the non-tractable ground truth conditional distribution determined by the data distribution and the encoders, and $KL(\cdot||\cdot)$ denotes Kullback–Leibler divergence.
Summing over all modalities, we can prove:
\begin{equation}
\begin{aligned}
-\E[L'] + C' \leq \sum_{k=1}^KI(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)}).
\end{aligned}
\end{equation}
\bigskip
\noindent\textbf{\MODEL{}.}
For \MODEL{}, the signal decoder takes additional inputs: masked spatial and structural positional embeddings. Let $\mathsf{S}_\mM^{(k)}$ and $\mathsf{R}_\mM^{(k)}$ denote the masked spatial and structural positions. Let $L_\text{sig}^{(k)}$ denote the signal reconstruction loss for modality $k$. Then, adjusting our reasoning above, we can modify Equation~\ref{eq:nll} to:
\begin{equation}
L_\mathrm{sig}^{(k)} + C = -\log P_\mathrm{dec}(\mathsf{X}^{(k)}=\tX^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)}, \mathsf{S}_\mM^{(k)}=\tS_\mM^{(k)}, \mathsf{R}_\mM^{(k)}=\tR_\mM^{(k)}).
\end{equation}
Since in \MODEL{}, the latent embeddings $\widetilde{\mZ}^{(k)}$ are calculated not only from unmasked signals, but also from unmasked spatial and structural positional embeddings, we can re-use our assumption above that the latent embeddings $\widetilde{\mZ}^{(k)}$ retain the full information of them. As a result, we can equivalently condition the likelihood on full spatial and structural positions:
\begin{equation}
\begin{aligned}
L_\mathrm{sig}^{(k)} + C &= -\log P_\mathrm{dec}(\mathsf{X}^{(k)}=\tX^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)}, \mathsf{S}_\mM^{(k)}=\tS_\mM^{(k)}, \mathsf{R}_\mM^{(k)}=\tR_\mM^{(k)})\\
&=-\log P_\mathrm{dec}(\mathsf{X}^{(k)}=\tX^{(k)}|\widetilde{\mathsf{Z}}^{(k)}=\widetilde{\mZ}^{(k)}, \mathsf{S}^{(k)}=\tS^{(k)}, \mathsf{R}^{(k)}=\tR^{(k)}).
\end{aligned}
\end{equation}
As the reasoning above, since the prior probability $P(\mathsf{X}^{(k)}=\tX^{(k)}|\mathsf{S}^{(k)}=\tS^{(k)}, \mathsf{R}^{(k)}=\tR^{(k)})$ is also independent of the model parameters, we can adjust Equation~\ref{eq:bound} to:
\begin{equation}
\begin{aligned}
\E[L_\mathrm{sig}^{(k)}] + C \geq - I(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)} | \mathsf{S}^{(k)}, \mathsf{R}^{(k)}).
\end{aligned}
\end{equation}
Since \MODEL{} treats spatial positions symmetrically to signals. We can apply the same reasoning on signal reconstruction loss to the spatial reconstruction loss $L_\mathrm{sp}^{(k)}$, yielding:
\begin{equation}
\begin{aligned}
\E[L_\mathrm{sp}^{(k)}] + C \geq - I(\mathsf{S}^{(k)}; \widetilde{\mathsf{Z}}^{(k)} | \mathsf{X}^{(k)}, \mathsf{R}^{(k)}).
\end{aligned}
\end{equation}
Summing over all modalities and both reconstruction losses, we can prove:
\begin{equation}
\begin{aligned}
-\E[L] + C \leq \sum_{k=1}^K I(\mathsf{X}^{(k)}; \widetilde{\mathsf{Z}}^{(k)} | \mathsf{S}^{(k)}, \mathsf{R}^{(k)}) + I(\mathsf{S}^{(k)}; \widetilde{\mathsf{Z}}^{(k)} | \mathsf{X}^{(k)}, \mathsf{R}^{(k)}).
\end{aligned}
\end{equation}
\end{proof}
\subsection{Proof of Proposition \ref{pp:occlusion}}
\begin{proof}\label{pf:occlusion}
\noindent\textbf{Classical MAE.}
Kong~\etal~\citep{kong2023understanding} provided a rigorous interpretation of classical MAE as a special case of contrastive learning, where the positive pair consists of two complementary masked views of the same input signals. For completeness and clarity, we briefly restate their reasoning here using our notation. For clarity, we focus on a single modality by omitting the superscript $(k)$ and the joint encoder $\fjointenc$; the extension to multiple modalities is straightforward.
Let $\fembencnk'$ denote the composition of the embedding layer and encoder in classical MAE, and let $\fdecnk'$ denote the decoder. Then, the reconstruction process can be written as:
\begin{equation}
\begin{aligned}
\widehat{\tX} = \fdecnk'(\fembencnk'(\mask(\tX;\mM))).
\end{aligned}
\end{equation}
Accordingly, the reconstruction loss of classical MAE can be rewritten as
\begin{equation}
\begin{aligned}
L' &= \| \mask(\tX-\widehat{\tX};\overline{\mM})\|_2^2 \\
&= \| \mask(\tX;\overline{\mM})-\mask(\widehat{\tX};\overline{\mM})\|_2^2 \\
& = \| \mask(\tX;\overline{\mM})- \mask(\fdecnk'(\fembencnk'(\mask(\tX;\mM)));\overline{\mM})\|_2^2 .
\end{aligned}
\end{equation}
Kong~\etal~\citep{kong2023understanding} assumes that due to the high dimension of the latent embeddings relative to the original signals, the latent embeddings produced by the $\fembencnk'$ may approximately preserve all the information of the input. This implies the existence of an alternative decoder $\fdecnktilde'$ that can satisfy:
\begin{equation}
\begin{aligned}
\mask(\tX;\overline{\mM}) \approx \mask(\fdecnktilde'(\fembencnk'(\mask(\tX;\overline{\mM})));\overline{\mM}),
\end{aligned}
\end{equation}
where $\fdecnktilde'$ can be optimized as:
\begin{equation}\label{eq:lfdec}
\begin{aligned}
L_{\fdecnktilde'} &= \|\mask(\tX;\overline{\mM}) - \mask(\fdecnktilde'(\fembencnk'(\mask(\tX;\overline{\mM})));\overline{\mM})\|_2^2 \\
\fdecnktilde' &= \argmin_{\fdecnktilde'} \E[L_{\fdecnktilde'}].
\end{aligned}
\end{equation}
Using this approximation, the classical MAE loss can be rewritten as:
\begin{equation}
\begin{aligned}
L' & \approx \| \mask(\fdecnktilde'(\fembencnk'(\mask(\tX;\overline{\mM})));\overline{\mM}) \\
& - \mask(\fdecnk'(\fembencnk'(\mask(\tX;\mM)));\overline{\mM})\|_2^2 .
\end{aligned}
\end{equation}
To draw a connection to contrastive learning, we define the following similarity measure:
\begin{equation}
\gG'(\mZ_1, \mZ_2) \overset{\underset{\mathrm{def}}{}}{=} \| \mask(\fdecnktilde'(\mZ_1);\overline{\mM})- \mask(\fdecnk'(\mZ_2);\overline{\mM})\|_2^2 .
\end{equation}
Then the classical MAE loss can be rewritten as:
\begin{equation}\label{eq:siamese}
L' \approx \gG'(\fembencnk'(\mask(\tX;\overline{\mM})), \fembencnk'(\mask(\tX;\mM))),
\end{equation}
where $\fembencnk'$ is ensured non-trivial by Equation~\ref{eq:lfdec}.
This reveals the contrastive learning view of classical MAE: $L'$ encourages the encoder $\fembencnk'$ to produce similar latent representations from two complementary masked views of the same input signals:
\begin{equation}
\left[\textnormal{\mask}({\tX};\mM), \quad \textnormal{\mask}({\tX};\overline{\mM})\right],
\end{equation}
which explicitly promotes the learning of occlusion-invariant representations in the signal domain.
\bigskip
\noindent\textbf{\MODEL{}.}
We now turn to \MODEL{}. To unify the components used in encoding, we define an extended encoder $\fencnktilde$ that encapsulates the signal, spatial, and structural embeddings, along with the encoder $\fembencnk$ and additional pre-decoder inputs:
\begin{equation}
\fencnktilde(\mask(\tX;\mM), \tS, \tR) \overset{\underset{\mathrm{def}}{}}{=} \left(\fencnk(\mask(\widetilde{\tX}+\widetilde{\tS}+\widetilde{\tR}; \mM)), \mask(\widetilde{\tS}+\widetilde{\tR} ; \overline{\mM})\right).
\end{equation}
By the same logic as for classical MAE, we can assume the existence of a decoder $\fsigdecnktilde$ that reconstructs $\mask(\tX; \overline{\mM})$ almost losslessly from the output of $\fencnktilde$:
\begin{equation}
\begin{aligned}
\mask(\tX;\overline{\mM}) \approx \mask(\fsigdecnktilde(\fencnktilde(\mask(\tX;\overline{\mM}), \tS, \tR));\overline{\mM}).
\end{aligned}
\end{equation}
We now define another similarity measure:
\begin{equation}
\gG_\mathrm{sig}(\mZ_1, \mZ_2) \overset{\underset{\mathrm{def}}{}}{=} \| \mask(\fsigdecnktilde(\mZ_1);\overline{\mM})- \mask(\fsigdecnk(\mZ_2);\overline{\mM})\|_2^2 .
\end{equation}
Let $L_{\mathrm{sig}}$ denote the signal reconstruction loss in \MODEL{}. Then we have the approximation similar to Equation~\ref{eq:siamese}:
\begin{equation}\label{eq:siamese_our}
L_\mathrm{sig} \approx \gG_\mathrm{sig}(\fencnktilde(\mask(\tX;\overline{\mM}), \tS, \tR), \fencnktilde(\mask(\tX;\mM), \tS, \tR)).
\end{equation}
Following the same argument of Kong~\etal~\citep{kong2023understanding}, this shows that $L_{\mathrm{sig}}$ in \MODEL{} can be viewed as a contrastive loss between two masked views of the signal, enriched with shared spatial and structural context:
\begin{equation}
\left[\left(\textnormal{\mask}({\tX};\mM),{\tS}, {\tR}\right) , \quad \left(\textnormal{\mask}({\tX};\overline{\mM}),{\tS}, {\tR}\right)\right].
\end{equation}
Since \MODEL{} treats spatial positions symmetrically with signals—both in embedding and reconstruction—we can apply the same reasoning to the spatial reconstruction loss $L_{\mathrm{sp}}$. This yields another type of contrastive pair:
\begin{equation}
\left[\left({\tX},\textnormal{\mask}({\tS};\mM), {\tR}\right) , \quad \left({\tX},\textnormal{\mask}({\tS};\overline{\mM}), {\tR}\right)\right].
\end{equation}
\end{proof}
\section{Additional Experimental Setup}\label{app:experiment}
\subsection{Baseline Methods Descriptions}\label{app:baselines}
Below, we provide detailed elaborations on the baseline methods introduced in Section~\ref{sec:experiment}.
\textbf{CMC}~\citep{tian2020contrastive} Learns shared representations by maximizing mutual information between views, enabling view-agnostic and scalable contrastive learning across multiple modalities.
\textbf{Cosmo}~\citep{ouyang2022cosmo} Integrates contrastive feature alignment with attention-based selective fusion to effectively capture shared and distinctive patterns from multimodal data under scarce labeling.
\textbf{SimCLR}~\citep{chen2020simple} Forms discriminative visual embeddings by aligning augmented image pairs through a nonlinear projection and optimizing the NT-Xent contrastive loss.
\textbf{AudioMAE}~\citep{huang2022masked} Applies masked autoencoding to audio by operating on spectrogram patches, using a Transformer to reconstruct masked regions and capture time-frequency patterns without relying on external modalities.
\textbf{CAV-MAE}~\citep{gong2022contrastive} Combines masked autoencoding and contrastive learning in a unified audio-visual framework, using modality-specific encoders and a joint decoder to learn both fused and aligned representations from spectrogram and image patches.
\textbf{FOCAL}~\citep{liu2023focal} Separates multimodal time-series signals into shared and private latent spaces, enforcing orthogonality and applying contrastive and temporal constraints to capture both modality-consistent and modality-exclusive features.
\textbf{FreqMAE}~\citep{kara2024freqmae} Enhances masked autoencoding for multimodal sensing by incorporating frequency-aware transformers, factorized fusion of shared and private features, and a weighted loss that prioritizes informative frequency bands and high-SNR samples.
\textbf{PhyMask}~\citep{kara2024phymask} Improves masked autoencoding by adaptively selecting time-frequency patches based on energy and coherence metrics, enabling efficient and informative masking tailored to physical sensing signals.
\subsection{Settings for Multi-modal Multi-node Vehicle Classification Dataset}
The Multi-Modality Multi-Node Vehicle Classification Dataset (M3N-VC)~\citep{li2025restoreml} (CC BY 4.0) comprises synchronized audio and vibration recordings of four vehicle types, along with background noise, collected from March 2023 to October 2024. Data were gathered across six distinct real-world scenes, each featuring diverse terrain types (asphalt, dirt, gravel, and concrete) and varying weather conditions (sunny, rainy, and windy).
Each scene is instrumented with a spatially distributed sensor network composed of 6 to 8 nodes. Every node includes a co-located microphone (sampled at 16 kHz) and a geophone (sampled at 200 Hz). Vehicle GPS trajectories were recorded at a rate of 1 Hz. All recordings are segmented into non-overlapping 2-second clips, resulting in a total of 21,694 samples. These clips are transformed into mel-scale spectrograms for model input. GPS coordinates are converted into meter-level spatial positions using the Local Tangent Plane approximation~\citep{united1987department}.
The dataset follows the official temporal split for training and validation (approximately 3:1). Unless otherwise noted, all models—including ours and the baselines—are pretrained on all six scenes.
We evaluate model performance on three downstream tasks:
\textbf{Single-Vehicle Classification.}
For this task, we use scene H24 for both fine-tuning and testing. A simple linear classifier is employed as the task head, trained using standard cross-entropy loss.
\textbf{Single-Vehicle Localization.}
This task also uses scene H24 for fine-tuning and testing. A single transformer layer is used as the task head and optimized with the mean squared error (MSE) loss.
\textbf{Multi-Vehicle Joint Classification and Localization.}
For multi-vehicle settings, we use scene I22, which contains multiple moving vehicles. A two-layer transformer serves as the task head, trained with a DETR-style loss function~\citep{carion2020end} to handle set-based predictions.
Additionally, we conduct a \textbf{fine-tuning on unseen placement} experiment, where models are pretrained on all scenes except H08 and H24 (which share similar placements). We then finetune and evaluate on scene H24.
\subsection{Settings for Ridgecrest Seismicity Dataset} This dataset contains seismic waveform recordings from 31,452 earthquake events (M > -0.5) occurring between January 1, 2020, and December 31, 2024, within an 80 km radius of (35.9°, -117.6°) in the Ridgecrest region of California. The data collection and processing procedures largely follow the methodology outlined by Si~\etal~\citep{si2024all}.
We obtained the earthquake event catalog by querying the Southern California Seismic Network (SCSN)~\citep{caltech1926socal} via the Southern California Earthquake Data Center (SCEDC)~\citep{center2013southern} online catalog. The selected events include magnitudes higher than -0.5 and depths larger than -5 km. For each event, we collected three-component (East, North, Vertical) waveform data from 16 stations in the California Integrated Seismic Network, using two modalities: high-gain broadband seismometers and high-gain accelerometers. All data are sampled at 100 Hz and retrieved in miniSEED format from the SCEDC Open Data repository.
For each event, we extract a 30.72-second window from all channels as model input. During preprocessing, we detrend the waveforms and apply the Short-Time Fourier Transform (STFT) to generate spectrograms. A 2-35-Hz band-pass filter is applied to remove low-frequency noise (e.g., oceanic and atmospheric microseisms) and high-frequency instrumental or environmental noise. We convert spatial positions of each station from GPS signals to kilometer-level positions using Local Tangent Plane projection~\citep{united1987department}.
We split the dataset temporally: events from 2020 and 2021 are used for training, while events from 2022 to 2024 form the validation set. This results in 22,360 events in the training set and 9,092 events in the validation set.
For the downstream task of earthquake localization, we employ a two-layer transformer as the task head, optimized using the mean squared error (MSE) loss.
\subsection{Settings for RealWorld-HAR Dataset}
The RealWorld Human Activity Recognition (HAR) dataset~\citep{sztyler2016body} comprises multi-modal activity signals collected from 15 participants. The dataset captures eight common activity types: walking, sitting, lying, climbing down, running, standing, climbing up, and jumping.
Sensor data were collected from seven body-mounted nodes, located at the chest, forearm, head, shin, thigh, upper arm, and waist. For our study, we focus on three sensing modalities: acceleration, gyroscope, and magnetic field. Due to substantial data loss in the forearm sensor, we exclude that position and retain six body locations for analysis. As the dataset does not provide explicit spatial coordinates, we manually assign approximate 3D spatial positions to each sensor based on standard anatomical placement on a standing person.
All sensor signals are resampled to 50Hz and segmented into non-overlapping 4-second windows, resulting in a total of 13,351 samples. To evaluate generalization to unseen individuals, we adopt a subject-based split: data from the first 10 participants are used for training, while data from the remaining 5 participants form the validation set.
For the downstream task of human activity recognition, we use a simple linear layer as the task head, trained with standard cross-entropy loss.
\end{document}
%%%%% NEW MATH DEFINITIONS %%%%%
\usepackage{amsmath,amsfonts,bm}
% Mark sections of captions for referring to divisions of figures
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
% Highlight a newly defined term
\newcommand{\newterm}[1]{{\bf #1}}
% Figure reference, lower-case.
\def\figref#1{figure~\ref{#1}}
% Figure reference, capital. For start of sentence
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
% Section reference, lower-case.
\def\secref#1{section~\ref{#1}}
% Section reference, capital.
\def\Secref#1{Section~\ref{#1}}
% Reference to two sections.
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
% Reference to three sections.
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
% Reference to an equation, lower-case.
\def\eqref#1{equation~\ref{#1}}
% Reference to an equation, upper case
\def\Eqref#1{Equation~\ref{#1}}
% A raw reference to an equation---avoid using if possible
\def\plaineqref#1{\ref{#1}}
% Reference to a chapter, lower-case.
\def\chapref#1{chapter~\ref{#1}}
% Reference to an equation, upper case.
\def\Chapref#1{Chapter~\ref{#1}}
% Reference to a range of chapters
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
% Reference to an algorithm, lower-case.
\def\algref#1{algorithm~\ref{#1}}
% Reference to an algorithm, upper case.
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
% Reference to a part, lower case
\def\partref#1{part~\ref{#1}}
% Reference to a part, upper case
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
% Random variables
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
% rm is already a command, just don't name any random variables m
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
% Random vectors
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
% Elements of random vectors
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
% Random matrices
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
% Elements of random matrices
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
% Vectors
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
% Elements of vectors
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
% Matrix
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
% Tensor
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
% I made this change
\newcommand{\tens}[1]{\bm{\mathcal{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
% Graph
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
% Sets
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
% Don't use a set called E, because this would be the same as our symbol
% for expectation.
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
% Entries of a matrix
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
% entries of a tensor
% Same font as tensor, without \bm wrapper
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
% The true underlying data generating distribution
\newcommand{\pdata}{p_{\rm{data}}}
% The empirical distribution defined by the training set
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
% The model distribution
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
% Stochastic autoencoder distributions
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} % Laplace distribution
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
% Wolfram Mathworld says $L^2$ is for function spaces and $\ell^2$ is for vectors
% But then they seem to use $L^2$ for vectors throughout the site, and so does
% wikipedia.
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} % See usage in notation.tex. Chosen to match Daphne's book.
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
% \usepackage[colorlinks=true]{hyperref}
% \hypersetup{%
% ,urlcolor=blue
% % ,citecolor=RawSienna
% ,citecolor=blue
% % ,linkcolor=MidnightBlue
% ,linkcolor=red
% }