Multi-scale Activation, Refinement, and Aggregation: Exploring Diverse Cues for Fine-Grained Bird Recognition
Extracted source text · arXiv 2504.09215
%File: anonymous-submission-latex-2025.tex
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai25} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\usepackage{amsmath}
% \usepackage{hyperref}
\usepackage{multirow}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{xcolor}
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%
% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
\usepackage{algorithm}
\usepackage{algorithmic}
%
% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
\usepackage{newfloat}
\usepackage{listings}
\DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
\lstset{%
basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
aboveskip=0pt,belowskip=0pt,%
showstringspaces=false,tabsize=2,breaklines=true}
\floatstyle{ruled}
\newfloat{listing}{tb}{lst}{}
\floatname{listing}{Listing}
%
% Keep the \pdfinfo as shown here. There's no need
% for you to add the /Title and /Author tags.
\pdfinfo{
/TemplateVersion (2025.1)
}
% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.
% The file aaai25.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%
% Title
% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Multi-scale Activation, Refinement, and Aggregation: Exploring Diverse Cues for Fine-Grained Bird Recognition}
\author{
Zhicheng Zhang\textsuperscript{\rm 1},
Hao Tang\textsuperscript{\rm 1,2},
Jinhui Tang\textsuperscript{\rm 1,}\thanks{Corresponding author.}
}
\affiliations{
%Afiliations
\textsuperscript{\rm 1}~Nanjing University of Science and Technology, China\\
\textsuperscript{\rm 2}~Centre for Smart Health, The Hong Kong Polytechnic University\\
\{zzc\_666, jinhuitang\}@njust.edu.cn, howard.haotang@gmail.com
%
% See more examples next
}
% REMOVE THIS: bibentry
% This is only needed to show inline citations in the guidelines document. You should not need it and can safely delete it.
\usepackage{bibentry}
% END REMOVE bibentry
\begin{document}
\maketitle
\begin{abstract}
Given the critical role of birds in ecosystems, Fine-Grained Bird Recognition (FGBR) has gained increasing attention, particularly in distinguishing birds within similar subcategories. Although Vision Transformer (ViT)-based methods often outperform Convolutional Neural Network (CNN)-based methods in FGBR, recent studies reveal that the limited receptive field of plain ViT model hinders representational richness and makes them vulnerable to scale variance. Thus, enhancing the multi-scale capabilities of existing ViT-based models to overcome this bottleneck in FGBR is a worthwhile pursuit. In this paper, we propose a novel framework for FGBR, namely Multi-scale Diverse Cues Modeling (MDCM), which explores diverse cues at different scales across various stages of a multi-scale Vision Transformer (MS-ViT) in an ``Activation-Selection-Aggregation'' paradigm. Specifically, we first propose a multi-scale cue activation module to ensure the discriminative cues learned at different stage are mutually different. Subsequently, a multi-scale token selection mechanism is proposed to remove redundant noise and highlight discriminative, scale-specific cues at each stage. Finally, the selected tokens from each stage are independently utilized for bird recognition, and the recognition results from multiple stages are adaptively fused through a multi-scale dynamic aggregation mechanism for final model decisions. Both qualitative and quantitative results demonstrate the effectiveness of our proposed MDCM, which outperforms CNN- and ViT-based models on several widely-used FGBR benchmarks.
\end{abstract}
\section{Introduction}
Birds play a crucial role in the biological chain.
However, the advancement of industrialization and consequent environmental degradation have led to a sharp decline in numerous bird species.
Statistics \cite{malhotra2022habitat} indicate that nearly half of all bird species are in decline. Due to their sensitivity to environmental changes, birds' activities can serve as indicators of environmental changes. Fine-Grained Bird Recognition (FGBR) has become a research hotspot, crucial for the conservation of bird species and natural habitats.
\begin{figure}[t!]
\centering
\includegraphics[width=\linewidth]{challenge4.pdf}
\caption{
The primary challenges in FGBR are evident from bird images. Figures (a) to (d) show subtle species differences, often obscured by complex backgrounds. Figures (e) to (h) highlight significant scale variations between distant and close-up shots, making the same body parts appear different and complicating the recognition task.
}\label{challenge}
\end{figure}
Fine-grained bird recognition is widely recognized as a challenging task due to the intrinsic characteristics of bird images~\cite{TangYLT22,JiangTL24}. These challenges can be summarized as follows:
\begin{itemize}
\item \textbf{Complex Backgrounds:}
Bird habitats, whether in the wild or urban environments, often characterized by complex and variable backgrounds that can obscure the subtle features distinguishing one species from another~\cite{ZhaTST23}. For example, as shown in Figure~\ref{challenge}, the key difference between the Yellow-Throated Vireo and the White-Eyed Vireo lies in the eye color. Complex backgrounds can misdirect the model's attention, causing it to focus on irrelevant background elements instead of critical features, leading to potential misclassification.
\item \textbf{Scale Variations:}
Bird images often exhibit significant scale variations due to differences in shooting distances, resulting in varying sizes of the same body part (\emph{e.g.,~}beak and eyes) across images (Figure~\ref{challenge}). These multi-scale differences complicate accurate recognition, requiring the model to consistently identify features such as the shape and texture of the beak or the color of the eyes, regardless of the scale at which they appear.
\end{itemize}
Recently, Vision Transformers (ViTs) \cite{DosovitskiyB0WZ21} have demonstrated impressive performance across various visual tasks~\cite{Jiang000L24,shen2023git}. ViT divides images into patches, projects these patches into tokens, and then feeds them into transformer blocks for image modeling. However, pure ViT architectures face challenges when applied to fine-grained recognition tasks~\cite{FangJTL24,10038499,fu2024cross}.
Firstly, ViT models often struggle to capture the discriminative body parts crucial for birds. To address this issue, researchers have developed methods that encourage the model to focus on object regions rather than background noise. These approaches include selecting specific image patches that contain relevant features \cite{HeCLKYBW22,WangY021} or designing mechanisms that emphasize discriminative cues \cite{10023961,ZhuKLLTS22}. Secondly, ViT's reliance on fixed-size image patches makes it less effective at capturing multi-scale features. To overcome this limitation, previous works have introduced multi-scale modules, which allow the model to observe images at different scales across multiple stages and use the features extracted from the final stage for downstream tasks \cite{LiuL00W0LG21,LiW0MXMF22,0007LYYL023,DongZYTT24}. However, significant scale variations in FGBR remain difficult to manage, as certain features learned at appropriate scales in earlier stages may be discarded during later computations, leading to the loss of valuable information.
Unlike previous works on FGBR that primarily exploit general bird features at a single scale, this work proposes a novel ``Activation-Selection-Aggregation'' paradigm. This approach captures diverse cues by extracting multi-scale features across various stages of a multi-scale Vision Transformer (MS-ViT).
%
To obtain these diverse cues, we first extract features from multiple stages and introduce a multi-scale cue activation module during the forward process to adjust and enhance cue distinctiveness. Consistent with this architecture, we then design the multi-scale token selection mechanism, which leverages deep semantic information to guide token selection in shallow layers, thereby enhancing robustness.
%
With cues learned across multiple stages, we could achieve recognition results at various scales. However, due to significant scale variations in FGBR, recognition results at inappropriate scales can negatively impact overall accuracy. To address this, we implement a gating mechanism that dynamically aggregates the recognition results.
%
We validate the effectiveness of our method on two popular bird datasets and a large-scale species recognition dataset. Additionally, we also provide intuitive visualizations demonstrating how our approach mitigates background noise and offer quantitative analyses of the multi-scale cues aggregation's performance.
\section{Related Work}
The primary challenges of FGBR include high visual similarity among subclasses, complex and variable background environments, and multi-scale representations. Naturally, researchers have explored part localization to extract part-level features, performing feature interaction and alignment for classification. Early approaches typically designed localization sub-networks for FGBR \cite{ZhangDGD14,LinSLJ15,ZhangXEHZEM16}, but these methods often required highly accurate part-level annotations for training, which limits its development. In response, later researchers shifted towards weakly supervised methods, such as Region Proposal Networks (RPN) and attention mechanisms \cite{TangLPT20,HeCLKYBW22,KeCCLG23,yan2023progressive,DongYTTZ24,tang2024divide}.
However, the localization process significantly increases computational cost. Given the crucial role of feature learning in visual tasks, researchers have sought to enhance feature learning to capture subtle differences. \cite{ShenSWJY22} introduces additional convolutional layers to assess the importance of different regions within the feature map, adjusting activation relationships to uncover rich cues. Similarly, \cite{SongY21} employs extra convolutional layers to determine the importance of the feature map, adjusts its activations to encourage the model to focus on more potential regions. Some other researchers focus on identifying the most salient features and leveraging them for interaction and recognition. \cite{HeCLKYBW22} selects the most important tokens before the last layer to encourage the model to focus on discriminative parts. \cite{WangY021} aggregates the important tokens from each transformer layer to capture discriminative features. Similar approaches have been explored by \cite{HuJZHZH021,abs-2303-06442}. Additionally, some methods have investigated pairwise feature interactions to derive a unified yet discriminative image representation from paired images of different species \cite{ZhuangW020,ZhuKLLTS22}.
\begin{figure*}
\centering
\includegraphics[width=\textwidth]{pipeline.pdf}
\caption{The framework of our MDCM. During the forward pass, MSCA adjusts the activation of feature map to ensure the cues learned at different stage are mutually different. Subsequently, MSTS extracts diverse cues from multiple stages and filters out noisy regions. Finally, MSDA dynamic aggregates the classification results for the final model decisions.}
\label{model}
\end{figure*}
\section{Multi-scale Diverse Cues Modeling}
Figure~\ref{model} illustrates the overall framework of our MDCM, which follows an ``Activation-Selection-Aggregation'' paradigm.
Firstly, the feature extractor is employed to capture cues at different scales across various stages. During the forward pass, we adjust the activation strength of these features to ensure that each stage learns distinct features. Secondly, to better focus on the salient parts of birds, we remove irrelevant background patches, highlighting regions crucial for classification. Finally, the multi-scale feature representations are fed into multiple classifiers to generate classification results, which are then aggregated through a gating mechanism for precise target classification.
\subsection{Multi-Scale Vision Transformer}
We utilize Multi-scale Vision Transformer (MS-ViT) \cite{LiW0MXMF22} as our primary backbone, due to its ability to capture multi-scale features from images effectively. Unlike traditional Vision Transformer~\cite{DosovitskiyB0WZ21}, which divides image into patches and project these patches into tokens, MS-ViT introduces a pooling operation within the multi-head self-attention mechanism, termed Multi-Head Pooling Attention (MHPA). This approach reduces the number of tokens, increases the number of channels, lowers the image resolution, and adjusts the scale.
Let $X_0 \in \mathbb{R}^{h_{0} \times w_{0} \times c_{0}}$ represents the bird image, where $h_{0}$, $w_{0}$, and $c_{0}$ denote the height, width, and channels of the input image, respectively. Initially, the input image is divided into patches of size $o \times o$, which are then reshaped into one-dimensional patches sequence and passed through the patch embedding layer. A trainable class (\texttt{cls}) token is appended to the patch sequence, and a learnable position embedding is added to retain positional information. The embedding process is expressed as follows:
\begin{equation}
\mathbf{Z} = [z^0, z^1 \mathbf{E}, z^2 \mathbf{E}, \cdots, z^N \mathbf{E}] + \mathbf{E}_{\text{pos}},
\end{equation}
where $\mathbf{E} \in \mathbb{R}^{(\frac{h_0 \cdot w_0}{o^2} \cdot c_0) \times D}$ is the patch embedding projection, $\mathbf{E}_{\text{pos}} \in \mathbb{R}^{(N+1) \cdot D}$ represents the position embedding, $N$ denotes the number of image patches, $z^0$ is the learnable \texttt{cls} token, and $z^n$ for $n \in \{1, 2, \ldots, N\}$ are the image patches. The MS-ViT consists of $L$ layers of Multi Head Pooling Attention (MHPA) and Multi-Layer Perceptron (MLP) blocks. The forward pass for the $l$-th layer is as follows:
\begin{align}
\mathbf{Z}'_l &= \text{MHPA}(\text{LN}(\mathbf{Z}_{l-1})) + \mathbf{Z}_{l-1}, & l \in 1, 2, \ldots, L, \\
\mathbf{Z}_l &= \text{MLP}(\text{LN}(\mathbf{Z}'_l)) + \mathbf{Z}'_l, & l \in 1, 2, \ldots, L,
\end{align}
where $\text{LN}(\cdot)$ denotes the layer normalization operation. MS-ViT utilizes the \texttt{cls} token of the last encoder layer, $z_L^0$, as the representation of the global feature, which is then forwarded to a classifier head for the final recognition.
\subsection{Multi-Scale Cue Activation}
If cues learned at different stages of MS-ViT are highly similar, it would impede the effective utilization of multiscale information in bird images. Inspired by the success of the attention-based erasing mechanism in CNN-based architectures~\cite{SongY21,ShenSWJY22}, we propose the parameter-free Multi-Scale Cue Activation (MSCA) module for MS-ViT. This module adjusts the activation of discriminative cues at each stage to explicitly learn multiple scale-specific representations, distinguishing it significantly from previous ViT-based methods.
During the forward propagation of the MS-ViT, the image is encoded into tokens for computation. Since the activation strength of cues should not be adjusted arbitrarily, it is crucial to first assess the importance of each token. Specifically, in MHPA, the tokens $z \in \mathbb{R}^{(N+1) \times D}$ are mapped and pooled into three matrices: $\mathbf{Q}$ (query), $\mathbf{K}$ (key), and $\mathbf{V}$ (value). The attention operation is performed as $\mathrm{Softmax}(\frac{\mathbf{Q}\mathbf{K}^\top}{\sqrt{d}})\mathbf{V}$, where $d$ is the dimension of the query vector. It is widely known that the \texttt{cls} token in ViTs tends to focus more on class-specific tokens rather than on tokens associated with non-object regions. Therefore, we propose utilizing the attentiveness of the \texttt{cls} token to other tokens as an activation map to identify the most important cues. Specifically, the activation map $\mathbf{A}$ is defined as $\mathbf{A}=\mathrm{Softmax}(\frac{\mathbf{q}_{class}\cdot \mathbf{K}^\top}{\sqrt{d}})$, where $\mathbf{q}_{class}$ represents the query vector of the \texttt{cls} token, and $\mathbf{K}$ and $\mathbf{V}$ correspond to the key and value matrices, respectively. Consequently, the activation value $a_i$ (\emph{i.e.}, the $i$-th entry in $\mathbf{A}$) reflects the importance of the $i$-th token.
Some studies~\cite{SerranoS19,AbnarZ20} indicate that high-level attention information may not always align with the true importance of the input tokens. To enhance robustness, we average the activation maps from previous stages to form the attention map for the current stage. To address the shape mismatch caused by pooling operations, we employ interpolation to adjust the shapes accordingly. Subsequently, we map the importance scores to an enhancement and suppression mask to adjust the activation of cues without introducing additional parameters. Specifically, for each element $a^{k}$ in $\mathbf{A}$ with the with index $k \in \{1,2,\cdots,N\}$, we apply the following operation to derive the scaling mask:
\begin{equation}\label{z-score}
m^k = \frac{a^k-\mathrm{mean}(\mathbf{A}')}{\mathrm{std}(\mathbf{A}')^\gamma},
\end{equation}
where $\mathbf{A}' \in \mathbb{R}^{N}$ represents all elements in $\mathbf{A}$ except the first one $a^0$, $\gamma$ is a hyperparameter that regularizes the degree of activation adjustment, and $\mathrm{mean}(\mathbf{A}')$ and $\mathrm{std}(\mathbf{A}')$ denote the mean and standard deviation of $\mathbf{A}'$. The resulting value, $m^k$, serves as the weight for the suppression and enhancement mask corresponding to the $k$-th token. We then apply the mask through element-wise multiplication with the tokens, yielding the activated tokens as $\mathbf{Z}' = \mathbf{Z} \odot \mathbf{M}$, where $\mathbf{M} = \{m^{1}, m^{2}, \cdots, m^N\}$.
\subsection{Multi-Scale Token Selection}
Considering that noisy tokens corresponding to complex background regions may misdirect the model’s attention, the Multi-scale Token Selection (MSTS) mechanism is proposed to filter out noisy tokens.
Let $\mathbf{Z'}_i = \{z_i^0,z_i^1,\cdots,z_i^{h_i\cdot w_i} \} $ denote the output from stage $i$. First, the \texttt{cls} token $z_i^0$ is detached, and the remaining patch tokens $\mathbf{P} = \{z_i^1,z_i^2,\cdots,z_i^{h_i \cdot w_i}\}$ are reshaped from $\mathbb{R}^{h_i \cdot w_i \times c_i}$ to $\mathbb{R}^{h_i \times w_i \times c_i}$. Next, we perform patch merging on $\mathbf{P}$ and flatten it to a 1D token sequence $\mathbf{P}' \in \mathbb{R}^{N_i \times c_i}$, where $N_i = (\frac{h_i\cdot w_i}{4})$. The patch merging method, inspired by Swin Transformer~\cite{LiuL00W0LG21}, expands the model's receptive field.
MSTS calculates scores $\mathbf{S}_i$ for tokens in $\mathbf{P}'$ to determine their importance and retains the top $k_i$ tokens.
The importance of each token $z_i \in \mathbb{R}^{c_i}$ is determined by averaging its activation across channels. The score $s$ for each token is computed as:
\begin{equation}
s = \frac{1}{c_i}\sum_{j=1}^{c_i}z(j),
\end{equation}
where $c_i$ is the number of channels. The scores $\mathbf{S}_i = \{s^1,s^2,\cdots ,s^{N_i}\}$ are sorted in descending order, and the top $k_i$ indices $\mathbf{I}_i$ are selected. The corresponding tokens are then gathered from $\mathbf{P}'$ as follows:
\begin{align}
\mathbf{I}_i &= \operatorname{topkIndex}(\mathbf{S}_i;k_i), \\
\hat{\mathbf{P}_i} &= \operatorname{gather}(\mathbf{P}',\mathbf{I}_i).
\end{align}
To leverage the rich semantic information from deeper layers for localization, MSTS selects tokens in shallow layers correspond to those chosen in deep layers, along with layer-specific tokens. For MS-ViT, Stages 1 and 2 are designated as shallow layers, while Stage 3 serves as the deep layer. Stage 3 captures rich semantic information with its higher resolution map, enabling flexible token selection.
%
For the \texttt{cls} token, instead of using the original \texttt{cls} token from each stage, we employ the \texttt{cls} Token Transfer (CTT) method~\cite{abs-2308-02161} to leverage the rich semantic information from deeper layers. Specifically, a linear layer projects the \texttt{cls} token $z_4^0$ from the last stage to the \texttt{cls} tokens of preceding stages, as defined:
\begin{equation}
{z}_i^{0} = \mathbf{W}_{i}^{1} \left(\mathrm{ReLU}\left(\mathrm{BN}\left(\mathbf{W}_{i}^{0} {z}_4^{0}\right)\right)\right).
\label{eq:layer_output}
\end{equation}
Finally, the $z_i^0$ and $\hat{\mathbf{P}}_i$ are concatenated and passed through a Squeeze and Excitation network~\cite{HuSS18} to explore cross-channel interactions. They are then processed through standard Transformer Blocks independently to obtain multi-scale \texttt{cls} tokens $\hat{z}_i^{0}$ for classification:
\begin{align}
\hat{\mathbf{Z}}_i &= \operatorname{Block}\left(\operatorname{SEM}\left(\operatorname{Concat}\left(z_i^0, \hat{\mathbf{P}}_i\right)\right)\right),\\
\eta_{i} &= \mathbf{W}_i(\hat{z}_i^{0}),
\end{align}
where $\hat{z}_i^{0}$ is the \texttt{cls} token in $\hat{\mathbf{Z}}_i$.
\subsection{Multi-Scale Dynamic Aggregation}
One of the primary challenges in FGBR arises from the scale differences caused by close-up and long-distance shots, making it difficult to rely on a single scale for effective recognition. To address this, our method derives recognition results from multi-scale cues and aggregates them. However, simple summation aggregation indiscriminately combines results from all scales, potentially incorporating inappropriate or even harmful cues. To resolve this, we introduce a gating mechanism that selectively aggregates the most relevant results from multi-scale cues. Details of the mechanism are as follows:
First, we concatenate all \texttt{cls} token $\hat{z}_i^0$ to construct the global multi-scale features $\mathcal{MF}$ of the image. Simultaneously, we stack the recognition results from the four stages to obtain the global classification result $\mathcal{MC} \in \mathbb{R}^{4\times n}$, where $n$ is the number of the classes:
\begin{align}
\mathcal{MF} &= \operatorname{Concat}(\hat{z}^0_{1},\hat{z}^0_{2},\hat{z}^0_{3},\hat{z}^0_{4}), \\
\mathcal{MC} &= \operatorname{Stack}(\eta_1,\eta_2,\eta_3,\eta_4).
\end{align}
Next, we compute a set of gating weights $\mathbf{G} \in \mathbb{R}^{4\cdot n}$ to guide the aggregation process:
\begin{equation}\label{gatingweight}
\mathbf{G} = \sigma(\mathbf{W}\mathcal{MF}+b),
\end{equation}
where $\mathbf{W}$ and $b$ are learnable transformation weights and biases, and $\sigma$ is the element-wise sigmoid activation function.
The gating weights $\mathbf{G}$ are then reshaped into $\mathcal{MG} \in \mathbb{R}^{4\times n}$ to align with the multi-scale recognition results $\mathcal{MC}$. Finally, element-wise multiplication is performed, followed by a summation to produce the final recognition results $\mathcal{MR} \in \mathbb{R}^n$:
\begin{equation}
\mathcal{MR} = \sum_{i=1}^{4}{\mathcal{MG}_i \cdot \mathcal{MC}_i}.
\end{equation}
The gating weights $\mathcal{MG}$ dynamically control the contribution of information from each scale, leveraging cues from all scales. For a particular scale, the proportion of classification results included in the aggregation depends on the corresponding value in $\mathcal{MG}$. The larger the value, the more ideal the recognition result at that scale. This dynamic selection ensures that the model aggregates only the most appropriate classification results from different scales.
Additionally, bird species exhibit statistical differences in their scale distributions. Our gating mechanism aggregates cues across all scales and finely adjusts the weights of the classification scores on all classes from all scales, effectively addressing this challenge.
\subsection{Loss Functions}
With the output from the gating mechanism, we obtain five recognition results (e.g. $\eta_1,\eta_2,\eta_3,\eta_4,\mathcal{MR}$). For clarity, we use $Y=\{ y_1,y_2,y_3,y_4,y_5\}$ to represent the prediction. For each result $y_i$, we use the cross-entropy loss function with label-smoothing \cite{SzegedyVISW16} for optimization, defined as:
\begin{align}
\mathcal{L}_s &= \sum_{y \in Y}\sum_{t=1}^{n}-\hat{y}^t_\beta\mathrm{log}y^t ,\\
\hat{y}_\beta^t&=
\begin{cases}
\beta, & t =\hat{t}, \\
\frac{1-\beta}{n}, & t \neq \hat{t} ,
\end{cases}
\end{align}
where $n$ is the number of classes, $t$ denotes the index of the label element, $\hat{t}$ is the index of the ground-truth class, and $\beta \in [0,1]$ is a smoothing factor. We set $\beta$ to incrementally increase in equal intervals of $0.1$, ranging from $0.6$ to $1$.
To enhance diversity among features and enable multi-scale results to better capture different variations, we incorporate an extra contrastive loss, specifically for $\hat{z_4^0}$ and $\hat{z_3^0}$:
\begin{equation}
\begin{split}
\mathcal{L}_{con} = \frac{1}{B^2}\sum_{i=1}^{B}[\sum_{j:y_i=y_j}^{B}(1-\operatorname{cos}(e^i,e^j)) +\\\sum_{j:y_i \neq y_j}^{B}(\operatorname{max}(\operatorname{cos}(e^i,e^j),0)],
\end{split}
\end{equation}
where $B$ denotes the batch size, $\operatorname{cos}(.,.)$ represents the cosine similarity function, and $e^i$ and $e^j$ refer to $\hat{z_4^0}$ or $\hat{z_3^0}$ extracted from the $i$-th and $j$-th images, respectively.
The final loss function is defined as:
\begin{align} \label{lossfunction}
\mathcal{L} = \mathcal{L}_s +\alpha\mathcal{L}_{con},
\end{align}
where $\alpha$ is the weighting factor for $\mathcal{L}_{con}$.
\section{Experiments}
\subsection{Datasets}
In our experiment, we demonstrate the effectiveness of our method on three fine-grained recognition datasets: CUB-200-2011 \cite{welinder2010caltech}, NABirds \cite{HornBFHBIPB15}, and the iNat2017 \cite{HornASCSSAPB18}. CUB-200-2011 is a widely used FGVC dataset, consists of $11,788$ images and 200 bird species, dividing $5,994$ images for training and $5,794$ images for testing. NABirds is a larger dataset, consists of $48,562$ images and 555 classes, dividing $23,929$ images for training and $24,633$ images for testing. iNat2017 is a large-scale dataset for fine-grained species recognition, consisting of $859,000$ images from over $5,000$ different species of plants and animals, dividing $579,184$ images for training and $95,986$ images for testing.
\subsection{Implementation Details}
For all experiments, we use the standard MS-ViT-Base model pre-trained on ImageNet21K as the backbone. Images are first resized to $600\times 600$ pixels and then uniformly cropped to $448\times 448$ pixels. For the training set, we apply random cropping and random horizontal flipping as data augmentation techniques, while for the test set, center cropping is employed. Our training strategy follows prior works~\cite{HeCLKYBW22}, employing an SGD optimizer with a momentum of $0.9$ and a weight decay of $0.0$.
%
The batch size is set to $16$ across all datasets. The learning rate is fixed at $0.045$, with the contrastive loss weight set to $0.001$. The hyperparameter $\gamma$ in Eq~\ref{z-score} and $\alpha$ in Eq~\ref{lossfunction} are set as $0.3$ and $0.1$, respectively. We use cosine annealing for learning rate decay during training. All experiments are performed in a CUDA 11.0 environment with PyTorch 1.7.1, utilizing two NVIDIA RTX 3090 GPUs.
\subsection{Main Results}
\begin{table}[t!]
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{l|c|c}
\toprule
Method & Backbone & Acc (\%) \\
\midrule
NTS-Net \cite{YangLWHGW18} &\multirow{10}*{ResNet50} & 87.5 \\
Cross-X \cite{LuoYML0LYL19} & & 87.7 \\
DCL \cite{WangCLJD20} & & 87.8 \\
PMG \cite{DuCBXMSG20} & & 89.6 \\
MCEN \cite{LiWZ21} & & 89.3\\
GaRD \cite{ZhaoYHL21} & & 89.6\\
CMN \cite{DengMGZ22} & & 88.2 \\
MA-ASN \cite{ZhangH022} & & 89.5 \\
GDSMP-Net \cite{KeCCLG23} & & 89.9 \\
% PMG-V2 \cite{DuXMCSG22} & ResNet101 & 90.2 \\
TA-CFN \cite{GuanYLZSS23} & & 90.5 \\
% CAP \cite{BeheraWHB21} & Xception & 91.8 \\
\midrule
ViT \cite{DosovitskiyB0WZ21} & \multirow{10}*{ViT-Base} & 90.6 \\
RAMS-Trans \cite{HuJZHZH021} & & 91.3\\
AF-Trans \cite{ZhangCZLWLC22} & & 91.6 \\
DCAL \cite{ZhuKLLTS22} & & 92.0 \\
TransFG \cite{HeCLKYBW22} & & 91.7 \\
SIM-Trans \cite{0002HP22} && 91.8 \\
IELT \cite{XuWJL23} && 91.8\\
MpT-Trans \cite{WangFM23} && 92.0 \\
ACC-ViT \cite{ZhangCWLX24} & & 91.8 \\
MP-FGVC \cite{Jiang0GDHL24} & & 91.8 \\
\midrule
TransIFC+ \cite{10023961} & \multirow{2}*{Swin-Base} & 91.0 \\
HERBS \cite{abs-2303-06442} & & 92.3 \\
\midrule
M2Former \cite{abs-2308-02161} & \multirow{2}*{MS-ViT-Base} & 92.4 \\
\textbf{MDCM (Ours)} & & \textbf{92.5} \\
\bottomrule
\end{tabular}}
\caption{Comparison with different methods on CUB-200-2011.}
\label{Result}
\end{table}
\begin{table}[t!]
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{l|c|c}
\toprule
Method & Backbone & Acc(\%) \\
\midrule
Cross-X \cite{LuoYML0LYL19} &\multirow{5}*{ResNet50}& 86.4 \\
PAIRS \cite{GuoF19} && 87.9 \\
GaRD \cite{ZhaoYHL21} && 88.0 \\
CMN \cite{DengMGZ22} && 87.8\\
GDSMP-Net \cite{KeCCLG23} & & 89.0 \\
\midrule
PMG-V2 \cite{DuXMCSG22} & \multirow{2}*{ResNet101} & 88.4 \\
MGE-CNN \cite{ZhangHLT19} && 88.6 \\
% CAP \cite{BeheraWHB21} & Xception & 91.0 \\
\midrule
ViT \cite{DosovitskiyB0WZ21} &\multirow{6}*{ViT-Base}& 89.6\\
TransFG \cite{HeCLKYBW22} & & 90.8 \\
IELT \cite{XuWJL23} && 90.8\\
MpT-Trans \cite{WangFM23} && 91.3 \\
MP-FGVC \cite{Jiang0GDHL24} & &91.0\\
ACC-ViT \cite{ZhangCWLX24} & &91.4 \\
\midrule
TransIFC+ \cite{10023961} & Swin-Base & 90.9 \\
\midrule
M2Former \cite{abs-2308-02161} &\multirow{2}*{MS-ViT-Base} & 91.1 \\
\textbf{MDCM (Ours)} & & \textbf{92.0} \\
\bottomrule
\end{tabular}}
\caption{Comparison with different methods on NABirds.}
\label{ResultonNABirds}
\end{table}
\subsubsection{Results on CUB-200-2011}
The main results on CUB-200-2011 are shown in Table~\ref{Result}. Our method achieves a top-1 accuracy of $92.5\%$, demonstrating a substantial improvement over CNN-based approaches. When compared to ViT-based methods such as TransFG~\cite{HeCLKYBW22}, DCAL~\cite{ZhuKLLTS22} , and M2Former~\cite{abs-2308-02161}, our method outperforms them by $0.8\%$, $0.5\%$, and $0.1\%$, respectively. Furthermore, it achieves a $1.5\%$ improvement over TransIFC~\cite{10023961}, highlighting the effectiveness of our proposed method.
\subsubsection{Results on NABirds}
The main results on the NABirds dataset are shown in Table~\ref{ResultonNABirds}. Our method achieves a top-1 accuracy of $92.0\%$, representing an improvement of at least $0.6\%$. Specifically, compared to recent ViT-based state-of-the-art methods~\cite{ZhangCWLX24,Jiang0GDHL24}, our approach delivers improvements of $0.6\%$ and $1.0\%$, respectively. When compared to the method~\cite{abs-2308-02161} using the same backbone, we observe a $0.9\%$ improvement. Additionally, against another bird recognition-focused method, our approach achieves a $1.1\%$ improvement. These results substantiate the effectiveness of our proposed method.
\begin{table}
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{l|c|c}
\toprule
Method & Backbone &Acc(\%) \\
\midrule
TASN \cite{ZhengFZL19} &\multirow{2}*{ResNet50}& 68.2 \\
SRGN \cite{WangWLCOT24} && 73.6\\
\midrule
SSN\cite{RecasensKSMT18} &\multirow{2}*{ResNet101}& 65.2 \\
IARG\cite{HuangL20} &&66.8 \\
% \midrule
% FixSENet-154 \cite{abs-2003-08237} & SENet-154 & 75.4 \\
\midrule
RAMS-Trans \cite{HuJZHZH021} &\multirow{6}*{ViT-Base}& 68.5\\
AF-Trans \cite{ZhangCZLWLC22} &&68.9 \\
SIM-Trans \cite{0002HP22} && 69.9\\
TransFG \cite{HeCLKYBW22} & & 71.7 \\
MFVT \cite{lv2022mfvt} & & 72.6 \\
ACC-ViT \cite{ZhangCWLX24} & & 77.0 \\
\midrule
M2Former \cite{abs-2308-02161} &\multirow{2}*{MS-ViT-Base} & 77.8 \\
\textbf{MDCM (Ours)} & & \textbf{79.8} \\
\bottomrule
\end{tabular}}
\caption{Comparison with different methods on iNat2017. }
\label{iNat2017}
\end{table}
\subsubsection{Results on iNat2017}
The main results on the large-scale iNat2017 dataset are shown in Table~\ref{iNat2017}. Our method achieves at least a $2.0\%$ improvement over other approaches, even when some of these methods employ significantly larger backbones~\cite{RyaliHB000ACPHM23,HeCXLDG22}. This highlights the efficiency and effectiveness of our approach.
\begin{table}
\centering
\label{NabridsResults}
\begin{tabular}{lc}
\toprule
Mechanisms & Score \\
\midrule
Addition Operation & 113 \\
Gating Operation & \textbf{173} \\
\bottomrule
\end{tabular}
\caption{Impact of different aggregation choices in MSDA.}
\label{Gatingmechanismscore}
\end{table}
\subsection{Ablation Study}
\subsubsection{Effectiveness of Proposed Modules}
We investigate the impact of the proposed modules, with results shown in Table~\ref{Ablationtable}. The baseline MS-ViT achieves a top-1 accuracy of $91.6\%$ on CUB-200-2011. Introducing multi-scale cues from multiple stages (\emph{i.e.,~}Table \ref{Ablationtable}(b)) results in a $0.4\%$ improvement, demonstrating the effectiveness of our token selection design. Notably, for Table~\ref{Ablationtable}(b), the top-1 accuracy corresponds to the best accuracy among the pre-aggregation results, whereas in experiments with MSDA, it refers to the output of MSDA.
%
Next, integrating our gating mechanism to dynamically aggregate recognition results from diverse multi-scale cues increases accuracy to $92.3\%$ (\emph{i.e.,~}Table \ref{Ablationtable}(c)). This finding highlights a limitation of prior methods, which process images at different scales across stages and rely on final features for downstream tasks, often failing to meet FGBR’s need for diverse features. By contrast, multi-scale cues learned at different stages provide sufficient diversity. However, correct classifications are dispersed across these multi-scale features. MSDA dynamically aggregates the correct results, yielding a significant performance boost. Finally, applying MSCA to adjust the activation of cues in each stage delivers an additional $0.2\%$ improvement, achieving a top-1 accuracy of $92.5\%$.
\begin{table}
\centering
\begin{tabular}{ccccc}
\toprule
& MSTS & MSDA & MSCA & Acc(\%) \\
\midrule
(a) & & & & 91.6 \\
(b) & \checkmark & & & 91.9 \\
(c) & \multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & & 92.3 \\
(d) & \multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & \textbf{92.5} \\
\bottomrule
\end{tabular}
\caption{Ablation study of our MDCM on CUB-200-2011.}
\label{Ablationtable}
\end{table}
\subsubsection{Contributions of MSDA}
We conduct additional experiments to analyze the effectiveness of the gating mechanism. First, we describe the evaluation approach for the gating mechanism. For each picture, five recognition results are generated, resulting in $2^5$ possible combinations of correct or incorrect results. A robust aggregation choice should aim to maintain high accuracy in the aggregated classification result, even when some pre-aggregation results are incorrect. To quantify this, we assign correction scores ranging from $-4$ to $+4$ across the $32$ possible combinations. The correction score for each image is computed as:
\begin{equation}\label{gatingscore}
\text{Score} =
\begin{cases}
\sum_{i=1}^{4}+(y_i\doteq y), & \text{if } y_5 = y \\
\sum_{i=1}^{4}-(y_i\oplus y), & \text{if } y_5 \neq y
\end{cases},
\end{equation}
where $y$ is the ground truth label, $\oplus$ denotes the exclusive OR operation, and $\doteq$ denotes the exclusive NOR operation. In brief, when the aggregated result is correct, the score reflects the number of incorrect pre-aggregation results that were corrected, assigning a corresponding positive value. Conversely, a negative score is assigned when the aggregated result is incorrect. This evaluation approach isolates the impact of pre-aggregation results, ensuring that the accuracy improvement is attributable to the aggregation mechanism rather than better pre-aggregation results.
According to Table~\ref{Gatingmechanismscore}, even a simply summation of recognition results from multi-scale cues learned at different stages yields positive scores, demonstrating that these cues are sufficiently diverse to address the requirements of FGBR. Moreover, our gating mechanism adaptively aggregates these results, effectively mitigating the negative impact of inappropriate cues and achieving higher accuracy.
\begin{figure}[t!]
\centering
\includegraphics[width=\linewidth]{patchselection.pdf}
\caption{
The visualization of the MSTS mechanism highlights the selected tokens marked with red rectangles.
}
\label{NPSMvisual}
\end{figure}
\subsubsection{Impact of Different Stage Cues}
We further analyze the impact of cues at each stage on recognition accuracy. Following COCO dataset\cite{LinMBHPRDZ14}, we categorize the images in CUB-200-2011 based on bounding box sizes into three groups: \texttt{Small}, \texttt{Medium}, and \texttt{Large}, according to quartiles. Detailed results are presented in Table~\ref{NPSM}.
%
The first row shows the performance of the baseline. The second row shows a slight improvement in accuracy, likely due to token selection reducing the influence of noise. Incorporating cues from Stage 3 into the recognition results enhances accuracy for medium and large-scale objects, indicating that Stage 3 cues provide valuable features for objects at these scales.
%
Additionally, cues from Stage 2 and Stage 1 improve recognition across all object sizes—small, medium, and large. This suggests that shallow-layer detail information not only aids in identifying small objects but also supplements the recognition of medium and large objects. Overall, capturing diverse multi-scale cues from different stages effectively addresses the challenges of FGBR.
\begin{table}[t!]
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{cccc|ccc|c}
\toprule
\multicolumn{4}{c|}{Cues from different stages} & \multicolumn{4}{c}{Acc(\%)} \\ \midrule
Stage $4$ & Stage $3$ & Stage $2$ & Stage $1$ & \multicolumn{1}{c}{\texttt{Large}} & \multicolumn{1}{c}{\texttt{Medium}} & \multicolumn{1}{c|}{\texttt{Small}} & Total \\ \midrule
& & & & 92.2 & 91.9 &90.4 & 91.6 \\
\multicolumn{1}{c}{\checkmark} & & & & 91.7 & 92.2 & 90.5 & 91.7 \\
\multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & & & 92.5 & 92.5 & 90.6 & 92.0 \\
\multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & & \textbf{92.9} & 92.7 & 91.2 & 92.4 \\
\multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c}{\checkmark} & \multicolumn{1}{c|}{\checkmark} & \textbf{92.9} & \textbf{92.8} & \textbf{91.5} & \textbf{92.5} \\
\bottomrule
\end{tabular}}
\caption{The impact of cues learned at different stages}
\vspace{-5mm}
\label{NPSM}
\end{table}
\subsection{Visulization}
Figure~\ref{NPSMvisual} shows the tokens selected at each stage by MSTS, with selected tokens highlighted by red rectangles. The first column displays the original image, while the subsequent columns, from left to right, represent tokens selected from the fourth stage to the first stage.
%
In deeper layers, MSTS focuses on primary features at larger scales, whereas in shallower layers, it captures subtle or edge features. For instance, in the third sample, the deeper layers highlight the prominent yellow parts of the wings, while the shallower layers complement this by capturing finer details such as the eyes, beak, and the white feathers beneath the beak.
%
Furthermore, with the guidance provided by MSTS, shallower modules effectively model deeper patches at smaller scales, enabling the capture of subtle differences. Overall, MSTS dynamically selects specific tokens across different stages, capturing discriminative representations in both shallow and deep layers, and providing rich cues for final decision-making.
\section{Conclusion}
This paper proposes a novel framework for fine-grained bird recognition, termed Multi-scale Diverse Cues Modeling (MDCM). The proposed framework captures diverse cues at different scales across various stages of a multi-scale Vision Transformer using an ``Activation-Selection-Aggregation'' paradigm. Specifically, the multi-scale cue activation module ensures that the discriminative cues learned at different stages are mutually distinct. Concurrently, a multi-scale token selection module is designed to remove redundant noise and emphasize discriminative, scale-specific cues at each stage. Finally, the selected tokens from each stage are independently utilized for bird recognition, with the recognition results adaptively fused through a multi-scale dynamic aggregation mechanism to make final model decisions. Qualitative and quantitative experiments consistently demonstrate the superiority of our MDCM for the FGBR task.
% \bigskip
% \section{Acknowledgements}
% This work was partially supported by the National Natural Science Foundation of China (Grant 62332010) and the Major Research Program of Jiangsu Province (Grant BG2024042).
\bibliography{aaai25}
\end{document}
\begin{thebibliography}{65}
\providecommand{\natexlab}[1]{#1}
\bibitem[{Abnar and Zuidema(2020)}]{AbnarZ20}
Abnar, S.; and Zuidema, W.~H. 2020.
\newblock Quantifying Attention Flow in Transformers.
\newblock In \emph{{ACL}}, 4190--4197.
\bibitem[{Chou, Kao, and Lin(2023)}]{abs-2303-06442}
Chou, P.; Kao, Y.; and Lin, C. 2023.
\newblock Fine-grained Visual Classification with High-temperature Refinement and Background Suppression.
\newblock \emph{CoRR}, abs/2303.06442.
\bibitem[{Deng et~al.(2022)Deng, Marsh, Gould, and Zheng}]{DengMGZ22}
Deng, W.; Marsh, J.; Gould, S.; and Zheng, L. 2022.
\newblock Fine-Grained Classification via Categorical Memory Networks.
\newblock \emph{{IEEE} Trans. Image Process.}, 4186--4196.
\bibitem[{Dong et~al.(2024{\natexlab{a}})Dong, Yan, Tang, Tang, and Zhang}]{DongYTTZ24}
Dong, N.; Yan, S.; Tang, H.; Tang, J.; and Zhang, L. 2024{\natexlab{a}}.
\newblock Multi-view Information Integration and Propagation for occluded person re-identification.
\newblock \emph{Inf. Fusion}, 102201.
\bibitem[{Dong et~al.(2024{\natexlab{b}})Dong, Zhang, Yan, Tang, and Tang}]{DongZYTT24}
Dong, N.; Zhang, L.; Yan, S.; Tang, H.; and Tang, J. 2024{\natexlab{b}}.
\newblock Erasing, Transforming, and Noising Defense Network for Occluded Person Re-Identification.
\newblock \emph{{IEEE} Trans. Circuits Syst. Video Technol.}, 4458--4472.
\bibitem[{Dosovitskiy et~al.(2021)Dosovitskiy, Beyer, Kolesnikov, Weissenborn, Zhai, Unterthiner, Dehghani, Minderer, Heigold, Gelly, Uszkoreit, and Houlsby}]{DosovitskiyB0WZ21}
Dosovitskiy, A.; Beyer, L.; Kolesnikov, A.; Weissenborn, D.; Zhai, X.; Unterthiner, T.; Dehghani, M.; Minderer, M.; Heigold, G.; Gelly, S.; Uszkoreit, J.; and Houlsby, N. 2021.
\newblock An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale.
\newblock In \emph{{ICLR}}.
\bibitem[{Du et~al.(2020)Du, Chang, Bhunia, Xie, Ma, Song, and Guo}]{DuCBXMSG20}
Du, R.; Chang, D.; Bhunia, A.~K.; Xie, J.; Ma, Z.; Song, Y.; and Guo, J. 2020.
\newblock Fine-Grained Visual Classification via Progressive Multi-granularity Training of Jigsaw Patches.
\newblock In \emph{{ECCV}}, 153--168.
\bibitem[{Du et~al.(2022)Du, Xie, Ma, Chang, Song, and Guo}]{DuXMCSG22}
Du, R.; Xie, J.; Ma, Z.; Chang, D.; Song, Y.; and Guo, J. 2022.
\newblock Progressive Learning of Category-Consistent Multi-Granularity Features for Fine-Grained Visual Classification.
\newblock \emph{{IEEE} Trans. Pattern Anal. Mach. Intell.}, 9521--9535.
\bibitem[{Fang et~al.(2024)Fang, Jiang, Tang, and Li}]{FangJTL24}
Fang, Z.; Jiang, X.; Tang, H.; and Li, Z. 2024.
\newblock Learning Contrastive Self-Distillation for Ultra-Fine-Grained Visual Categorization Targeting Limited Samples.
\newblock \emph{{IEEE} Trans. Circuits Syst. Video Technol.}, 7135--7148.
\bibitem[{Fu et~al.(2024)Fu, Wang, Pan, Huai, Qiu, Shangguan, Liu, Fu, Van~Gool, and Jiang}]{fu2024cross}
Fu, Y.; Wang, Y.; Pan, Y.; Huai, L.; Qiu, X.; Shangguan, Z.; Liu, T.; Fu, Y.; Van~Gool, L.; and Jiang, X. 2024.
\newblock Cross-domain few-shot object detection via enhanced open-set object detector.
\newblock In \emph{ECCV}, 247--264. Springer.
\bibitem[{Guan et~al.(2023)Guan, Yang, Li, Zhu, Song, and Shen}]{GuanYLZSS23}
Guan, X.; Yang, Y.; Li, J.; Zhu, X.; Song, J.; and Shen, H.~T. 2023.
\newblock On the Imaginary Wings: Text-Assisted Complex-Valued Fusion Network for Fine-Grained Visual Classification.
\newblock \emph{{IEEE} Trans. Neural Networks Learn. Syst.}, 5112--5121.
\bibitem[{Guo and Farrell(2019)}]{GuoF19}
Guo, P.; and Farrell, R. 2019.
\newblock Aligned to the Object, Not to the Image: {A} Unified Pose-Aligned Representation for Fine-Grained Recognition.
\newblock In \emph{{WACV}}, 1876--1885.
\bibitem[{He et~al.(2022{\natexlab{a}})He, Chen, Liu, Kortylewski, Yang, Bai, and Wang}]{HeCLKYBW22}
He, J.; Chen, J.; Liu, S.; Kortylewski, A.; Yang, C.; Bai, Y.; and Wang, C. 2022{\natexlab{a}}.
\newblock TransFG: {A} Transformer Architecture for Fine-Grained Recognition.
\newblock In \emph{{AAAI}}, 852--860.
\bibitem[{He et~al.(2022{\natexlab{b}})He, Chen, Xie, Li, Doll{\'{a}}r, and Girshick}]{HeCXLDG22}
He, K.; Chen, X.; Xie, S.; Li, Y.; Doll{\'{a}}r, P.; and Girshick, R.~B. 2022{\natexlab{b}}.
\newblock Masked Autoencoders Are Scalable Vision Learners.
\newblock In \emph{{CVPR}}, 15979--15988.
\bibitem[{Horn et~al.(2018)Horn, Aodha, Song, Cui, Sun, Shepard, Adam, Perona, and Belongie}]{HornASCSSAPB18}
Horn, G.~V.; Aodha, O.~M.; Song, Y.; Cui, Y.; Sun, C.; Shepard, A.; Adam, H.; Perona, P.; and Belongie, S.~J. 2018.
\newblock The INaturalist Species Classification and Detection Dataset.
\newblock In \emph{{CVPR}}, 8769--8778.
\bibitem[{Horn et~al.(2015)Horn, Branson, Farrell, Haber, Barry, Ipeirotis, Perona, and Belongie}]{HornBFHBIPB15}
Horn, G.~V.; Branson, S.; Farrell, R.; Haber, S.; Barry, J.; Ipeirotis, P.; Perona, P.; and Belongie, S.~J. 2015.
\newblock Building a bird recognition app and large scale dataset with citizen scientists: The fine print in fine-grained dataset collection.
\newblock In \emph{{CVPR}}, 595--604.
\bibitem[{Hu, Shen, and Sun(2018)}]{HuSS18}
Hu, J.; Shen, L.; and Sun, G. 2018.
\newblock Squeeze-and-Excitation Networks.
\newblock In \emph{{CVPR}}, 7132--7141.
\bibitem[{Hu et~al.(2021)Hu, Jin, Zhang, Hong, Zhang, He, and Xue}]{HuJZHZH021}
Hu, Y.; Jin, X.; Zhang, Y.; Hong, H.; Zhang, J.; He, Y.; and Xue, H. 2021.
\newblock RAMS-Trans: Recurrent Attention Multi-scale Transformer for Fine-grained Image Recognition.
\newblock In \emph{{ACM} Multimedia}, 4239--4248.
\bibitem[{Huang and Li(2020)}]{HuangL20}
Huang, Z.; and Li, Y. 2020.
\newblock Interpretable and Accurate Fine-grained Recognition via Region Grouping.
\newblock In \emph{{CVPR}}, 8659--8669.
\bibitem[{Jiang et~al.(2024{\natexlab{a}})Jiang, Tang, Gao, Du, He, and Li}]{Jiang0GDHL24}
Jiang, X.; Tang, H.; Gao, J.; Du, X.; He, S.; and Li, Z. 2024{\natexlab{a}}.
\newblock Delving into Multimodal Prompting for Fine-Grained Visual Classification.
\newblock In \emph{{AAAI}}, 2570--2578.
\bibitem[{Jiang, Tang, and Li(2024)}]{JiangTL24}
Jiang, X.; Tang, H.; and Li, Z. 2024.
\newblock Global Meets Local: Dual Activation Hashing Network for Large-Scale Fine-Grained Image Retrieval.
\newblock \emph{{IEEE} Trans. Knowl. Data Eng.}, 6266--6279.
\bibitem[{Jiang et~al.(2024{\natexlab{b}})Jiang, Tang, Yan, Tang, and Li}]{Jiang000L24}
Jiang, X.; Tang, H.; Yan, R.; Tang, J.; and Li, Z. 2024{\natexlab{b}}.
\newblock {DVF:} Advancing Robust and Accurate Fine-Grained Image Retrieval with Retrieval Guidelines.
\newblock In \emph{{ACM} Multimedia}, 2379--2388.
\bibitem[{Ke et~al.(2023)Ke, Cai, Chen, Liu, and Guo}]{KeCCLG23}
Ke, X.; Cai, Y.; Chen, B.; Liu, H.; and Guo, W. 2023.
\newblock Granularity-aware distillation and structure modeling region proposal network for fine-grained image classification.
\newblock \emph{Pattern Recognit.}, 109305.
\bibitem[{Li, Wang, and Zhu(2021)}]{LiWZ21}
Li, G.; Wang, Y.; and Zhu, F. 2021.
\newblock Multi-branch Channel-wise Enhancement Network for Fine-grained Visual Recognition.
\newblock In \emph{{ACM} Multimedia}, 5273--5280.
\bibitem[{Li et~al.(2022)Li, Wu, Fan, Mangalam, Xiong, Malik, and Feichtenhofer}]{LiW0MXMF22}
Li, Y.; Wu, C.; Fan, H.; Mangalam, K.; Xiong, B.; Malik, J.; and Feichtenhofer, C. 2022.
\newblock MViTv2: Improved Multiscale Vision Transformers for Classification and Detection.
\newblock In \emph{{CVPR}}, 4794--4804.
\bibitem[{Li et~al.(2023)Li, Tang, Peng, Qi, and Tang}]{10038499}
Li, Z.; Tang, H.; Peng, Z.; Qi, G.-J.; and Tang, J. 2023.
\newblock Knowledge-Guided Semantic Transfer Network for Few-Shot Image Recognition.
\newblock \emph{{IEEE} Trans. Neural Networks Learn. Syst.}, 1--15.
\bibitem[{Lin et~al.(2015)Lin, Shen, Lu, and Jia}]{LinSLJ15}
Lin, D.; Shen, X.; Lu, C.; and Jia, J. 2015.
\newblock Deep {LAC:} Deep localization, alignment and classification for fine-grained recognition.
\newblock In \emph{{CVPR}}, 1666--1674.
\bibitem[{Lin et~al.(2014)Lin, Maire, Belongie, Hays, Perona, Ramanan, Doll{\'{a}}r, and Zitnick}]{LinMBHPRDZ14}
Lin, T.; Maire, M.; Belongie, S.~J.; Hays, J.; Perona, P.; Ramanan, D.; Doll{\'{a}}r, P.; and Zitnick, C.~L. 2014.
\newblock Microsoft {COCO:} Common Objects in Context.
\newblock In \emph{{ECCV}}, volume 8693, 740--755.
\bibitem[{Liu et~al.(2023)Liu, Zhang, Deng, Xie, Liu, and Li}]{10023961}
Liu, H.; Zhang, C.; Deng, Y.; Xie, B.; Liu, T.; and Li, Y.-F. 2023.
\newblock TransIFC: Invariant Cues-aware Feature Concentration Learning for Efficient Fine-grained Bird Image Classification.
\newblock \emph{IEEE Transactions on Multimedia}, 1--14.
\bibitem[{Liu et~al.(2021)Liu, Lin, Cao, Hu, Wei, Zhang, Lin, and Guo}]{LiuL00W0LG21}
Liu, Z.; Lin, Y.; Cao, Y.; Hu, H.; Wei, Y.; Zhang, Z.; Lin, S.; and Guo, B. 2021.
\newblock Swin Transformer: Hierarchical Vision Transformer using Shifted Windows.
\newblock In \emph{{ICCV}}, 9992--10002.
\bibitem[{Luo et~al.(2019)Luo, Yang, Mo, Lu, Davis, Li, Yang, and Lim}]{LuoYML0LYL19}
Luo, W.; Yang, X.; Mo, X.; Lu, Y.; Davis, L.; Li, J.; Yang, J.; and Lim, S. 2019.
\newblock Cross-X Learning for Fine-Grained Visual Categorization.
\newblock In \emph{ICCV}, 8241--8250.
\bibitem[{Lv et~al.(2022)Lv, Xia, Li, Li, and Lan}]{lv2022mfvt}
Lv, X.; Xia, H.; Li, N.; Li, X.; and Lan, R. 2022.
\newblock Mfvt: Multilevel feature fusion vision transformer and ramix data augmentation for fine-grained visual categorization.
\newblock \emph{Electronics}, 3552.
\bibitem[{Malhotra(2022)}]{malhotra2022habitat}
Malhotra, R. 2022.
\newblock Habitat loss pushing more bird species to near extinction.
\newblock \emph{Nature India}, 18.
\bibitem[{Moon et~al.(2023)Moon, Lee, Lee, and Park}]{abs-2308-02161}
Moon, J.; Lee, J.; Lee, Y.; and Park, S. 2023.
\newblock M2Former: Multi-Scale Patch Selection for Fine-Grained Visual Recognition.
\newblock \emph{CoRR}, abs/2308.02161.
\bibitem[{Recasens et~al.(2018)Recasens, Kellnhofer, Stent, Matusik, and Torralba}]{RecasensKSMT18}
Recasens, A.; Kellnhofer, P.; Stent, S.; Matusik, W.; and Torralba, A. 2018.
\newblock Learning to Zoom: {A} Saliency-Based Sampling Layer for Neural Networks.
\newblock In \emph{{ECCV}}, 52--67.
\bibitem[{Ryali et~al.(2023)Ryali, Hu, Bolya, Wei, Fan, Huang, Aggarwal, Chowdhury, Poursaeed, Hoffman, Malik, Li, and Feichtenhofer}]{RyaliHB000ACPHM23}
Ryali, C.; Hu, Y.; Bolya, D.; Wei, C.; Fan, H.; Huang, P.; Aggarwal, V.; Chowdhury, A.; Poursaeed, O.; Hoffman, J.; Malik, J.; Li, Y.; and Feichtenhofer, C. 2023.
\newblock Hiera: {A} Hierarchical Vision Transformer without the Bells-and-Whistles.
\newblock In \emph{{ICML}}, 29441--29454.
\bibitem[{Serrano and Smith(2019)}]{SerranoS19}
Serrano, S.; and Smith, N.~A. 2019.
\newblock Is Attention Interpretable?
\newblock In \emph{{ACL}}, 2931--2951.
\bibitem[{Shen et~al.(2023)Shen, Xie, Zhu, Zhu, and Zeng}]{shen2023git}
Shen, F.; Xie, Y.; Zhu, J.; Zhu, X.; and Zeng, H. 2023.
\newblock Git: Graph interactive transformer for vehicle re-identification.
\newblock \emph{{IEEE} Trans. Image Process.}, 1039--1051.
\bibitem[{Shen et~al.(2022)Shen, Sun, Wei, Jiang, and Yang}]{ShenSWJY22}
Shen, Y.; Sun, X.; Wei, X.; Jiang, Q.; and Yang, J. 2022.
\newblock {SEMICON:} {A} Learning-to-Hash Solution for Large-Scale Fine-Grained Image Retrieval.
\newblock In \emph{{ECCV}}, 531--548.
\bibitem[{Song and Yang(2021)}]{SongY21}
Song, J.; and Yang, R. 2021.
\newblock Feature Boosting, Suppression, and Diversification for Fine-Grained Visual Classification.
\newblock In \emph{IJCNN}, 1--8.
\bibitem[{Sun, He, and Peng(2022)}]{0002HP22}
Sun, H.; He, X.; and Peng, Y. 2022.
\newblock SIM-Trans: Structure Information Modeling Transformer for Fine-grained Visual Categorization.
\newblock In \emph{{ACM} Multimedia}, 5853--5861.
\bibitem[{Szegedy et~al.(2016)Szegedy, Vanhoucke, Ioffe, Shlens, and Wojna}]{SzegedyVISW16}
Szegedy, C.; Vanhoucke, V.; Ioffe, S.; Shlens, J.; and Wojna, Z. 2016.
\newblock Rethinking the Inception Architecture for Computer Vision.
\newblock In \emph{{CVPR}}, 2818--2826.
\bibitem[{Tang et~al.(2020)Tang, Li, Peng, and Tang}]{TangLPT20}
Tang, H.; Li, Z.; Peng, Z.; and Tang, J. 2020.
\newblock BlockMix: Meta Regularization and Self-Calibrated Inference for Metric-Based Meta-Learning.
\newblock In \emph{{ACM} Multimedia}, 610--618.
\bibitem[{Tang et~al.(2025)Tang, Li, Zhang, He, and Tang}]{tang2024divide}
Tang, H.; Li, Z.; Zhang, D.; He, S.; and Tang, J. 2025.
\newblock Divide-and-Conquer: Confluent Triple-Flow Network for RGB-T Salient Object Detection.
\newblock \emph{{IEEE} Trans. Pattern Anal. Mach. Intell.}, 47(3): 1958--1974.
\bibitem[{Tang et~al.(2023)Tang, Liu, Yan, Yan, Li, and Tang}]{0007LYYL023}
Tang, H.; Liu, J.; Yan, S.; Yan, R.; Li, Z.; and Tang, J. 2023.
\newblock M3Net: Multi-view Encoding, Matching, and Fusion for Few-shot Fine-grained Action Recognition.
\newblock In \emph{{ACM} Multimedia}, 1719--1728.
\bibitem[{Tang et~al.(2022)Tang, Yuan, Li, and Tang}]{TangYLT22}
Tang, H.; Yuan, C.; Li, Z.; and Tang, J. 2022.
\newblock Learning attention-guided pyramidal features for few-shot fine-grained recognition.
\newblock \emph{Pattern Recognit.}, 108792.
\bibitem[{Wang, Fu, and Ma(2023)}]{WangFM23}
Wang, C.; Fu, H.; and Ma, H. 2023.
\newblock Multi-Part Token Transformer with Dual Contrastive Learning for Fine-grained Image Classification.
\newblock In \emph{{ACM} Multimedia}, 7648--7656.
\bibitem[{Wang, Yu, and Gao(2021)}]{WangY021}
Wang, J.; Yu, X.; and Gao, Y. 2021.
\newblock Feature Fusion Vision Transformer for Fine-Grained Visual Categorization.
\newblock In \emph{{BMVC}}, 170.
\bibitem[{Wang et~al.(2024)Wang, Wang, Li, Chang, Ouyang, and Tian}]{WangWLCOT24}
Wang, S.; Wang, Z.; Li, H.; Chang, J.; Ouyang, W.; and Tian, Q. 2024.
\newblock Accurate Fine-Grained Object Recognition with Structure-Driven Relation Graph Networks.
\newblock \emph{Int. J. Comput. Vis.}, 137--160.
\bibitem[{Wang et~al.(2020)Wang, Cui, Li, Jiang, and Deng}]{WangCLJD20}
Wang, W.; Cui, Y.; Li, G.; Jiang, C.; and Deng, S. 2020.
\newblock A self-attention-based destruction and construction learning fine-grained image classification method for retail product recognition.
\newblock \emph{Neural Comput. Appl.}, 14613--14622.
\bibitem[{Welinder et~al.(2010)Welinder, Branson, Mita, Wah, Schroff, Belongie, and Perona}]{welinder2010caltech}
Welinder, P.; Branson, S.; Mita, T.; Wah, C.; Schroff, F.; Belongie, S.; and Perona, P. 2010.
\newblock Caltech-UCSD birds 200.
\bibitem[{Xu et~al.(2023)Xu, Wang, Jiang, and Luo}]{XuWJL23}
Xu, Q.; Wang, J.; Jiang, B.; and Luo, B. 2023.
\newblock Fine-Grained Visual Classification via Internal Ensemble Learning Transformer.
\newblock \emph{{IEEE} Trans. Multim.}, 9015--9028.
\bibitem[{Yan et~al.(2023)Yan, Xie, Shu, Zhang, and Tang}]{yan2023progressive}
Yan, R.; Xie, L.; Shu, X.; Zhang, L.; and Tang, J. 2023.
\newblock Progressive instance-aware feature learning for compositional action recognition.
\newblock \emph{{IEEE} Trans. Pattern Anal. Mach. Intell.}, 10317--10330.
\bibitem[{Yang et~al.(2018)Yang, Luo, Wang, Hu, Gao, and Wang}]{YangLWHGW18}
Yang, Z.; Luo, T.; Wang, D.; Hu, Z.; Gao, J.; and Wang, L. 2018.
\newblock Learning to Navigate for Fine-Grained Classification.
\newblock In \emph{{ECCV}}, 438--454.
\bibitem[{Zha et~al.(2023)Zha, Tang, Sun, and Tang}]{ZhaTST23}
Zha, Z.; Tang, H.; Sun, Y.; and Tang, J. 2023.
\newblock Boosting Few-Shot Fine-Grained Recognition With Background Suppression and Foreground Alignment.
\newblock \emph{{IEEE} Trans. Circuits Syst. Video Technol.}, 3947--3961.
\bibitem[{Zhang et~al.(2016)Zhang, Xu, Elhoseiny, Huang, Zhang, Elgammal, and Metaxas}]{ZhangXEHZEM16}
Zhang, H.; Xu, T.; Elhoseiny, M.; Huang, X.; Zhang, S.; Elgammal, A.~M.; and Metaxas, D.~N. 2016.
\newblock {SPDA-CNN:} Unifying Semantic Part Detection and Abstraction for Fine-Grained Recognition.
\newblock In \emph{{CVPR}}, 1143--1152.
\bibitem[{Zhang, Huang, and Liu(2022)}]{ZhangH022}
Zhang, L.; Huang, S.; and Liu, W. 2022.
\newblock Enhancing Mixture-of-Experts by Leveraging Attention for Fine-Grained Recognition.
\newblock \emph{{IEEE} Trans. Multim.}, 4409--4421.
\bibitem[{Zhang et~al.(2019)Zhang, Huang, Liu, and Tao}]{ZhangHLT19}
Zhang, L.; Huang, S.; Liu, W.; and Tao, D. 2019.
\newblock Learning a Mixture of Granularity-Specific Experts for Fine-Grained Categorization.
\newblock In \emph{{ICCV}}, 8330--8339.
\bibitem[{Zhang et~al.(2014)Zhang, Donahue, Girshick, and Darrell}]{ZhangDGD14}
Zhang, N.; Donahue, J.; Girshick, R.~B.; and Darrell, T. 2014.
\newblock Part-Based R-CNNs for Fine-Grained Category Detection.
\newblock In \emph{{ECCV}}, 834--849.
\bibitem[{Zhang et~al.(2022)Zhang, Cao, Zhang, Liu, Wang, Ling, and Chen}]{ZhangCZLWLC22}
Zhang, Y.; Cao, J.; Zhang, L.; Liu, X.; Wang, Z.; Ling, F.; and Chen, W. 2022.
\newblock A free lunch from ViT: adaptive attention multi-scale fusion Transformer for fine-grained visual recognition.
\newblock In \emph{{ICASSP}}, 3234--3238.
\bibitem[{Zhang et~al.(2024)Zhang, Chen, Wang, Luo, and Xu}]{ZhangCWLX24}
Zhang, Z.; Chen, Z.; Wang, Y.; Luo, X.; and Xu, X. 2024.
\newblock A vision transformer for fine-grained classification by reducing noise and enhancing discriminative information.
\newblock \emph{Pattern Recognit.}, 145: 109979.
\bibitem[{Zhao et~al.(2021)Zhao, Yan, Huang, and Li}]{ZhaoYHL21}
Zhao, Y.; Yan, K.; Huang, F.; and Li, J. 2021.
\newblock Graph-Based High-Order Relation Discovery for Fine-Grained Recognition.
\newblock In \emph{{CVPR}}, 15079--15088.
\bibitem[{Zheng et~al.(2019)Zheng, Fu, Zha, and Luo}]{ZhengFZL19}
Zheng, H.; Fu, J.; Zha, Z.; and Luo, J. 2019.
\newblock Looking for the Devil in the Details: Learning Trilinear Attention Sampling Network for Fine-Grained Image Recognition.
\newblock In \emph{{CVPR}}, 5012--5021.
\bibitem[{Zhu et~al.(2022)Zhu, Ke, Li, Liu, Tian, and Shan}]{ZhuKLLTS22}
Zhu, H.; Ke, W.; Li, D.; Liu, J.; Tian, L.; and Shan, Y. 2022.
\newblock Dual Cross-Attention Learning for Fine-Grained Visual Categorization and Object Re-Identification.
\newblock In \emph{{CVPR}}, 4682--4692.
\bibitem[{Zhuang, Wang, and Qiao(2020)}]{ZhuangW020}
Zhuang, P.; Wang, Y.; and Qiao, Y. 2020.
\newblock Learning Attentive Pairwise Interaction for Fine-Grained Classification.
\newblock In \emph{{AAAI}}, 13130--13137.
\end{thebibliography}