TIMA: Text-Image Mutual Awareness for Balancing Zero-Shot Adversarial Robustness and Generalization Ability
Extracted source text · arXiv 2405.17678
%% The first command in your LaTeX source must be the \documentclass command.review,
% \documentclass[sigconf, review,anonymous]{acmart}
\documentclass[sigconf]{acmart}
% ,review,authordraft
% \renewcommand\footnotetextcopyrightpermission[1]{}
\settopmatter{printacmref=false} %remove ACM reference format
\usepackage{graphicx}
\usepackage{amsmath}
\let\Bbbk\relax
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{booktabs}
\usepackage{diagbox}
\usepackage{multirow}
\usepackage{graphicx}
\usepackage[subfigure]{graphfig}
\usepackage{subfigure}
%% NOTE that a single column version may required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%% \BibTeX command to typeset BibTeX logo in the docs
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\usepackage{graphicx}
\usepackage{amsmath}
%\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{diagbox}
\usepackage{makecell}
\usepackage{comment}
\usepackage{bbding}
\usepackage{xcolor}
\usepackage{colortbl}
\usepackage{pifont}
\AtBeginDocument{%
\providecommand\BibTeX{{%
Bib\TeX}}}
\setcopyright{acmlicensed}
\copyrightyear{2018}
\acmYear{2018}
\acmDOI{XXXXXXX.XXXXXXX}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[Conference acronym 'XX]{Make sure to enter the correct
conference title from your rights confirmation emai}{June 03--05,
2018}{Woodstock, NY}
%%
\acmISBN{978-1-4503-XXXX-X/18/06}
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title[TIMA for Balancing Zero-Shot Adversarial Robustness and Generalization]{TIMA: Text-Image Mutual Awareness for Balancing Zero-Shot Adversarial Robustness and Generalization Ability}
% \author{Anonymous Authors}
% \author[]{}
\settopmatter{printacmref=false}
\renewcommand\footnotetextcopyrightpermission[1]{}
\author{\large Fengji Ma$^{1}$, Li Liu$^{1\dagger}$, Hei Victor Cheng$^{2}$}
\affiliation{%
\institution{\large $^{1}$The Hong Kong University of Science and Technology (Guangzhou) \ \
\ \ $^{2}$Aarhus University}
\streetaddress{}
\city{}
\country{}
}
\thanks{{${\dagger}$ Corresponding author \tt(avrillliu@hkust-gz.edu.cn).}}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\input{sections/abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
%\begin{CCSXML}
%\end{CCSXML}
%\ccsdesc[500]{Do Not Use This Code~Generate the Correct Terms for Your Paper}
%\ccsdesc[300]{Do Not Use This Code~Generate the Correct Terms for Your Paper}
%\ccsdesc{Do Not Use This Code~Generate the Correct Terms for Your Paper}
%\ccsdesc[100]{Do Not Use This Code~Generate the Correct Terms for Your Paper}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10010147.10010178.10010224.10010225</concept_id>
<concept_desc>Computing methodologies~Computer vision tasks</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[100]{Computing methodologies~Computer vision tasks}
\keywords{Adversarial Robustness, Multimodal Learning, Zero-Shot Learning, Vision-Language Models}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\begin{teaserfigure}
\centering
\subfigure[Clean Accuracy]
{
\begin{minipage}[t]{0.24\linewidth}
\centering
\includegraphics[width=\linewidth]{images/eps0-radar-bold-MM.png}
%\caption{fig1}
\label{fig-radar-eps0}
\end{minipage}%
}
%\vspace{-0.1m}
\subfigure[Small perturbation radius 1/255]
{
\begin{minipage}[t]{0.24\linewidth}
\centering
\includegraphics[width=\linewidth]{images/eps1-radar-bold-MM.png}
% \caption{fig2}
\label{fig-radar-eps1}
\end{minipage}%
}
\subfigure[Large perturbation radius 4/255]
{
\begin{minipage}[t]{0.24\linewidth}
\centering
\includegraphics[width=\linewidth]{images/eps4-radar-bold-MM.png}
%\caption{fig1}
\label{fig-radar-eps4}
\end{minipage}%
}
\subfigure[Large perturbation radius 8/255]
{
\begin{minipage}[t]{0.24\linewidth}
\centering
\includegraphics[width=\linewidth]{images/eps8-radar-bold-MM.png}
% \caption{fig2}
\label{fig-radar-eps8}
\end{minipage}%
}
\centering
\caption{Zero-shot accuracies of compared methods and our proposed method. We show zero-shot robust accuracy of different methods under different perturbation radius. Numbers in the bracket denote the best accuracy among different methods. The accuracy of all methods is normalized with the highest method in the corresponding dataset. Our method (TIMA) achieves better results than existing methods (TeCoA~\cite{TeCoA}, PMG~\cite{PMG}, and LAAT~\cite{LAAT}), under both small and large adversarial perturbations.}
%\vspace{-2.4em}
\label{fig-radar}
\end{teaserfigure}
\maketitle
\input{sections/introduction}
% \vspace{0.3em}
% \vspace{-0.5mm}
\input{sections/relatedworks}
\input{sections/methods}
\input{sections/experiments}
% \vspace{0.2em}
% \vspace{-0.5mm}
\input{sections/conclusion}
% \vspace{0.2em}
\clearpage
\bibliographystyle{ACM-Reference-Format}
\bibliography{egbib}
%\input{sections/X_suppl}
\end{document}
\endinput
%%
%% End of file `sample-authordraft.tex'.
\begin{abstract}
This work addresses the challenge of achieving zero-shot adversarial robustness while preserving zero-shot generalization in large-scale foundation models, with a focus on the popular Contrastive Language-Image Pre-training (CLIP). Although foundation models were reported to have exceptional zero-shot generalization, they are highly vulnerable to adversarial perturbations. Existing methods achieve a comparable good tradeoff between zero-shot adversarial robustness and generalization under small adversarial perturbations. However, they fail to achieve a good tradeoff under large adversarial perturbations. To this end, we propose a novel \textbf{Text-Image Mutual Awareness (TIMA)} method that strikes a balance between zero-shot adversarial robustness and generalization.
More precisely, we propose an Image-Aware Text (IAT) tuning mechanism that increases the inter-class distance of text embeddings by incorporating the Minimum Hyperspherical Energy (MHE). Simultaneously, fixed pre-trained image embeddings are used as cross-modal auxiliary supervision to maintain the similarity between the MHE-tuned and original text embeddings by the knowledge distillation, preserving semantic information between different classes. Besides, we introduce a Text-Aware Image (TAI) tuning mechanism, which increases inter-class distance between image embeddings during the training stage by Text-distance based Adaptive Margin (TAM). Similarly, a knowledge distillation is utilized to retain the similarity between fine-tuned and pre-trained image embeddings.
%preserving the semantic relationship between pre-trained image-text embeddings.
Extensive experimental results demonstrate the effectiveness of our approach, showing impressive zero-shot performance against a wide range of adversarial perturbations while preserving the zero-shot generalization capabilities of the original CLIP model.
% Our proposed TIMA improves CLIP’s zero-shot adversarial robustness, particularly under large perturbations, without sacrificing its zero-shot generalization.
% In particular, an Image-Aware Text (IAT) tuning mechanism is proposed to increase the inter-class distance of text embeddings while maintaining the generalization ability of text embedding from original pre-trained CLIP. Besides, a Text-Aware Image (TAI) tuning mechanism is proposed to increase the logit space margin via text-distance adaptive weight for improving adversarial robustness while distilling the knowledge from cross-modality frozen text embeddings.
% \textbf{We propose an Image-Aware Text (IAT) tuning mechanism that increases the inter-class distance of text embeddings by incorporating the Minimum Hypersphere Energy (MHE). Simultanously, fixed pre-trained image embeddings are used as cross-modal auxiliary supervision to maintain the similarity between the MHE-tuned and original text embeddings by the knowledge distillation, preserving semantic information between different classes. Moreover, we introduce a Text-Aware Image (TAI) tuning mechanism, which increases the logit space margin through text distance adaptive weight rather than directly enlarging the inter-class distance of image embeddings. Similar to IAT, TAI uses fixed pre-trained text embeddings to help retain the similarity between fine-tuned and originally pre-trained image embeddings, preserving the semantic relationship between pre-trained image-text embeddings, thus enhancing adversarial robustness while maintaining zero-shot generalization capabilities. Our proposed Text-Image Mutual Awareness(TIMA), integration IAT and TAI, successfully increases the adversarial robustness of CLIP against significant perturbations without sacrificing its zero-shot generalization capabilities.}
\end{abstract}
\section{Introduction}
\label{sec-intro}
%\vspace{-0.6em}
Large-scale foundation models~\cite{CLIP,foundations1,foundations2,foundations3} have recently garnered significant attention due to their excellent zero-shot generalization ability. Contrastive Language-Image Pre-training (CLIP)~\cite{CLIP}, the most widely used vision-language model, demonstrates the ability to accurately classify new classes with only simple text captions, even those that have not been encountered before. However, recent studies~\cite{TeCoA,LVL-AR,Robust-CLIP,CLIP-OOD-detecors,reading,ChatGPT-Robustness,Attack-foundation-vision} revealed that while these foundation models exhibit strong generalization performance, they are extremely vulnerable to adversarial perturbation. Notably, existing adversarial methods ~\cite{AT,CW,AA} have effectively attacked these foundation models, resulting in performance degradation of up to 90\% on different datasets, even with minor attacks.
Traditionally, time-intensive and resource-heavy adversarial training is needed to achieve adversarial robustness. To bypass the retraining across new tasks or datasets, and to harvest the full potential of foundation models, the concept of zero-shot adversarial robustness in foundation models~\cite{TeCoA,LAAT,PMG} has emerged as a new and urgent research topic. The objective of zero-shot adversarial robustness is twofold: to \textbf{transfer adversarial robustness} in a zero-shot learning manner, and to \textbf{preserve zero-shot generalization} capabilities of foundational models. Here, adversarial robustness refers to a model's ability to withstand adversarial attacks with various perturbations. Zero-shot generalization, on the other hand, denotes the model's capacity to achieve high accuracy on previously unseen classes without prior exposure.
In pursuit of a zero-shot adversarial robust CLIP model, the pioneer work TeCoA~\cite{TeCoA} proposes to align adversarial image embeddings with their corresponding text embeddings. This eliminates the need for retraining in subsequent visual tasks. Building on this idea, PMG~\cite{PMG} introduces additional constraints from pre-trained models and clean samples into the objective function, encouraging the preservation of generalized pre-trained information.
Despite these advancements, TeCoA~\cite{TeCoA} and PMG~\cite{PMG} fall short under larger adversarial attacks, as their effectiveness wanes significantly with increased perturbation radius, which can be seen from Fig~\ref{fig-radar}. The reason is that both methods fail to enlarge the embedding distances between classes. To address this, LAAT~\cite{LAAT} proposes an algorithm to expand the distance between fixed textual embeddings in hyperspherical space, showing promise against a larger perturbation radius. Unfortunately, disrupting the semantic information within the text embedding space and the contrastive relationship between text and image embeddings sacrifices zero-shot generalization on clean samples.
To address the challenge of balancing adversarial robustness and generalization in the zero-shot setting, this work proposes the Text-Image Mutual Awareness (TIMA) mechanism. TIMA achieves state-of-the-art results on clean datasets and datasets under both small and large perturbations, as demonstrated in Fig. ~\ref{fig-radar}. TIMA overcomes the limitation identified in PMG~\cite{PMG}, TeCoA~\cite{TeCoA}, and \cite{LAAT} by ensuring robustness against larger perturbations while preserving zero-shot generalization, achieving a satisfactory balance between zero-shot adversarial robustness and generalization. Central to TIMA's innovation is the hypothesis:
\textit{Increasing the inter-class distances within the pretrained CLIP text and image embeddings is the key to improving zero-shot adversarial robustness, especially under large perturbation}.
Building on this, we propose two corresponding modules for enhancing the inter-class distance for the text and image embeddings, respectively. First, for the text embeddings, unlike LAAT~\cite{LAAT}, TIMA employs a learning-based approach named Image-Aware Text (IAT) tuning mechanism during adversarial training. This approach applies the Minimum Hyperspherical Energy (MHE) principle~\cite{MHE,MHE2,MHE3,MHE4,MHE5} to increase the distances between classes in text embeddings, meanwhile ensuring the preservation of the semantic information and coherent relationship between text and image embeddings inherent in the original pre-trained CLIP model using an image-to-text distillation module.
Additionally, we illustrate the importance of adjusting the inter-class distance of image embeddings, which has been ignored in prior work. TIMA proposes a Text-Aware Image (TAI) tuning mechanism, which adds text distance based adaptive margins to different classes based on the classification results and hence enlarges the inter-class distance of the image embeddings during contrastive adversarial fine-tuning.
Similarly, an image-to-text distillation module is used to transfer the semantic information and interaction dynamics from the original CLIP model.
Combining IAT with TAI mechanisms, TIMA achieves effective multimodal training by enlarging the inter-class distance in image and text embeddings and utilizing the contrastive and interactive information between text and image modalities to strike an optimal balance between zero-shot robustness and generalization.
Our main contributions can be summarized as follows:
% \vspace{0.1em}
\begin{itemize}
\item The proposed MHE method finds the largest inter-class distances for text embeddings, making CLIP more resilient to both small and large perturbations.
\item This is the first work to propose a working mechanism for enlarging the inter-class distance in image embeddings. The proposed TAI can be applied in a plug-and-play manner to combine with any text embedding adjustment schemes.
\item We design novel knowledge distillation methods for text and image embeddings to keep semantic and interactive information between the modalities after enlarging inter-class distances.
\item Extensive experiments show the proposed TIMA's superiority in zero-shot robust accuracy and clean accuracy across multiple datasets and under different perturbations, establishing its excellent efficacy.
\end{itemize}
\begin{figure*}[h]
%\vspace{-1.0em}
\includegraphics[width=0.85\linewidth]{images/Framework-MM-0411.pdf}
\centering
%\vspace{-0.9em}
\caption{\textcolor{black}{The framework of our proposed method Text-Image Mutual Awareness (TIMA).} $\mathcal{L}_{\mathrm{MHE}}$ is Minimum Hyperspherical Energy loss, $\mathcal{L}_{\mathrm{IAKD}}$ is Image-Aware Knowledge Distillation loss, $\mathcal{L}_{\mathrm{TAW}}$ is Text-distance Adaptive Margin loss and $\mathcal{L}_{\mathrm{TAKD}}$ is Text-Aware Knowledge Distillation loss.}
%\vspace{-1.5em}
\label{fig-framework}
\end{figure*}
%\vspace{-4mm}
% \vspace{-2.7em}
\section{Related Works}
\label{sec-relatedworks}
This paper explores the trade-off between zero-shot adversarial robustness and zero-shot generalization in vision-language models (VLMs), particularly CLIP. Numerous studies~\cite{VLM-distillation,VLM-distillation2,VLM-distillation3,VLM-distillation4,VLM-distillation5,VLM-distillation6} have shown that knowledge distillation preserves the zero-shot generalization of VLMs. In this paper, our focus is primarily on improving zero-shot adversarial robustness.
Existing zero-shot adversarial robustness approaches~\cite{TeCoA,PMG,LAAT} fall into two categories based on the adjustment of inter-class distances of pretrained CLIP text embeddings. The first category does not adjust these distances and includes methods like TeCoA~\cite{TeCoA}, which uses adversarial fine-tuning to align adversarial image embeddings with fixed CLIP text embeddings, and PMG~\cite{PMG}, which improves the similarity between clean and adversarial image embeddings through knowledge distillation. These methods, however, fail under large-scale perturbation adversarial attacks. The second category, exemplified by LAAT~\cite{LAAT}, increasing text embedding distances but often at the cost of losing the original semantic relationships, leading to zero-shot generalization significant decrease.
Our proposed method diverges from these by hypothesizing that increasing inter-class distance of both text and image embedding can improve zero-shot adversarial robustness, especially under large perturbations. By increasing both types of inter-class distances and leveraging cross-modal auxiliary supervision information, we aim to preserve the semantic information of pretrained CLIP, thereby sustaining zero-shot generalization.
\section{Methods}
\label{sec-methods}
\subsection{Preliminaries and Problem Setup}
\label{sec-background}
We choose CLIP, a popular large-scale pre-trained vision-language model for zero-shot recognition, as our base model. Let $F_{\theta}(\cdot)$ be the CLIP image encoder parameterized by
$\theta$ and $F_{\phi}(\cdot)$ be the CLIP text encoder parameterized by $\phi$. Given an input image $x$ and a textual description $\mathcal{T}$ about its class, such as ”This is a photo of a {}”, denoted as $t$, the model will output an image embedding $F_{\theta}(x)$ and a text embedding $F_{\phi}(\mathcal{T})$. We denote the image embedding as $z = F_{\theta}(x)$ and text embedding as $ t = F_{\phi}(\mathcal{T})$. The similarity between image embedding and each candidate class of text embedding is computed. For the $j$-th class, the text embedding can be described as $z_{j}=F_{\phi}(\mathcal{T}_{j})$. For the CLIP model, the generation of adversarial examples can be described as:
\vspace{-1mm}
\begin{align}
x^{(a)} = \underset{x^{(a)}}{\mathrm{argmax}} \; \mathcal{L}_{\mathrm{CE}}(x^{(a)},\mathcal{T},y), \qquad s.t. \:||x^{(a)}-x||_{p}\le \varepsilon,
\label{eq-AE}
% \vspace{-0.5mm}
\end{align}
where $\mathcal{L}_{\mathrm{CE}}$ represents cross-entropy loss. The additive noise is bounded within a $p$-norm ball of radius $\varepsilon$. For an adversarial example $x^{(a)}$, the adversarial image embedding is given by $z^{(a)}=F_{\theta}(x^{(a)})$. The generation of adversarial examples is by PGD~\cite{AT}, which is shown in Fig.~\ref{fig-framework}.
We follow the definitions of zero-shot adversarial robustness and generalization as outlined in existing studies \cite{TeCoA,PMG,LAAT}. Our experiments use zero-shot robust accuracy as the metric for assessing zero-shot adversarial robustness. The framework of the proposed method, TIMA, is shown in Fig.~\ref{fig-framework}.
%\vspace{-0.5em}
\subsection{Image-Aware Text Tuning Mechanism}
\label{sec-MHE}
\begin{figure}[t!]
\centering
\subfigure[CLIP~\cite{CLIP}]
{
\begin{minipage}[t]{0.3\linewidth}
\centering
\includegraphics[width=\linewidth]{images/CLIP_t2t_min0.87max1.0_cifar10.png}
\includegraphics[width=\linewidth]{images/CLIP_clean2t_min0.35max0.40_cifar10.png}
%\caption{fig1}
\label{fig-CLIP-T2T-I2T}
\end{minipage}%
}
\subfigure[LAAT~\cite{LAAT}]
{
\begin{minipage}[t]{0.3\linewidth}
\centering
\includegraphics[width=\linewidth]{images/LAAT_t2t_min0.87max1.0_cifar10.png}
\includegraphics[width=\linewidth]{images/LAAT_clean2t_0.35-0.4_cifar10.png}
% \caption{fig2}
\label{fig-LAAT-T2T-I2T}
\end{minipage}%
}
\subfigure[Ours]
{
\begin{minipage}[t]{0.3\linewidth}
\centering
\includegraphics[width=\linewidth]{images/ours_t2t_min0.87max1.0_cifar10.png}
\includegraphics[width=\linewidth]{images/ours_clean2t_0.35-0.4_cifar10.png}
% \caption{fig2}
\label{fig-ours-T2T-I2T}
\end{minipage}%
}
\centering
\vspace{-0.5em}
\caption{Text-text embedding pairs and (clean) image-text embedding pairs cosine similarity matrix on CIFAR10. The first row shows the text-text embedding pairs similarity of different methods. The second row shows the similarity among clean image-text embedding pairs for different methods.}
\label{fig-CLIP-LAAT-ours-text-text}
\end{figure}
Prior research~\cite{LAAT} demonstrated that increasing the inter-class distance of text embeddings can significantly enhance a model's zero-shot adversarial robustness. Inspired by it, we aim to strengthen zero-shot adversarial robustness under the large perturbation radius by increasing the inter-class distance of text embeddings. The idea of MHE (Minimum Hyperspherical Energy)~\cite{MHE,MHE2,MHE3,MHE4,MHE5} is applied to fine-tune text embeddings, enforcing embeddings from different classes to be distributed uniformly over the hypersphere representing the text embedding space, thus maximizing their inter-class distances.
%Consequently, we introduce the MHE in the proposed IAT tuning mechanism to increase the inter-class distances of text embeddings.
On the other hand, as illustrated in Fig.~\ref{fig-CLIP-LAAT-ours-text-text}, the text embedding expansion algorithm in LAAT disrupts the semantic relationships between similar classes. To address this, we utilize the (fixed) pre-trained image embeddings as auxiliary supervisory information for cross-modal supervision distillation.
\subsubsection{MHE-Guided Text Distance Enlargement}
% \textbf{MHE-Guided Text Distance Modulation}.
In pursuit of amplifying inter-class distances, thereby improving the separability of text embeddings, we introduce MHE,
%This method aims to enhance uniformity within hyperspherical spaces, offering a systematic approach to increase the distance between classes in the embedding space,
which can be described as:
\begin{align}
\mathcal{L}_{\mathrm{MHE}}=\mathbb{E}_{j\neq k}\left[\frac{1}{d^{\alpha}(t_{j},t_{k})}\right],
\end{align}
where $t_{j},t_{k}$ represents the text embedding of the text encoder, \textit{i.e.}, $t_{j}=F_{\phi}(\mathcal{T}_{j})$, and $d(t_{j},t_{k})$ is the Euclidean distance between text embeddings of different classes with $\alpha =2$. Different from the original form of MHE loss, to prevent gradient explosion, we adopted the MHE loss as $\mathcal{L}_{\mathrm{MHE}}=\mathbb{E}_{j\neq k}\left[\frac{1}{1+d^{\alpha}(t_{j},t_{k})}\right]$.
\subsubsection{Image-Aware Knowledge Distillation for Text Semantic Consistency}
Previous research~\cite{LAAT} solely focuses on increasing the inter-class distances of text embeddings to improve zero-shot adversarial robustness, which destroys semantic information between different classes of text embeddings and sacrifices the generalization performance on clean datasets. We propose an Image-Aware Knowledge Distillation (IAKD) to circumvent the issue of semantic information destruction inherent in enlarging the inter-class distance in text embeddings. The idea is to employ the image embeddings from fixed pretrained CLIP as supervision information, we distill text embeddings to maintain their semantic consistency, thereby preserving zero-shot generalization. It's important to note that our approach does not directly distill between the pretrained CLIP text embeddings and the fine-tuned text embeddings.
Denote the teacher contrastive distribution as $p_{ij}$ and the student contrastive distribution as $q_{ij}$, they are given by:
\begin{equation}
\begin{aligned}
p_{ij} = \frac{\mathrm{exp}({s(\hat{z}_{i},\hat{t}_{j})}/\tau)}{{\sum_{k}}\mathrm{exp}(s(\hat{z}_{i},\hat{t}_{k})/\tau)}, \qquad
q_{ij} = \frac{\mathrm{exp}({s(\hat{z}_{i},t_{j})}/\tau)}{{\sum_{k}}\mathrm{exp}(s(\hat{z}_{i},t_{k})/\tau)}.
\end{aligned}
\label{eq-teacher-student-1}
\end{equation}
where $\tau$ is the temperature of CLIP, $\hat{t}_{j}$ represents the $j$-th text embedding of the fixed pretrained CLIP text encoder, $t$ is the text embedding of our fine-tuning CLIP text encoder and $\hat{z}$ is the image embedding of fixed pre-trained CLIP image encoder.
Specifically, we first calculate the softmax distribution of the cosine similarity between the fixed pretrained CLIP image embeddings and the fixed pretrained CLIP text embeddings to serve as the teacher contrastive distribution. Subsequently, we compute the softmax distribution of the cosine similarity between the fixed pretrained CLIP image embeddings and the fine-tuned text embeddings as the student contrastive distribution. Then, we compute KL divergence between student contrastive distribution and teacher contrastive distribution. The loss function $\mathcal{L}_{\mathrm{IAKD}}$ of image-awareness supervision text embedding distillation is described as:
\begin{align}
\mathcal{L}_{\mathrm{IAKD}}= \mathrm{D}_{\mathrm{KL}}(p_{ij}||q_{ij}).
\end{align}
Therefore, the loss function of our Image-Aware Text (IAT) tuning mechanism can be described as $\mathcal{L}_{\mathrm{IAT}}=\mathcal{L}_{\mathrm{MHE}}+\lambda_{\mathrm{T}}\mathcal{L}_{\mathrm{IAKD}}$.
where $\lambda_{\mathrm{T}}$ is a hyper-parameter that balances the text embedding separation and zero-shot generalization.
As illustrated in Fig.~\ref{fig-CLIP-LAAT-ours-text-text}, the text embedding expansion algorithm proposed by LAAT~\cite{LAAT} significantly diminishes the semantic relationship between image and text embeddings on clean samples. Furthermore, the inherent semantic relatedness between similar classes
% (\textit{e.g.}, cat and dog, automobile and trucks, airplane and ship, deer and dog)
is lost due to the expansion algorithm's neglect of inter-class relationships. In this perspective, we improve zero-shot adversarial robustness via increasing inter-class distance of text embeddings without sacrificing the zero-shot generalization performance. \textbf{For a detailed analysis of why increasing the inter-class distances of text embeddings by MHE principle improves CLIP's zero-shot adversarial robustness, please refer to the supplementary materials.}
%%%adv-text
\subsection{Text-Aware Image Tuning Mechanism}
\label{sec-AW}
%%% adv-adv
\begin{figure}[t!]
\centering
\subfigure[TeCoA~\cite{TeCoA}]
{
\begin{minipage}[t]{0.32\linewidth}
\centering
\includegraphics[width=\linewidth]{images/TeCoA_adv2adv_min0.95max1.0_cifar10_l.png}
%\caption{fig1}
\label{fig-TeCoA-adv-adv-large}
\end{minipage}%
}
\subfigure[LAAT~\cite{LAAT}]
{
\begin{minipage}[t]{0.32\linewidth}
\centering
\includegraphics[width=\linewidth]{images/LAAT_adv2adv_min0.95max1.0_cifar10_l.png}
% \caption{fig2}
\label{fig-LAAT-adv-adv-large}
\end{minipage}%
}%
\subfigure[Ours]
{
\begin{minipage}[t]{0.32\linewidth}
\centering
\includegraphics[width=\linewidth]{images/ours_adv2adv_min0.95max1.0_cifar10_l.png}
% \caption{fig2}
\label{fig-ours-adv-adv-large}
\end{minipage}%
}%
\centering
\vspace{-0.5em}
\caption{Adversarial image \textit{vs.} adversarial image embedding pairs cosine similarity matrix on CIFAR10 under large perturbation radius ($\varepsilon=8/255$) adversarial attack.}
\vspace{-1.8em}
\label{fig-large-perturbation-adv-adv}
\end{figure}
Previous research in unimodal image classification~\cite{inter-class1,inter-class2,inter-class3,inter-class4,inter-class5,margin3} underscores the importance of increasing inter-class distances in image features to improve model robustness. However, existing methods for adjusting CLIP such as TeCoA, PMG, and LAAT lack adjustments in the inter-class distances of image embeddings. This oversight results in exaggerated cosine distances between adversarial image embeddings of different classes under large perturbation attacks, as shown in Fig.~\ref{fig-large-perturbation-adv-adv}. The results of TeCoA and LAAT from Fig.~\ref{fig-large-perturbation-adv-adv} show high cosine similarity between different classes of adversarial image embeddings, \textit{i.e.} small inter-class distance of adversarial image embeddings, which leads to compromising model robustness under large perturbation (shown in Fig.~\ref{fig-radar-eps8}).
%To address this, we hypothesize that increasing the inter-class distances for both image and text embeddings improves the zero-shot adversarial robustness of the CLIP model, especially against attacks involving large perturbations.
Similar to the IAT tuning mechanism, which expands the inter-class distance of text embeddings while preserving semantic relationships to boost zero-shot generalization and robustness, we introduce Text-Aware Image (TAI) tuning mechanism, a novel strategy aiming to enhance zero-shot adversarial robustness by increasing the inter-class distances of fine-tuned adversarial image embeddings without sacrificing semantic information. Our method involves the introduction of an adaptive margin based on text distances to increase the inter-class distance of image embeddings, complemented by the utilization of fixed pre-trained text embeddings for cross-modal supervision in the distillation of image embeddings.
\subsubsection{Text-distance based Adaptive Margin}
% \textbf{Text-distance Adaptive Margin.}
Numerous studies~\cite{inter-class4,inter-class5,margin1, margin2, margin3,margin-softmax} have demonstrated that incorporating margin penalties can increase the distance between classes, thereby enhancing adversarial robustness. However, these studies have applied a fixed margin that treats all negative samples equally, overlooking the intrinsic semantic relationships between classes. This oversight leads to misclassifications among semantically similar classes under large perturbation adversarial attacks.
To make use of semantic information and the likelihood of misclassification and potential misclassification among different classes, our approach apply margins to incorrectly classified classes and potentially misclassified, where the added margin increases as the semantic proximity between the incorrect and correct classes increases.
Considering rich semantic information between different classes captured by pre-trained CLIP text embeddings, we propose the text-distance based adaptive margin. When the cosine similarity between the image embedding and the corresponding text embedding falls below a threshold, indicating a significant semantic difference, then no additional margin is added. Conversely, when the similarity exceeds the threshold, indicating semantic proximity, an additional margin is added to these classes. The margin size depends on the level of semantic similarity to the correct class. Therefore, we design a margin is as follows:
\begin{align}
\mathcal{M}_{ik} =\left\{\begin{matrix}
m\circ s(\hat{t}_{i},\hat{t}_{k}), &s(\hat{z}_{i},\hat{t}_{k})\ge \eta\cdot s(\hat{z}_{i},\hat{t}_{j}) \\
0, &s(\hat{z}_{i},\hat{t}_{k})< \eta\cdot s(\hat{z}_{i},\hat{t}_{j})
\end{matrix},\right.
\end{align}
where $\circ$ is element-wise multiplication and $\hat{z}_{i}$ are the frozen image embeddings of the original CLIP image encoder. Margin value $m$ and threshold $\eta$ are hyper-parameters of adaptive margin. Specifically, when the cosine similarity between image embedding and the incorrect class text embedding is higher than that between image embedding and the corresponding correct class text embedding, $\textit{i.e.},$ $s(\hat{z}_{i},\hat{t}_{k})>s(\hat{z}_{i},\hat{t}_{j})$, which means that is a misclassified class of text embedding.
The value $\eta$ should be set as $\eta < 1$. When $\eta \cdot s(\hat{z}_{i},\hat{t}_{j}) \le s(\hat{z}_{i},\hat{t}_{k})<s(\hat{z}_{i},\hat{t}_{j})$, the text embedding class represents the potential class that may be classified incorrectly. Such a class is semantically close to the correct class, and the text distance adaptive margin needs to be added. From the experimental results in Sec.~\ref{sec-experiment}, the hyper-parameters, margin value $m$ and threshold $\eta$ are set as 0.1 and 0.95. respectively. Putting together, our proposed \textbf{Text-distance Adaptive Margin (TAM)} can be combined with adversarial fine-tuning:
\begin{align}
\mathcal{L}_{\mathrm{TAM}} = -\mathbb{E}_{i,j}\left[y_{ij}\mathrm{log}\frac{\mathrm{exp}({[s(z^{(a)}_{i},\hat{t}_{j})-\mathcal{M}_{ij}]}/\tau)}{{\sum_{k}}\mathrm{exp}([s(z^{(a)}_{i},\hat{t}_{k})-\mathcal{M}_{ik}]/\tau)}\right],
\label{eq-AW}
\end{align}
where $\mathcal{M}$ is the text-distance adaptive margin. $\tau$ is the temperature of CLIP. $\hat{t}$ is the fixed text embedding from the pre-trained CLIP. $z^{(a)}$ is the adversarial image embedding, \textit{i.e.}, $z^{(a)}=F_{\theta}(x^{(a)})$, where $x^{(a)}$ is the adversarial image generated by Eq.~\ref{eq-AE}. The proposed text distance adaptive margin for cross-modal image-text alignment leverages pre-trained text embeddings to increase the inter-class distance of image embeddings (which can be seen in Fig.~\ref{fig-ours-adv-adv-large}), thereby enhancing adversarial robustness, particularly under large perturbations.
% \begin{align}
% \mathcal{M}_{ik} =\left\{\begin{matrix}
% m\circ s(\hat{t}_{i},\hat{t}_{j}), &s(\hat{z}_{i},\hat{t}_{k})\ge \eta\cdot s(\hat{z}_{i},\hat{t}_{j}) \\
% 0, &s(\hat{z}_{i},\hat{t}_{k})< \eta\cdot s(\hat{z}_{i},\hat{t}_{j})
% \end{matrix}\right.
% \end{align}
% where $\circ$ represents element-wise multiplication and $\hat{z}_{i}$ are the frozen image embeddings of original CLIP image encoder. Margin value $m$ and threshold $\eta$ are hyper-parameters of adaptive margin.
% Under this setting, $\eta$ is less than 1.
% In the interval of $[\eta,1)$, the score of $s$ function represents potential hard negatives. The gate value is set as 1 which introduces a text-text guided margin.
% $\mathcal{G}_{ik}$ is the gate from the image-text pair of the frozen original CLIP model. The gate is described as follows:
% \begin{align}
% % \mathcal{G}_{ik} = \frac{g}{2}[\mathrm{sgn}(s(z_{i},\hat{t}_{k})-\eta\cdot s(z_{i},\hat{t}_{j}))+1]
% \mathcal{G}_{ik}=\left\{\begin{matrix}
% g, &s(\hat{z}_{i},\hat{t}_{k})\ge \eta\cdot s(\hat{z}_{i},\hat{t}_{j}) \\
% 0, &s(\hat{z}_{i},\hat{t}_{k})< \eta\cdot s(\hat{z}_{i},\hat{t}_{j})
% \end{matrix}\right.,
% \end{align}
% \begin{align}
% &\mathcal{L}_{\mathrm{AW}} =\nonumber\\
% &-\mathbb{E}_{i,j}\left[y_{ij}\mathrm{log}\frac{\mathrm{exp}({s(z_{i},\hat{t}_{j})}/\tau+\mathcal{G}_{ij}\circ (1-s(\hat{t}_{i},\hat{t}_{j}))/\tau)}{{\sum_{k}}\mathrm{exp}(s(z_{i},\hat{t}_{k})/\tau+\mathcal{G}_{ik}\circ (1-s(\hat{t}_{i},\hat{t}_{k}))/\tau)}\right].
% \label{eq-AW2}
% \end{align}
% \begin{align}\tiny
% &\mathcal{L}_{\mathrm{AW}} =\nonumber\\
% &-\mathbb{E}_{i,j}\left[y_{ij}\mathrm{log}\frac{\mathrm{exp}({s(z_{i},\hat{t}_{j})}/\tau+\mathcal{G}_{ij}\circ (1-s(\hat{t}_{i},\hat{t}_{j}))/\tau)}{{\sum_{k}}\mathrm{exp}(s(z_{i},\hat{t}_{k})/\tau+\mathcal{G}_{ik}\circ (1-s(\hat{t}_{i},\hat{t}_{k}))/\tau)}\right].
% \mathcal{L}_{\mathrm{AW}} =-\mathbb{E}_{i,j}\left[y_{ij}\mathrm{log}\frac{\mathrm{exp}({[s(z_{i},\hat{t}_{j})}+\mathcal{G}_{ij}\circ (1-s(\hat{t}_{i},\hat{t}_{j}))]/\tau)}{{\sum_{k}}\mathrm{exp}([s(z_{i},\hat{t}_{k})+\mathcal{G}_{ik}\circ (1-s(\hat{t}_{i},\hat{t}_{k}))]/\tau)}\right]
% \label{eq-AW2}
% \end{align}
% Similar to the definition of text embedding distance, we can define the image embedding distance $\gamma_{i}$ as a logit margin, which means the distance metric between the probability distribution of given class and the closet probability distribution of the wrong class. For image embedding of the image input $i$ for TeCoA loss,
% \begin{align}
% \gamma_{i}=p_{ij}-p_{ik^{\star}}=\frac{e^{s(z_{i},\hat{t}_{j})} -e^{s(z_{i},\hat{t}_{k^{\star}})}}{\sum_{k}e^{s(z_{i},\hat{t}_{k})}}
% \label{eq-gamma}
% \end{align}
% where $k^{\star}$ means the closet wrong class of text embedding.
% \begin{align}
% \bar{\gamma}_{i}=\bar{p}_{ij}-\bar{p}_{ik^{\star}}=\frac{\omega_{ij}e^{s(z_{i},\hat{t}_{j})} -\omega_{ik^{\star}}e^{s(z_{i},\hat{t}_{k^{\star}})}}{\sum_{k}\omega_{ik}e^{s(z_{i},\hat{t}_{k})}}
% \label{eq-gamma}
% \end{align}
% From the above definition, we can have a proposition about the two inter-class distances:\\
% \noindent{\textbf{Proposition 2}:} Adaptive weighted contrastive adversarial training provides a lower bound for text-guided contrastive adversarial training. ($\gamma_{i}\ge \bar{\gamma}_{i}$)
% From the proposition 2, we can obviously get the conclusion that a fully trained by $\mathcal{L}_{AW}$, the final inter-class separation(distance) in the image embedding space would be increased, which would help to improve adversarial robustness.
% From another perspective, in contrast to TeCoA~\cite{TeCoA}'s equal weighting for all classes, the use of adaptive weights implies that classes with significant semantic distance from the correct class, easily discernible during adversarial attacks, are not overly emphasized. Conversely, classes with smaller semantic distances between correct and easily misclassified (potentially misclassified) classes are assigned higher weights, facilitating better differentiation during the adversarial training process. The hard negatives and potential hard negatives need to be emphasized to increase their distances and thus the inter-class variance in image embedding space increases. This adaptive weighting mechanism enhances the model's ability to discern semantic nuances among classes and tailor the adversarial training process accordingly.
% \subsubsection{AW-based Knowledge Distillation}
% \textbf{Text-Aware Knowledge Distillation for Image Semantic Consistency.}
% Similar to the cross-modal aware supervision employed in the IAT tuning mechanism, we leverage frozen pre-trained text embeddings as auxiliary supervisory information. This involves calculating the similarity between the finetuned image embedding and the pretrained text embedding. \textcolor{red}{By aligning the \textbf{student contrastive distribution} (\textit{i.e.}, adversarial image-to-text) with the \textbf{teacher contrastive distribution} (\textit{i.e.}, the fixed pretrained image-text embedding distribution)}, we aim to enhance the zero-shot generalization performance of the finetuned encoder. This method underscores the importance of utilizing established knowledge within pretrained models to guide the refinement of model representations, thereby improving their ability to generalize to new, unseen data in a zero-shot learning context.
% In the cross-modality awareness supervision distillation, the \textbf{teacher contrastive distribution} $p_{ji}$ is the same as in Eq.~\ref{eq-teacher-student-1}. The \textbf{student contrastive distribution} $r_{ij}$ can be described as:
% \begin{align}
% p^{'}_{ij} = \frac{\mathrm{exp}({s(\hat{z}_{i},\hat{t}_{j})}/\tau)}{{\sum_{k}}\mathrm{exp}(s(\hat{z}_{i},\hat{t}_{k})/\tau)},
% \end{align}
% where $\hat{t}_{j}$ represents the $j$-th text embedding of the frozen CLIP text encoder and the student text-to-image contrastive distributions $q_{ji}$ can be written as:
\subsubsection{Text-Aware Knowledge Distillation for Image Semantic Consistency}
Similar to Image-Aware Knowledge Distillation (IAKD) for text semantic consistency, we propose a Text-Aware Knowledge Distillation (TAKD) for image semantic consistency. In this paper, we denote the teacher contrastive distribution as $p_{ij}$ in TAI tuning mechanism (which is the same as teacher contrastive distribution of Eq.~\ref{eq-teacher-student-1} in IAT tuning mechanism) and the student contrastive distribution as $r_{ij}$ in tuning mechanism which can be described as:
\begin{align}
r_{ij} = \frac{\mathrm{exp}({s(z^{(a)}_{i},\hat{t}_{j})}/\tau)}{{\sum_{k}}\mathrm{exp}(s(z^{(a)}_{i},\hat{t}_{k})/\tau)},
\label{eq-teacher-student-2}
\end{align}
where $z^{(a)}$ is the adversarial image embedding of the fine-tuning CLIP image encoder and $\hat{t}$ is the text embedding of fixed pre-trained CLIP text encoder. Specifically, we first calculate the softmax distribution of the cosine similarity between the fixed pretrained CLIP clean image embeddings and the fixed pretrained CLIP text embeddings to serve as the teacher contrastive distribution. Subsequently, we compute the softmax distribution of the cosine similarity between the adversarial image embeddings and the fixed pretrained text embeddings as the student contrastive distribution. Then, we conduct KL divergence between student contrastive distribution and teacher contrastive distribution. The loss function $\mathcal{L}_{\mathrm{TAKD}}$ of Text-Aware Knowledge Distillation for image semantic consistency is described as:
\begin{align}
\mathcal{L}_{\mathrm{TAKD}}= \mathrm{D}_{\mathrm{KL}}(p_{ij}||r_{ij}).
\end{align}
Thus the loss function of our Text-Aware Image (TAI) tuning mechanism can be described as $\mathcal{L}_{\mathrm{TAI}}=\mathcal{L}_{\mathrm{TAM}}+\lambda_{\mathrm{V}}\mathcal{L}_{\mathrm{TAKD}}$.
$\lambda_{\mathrm{V}}$ is a hyper-parameter, which balances the text distance adaptive margin loss and knowledge distillation loss. The total loss function is:
\begin{align}
% \begin{matrix}
\mathcal{L}_{\mathrm{TIMA}} & = \mathcal{L}_{\mathrm{TAI}}+\lambda \mathcal{L}_\mathcal{\mathrm{IAT}} \nonumber\\
&=\mathcal{L}_{\mathrm{TAM}}+\lambda_{\mathrm{V}}\mathcal{L}_{\mathrm{TAKD}}+\lambda (\mathcal{L}_{\mathrm{MHE}}+\lambda_{\mathrm{T}}\mathcal{L}_{\mathrm{IAKD}}),
% \end{matrix}
\end{align}
where $\lambda$ is the hyper-parameter, which balances the $\mathcal{L}_{\mathrm{TAI}}$ and $\mathcal{L}_{\mathrm{IAT}}$, $\lambda_{\mathrm{V}}$ and $\lambda_{\mathrm{T}}$ are hyper-paeameters in TAI and IAT tuning.
\begin{table*}[tbp!]
\caption{Result of zero-shot accuracies. To evaluate robust accuracy, we utilize two white-box attacks, PGD~\cite{AT} and AutoAttack~\cite{AA}, by adversarial fine-tuning the model on \textbf{ImageNet}. The average performance of each method is presented in the last column. The best accuracy among SOTA methods (TeCoA, PMG, LAAT) and our proposed method (TIMA) are in bold.}
\small
\begin{tabular}{ccccccccccccc}
\hline
% \multirow{2}{*}{} & \multirow{2}{*}{} & \multicolumn{10}{c}{} & \\
& & \rotatebox{90}{ImageNet} &\rotatebox{90}{ CIFAR10} & \rotatebox{90}{CIFAR100} & \rotatebox{90}{STL10} & \rotatebox{90}{Tiny-ImageNet} & \rotatebox{90}{OxfordPet} & \rotatebox{90}{Food101} & \rotatebox{90}{EuroSAT} & \rotatebox{90}{SUN} & \rotatebox{90}{DTD} & \rotatebox{90}{Average} \\ \hline
\multirow{5}{*}{PGD}
& CLIP & 1.72 & 9.57 & 4.55 & 35.40 & 3.50 & 2.72 & 3.95 & 0.11 & 1.02 & 2.50 & 6.51 \\ \cline{2-13}
& TeCoA & 41.00 & 60.61 & 34.17 & 83.49 & 40.99 & 62.31 & 28.16 & 14.72 & 31.13 & 22.77 & 41.93 \\
& PMG & 45.53 & 62.08 & 35.02 & 84.66 & 45.59 & 64.60 & 27.81 & \textbf{17.07} & 31.92 & 23.40 & 43.77 \\
& LAAT & 40.73 & 58.54 & 35.02 & 33.43 & 70.56 & 41.57 & 21.11 & 8.86 & 31.35 & 13.14 & 35.43 \\
& \textbf{TIMA (Ours)} & \textbf{46.09} & \textbf{65.43} & \textbf{43.95} & \textbf{84.96} & \textbf{47.85} & \textbf{67.92} & \textbf{28.59} & 15.69 & \textbf{33.88} & \textbf{29.68} & \textbf{46.41} \\ \hline
\multirow{5}{*}{AutoAttack}
& CLIP & 0.00 & 2.54 & 2.34 & 2.93& 0.20 & 0.00 & 0.00 & 0.26 & 0.11 & 0.00 & 0.84 \\ \cline{2-13}
& TeCoA & 40.88 & 59.28 & 32.13 & 82.45 & 41.95 & 61.61 & 27.99 & 12.69 & 29.34 & 22.18 & 41.05 \\
& PMG & 41.03 & 60.33 & 36.89 & 82.98 & 42.16 & 61.05 & \textbf{29.86} & 13.97 & 30.59 & 25.56 & 42.44 \\
& LAAT & 38.75 & 51.56 & 33.20 & 77.34 & 41.21 & 23.05 & 24.77 & 4.69 & 32.90 & 15.89 & 34.34 \\
& \textbf{TIMA (Ours)} & \textbf{46.17} & \textbf{64.06} & \textbf{42.19} & \textbf{84.96} & \textbf{46.48} & \textbf{65.31} & 28.67 & \textbf{15.17} & \textbf{33.86} & \textbf{28.13} & \textbf{44.54} \\ \hline
\hline
\multirow{5}{*}{Clean Accuracy}
& CLIP & 59.12 & 89.06 & 62.32 & 97.16 & 60.74 & 85.82 & 83.15 & 38.26 & 57.67 & 40.52 & 67.38 \\ \cline{2-13}
& TeCoA & 63.21 & 78.63 &50.07 & \textbf{93.66} & 61.37 & 81.58 & 55.93 & 26.29 & 53.01 & 34.40 & 59.85 \\
& PMG & 63.27 & 78.23 & 48.94 & 92.83 & 61.93 & 80.00 & 48.83 & \textbf{27.28} & 49.36 & 32.45 & 58.31 \\
& LAAT & 62.66 & 71.66 & 49.90 & 91.86 & 62.34 & 50.75 & 41.48 & 9.53 & 54.20 & 21.65 & 51.60 \\
& \textbf{TIMA (Ours)} & \textbf{69.25} & \textbf{85.74} & \textbf{63.87} & 93.36 & \textbf{68.36} & \textbf{84.52} & \textbf{61.88} & 20.38 & \textbf{57.63} & \textbf{38.67} & \textbf{64.37} \\ \hline
\end{tabular}
\label{tab-ImageNet}
\end{table*}
\section{Experiments}
\label{sec-experiment}
\subsection{Experimental Setup}
\label{exp-datasets}
\subsubsection{Datasets and Baselines}
In our study, we assess the zero-shot adversarial robustness of our proposed method, TIMA, alongside several state-of-the-art (SOTA) methods, TeCoA~\cite{TeCoA}, PMG~\cite{PMG}, and LAAT~\cite{LAAT}, adversarial fine-tuned on both ImageNet~\cite{ImageNet} and Tiny-ImageNet~\cite{ImageNet} as two different versions. The two datasets were selected to evaluate the adaptability of TIMA to both large-scale datasets, like ImageNet, and smaller-scale datasets, such as Tiny-ImageNet. Performance will be analyzed using a variety of zero-shot test datasets besides the two datasets. Specifically, we include CIFAR10, CIFAR100~\cite{CIFAR100}, STL10~\cite{STL10} for generic classification; OxfordPets~\cite{OxfordPets}, Food101~\cite{Food101} for fine-grained classification; SUN397~\cite{SUN397} for scene recognition; DTD~\cite{DTD} for texture recognition, and finally, EuroSAT~\cite{EuroSAT} for domain-specialized task.
For baselines, we utilize the foundational CLIP model and the aforementioned SOTA methods under the same zero-shot settings previously established by these methodologies. We report the performance of these models after fine-tuning on both ImageNet and Tiny-ImageNet, ensuring that learning rate and adversarial training duration are consistent with our experimental setup for fair comparison. Detailed experimental results are presented in Tab.~\ref{tab-ImageNet} and Tab.~\ref{tab-Tiny-ImageNet}, with additional information available in the supplementary materials to provide a comprehensive evaluation of both our method and the compared methods across the selected datasets.
% \subsubsection{Datasets}
% We evaluate the zero-shot adversarial robustness conferred by our proposed method TIMA and several SOTA methods, such as TeCoA~\cite{TeCoA}, PMG~\cite{PMG} and LAAT~\cite{LAAT} trained on ImageNet and Tiny-ImageNet as two different versions. The rationale for adversarial fine-tuning on two datasets is to test whether the proposed TIMA can adapt effectively to both large-scale datasets (ImageNet) and smaller-scale datasets (Tiny-ImageNet). Therefore, we would report the performance of the models on the ImageNet test set and several zero-shot test datasets, covering a diverse range of recognition tasks. In addition to these findings, further experiments will be conducted on Tiny-ImageNet to demonstrate the superiority of our proposed method. Specifically, we include CIFAR10, CIFAR100~\cite{CIFAR100}, STL10~\cite{STL10} for generic classification; OxfordPets~\cite{OxfordPets}, Food101~\cite{Food101} for fine-grained classification; SUN397~\cite{SUN397} for scene recognition; DTD~\cite{DTD} for texture recognition, and finally, EuroSAT~\cite{EuroSAT} for domain-specialized task.
% %(satellite image classification).
% \subsubsection{Baselines}
% We choose the foundational CLIP model and three existing SOTA methods, TeCoA~\cite{TeCoA}, PMG~\cite{PMG} and LAAT~\cite{LAAT}, all of which investigate zero-shot adversarial robustness. Simultaneously, we adopt the zero-shot settings employed by these methods, specifically, the generalized zero-shot learning framework. The experimental results in Tab.~\ref{tab-ImageNet} present our re-reported outcomes of fine-tuning model for TeCoA, PMG and LAAT on ImageNet~\cite{ImageNet}. The experimental results in Tab.~\ref{tab-Tiny-ImageNet} present our re-reported outcomes of fine-tuning model for TeCoA, PMG and LAAT on Tiny-ImageNet~\cite{ImageNet}. This ensures a fair comparison by maintaining consistency in parameters such as learning rate and the number of adversarial training epochs with our proposed method. Due to space constraints, additional details are provided in the supplementary materials to ensure a comprehensive evaluation. This includes a thorough examination of our method and compared methods including TeCoA, PMG and LAAT on ImageNet and Tiny-ImageNet.
\subsubsection{Implementation Details}
We utilized the CLIP-B/32 architecture for our experiments, with the prompt text configured as "This is a photo of a {}". The model was optimized using an SGD optimizer with a momentum of 0.9. We conducted adversarial fine-tuning using $l_{\infty}$ bounded attacks with $\varepsilon= 1/255$ through a 2-step PGD attack~\cite{AT}, also with a step size of 1/255. The hyperparameters $m$ and $\eta$ for the Text-distance adaptive margin in the TAI tuning mechanism were uniformly set at 0.1 and 0.95. The hyper-parameters $\lambda, \lambda_{\mathrm{T}}, \lambda_{\mathrm{V}}$ were set at $10^{-3}, 1, 1$ for ImageNet and $1, 1, 1$ for Tiny-ImageNet. For adversarial fine-tuning on ImageNet, we trained the model for 5 epochs using a learning rate of $10^{-4}$ and a batch size of 256. In the case of Tiny-ImageNet, the model was trained over 10 epochs with a learning rate $10^{-4}$ but a batch size of 512.
\begin{table*}[tbp!]
\caption{Result of zero-shot accuracies. Robust accuracy is under PGD-10~\cite{AT}. All SOTA methods and our proposed method TIMA are adversarial fine-tuned on \textbf{Tiny-ImageNet}. We show the average performance of each method in the last column. The best accuracy is in bolded.}
% \small
\begin{tabular}{ccccccccccccc}
\hline
% \multirow{2}{*}{} & \multirow{2}{*}{} & \multicolumn{10}{c}{Datasets} & \\ \cline{3-13}
& & \rotatebox{90}{ImageNet} &\rotatebox{90}{ CIFAR10} & \rotatebox{90}{CIFAR100} & \rotatebox{90}{STL10} & \rotatebox{90}{Tiny-ImageNet} & \rotatebox{90}{OxfordPet} & \rotatebox{90}{Food101} & \rotatebox{90}{EuroSAT} & \rotatebox{90}{SUN} & \rotatebox{90}{DTD} & \rotatebox{90}{Average} \\ \hline
\multirow{4}{*}{Robust Accuracy}
& TeCoA & 12.98 & 54.91 & 30.15 & 75.14 & 56.63 & 29.76 & 9.59 & 14.95 & 13.86 & 10.77 & 30.87 \\
& PMG & 15.17
& 56.67 & \textbf{34.19} & 76.89 & 52.52 & 31.45 & 13.10 & 15.11 &\textbf{16.30} & \textbf{10.82} & 32.22 \\
& LAAT & 13.30 & 57.50 & 33.15 & 76.54 & \textbf{56.72} & 28.35 & 12.83 & 12.33 & 13.53 & 8.51 & 31.28 \\
& \textbf{TIMA (Ours)} & \textbf{15.65} &\textbf{58.70} & 33.37 &\textbf{77.40} & 56.41 & \textbf{38.87} & \textbf{14.23} & \textbf{15.21} & 13.52 & 8.31 & \textbf{33.17}\\ \hline
\hline
\multirow{4}{*}{Clean Accuracy}
& TeCoA & 43.87 & 76.10 & 46.00 & 91.39 & 76.00 & 75.33 & 43.11 & 17.83 & 46.15 & 29.73 & 54.55 \\
& PMG & 46.08
& 78.47 & 50.31 & 92.10 & 70.98 & 76.62 & 53.45 &23.72 & 45.85 & 31.12 & 56.87 \\
& LAAT & 42.72
& 68.08 & 42.72 & 87.75 & 75.84 & 51.21 & 34.81 & 9.44 & 43.92 & 18.14 & 47.46 \\
& \textbf{TIMA (Ours)} & \textbf{47.33} & \textbf{82.92} & \textbf{55.92} & \textbf{92.99} & \textbf{76.98} & \textbf{77.30} & \textbf{55.64} & \textbf{24.42} & \textbf{47.33} & \textbf{32.39} & \textbf{59.32} \\ \hline
\end{tabular}
\label{tab-Tiny-ImageNet}
\end{table*}
\vspace{-0.5mm}
\subsection{Comparison Results}
\label{exp-results}
We present an assessment of the robustness of our proposed method against prevalent white-box adversarial attacks, encompassing PGD-10~\cite{AT} (10 steps, step size of 1/255), and AutoAttack~\cite{AA} (AA), an effective attack used for evaluating adversarial robustness. Tab.~\ref{tab-ImageNet} and~\ref{tab-Tiny-ImageNet} display the robust accuracy (zero-shot adversarial robustness) and clean accuracy (zero-shot generalization) of state-of-the-art methods and our proposed approach under the $l_{\infty}$ setting, with the maximum adversarial perturbation set to $\varepsilon=1/255$. The best results for clean accuracy and adversarial robustness under various attack methods for each dataset are bolded. The final column in the table showcases the average adversarial robustness (and clean accuracy) results across these datasets.
Tab.~\ref{tab-ImageNet} and~\ref{tab-Tiny-ImageNet} present the clean accuracy and robust accuracy of adversarial fine-tuning on ImageNet and Tiny-ImageNet, comparing existing methods with the proposed TIMA. In Tab.~\ref{tab-ImageNet}, we showcase the robust accuracy under PGD-10~\cite{AT} (step size as 1/255) and AutoAttack~\cite{AA}, with a perturbation radius of $1/255$. Due to space constraints, Tab.~\ref{tab-Tiny-ImageNet} only displays robust accuracy at a perturbation budget of $1/255$ under the PGD-10 attack. For AutoAttack robust accuracy of existing methods and our TIMA after adversarial fine-tuning on Tiny-ImageNet, please refer to the supplementary materials.
From Tab.~\ref{tab-ImageNet}, it is evident that our method, despite a marginal 3.01\% (from 67.38\% to 64.37\%) decline in zero-shot clean accuracy (generalization) compared to CLIP, achieves a remarkable 39.9\% (from 6.51\% to 46.41\%) improvement in performance under PGD-10 attacks and a substantial 43.7\% (from 0.84\% to 44.54\%) enhancement against AutoAttack. In comparison with existing SOTA methods, our method improves zero-shot clean accuracy by 4.52\% (from 59.85\% to 64.37\%) over TeCoA, 6.06\% (from 58.31\% to 64.37\%) over PMG, and by a significant 12.77\% (from 51.60\% to 64.37\%) over LAAT.
Regarding zero-shot robust accuracy, under PGD attack, our proposed method's zero-shot robust accuracy exceeds TeCoA by 4.48\% (from 41.93\% to 46.41\%), PMG by 2.64\% (from 43.77\% to 46.41\%), and LAAT by 10.98\% (from 35.43\% to 46.41\%). Under AutoAttack, our method's zero-shot robust accuracy is 3.49\% higher than TeCoA's, 2.1\% more than PMG's, and 10.2\% above LAAT's. The data from Tab.~\ref{tab-ImageNet} leads to the conclusion that, when adversarially fine-tuned on the ImageNet dataset, our method achieves a favorable trade-off between zero-shot adversarial robustness and zero-shot generalization. Moreover, the results from Tab.~\ref{tab-Tiny-ImageNet}
show that adversarial training on a smaller scale of dataset also works our proposed method. From Tab.~\ref{tab-Tiny-ImageNet}, we conclude that our proposed method also a favorable trade-off between zero-shot adversarial robustness and zero-shot generalization, when adversarially fine-tuned on the Tiny-ImageNet dataset. The scale of adversarial fine-tuning dataset is not a constraint of our proposed method.
\begin{table*}[tbp!]
\caption{Zero-shot robust accuracy under different perturbation radius $\varepsilon$. We adversarial fine-tune the model on \textbf{Tiny-ImageNet}. The average performance of each method is shown in the last column. The best accuracies are bolded.}
% \small
\begin{tabular}{ccccccccccccc}
\hline
% \multirow{2}{*}{} & \multirow{2}{*}{} & \multicolumn{10}{c}{Datasets} & \\ \cline{3-13}
& & \rotatebox{90}{ImageNet} &\rotatebox{90}{ CIFAR10} & \rotatebox{90}{CIFAR100} & \rotatebox{90}{STL10} & \rotatebox{90}{Tiny-ImageNet} & \rotatebox{90}{OxfordPet} & \rotatebox{90}{Food101} & \rotatebox{90}{EuroSAT} & \rotatebox{90}{SUN} & \rotatebox{90}{DTD} & \rotatebox{90}{Average} \\ \hline
\multirow{4}{*}{$\varepsilon=4/255$}
& TeCoA & 1.99
& 10.62 & 6.71 & 42.91 & 17.97 & 6.62 & 1.40 & \textbf{0.53} & 1.22 & 1.54 & 9.15 \\
& PMG & 0.72
& 11.68 & 6.26 & 34.34 & 8.69 & 2.34 & 1.09 & 0.00 & 0.57 & 1.06 & 6.68 \\
& LAAT & 1.25
& 24.22 & 10.16 & 58.34 & 16.99 & 4.23 & 2.60 & 0.00 & 1.55 & 1.03 & 12.04 \\
& Ours & \textbf{11.89} & \textbf{32.76} & \textbf{15.80} &\textbf{69.26} &\textbf{44.94} & \textbf{35.11} & \textbf{11.49} & 0.00 &\textbf{7.04} & \textbf{3.30} & \textbf{23.16} \\ \hline
\multirow{4}{*}{$\varepsilon=8/255$}
& TeCoA & 1.85 & 7.06 & 3.95 & 40.93 & 15.29 & 6.54 & 1.45 & 0.00 & 1.10 & 0.53 & 7.87 \\
& PMG & 0.54 & 5.51 & 2.07 & 30.06 & 4.13 & 2.34 & 1.13 & 0.00 & 0.42 & 0.37 & 4.66 \\
& LAAT & 1.04 & 23.44 & 9.34 & 58.59 & 14.67 & 4.11 & 2.67 & 0.00 & 1.62 & 0.32 & 11.58 \\
& Ours & \textbf{11.89} & \textbf{30.74} & \textbf{14.80} & \textbf{68.59} & \textbf{44.41} & \textbf{34.67} &\textbf{11.55} & 0.00 &\textbf{7.00} &\textbf{2.71} & \textbf{22.64} \\ \hline
\end{tabular}
\label{tab-Perturbation}
\end{table*}
% \vspace{-4mm}
\subsection{Result Analysis on $\varepsilon$ and $\tau$}
\label{exp-analysis}
\subsubsection{Zero-shot Adversarial Robustness under Larger Perturbations}
Tab.~\ref{tab-Perturbation} illustrates the zero-shot adversarial robustness under the larger perturbation radius ($\varepsilon=4/255, 8/255$) with the robust accuracy by PGD-10~\cite{AT} attack. For results on zero-shot adversarial robustness with larger perturbations using AutoAttack, please refer to the supplementary materials. The adversarial robustness at a smaller perturbation radius of $1/255$ and the clean sample accuracy (zero-shot generalization) results are shown in Tab.~\ref{tab-Tiny-ImageNet}. From Tab.~\ref{tab-Perturbation}, under large perturbation $\varepsilon=8/255$, our zero-shot robust accuracy improved by 14.77\% (from 7.87\% to 22.64\%) compared to the TeCoA method and by 11.06\% (from 11.58\% to 22.64\%) relative to the LAAT.
\begin{figure}[tbp!]
\centering
\subfigure[Robust Accuracy]
{
\begin{minipage}[t]{0.4\linewidth}
\centering
\includegraphics[width=\linewidth]{images/temp1-robust.png}
%\caption{fig1}
\label{fig-robust-accuracy-t}
\end{minipage}%
}
\subfigure[Clean Accuracy]
{
\begin{minipage}[t]{0.4\linewidth}
\centering
\includegraphics[width=\linewidth]{images/temp1-clean.png}
% \caption{fig2}
\label{fig-clean-accuracy-t}
\end{minipage}%
}
\centering
\vspace{-0.5em}
\caption{Zero-shot robust/clean accuracy of the proposed method under the CLIP temperature $\tau = 1$.}
% \vspace{-1.8em}
\label{fig-tem}
\end{figure}
\begin{table*}[tbp!]
\caption{Zero-shot accuracies of TeCoA~\cite{TeCoA} and LAAT~\cite{LAAT} with our proposed tuning mechanism. We adversarial fine-tune the method on \textbf{Tiny-ImageNet}. CA and RA represent clean accuracy and robust accuracy. See 4/255 case in appendix.}
%The average performance of each method is shown in the last column.
% \small
\begin{tabular}{ccccccccccccc}
\hline
% \multirow{2}{*}{} & \multirow{2}{*}{} & \multicolumn{10}{c}{Datasets} & \\ \cline{3-13}
& & \rotatebox{90}{ImageNet} &\rotatebox{90}{ CIFAR10} & \rotatebox{90}{CIFAR100} & \rotatebox{90}{STL10} & \rotatebox{90}{Tiny-ImageNet} & \rotatebox{90}{OxfordPet} & \rotatebox{90}{Food101} & \rotatebox{90}{EuroSAT} & \rotatebox{90}{SUN} & \rotatebox{90}{DTD} & \rotatebox{90}{Average} \\ \hline
\multirow{3}{*}{TeCoA}
% & CLIP & 1.72 & 9.57 & 4.55 & 35.40 & 3.50 & 2.72 & 3.95 & 1.19 & 1.02 & 2.50 & 6.61 \\
& CA & 43.87 & 76.10 & 46.00 & 91.39 & 76.00 & 75.33 & 43.11 & 17.83 & 46.15 & 29.73 & 54.55 \\
& RA ($\varepsilon=1/255$) & 12.98 & 54.91 & 30.15 & 75.14 & 56.63 & 62.61 & 9.59 & 10.77 & 13.86 & 14.95 & 30.87 \\ \\
& RA ($\varepsilon=8/255$) & 1.85 & 7.06 & 3.95 & 40.93 & 15.29 & 6.54 & 1.45 & 0.00 & 1.10 & 0.53 & 7.87 \\
\hline
\multirow{3}{*}{TeCoA + IAT}
& CA &44.50 &76.74 &45.92 &91.94 &77.11 &76.75 &41.47 &20.16 &44.18 &28.62 & 54.74 \\
& RA ($\varepsilon=1/255$) &12.34 &55.16 &30.19 &75.11 &58.07 &28.73 &9.21 &14.99 &13.04 &13.51 & 31.04 \\
& RA ($\varepsilon=8/255$) &5.64 &23.62 &14.08 &48.97 &19.19 &6.49 &1.65 &0.00 &1.93 &0.78 &12.24 \\
\hline
\multirow{3}{*}{TeCoA +TAI}
& CA &46.26 &83.80 &55.50 &93.13 &77.20 &77.19 &55.73 &19.63 &47.77 &33.46 & 58.97 \\
& RA ($\varepsilon=1/255$) &15.00 &60.11 &33.98 &77.44 &56.43 &37.89 &13.79 &1.39 &13.47 &15.43 & 32.49 \\
& RA ($\varepsilon=8/255$ ) & 8.36 &22.66 &14.06 &66.02 &41.89 &19.92 &10.61 &0.00 &8.08 &2.93 & 19.45 \\
\hline
\hline
\multirow{3}{*}{LAAT}
& CA & 42.72
& 68.08 & 42.72 & 87.75 & 75.84 & 51.21 & 34.81 & 9.44 & 43.92 & 18.14 & 47.46 \\
& RA ($\varepsilon=1/255$) & 13.30 & 57.50 & 33.15 & 76.54 & 56.72 & 28.35 & 12.83 & 12.33 & 13.53 & 8.51 & 31.28 \\
& RA ($\varepsilon=8/255$) & 1.04 & 23.44 & 9.34 & 58.59 & 14.67 & 4.11 & 2.67 & 0.00 & 1.62 & 0.32 & 11.58 \\
\hline
\multirow{3}{*}{LAAT + TAI}
& CA &44.96 &74.51 &49.27 &89.26 &77.34 &73.63 &55.80 &23.07 &45.36 &25.12 &55.84 \\
& RA ($\varepsilon=1/255$) &13.56 &57.81 &31.64 &77.15 &58.20 &32.37 &15.35 &14.76 &12.82 &8.79 &32.25 \\
& RA ($\varepsilon=8/255$) &10.78 &30.04 &12.85 &69.17 &45.22 &29.34 &7.40 &0.00 &10.39 &1.37 &21.66 \\
\hline
\hline
\multirow{3}{*}{TIMA (Ours)}
& CA & 47.33 & 82.92 & 55.92 & 92.99 & 76.98 & 77.30 & 55.64 & 24.42 & 47.33 & 32.39 & 59.32 \\
& RA ($\varepsilon=1/255$) & 15.65 &58.70 & 33.37 &77.40 & 56.41 & 38.87 & 14.23 & 15.21 & 13.52 & 8.31 & 33.17 \\
&RA ($\varepsilon=8/255$) & 11.89 & 30.74 & 14.80 & 68.59 & 44.41 & 34.67 & 11.55 & 0.00 & 7.00 & 2.71 & 22.64 \\
\hline
\end{tabular}
\label{tab-ablation}
\end{table*}
% %%%% temperature
\subsubsection{Temperature Effect for Zero-shot Adversarial Robustness and Generalization}
In addition to its effectiveness against large perturbative adversarial attacks, we observe that the proposed TIMA method remains efficacious across various CLIP temperature $\tau$. As depicted in Fig.~\ref{fig-tem}, an increase in the CLIP temperature $\tau$ to 1 (contrary to other experiments within this study where the default temperature coefficients for existing methods and our proposed TIMA method are maintained at 0.01), the results demonstrate that our TIMA notably excels over current SOTA methods in both robust and clean accuracy. This result affirms the versatility and superior performance of the TIMA method across a broad range of CLIP temperature $\tau$.
% %%%% hyper-meters eta g
\begin{figure}[t!]
\vspace{-1em}
\centering
\subfigure[Robust Accuracy \textit{vs.} $m$]
{
\begin{minipage}[t]{0.4\linewidth}
\centering
\includegraphics[width=\linewidth]{images/1-margin-ours.png}
%\caption{fig1}
\label{fig-hyper-g}
\end{minipage}%
}
\subfigure[Robust Accuracy \textit{vs.} $\eta$]
{
\begin{minipage}[t]{0.4\linewidth}
\centering
\includegraphics[width=\linewidth]{images/1-margin-eta-ours.png}
% \caption{fig2}
\label{fig-hyper-eta}
\end{minipage}%
}
\centering
\vspace{-0.5em}
\caption{Zero-shot robust accuracy of our proposed method with different hyper-parameter $m$ and $\eta$ in our proposed Text-distance Adaptive Margin.}
% \vspace{-1.8em}
\label{fig-hyper}
\end{figure}
\subsection{Ablation Study}
\label{exp-ablation}
% \textcolor{red}{Unfinished.}
% Limited by computing resources,
The ablation study is based on the proposed model which is adversarial fine-tuning on Tiny-ImageNet. It should be noted that the robust accuracy in Sec.~\ref{exp-ablation} is evaluated by PGD-10.
\subsubsection{Contribution of Each Tuning Mechanism}
Tab.~\ref{tab-ablation} presents the results of an ablation study on tuning mechanisms (IAT and TAI), showing the zero-shot clean accuracy and zero-shot robust accuracy zero-shot clean accuracy across multiple test datasets under various perturbation budgets. The results clearly demonstrate that IAT markedly improves adversarial robustness under various perturbations. TAI is a plug-and-play tuning mechanism which can be introduced into TeCoA and LAAT, leading to increasing both zero-shot adversarial robustness and generalization. It should be noted that IAT tuning mechanism is a learn-based text emmbedding adjustment algorithm, while LAAT has fixed text expansion algorithms, so IAT cannot be directly introduced into LAAT.
Comparing the results between TeCoA (LAAT) and TeCoA+TAI (LAAT+TAI), it is evident that increasing the inter-class distances of image embeddings improves zero-shot adversarial robustness. Further, the results between TeCoA+IAT, TeCoA+TAI and TIMA demonstrate that increasing both text and image embedding inter-class distances improves zero-shot adversarial robustness, particularly under large perturbations, which can be \textbf{the evidence of our hypothesis in Sec.~\ref{sec-intro}}.
\vspace{-0.5em}
\subsubsection{Hypermeters $\beta, m$ in TAI Tuning Mechanism}
We also present experimental results for the remaining hyper-parameters of Text-distance Adaptive Margin in the TAI tuning mechanism in Fig.~\ref{fig-hyper-g} and~\ref{fig-hyper-eta}, which showcase the outcomes of experiments concerning the hyper-parameters $\eta$ and $m$ in the Text-distance Adaptive Margin. As depicted in Fig.~\ref{fig-hyper-g}, the optimal performance for zero-shot robust accuracy is observed when the value of $m$ is set to 0.1. Similarly, from Fig.~\ref{fig-hyper-eta}, it is evident that the optimal and relatively stable performance is achieved when $\eta$ falls within the range of 0.9 to 0.95. Therefore, in our experiments, we set $\eta$ and $m$ to 0.95 and 0.1.
% \vspace{-3.5mm}
% \vspace{0.2em}
\section{Conclusion}
\label{sec-conclusion}
This study proposes an adversarial fine-tuning approach with two tuning mechanisms (IAT and TAI) to balance zero-shot adversarial robustness and generalization. IAT tuning achieves a balance between increasing inter-class distance and semantic consistency between text embeddings by combining MHE and IAKD. TAI tuning balances the enhanced inter-class distance and semantic consistency between image embeddings by combining TAW and TAKD. Our results affirm the importance at the inter-class distance for adversarial robustness and highlight the role of semantic alignments for generalization. Both aspects deserve future investigation.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{44}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Ahn et~al\mbox{.}(2022)]%
{foundations2}
\bibfield{author}{\bibinfo{person}{Michael Ahn}, \bibinfo{person}{Anthony Brohan}, \bibinfo{person}{Noah Brown}, \bibinfo{person}{Yevgen Chebotar}, \bibinfo{person}{Omar Cortes}, \bibinfo{person}{Byron David}, \bibinfo{person}{Chelsea Finn}, \bibinfo{person}{Chuyuan Fu}, \bibinfo{person}{Keerthana Gopalakrishnan}, \bibinfo{person}{Karol Hausman}, {et~al\mbox{.}}} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Do as i can, not as i say: Grounding language in robotic affordances}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2204.01691}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Bossard et~al\mbox{.}(2014)]%
{Food101}
\bibfield{author}{\bibinfo{person}{Lukas Bossard}, \bibinfo{person}{Matthieu Guillaumin}, {and} \bibinfo{person}{Luc Van~Gool}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Food-101--mining discriminative components with random forests}. In \bibinfo{booktitle}{\emph{Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part VI 13}}. Springer, \bibinfo{pages}{446--461}.
\newblock
\bibitem[Bui et~al\mbox{.}(2021)]%
{inter-class1}
\bibfield{author}{\bibinfo{person}{Anh Bui}, \bibinfo{person}{Trung Le}, \bibinfo{person}{He Zhao}, \bibinfo{person}{Paul Montague}, \bibinfo{person}{Seyit Camtepe}, {and} \bibinfo{person}{Dinh Phung}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Understanding and achieving efficient robustness with adversarial supervised contrastive learning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2101.10027}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Carlini and Wagner(2017)]%
{CW}
\bibfield{author}{\bibinfo{person}{Nicholas Carlini} {and} \bibinfo{person}{David Wagner}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Towards evaluating the robustness of neural networks}. In \bibinfo{booktitle}{\emph{2017 ieee symposium on security and privacy (sp)}}. Ieee, \bibinfo{pages}{39--57}.
\newblock
\bibitem[Cimpoi et~al\mbox{.}(2014)]%
{DTD}
\bibfield{author}{\bibinfo{person}{Mircea Cimpoi}, \bibinfo{person}{Subhransu Maji}, \bibinfo{person}{Iasonas Kokkinos}, \bibinfo{person}{Sammy Mohamed}, {and} \bibinfo{person}{Andrea Vedaldi}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Describing textures in the wild}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE conference on computer vision and pattern recognition}}. \bibinfo{pages}{3606--3613}.
\newblock
\bibitem[Coates et~al\mbox{.}(2011)]%
{STL10}
\bibfield{author}{\bibinfo{person}{Adam Coates}, \bibinfo{person}{Andrew Ng}, {and} \bibinfo{person}{Honglak Lee}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{An Analysis of Single-Layer Networks in Unsupervised Feature Learning}. In \bibinfo{booktitle}{\emph{Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics}} \emph{(\bibinfo{series}{Proceedings of Machine Learning Research}, Vol.~\bibinfo{volume}{15})}, \bibfield{editor}{\bibinfo{person}{Geoffrey Gordon}, \bibinfo{person}{David Dunson}, {and} \bibinfo{person}{Miroslav Dudík}} (Eds.). \bibinfo{publisher}{PMLR}, \bibinfo{address}{Fort Lauderdale, FL, USA}, \bibinfo{pages}{215--223}.
\newblock
\urldef\tempurl%
\url{https://proceedings.mlr.press/v15/coates11a.html}
\showURL{%
\tempurl}
\bibitem[Croce and Hein(2020)]%
{AA}
\bibfield{author}{\bibinfo{person}{Francesco Croce} {and} \bibinfo{person}{Matthias Hein}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks}. In \bibinfo{booktitle}{\emph{International conference on machine learning}}. PMLR, \bibinfo{pages}{2206--2216}.
\newblock
\bibitem[Deng et~al\mbox{.}(2009)]%
{ImageNet}
\bibfield{author}{\bibinfo{person}{Jia Deng}, \bibinfo{person}{Wei Dong}, \bibinfo{person}{Richard Socher}, \bibinfo{person}{Li-Jia Li}, \bibinfo{person}{Kai Li}, {and} \bibinfo{person}{Li Fei-Fei}.} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{Imagenet: A large-scale hierarchical image database}. In \bibinfo{booktitle}{\emph{2009 IEEE conference on computer vision and pattern recognition}}. Ieee, \bibinfo{pages}{248--255}.
\newblock
\bibitem[Ding et~al\mbox{.}(2019)]%
{margin1}
\bibfield{author}{\bibinfo{person}{Gavin~Weiguang Ding}, \bibinfo{person}{Yash Sharma}, \bibinfo{person}{Kry Yik~Chau Lui}, {and} \bibinfo{person}{Ruitong Huang}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{MMA Training: Direct Input Space Margin Maximization through Adversarial Training}. In \bibinfo{booktitle}{\emph{International Conference on Learning Representations}}.
\newblock
\bibitem[Helber et~al\mbox{.}(2019)]%
{EuroSAT}
\bibfield{author}{\bibinfo{person}{Patrick Helber}, \bibinfo{person}{Benjamin Bischke}, \bibinfo{person}{Andreas Dengel}, {and} \bibinfo{person}{Damian Borth}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Eurosat: A novel dataset and deep learning benchmark for land use and land cover classification}.
\newblock \bibinfo{journal}{\emph{IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing}} \bibinfo{volume}{12}, \bibinfo{number}{7} (\bibinfo{year}{2019}), \bibinfo{pages}{2217--2226}.
\newblock
\bibitem[Hou et~al\mbox{.}(2023)]%
{margin3}
\bibfield{author}{\bibinfo{person}{Pengyue Hou}, \bibinfo{person}{Jie Han}, {and} \bibinfo{person}{Xingyu Li}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Improving adversarial robustness with self-paced hard-class pair reweighting}. In \bibinfo{booktitle}{\emph{Proceedings of the AAAI Conference on Artificial Intelligence}}, Vol.~\bibinfo{volume}{37}. \bibinfo{pages}{14883--14891}.
\newblock
\bibitem[Huang et~al\mbox{.}(2023)]%
{VLM-distillation4}
\bibfield{author}{\bibinfo{person}{Zeyi Huang}, \bibinfo{person}{Andy Zhou}, \bibinfo{person}{Zijian Ling}, \bibinfo{person}{Mu Cai}, \bibinfo{person}{Haohan Wang}, {and} \bibinfo{person}{Yong~Jae Lee}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{A Sentence Speaks a Thousand Images: Domain Generalization through Distilling CLIP with Language Guidance}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}}. \bibinfo{pages}{11685--11695}.
\newblock
\bibitem[Inkawhich et~al\mbox{.}(2023)]%
{Attack-foundation-vision}
\bibfield{author}{\bibinfo{person}{Nathan Inkawhich}, \bibinfo{person}{Gwendolyn McDonald}, {and} \bibinfo{person}{Ryan Luley}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Adversarial Attacks on Foundational Vision Models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2308.14597}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Jia et~al\mbox{.}(2021)]%
{foundations1}
\bibfield{author}{\bibinfo{person}{Chao Jia}, \bibinfo{person}{Yinfei Yang}, \bibinfo{person}{Ye Xia}, \bibinfo{person}{Yi-Ting Chen}, \bibinfo{person}{Zarana Parekh}, \bibinfo{person}{Hieu Pham}, \bibinfo{person}{Quoc Le}, \bibinfo{person}{Yun-Hsuan Sung}, \bibinfo{person}{Zhen Li}, {and} \bibinfo{person}{Tom Duerig}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Scaling up visual and vision-language representation learning with noisy text supervision}. In \bibinfo{booktitle}{\emph{International conference on machine learning}}. PMLR, \bibinfo{pages}{4904--4916}.
\newblock
\bibitem[Krizhevsky et~al\mbox{.}(2009)]%
{CIFAR100}
\bibfield{author}{\bibinfo{person}{Alex Krizhevsky}, \bibinfo{person}{Geoffrey Hinton}, {et~al\mbox{.}}} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{Learning multiple layers of features from tiny images}.
\newblock (\bibinfo{year}{2009}).
\newblock
\bibitem[Li et~al\mbox{.}(2023a)]%
{VLM-distillation}
\bibfield{author}{\bibinfo{person}{Xuanlin Li}, \bibinfo{person}{Yunhao Fang}, \bibinfo{person}{Minghua Liu}, \bibinfo{person}{Zhan Ling}, \bibinfo{person}{Zhuowen Tu}, {and} \bibinfo{person}{Hao Su}.} \bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{Distilling large vision-language model with out-of-distribution generalizability}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}}. \bibinfo{pages}{2492--2503}.
\newblock
\bibitem[Li et~al\mbox{.}(2021)]%
{inter-class3}
\bibfield{author}{\bibinfo{person}{Xin Li}, \bibinfo{person}{Xiangrui Li}, \bibinfo{person}{Deng Pan}, {and} \bibinfo{person}{Dongxiao Zhu}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Improving adversarial robustness via probabilistically compact loss with logit constraints}. In \bibinfo{booktitle}{\emph{Proceedings of the AAAI conference on artificial intelligence}}, Vol.~\bibinfo{volume}{35}. \bibinfo{pages}{8482--8490}.
\newblock
\bibitem[Li et~al\mbox{.}(2023b)]%
{LAAT}
\bibfield{author}{\bibinfo{person}{Xiao Li}, \bibinfo{person}{Wei Zhang}, \bibinfo{person}{Yining Liu}, \bibinfo{person}{Zhanhao Hu}, \bibinfo{person}{Bo Zhang}, {and} \bibinfo{person}{Xiaolin Hu}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{Anchor-Based Adversarially Robust Zero-Shot Learning Driven by Language}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2301.13096}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Lin et~al\mbox{.}(2020)]%
{MHE3}
\bibfield{author}{\bibinfo{person}{Rongmei Lin}, \bibinfo{person}{Weiyang Liu}, \bibinfo{person}{Zhen Liu}, \bibinfo{person}{Chen Feng}, \bibinfo{person}{Zhiding Yu}, \bibinfo{person}{James~M Rehg}, \bibinfo{person}{Li Xiong}, {and} \bibinfo{person}{Le Song}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Regularizing neural networks via minimizing hyperspherical energy}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}}. \bibinfo{pages}{6917--6927}.
\newblock
\bibitem[Liu et~al\mbox{.}(2018)]%
{MHE}
\bibfield{author}{\bibinfo{person}{Weiyang Liu}, \bibinfo{person}{Rongmei Lin}, \bibinfo{person}{Zhen Liu}, \bibinfo{person}{Lixin Liu}, \bibinfo{person}{Zhiding Yu}, \bibinfo{person}{Bo Dai}, {and} \bibinfo{person}{Le Song}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Learning towards minimum hyperspherical energy}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}} \bibinfo{volume}{31} (\bibinfo{year}{2018}).
\newblock
\bibitem[Liu et~al\mbox{.}(2021)]%
{MHE2}
\bibfield{author}{\bibinfo{person}{Weiyang Liu}, \bibinfo{person}{Rongmei Lin}, \bibinfo{person}{Zhen Liu}, \bibinfo{person}{Li Xiong}, \bibinfo{person}{Bernhard Sch{\"o}lkopf}, {and} \bibinfo{person}{Adrian Weller}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Learning with hyperspherical uniformity}. In \bibinfo{booktitle}{\emph{International Conference On Artificial Intelligence and Statistics}}. PMLR, \bibinfo{pages}{1180--1188}.
\newblock
\bibitem[Liu et~al\mbox{.}(2016)]%
{margin-softmax}
\bibfield{author}{\bibinfo{person}{Weiyang Liu}, \bibinfo{person}{Yandong Wen}, \bibinfo{person}{Zhiding Yu}, {and} \bibinfo{person}{Meng Yang}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Large-margin softmax loss for convolutional neural networks}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1612.02295}} (\bibinfo{year}{2016}).
\newblock
\bibitem[Liu et~al\mbox{.}(2022)]%
{MHE4}
\bibfield{author}{\bibinfo{person}{Weiyang Liu}, \bibinfo{person}{Longhui Yu}, \bibinfo{person}{Adrian Weller}, {and} \bibinfo{person}{Bernhard Sch{\"o}lkopf}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Generalizing and Decoupling Neural Collapse via Hyperspherical Uniformity Gap}. In \bibinfo{booktitle}{\emph{The Eleventh International Conference on Learning Representations}}.
\newblock
\bibitem[Liu and Chan(2022)]%
{margin2}
\bibfield{author}{\bibinfo{person}{Ziquan Liu} {and} \bibinfo{person}{Antoni~B Chan}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Boosting adversarial robustness from the perspective of effective margin regularization}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2210.05118}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Madry et~al\mbox{.}(2017)]%
{AT}
\bibfield{author}{\bibinfo{person}{Aleksander Madry}, \bibinfo{person}{Aleksandar Makelov}, \bibinfo{person}{Ludwig Schmidt}, \bibinfo{person}{Dimitris Tsipras}, {and} \bibinfo{person}{Adrian Vladu}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Towards deep learning models resistant to adversarial attacks}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1706.06083}} (\bibinfo{year}{2017}).
\newblock
\bibitem[Mao et~al\mbox{.}(2022)]%
{TeCoA}
\bibfield{author}{\bibinfo{person}{Chengzhi Mao}, \bibinfo{person}{Scott Geng}, \bibinfo{person}{Junfeng Yang}, \bibinfo{person}{Xin Wang}, {and} \bibinfo{person}{Carl Vondrick}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Understanding zero-shot adversarial robustness for large-scale models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2212.07016}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Mao et~al\mbox{.}(2019)]%
{inter-class4}
\bibfield{author}{\bibinfo{person}{Chengzhi Mao}, \bibinfo{person}{Ziyuan Zhong}, \bibinfo{person}{Junfeng Yang}, \bibinfo{person}{Carl Vondrick}, {and} \bibinfo{person}{Baishakhi Ray}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Metric learning for adversarial robustness}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}} \bibinfo{volume}{32} (\bibinfo{year}{2019}).
\newblock
\bibitem[Michels et~al\mbox{.}(2023)]%
{CLIP-OOD-detecors}
\bibfield{author}{\bibinfo{person}{Felix Michels}, \bibinfo{person}{Nikolas Adaloglou}, \bibinfo{person}{Tim Kaiser}, {and} \bibinfo{person}{Markus Kollmann}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Contrastive Language-Image Pretrained (CLIP) Models are Powerful Out-of-Distribution Detectors}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2303.05828}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Mustafa et~al\mbox{.}(2019)]%
{inter-class2}
\bibfield{author}{\bibinfo{person}{Aamir Mustafa}, \bibinfo{person}{Salman Khan}, \bibinfo{person}{Munawar Hayat}, \bibinfo{person}{Roland Goecke}, \bibinfo{person}{Jianbing Shen}, {and} \bibinfo{person}{Ling Shao}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Adversarial defense by restricting the hidden space of deep neural networks}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}}. \bibinfo{pages}{3385--3394}.
\newblock
\bibitem[Noever and Noever(2021)]%
{reading}
\bibfield{author}{\bibinfo{person}{David~A Noever} {and} \bibinfo{person}{Samantha E~Miller Noever}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Reading Isn't Believing: Adversarial Attacks On Multi-Modal Neurons}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2103.10480}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Parkhi et~al\mbox{.}(2012)]%
{OxfordPets}
\bibfield{author}{\bibinfo{person}{Omkar~M Parkhi}, \bibinfo{person}{Andrea Vedaldi}, \bibinfo{person}{Andrew Zisserman}, {and} \bibinfo{person}{CV Jawahar}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Cats and dogs}. In \bibinfo{booktitle}{\emph{2012 IEEE conference on computer vision and pattern recognition}}. IEEE, \bibinfo{pages}{3498--3505}.
\newblock
\bibitem[Radenovic et~al\mbox{.}(2023)]%
{VLM-distillation2}
\bibfield{author}{\bibinfo{person}{Filip Radenovic}, \bibinfo{person}{Abhimanyu Dubey}, \bibinfo{person}{Abhishek Kadian}, \bibinfo{person}{Todor Mihaylov}, \bibinfo{person}{Simon Vandenhende}, \bibinfo{person}{Yash Patel}, \bibinfo{person}{Yi Wen}, \bibinfo{person}{Vignesh Ramanathan}, {and} \bibinfo{person}{Dhruv Mahajan}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Filtering, distillation, and hard negatives for vision-language pre-training}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}}. \bibinfo{pages}{6967--6977}.
\newblock
\bibitem[Radford et~al\mbox{.}(2021)]%
{CLIP}
\bibfield{author}{\bibinfo{person}{Alec Radford}, \bibinfo{person}{Jong~Wook Kim}, \bibinfo{person}{Chris Hallacy}, \bibinfo{person}{Aditya Ramesh}, \bibinfo{person}{Gabriel Goh}, \bibinfo{person}{Sandhini Agarwal}, \bibinfo{person}{Girish Sastry}, \bibinfo{person}{Amanda Askell}, \bibinfo{person}{Pamela Mishkin}, \bibinfo{person}{Jack Clark}, {et~al\mbox{.}}} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Learning transferable visual models from natural language supervision}. In \bibinfo{booktitle}{\emph{International conference on machine learning}}. PMLR, \bibinfo{pages}{8748--8763}.
\newblock
\bibitem[Ramesh et~al\mbox{.}(2022)]%
{foundations3}
\bibfield{author}{\bibinfo{person}{Aditya Ramesh}, \bibinfo{person}{Prafulla Dhariwal}, \bibinfo{person}{Alex Nichol}, \bibinfo{person}{Casey Chu}, {and} \bibinfo{person}{Mark Chen}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Hierarchical text-conditional image generation with clip latents}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2204.06125}} \bibinfo{volume}{1}, \bibinfo{number}{2} (\bibinfo{year}{2022}), \bibinfo{pages}{3}.
\newblock
\bibitem[Wang et~al\mbox{.}(2023)]%
{ChatGPT-Robustness}
\bibfield{author}{\bibinfo{person}{Jindong Wang}, \bibinfo{person}{Xixu Hu}, \bibinfo{person}{Wenxin Hou}, \bibinfo{person}{Hao Chen}, \bibinfo{person}{Runkai Zheng}, \bibinfo{person}{Yidong Wang}, \bibinfo{person}{Linyi Yang}, \bibinfo{person}{Haojun Huang}, \bibinfo{person}{Wei Ye}, \bibinfo{person}{Xiubo Geng}, {et~al\mbox{.}}} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{On the robustness of chatgpt: An adversarial and out-of-distribution perspective}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2302.12095}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Wang et~al\mbox{.}(2024)]%
{PMG}
\bibfield{author}{\bibinfo{person}{Sibo Wang}, \bibinfo{person}{Jie Zhang}, \bibinfo{person}{Zheng Yuan}, {and} \bibinfo{person}{Shiguang Shan}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Pre-trained Model Guided Fine-Tuning for Zero-Shot Adversarial Robustness}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2401.04350}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Wang et~al\mbox{.}(2020)]%
{MHE5}
\bibfield{author}{\bibinfo{person}{Zhennan Wang}, \bibinfo{person}{Canqun Xiang}, \bibinfo{person}{Wenbin Zou}, {and} \bibinfo{person}{Chen Xu}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{MMA regularization: Decorrelating weights of neural networks by maximizing the minimal angles}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} \bibinfo{volume}{33} (\bibinfo{year}{2020}), \bibinfo{pages}{19099--19110}.
\newblock
\bibitem[Xiao et~al\mbox{.}(2010)]%
{SUN397}
\bibfield{author}{\bibinfo{person}{Jianxiong Xiao}, \bibinfo{person}{James Hays}, \bibinfo{person}{Krista~A Ehinger}, \bibinfo{person}{Aude Oliva}, {and} \bibinfo{person}{Antonio Torralba}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{Sun database: Large-scale scene recognition from abbey to zoo}. In \bibinfo{booktitle}{\emph{2010 IEEE computer society conference on computer vision and pattern recognition}}. IEEE, \bibinfo{pages}{3485--3492}.
\newblock
\bibitem[Yang et~al\mbox{.}(2023)]%
{VLM-distillation6}
\bibfield{author}{\bibinfo{person}{Chuanguang Yang}, \bibinfo{person}{Zhulin An}, \bibinfo{person}{Libo Huang}, \bibinfo{person}{Junyu Bi}, \bibinfo{person}{Xinqiang Yu}, \bibinfo{person}{Han Yang}, {and} \bibinfo{person}{Yongjun Xu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{CLIP-KD: An Empirical Study of Distilling CLIP Models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2307.12732}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Yang and Mirzasoleiman(2023)]%
{Robust-CLIP}
\bibfield{author}{\bibinfo{person}{Wenhan Yang} {and} \bibinfo{person}{Baharan Mirzasoleiman}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Robust Contrastive Language-Image Pretraining against Adversarial Attacks}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2303.06854}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Zang et~al\mbox{.}(2023)]%
{VLM-distillation5}
\bibfield{author}{\bibinfo{person}{Yuhang Zang}, \bibinfo{person}{Hanlin Goh}, \bibinfo{person}{Joshua~M Susskind}, {and} \bibinfo{person}{Chen Huang}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Overcoming the Pitfalls of Vision-Language Model Finetuning for OOD Generalization}. In \bibinfo{booktitle}{\emph{The Twelfth International Conference on Learning Representations}}.
\newblock
\bibitem[Zhao et~al\mbox{.}(2023)]%
{LVL-AR}
\bibfield{author}{\bibinfo{person}{Yunqing Zhao}, \bibinfo{person}{Tianyu Pang}, \bibinfo{person}{Chao Du}, \bibinfo{person}{Xiao Yang}, \bibinfo{person}{Chongxuan Li}, \bibinfo{person}{Ngai-Man Cheung}, {and} \bibinfo{person}{Min Lin}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{On evaluating adversarial robustness of large vision-language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2305.16934}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Zhou et~al\mbox{.}(2024)]%
{VLM-distillation3}
\bibfield{author}{\bibinfo{person}{Andy Zhou}, \bibinfo{person}{Jindong Wang}, \bibinfo{person}{Yu-Xiong Wang}, {and} \bibinfo{person}{Haohan Wang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Distilling Out-of-Distribution Robustness from Vision-Language Foundation Models}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} \bibinfo{volume}{36} (\bibinfo{year}{2024}).
\newblock
\bibitem[Zhou and Patel(2022)]%
{inter-class5}
\bibfield{author}{\bibinfo{person}{Mo Zhou} {and} \bibinfo{person}{Vishal~M Patel}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Enhancing adversarial robustness for deep metric learning}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}}. \bibinfo{pages}{15325--15334}.
\newblock
\end{thebibliography}