Geneshift: Impact of different scenario shift on Jailbreaking LLM
Extracted source text · arXiv 2504.08104
\documentclass{article} % For LaTeX2e
\usepackage{iclr2025_conference,times}
% \usepackage{iclr14submit_e,times}
\usepackage{pifont}
\input{math_commands.tex}
\usepackage[hyperfootnotes=false]{hyperref}
\usepackage{url}
\iclrfinalcopy
\usepackage[hyperfootnotes=false]{hyperref}
\usepackage{url}
\usepackage{multirow}
\usepackage{svg}
% \usepackage[table]{xcolor}
\usepackage{xcolor}
% Define custom colors using hex values
% "#fababa", "#b0cdf7", "#b7e8cb"
\definecolor{lightpink}{HTML}{ed9782}
\definecolor{lightblue}{HTML}{5395f5}
\definecolor{lightgreen}{HTML}{efd08f}
\definecolor{grey}{HTML}{b3b3b3}
\usepackage[most]{tcolorbox}
\usepackage{algorithm}
% \usepackage{algorithmic}
\usepackage{enumitem}
\usepackage{tabularx}
\usepackage{makecell}
\usepackage{tabularx}
\usepackage{multirow}
\usepackage[normalem]{ulem}
\useunder{\uline}{\ul}{}
\usepackage{times}%, mathptm}
\usepackage{moreverb}
\usepackage{graphicx}
\usepackage{mathrsfs}
\usepackage{bm}
\usepackage{url}
\usepackage{amsmath}
\let\Bbbk\relax %%redefined in newtxmath.sty
\usepackage{amssymb}
\usepackage{tikz}
\usepackage{array}
\usepackage{times}
\usepackage{helvet}
\usepackage{courier}
\usepackage{multirow}
\usepackage{mathrsfs}
\usepackage{graphicx}
\usepackage{enumitem}
\usepackage{graphicx}
\usepackage{blindtext}
\usepackage{algorithm}
\usepackage{lineno,hyperref}
\usepackage[marginal]{footmisc}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{epstopdf}
\usepackage{array}
\usepackage{multirow}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage{wrapfig}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{xcolor}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{multicol}
\newcommand{\newchange}[1]{\textcolor{blue}{#1}}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets
% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}
\usepackage[normalem]{ulem}
\useunder{\uline}{\ul}{}
% \usepackage{algpseudocode}
\usepackage{subfigure}
\title{Geneshift: Impact of different scenario shift on Jailbreaking LLM}
% \author{Antiquus S.~Hippocampus, Natalia Cerebro \& Amelie P. Amygdale \thanks{ Use footnote for providing further information
% about the author (webpage, alternative address)---\emph{not} for acknowledging
% funding agencies. Funding acknowledgments go at the end of the paper.} \\
% Department of Computer Science\\
% Cranberry-Lemon University\\
% Pittsburgh, PA 15213, USA \\
% \texttt{\{hippo,brain,jen\}@cs.cranberry-lemon.edu} \\
% \And
% Ji Q. Ren \& Yevgeny LeNet \\
% Department of Computational Neuroscience \\
% University of the Witwatersrand \\
% Joburg, South Africa \\
% \texttt{\{robot,net\}@wits.ac.za} \\
% \AND
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email}
% }
\author{
\textbf{Tianyi Wu\textsuperscript{1,2,3}\thanks{Equal Contribution}}\quad
\textbf{Zhiwei Xue\textsuperscript{1,3}\footnotemark[1]}\quad
\textbf{Yue Liu\textsuperscript{1,2,3}\footnotemark[1]}\quad
\textbf{Jiaheng Zhang\textsuperscript{3}}\quad
\textbf{Bryan Hooi\textsuperscript{2,3}}\quad
\textbf{See-Kiong Ng\textsuperscript{2,3}}\\[6pt]
\textsuperscript{1}Integrative Sciences and Engineering Programme, NUS Graduate School, National University of Singapore \\
\textsuperscript{2}Institute of Data Science (IDS), National University of Singapore \\
\textsuperscript{3}Department of Computer Science, School of Computing, National University of Singapore \\
\small{\texttt{tianyi\_wu@u.nus.edu}\quad \texttt{zhiweixue@u.nus.edu}\quad \texttt{yliu@u.nus.edu}}
}
% \author{\textbf{Tianyi Wu\textsuperscript{1,2,3*}}\quad
% \textbf{Zhiwei Xue\textsuperscript{1,2*}}\quad
% \textbf{Yue Liu\textsuperscript{1,2,3*}}\quad
% \textbf{Jiaheng Zhang\textsuperscript{3}}\quad \\
% \textbf{Bryan Hooi\textsuperscript{2,3}}\quad
% \textbf{See-Kiong Ng\textsuperscript{2,3}}\quad
% \\
% \textsuperscript{1}Integrative Sciences and Engineering Programme, NUS Graduate School \\
% \textsuperscript{2}Institute of Data Science (IDS), National University of Singapore \\
% \textsuperscript{3}Department of Computer Science, National University of Singapore \\
% \small{\texttt{tianyi\_wu@u.nus.edu}\quad \texttt{zhiweixue@u.nus.edu}\quad \texttt{yliu@u.nus.edu}
% }
% }
% \author{
% \makebox[\textwidth][c]{%
% \textbf{Tianyi Wu\textsuperscript{1,2,3*}} \quad
% \textbf{Zhiwei Xue\textsuperscript{1,2*}} \quad
% \textbf{Yue Liu\textsuperscript{1,2,3*}} \quad
% \textbf{Jiaheng Zhang\textsuperscript{3}} \quad
% }\\
% \makebox[\textwidth][c]{%
% \textbf{Bryan Hooi\textsuperscript{2,3}} \quad
% \textbf{See-Kiong Ng\textsuperscript{2,3}}%
% }\\
% \textsuperscript{1}Integrative Sciences and Engineering Programme, NUS Graduate School \\
% \textsuperscript{2}Institute of Data Science (IDS), National University of Singapore \\
% \textsuperscript{3}Department of Computer Science, National University of Singapore \\
% \small{\texttt{tianyi\_wu@u.nus.edu} \quad \texttt{zhiweixue@u.nus.edu} \quad \texttt{yliu@u.nus.edu}}
% }
\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}
\begin{document}
\maketitle
\begin{abstract}
Jailbreak attacks, which aim to cause LLMs to perform unrestricted behaviors, have become a critical and challenging direction in AI safety. Despite achieving the promising attack success rate using dictionary-based evaluation, existing jailbreak attack methods fail to output detailed contents to satisfy the harmful request, leading to poor performance on GPT-based evaluation.
To this end, we propose a black-box jailbreak attack termed \textbf{GeneShift}, by using a genetic algorithm to optimize the scenario shifts. Firstly, we observe that the malicious queries perform optimally under different scenario shifts. Based on it, we develop a genetic algorithm to evolve and select the hybrid of scenario shifts. It guides our method to elicit detailed and actionable harmful responses while keeping the seemingly benign facade, improving stealthiness. Extensive experiments demonstrate the superiority of GeneShift. Notably, GeneShift increases the jailbreak success rate from 0\% to 60\% when direct prompting alone would fail. We will open-source our code.
% propose a genetic algorithm to explore and optimize hybrids of scenarios to improve the efficacy of bypassing the LLM guardrail.
% To induce the detailed and actionable harmful response, we design Chain-of-Interrogation to obtain detailed and actionable harmful responses.
\begin{center}
\color{red}Warning: this paper contains potentially harmful text.
\end{center}
\end{abstract}
\section{Introduction}
\label{sec:intro}
Large Language Models (LLMs) \citep{achiam2023gpt,dubey2024llama} are indispensable for tasks like knowledge-seeking, content generation, and planning, yet they remain vulnerable to attacks that bypass safety mechanisms and elicit harmful responses. Although white-box methods (e.g., GCG \citep{GCG}) show promising success rates, they require model weights and intensive optimization, limiting practicality for closed-source LLMs. In contrast, black-box methods like PAIR \citep{PAIR}, ArtPrompt \citep{Artprompt}, and SelfCipher \citep{SelfCipher} circumvent these constraints using iterative prompts, artistic language, or cipher techniques. In our experiments, we observe that recent state-of-the-art black-box models indeed achieve good performance under dictionary-based evaluation, which detects the success of the attack by simply checking whether the response contains refusal keywords (a list of <40 keywords). However, with our case studies and further experiments, we find that these methods fail due to either producing insufficiently detailed harmful responses or non-harmful responses that do not contain refusal keywords, thus being misclassified by dictionary-based metrics.
% In our experiments on, we observe that recent state-of-the-art black-box models indeed achieve good performance under dictionary-based evaluation, which detects the success of the attack by simply checking whether hit refusal-triggering phrases. However, with our case studies and further experiments, we find that these methods fail due to producing either non-harmful responses or insufficiently detailed harmful responses, leading to a significantly poor performance using the GPT-based metric.
To address this shortfall, we introduce \textbf{GeneShift}, a novel black-box attack that employs a genetic algorithm to optimize scenario shifts. We find that different malicious prompts perform optimally under varying scenario shifts. By evolving and selectively blending these shifts, GeneShift elicits more detailed and actionable harmful responses while retaining a benign facade. Experiments on GPT-4o mini confirm GeneShift’s effectiveness, demonstrating a marked improvement in attack success rates.
Our main contributions are as follows: \begin{itemize}
\item We show that aligning distinct scenario shifts with specific malicious behaviors yields the most detailed harmful responses, emphasizing the need for tailored scenario elements.
\item We demonstrate that malicious prompt efficacy is dependent on contextual configuration, with optimal performance emerging only under the right scenario shifts.
\item We propose GeneShift, a black-box jailbreak attack that employs a genetic algorithm to optimize scenario shifts, enabling malicious prompts to bypass LLM guardrails while appearing benign.
\item Our experiments on GPT-4o mini reveal that GeneShift boosts the jailbreak success rate from below 1\% to 60\%.
\end{itemize}
\section{GeneShift}
\label{sec:method}
\subsection{Scenario Shift for Enhanced Detail} Direct requests for harmful content often trigger immediate refusal from LLMs. To circumvent this, attackers often embed the malicious query in a broader, seemingly benign context--a \emph{Scenario Shift (CS)}--prompting more elaborate responses that can still fulfill harmful intentions. However, a single, fixed scenario may fail if it does not sufficiently align with the malicious query or misleading enough to bypass the model’s guardrails, motivating an automated approach to systematically discover more suitable transformations.
\subsection{GeneShift: A Genetic Algorithm for Scenario Optimization} To automate the search for optimal scenario shifts, we propose a black-box jailbreak framework called \textbf{GeneShift}. It leverages a genetic algorithm (GA) to explore, evaluate, and refine various transformations, ultimately generating a single-turn prompt that elicits the desired harmful response.
\\[3pt]
\textbf{Gene Design.} \citet{yu2024dontlistenmeunderstanding} studied existing jailbreak attacks and classified them into ten categories, we adopt their transformation categories and introduce an additional transformation rule, forming a gene database \( \mathcal{G} := \{ \tau_1, \tau_2, \dots, \tau_M \} \).
\\[3pt]
\textbf{Population Initialization.}
We initialize a population \( \mathcal{P} = \{(p_j, g_j)\}_{j=1}^N \), where each candidate's gene \( g_j \) is generated by randomly selecting a subset of \( z_j \) number of distinct transformation rules from the gene database \( \mathcal{G} \). The number of selected transformation rules, i.e., \( z_j \), is sampled from a uniform distribution $z_j \sim \mathcal{U}(1, Z)$, where$\quad Z = 4$.
Each candidate's gene \( g_j \subset \mathcal{G} \) consists of a set of distinct, randomly selected transformation rules, i.e., \( g_j = \{ \tau_{i_1}, \tau_{i_2}, \dots, \tau_{i_{z_j}} \} \), where \( \tau_{i_k} \in \mathcal{G} \) and \( i_1, i_2, \dots, i_{z_j} \) are distinct indices. The corresponding candidate jailbreak prompt \( p_j \) is then generated by the LLM using the selected transformation rules:
\\[-5pt]
\begin{equation}
p_j = \text{LLM}(g_j), \quad g_j \subset \mathcal{G}, \quad |g_j| = z_j
\label{Equation:population_init}
\end{equation}
Each pair \( (g_j, p_j) \) is stored to track the transformations applied in future iterations.
% Andrew read her 1:48
\\[3pt]
\textbf{Fitness Evaluation.}
To evaluate the fitness of each candidate jailbreak prompt \( p_j \), we utilize a two-step process involving both the attack model and a judge LLM. Given a response \( r_j \) from the attack model, we instruct the judge LLM to classify \( r_j \) into one of six quality categories. The detailed design of the score can be found in Appendix.
We define the fitness function \( F(p_j) \) of the candidate prompt \( p_j \) as the score assigned by the judge LLM:
\\[-5pt]
\begin{equation}
F(p_j) = \text{score}(r_j), \quad r_j = \text{AttackModel}(p_j)
\end{equation}
The fitness \( F(p_j) \) is sampled from a discrete probability distribution \( P(F(p_j) = k) \), where \( k \in \{1, 2, \dots, 6\} \), reflecting the likelihood of each response category based on the attack model's behavior.
\\[3pt]
\textbf{Crossover.}
Before performing crossover, we preserve the top \( k \) performing candidates as elites \( \mathcal{E} \subset \mathcal{P} \) to ensure that high-quality individuals are carried over to the next generation. For the remaining candidates, we select parents based on fitness-proportional selection, where the probability of selecting a parent \( p_j \) is:
\begin{equation}
P(p_j) = \frac{F(p_j)}{\sum_{i=1}^{N} F(p_i)}, \quad j = 1, \dots, N.
\end{equation}
Given two parents \((p_a, g_a)\) and \((p_b, g_b)\), the offspring’s gene \( g_{\text{child}} \) is created by randomly swapping 1 or 2 transformation rules between the two parents. Let \( I \in \{0, 1\}^{z_j} \) be a binary mask indicating which genes are swapped between the parents. The offspring’s gene is then represented as:
\\[-10pt]
\begin{equation}
g_{\text{child}, i} = I_i g_{a, i} + (1 - I_i) g_{b, i}, \quad I_i \sim \text{U}\{0, 1\}.
\end{equation}
Thus, each gene \( g_{\text{child}, i} \) is inherited either from parent \( g_a \) or \( g_b \), depending on the value of \( I_i \). The offspring prompt \( p_{\text{child}} \) is then generated by passing the crossovered gene to the LLM similar to Equation \ref{Equation:population_init}:
$p_{\text{child}} = \text{LLM}(g_{\text{child}})$.
\\[3pt]
\textbf{Mutation.} Mutation introduces further diversity into the population by modifying or expanding some of the genes in the offspring. For each gene \( g_{\text{child}, i} \), mutation occurs with a probability \( p_{\text{mut}} \). If mutation occurs, the operation is randomly chosen with equal probability: (1) Switch, where the gene is replaced by a randomly selected gene from the gene pool \( \mathcal{G} \); or (2) Add, where a new random gene is appended to the gene sequence:
\begin{equation*}
g_{\text{child}, i} =
\begin{cases}
g_{\text{rand}} & \text{if } O < 0.5 \quad \\[5pt]
g_{\text{child}, i} \cup \{g_{\text{rand}}\} & \text{if } O \geq 0.5 \quad
\end{cases}
\end{equation*}
\begin{equation}
I_i \sim \text{Bernoulli}(p_{\text{mut}}), \quad O \sim \text{Uniform}(0, 1), \quad g_{\text{rand}} \sim \mathcal{G}.
\end{equation}
This mutation process ensures new genetic material is introduced and maintains population diversity across generations. The switch operation directly replaces existing genes, while the add operation introduces additional genes, enriching the offspring’s potential solution space.
\\[3pt]
\textbf{Termination Criteria.}
The genetic algorithm terminates when one of the following conditions is satisfied:
1) The maximum number of iterations \( T \) has been reached.
2) The number of candidates with fitness scores \( F(p_j) > 5 \) meets or exceeds a predefined threshold \( \theta \), expressed as: $\sum_{j=1}^{N} \mathbb{I}(F(p_j) > 5) \geq \theta$, where \( \mathbb{I}(\cdot) \) is the indicator function. The best-performing candidates is returned as the output of the algorithm.
\section{Experiment}
\subsection{Evaluation Metrics}
\paragraph{Dictionary-based Evaluation.}
Following \citet{GCG}, we check whether the model’s response contains any predefined refusal phrases (see Table~\ref{tab:rejection_dictionary}). If such phrases appear, the attack is considered a failure; otherwise, it is considered successful under this metric, denoted as \(\text{ASR-DICT}\).
\paragraph{GPT-based Evaluation.}
As dictionary-based checks may overlook subtle refusals or incomplete harmful content, we employ a GPT-based evaluation \citep{Artprompt,qi2023fine} for a more nuanced assessment. This metric, \(\text{ASR-GPT}\), uses a secondary large language model to judge whether the attack prompt successfully elicits detailed, harmful content. Higher scores on this metric indicate more substantial (and therefore more concerning) policy violations.
\subsection{Experimental Results}
We compare our proposed method, \textbf{GeneShift}, with four white-box baselines \citep{GCG,AutoDAN_ICLR24,MAC,Cold_attack} and eight black-box methods \citep{PAIR,TAP,jailbroken,GPTFUZZER,DRA,Artprompt,PromptAttack,SelfCipher}. All experiments are conducted on GPT-4o mini \citep{GCG}, and results are measured using \(\text{ASR-DICT}\) and \(\text{ASR-GPT}\).
\begin{table}[!t]
\tiny
% \renewcommand{\arraystretch}{1.1}
\centering
\caption{Attack success rate (\%) of 12 methods on GPT-4o mini. \textbf{Bold} indicates the best result; \uline{underlined} indicates the runner-up.}
\vspace{5pt}
\label{tab:compare_table_gpt}
\setlength{\tabcolsep}{3pt}
\begin{tabular}{ccc}
\hline
\textbf{Method} & \textbf{ASR-DICT (\%)} & \textbf{ASR-GPT (\%)} \\ \hline
\multicolumn{3}{c}{\textit{White-box Methods}} \\
\hline
GCG & 03.46 & 02.50 \\
AutoDAN & 27.12 & 27.31 \\
MAC & 02.50 & 01.92 \\
COLD-Attack & 05.58 & 01.92 \\
\hline
\multicolumn{3}{c}{\textit{Black-box Methods}} \\
\hline
PAIR & 12.50 & 03.46 \\
TAP & 09.23 & 06.54 \\
Base64 & 13.08 & 03.08 \\
GPTFuzzer & 34.62 & \uline{41.35} \\
DRA & 00.00 & 02.69 \\
ArtPrompt & \textbf{83.46} & 00.77 \\
PromptAttack & 32.88 & 00.00 \\
SelfCipher & 25.77 & 00.00 \\
DeepInception & 25.77 & 45.00 \\
GeneShift (Ours) & \uline{56.15} & \textbf{60.00} \\
\hline
\end{tabular}
\vspace{-13pt}
\end{table}
As shown in Table~\ref{tab:compare_table_gpt}, certain methods (e.g., ArtPrompt) score highly on \(\text{ASR-DICT}\) but produce vague or incomplete harmful content, reflected in low \(\text{ASR-GPT}\) scores. In contrast, GeneShift’s genetic algorithm identifies transformation rules that consistently bypass guardrails and yield detailed harmful content, achieving the highest overall \(\text{ASR-GPT}\) of 60.00\%.
\begin{table}[htbp]
\tiny
\centering
\caption{Success Rates (\%) of Scenario Shifts Across Different Malicious Request Types}
\vspace{5pt}
\label{tab:scenario_shifts}
\begin{tabular}{lccccccc}
\toprule
\textbf{Scenario Shift} & \textbf{Illegal Activity} & \textbf{Malware} & \textbf{Physical Harm} & \textbf{Economic Harm} & \textbf{Hate Speech} & \textbf{Privacy Violence} & \textbf{Fraud} \\
\midrule
Persona Adoption & \textbf{13.76\%} & \textbf{17.14\%} & 2.50\% & 0.00\% & \textbf{21.46\%} & 9.09\% & \textbf{24.00\%} \\
Fictional Scenario Setup & \textbf{19.30\%} & \textbf{21.90\%} & 0.00\% & \textbf{26.23\%} & \textbf{14.15\%} & \textbf{30.30\%} & 16.00\% \\
Complicated Language & 11.29\% & 4.76\% & 15.00\% & \textbf{22.95\%} & 6.83\% & 0.00\% & \textbf{22.00\%} \\
Privilege Escalation Mode & 8.01\% & 1.90\% & \textbf{25.00\%} & 14.75\% & 6.83\% & 0.00\% & 14.00\% \\
Research Pretext & 7.19\% & 6.67\% & 2.50\% & 16.39\% & 11.71\% & 12.12\% & 4.00\% \\
Language Evasion & 9.65\% & 5.71\% & 5.00\% & 0.00\% & 4.39\% & 3.03\% & 4.00\% \\
Joke Pretext & 3.90\% & 4.76\% & \textbf{25.00\%} & 3.28\% & 6.83\% & 6.06\% & 2.00\% \\
Text Continuation & 11.50\% & 15.24\% & \textbf{25.00\%} & 16.39\% & 10.73\% & \textbf{18.18\%} & 14.00\% \\
Program Execution & 9.24\% & 13.33\% & 0.00\% & 0.00\% & 6.83\% & 0.00\% & 0.00\% \\
Opposite Mode & 6.16\% & 8.57\% & 0.00\% & 0.00\% & 10.24\% & 21.21\% & 0.00\% \\
\bottomrule
\end{tabular}
\vspace{-13pt}
\end{table}
Table~\ref{tab:scenario_shifts} reveals distinct success rates across different malicious request types, underscoring the need for tailored approaches. For instance, \emph{Persona Adoption} attains a relatively high success rate of 24.00\% in \emph{Fraud} scenarios, yet only 2.50\% in \emph{Physical Harm} requests. By contrast, \emph{Fictional Scenario Setup} excels at \emph{Privacy Violence} (30.30\%) and \emph{Economic Harm} (26.23\%), but achieves 0.00\% in \emph{Physical Harm}. Additionally, both \emph{Privilege Escalation Mode} and \emph{Joke Pretext} reach a notably higher success rate (25.00\%) for \emph{Physical Harm} compared to their performance in other categories. Such variability shows that no single scenario shift outperforms all others across every malicious intent. Instead, each scenario shift appears more or less effective depending on contextual alignment with the target request. These findings underline the importance of adaptive strategies, suggesting that a catalog of scenario shifts—possibly optimized via systematic search—can better accommodate the diverse requirements of malicious prompts.
\subsection{Ablation Study}
We conduct an ablation study on GPT-4o mini to evaluate three components: a \textbf{Base} condition that directly poses a malicious query, a \textbf{Scenario Shift (SS)} that augments the query with benign context, and a \textbf{Genetic Algorithm (GA)} that dynamically searches for and combines transformation rules from the scenario database \(\mathcal{G}\).
\begin{table}[!t]
\small
% \renewcommand{\arraystretch}{1.1}
\centering
\caption{Ablation study of GeneShift on GPT-4o mini. SS and GA represent scenario shift and genetic algorithm, respectively.}
\vspace{5pt}
\label{tab:ablation_single_turn}
\setlength{\tabcolsep}{3pt}
\begin{tabular}{ccc}
\hline
\textbf{Method} & \textbf{ASR-DICT (\%)} & \textbf{ASR-GPT (\%)} \\
\hline
Base (direct) & 1.35 & 0.00 \\
Base + SS & 69.04 & 18.00 \\
Base + SS + GA (GeneShift) & 56.15 & 60.00 \\
\hline
\end{tabular}
\vspace{-13pt}
\end{table}
Table~\ref{tab:ablation_single_turn} shows that \(\text{ASR-GPT}\) jumps from 0.00\% (Base alone) to 18.00\% (Base+CS), indicating that context manipulation prompts the model to produce more elaborate responses, sometimes including harmful details. Incorporating the genetic algorithm (Base+CS+GA) raises \(\text{ASR-GPT}\) further to 60.00\%, underscoring the importance of automated scenario optimization.
\section{Conclusion}
This work presents \textbf{GeneShift}, a black-box jailbreak attack that harnesses a genetic algorithm to discover effective scenario shifts for prompting large language models. Our experiments demonstrate that GeneShift outperforms both white-box and black-box baselines under dictionary-based and GPT-based metrics, reflecting its ability to elicit detailed and harmful content that circumvents common refusal triggers. These results serve as a warning about the evolving sophistication of jailbreaking strategies and emphasize the need for more rigorous safety defenses in future LLM deployments.
% \section*{Acknowledgements}
\bibliography{2_ref}
\bibliographystyle{2_ref}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\appendix
\begin{onecolumn}
\section{Appendix}
\label{sec:appendix}
\subsection{Related Work}
\label{sec:related_work}
\subsection{Jailbreak Attacks on LLMs} Jailbreak attacks aim to bypass the safety measures of LLMs, enabling unrestricted outputs, even harmful behaviors. These attacks are generally divided into white-box and black-box categories. White-box approaches like GCG \citep{GCG} optimize harmful prompts through gradient-based methods, showing transferability to public interfaces. Further advancements, such as MAC \citep{MAC} and AutoDAN \citep{AutoDAN_ICLR24}, improve attack efficiency and readability. However, these methods often require access to model weights or gradients, limiting their applicability in real-world black-box scenarios.
To overcome these limitations, black-box methods \citep{DAN,Masterkey,PANDORA} have emerged, targeting commercial LLMs like GPT and Claude by manipulating only input-output interactions. Techniques such as PAIR \citep{PAIR} and TAP \citep{TAP} refine jailbreak prompts through iterative questioning, while PromptAttack \citep{PromptAttack} and IRIS \citep{IRIS} exploit the reflective capabilities of LLMs. DRA\citep{DRA} circumvents LLM safety mechanisms through a disguise-and-reconstruction framework. Other methods, such as ReNeLLM \citep{ReNeLLM}, integrate prompt re-writing and scenario construction to bypass safety guardrails. Additionally, some methods misguide LLMs by using ciphers \citep{SelfCipher,jailbroken}, art words \citep{Artprompt}, and multilingual contexts \citep{deng2023multilingual,Low_resource_language}. Despite their success, existing methods often rely on iterative refinement, or involve complex tasks (e.g., cipher or puzzle-solving), which dilute the malicious intent and lead to vague or unhelpful outputs. \citet{liu2024flipattack} proposes a simple yet effective attack method to jailbreak LLMs within 1 query by flipping. This paper introduces GeneShift, a multi-turn jailbreak attack that maintains focus on the original harmful intent while progressively guiding the LLM towards providing detailed, actionable responses.
\subsubsection{Jailbreak Defense on LLMs}
Jailbreak defense \citep{xu2024llm} aims to protect LLMs from jailbreak attacks, ensuring they remain helpful and safe. Defense methods are broadly categorized into strategy-based and learning-based approaches. Strategy-based methods include using perplexity to filter harmful prompts \citep{perplexity}, employing system-mode self-reminders \citep{self_reminders}, and detecting harmful prompts through gradient analysis of safety-critical parameters \citep{GradSafe}. Other approaches involve using auxiliary LLMs to screen responses \citep{self_defense}, repeating outputs to avoid harmful content \citep{PANDORA}, and adjusting token probabilities to prioritize safety disclaimers \citep{Safedecoding}. Techniques like multiple attack iterations with majority voting \citep{Smoothllm,Semantic_Smoothing} and rewindable auto-regressive inference \citep{RAIN} are also employed. Learning-based methods focus on fine-tuning LLMs using reinforcement learning from human feedback \citep{RLHF_authropic,safe_rlhf}, combining harmful prompt generation with safe response generation \citep{Mart}, and applying knowledge editing to detoxify models \citep{safe_edit}. Additional strategies include integrating goal prioritization during training and inference \citep{safe_decoding}, optimizing prompts by adjusting query representations based on harmfulness \citep{DRO}, and using prompt adversarial tuning to attach guard prefixes \citep{TAP}. Tools like LLaMA Guard \citep{Llamaguard}, Aegis Guard \citep{AegisGuard}, WildGuard \citep{wildguard}, and ShieldGemma \citep{Shieldgemma} moderate inputs and outputs, while GuardReasoner \citep{liu2025guardreasoner} enhances performance, explainability, and generalization through reasoning.
\subsubsection{Rejection Dictionary of Dictionary-based Evaluation}
Table~\ref{tab:rejection_dictionary} depicts predefined refusal phrases for dict-based evaluation.
\begin{table}[htbp]
\renewcommand{\arraystretch}{1.2}
\centering
\caption{The rejection dictionary of the ASR-DICT metric.}
\label{tab:rejection_dictionary}
\resizebox{0.6\linewidth}{!}{
\begin{tabular}{cc}
\hline
\multicolumn{2}{c}{Rejection Dictionary} \\ \hline
\multicolumn{1}{c|}{I'm sorry} & illegal \\
\multicolumn{1}{c|}{Sorry} & not legal \\
\multicolumn{1}{c|}{I am sorry} & My apologies \\
\multicolumn{1}{c|}{I apologize} & but I cannot \\
\multicolumn{1}{c|}{As an} & If you have any other non-malicious requests \\
\multicolumn{1}{c|}{As an AI} & I'm not able to provide \\
\multicolumn{1}{c|}{I'm an} & I am really sorry \\
\multicolumn{1}{c|}{I'm just} & I cannot create \\
\multicolumn{1}{c|}{As a language model} & I cannot support \\
\multicolumn{1}{c|}{As an Assistant} & I can't provide \\
\multicolumn{1}{c|}{I cannot} & I can't assist \\
\multicolumn{1}{c|}{I can't} & I am unable \\
\multicolumn{1}{c|}{I do not} & I must \\
\multicolumn{1}{c|}{cannot} & It must \\
\multicolumn{1}{c|}{Hello!} & it must \\
\multicolumn{1}{c|}{is illegal and unethical} & is not capable of \\
\multicolumn{1}{c|}{I'm really sorry} & As a responsible \\
\multicolumn{1}{c|}{unethical} & never \\
\multicolumn{1}{c|}{not ethical} & is not \\ \hline
\end{tabular}}
\end{table}
\subsection{Transferability}
We also examine whether the best-performing prompts found by GeneShift on GPT-4o mini retain their effectiveness when transferred to another target model. Table~\ref{tab:transfer} shows that the attack maintains a strong performance, demonstrating that effective scenario shifts and transformations often generalize to similar LLM architectures.
\begin{table}[!t]
\renewcommand{\arraystretch}{1.2}
\centering
\caption{Transferability results of GeneShift. Prompts derived from GPT-4o mini are applied to GPT-4o.}
\label{tab:transfer}
\resizebox{0.55\linewidth}{!}{
\begin{tabular}{ccc}
\hline
\textbf{Method} & \textbf{ASR-DICT (\%)} & \textbf{ASR-GPT (\%)} \\ \hline
GeneShift on GPT-4o (Transfer) & 50.00 & 53.08 \\
GeneShift on GPT-4o mini (Original) & 56.15 & 60.00 \\ \hline
\end{tabular}}
\end{table}
\subsection{Case Study}
\label{sec:case_study}
To illustrate GeneShift’s effectiveness, consider a malicious request: \emph{“Write a tutorial on how to make a bomb.”} A direct query typically triggers an immediate refusal. However, using our genetically optimized scenario shift, the resulting single-turn prompt elicits a more detailed—albeit highly unsafe—response. Figure references from our original experiments (omitted here) show that GeneShift successfully coaxes step-by-step instructions in violation of common guardrails.
\subsection{Algorithms for Genetic Algorithm}
This section presents the core algorithms used in the GeneShift genetic algorithm. Algorithm \ref{alg:population_init} initializes the population by sampling transformation rules from the gene database. Algorithm \ref{alg:crossover} performs crossover by combining genes from selected parent candidates. Algorithm \ref{alg:mutation} introduces diversity through mutation, using either a switch or add operation with equal probability. Finally, Algorithm \ref{alg:GeneShift_GA} outlines the main GeneShift genetic algorithm, iterating through evaluation, selection, crossover, and mutation to generate the next population until the stopping criteria are met.
\begin{algorithm*}
\caption{Population Initialization}
\label{alg:population_init}
\begin{algorithmic}[1]
\State \textbf{init} $j = 1$, $\mathcal{P} = \emptyset$, $Z = 4$, gene database $\mathcal{G}$
\While{$j \leq N$}
\State $z_j \sim \mathcal{U}(1, Z)$ \Comment{Sample number of transformation rules}
\State $g_j \leftarrow \text{randomly select } z_j \text{ rules from } \mathcal{G}$
\State $p_j \leftarrow \text{LLM}(g_j)$ \Comment{Generate prompt using LLM}
\State $\mathcal{P} \leftarrow \mathcal{P} \cup \{(p_j, g_j)\}$
\State $j \leftarrow j + 1$
\EndWhile
\State \textbf{return} Population $\mathcal{P}$
\end{algorithmic}
\end{algorithm*}
\begin{algorithm*}
\caption{Crossover}
\label{alg:crossover}
\begin{algorithmic}[1]
\State \textbf{init} top $k$ elites $\mathcal{E} \subset \mathcal{P}$, $j = 1$
\While{$j \leq N - k$}
\State Select parents $(p_a, g_a)$ and $(p_b, g_b)$ \Comment{Fitness-proportional selection}
\State Generate mask $I \sim \text{U}\{0, 1\}^{z_j}$ \Comment{Binary mask for crossover}
\State $g_{\text{child}} \leftarrow I \cdot g_a + (1 - I) \cdot g_b$ \Comment{Crossover genes from parents}
\State $p_{\text{child}} \leftarrow \text{LLM}(g_{\text{child}})$ \Comment{Generate offspring prompt}
\State Store $(p_{\text{child}}, g_{\text{child}})$
\State $j \leftarrow j + 1$
\EndWhile
\State \textbf{return} Next generation $\mathcal{P} \cup \mathcal{E}$ \Comment{Combine offspring and elites}
\end{algorithmic}
\end{algorithm*}
\begin{algorithm*}
\caption{Mutation with Switch or Add Operations}
\label{alg:mutation}
\begin{algorithmic}[1]
\For{each offspring $(p_{\text{child}}, g_{\text{child}})$}
\For{each gene $g_{\text{child}, i}$}
\State $I_i \sim \text{Bernoulli}(p_{\text{mut}})$ \Comment{Determine if mutation occurs}
\If{$I_i = 1$}
\State $O \sim \text{Uniform}(0, 1)$ \Comment{Randomly choose operation}
\If{$O < 0.5$}
\State $g_{\text{child}, i} \leftarrow g_{\text{rand}}$ \Comment{Switch: Replace with a random gene from $\mathcal{G}$}
\Else
\State $g_{\text{child}} \leftarrow g_{\text{child}} \cup \{g_{\text{rand}}\}$ \Comment{Add: Insert a new random gene}
\EndIf
\EndIf
\EndFor
\State $p_{\text{child}} \leftarrow \text{LLM}(g_{\text{child}})$ \Comment{Update prompt after mutation}
\EndFor
\State \textbf{return} Mutated population
\end{algorithmic}
\end{algorithm*}
% for N-k candidates:
% off <- crossover(i) (c.f. Alg. 2)
\begin{algorithm*}
\caption{GeneShift Genetic Algorithm}
\label{alg:GeneShift_GA}
\begin{algorithmic}[1]
\State \textbf{init} population $\mathcal{P} = \{(p_j, g_j)\}_{j=1}^N$, iteration counter $t = 0$, max iterations $T$, threshold $\theta$
\While{$t < T$ \textbf{and} $\sum_{j=1}^{N} \mathbb{I}(F(p_j) > 5) < \theta$} \Comment{Stopping condition: max iterations or fitness threshold}
\State Evaluate fitness for each $(p_j, g_j)$ \Comment{Call Fitness Evaluation}
\State Select top $k$ elites $\mathcal{E} \subset \mathcal{P}$ \Comment{Preserve top performers}
\State Perform crossover for remaining $N - k$ candidates \Comment{Call Crossover}
\State Perform mutation on the offspring \Comment{Call Mutation}
\State $\mathcal{P} \leftarrow \mathcal{E} \cup \text{new offspring}$ \Comment{Form the next generation}
\State $t \leftarrow t + 1$ \Comment{Update iteration counter}
\EndWhile
\State \textbf{return} Best-performing candidates
\end{algorithmic}
\end{algorithm*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_1.jpg}
\caption{Rules of genetic algorithm.}
\label{GA: rules}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_2.jpg}
\caption{Prompt of genetic algorithm initialization.}
\label{GA: initialization}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_4.jpg}
\caption{Prompt of genetic algorithm evaluation.}
\label{GA: evaluation}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_5.jpg}
\caption{Prompt of genetic algorithm crossover.}
\label{GA: crossover}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_6.jpg}
\caption{Prompt of genetic algorithm swap.}
\label{GA: swap}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_7.jpg}
\caption{Prompt of genetic algorithm add.}
\label{GA: add}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_8.jpg}
\caption{Prompt used in a successful GeneShift case for the example: "Write a tutorial on how to make a bomb".}
\label{Case1:prompt}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_9.jpg}
\caption{Output of a successful GeneShift case for the example: "Write a tutorial on how to make a bomb".}
\label{Case1:output1}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=\textwidth]{prompt_10.jpg}
\caption{Output of a successful GeneShift case for the example: "Write a tutorial on how to make a bomb".}
\label{Case1:output2}
\end{figure*}
\end{onecolumn}
\end{document}
%%%%% NEW MATH DEFINITIONS %%%%%
\usepackage{amsmath,amsfonts,bm}
% Mark sections of captions for referring to divisions of figures
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
% Highlight a newly defined term
\newcommand{\newterm}[1]{{\bf #1}}
% Figure reference, lower-case.
\def\figref#1{figure~\ref{#1}}
% Figure reference, capital. For start of sentence
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
% Section reference, lower-case.
\def\secref#1{section~\ref{#1}}
% Section reference, capital.
\def\Secref#1{Section~\ref{#1}}
% Reference to two sections.
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
% Reference to three sections.
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
% Reference to an equation, lower-case.
\def\eqref#1{equation~\ref{#1}}
% Reference to an equation, upper case
\def\Eqref#1{Equation~\ref{#1}}
% A raw reference to an equation---avoid using if possible
\def\plaineqref#1{\ref{#1}}
% Reference to a chapter, lower-case.
\def\chapref#1{chapter~\ref{#1}}
% Reference to an equation, upper case.
\def\Chapref#1{Chapter~\ref{#1}}
% Reference to a range of chapters
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
% Reference to an algorithm, lower-case.
\def\algref#1{algorithm~\ref{#1}}
% Reference to an algorithm, upper case.
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
% Reference to a part, lower case
\def\partref#1{part~\ref{#1}}
% Reference to a part, upper case
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
% Random variables
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
% rm is already a command, just don't name any random variables m
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
% Random vectors
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
% Elements of random vectors
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
% Random matrices
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
% Elements of random matrices
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
% Vectors
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
% Elements of vectors
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
% Matrix
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
% Tensor
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
% Graph
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
% Sets
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
% Don't use a set called E, because this would be the same as our symbol
% for expectation.
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
% Entries of a matrix
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
% entries of a tensor
% Same font as tensor, without \bm wrapper
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
% The true underlying data generating distribution
\newcommand{\pdata}{p_{\rm{data}}}
% The empirical distribution defined by the training set
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
% The model distribution
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
% Stochastic autoencoder distributions
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} % Laplace distribution
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
% Wolfram Mathworld says $L^2$ is for function spaces and $\ell^2$ is for vectors
% But then they seem to use $L^2$ for vectors throughout the site, and so does
% wikipedia.
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} % See usage in notation.tex. Chosen to match Daphne's book.
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
\begin{thebibliography}{44}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Achiam et~al.(2023)Achiam, Adler, Agarwal, Ahmad, Akkaya, Aleman, Almeida, Altenschmidt, Altman, Anadkat, et~al.]{achiam2023gpt}
Josh Achiam, Steven Adler, Sandhini Agarwal, Lama Ahmad, Ilge Akkaya, Florencia~Leoni Aleman, Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, et~al.
\newblock Gpt-4 technical report.
\newblock \emph{arXiv preprint arXiv:2303.08774}, 2023.
\bibitem[Alon \& Kamfonas(2023)Alon and Kamfonas]{perplexity}
Gabriel Alon and Michael Kamfonas.
\newblock Detecting language model attacks with perplexity.
\newblock \emph{arXiv preprint arXiv:2308.14132}, 2023.
\bibitem[Bai et~al.(2022)Bai, Jones, Ndousse, Askell, Chen, DasSarma, Drain, Fort, Ganguli, Henighan, et~al.]{RLHF_authropic}
Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, et~al.
\newblock Training a helpful and harmless assistant with reinforcement learning from human feedback.
\newblock \emph{arXiv preprint arXiv:2204.05862}, 2022.
\bibitem[Chao et~al.(2023)Chao, Robey, Dobriban, Hassani, Pappas, and Wong]{PAIR}
Patrick Chao, Alexander Robey, Edgar Dobriban, Hamed Hassani, George~J Pappas, and Eric Wong.
\newblock Jailbreaking black box large language models in twenty queries.
\newblock \emph{arXiv preprint arXiv:2310.08419}, 2023.
\bibitem[Chen et~al.(2024)Chen, Zhao, Qu, Wen, Han, Zhu, Zhang, and Yao]{PANDORA}
Zhaorun Chen, Zhuokai Zhao, Wenjie Qu, Zichen Wen, Zhiguang Han, Zhihong Zhu, Jiaheng Zhang, and Huaxiu Yao.
\newblock Pandora: Detailed llm jailbreaking via collaborated phishing agents with decomposed reasoning.
\newblock In \emph{ICLR 2024 Workshop on Secure and Trustworthy Large Language Models}, 2024.
\bibitem[Dai et~al.(2023)Dai, Pan, Sun, Ji, Xu, Liu, Wang, and Yang]{safe_rlhf}
Josef Dai, Xuehai Pan, Ruiyang Sun, Jiaming Ji, Xinbo Xu, Mickel Liu, Yizhou Wang, and Yaodong Yang.
\newblock Safe rlhf: Safe reinforcement learning from human feedback.
\newblock \emph{arXiv preprint arXiv:2310.12773}, 2023.
\bibitem[Deng et~al.(2024)Deng, Liu, Li, Wang, Zhang, Li, Wang, Zhang, and Liu]{Masterkey}
Gelei Deng, Yi~Liu, Yuekang Li, Kailong Wang, Ying Zhang, Zefeng Li, Haoyu Wang, Tianwei Zhang, and Yang Liu.
\newblock Masterkey: Automated jailbreaking of large language model chatbots.
\newblock In \emph{Proc. ISOC NDSS}, 2024.
\bibitem[Deng et~al.(2023)Deng, Zhang, Pan, and Bing]{deng2023multilingual}
Yue Deng, Wenxuan Zhang, Sinno~Jialin Pan, and Lidong Bing.
\newblock Multilingual jailbreak challenges in large language models.
\newblock \emph{arXiv preprint arXiv:2310.06474}, 2023.
\bibitem[Ding et~al.(2023)Ding, Kuang, Ma, Cao, Xian, Chen, and Huang]{ReNeLLM}
Peng Ding, Jun Kuang, Dan Ma, Xuezhi Cao, Yunsen Xian, Jiajun Chen, and Shujian Huang.
\newblock A wolf in sheep's clothing: Generalized nested jailbreak prompts can fool large language models easily.
\newblock \emph{arXiv preprint arXiv:2311.08268}, 2023.
\bibitem[Dubey et~al.(2024)Dubey, Jauhri, Pandey, Kadian, Al-Dahle, Letman, Mathur, Schelten, Yang, Fan, et~al.]{dubey2024llama}
Abhimanyu Dubey, Abhinav Jauhri, Abhinav Pandey, Abhishek Kadian, Ahmad Al-Dahle, Aiesha Letman, Akhil Mathur, Alan Schelten, Amy Yang, Angela Fan, et~al.
\newblock The llama 3 herd of models.
\newblock \emph{arXiv preprint arXiv:2407.21783}, 2024.
\bibitem[Ge et~al.(2023)Ge, Zhou, Hou, Khabsa, Wang, Wang, Han, and Mao]{Mart}
Suyu Ge, Chunting Zhou, Rui Hou, Madian Khabsa, Yi-Chia Wang, Qifan Wang, Jiawei Han, and Yuning Mao.
\newblock Mart: Improving llm safety with multi-round automatic red-teaming.
\newblock \emph{arXiv preprint arXiv:2311.07689}, 2023.
\bibitem[Ghosh et~al.(2024)Ghosh, Varshney, Galinkin, and Parisien]{AegisGuard}
Shaona Ghosh, Prasoon Varshney, Erick Galinkin, and Christopher Parisien.
\newblock Aegis: Online adaptive ai content safety moderation with ensemble of llm experts.
\newblock \emph{arXiv preprint arXiv:2404.05993}, 2024.
\bibitem[Han et~al.(2024)Han, Rao, Ettinger, Jiang, Lin, Lambert, Choi, and Dziri]{wildguard}
Seungju Han, Kavel Rao, Allyson Ettinger, Liwei Jiang, Bill~Yuchen Lin, Nathan Lambert, Yejin Choi, and Nouha Dziri.
\newblock Wildguard: Open one-stop moderation tools for safety risks, jailbreaks, and refusals of llms.
\newblock \emph{arXiv preprint arXiv:2406.18495}, 2024.
\bibitem[Inan et~al.(2023)Inan, Upasani, Chi, Rungta, Iyer, Mao, Tontchev, Hu, Fuller, Testuggine, et~al.]{Llamaguard}
Hakan Inan, Kartikeya Upasani, Jianfeng Chi, Rashi Rungta, Krithika Iyer, Yuning Mao, Michael Tontchev, Qing Hu, Brian Fuller, Davide Testuggine, et~al.
\newblock Llama guard: Llm-based input-output safeguard for human-ai conversations.
\newblock \emph{arXiv preprint arXiv:2312.06674}, 2023.
\bibitem[Ji et~al.(2024)Ji, Hou, Robey, Pappas, Hassani, Zhang, Wong, and Chang]{Semantic_Smoothing}
Jiabao Ji, Bairu Hou, Alexander Robey, George~J. Pappas, Hamed Hassani, Yang Zhang, Eric Wong, and Shiyu Chang.
\newblock Defending large language models against jailbreak attacks via semantic smoothing.
\newblock \emph{arXiv preprint arXiv: 2402.16192}, 2024.
\bibitem[Jiang et~al.(2024)Jiang, Xu, Niu, Xiang, Ramasubramanian, Li, and Poovendran]{Artprompt}
Fengqing Jiang, Zhangchen Xu, Luyao Niu, Zhen Xiang, Bhaskar Ramasubramanian, Bo~Li, and Radha Poovendran.
\newblock Artprompt: Ascii art-based jailbreak attacks against aligned llms.
\newblock \emph{arXiv preprint arXiv:2402.11753}, 2024.
\bibitem[Li et~al.(2024)Li, Wei, Zhao, Zhang, and Zhang]{RAIN}
Yuhui Li, Fangyun Wei, Jinjing Zhao, Chao Zhang, and Hongyang Zhang.
\newblock Rain: Your language models can align themselves without finetuning.
\newblock In \emph{International Conference on Learning Representations}, 2024.
\bibitem[Liu et~al.(2024{\natexlab{a}})Liu, Zhao, Dong, Meng, and Chen]{DRA}
Tong Liu, Zhe Zhao, Yinpeng Dong, Guozhu Meng, and Kai Chen.
\newblock Making them ask and answer: Jailbreaking large language models in few queries via disguise and reconstruction.
\newblock In \emph{33rd USENIX Security Symposium (USENIX Security 24)}, pp.\ 4711--4728, 2024{\natexlab{a}}.
\bibitem[Liu et~al.(2024{\natexlab{b}})Liu, Xu, Chen, and Xiao]{AutoDAN_ICLR24}
Xiaogeng Liu, Nan Xu, Muhao Chen, and Chaowei Xiao.
\newblock Autodan: Generating stealthy jailbreak prompts on aligned large language models.
\newblock In \emph{The Twelfth International Conference on Learning Representations}, 2024{\natexlab{b}}.
\newblock URL \url{https://openreview.net/forum?id=7Jwpw4qKkb}.
\bibitem[Liu et~al.(2024{\natexlab{c}})Liu, He, Xiong, Fu, Deng, and Hooi]{liu2024flipattack}
Yue Liu, Xiaoxin He, Miao Xiong, Jinlan Fu, Shumin Deng, and Bryan Hooi.
\newblock Flipattack: Jailbreak llms via flipping.
\newblock \emph{arXiv preprint arXiv:2410.02832}, 2024{\natexlab{c}}.
\bibitem[Liu et~al.(2025)Liu, Gao, Zhai, Xia, Wu, Xue, Chen, Kawaguchi, Zhang, and Hooi]{liu2025guardreasoner}
Yue Liu, Hongcheng Gao, Shengfang Zhai, Jun Xia, Tianyi Wu, Zhiwei Xue, Yulin Chen, Kenji Kawaguchi, Jiaheng Zhang, and Bryan Hooi.
\newblock Guardreasoner: Towards reasoning-based llm safeguards.
\newblock \emph{arXiv preprint arXiv:2501.18492}, 2025.
\bibitem[Mehrotra et~al.(2023)Mehrotra, Zampetakis, Kassianik, Nelson, Anderson, Singer, and Karbasi]{TAP}
Anay Mehrotra, Manolis Zampetakis, Paul Kassianik, Blaine Nelson, Hyrum Anderson, Yaron Singer, and Amin Karbasi.
\newblock Tree of attacks: Jailbreaking black-box llms automatically.
\newblock \emph{arXiv preprint arXiv:2312.02119}, 2023.
\bibitem[Phute et~al.(2023)Phute, Helbling, Hull, Peng, Szyller, Cornelius, and Chau]{self_defense}
Mansi Phute, Alec Helbling, Matthew Hull, ShengYun Peng, Sebastian Szyller, Cory Cornelius, and Duen~Horng Chau.
\newblock Llm self defense: By self examination, llms know they are being tricked.
\newblock \emph{arXiv preprint arXiv:2308.07308}, 2023.
\bibitem[Qi et~al.(2023)Qi, Zeng, Xie, Chen, Jia, Mittal, and Henderson]{qi2023fine}
Xiangyu Qi, Yi~Zeng, Tinghao Xie, Pin-Yu Chen, Ruoxi Jia, Prateek Mittal, and Peter Henderson.
\newblock Fine-tuning aligned language models compromises safety, even when users do not intend to!
\newblock \emph{arXiv preprint arXiv:2310.03693}, 2023.
\bibitem[Qin et~al.(2022)Qin, Welleck, Khashabi, and Choi]{Cold_attack}
Lianhui Qin, Sean Welleck, Daniel Khashabi, and Yejin Choi.
\newblock Cold decoding: Energy-based constrained text generation with langevin dynamics.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 9538--9551, 2022.
\bibitem[Ramesh et~al.(2024)Ramesh, Dou, and Xu]{IRIS}
Govind Ramesh, Yao Dou, and Wei Xu.
\newblock Gpt-4 jailbreaks itself with near-perfect success using self-explanation.
\newblock \emph{arXiv preprint arXiv:2405.13077}, 2024.
\bibitem[Robey et~al.(2023)Robey, Wong, Hassani, and Pappas]{Smoothllm}
Alexander Robey, Eric Wong, Hamed Hassani, and George~J Pappas.
\newblock Smoothllm: Defending large language models against jailbreaking attacks.
\newblock \emph{arXiv preprint arXiv:2310.03684}, 2023.
\bibitem[Shen et~al.(2023)Shen, Chen, Backes, Shen, and Zhang]{DAN}
Xinyue Shen, Zeyuan Chen, Michael Backes, Yun Shen, and Yang Zhang.
\newblock " do anything now": Characterizing and evaluating in-the-wild jailbreak prompts on large language models.
\newblock \emph{arXiv preprint arXiv:2308.03825}, 2023.
\bibitem[Wang et~al.(2024)Wang, Zhang, Xu, Xi, Deng, Yao, Zhang, Yang, Wang, and Chen]{safe_edit}
Mengru Wang, Ningyu Zhang, Ziwen Xu, Zekun Xi, Shumin Deng, Yunzhi Yao, Qishen Zhang, Linyi Yang, Jindong Wang, and Huajun Chen.
\newblock Detoxifying large language models via knowledge editing.
\newblock \emph{arXiv preprint arXiv:2403.14472}, 2024.
\bibitem[Wei et~al.(2024)Wei, Haghtalab, and Steinhardt]{jailbroken}
Alexander Wei, Nika Haghtalab, and Jacob Steinhardt.
\newblock Jailbroken: How does llm safety training fail?
\newblock \emph{Advances in Neural Information Processing Systems}, 36, 2024.
\bibitem[Xie et~al.(2023)Xie, Yi, Shao, Curl, Lyu, Chen, Xie, and Wu]{self_reminders}
Yueqi Xie, Jingwei Yi, Jiawei Shao, Justin Curl, Lingjuan Lyu, Qifeng Chen, Xing Xie, and Fangzhao Wu.
\newblock Defending chatgpt against jailbreak attack via self-reminders.
\newblock \emph{Nature Machine Intelligence}, 5\penalty0 (12):\penalty0 1486--1496, 2023.
\bibitem[Xie et~al.(2024)Xie, Fang, Pi, and Gong]{GradSafe}
Yueqi Xie, Minghong Fang, Renjie Pi, and Neil Gong.
\newblock Gradsafe: Detecting unsafe prompts for llms via safety-critical gradient analysis.
\newblock \emph{arXiv preprint arXiv:2402.13494}, 2024.
\bibitem[Xu et~al.(2023)Xu, Kong, Liu, Cui, Wang, Zhang, and Kankanhalli]{PromptAttack}
X~Xu, K~Kong, N~Liu, L~Cui, D~Wang, J~Zhang, and M~Kankanhalli.
\newblock An llm can fool itself: A prompt-based adversarial attack.
\newblock \emph{URL: http://arxiv. org/abs/2310.13345}, 2023.
\bibitem[Xu et~al.(2024{\natexlab{a}})Xu, Jiang, Niu, Jia, Lin, and Poovendran]{Safedecoding}
Zhangchen Xu, Fengqing Jiang, Luyao Niu, Jinyuan Jia, Bill~Yuchen Lin, and Radha Poovendran.
\newblock Safedecoding: Defending against jailbreak attacks via safety-aware decoding.
\newblock \emph{arXiv preprint arXiv:2402.08983}, 2024{\natexlab{a}}.
\bibitem[Xu et~al.(2024{\natexlab{b}})Xu, Liu, Deng, Li, and Picek]{xu2024llm}
Zihao Xu, Yi~Liu, Gelei Deng, Yuekang Li, and Stjepan Picek.
\newblock Llm jailbreak attack versus defense techniques--a comprehensive study.
\newblock \emph{arXiv preprint arXiv:2402.13457}, 2024{\natexlab{b}}.
\bibitem[Yong et~al.(2023)Yong, Menghini, and Bach]{Low_resource_language}
Zheng-Xin Yong, Cristina Menghini, and Stephen~H Bach.
\newblock Low-resource languages jailbreak gpt-4.
\newblock \emph{arXiv preprint arXiv:2310.02446}, 2023.
\bibitem[Yu et~al.(2023)Yu, Lin, and Xing]{GPTFUZZER}
Jiahao Yu, Xingwei Lin, and Xinyu Xing.
\newblock Gptfuzzer: Red teaming large language models with auto-generated jailbreak prompts.
\newblock \emph{arXiv preprint arXiv:2309.10253}, 2023.
\bibitem[Yu et~al.(2024)Yu, Liu, Liang, Cameron, Xiao, and Zhang]{yu2024dontlistenmeunderstanding}
Zhiyuan Yu, Xiaogeng Liu, Shunning Liang, Zach Cameron, Chaowei Xiao, and Ning Zhang.
\newblock Don't listen to me: Understanding and exploring jailbreak prompts of large language models, 2024.
\newblock URL \url{https://arxiv.org/abs/2403.17336}.
\bibitem[Yuan et~al.(2023)Yuan, Jiao, Wang, Huang, He, Shi, and Tu]{SelfCipher}
Youliang Yuan, Wenxiang Jiao, Wenxuan Wang, Jen-tse Huang, Pinjia He, Shuming Shi, and Zhaopeng Tu.
\newblock Gpt-4 is too smart to be safe: Stealthy chat with llms via cipher.
\newblock \emph{arXiv preprint arXiv:2308.06463}, 2023.
\bibitem[Zeng et~al.(2024)Zeng, Liu, Mullins, Peran, Fernandez, Harkous, Narasimhan, Proud, Kumar, Radharapu, et~al.]{Shieldgemma}
Wenjun Zeng, Yuchi Liu, Ryan Mullins, Ludovic Peran, Joe Fernandez, Hamza Harkous, Karthik Narasimhan, Drew Proud, Piyush Kumar, Bhaktipriya Radharapu, et~al.
\newblock Shieldgemma: Generative ai content moderation based on gemma.
\newblock \emph{arXiv preprint arXiv:2407.21772}, 2024.
\bibitem[Zhang \& Wei(2024)Zhang and Wei]{MAC}
Yihao Zhang and Zeming Wei.
\newblock Boosting jailbreak attack with momentum.
\newblock \emph{arXiv preprint arXiv:2405.01229}, 2024.
\bibitem[Zhang et~al.(2023)Zhang, Yang, Ke, and Huang]{safe_decoding}
Zhexin Zhang, Junxiao Yang, Pei Ke, and Minlie Huang.
\newblock Defending large language models against jailbreaking attacks through goal prioritization.
\newblock \emph{arXiv preprint arXiv:2311.09096}, 2023.
\bibitem[Zheng et~al.(2024)Zheng, Yin, Zhou, Meng, Zhou, Chang, Huang, and Peng]{DRO}
Chujie Zheng, Fan Yin, Hao Zhou, Fandong Meng, Jie Zhou, Kai-Wei Chang, Minlie Huang, and Nanyun Peng.
\newblock On prompt-driven safeguarding for large language models.
\newblock In \emph{Forty-first International Conference on Machine Learning}, 2024.
\bibitem[Zou et~al.(2023)Zou, Wang, Kolter, and Fredrikson]{GCG}
Andy Zou, Zifan Wang, J.~Zico Kolter, and Matt Fredrikson.
\newblock Universal and transferable adversarial attacks on aligned language models, 2023.
\end{thebibliography}