SafetyPrompts: a Systematic Review of Open Datasets for Evaluating and Improving Large Language Model Safety
Extracted source text · arXiv 2404.05399
%File: anonymous-submission-latex-2025.tex
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage[draft]{aaai25} % DO NOT CHANGE THIS
\usepackage{times} % DO NOT CHANGE THIS
\usepackage{helvet} % DO NOT CHANGE THIS
\usepackage{courier} % DO NOT CHANGE THIS
\usepackage[hyphens]{url} % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm} % DO NOT CHANGE THIS
\usepackage{natbib} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%
% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
\usepackage{algorithm}
\usepackage{algorithmic}
%
% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
\usepackage{newfloat}
\usepackage{listings}
\DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
\lstset{%
basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
aboveskip=0pt,belowskip=0pt,%
showstringspaces=false,tabsize=2,breaklines=true}
\floatstyle{ruled}
\newfloat{listing}{tb}{lst}{}
\floatname{listing}{Listing}
%
% Keep the \pdfinfo as shown here. There's no need
% for you to add the /Title and /Author tags.
\pdfinfo{
/TemplateVersion (2025.1)
}
% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference
% PR custom packages
\usepackage{graphicx}
\usepackage{microtype}
\usepackage{url}
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{colortbl}
\usepackage{multirow}
\usepackage{xcolor}
\usepackage{bibentry}
%PR custom colours
\definecolor{lightbrown}{HTML}{cba487}
\definecolor{lightorange}{HTML}{f6c4a2}
\definecolor{lightyellow}{HTML}{fce597}
\definecolor{lightpurple}{HTML}{decbff}
\definecolor{lightgrey}{HTML}{d4d4d4}
\hyphenation{Anthropic-RedTeam}
\setcounter{secnumdepth}{2} %May be changed to 1 or 2 if section numbers are desired.
% The file aaai25.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%
% Title
% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{SafetyPrompts: a Systematic Review of Open Datasets for\\ Evaluating and Improving Large Language Model Safety}
\author{
%Authors
% All authors must be in the same font size and format.
Paul Röttger\textsuperscript{\rm 1}, Fabio Pernisi\textsuperscript{\rm 1}, Bertie Vidgen\textsuperscript{\rm 2}, and Dirk Hovy\textsuperscript{\rm 1}
}
\affiliations{
\textsuperscript{\rm 1}Bocconi University,
\textsuperscript{\rm 2}Contextual AI
}
\begin{document}
%\nobibliography{custom}
\maketitle
\begin{abstract}
The last two years have seen a rapid growth in concerns around the safety of large language models (LLMs).
Researchers and practitioners have met these concerns by creating an abundance of datasets for evaluating and improving LLM safety.
However, much of this work has happened in parallel, and with very different goals in mind, ranging from the mitigation of near-term risks around bias and toxic content generation to the assessment of longer-term catastrophic risk potential.
This makes it difficult for researchers and practitioners to find the most relevant datasets for their use case, and to identify gaps in dataset coverage that future work may fill.
To remedy these issues, we conduct a first systematic review of open datasets for evaluating and improving LLM safety.
We review 144 datasets, which we identified through an iterative and community-driven process over the course of several months.
We highlight patterns and trends, such as a trend towards fully synthetic datasets, as well as gaps in dataset coverage, such as a clear lack of non-English and naturalistic datasets.
We also examine how LLM safety datasets are used in practice -- in LLM release publications and popular LLM benchmarks -- finding that current evaluation practices are highly idiosyncratic and make use of only a small fraction of available datasets.
Our contributions are based on SafetyPrompts.com, a living catalogue of open datasets for LLM safety, which we plan to update continuously as the field of LLM safety develops.
\end{abstract}
% Uncomment the following to link to your code, datasets, an extended version or similar.
%
% \begin{links}
% \link{Code}{https://aaai.org/example/code}
% \link{Datasets}{https://aaai.org/example/datasets}
% \link{Extended version}{https://aaai.org/example/extended-version}
% \end{links}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
Ensuring the safety of large language models (LLMs) has become as a key priority for model developers and regulators.
Consequently, in recent years, researchers and practitioners have created an abundance of datasets for evaluating and improving LLM safety.
Safety, however, is a multi-faceted and contextual concept that lacks a unifying definition \citep{aroyo2023dices}.
This complexity is reflected in the current landscape of safety datasets, which is broad, diverse, and fast-changing.
Within just two months of 2024, for example, researchers published datasets for evaluating near-term risks from LLMs, such as sociodemographic bias \citep{gupta2024calm} and toxic content generation \citep{bianchi2024safetytuned}, as well as datasets for evaluating long-term societal risk potential, around power-seeking \citep{mazeika2024harmbench} and sycophantic behaviours \citep{sharma2024sycophancyeval}.
This rapid pace of dataset creation, and the variety of purposes served by different datasets, make it difficult for researchers and practitioners to find the most relevant datasets for different use cases, and to identify gaps in dataset coverage that future work may fill.
In this paper, we seek to address these issues by conducting a \textbf{first systematic review of open datasets for evaluating and improving LLM safety}.
We identify 144 datasets published between June 2018 and December 2024 based on clear inclusion criteria (\S\ref{subsec: methods - inclusion criteria}) using a comprehensive community-driven search method (\S\ref{subsec: methods - finding datasets}).
We examine these 144 datasets along several key dimensions, including their purpose (\S\ref{subsec: findings - purpose}), intended use (\S\ref{subsec: findings - intended use}), format and size (\S\ref{subsec: findings - format}), their creation (\S\ref{subsec: findings - creation}), language (\S\ref{subsec: findings - language}), licensing and access (\S\ref{subsec: findings - access}), and publication (\S\ref{subsec: findings - publication}).
Our review shows that a growing interest in LLM safety is driving the creation of more and more diverse open LLM safety datasets, with most contributions coming from academia.
Major outstanding challenges, on the other hand, include a clear lack of safety datasets in non-English languages as well as a lack of naturalistic safety evaluations.
We also review how open LLM safety datasets are used in practice -- in model release publications (\S\ref{sec: datasets in model releases}) and popular LLM benchmarks (\S\ref{sec: datasets in model benchmarks}) -- finding that current evaluation practices are highly idiosyncratic and leverage only a small fraction of available datasets.
We argue that this creates clear scope for standardisation in LLM safety evaluations, and that model safety evaluations in general could be improved by better leveraging recent progress in dataset creation (\S\ref{sec: discussion}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dataset Review Methodology}
\label{sec: dataset review methods}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Inclusion Criteria}
\label{subsec: methods - inclusion criteria}
At a high level, we restrict our review to \emph{open} datasets that are relevant to \emph{LLMs}, and specifically relevant to evaluating and improving LLM \emph{safety}.
More specifically, this means:
In terms of \textbf{data modality}, we only include text datasets.
We do not include image datasets \citep[e.g.][]{schwemmer2020diagnosing, zhao2021understanding, ricker2022deepfakes} or audio datasets \citep[e.g.][]{reimao2019dataset,koenecke2020racial,meyer2020artie}.
We also do not include datasets targeted at multimodal models, even if one modality is text, such as in the case of vision-language \citep[e.g.][]{carlini2023aligned, hall2023visogender,wolfe2023visionlanguagebias} or text-to-image models \citep[e.g.][]{bianchi2023texttoimage,parrish2023adversarial,luccioni2024stable}.
Further, we do not include datasets targeted at code generation models \citep[e.g.][]{siddiq2022securityeval, bhatt2023purple}.
These modalities and models constitute natural expansions for future work.
We make only minimal restrictions in terms of \textbf{data format}.
Real-world user interactions with LLMs usually take the form of text chat
\citep{ouyang2023shifted,zheng2024lmsys,zhao2024inthewildchat}, so we are most interested in datasets that naturally fit a chat format, like open-ended questions and instructions, but we also consider any other dataset that can meaningfully be expressed in a prompt format.
This includes multiple-choice questions or autocomplete-style text snippets.
We do not make any restrictions on dataset language.
For \textbf{data access}, we only include datasets that are publicly available for download via GitHub and/or Hugging Face.
%In practice, we found that, if data is made available, it is almost always on one or both of these platforms.
We do not make restrictions based on how data is licensed.
Finally, we require that all datasets are \textbf{relevant to safety}.
For the purposes of our review, we adopt a wide and open definition of safety.
Broadly speaking, we include datasets that relate to representational, political or other forms of sociodemographic bias; to toxicity, malicious instructions or harmful advice; to hazardous behaviours like sycophancy or power-seeking; to alignment with social, moral or ethical values; or to adversarial LLM usage (e.g.\ red-teaming, jailbreaking, prompt hacking).
We only include datasets that explicitly focus on one or multiple of these aspects of LLM safety.
We do not include datasets that target general LLM capabilities like reasoning, language understanding, or code completion \citep[e.g.][]{dua2019drop,hendrycks2020mmlu,chen2021humaneval}.
We also do not include datasets that target factuality in LLMs, unless they directly relate to safety, like in the case of generating misinformation \citep[][]{souly2024strongreject} or measuring truthfulness \citep{lin2022truthfulqa}.%
\footnote{Each dataset candidate was reviewed by two authors of this paper.
%For reproducibility and transparency of our selection,
We make detailed information on each dataset, as well as excluded datasets + exclusion reasons, available in the project repo.}
\textbf{The cutoff date for our review is December 17th, 2024}.
We only included datasets that were first published (i.e.\ made publicly available online) before this date.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Finding Dataset Candidates}
\label{subsec: methods - finding datasets}
We used an iterative and community-driven approach combined with snowball search to identify dataset candidates for inclusion in our review.
In January 2024, we released a first version of SafetyPrompts.com, with an initial list of 44 datasets that we had compiled in a heuristic fashion based on prior work and our knowledge of the LLM safety field.
Over the next months, we marketed the website to the LLM safety community on Twitter and Reddit, to solicit feedback and further dataset suggestions.
This resulted in 51 additional datasets, suggested by many different researchers and practitioners.
We then used these 95 datasets as a starting point for snowball search, wherein we reviewed each publication corresponding to each dataset for references to other potentially relevant datasets.
Whenever we identified a new dataset, we repeated this process.
This resulted in 49 additional datasets.
\textbf{Overall, our review includes 144 open datasets for evaluating and improving LLM safety}, which were first published between June 2018 and December 2024.
We chose this review method because of two main reasons.
First, LLM safety is a very fast-moving field with contributions from across academia and industry.
By sharing intermittent results of our review on SafetyPrompts.com, we were able to solicit feedback from a broad range of stakeholders and expand our review, while also providing a useful resource to the community well ahead of the release of this paper.
Second, traditional systematic review methods like keyword search are ill-suited to the scope of our review.
Combinations of relevant keywords like ``language model'', ``safety'' and ``dataset'' return thousands of results on Google Scholar and similar platforms -- and still fail to capture the many types of datasets that may not mention ``safety'' but are highly relevant to it regardless, like toxic conversation datasets or bias evaluations.
Despite our best efforts, it is likely that our review is missing at least some relevant datasets.
We are committed to adding these datasets, along with future relevant dataset releases, to SafetyPrompts.com.%
%\footnote{We have been receiving regular community suggestions for SafetyPrompts.com, which has been accessed by 100s of people since release. We also have funding in place to ensure future updates.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Recording Structured Information}
\label{subsec: methods - recording info}
For each of the 144 datasets in our review, we recorded 23 pieces of structured information.
At a high level, our goal was to capture the full development pipeline of each dataset: from how the dataset was created, to what entries in the dataset look like, what the dataset can or should be used for, how it can be accessed, and where it was published.
%We show the full codebook in the Appendix, which describes the structure and content of our main review spreadsheet.
We show the full codebook in Appendix~\ref{app: code book}, which describes the structure and content of our main review spreadsheet.
We make the spreadsheet available along with code to reproduce our analyses at github.com/paul-rottger/safetyprompts-paper.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dataset Review Findings}
\label{sec: dataset review findings}
\begin{figure*}[ht]
\includegraphics[width=0.9\textwidth]{figures/year_purpose_type.png}
\vspace{0.3cm}
\raggedleft
\footnotesize
\renewcommand{\arraystretch}{1.6}
\begin{tabularx}{0.99\linewidth}{p{0.3cm}|X}
\multirow{5}{*}{\rotatebox[origin=c]{90}{Example datasets}} &
\colorbox{lightbrown}{DecodingTrust} by \citet{wang2024decodingtrust} evaluates LLM safety across 8 ``perspectives''\\
& \colorbox{lightorange}{ConfAIde} by \citet{mireshghallah2024confaide} evaluates the privacy-reasoning capabilities of LLMs\\
& \colorbox{lightyellow}{MoralChoice} by \citet{scherrer2023evaluating} evaluates moral beliefs encoded in LLMs\\
& \colorbox{lightpurple}{BBQ} by \citet{parrish2022bbq} evaluates social biases of LLMs in question answering\\
& \colorbox{lightgray}{ToxicChat} by \citet{lin2023toxicchat} trains and evaluates content moderation systems for LLMs\\
\end{tabularx}
\caption{(top)
\textbf{Number of datasets published per year, grouped by their primary purpose.}
Our review includes 144 datasets published between June 2018 and December 2024.
(bottom)
\textbf{Example datasets} for each type of dataset purpose (\S\ref{subsec: findings - purpose}).
}
\label{fig:purpose_type}
\end{figure*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{History and Growth of Safety Datasets}
%\subsection{How did the field of LLM safety grow and develop over time?}
\label{subsec: findings - timeline}
Our review shows that \textbf{LLM safety builds on a rich history of research into the risks and biases of language models and dialogue agents}.
The first datasets in our review were published in mid-2018, and focus on evaluating gender bias -- originally for co-reference resolution systems, but equally applicable to current LLMs \citep{zhao2018winobias,rudinger2018winogender}.
These datasets, in turn, build on earlier works on biases in word embeddings \citep[e.g.][]{bolukbasi2016man,caliskan2017semantics,garg2018word}, which illustrates that concerns around the negative social impacts of language models are far from new.
Even the term ``safety'' was already used by \citet{dinan2019build} and \citet{rashkin2019empatheticdialogues}, among others, who introduced datasets for evaluating and improving the safety of dialogue agents well before the current generative LLM paradigm.
By today's standards, however, interest in safety was relatively low at the time, with only 9 out of the 144 datasets in our review (6.3\%) published in or before 2020, as shown in Figure~\ref{fig:purpose_type}.
\textbf{LLM safety experienced a first moderate growth phase in 2021 and 2022}.
These two years, respectively, saw the publication of 16 and 17 open LLM safety datasets.
This coincides with increased interest in generative language models, particularly among researchers, following the release of GPT-3 in mid-2020 \citep{brown2020language}.
Right now, \textbf{research into LLM safety is experiencing unprecedented growth}.
This coincides with a surge in public interest in LLMs as well as concerns around LLM safety following the release of ChatGPT in November 2022.
43 out of the 144 datasets in our review (29.9\%) were published in 2023, and 2024 saw even more activity, with 59 datasets published.
Accordingly, it seems very likely that 2025 will surpass this record once more.
%With 38 datasets published in just the first half of 2024 -- i.e.\ up to our review cutoff of July 1st, 2024 -- it is certain that more open datasets for LLM safety will be published in 2024 than ever before.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Purpose of Datasets}
\label{subsec: findings - purpose}
In our review, we differentiate between five distinct dataset purposes:
\colorbox{lightbrown}{\textbf{Broad safety}} (n=53) denotes datasets that cover several aspects of LLM safety.
This includes structured evaluation datasets like SafetyKit \citep{dinan2022safetykit} or SimpleSafetyTests \citep{vidgen2024simplesafetytests} as well as broad-scope red-teaming datasets like BAD \citep{xu2021botadversarialdialogue} or AnthropicRedTeam \citep{ganguli2022red}.
\colorbox{lightorange}{\textbf{Narrow safety}} (n=25), conversely, denotes datasets that focus only on one specific aspect of LLM safety.
SafeText \citep{levy2022safetext}, for example, focuses only on commonsense physical safety, while SycophancyEval \citep{sharma2024sycophancyeval} focuses on sycophantic behaviour.
\colorbox{lightyellow}{\textbf{Value alignment}} (n=23) refers to datasets that are concerned with the ethical, moral or social behaviour of LLMs.
This includes datasets that seek to evaluate LLM understanding of ethical norms, like Scruples \citep{lourie2021scruples} and ETHICS \citep{hendrycks2021ethics}, as well as opinion surveys like GlobalOpinionQA \citep{durmus2024globalopinionqa}.
\colorbox{lightpurple}{\textbf{Bias}} (n=33) refers to datasets for evaluating sociodemographic biases in LLMs.
BOLD \citep{dhamala2021bold}, for example, evaluates bias in text completions, whereas DiscrimEval \citep{tamkin2023evaluating} evaluates biases in situated LLM decision-making.
\colorbox{lightgrey}{\textbf{Other}} (n=10), in our review, includes datasets for developing LLM chat moderation systems, like FairPrism \citep{fleisig2023fairprism} and ToxicChat \citep{lin2023toxicchat}, as well as specialised prompts from public competitions like Gandalf \citep{lakera2023gandalf}, Mosscap \citep{lakera2023mosscap} or HackAPrompt \citep{schulhoff2023hackaprompt}.
Figure~\ref{fig:purpose_type} shows that \textbf{early safety datasets were primarily concerned with evaluating biases}.
14 out of 25 datasets (56.0\%) published between 2018 and 2021 were created to identify and analyse sociodemographic biases in language models.
13 of these datasets evaluate gender biases, either exclusively \citep[e.g.][]{nozza2021honest} or along with other categories of bias such as race and sexual orientation \citep[e.g.][]{sheng2019woman} or religion \citep[e.g.][]{li2020unqovering}.
\textbf{Broad safety emerged as a prominent theme in 2022, driven by industry contributions}.
Anthropic, for instance, released two broad-scope red-teaming datasets \citep{ganguli2022red,bai2022training}, while Meta published datasets on positive LLM conversations \citep{ung2022saferdialogues} and general safety evaluation \citep{dinan2022safetykit}.
Most recently, broad safety has shifted towards more structured evaluation, as exemplified by benchmarks like DecodingTrust \citep{wang2024decodingtrust} or HarmBench \citep{mazeika2024harmbench}.
\textbf{There is now a trend towards more specialised safety evaluations}.
Narrow safety evaluations did not emerge until 2022, but now make up a significant portion of all new datasets.
In 2024 alone, 15 of the 59 datasets in our review (25.4\%) were concerned with specific aspects of LLM safety, like rule-following \citep{mu2024rules} or privacy-reasoning ability \citep{mireshghallah2024confaide}.
Relatedly, \textbf{there is an increasing focus on adversarial robustness and LLM ``jailbreaking''}.
A growing body of work introduces safety datasets that are explicitly designed to elicit unsafe responses even from models trained to respond safely, often using elaborate prompting strategies, also known as ``jailbreaks'' \citep{wei2023jailbroken}.
Since August 2023, we record 13 such datasets, which differ from most other datasets that seek to evaluate safety in response to simpler, more naturalistic prompts \citep[e.g.][]{bhardwaj2023redteaming,shaikh2023harmfulq,vidgen2024simplesafetytests}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Intended Use of Datasets}
\label{subsec: findings - intended use}
\textbf{Most datasets are intended for model evaluation only}.
112 out of the 144 datasets in our review (77.8\%) were created for benchmarking or evaluation.
Only 6 datasets (4.2\%), by contrast, comprise examples of positive interactions between users and LLMs, created specifically for model training \citep{rashkin2019empatheticdialogues,ung2022saferdialogues,kim2022prosocialdialog,bianchi2024safetytuned,guo2024controllablepreferenceoptimization,jiang2024wildteaming}.
Overall, there is much more work on evaluating LLM safety than there is work on improving LLM safety.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Format and Size}
%\subsection{What formats do different open LLM safety datasets have?}
\label{subsec: findings - format}
\textbf{The format of LLM safety datasets has changed alongside general trends in LLM development}.
Early datasets, especially those created for bias evaluation, often used autocomplete-style formats (n=9), where models are tasked to either fill a masked word or finish a sentence snippet \citep[e.g.][]{sheng2019woman,dhamala2021bold,kirk2021bias}.
Such formats, which are most compatible with early language models like BERT or GPT-2, have since mostly been replaced by chat-style prompts, like open-ended questions and instructions (n=88), as well as promptable multiple-choice questions (n=24), which better suit current generative LLMs.
Dataset size varies substantially across the 144 datasets in our review.
Smaller datasets ($\le$1k entries, n=42) tend to be narrow safety evaluation datasets, written by the dataset authors.
For example, the smallest dataset -- ControversialInstructions by \citet{bianchi2024safetytuned} -- comprises 40 author-written prompts to test the propensity of LLMs to generate hate speech.
Larger datasets ($\ge$100k entries, n=21), on the other hand, are often broader in scope, designed for training and evaluation, and created through templating or LLM generation (\S\ref{subsec: findings - creation}).
For example, the largest dataset -- WorldValuesBench by \citet{zhao2024worldvaluesbench} -- contains 21.5m multiple-choice prompts created programmatically from individual responses to the large-scale World Value Survey.
%The largest dataset, HackAPrompt by \citet{schulhoff2023hackaprompt}, contains 601,757 human-written prompts recorded as part of a public prompt hacking competition.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Creation}
\label{subsec: findings - creation}
\textbf{The use of templates is a consistently popular method for generating safety evaluation data}.
26~out of the 144 datasets in our review (18.1\%) consist of human-written phrase or prompt templates, which are expanded through combination into larger evaluation datasets.
HolisticBias \citep{smith2022holisticbias}, for example, comprises 26 sentence templates (e.g.\ ``I am an [attribute1] who is [attribute2].''), which are combined with around 600 individual descriptor terms to create 459.8k test cases.
Generally, template approaches are most popular for bias evaluation -- 16 out of 33 bias evaluation datasets in our review use a template approach -- but recent work has also used templates for evaluating broad-scope LLM safety \citep{wang2024decodingtrust} or specific concerns such as privacy reasoning \citep{mireshghallah2024confaide}.
\textbf{A significant portion of recently-released datasets is fully synthetic}.
%Earlier safety datasets collected human-written prompts \citep[e.g.][]{dinan2019build,cercas2021convabuse,xu2021botadversarialdialogue},
%2023 saw the release of the first datasets with fully LLM-generated prompts.
21 of the 102 datasets released since 2023 (20.6\%) consist of chat prompts or multiple-choice questions generated entirely by LLMs -- most commonly by some variant of GPT-3.5.
\citet{shaikh2023harmfulq}, for example, prompt GPT-3.5 to generate 200 harmful questions, which they use to explore safety in chain-of-thought question answering.
The resulting dataset resembles the human-written SimpleSafetyTests \citep{vidgen2024simplesafetytests}.
Generally, synthetic datasets vary in size and scope much like human-written datasets do.
Relatedly, instead of relying on static templates for data creation, \textbf{many recent datasets are more flexibly augmented using LLMs}.
\citet{bhatt2023purple}, for instance, expand a small expert-written set of cyberattack instructions into a larger set of 1,000 prompts using Llama-70b-chat \citep{touvron2023llama}.
\citet{wang2024decodingtrust} take a similar approach to build their large-scale DecodingTrust benchmark.
By contrast, \textbf{very few datasets comprise naturalistic user interactions with LLMs}.
Around half of the 144 datasets in our review are hand-written (n=70, 48.6\%), but mostly by the authors of the corresponding dataset publications.
A much smaller proportion of datasets is created by humans -- either crowdworkers \citep[e.g.][]{ganguli2022red,aroyo2023dices,kirk2024prism} or participants of online competitions \citep[e.g.][]{lakera2023gandalf,lakera2023mosscap} -- interacting more naturally with LLMs.
Finally, \textbf{there is a trend towards reusing existing datasets rather than creating original data}.
34~out of the 59 datasets published in 2024 (57.6\%) were entirely~(n=10) or partially~(n=24) sampled from older datasets.
QHarm by \citet{bianchi2024safetytuned}, for example, is taken entirely from AnthropicHarmlessBase \citep{bai2022training}.
Another common approach is to sample existing data and augment it with templates \citep[e.g.][]{tedeschi2024alert} or LLM generations \citep[e.g.][]{yu2024cosafe}.
By contrast, just 10 out of 43 datasets published in 2023 (23.3\%) reused existing data.
This shift suggests a growing maturity of the LLM safety dataset landscape.
\begin{table*}[ht]
\centering
\footnotesize
\renewcommand{\arraystretch}{1.4}
\begin{minipage}{.5\linewidth}
\begin{tabularx}{0.95\linewidth}{p{0.3cm}p{4.3cm}X}
\hline
\rowcolor[HTML]{decbff}
& \textbf{Academic / Non-Profit Org.} & \textbf{n} \\
\hline
1 & Stanford University & 15\\
\rowcolor[rgb]{0.95,0.95,0.95}
1 & Allen Institute for AI & 15\\
3 & UC Berkeley & 14\\
\rowcolor[rgb]{0.95,0.95,0.95}
4 & University of Washington & 13\\
5 & Carnegie Mellon University & 12\\
\bottomrule
\end{tabularx}
\end{minipage}%
\begin{minipage}{.5\linewidth}
\raggedleft
\begin{tabularx}{0.95\linewidth}{p{0.3cm}p{4.3cm}X}
\hline
\rowcolor[HTML]{FCE597}
& \textbf{Industry Org.} & \textbf{n} \\
\hline
1 & Meta* (prev. Facebook) & 13\\
\rowcolor[rgb]{0.95,0.95,0.95}
2 & Anthropic & 9\\
3 & Microsoft* (incl. Research) & 7\\
\rowcolor[rgb]{0.95,0.95,0.95}
4 & Google* (incl. DeepMind) & 6\\
5 & Alibaba & 4\\
\bottomrule
\end{tabularx}
\end{minipage}
\caption{\textbf{Organisations that published the most open LLM safety datasets} among the 144 datasets in our review.
For each dataset, we count all affiliations for all co-authors.
Stanford at n=15, for example, means that 15 datasets had a Stanford author.
}
\label{tab:prolific_orgs}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Languages}
%\subsection{What languages are represented in open LLM safety datasets?}
\label{subsec: findings - language}
\textbf{The vast majority of open LLM safety datasets uses English language only}.
113 out of 144 datasets in our review (78.5\%) are exclusively in English.
10~datasets (6.9\%) focus only on Chinese \citep[e.g.][]{zhou2022cdial,xu2023cvalues,zhao2023chbias}, and one dataset each focuses only on Arabic, Swedish, Hindi, Korean, and French.
The 16 other datasets (11.1\%) cover English along with one or more other languages.
\citet{jin2024multilingualtrolleyproblems} cover 106 languages, although test cases are auto-translated from the English.
The other 143 datasets in our review together cover 31 languages.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Data Licensing and Access}
\label{subsec: findings - access}
\textbf{When data is shared, usage licenses are mostly permissive}.
The most common license is the very permissive MIT License, which is used for 58 out of 144 datasets (40.3\%).
23 datasets (16.0\%) use the Apache 2.0 License, which provides additional patent protections.
43 datasets (29.9\%) use variants of a CC BY 4.0 License, which requires dataset users to provide appropriate credit and indicate if changes were made to the dataset.
Notably, 15 datasets (10.4\%) prohibit commercial usage with a CC BY-NC License.
Only 2 datasets (1.4\%) use a more restrictive custom license.
As of December 17th, 2024, 15 datasets (10.4\%) do not specify any license.%
\footnote{While conducting our review, we reached out to authors of all datasets that had not specified a license and encouraged them to add one. At least five authors added a license as a result.}
\textbf{GitHub is still the most popular platform for sharing LLM safety data}.
72 datasets (50.0\%) are available only on GitHub.
However, 55 datasets (38.2\%) are available on both GitHub and HuggingFace, and several recent datasets are shared on Hugging Face only \citep[e.g.][]{han2024wildguard,manerba2024social}, suggesting a shift in how data is shared.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Dataset Publication Authors and Venues}
%\subsection{Who is publishing open LLM safety datasets, and where?}
\label{subsec: findings - publication}
\textbf{Academic and non-profit organisations drive most of the creation of open LLM safety datasets}.
For 69 out of 144 datasets in our review (47.9\%), all authors of the corresponding publication were affiliated only with academic or non-profit organisations.
45 datasets (31.3\%) included authors from industry and academia, and only 30 datasets (20.8\%) were published by fully industry teams.
\textbf{The creation of LLM safety datasets is concentrated in few research hubs} (Table~\ref{tab:prolific_orgs}).
There are 156 unique affiliations across the authors of the 144 datasets in our review.
101 affiliations (64.7\%) are associated with just a single dataset.
The five most prolific organisations, on the other hand, are each associated with at least 13 datasets.
All of the twenty most prolific organisations are located and/or headquartered in the US, with the exception of Bocconi University (Italy, n=11), the University of Oxford (UK, n=5), CUHK (HK, n=4), and Tsinghua University (China, n=4).
\textbf{Most LLM safety datasets so far have been published at *ACL conferences}.
68 out of the 144 datasets in our review (47.2\%) were published at either ACL (n=27), EMNLP (n=30) or other *ACL venues.
33 datasets (19.7\%) were published at more ML-focused conferences, i.e.\ NeurIPS (n=18), ICLR (n=10), or ICML (n=5).
Only 11 datasets (7.6\%) were published at other venues, and only 2 datasets appeared in journal publications \citep{jin2024kobbq,yu2024beyond}.
28 datasets (19.4\%), on the other hand, were accompanied only by arXiv preprints, and 4 (2.7\%) only by blog posts, meaning they did not receive traditional peer review.
Generally, we observe a slight trend away from *ACL conferences and towards more ML-focused venues as well as arXiv-only publication, although this could in part be explained by recent arXiv preprints still being under review.
\begin{table*}[ht]
\centering
\footnotesize
\renewcommand{\arraystretch}{1.4}
\begin{tabularx}{\linewidth}{p{0.4cm}p{5.6cm}Xp{0.4cm}}
\hline
\rowcolor[HTML]{f6c4a2}
& \textbf{Dataset} & \textbf{Purpose} & \textbf{n} \\
\hline
1 & TruthfulQA \citep{lin2022truthfulqa} & evaluate tendency to mimic human falsehoods & 8 \\
\rowcolor[rgb]{0.95,0.95,0.95}
2 & BBQ \citep{parrish2022bbq} & evaluate social bias in question answering & 6 \\
3 & AnthropicRedTeam \citep{ganguli2022red} & evaluate responses to diverse red-team attacks & 3 \\
\rowcolor[rgb]{0.95,0.95,0.95}
4 & ToxiGen \citep{hartvigsen2022toxigen} & evaluate toxicity in text completions & 3 \\
5 & RealToxicityPrompts \citep{gehman2020realtoxicityprompts} & evaluate toxicity in text completions & 3 \\
\bottomrule
\end{tabularx}
\caption{\textbf{Most popular open LLM safety datasets}, based on how often
release publications for SOTA LLMs reported results on each dataset.
BBQ at n=6, for example, means that 6 out of the 29 model release publications (\S\ref{subsec: datasets in model releases - findings}) reported BBQ results.
}
\label{tab:model_top_datasets}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Datasets in Model Release Publications}
\label{sec: datasets in model releases}
In the following, we briefly examine \textbf{how open LLM safety datasets are used in practice}.
In particular, we examine in this section which safety datasets are used to evaluate current state-of-the-art LLMs ahead of their release, as documented in model release publications.
In the next section (\S\ref{sec: datasets in model benchmarks}), we then examine which safety datasets are included in popular LLM benchmarks and leaderboards.
%\footnote{We make the full review spreadsheets of our model release and benchmark reviews available at [REDACTED\_URL].}
This is to characterise current norms and common practices in evaluating LLM safety, so that we can then discuss (\S\ref{sec: discussion}) these norms and practices in relation to the findings of our dataset review (\S\ref{sec: dataset review findings}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Scope of our Model Release Review}
\label{subsec: datasets in model releases - scope}
We examine the top 50 best-performing LLMs listed on the LMSYS Chatbot Arena Leaderboard \citep{chiang2024chatbotarena} as of July 25th, 2024.%
\footnote{\url{https://lmarena.ai/?leaderboard}}
The LMSYS Chatbot Arena Leaderboard is a crowdsourced platform for LLM evaluation, which ranks models based on model Elo scores calculated from over one million pairwise human preference votes.
We use this leaderboard for setting the scope of our review because it is held in high regard in the LLM community, and it has up-to-date coverage of recent model releases.
The top 50 entries on the LMSYS leaderboard correspond to 29 unique model releases.%
\footnote{A model release may comprise multiple model versions, such as GPT-4-0314 and GPT-4-0613.}
Of these 29 models, 16 (55.2\%) are proprietary models only accessible via an API, released by companies such as OpenAI (GPT), Anthropic (Claude), Google (Gemini) and 01 AI (Yi).
The other 13 models (44.8\%) are open models, for which weights are publicly accessible via Hugging Face.
Proprietary models generally outrank open models on the leaderboard, with Gemma 2 27b \citep{deepmind2024gemma2} being the best open model at rank 14.
All 29 models were released by industry labs.%
%\footnote{In a March 2024 version of this analysis, i.e.~4 months earlier, the proportion of proprietary models in the LMSYS top 50 was substantially lower. Also, at least some models in the top 50 originated from academia or non-profit organisations, rather than industry.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Findings of our Model Release Review}
\label{subsec: datasets in model releases - findings}
\textbf{Around half of state-of-the-art LLMs are evaluated for safety ahead of their release}, with substantial variation in the extent and nature of safety evaluations.
For 16 out of 29 model releases (55.2\%), model developers report at least some quantitative safety evaluation in the associated technical report, blog post or model card.
13 model release publications (44.8\%) report results on at least one open LLM safety dataset.
The Claude 3.5 model card, for example, reports results on WildChat \citep{zhao2024inthewildchat} and XSTest \citep{rottger2024xstest}, whereas Gemma 2 \citep{deepmind2024gemma2} is evaluated on 7 different open LLM safety datasets.
13 out of the 29 models (44.8\%), on the other hand, do not report any safety evaluations.
This includes the strongest model, the proprietary GPT-4o, which mentions safety measures in its release blog post, but lacks any concrete quantitative safety evaluations.
Other models, both proprietary (e.g.\ Yi-Large, Reka Core) and open (e.g.\ Mixtral, Command R), do not mention safety at all in their release publications.
When safety is evaluated, \textbf{proprietary data plays a large role in model release safety evaluations}.
Out of the 16 model releases that report safety evaluation results, 11 (68.8\%) use undisclosed proprietary data for evaluating model safety.
3 of these releases -- Llama 3 \citep{meta2024llama3}, Qwen 2 \citep{yang2024qwen2}, and Phi 3 \citep{abdin2024phi3} -- report results only on proprietary safety datasets.
Finally, \textbf{the diversity of open LLM safety datasets used in model release evaluations is very limited}.
Only 14 open LLM safety datasets are used across the 29 model releases, and 6 of these 14 datasets are used only once.
Table~\ref{tab:model_top_datasets} shows the 5 datasets that are used most often.
Notably, TruthfulQA \citep{lin2022truthfulqa} is used in 8 out of the 16 model release publications that report any safety evaluation results (50.0\%), and it is often framed by model developers as a capability rather than a safety evaluation.
We discuss the implications of these findings in \S\ref{sec: discussion}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Datasets used in Popular Benchmarks}
\label{sec: datasets in model benchmarks}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Scope of our Benchmark Review}
\label{subsec: datasets in model benchmarks - scope}
To complement our model release review, we briefly examine 9 popular LLM benchmarks for which safety datasets they include.
6 benchmarks are widely-used general-purpose benchmarking suites: Stanford's HELM Classic \citep{liang2023helm} and HELM Instruct \citep{zhang2024helminstruct}, Hugging Face's Open LLM Leaderboard \citep{beeching2023openllmleaderboard}, AllenAI's RewardBench \citep{lambert2024rewardbench}, Eleuther AI's Evaluation Harness \citep{gao2021evalharness}, and BIG-Bench \citep{srivastava2023bigbench}.
3 benchmarks focus explicitly on LLM safety: TrustLLM \citep{sun2024trustllm}, HELM Safety \citep{kaiyom2024helmsafety}, and the LLM Safety Leaderboard.\footnote{\url{https://huggingface.co/spaces/AI-Secure/llm-trustworthy-leaderboard}, based on \citet{wang2024decodingtrust}.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Findings of our Benchmark Review}
\label{subsec: datasets in model benchmarks - findings}
\textbf{There are large differences in how different benchmarks evaluate LLM safety}.
The 9 benchmarks make use of 26 open LLM safety datasets.
17 of these datasets are used in just one benchmark.
TrustLLM \citep{sun2024trustllm}, for example, combines 11 open LLM safety datasets, of which 5 are not used in any other benchmark.
The only open LLM safety datasets that are used in more than 2 benchmarks are TruthfulQA \citep{lin2022truthfulqa}, which is used in 4 benchmarks, as well as RealToxicityPrompts \citep{gehman2020realtoxicityprompts}, BBQ \citep{parrish2022bbq}, ETHICS \citep{hendrycks2021ethics}, XSTest \citep{rottger2024xstest}, and DoNotAnswer \citep{wang2024donotanswer}, which are each used in 3 benchmarks.
\textbf{There is currently no single LLM safety benchmark with a truly comprehensive scope}.
The TrustLLM benchmark \citep{sun2024trustllm}, has the broadest scope relevant to safety among the 9 benchmarks we examined, covering malicious instruction-following, bias, and value alignment.
However, it does not, for example, test for longer-term risk potential with evaluations for sycophancy \citep{perez2023discovering} or chemical weapon knowledge \citep{li2024wmdp}, as the Evaluation Harness does \citep{gao2021evalharness}.
Similarly, the LLM Safety Leaderboard \citep{beeching2023openllmleaderboard} tests for toxic content generation and malicious instruction-following, but not for false refusal \citep[e.g.][]{rottger2024xstest} or sociodemographic biases \citep[e.g.][]{parrish2022bbq}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion \& Future Directions}
\label{sec: discussion}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{The State of the Safety Dataset Landscape}
\label{subsec: discussion - datasets}
Overall, our review shows that \textbf{growing interest in LLM safety is driving the creation of more and more diverse open LLM safety datasets}.
More datasets were published in 2023 than ever before, and 2024 again surpassed this record (\S\ref{subsec: findings - timeline}).
Existing datasets span varied purposes (\S\ref{subsec: findings - purpose}) and formats (\S\ref{subsec: findings - format}), which have adapted over time to meet the needs and requirements of LLM users and developers.
Researchers and practitioners are making creative use of new methods for dataset creation (\S\ref{subsec: findings - creation}), and when data is shared, usage licenses are mostly permissive (\S\ref{subsec: findings - access}).
These are encouraging signs for the health of the open LLM safety community and its ability to address emerging challenges and fill gaps in dataset coverage as they become apparent.
Among these gaps, the most apparent today is that \textbf{there is a clear lack of safety datasets in non-English languages}.
We found that English dominates the current safety dataset landscape (\S\ref{subsec: findings - language}), mirroring long-standing trends in NLP research \citep{bender2011achieving,joshi2020state,holtermann2024evaluating}.
To some extent, this imbalance also reflects an imbalance in who is publishing safety datasets (\S\ref{subsec: findings - publication}).
The lack of non-English resources for evaluating and improving LLM safety is a problem because it means that billions of non-English speakers across the world are potentially more at risk of harm when using current language technologies.
Further, interpretations of safety are known to vary across cultures and geographies \citep{aroyo2023dices,kirk2024prism}.
These two factors create an urgent need for the LLM safety community to prioritise the creation of non-English datasets going forward.
Recent datasets like AyaRedTeaming \citep{aakanksha2024multilingual}, which were created for language-specific cultural contexts with the involvement of local stakeholders, may serve as a useful blueprint for future work in this direction.
The second major concern apparent from our review is that \textbf{current safety evaluation datasets largely fail to reflect real-world LLM usage}, which undermines their ecological validity.
Data generation and augmentation, from templates to more recent LLM-based methods, have enabled rapid dataset creation (\S\ref{subsec: findings - creation}), but it is not clear that they can emulate the ``messiness'' and diversity that is apparent in real-world user interactions with LLMs \citep{ouyang2023shifted,zhao2024inthewildchat,zheng2024lmsys}.
An increasing focus on safety against ``jailbreaking'' (\S\ref{subsec: findings - purpose}) may match the behaviour of sophisticated adversarial users, but does not capture the potential risks from vulnerable populations interacting with LLMs without malicious intent \citep{vidgen2024mlc05}.
Addressing these concerns will require the creation of more naturalistic evaluations, and clearer communication about which user personas (e.g.\ vulnerable, malicious, or adversarial) are targeted by individual safety evaluations.
Encouraging developments in this direction include WildGuard \citep{han2024wildguard} and WildJailBreak \citep{jiang2024wildteaming}, which provide evaluations based on real user prompts.
Future work may similarly look to collections of real user interactions with LLMs, such as WildChat \citep{zhao2024inthewildchat} and LMSYS \citep{zheng2024lmsys}, to improve the ecological validity of LLM safety evaluations.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{The Use of Safety Datasets in Practice}
\label{subsec: discussion - benchmarks}
Our analysis of how open LLM safety datasets are used in practice shows that \textbf{there is clear scope for standardisation in LLM safety evaluations}.
Safety is a key priority for model developers and users, as evidenced by the inclusion of safety evaluations in the majority of model release publications (\S\ref{sec: datasets in model releases}) and popular LLM benchmarks (\S\ref{sec: datasets in model benchmarks}).
However, there is much idiosyncrasy in \textit{which} datasets are used for evaluating safety across model release publications and benchmarks.
For commercial model releases, these datasets are often proprietary and undisclosed.
More standardised and open evaluations, on the other hand, would enable more meaningful model comparisons and incentivise the development of safer LLMs.
We see three main directions towards this goal.
First, current evaluation practices could \textbf{better leverage recent progress in safety dataset creation}.
The most popular open datasets for evaluating safety in model release publications, for example, are all from 2020 or 2022 (Table~\ref{tab:model_top_datasets}), despite more than two thirds of the datasets in our review being published in or after 2023.
Prior publication date is not a sign of lacking quality, but older autocomplete-style datasets like RealToxicityPrompts \citep{gehman2020realtoxicityprompts} or ToxiGen \citep{hartvigsen2022toxigen} simply do not reflect realistic usage of current chat-optimised LLMs \citep{ouyang2023shifted,zheng2024lmsys,zhao2024inthewildchat}, which undermines their ecological validity.
Our review highlights many datasets that could be used in their stead.
%
Second, there is a clear need for \textbf{unified standards of dataset quality}.
In this paper, we refrained from making quality judgments about the datasets we reviewed, mainly because different datasets serve different purposes, so that their utility is highly context dependent.
However, recent efforts like BetterBench \citep{reuel2024betterbench} as applied to safety datasets could help create consensus on which datasets to use for safety benchmarking.
%
Third, safety research would benefit from \textbf{unified safety evaluation protocols}.
The open-ended nature of safety-relevant behaviours complicates standardised evaluation compared to many factual tasks.
Frameworks like Eleuther AI's Evaluation Harness, adapted for safety tasks, would make it easier for model developers and users to run the same safety evaluations with reproducible results, and thus serve to reduce idiosyncrasy in prevailing safety evaluation practices.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec: conclusion}
In recent years, researchers and practitioners have sought to meet concerns around the safety of large language models by creating an abundance of datasets for evaluating and improving LLM safety.
However, the rapid pace of dataset creation and the variety of purposes served by different datasets have made it difficult for researchers and practitioners to find the most relevant datasets for different use cases, and to identify gaps in dataset coverage that future work may fill.
In this paper, we addressed these issues by conducting a first systematic review of open LLM safety datasets.
In our review, which includes 144 datasets published between June 2018 and December 2024, we showed that, encouragingly, existing datasets span varied purposes and formats, which have adapted over time to meet the changing needs and requirements of LLM users and developers.
However, we also highlighted major outstanding challenges, including a clear lack of non-English datasets as well as naturalistic safety evaluations.
Further, when examining how open LLM safety datasets are used in practice -- in model release publications and popular LLM benchmarks -- we found that current evaluation practices are highly idiosyncratic and make use of only a small fraction of available datasets, which presents clear scope for rejuvenation and standardisation.
Overall, we hope that with our review, as well as the living dataset catalogue we make available on SafetyPrompts.com, we can enable such positive change, by helping researchers and practitioners make the best use of existing datasets, and providing a strong foundation for future dataset development.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Acknowledgments}
Thank you for feedback and dataset suggestions to
Giuseppe Attanasio, Steven Basart, Federico Bianchi, Marta R. Costa-Jussà, Daniel Hershcovic, Kexin Huang, Hyunwoo Kim, George Kour, Bo Li, Hannah Lucas, Marta Marchiori Manerba, Norman Mu, Niloofar Mireshghallah, Matus Pikuliak, Verena Rieser, Felix Röttger, Sam Toyer, Ryan Tsang, Pranav Venkit, Laura Weidinger, and Linhao Yu.
Special thanks to Hannah Rose Kirk for the initial logo suggestion.
PR, FP, and DH are members of the Data and Marketing Insights research unit of the Bocconi Institute for Data Science and Analysis, and are supported by a MUR FARE 2020 initiative under grant agreement Prot.\ R20YSMBZ8S (INDOMITA) and the European Research Council (ERC) under the European Union’s Horizon 2020 research and innovation program (No.\ 949944, INTEGRATOR).
\bibliography{custom}
\appendix
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Review Code Book}
\label{app: code book}
For each of the 144 datasets in our review, we recorded 23 pieces of structured information.
At a high level, our goal was to capture the full development pipeline of each dataset: from how the dataset was created, to what entries in the dataset look like, what the dataset can or should be used for, how it can be accessed, and where it was published.
We show the full codebook in Table~\ref{tab:code_book} below, which describes the structure and content of our main review spreadsheet.
We make the spreadsheet available along with code to reproduce our analyses at github.com/paul-rottger/safetyprompts-paper.
\begin{table*}[ht]
\centering
\footnotesize
\renewcommand{\arraystretch}{1.4}
\begin{tabularx}{\textwidth}{p{1.9cm}p{6.5cm}X}
\hline
\rowcolor[HTML]{FCE597}
\multicolumn{3}{c}{\textbf{\texttt{purpose}}: what can I use this dataset for?} \\
\hline
\texttt{\_type} & High-level type of purpose / \newline general area of application & single choice: [broad safety, narrow safety, bias, value alignment, other] \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_tags} & Additional tags to specify purpose & comma-separated list of text tags \\
\texttt{\_stated} & Exact purpose as stated by authors & free text \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_llmdev} & Intended use of the dataset within the LLM development pipeline & single choice: [eval only, train and eval, train only, other] \\
\hline
\rowcolor[HTML]{FCE597}
\multicolumn{3}{c}{\textbf{\texttt{entries}}: what do entries in this dataset look like?} \\
\hline
\texttt{\_type} & High-level type of entry / format & single choice: [chat, multiple choice, autocomplete, other] \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_languages} & Languages in the dataset & comma-separated list of free-text language names \\
\texttt{\_n} & Number of entries & integer \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_unit} & Unit of entries (e.g.\ conversation) & free text \\
\texttt{\_detail} & Additional detail on entry format & free text \\
\hline
\rowcolor[HTML]{FCE597}
\multicolumn{3}{c}{\textbf{\texttt{creation}}: who created this dataset / where is it sampled from?} \\
\hline
\texttt{\_creator\_type} & Type of creator, i.e.\ who or what created the data & single choice: [human, machine, hybrid] \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_source\_type} & Type of data source, i.e.\ where the data is taken from & single choice: [original, sampled, mixed] \\
\texttt{\_detail} & Additional detail on creator/source & free text \\
\hline
\rowcolor[HTML]{FCE597}
\multicolumn{3}{c}{\textbf{\texttt{access}}: where can I download this dataset, and how is it licensed?} \\
\hline
\texttt{\_git\_url} & GitHub repo URL & URL \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_hf\_url} & Hugging Face dataset URL & URL \\
\texttt{\_license} & Dataset license & free text \\
\hline
\rowcolor[HTML]{FCE597}
\multicolumn{3}{c}{\textbf{\texttt{publication}}: when, where, and by whom was this dataset published?} \\
\hline
\texttt{\_date} & Publication date (most recent version) & dd-mmm-yyyy \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_affils} & Author affiliations & comma-separated list of institutions \\
\texttt{\_sector} & Sector from which the publication originated & single choice: [academia, industry, mixed] \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_name} & Publication name / reference & free text \\
\texttt{\_venue} & Publication venue & free text \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_url} & Publication URL & URL \\
\hline
\rowcolor[HTML]{FCE597}
\multicolumn{3}{c}{\textbf{\texttt{other}}: what is worth noting beyond the scope of this review?} \\
\hline
\texttt{\_notes} & Additional notes & free text \\
\rowcolor[rgb]{0.95,0.95,0.95}
\texttt{\_date\_added} & Date on which the dataset was added to SafetyPrompts.com & dd-mmm-yyyy \\
\bottomrule
\end{tabularx}
\caption{\textbf{Codebook for our main review spreadsheet}.
For each of the 144 datasets included in our review, we recorded 23 pieces of structured information.}
\label{tab:code_book}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{List of All Datasets}
\label{app: datasets}
In total, our review covers 144 datasets published before our cutoff date of December 17th, 2024.
We did not cite all dataset papers in the main body of our review, so we cite all datasets here, in ascending order by year of publication.
\begin{enumerate}
\item JBBBehaviours \citep{chao2024jailbreakbench}
\item MedSafetyBench \citep{han2024medsafetybench}
\item PRISM \citep{kirk2024prism}
\item WildGuardMix \citep{han2024wildguard}
\item SGBench \citep{mou2024sgbench}
\item StrongREJECT \citep{souly2024strongreject}
\item CoCoNot \citep{brahman2024coconot}
\item WildJailBreak \citep{jiang2024wildteaming}
\item MultiTP \citep{jin2024multilingualtrolleyproblems}
\item DoAnythingNow \citep{shen2024dan}
\item ForbiddenQuestions \citep{shen2024dan}
\item CoSafe \citep{yu2024cosafe}
\item SoFa \citep{manerba2024social}
\item ArabicAdvBench \citep{ghanim2024arabicadvbench}
\item AyaRedTeaming \citep{aakanksha2024multilingual}
\item UltraSafety \citep{guo2024controllablepreferenceoptimization}
\item GEST \citep{pikuliak2024gest}
\item DeMET \citep{levy2024demet}
\item GenMO \citep{bajaj2024genmo}
\item SGXSTest \citep{gupta2024walledeval}
\item HindiXSTest \citep{gupta2024walledeval}
\item CIVICS \citep{pistilli2024civics}
\item GlobalOpinionQA \citep{durmus2024globalopinionqa}
\item CALM \citep{gupta2024calm}
\item ChiSafetyBench \citep{zhang2024chisafetybench}
\item SafetyBench \citep{zhang2024safetybench}
\item CatQA \citep{bhardwaj2024catqa}
\item KorNAT \citep{lee2024kornat}
\item CMoralEval \citep{yu2024cmoraleval}
\item XSafety \citep{wang2024xsafety}
\item SaladBench \citep{li2024saladbench}
\item MMHB \citep{tan2024mmhb}
\item GPTFuzzer \citep{yu2024gptfuzzer}
\item WMDP \citep{li2024wmdp}
\item ALERT \citep{tedeschi2024alert}
\item ORBench \citep{cui2024orbench}
\item SorryBench \citep{xie2024sorrybench}
\item XSTest \citep{rottger2024xstest}
\item Flames \citep{huang2024flames}
\item SEval \citep{yuan2024seval}
\item WorldValuesBench \citep{zhao2024worldvaluesbench}
\item CBBQ \citep{huang2024cbbq}
\item KoBBQ \citep{jin2024kobbq}
\item SAFE \citep{yu2024safe}
\item AegisAIContentSafety \citep{ghosh2024aegis}
\item DoNotAnswer \citep{wang2024donotanswer}
\item RuLES \citep{mu2024rules}
\item HarmBench \citep{mazeika2024harmbench}
\item DecodingTrust \citep{wang2024decodingtrust}
\item SimpleSafetyTests \citep{vidgen2024simplesafetytests}
\item CoNA \citep{bianchi2024safetytuned}
\item ControversialInstructions \citep{bianchi2024safetytuned}
\item QHarm \citep{bianchi2024safetytuned}
\item MaliciousInstructions \citep{bianchi2024safetytuned}
\item PhysicalSafetyInstructions \citep{bianchi2024safetytuned}
\item SafetyInstructions \citep{bianchi2024safetytuned}
\item HExPHI \citep{qi2024hexphi}
\item MaliciousInstruct \citep{huang2024maliciousinstruct}
\item SycophancyEval \citep{sharma2024sycophancyeval}
\item ConfAIde \citep{mireshghallah2024confaide}
\item OKTest \citep{shi2024oktest}
\item AdvBench \citep{zou2023advbench}
\item Mosscap \citep{lakera2023mosscap}
\item TDCRedTeaming \citep{mazeika2023tdc}
\item JADE \citep{zhang2023jade}
\item CPAD \citep{liu2023cpad}
\item CyberattackAssistance \citep{bhatt2023purple}
\item AttaQ \citep{kour2023attaq}
\item AART \citep{radharapu2023aart}
\item DELPHI \citep{sun2023delphi}
\item AdvPromptSet \citep{esiobu2023robbie}
\item HolisticBiasR \citep{esiobu2023robbie}
\item DiscrimEval \citep{tamkin2023discrimeval}
\item HackAPrompt \citep{schulhoff2023hackaprompt}
\item ToxicChat \citep{lin2023toxicchat}
\item SPMisconceptions \citep{chen2023spmisconceptions}
\item FFT \citep{cui2023fft}
\item MoralChoice \citep{scherrer2023evaluating}
\item DICES350 \citep{aroyo2023dices}
\item DICES990 \citep{aroyo2023dices}
\item BeaverTails \citep{ji2023beavertails}
\item PromptExtractionRobustness \citep{toyer2023tensor}
\item PromptHijackingRobustness \citep{toyer2023tensor}
\item GandalfIgnoreInstructions \citep{lakera2023gandalf}
\item GandalfSummarization \citep{lakera2023gandalf}
\item HarmfulQA \citep{bhardwaj2023redteaming}
\item LatentJailbreak \citep{qiu2023latentjailbreak}
\item OpinionQA \citep{santurkar2023opinionqa}
\item CValuesResponsibilityMC \citep{xu2023cvalues}
\item CValuesResponsibilityPrompts \citep{xu2023cvalues}
\item CHBias \citep{zhao2023chbias}
\item HarmfulQ \citep{shaikh2023harmfulq}
\item FairPrism \citep{fleisig2023fairprism}
\item ModelWrittenSycophancy \citep{perez2023discovering}
\item ModelWrittenAdvancedAIRisk \citep{perez2023discovering}
\item ModelWrittenPersona \citep{perez2023discovering}
\item WinoGenerated \citep{perez2023discovering}
\item SeeGULL \citep{jha2023seegull}
\item WinoQueer \citep{felkner2023winoqueer}
\item Machiavelli \citep{pan2023machiavelli}
\item SafetyPrompts \citep{sun2023safetyprompts}
\item OIGModeration \citep{ontocord2023oigmoderation}
\item CDialBias \citep{zhou2022cdial}
\item HolisticBias \citep{smith2022holisticbias}
\item PersonalInfoLeak \citep{huang2022personalinfoleak}
\item ProsocialDialog \citep{kim2022prosocialdialog}
\item SafeText \citep{levy2022safetext}
\item AnthropicRedTeam \citep{ganguli2022red}
\item IndianStereotypes \citep{bhatt2022indianstereotypes}
\item MoralExceptQA \citep{jin2022moralexceptqa}
\item BBQ \citep{parrish2022bbq}
\item FrenchCrowSPairs \citep{neveol2022frenchcrows}
\item MIC \citep{ziems2022moralintegrity}
\item SaFeRDialogues \citep{ung2022saferdialogues}
\item SafetyKit \citep{dinan2022safetykit}
\item ToxiGen \citep{hartvigsen2022toxigen}
\item DiaSafety \citep{sun2022diasafety}
\item TruthfulQA \citep{lin2022truthfulqa}
\item AnthropicHarmlessBase \citep{bai2022training}
\item JiminyCricket \citep{hendrycks2022jiminy}
\item BiasOutOfTheBox \citep{kirk2021bias}
\item EthnicBias \citep{ahn2021ethnicbias}
\item ConvAbuse \citep{cercas2021convabuse}
\item MoralStories \citep{emelin2021moralstories}
\item RedditBias \citep{barikeri2021redditbias}
\item HypothesisStereotypes \citep{sotnikova2021hypothesisstereotypes}
\item StereoSet \citep{nadeem2021stereoset}
\item ETHICS \citep{hendrycks2021ethics}
\item BAD \citep{xu2021botadversarialdialogue}
\item HONEST \citep{nozza2021honest}
\item SweWinoGender \citep{hansson2021swewinogender}
\item LMBias \citep{liang2021lmbias}
\item ScruplesAnecdotes \citep{lourie2021scruples}
\item ScruplesDilemmas \citep{lourie2021scruples}
\item BOLD \citep{dhamala2021bold}
\item CrowSPairs \citep{nangia2020crows}
\item UnQover \citep{li2020unqovering}
\item RealToxicityPrompts \citep{gehman2020realtoxicityprompts}
\item SocialChemistry101 \citep{forbes2020socialchemistry}
\item Regard \citep{sheng2019woman}
\item ParlAIDialogueSafety \citep{dinan2019build}
\item EmpatheticDialogues \citep{rashkin2019empatheticdialogues}
\item WinoBias \citep{zhao2018winobias}
\item WinoGender \citep{rudinger2018winogender}
\end{enumerate}
\end{document}
\ifdefined\DeclarePrefChars\DeclarePrefChars{'’-}\else\fi
\begin{thebibliography}{173}
\providecommand{\natexlab}[1]{#1}
\bibitem[{{Aakanksha} et~al.(2024){Aakanksha}, Ahmadian, Ermis, Goldfarb-Tarrant, Kreutzer, Fadaee, and Hooker}]{aakanksha2024multilingual}
{Aakanksha}; Ahmadian, A.; Ermis, B.; Goldfarb-Tarrant, S.; Kreutzer, J.; Fadaee, M.; and Hooker, S. 2024.
\newblock The Multilingual Alignment Prism: Aligning Global and Local Preferences to Reduce Harm.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, 12027--12049. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Abdin et~al.(2024)Abdin, Jacobs, Awan, Aneja, Awadallah, Awadalla, Bach, Bahree, Bakhtiari, Bao, Behl, Benhaim, Bilenko, Bjorck, Bubeck, Cai, Cai, Mendes, Chen, Chaudhary, Chen, Chen, Chen, Chen, Chopra, Dai, Giorno, de~Rosa, Dixon, Eldan, Fragoso, Iter, Gao, Gao, Gao, Garg, Goswami, Gunasekar, Haider, Hao, Hewett, Huynh, Javaheripi, Jin, Kauffmann, Karampatziakis, Kim, Khademi, Kurilenko, Lee, Lee, Li, Li, Liang, Liden, Liu, Liu, Liu, Lin, Lin, Luo, Madan, Mazzola, Mitra, Modi, Nguyen, Norick, Patra, Perez-Becker, Portet, Pryzant, Qin, Radmilac, Rosset, Roy, Ruwase, Saarikivi, Saied, Salim, Santacroce, Shah, Shang, Sharma, Shukla, Song, Tanaka, Tupini, Wang, Wang, Wang, Wang, Ward, Wang, Witte, Wu, Wyatt, Xiao, Xu, Xu, Xu, Yadav, Yang, Yang, Yang, Yang, Yu, Yuan, Zhang, Zhang, Zhang, Zhang, Zhang, Zhang, Zhang, and Zhou}]{abdin2024phi3}
Abdin, M.; Jacobs, S.~A.; Awan, A.~A.; Aneja, J.; Awadallah, A.; Awadalla, H.; Bach, N.; Bahree, A.; Bakhtiari, A.; Bao, J.; Behl, H.; Benhaim, A.; Bilenko, M.; Bjorck, J.; Bubeck, S.; Cai, Q.; Cai, M.; Mendes, C. C.~T.; Chen, W.; Chaudhary, V.; Chen, D.; Chen, D.; Chen, Y.-C.; Chen, Y.-L.; Chopra, P.; Dai, X.; Giorno, A.~D.; de~Rosa, G.; Dixon, M.; Eldan, R.; Fragoso, V.; Iter, D.; Gao, M.; Gao, M.; Gao, J.; Garg, A.; Goswami, A.; Gunasekar, S.; Haider, E.; Hao, J.; Hewett, R.~J.; Huynh, J.; Javaheripi, M.; Jin, X.; Kauffmann, P.; Karampatziakis, N.; Kim, D.; Khademi, M.; Kurilenko, L.; Lee, J.~R.; Lee, Y.~T.; Li, Y.; Li, Y.; Liang, C.; Liden, L.; Liu, C.; Liu, M.; Liu, W.; Lin, E.; Lin, Z.; Luo, C.; Madan, P.; Mazzola, M.; Mitra, A.; Modi, H.; Nguyen, A.; Norick, B.; Patra, B.; Perez-Becker, D.; Portet, T.; Pryzant, R.; Qin, H.; Radmilac, M.; Rosset, C.; Roy, S.; Ruwase, O.; Saarikivi, O.; Saied, A.; Salim, A.; Santacroce, M.; Shah, S.; Shang, N.; Sharma, H.; Shukla, S.; Song, X.; Tanaka, M.; Tupini, A.;
Wang, X.; Wang, L.; Wang, C.; Wang, Y.; Ward, R.; Wang, G.; Witte, P.; Wu, H.; Wyatt, M.; Xiao, B.; Xu, C.; Xu, J.; Xu, W.; Yadav, S.; Yang, F.; Yang, J.; Yang, Z.; Yang, Y.; Yu, D.; Yuan, L.; Zhang, C.; Zhang, C.; Zhang, J.; Zhang, L.~L.; Zhang, Y.; Zhang, Y.; Zhang, Y.; and Zhou, X. 2024.
\newblock Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone.
\newblock arXiv:2404.14219.
\bibitem[{Ahn and Oh(2021)}]{ahn2021ethnicbias}
Ahn, J.; and Oh, A. 2021.
\newblock Mitigating Language-Dependent Ethnic Bias in {BERT}.
\newblock In Moens, M.-F.; Huang, X.; Specia, L.; and Yih, S. W.-t., eds., \emph{Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, 533--549. Online and Punta Cana, Dominican Republic: Association for Computational Linguistics.
\bibitem[{Aroyo et~al.(2023)Aroyo, Taylor, Diaz, Homan, Parrish, Serapio-Garc{\'\i}a, Prabhakaran, and Wang}]{aroyo2023dices}
Aroyo, L.; Taylor, A.; Diaz, M.; Homan, C.; Parrish, A.; Serapio-Garc{\'\i}a, G.; Prabhakaran, V.; and Wang, D. 2023.
\newblock Dices dataset: Diversity in conversational ai evaluation for safety.
\newblock \emph{Advances in Neural Information Processing Systems}, 36.
\bibitem[{Bai et~al.(2022)Bai, Jones, Ndousse, Askell, Chen, DasSarma, Drain, Fort, Ganguli, Henighan et~al.}]{bai2022training}
Bai, Y.; Jones, A.; Ndousse, K.; Askell, A.; Chen, A.; DasSarma, N.; Drain, D.; Fort, S.; Ganguli, D.; Henighan, T.; et~al. 2022.
\newblock Training a helpful and harmless assistant with reinforcement learning from human feedback.
\newblock \emph{arXiv preprint arXiv:2204.05862}.
\bibitem[{Bajaj et~al.(2024)Bajaj, Lei, Tong, and Huang}]{bajaj2024genmo}
Bajaj, D.; Lei, Y.; Tong, J.; and Huang, R. 2024.
\newblock Evaluating Gender Bias of {LLM}s in Making Morality Judgements.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2024}, 15804--15818. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Barikeri et~al.(2021)Barikeri, Lauscher, Vuli{\'c}, and Glava{\v{s}}}]{barikeri2021redditbias}
Barikeri, S.; Lauscher, A.; Vuli{\'c}, I.; and Glava{\v{s}}, G. 2021.
\newblock {R}eddit{B}ias: A Real-World Resource for Bias Evaluation and Debiasing of Conversational Language Models.
\newblock In Zong, C.; Xia, F.; Li, W.; and Navigli, R., eds., \emph{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}, 1941--1955. Online: Association for Computational Linguistics.
\bibitem[{Beeching et~al.(2023)Beeching, Fourrier, Habib, Han, Lambert, Rajani, Sanseviero, Tunstall, and Wolf}]{beeching2023openllmleaderboard}
Beeching, E.; Fourrier, C.; Habib, N.; Han, S.; Lambert, N.; Rajani, N.; Sanseviero, O.; Tunstall, L.; and Wolf, T. 2023.
\newblock Open LLM Leaderboard.
\newblock \url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}.
\bibitem[{Bender(2011)}]{bender2011achieving}
Bender, E.~M. 2011.
\newblock On achieving and evaluating language-independence in NLP.
\newblock \emph{Linguistic Issues in Language Technology}, 6.
\bibitem[{Bhardwaj, Do, and Poria(2024)}]{bhardwaj2024catqa}
Bhardwaj, R.; Do, D.~A.; and Poria, S. 2024.
\newblock Language Models are {H}omer Simpson! Safety Re-Alignment of Fine-tuned Language Models through Task Arithmetic.
\newblock In Ku, L.-W.; Martins, A.; and Srikumar, V., eds., \emph{Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 14138--14149. Bangkok, Thailand: Association for Computational Linguistics.
\bibitem[{Bhardwaj and Poria(2023)}]{bhardwaj2023redteaming}
Bhardwaj, R.; and Poria, S. 2023.
\newblock Red-Teaming Large Language Models using Chain of Utterances for Safety-Alignment.
\newblock arXiv:2308.09662.
\bibitem[{Bhatt et~al.(2023)Bhatt, Chennabasappa, Nikolaidis, Wan, Evtimov, Gabi, Song, Ahmad, Aschermann, Fontana et~al.}]{bhatt2023purple}
Bhatt, M.; Chennabasappa, S.; Nikolaidis, C.; Wan, S.; Evtimov, I.; Gabi, D.; Song, D.; Ahmad, F.; Aschermann, C.; Fontana, L.; et~al. 2023.
\newblock Purple llama cyberseceval: A secure coding benchmark for language models.
\newblock \emph{arXiv preprint arXiv:2312.04724}.
\bibitem[{Bhatt et~al.(2022)Bhatt, Dev, Talukdar, Dave, and Prabhakaran}]{bhatt2022indianstereotypes}
Bhatt, S.; Dev, S.; Talukdar, P.; Dave, S.; and Prabhakaran, V. 2022.
\newblock Re-contextualizing Fairness in {NLP}: The Case of {I}ndia.
\newblock In He, Y.; Ji, H.; Li, S.; Liu, Y.; and Chang, C.-H., eds., \emph{Proceedings of the 2nd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics and the 12th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}, 727--740. Online only: Association for Computational Linguistics.
\bibitem[{Bianchi et~al.(2023)Bianchi, Kalluri, Durmus, Ladhak, Cheng, Nozza, Hashimoto, Jurafsky, Zou, and Caliskan}]{bianchi2023texttoimage}
Bianchi, F.; Kalluri, P.; Durmus, E.; Ladhak, F.; Cheng, M.; Nozza, D.; Hashimoto, T.; Jurafsky, D.; Zou, J.; and Caliskan, A. 2023.
\newblock Easily Accessible Text-to-Image Generation Amplifies Demographic Stereotypes at Large Scale.
\newblock In \emph{Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency}, FAccT '23, 1493–1504. New York, NY, USA: Association for Computing Machinery.
\newblock ISBN 9798400701924.
\bibitem[{Bianchi et~al.(2024)Bianchi, Suzgun, Attanasio, Rottger, Jurafsky, Hashimoto, and Zou}]{bianchi2024safetytuned}
Bianchi, F.; Suzgun, M.; Attanasio, G.; Rottger, P.; Jurafsky, D.; Hashimoto, T.; and Zou, J. 2024.
\newblock Safety-Tuned {LL}a{MA}s: Lessons From Improving the Safety of Large Language Models that Follow Instructions.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Bolukbasi et~al.(2016)Bolukbasi, Chang, Zou, Saligrama, and Kalai}]{bolukbasi2016man}
Bolukbasi, T.; Chang, K.-W.; Zou, J.~Y.; Saligrama, V.; and Kalai, A.~T. 2016.
\newblock Man is to computer programmer as woman is to homemaker? debiasing word embeddings.
\newblock \emph{Advances in neural information processing systems}, 29.
\bibitem[{Brahman et~al.(2024)Brahman, Kumar, Balachandran, Dasigi, Pyatkin, Ravichander, Wiegreffe, Dziri, Chandu, Hessel, Tsvetkov, Smith, Choi, and Hajishirzi}]{brahman2024coconot}
Brahman, F.; Kumar, S.; Balachandran, V.; Dasigi, P.; Pyatkin, V.; Ravichander, A.; Wiegreffe, S.; Dziri, N.; Chandu, K.; Hessel, J.; Tsvetkov, Y.; Smith, N.~A.; Choi, Y.; and Hajishirzi, H. 2024.
\newblock The Art of Saying No: Contextual Noncompliance in Language Models.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Brown et~al.(2020)Brown, Mann, Ryder, Subbiah, Kaplan, Dhariwal, Neelakantan, Shyam, Sastry, Askell et~al.}]{brown2020language}
Brown, T.; Mann, B.; Ryder, N.; Subbiah, M.; Kaplan, J.~D.; Dhariwal, P.; Neelakantan, A.; Shyam, P.; Sastry, G.; Askell, A.; et~al. 2020.
\newblock Language models are few-shot learners.
\newblock \emph{Advances in neural information processing systems}, 33: 1877--1901.
\bibitem[{Caliskan, Bryson, and Narayanan(2017)}]{caliskan2017semantics}
Caliskan, A.; Bryson, J.~J.; and Narayanan, A. 2017.
\newblock Semantics derived automatically from language corpora contain human-like biases.
\newblock \emph{Science}, 356(6334): 183--186.
\bibitem[{Carlini et~al.(2023)Carlini, Nasr, Choquette-Choo, Jagielski, Gao, Koh, Ippolito, Tram{\`e}r, and Schmidt}]{carlini2023aligned}
Carlini, N.; Nasr, M.; Choquette-Choo, C.~A.; Jagielski, M.; Gao, I.; Koh, P.~W.; Ippolito, D.; Tram{\`e}r, F.; and Schmidt, L. 2023.
\newblock Are aligned neural networks adversarially aligned?
\newblock In \emph{Thirty-seventh Conference on Neural Information Processing Systems}.
\bibitem[{Cercas~Curry, Abercrombie, and Rieser(2021)}]{cercas2021convabuse}
Cercas~Curry, A.; Abercrombie, G.; and Rieser, V. 2021.
\newblock {C}onv{A}buse: Data, Analysis, and Benchmarks for Nuanced Detection in Conversational {AI}.
\newblock In \emph{Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, 7388--7403. Online and Punta Cana, Dominican Republic: Association for Computational Linguistics.
\bibitem[{Chao et~al.(2024)Chao, Debenedetti, Robey, Andriushchenko, Croce, Sehwag, Dobriban, Flammarion, Pappas, Tram{\`e}r, Hassani, and Wong}]{chao2024jailbreakbench}
Chao, P.; Debenedetti, E.; Robey, A.; Andriushchenko, M.; Croce, F.; Sehwag, V.; Dobriban, E.; Flammarion, N.; Pappas, G.~J.; Tram{\`e}r, F.; Hassani, H.; and Wong, E. 2024.
\newblock JailbreakBench: An Open Robustness Benchmark for Jailbreaking Large Language Models.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Chen et~al.(2021)Chen, Tworek, Jun, Yuan, Pinto, Kaplan, Edwards, Burda, Joseph, Brockman et~al.}]{chen2021humaneval}
Chen, M.; Tworek, J.; Jun, H.; Yuan, Q.; Pinto, H. P. d.~O.; Kaplan, J.; Edwards, H.; Burda, Y.; Joseph, N.; Brockman, G.; et~al. 2021.
\newblock Evaluating large language models trained on code.
\newblock \emph{arXiv preprint arXiv:2107.03374}.
\bibitem[{Chen, Arunasalam, and Celik(2023)}]{chen2023spmisconceptions}
Chen, Y.; Arunasalam, A.; and Celik, Z.~B. 2023.
\newblock Can Large Language Models Provide Security \& Privacy Advice? Measuring the Ability of LLMs to Refute Misconceptions.
\newblock In \emph{Proceedings of the 39th Annual Computer Security Applications Conference}, ACSAC '23, 366–378. New York, NY, USA: Association for Computing Machinery.
\newblock ISBN 9798400708862.
\bibitem[{Chiang et~al.(2024)Chiang, Zheng, Sheng, Angelopoulos, Li, Li, Zhang, Zhu, Jordan, Gonzalez, and Stoica}]{chiang2024chatbotarena}
Chiang, W.-L.; Zheng, L.; Sheng, Y.; Angelopoulos, A.~N.; Li, T.; Li, D.; Zhang, H.; Zhu, B.; Jordan, M.; Gonzalez, J.~E.; and Stoica, I. 2024.
\newblock Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference.
\newblock arXiv:2403.04132.
\bibitem[{Cui et~al.(2024)Cui, Chiang, Stoica, and Hsieh}]{cui2024orbench}
Cui, J.; Chiang, W.-L.; Stoica, I.; and Hsieh, C.-J. 2024.
\newblock OR-Bench: An Over-Refusal Benchmark for Large Language Models.
\newblock \emph{arXiv preprint arXiv:2405.20947}.
\bibitem[{Cui et~al.(2023)Cui, Zhang, Chen, Zhang, Liu, Wang, and Liu}]{cui2023fft}
Cui, S.; Zhang, Z.; Chen, Y.; Zhang, W.; Liu, T.; Wang, S.; and Liu, T. 2023.
\newblock Fft: Towards harmlessness evaluation and analysis for llms with factuality, fairness, toxicity.
\newblock \emph{arXiv preprint arXiv:2311.18580}.
\bibitem[{DeepMind(2024)}]{deepmind2024gemma2}
DeepMind, G. 2024.
\newblock Gemma 2: Improving Open Language Models at a Practical Size.
\bibitem[{Dhamala et~al.(2021)Dhamala, Sun, Kumar, Krishna, Pruksachatkun, Chang, and Gupta}]{dhamala2021bold}
Dhamala, J.; Sun, T.; Kumar, V.; Krishna, S.; Pruksachatkun, Y.; Chang, K.-W.; and Gupta, R. 2021.
\newblock BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation.
\newblock In \emph{Proceedings of the 2021 ACM Conference on Fairness, Accountability, and Transparency}, FAccT '21, 862–872. New York, NY, USA: Association for Computing Machinery.
\newblock ISBN 9781450383097.
\bibitem[{Dinan et~al.(2022)Dinan, Abercrombie, Bergman, Spruit, Hovy, Boureau, and Rieser}]{dinan2022safetykit}
Dinan, E.; Abercrombie, G.; Bergman, A.; Spruit, S.; Hovy, D.; Boureau, Y.-L.; and Rieser, V. 2022.
\newblock {S}afety{K}it: First Aid for Measuring Safety in Open-domain Conversational Systems.
\newblock In \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 4113--4133. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Dinan et~al.(2019)Dinan, Humeau, Chintagunta, and Weston}]{dinan2019build}
Dinan, E.; Humeau, S.; Chintagunta, B.; and Weston, J. 2019.
\newblock Build it Break it Fix it for Dialogue Safety: Robustness from Adversarial Human Attack.
\newblock In \emph{Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)}, 4537--4546. Hong Kong, China: Association for Computational Linguistics.
\bibitem[{Dua et~al.(2019)Dua, Wang, Dasigi, Stanovsky, Singh, and Gardner}]{dua2019drop}
Dua, D.; Wang, Y.; Dasigi, P.; Stanovsky, G.; Singh, S.; and Gardner, M. 2019.
\newblock {DROP}: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs.
\newblock In Burstein, J.; Doran, C.; and Solorio, T., eds., \emph{Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)}, 2368--2378. Minneapolis, Minnesota: Association for Computational Linguistics.
\bibitem[{Durmus et~al.(2024)Durmus, Nguyen, Liao, Schiefer, Askell, Bakhtin, Chen, Hatfield-Dodds, Hernandez, Joseph, Lovitt, McCandlish, Sikder, Tamkin, Thamkul, Kaplan, Clark, and Ganguli}]{durmus2024globalopinionqa}
Durmus, E.; Nguyen, K.; Liao, T.; Schiefer, N.; Askell, A.; Bakhtin, A.; Chen, C.; Hatfield-Dodds, Z.; Hernandez, D.; Joseph, N.; Lovitt, L.; McCandlish, S.; Sikder, O.; Tamkin, A.; Thamkul, J.; Kaplan, J.; Clark, J.; and Ganguli, D. 2024.
\newblock Towards Measuring the Representation of Subjective Global Opinions in Language Models.
\newblock In \emph{First Conference on Language Modeling}.
\bibitem[{Emelin et~al.(2021)Emelin, Le~Bras, Hwang, Forbes, and Choi}]{emelin2021moralstories}
Emelin, D.; Le~Bras, R.; Hwang, J.~D.; Forbes, M.; and Choi, Y. 2021.
\newblock Moral Stories: Situated Reasoning about Norms, Intents, Actions, and their Consequences.
\newblock In Moens, M.-F.; Huang, X.; Specia, L.; and Yih, S. W.-t., eds., \emph{Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, 698--718. Online and Punta Cana, Dominican Republic: Association for Computational Linguistics.
\bibitem[{Esiobu et~al.(2023)Esiobu, Tan, Hosseini, Ung, Zhang, Fernandes, Dwivedi-Yu, Presani, Williams, and Smith}]{esiobu2023robbie}
Esiobu, D.; Tan, X.; Hosseini, S.; Ung, M.; Zhang, Y.; Fernandes, J.; Dwivedi-Yu, J.; Presani, E.; Williams, A.; and Smith, E. 2023.
\newblock {ROBBIE}: Robust Bias Evaluation of Large Generative Language Models.
\newblock In Bouamor, H.; Pino, J.; and Bali, K., eds., \emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, 3764--3814. Singapore: Association for Computational Linguistics.
\bibitem[{Felkner et~al.(2023)Felkner, Chang, Jang, and May}]{felkner2023winoqueer}
Felkner, V.; Chang, H.-C.~H.; Jang, E.; and May, J. 2023.
\newblock {W}ino{Q}ueer: A Community-in-the-Loop Benchmark for Anti-{LGBTQ}+ Bias in Large Language Models.
\newblock In Rogers, A.; Boyd-Graber, J.; and Okazaki, N., eds., \emph{Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 9126--9140. Toronto, Canada: Association for Computational Linguistics.
\bibitem[{Fleisig et~al.(2023)Fleisig, Amstutz, Atalla, Blodgett, Daum{\'e}~III, Olteanu, Sheng, Vann, and Wallach}]{fleisig2023fairprism}
Fleisig, E.; Amstutz, A.; Atalla, C.; Blodgett, S.~L.; Daum{\'e}~III, H.; Olteanu, A.; Sheng, E.; Vann, D.; and Wallach, H. 2023.
\newblock {F}air{P}rism: Evaluating Fairness-Related Harms in Text Generation.
\newblock In Rogers, A.; Boyd-Graber, J.; and Okazaki, N., eds., \emph{Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 6231--6251. Toronto, Canada: Association for Computational Linguistics.
\bibitem[{Forbes et~al.(2020)Forbes, Hwang, Shwartz, Sap, and Choi}]{forbes2020socialchemistry}
Forbes, M.; Hwang, J.~D.; Shwartz, V.; Sap, M.; and Choi, Y. 2020.
\newblock Social Chemistry 101: Learning to Reason about Social and Moral Norms.
\newblock In Webber, B.; Cohn, T.; He, Y.; and Liu, Y., eds., \emph{Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)}, 653--670. Online: Association for Computational Linguistics.
\bibitem[{Ganguli et~al.(2022)Ganguli, Lovitt, Kernion, Askell, Bai, Kadavath, Mann, Perez, Schiefer, Ndousse et~al.}]{ganguli2022red}
Ganguli, D.; Lovitt, L.; Kernion, J.; Askell, A.; Bai, Y.; Kadavath, S.; Mann, B.; Perez, E.; Schiefer, N.; Ndousse, K.; et~al. 2022.
\newblock Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned.
\newblock \emph{arXiv preprint arXiv:2209.07858}.
\bibitem[{Gao et~al.(2021)Gao, Tow, Biderman, Black, DiPofi, Foster, Golding, Hsu, McDonell, Muennighoff, Phang, Reynolds, Tang, Thite, Wang, Wang, and Zou}]{gao2021evalharness}
Gao, L.; Tow, J.; Biderman, S.; Black, S.; DiPofi, A.; Foster, C.; Golding, L.; Hsu, J.; McDonell, K.; Muennighoff, N.; Phang, J.; Reynolds, L.; Tang, E.; Thite, A.; Wang, B.; Wang, K.; and Zou, A. 2021.
\newblock A framework for few-shot language model evaluation.
\newblock In \emph{Zenodo}. Zenodo.
\bibitem[{Garg et~al.(2018)Garg, Schiebinger, Jurafsky, and Zou}]{garg2018word}
Garg, N.; Schiebinger, L.; Jurafsky, D.; and Zou, J. 2018.
\newblock Word embeddings quantify 100 years of gender and ethnic stereotypes.
\newblock \emph{Proceedings of the National Academy of Sciences}, 115(16): E3635--E3644.
\bibitem[{Gehman et~al.(2020)Gehman, Gururangan, Sap, Choi, and Smith}]{gehman2020realtoxicityprompts}
Gehman, S.; Gururangan, S.; Sap, M.; Choi, Y.; and Smith, N.~A. 2020.
\newblock {R}eal{T}oxicity{P}rompts: Evaluating Neural Toxic Degeneration in Language Models.
\newblock In \emph{Findings of the Association for Computational Linguistics: EMNLP 2020}, 3356--3369. Online: Association for Computational Linguistics.
\bibitem[{Ghanim et~al.(2024)Ghanim, Almohaimeed, Zheng, Solihin, and Lou}]{ghanim2024arabicadvbench}
Ghanim, M.~A.; Almohaimeed, S.; Zheng, M.; Solihin, Y.; and Lou, Q. 2024.
\newblock Jailbreaking {LLM}s with {A}rabic Transliteration and {A}rabizi.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, 18584--18600. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Ghosh et~al.(2024)Ghosh, Varshney, Galinkin, and Parisien}]{ghosh2024aegis}
Ghosh, S.; Varshney, P.; Galinkin, E.; and Parisien, C. 2024.
\newblock AEGIS: Online Adaptive AI Content Safety Moderation with Ensemble of LLM Experts.
\newblock \emph{arXiv preprint arXiv:2404.05993}.
\bibitem[{Guo et~al.(2024)Guo, Cui, Yuan, Ding, Sun, Sun, Chen, Xie, Zhou, Lin, Liu, and Sun}]{guo2024controllablepreferenceoptimization}
Guo, Y.; Cui, G.; Yuan, L.; Ding, N.; Sun, Z.; Sun, B.; Chen, H.; Xie, R.; Zhou, J.; Lin, Y.; Liu, Z.; and Sun, M. 2024.
\newblock Controllable Preference Optimization: Toward Controllable Multi-Objective Alignment.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, 1437--1454. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Gupta et~al.(2024{\natexlab{a}})Gupta, Yau, Low, Lee, Lim, Teoh, Hng, Liew, Bhardwaj, Bhardwaj, and Poria}]{gupta2024walledeval}
Gupta, P.; Yau, L.~Q.; Low, H.~H.; Lee, I.-S.; Lim, H.~M.; Teoh, Y.~X.; Hng, K.~J.; Liew, D.~W.; Bhardwaj, R.; Bhardwaj, R.; and Poria, S. 2024{\natexlab{a}}.
\newblock {W}alled{E}val: A Comprehensive Safety Evaluation Toolkit for Large Language Models.
\newblock In Hernandez~Farias, D.~I.; Hope, T.; and Li, M., eds., \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: System Demonstrations}, 397--407. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Gupta et~al.(2024{\natexlab{b}})Gupta, Venkit, Lauren{\c{c}}on, Wilson, and Passonneau}]{gupta2024calm}
Gupta, V.; Venkit, P.~N.; Lauren{\c{c}}on, H.; Wilson, S.; and Passonneau, R.~J. 2024{\natexlab{b}}.
\newblock {CALM} : A Multi-task Benchmark for Comprehensive Assessment of Language Model Bias.
\newblock In \emph{First Conference on Language Modeling}.
\bibitem[{Hall et~al.(2023)Hall, Abrantes, Zhu, Sodunke, Shtedritski, and Kirk}]{hall2023visogender}
Hall, S.~M.; Abrantes, F.~G.; Zhu, H.; Sodunke, G.; Shtedritski, A.; and Kirk, H.~R. 2023.
\newblock VisoGender: A dataset for benchmarking gender bias in image-text pronoun resolution.
\newblock In \emph{Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Han et~al.(2024{\natexlab{a}})Han, Rao, Ettinger, Jiang, Lin, Lambert, Choi, and Dziri}]{han2024wildguard}
Han, S.; Rao, K.; Ettinger, A.; Jiang, L.; Lin, B.~Y.; Lambert, N.; Choi, Y.; and Dziri, N. 2024{\natexlab{a}}.
\newblock WildGuard: Open One-stop Moderation Tools for Safety Risks, Jailbreaks, and Refusals of LLMs.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Han et~al.(2024{\natexlab{b}})Han, Kumar, Agarwal, and Lakkaraju}]{han2024medsafetybench}
Han, T.; Kumar, A.; Agarwal, C.; and Lakkaraju, H. 2024{\natexlab{b}}.
\newblock MedSafetyBench: Evaluating and Improving the Medical Safety of Large Language Models.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Hansson et~al.(2021)Hansson, Mavromatakis, Adesam, Bouma, and Dann{\'e}lls}]{hansson2021swewinogender}
Hansson, S.; Mavromatakis, K.; Adesam, Y.; Bouma, G.; and Dann{\'e}lls, D. 2021.
\newblock The {S}wedish {W}inogender Dataset.
\newblock In Dobnik, S.; and {\O}vrelid, L., eds., \emph{Proceedings of the 23rd Nordic Conference on Computational Linguistics (NoDaLiDa)}, 452--459. Reykjavik, Iceland (Online): Link{\"o}ping University Electronic Press, Sweden.
\bibitem[{Hartvigsen et~al.(2022)Hartvigsen, Gabriel, Palangi, Sap, Ray, and Kamar}]{hartvigsen2022toxigen}
Hartvigsen, T.; Gabriel, S.; Palangi, H.; Sap, M.; Ray, D.; and Kamar, E. 2022.
\newblock {T}oxi{G}en: A Large-Scale Machine-Generated Dataset for Adversarial and Implicit Hate Speech Detection.
\newblock In \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 3309--3326. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Hendrycks et~al.(2021)Hendrycks, Burns, Basart, Critch, Li, Song, and Steinhardt}]{hendrycks2021ethics}
Hendrycks, D.; Burns, C.; Basart, S.; Critch, A.; Li, J.; Song, D.; and Steinhardt, J. 2021.
\newblock Aligning AI With Shared Human Values.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Hendrycks et~al.(2020)Hendrycks, Burns, Basart, Zou, Mazeika, Song, and Steinhardt}]{hendrycks2020mmlu}
Hendrycks, D.; Burns, C.; Basart, S.; Zou, A.; Mazeika, M.; Song, D.; and Steinhardt, J. 2020.
\newblock Measuring Massive Multitask Language Understanding.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Hendrycks et~al.(2022)Hendrycks, Mazeika, Zou, Patel, Zhu, Navarro, Song, Li, and Steinhardt}]{hendrycks2022jiminy}
Hendrycks, D.; Mazeika, M.; Zou, A.; Patel, S.; Zhu, C.; Navarro, J.; Song, D.; Li, B.; and Steinhardt, J. 2022.
\newblock What Would Jiminy Cricket Do? Towards Agents That Behave Morally.
\newblock In \emph{Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}.
\bibitem[{Holtermann et~al.(2024)Holtermann, R{\"o}ttger, Dill, and Lauscher}]{holtermann2024evaluating}
Holtermann, C.; R{\"o}ttger, P.; Dill, T.; and Lauscher, A. 2024.
\newblock Evaluating the Elementary Multilingual Capabilities of Large Language Models with MultiQ.
\newblock \emph{arXiv preprint arXiv:2403.03814}.
\bibitem[{Huang, Shao, and Chang(2022)}]{huang2022personalinfoleak}
Huang, J.; Shao, H.; and Chang, K. C.-C. 2022.
\newblock Are Large Pre-Trained Language Models Leaking Your Personal Information?
\newblock In Goldberg, Y.; Kozareva, Z.; and Zhang, Y., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2022}, 2038--2047. Abu Dhabi, United Arab Emirates: Association for Computational Linguistics.
\bibitem[{Huang et~al.(2024{\natexlab{a}})Huang, Liu, Guo, Sun, Sun, Wang, Zhou, Wang, Teng, Qiu, Wang, and Lin}]{huang2024flames}
Huang, K.; Liu, X.; Guo, Q.; Sun, T.; Sun, J.; Wang, Y.; Zhou, Z.; Wang, Y.; Teng, Y.; Qiu, X.; Wang, Y.; and Lin, D. 2024{\natexlab{a}}.
\newblock Flames: Benchmarking Value Alignment of {LLM}s in {C}hinese.
\newblock In Duh, K.; Gomez, H.; and Bethard, S., eds., \emph{Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)}, 4551--4591. Mexico City, Mexico: Association for Computational Linguistics.
\bibitem[{Huang et~al.(2024{\natexlab{b}})Huang, Gupta, Xia, Li, and Chen}]{huang2024maliciousinstruct}
Huang, Y.; Gupta, S.; Xia, M.; Li, K.; and Chen, D. 2024{\natexlab{b}}.
\newblock Catastrophic Jailbreak of Open-source {LLM}s via Exploiting Generation.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Huang and Xiong(2024)}]{huang2024cbbq}
Huang, Y.; and Xiong, D. 2024.
\newblock {CBBQ}: A {C}hinese Bias Benchmark Dataset Curated with Human-{AI} Collaboration for Large Language Models.
\newblock In Calzolari, N.; Kan, M.-Y.; Hoste, V.; Lenci, A.; Sakti, S.; and Xue, N., eds., \emph{Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)}, 2917--2929. Torino, Italia: ELRA and ICCL.
\bibitem[{Jha et~al.(2023)Jha, Mostafazadeh~Davani, Reddy, Dave, Prabhakaran, and Dev}]{jha2023seegull}
Jha, A.; Mostafazadeh~Davani, A.; Reddy, C.~K.; Dave, S.; Prabhakaran, V.; and Dev, S. 2023.
\newblock {S}ee{GULL}: A Stereotype Benchmark with Broad Geo-Cultural Coverage Leveraging Generative Models.
\newblock In Rogers, A.; Boyd-Graber, J.; and Okazaki, N., eds., \emph{Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 9851--9870. Toronto, Canada: Association for Computational Linguistics.
\bibitem[{Ji et~al.(2023)Ji, Liu, Dai, Pan, Zhang, Bian, Chen, Sun, Wang, and Yang}]{ji2023beavertails}
Ji, J.; Liu, M.; Dai, J.; Pan, X.; Zhang, C.; Bian, C.; Chen, B.; Sun, R.; Wang, Y.; and Yang, Y. 2023.
\newblock BeaverTails: Towards Improved Safety Alignment of {LLM} via a Human-Preference Dataset.
\newblock In \emph{Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Jiang et~al.(2024)Jiang, Rao, Han, Ettinger, Brahman, Kumar, Mireshghallah, Lu, Sap, Choi, and Dziri}]{jiang2024wildteaming}
Jiang, L.; Rao, K.; Han, S.; Ettinger, A.; Brahman, F.; Kumar, S.; Mireshghallah, N.; Lu, X.; Sap, M.; Choi, Y.; and Dziri, N. 2024.
\newblock WildTeaming at Scale: From In-the-Wild Jailbreaks to (Adversarially) Safer Language Models.
\newblock In \emph{The Thirty-eighth Annual Conference on Neural Information Processing Systems}.
\bibitem[{Jin et~al.(2024{\natexlab{a}})Jin, Kim, Lee, Yoo, Oh, and Lee}]{jin2024kobbq}
Jin, J.; Kim, J.; Lee, N.; Yoo, H.; Oh, A.; and Lee, H. 2024{\natexlab{a}}.
\newblock {K}o{BBQ}: {K}orean Bias Benchmark for Question Answering.
\newblock \emph{Transactions of the Association for Computational Linguistics}, 12: 507--524.
\bibitem[{Jin et~al.(2024{\natexlab{b}})Jin, Kleiman-Weiner, Piatti, Levine, Liu, Adauto, Ortu, Strausz, Sachan, Mihalcea, Choi, and Sch{\"o}lkopf}]{jin2024multilingualtrolleyproblems}
Jin, Z.; Kleiman-Weiner, M.; Piatti, G.; Levine, S.; Liu, J.; Adauto, F.~G.; Ortu, F.; Strausz, A.; Sachan, M.; Mihalcea, R.; Choi, Y.; and Sch{\"o}lkopf, B. 2024{\natexlab{b}}.
\newblock Multilingual Trolley Problems for Language Models.
\newblock In \emph{Pluralistic Alignment Workshop at NeurIPS 2024}.
\bibitem[{Jin et~al.(2022)Jin, Levine, Adauto, Kamal, Sap, Sachan, Mihalcea, Tenenbaum, and Sch{\"o}lkopf}]{jin2022moralexceptqa}
Jin, Z.; Levine, S.; Adauto, F.~G.; Kamal, O.; Sap, M.; Sachan, M.; Mihalcea, R.; Tenenbaum, J.~B.; and Sch{\"o}lkopf, B. 2022.
\newblock When to Make Exceptions: Exploring Language Models as Accounts of Human Moral Judgment.
\newblock In Oh, A.~H.; Agarwal, A.; Belgrave, D.; and Cho, K., eds., \emph{Advances in Neural Information Processing Systems}.
\bibitem[{Joshi et~al.(2020)Joshi, Santy, Budhiraja, Bali, and Choudhury}]{joshi2020state}
Joshi, P.; Santy, S.; Budhiraja, A.; Bali, K.; and Choudhury, M. 2020.
\newblock The State and Fate of Linguistic Diversity and Inclusion in the {NLP} World.
\newblock In \emph{Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, 6282--6293. Online: Association for Computational Linguistics.
\bibitem[{Kaiyom et~al.(2024)Kaiyom, Ahmed, Mai, Klyman, Bommasani, and Liang}]{kaiyom2024helmsafety}
Kaiyom, F.; Ahmed, A.; Mai, Y.; Klyman, K.; Bommasani, R.; and Liang, P. 2024.
\newblock HELM Safety: Towards Standardized Safety Evaluations of Language Models.
\newblock \emph{Stanford Center for Research on Foundation Models}.
\bibitem[{Kim et~al.(2022)Kim, Yu, Jiang, Lu, Khashabi, Kim, Choi, and Sap}]{kim2022prosocialdialog}
Kim, H.; Yu, Y.; Jiang, L.; Lu, X.; Khashabi, D.; Kim, G.; Choi, Y.; and Sap, M. 2022.
\newblock {P}rosocial{D}ialog: A Prosocial Backbone for Conversational Agents.
\newblock In Goldberg, Y.; Kozareva, Z.; and Zhang, Y., eds., \emph{Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, 4005--4029. Abu Dhabi, United Arab Emirates: Association for Computational Linguistics.
\bibitem[{Kirk et~al.(2021)Kirk, Jun, Volpin, Iqbal, Benussi, Dreyer, Shtedritski, and Asano}]{kirk2021bias}
Kirk, H.~R.; Jun, Y.; Volpin, F.; Iqbal, H.; Benussi, E.; Dreyer, F.; Shtedritski, A.; and Asano, Y. 2021.
\newblock Bias out-of-the-box: An empirical analysis of intersectional occupational biases in popular generative language models.
\newblock \emph{Advances in neural information processing systems}, 34: 2611--2624.
\bibitem[{Kirk et~al.(2024)Kirk, Whitefield, R{\"o}ttger, Bean, Margatina, Mosquera, Ciro, Bartolo, Williams, He, Vidgen, and Hale}]{kirk2024prism}
Kirk, H.~R.; Whitefield, A.; R{\"o}ttger, P.; Bean, A.~M.; Margatina, K.; Mosquera, R.; Ciro, J.~M.; Bartolo, M.; Williams, A.; He, H.; Vidgen, B.; and Hale, S.~A. 2024.
\newblock The {PRISM} Alignment Dataset: What Participatory, Representative and Individualised Human Feedback Reveals About the Subjective and Multicultural Alignment of Large Language Models.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Koenecke et~al.(2020)Koenecke, Nam, Lake, Nudell, Quartey, Mengesha, Toups, Rickford, Jurafsky, and Goel}]{koenecke2020racial}
Koenecke, A.; Nam, A.; Lake, E.; Nudell, J.; Quartey, M.; Mengesha, Z.; Toups, C.; Rickford, J.~R.; Jurafsky, D.; and Goel, S. 2020.
\newblock Racial disparities in automated speech recognition.
\newblock \emph{Proceedings of the National Academy of Sciences}, 117(14): 7684--7689.
\bibitem[{Kour et~al.(2023)Kour, Zalmanovici, Zwerdling, Goldbraich, Fandina, Anaby~Tavor, Raz, and Farchi}]{kour2023attaq}
Kour, G.; Zalmanovici, M.; Zwerdling, N.; Goldbraich, E.; Fandina, O.; Anaby~Tavor, A.; Raz, O.; and Farchi, E. 2023.
\newblock Unveiling Safety Vulnerabilities of Large Language Models.
\newblock In Gehrmann, S.; Wang, A.; Sedoc, J.; Clark, E.; Dhole, K.; Chandu, K.~R.; Santus, E.; and Sedghamiz, H., eds., \emph{Proceedings of the Third Workshop on Natural Language Generation, Evaluation, and Metrics (GEM)}, 111--127. Singapore: Association for Computational Linguistics.
\bibitem[{LakeraAI(2023{\natexlab{a}})}]{lakera2023gandalf}
LakeraAI. 2023{\natexlab{a}}.
\newblock Gandalf Prompt Injection.
\newblock In \emph{Hugging Face Dataset}.
\bibitem[{LakeraAI(2023{\natexlab{b}})}]{lakera2023mosscap}
LakeraAI. 2023{\natexlab{b}}.
\newblock Mosscap Prompt Injection.
\newblock In \emph{Hugging Face Dataset}.
\bibitem[{Lambert et~al.(2024)Lambert, Pyatkin, Morrison, Miranda, Lin, Chandu, Dziri, Kumar, Zick, Choi, Smith, and Hajishirzi}]{lambert2024rewardbench}
Lambert, N.; Pyatkin, V.; Morrison, J.; Miranda, L.; Lin, B.~Y.; Chandu, K.; Dziri, N.; Kumar, S.; Zick, T.; Choi, Y.; Smith, N.~A.; and Hajishirzi, H. 2024.
\newblock RewardBench: Evaluating Reward Models for Language Modeling.
\newblock arXiv:2403.13787.
\bibitem[{Lee et~al.(2024)Lee, Kim, Kim, Kim, Won, Lee, and Choi}]{lee2024kornat}
Lee, J.; Kim, M.; Kim, S.; Kim, J.; Won, S.; Lee, H.; and Choi, E. 2024.
\newblock {K}or{NAT}: {LLM} Alignment Benchmark for {K}orean Social Values and Common Knowledge.
\newblock In Ku, L.-W.; Martins, A.; and Srikumar, V., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2024}, 11177--11213. Bangkok, Thailand: Association for Computational Linguistics.
\bibitem[{Levy et~al.(2024)Levy, Adler, Karver, Dredze, and Kaufman}]{levy2024demet}
Levy, S.; Adler, W.; Karver, T.~S.; Dredze, M.; and Kaufman, M.~R. 2024.
\newblock Gender Bias in Decision-Making with Large Language Models: A Study of Relationship Conflicts.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2024}, 5777--5800. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Levy et~al.(2022)Levy, Allaway, Subbiah, Chilton, Patton, McKeown, and Wang}]{levy2022safetext}
Levy, S.; Allaway, E.; Subbiah, M.; Chilton, L.; Patton, D.; McKeown, K.; and Wang, W.~Y. 2022.
\newblock {S}afe{T}ext: A Benchmark for Exploring Physical Safety in Language Models.
\newblock In \emph{Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, 2407--2421. Abu Dhabi, United Arab Emirates: Association for Computational Linguistics.
\bibitem[{Li et~al.(2024{\natexlab{a}})Li, Dong, Wang, Hu, Zuo, Lin, Qiao, and Shao}]{li2024saladbench}
Li, L.; Dong, B.; Wang, R.; Hu, X.; Zuo, W.; Lin, D.; Qiao, Y.; and Shao, J. 2024{\natexlab{a}}.
\newblock {SALAD}-Bench: A Hierarchical and Comprehensive Safety Benchmark for Large Language Models.
\newblock In Ku, L.-W.; Martins, A.; and Srikumar, V., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2024}, 3923--3954. Bangkok, Thailand: Association for Computational Linguistics.
\bibitem[{Li et~al.(2024{\natexlab{b}})Li, Pan, Gopal, Yue, Berrios, Gatti, Li, Dombrowski, Goel, Mukobi, Helm-Burger, Lababidi, Justen, Liu, Chen, Barrass, Zhang, Zhu, Tamirisa, Bharathi, Herbert-Voss, Breuer, Zou, Mazeika, Wang, Oswal, Lin, Hunt, Tienken-Harder, Shih, Talley, Guan, Steneker, Campbell, Jokubaitis, Basart, Fitz, Kumaraguru, Karmakar, Tupakula, Varadharajan, Shoshitaishvili, Ba, Esvelt, Wang, and Hendrycks}]{li2024wmdp}
Li, N.; Pan, A.; Gopal, A.; Yue, S.; Berrios, D.; Gatti, A.; Li, J.~D.; Dombrowski, A.-K.; Goel, S.; Mukobi, G.; Helm-Burger, N.; Lababidi, R.; Justen, L.; Liu, A.~B.; Chen, M.; Barrass, I.; Zhang, O.; Zhu, X.; Tamirisa, R.; Bharathi, B.; Herbert-Voss, A.; Breuer, C.~B.; Zou, A.; Mazeika, M.; Wang, Z.; Oswal, P.; Lin, W.; Hunt, A.~A.; Tienken-Harder, J.; Shih, K.~Y.; Talley, K.; Guan, J.; Steneker, I.; Campbell, D.; Jokubaitis, B.; Basart, S.; Fitz, S.; Kumaraguru, P.; Karmakar, K.~K.; Tupakula, U.; Varadharajan, V.; Shoshitaishvili, Y.; Ba, J.; Esvelt, K.~M.; Wang, A.; and Hendrycks, D. 2024{\natexlab{b}}.
\newblock The {WMDP} Benchmark: Measuring and Reducing Malicious Use with Unlearning.
\newblock In \emph{Forty-first International Conference on Machine Learning}.
\bibitem[{Li et~al.(2020)Li, Khashabi, Khot, Sabharwal, and Srikumar}]{li2020unqovering}
Li, T.; Khashabi, D.; Khot, T.; Sabharwal, A.; and Srikumar, V. 2020.
\newblock {UNQOVER}ing Stereotyping Biases via Underspecified Questions.
\newblock In Cohn, T.; He, Y.; and Liu, Y., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2020}, 3475--3489. Online: Association for Computational Linguistics.
\bibitem[{Liang et~al.(2023)Liang, Bommasani, Lee, Tsipras, Soylu, Yasunaga, Zhang, Narayanan, Wu, Kumar et~al.}]{liang2023helm}
Liang, P.; Bommasani, R.; Lee, T.; Tsipras, D.; Soylu, D.; Yasunaga, M.; Zhang, Y.; Narayanan, D.; Wu, Y.; Kumar, A.; et~al. 2023.
\newblock Holistic Evaluation of Language Models.
\newblock \emph{Transactions on Machine Learning Research}.
\bibitem[{Liang et~al.(2021)Liang, Wu, Morency, and Salakhutdinov}]{liang2021lmbias}
Liang, P.~P.; Wu, C.; Morency, L.-P.; and Salakhutdinov, R. 2021.
\newblock Towards understanding and mitigating social biases in language models.
\newblock In \emph{International Conference on Machine Learning}, 6565--6576. PMLR.
\bibitem[{Lin, Hilton, and Evans(2022)}]{lin2022truthfulqa}
Lin, S.; Hilton, J.; and Evans, O. 2022.
\newblock {T}ruthful{QA}: Measuring How Models Mimic Human Falsehoods.
\newblock In Muresan, S.; Nakov, P.; and Villavicencio, A., eds., \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 3214--3252. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Lin et~al.(2023)Lin, Wang, Tong, Wang, Guo, Wang, and Shang}]{lin2023toxicchat}
Lin, Z.; Wang, Z.; Tong, Y.; Wang, Y.; Guo, Y.; Wang, Y.; and Shang, J. 2023.
\newblock {T}oxic{C}hat: Unveiling Hidden Challenges of Toxicity Detection in Real-World User-{AI} Conversation.
\newblock In Bouamor, H.; Pino, J.; and Bali, K., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2023}, 4694--4702. Singapore: Association for Computational Linguistics.
\bibitem[{Liu et~al.(2023)Liu, Zhao, Qing, Kang, Sun, Kuang, and Wu}]{liu2023cpad}
Liu, C.; Zhao, F.; Qing, L.; Kang, Y.; Sun, C.; Kuang, K.; and Wu, F. 2023.
\newblock A chinese prompt attack dataset for llms with evil content.
\newblock \emph{arXiv preprint arXiv:2309.11830}.
\bibitem[{Lourie, Le~Bras, and Choi(2021)}]{lourie2021scruples}
Lourie, N.; Le~Bras, R.; and Choi, Y. 2021.
\newblock Scruples: A corpus of community ethical judgments on 32,000 real-life anecdotes.
\newblock In \emph{Proceedings of the AAAI Conference on Artificial Intelligence}, volume~35, 13470--13479.
\bibitem[{Luccioni et~al.(2024)Luccioni, Akiki, Mitchell, and Jernite}]{luccioni2024stable}
Luccioni, S.; Akiki, C.; Mitchell, M.; and Jernite, Y. 2024.
\newblock Stable bias: Evaluating societal representations in diffusion models.
\newblock \emph{Advances in Neural Information Processing Systems}, 36.
\bibitem[{Manerba et~al.(2024)Manerba, Stanczak, Guidotti, and Augenstein}]{manerba2024social}
Manerba, M.; Stanczak, K.; Guidotti, R.; and Augenstein, I. 2024.
\newblock Social Bias Probing: Fairness Benchmarking for Language Models.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, 14653--14671. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Mazeika et~al.(2024)Mazeika, Phan, Yin, Zou, Wang, Mu, Sakhaee, Li, Basart, Li, Forsyth, and Hendrycks}]{mazeika2024harmbench}
Mazeika, M.; Phan, L.; Yin, X.; Zou, A.; Wang, Z.; Mu, N.; Sakhaee, E.; Li, N.; Basart, S.; Li, B.; Forsyth, D.; and Hendrycks, D. 2024.
\newblock HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal.
\newblock In \emph{Forty-first International Conference on Machine Learning}.
\bibitem[{Mazeika et~al.(2023)Mazeika, Zou, Mu, Phan, Wang, Yu, Khoja, Jiang, O'Gara, Sakhaee, Xiang, Rajabi, Hendrycks, Poovendran, Li, and Forsyth}]{mazeika2023tdc}
Mazeika, M.; Zou, A.; Mu, N.; Phan, L.; Wang, Z.; Yu, C.; Khoja, A.; Jiang, F.; O'Gara, A.; Sakhaee, E.; Xiang, Z.; Rajabi, A.; Hendrycks, D.; Poovendran, R.; Li, B.; and Forsyth, D. 2023.
\newblock TDC 2023 (LLM Edition): The Trojan Detection Challenge.
\newblock In \emph{NeurIPS Competition Track}.
\bibitem[{Meta(2024)}]{meta2024llama3}
Meta. 2024.
\newblock The Llama 3 Herd of Models.
\bibitem[{Meyer et~al.(2020)Meyer, Rauchenstein, Eisenberg, and Howell}]{meyer2020artie}
Meyer, J.; Rauchenstein, L.; Eisenberg, J.~D.; and Howell, N. 2020.
\newblock Artie Bias Corpus: An Open Dataset for Detecting Demographic Bias in Speech Applications.
\newblock In Calzolari, N.; B{\'e}chet, F.; Blache, P.; Choukri, K.; Cieri, C.; Declerck, T.; Goggi, S.; Isahara, H.; Maegaard, B.; Mariani, J.; Mazo, H.; Moreno, A.; Odijk, J.; and Piperidis, S., eds., \emph{Proceedings of the Twelfth Language Resources and Evaluation Conference}, 6462--6468. Marseille, France: European Language Resources Association.
\newblock ISBN 979-10-95546-34-4.
\bibitem[{Mireshghallah et~al.(2024)Mireshghallah, Kim, Zhou, Tsvetkov, Sap, Shokri, and Choi}]{mireshghallah2024confaide}
Mireshghallah, N.; Kim, H.; Zhou, X.; Tsvetkov, Y.; Sap, M.; Shokri, R.; and Choi, Y. 2024.
\newblock Can {LLM}s Keep a Secret? Testing Privacy Implications of Language Models via Contextual Integrity Theory.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Mou, Zhang, and Ye(2024)}]{mou2024sgbench}
Mou, Y.; Zhang, S.; and Ye, W. 2024.
\newblock {SG}-Bench: Evaluating {LLM} Safety Generalization Across Diverse Tasks and Prompt Types.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Mu et~al.(2024)Mu, Chen, Wang, Chen, Karamardian, Aljeraisy, Alomair, Hendrycks, and Wagner}]{mu2024rules}
Mu, N.; Chen, S.; Wang, Z.; Chen, S.; Karamardian, D.; Aljeraisy, L.; Alomair, B.; Hendrycks, D.; and Wagner, D. 2024.
\newblock Can LLMs Follow Simple Rules?
\newblock arXiv:2311.04235.
\bibitem[{Nadeem, Bethke, and Reddy(2021)}]{nadeem2021stereoset}
Nadeem, M.; Bethke, A.; and Reddy, S. 2021.
\newblock {S}tereo{S}et: Measuring stereotypical bias in pretrained language models.
\newblock In Zong, C.; Xia, F.; Li, W.; and Navigli, R., eds., \emph{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}, 5356--5371. Online: Association for Computational Linguistics.
\bibitem[{Nangia et~al.(2020)Nangia, Vania, Bhalerao, and Bowman}]{nangia2020crows}
Nangia, N.; Vania, C.; Bhalerao, R.; and Bowman, S.~R. 2020.
\newblock {C}row{S}-Pairs: A Challenge Dataset for Measuring Social Biases in Masked Language Models.
\newblock In Webber, B.; Cohn, T.; He, Y.; and Liu, Y., eds., \emph{Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)}, 1953--1967. Online: Association for Computational Linguistics.
\bibitem[{N{\'e}v{\'e}ol et~al.(2022)N{\'e}v{\'e}ol, Dupont, Bezan{\c{c}}on, and Fort}]{neveol2022frenchcrows}
N{\'e}v{\'e}ol, A.; Dupont, Y.; Bezan{\c{c}}on, J.; and Fort, K. 2022.
\newblock {F}rench {C}row{S}-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than {E}nglish.
\newblock In Muresan, S.; Nakov, P.; and Villavicencio, A., eds., \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 8521--8531. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Nozza, Bianchi, and Hovy(2021)}]{nozza2021honest}
Nozza, D.; Bianchi, F.; and Hovy, D. 2021.
\newblock {HONEST}: Measuring Hurtful Sentence Completion in Language Models.
\newblock In \emph{Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}, 2398--2406. Online: Association for Computational Linguistics.
\bibitem[{Ontocord.AI(2023)}]{ontocord2023oigmoderation}
Ontocord.AI. 2023.
\newblock OIG-Moderation.
\newblock In \emph{Hugging Face Dataset}.
\bibitem[{Ouyang et~al.(2023)Ouyang, Wang, Liu, Zhong, Jiao, Iter, Pryzant, Zhu, Ji, and Han}]{ouyang2023shifted}
Ouyang, S.; Wang, S.; Liu, Y.; Zhong, M.; Jiao, Y.; Iter, D.; Pryzant, R.; Zhu, C.; Ji, H.; and Han, J. 2023.
\newblock The Shifted and The Overlooked: A Task-oriented Investigation of User-{GPT} Interactions.
\newblock In Bouamor, H.; Pino, J.; and Bali, K., eds., \emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, 2375--2393. Singapore: Association for Computational Linguistics.
\bibitem[{Pan et~al.(2023)Pan, Chan, Zou, Li, Basart, Woodside, Zhang, Emmons, and Hendrycks}]{pan2023machiavelli}
Pan, A.; Chan, J.~S.; Zou, A.; Li, N.; Basart, S.; Woodside, T.; Zhang, H.; Emmons, S.; and Hendrycks, D. 2023.
\newblock Do the rewards justify the means? measuring trade-offs between rewards and ethical behavior in the machiavelli benchmark.
\newblock In \emph{International Conference on Machine Learning}, 26837--26867. PMLR.
\bibitem[{Parrish et~al.(2022)Parrish, Chen, Nangia, Padmakumar, Phang, Thompson, Htut, and Bowman}]{parrish2022bbq}
Parrish, A.; Chen, A.; Nangia, N.; Padmakumar, V.; Phang, J.; Thompson, J.; Htut, P.~M.; and Bowman, S. 2022.
\newblock {BBQ}: A hand-built bias benchmark for question answering.
\newblock In Muresan, S.; Nakov, P.; and Villavicencio, A., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2022}, 2086--2105. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Parrish et~al.(2023)Parrish, Kirk, Quaye, Rastogi, Bartolo, Inel, Ciro, Mosquera, Howard, Cukierski et~al.}]{parrish2023adversarial}
Parrish, A.; Kirk, H.~R.; Quaye, J.; Rastogi, C.; Bartolo, M.; Inel, O.; Ciro, J.; Mosquera, R.; Howard, A.; Cukierski, W.; et~al. 2023.
\newblock Adversarial Nibbler: A Data-Centric Challenge for Improving the Safety of Text-to-Image Models.
\newblock \emph{arXiv preprint arXiv:2305.14384}.
\bibitem[{Perez et~al.(2023)Perez, Ringer, Lukosiute, Nguyen, Chen, Heiner, Pettit, Olsson, Kundu, Kadavath, Jones, Chen, Mann, Israel, Seethor, McKinnon, Olah, Yan, Amodei, Amodei, Drain, Li, Tran-Johnson, Khundadze, Kernion, Landis, Kerr, Mueller, Hyun, Landau, Ndousse, Goldberg, Lovitt, Lucas, Sellitto, Zhang, Kingsland, Elhage, Joseph, Mercado, DasSarma, Rausch, Larson, McCandlish, Johnston, Kravec, El~Showk, Lanham, Telleen-Lawton, Brown, Henighan, Hume, Bai, Hatfield-Dodds, Clark, Bowman, Askell, Grosse, Hernandez, Ganguli, Hubinger, Schiefer, and Kaplan}]{perez2023discovering}
Perez, E.; Ringer, S.; Lukosiute, K.; Nguyen, K.; Chen, E.; Heiner, S.; Pettit, C.; Olsson, C.; Kundu, S.; Kadavath, S.; Jones, A.; Chen, A.; Mann, B.; Israel, B.; Seethor, B.; McKinnon, C.; Olah, C.; Yan, D.; Amodei, D.; Amodei, D.; Drain, D.; Li, D.; Tran-Johnson, E.; Khundadze, G.; Kernion, J.; Landis, J.; Kerr, J.; Mueller, J.; Hyun, J.; Landau, J.; Ndousse, K.; Goldberg, L.; Lovitt, L.; Lucas, M.; Sellitto, M.; Zhang, M.; Kingsland, N.; Elhage, N.; Joseph, N.; Mercado, N.; DasSarma, N.; Rausch, O.; Larson, R.; McCandlish, S.; Johnston, S.; Kravec, S.; El~Showk, S.; Lanham, T.; Telleen-Lawton, T.; Brown, T.; Henighan, T.; Hume, T.; Bai, Y.; Hatfield-Dodds, Z.; Clark, J.; Bowman, S.~R.; Askell, A.; Grosse, R.; Hernandez, D.; Ganguli, D.; Hubinger, E.; Schiefer, N.; and Kaplan, J. 2023.
\newblock Discovering Language Model Behaviors with Model-Written Evaluations.
\newblock In Rogers, A.; Boyd-Graber, J.; and Okazaki, N., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2023}, 13387--13434. Toronto, Canada: Association for Computational Linguistics.
\bibitem[{Pikuliak et~al.(2024)Pikuliak, Oresko, Hrckova, and Simko}]{pikuliak2024gest}
Pikuliak, M.; Oresko, S.; Hrckova, A.; and Simko, M. 2024.
\newblock Women Are Beautiful, Men Are Leaders: Gender Stereotypes in Machine Translation and Language Modeling.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2024}, 3060--3083. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Pistilli et~al.(2024)Pistilli, Leidinger, Jernite, Kasirzadeh, Luccioni, and Mitchell}]{pistilli2024civics}
Pistilli, G.; Leidinger, A.; Jernite, Y.; Kasirzadeh, A.; Luccioni, A.~S.; and Mitchell, M. 2024.
\newblock CIVICS: Building a Dataset for Examining Culturally-Informed Values in Large Language Models.
\newblock \emph{Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society}, 7(1): 1132--1144.
\bibitem[{Qi et~al.(2024)Qi, Zeng, Xie, Chen, Jia, Mittal, and Henderson}]{qi2024hexphi}
Qi, X.; Zeng, Y.; Xie, T.; Chen, P.-Y.; Jia, R.; Mittal, P.; and Henderson, P. 2024.
\newblock Fine-tuning Aligned Language Models Compromises Safety, Even When Users Do Not Intend To!
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Qiu et~al.(2023)Qiu, Zhang, Li, He, and Lan}]{qiu2023latentjailbreak}
Qiu, H.; Zhang, S.; Li, A.; He, H.; and Lan, Z. 2023.
\newblock Latent jailbreak: A benchmark for evaluating text safety and output robustness of large language models.
\newblock \emph{arXiv preprint arXiv:2307.08487}.
\bibitem[{Radharapu et~al.(2023)Radharapu, Robinson, Aroyo, and Lahoti}]{radharapu2023aart}
Radharapu, B.; Robinson, K.; Aroyo, L.; and Lahoti, P. 2023.
\newblock {AART}: {AI}-Assisted Red-Teaming with Diverse Data Generation for New {LLM}-powered Applications.
\newblock In Wang, M.; and Zitouni, I., eds., \emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: Industry Track}, 380--395. Singapore: Association for Computational Linguistics.
\bibitem[{Rashkin et~al.(2019)Rashkin, Smith, Li, and Boureau}]{rashkin2019empatheticdialogues}
Rashkin, H.; Smith, E.~M.; Li, M.; and Boureau, Y.-L. 2019.
\newblock Towards Empathetic Open-domain Conversation Models: A New Benchmark and Dataset.
\newblock In Korhonen, A.; Traum, D.; and M{\`a}rquez, L., eds., \emph{Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, 5370--5381. Florence, Italy: Association for Computational Linguistics.
\bibitem[{Reimao and Tzerpos(2019)}]{reimao2019dataset}
Reimao, R.; and Tzerpos, V. 2019.
\newblock For: A dataset for synthetic speech detection.
\newblock In \emph{2019 International Conference on Speech Technology and Human-Computer Dialogue (SpeD)}, 1--10. IEEE.
\bibitem[{Reuel et~al.(2024)Reuel, Hardy, Smith, Lamparth, Hardy, and Kochenderfer}]{reuel2024betterbench}
Reuel, A.; Hardy, A.; Smith, C.; Lamparth, M.; Hardy, M.; and Kochenderfer, M. 2024.
\newblock BetterBench: Assessing {AI} Benchmarks, Uncovering Issues, and Establishing Best Practices.
\newblock In \emph{The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track}.
\bibitem[{Ricker et~al.(2022)Ricker, Damm, Holz, and Fischer}]{ricker2022deepfakes}
Ricker, J.; Damm, S.; Holz, T.; and Fischer, A. 2022.
\newblock Towards the detection of diffusion model deepfakes.
\newblock \emph{arXiv preprint arXiv:2210.14571}.
\bibitem[{R{\"o}ttger et~al.(2024)R{\"o}ttger, Kirk, Vidgen, Attanasio, Bianchi, and Hovy}]{rottger2024xstest}
R{\"o}ttger, P.; Kirk, H.; Vidgen, B.; Attanasio, G.; Bianchi, F.; and Hovy, D. 2024.
\newblock {XST}est: A Test Suite for Identifying Exaggerated Safety Behaviours in Large Language Models.
\newblock In Duh, K.; Gomez, H.; and Bethard, S., eds., \emph{Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)}, 5377--5400. Mexico City, Mexico: Association for Computational Linguistics.
\bibitem[{Rudinger et~al.(2018)Rudinger, Naradowsky, Leonard, and Van~Durme}]{rudinger2018winogender}
Rudinger, R.; Naradowsky, J.; Leonard, B.; and Van~Durme, B. 2018.
\newblock Gender Bias in Coreference Resolution.
\newblock In Walker, M.; Ji, H.; and Stent, A., eds., \emph{Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)}, 8--14. New Orleans, Louisiana: Association for Computational Linguistics.
\bibitem[{Santurkar et~al.(2023)Santurkar, Durmus, Ladhak, Lee, Liang, and Hashimoto}]{santurkar2023opinionqa}
Santurkar, S.; Durmus, E.; Ladhak, F.; Lee, C.; Liang, P.; and Hashimoto, T. 2023.
\newblock Whose opinions do language models reflect?
\newblock In \emph{Proceedings of the 40th International Conference on Machine Learning}, ICML'23. JMLR.org.
\bibitem[{Scherrer et~al.(2023)Scherrer, Shi, Feder, and Blei}]{scherrer2023evaluating}
Scherrer, N.; Shi, C.; Feder, A.; and Blei, D. 2023.
\newblock Evaluating the Moral Beliefs Encoded in LLMs.
\newblock In \emph{Thirty-seventh Conference on Neural Information Processing Systems}.
\bibitem[{Schulhoff et~al.(2023)Schulhoff, Pinto, Khan, Bouchard, Si, Anati, Tagliabue, Kost, Carnahan, and Boyd-Graber}]{schulhoff2023hackaprompt}
Schulhoff, S.; Pinto, J.; Khan, A.; Bouchard, L.-F.; Si, C.; Anati, S.; Tagliabue, V.; Kost, A.; Carnahan, C.; and Boyd-Graber, J. 2023.
\newblock Ignore This Title and {H}ack{AP}rompt: Exposing Systemic Vulnerabilities of {LLM}s Through a Global Prompt Hacking Competition.
\newblock In Bouamor, H.; Pino, J.; and Bali, K., eds., \emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, 4945--4977. Singapore: Association for Computational Linguistics.
\bibitem[{Schwemmer et~al.(2020)Schwemmer, Knight, Bello-Pardo, Oklobdzija, Schoonvelde, and Lockhart}]{schwemmer2020diagnosing}
Schwemmer, C.; Knight, C.; Bello-Pardo, E.~D.; Oklobdzija, S.; Schoonvelde, M.; and Lockhart, J.~W. 2020.
\newblock Diagnosing gender bias in image recognition systems.
\newblock \emph{Socius}, 6: 2378023120967171.
\bibitem[{Shaikh et~al.(2023)Shaikh, Zhang, Held, Bernstein, and Yang}]{shaikh2023harmfulq}
Shaikh, O.; Zhang, H.; Held, W.; Bernstein, M.; and Yang, D. 2023.
\newblock On Second Thought, Let{'}s Not Think Step by Step! Bias and Toxicity in Zero-Shot Reasoning.
\newblock In Rogers, A.; Boyd-Graber, J.; and Okazaki, N., eds., \emph{Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 4454--4470. Toronto, Canada: Association for Computational Linguistics.
\bibitem[{Sharma et~al.(2024)Sharma, Tong, Korbak, Duvenaud, Askell, Bowman, DURMUS, Hatfield-Dodds, Johnston, Kravec, Maxwell, McCandlish, Ndousse, Rausch, Schiefer, Yan, Zhang, and Perez}]{sharma2024sycophancyeval}
Sharma, M.; Tong, M.; Korbak, T.; Duvenaud, D.; Askell, A.; Bowman, S.~R.; DURMUS, E.; Hatfield-Dodds, Z.; Johnston, S.~R.; Kravec, S.~M.; Maxwell, T.; McCandlish, S.; Ndousse, K.; Rausch, O.; Schiefer, N.; Yan, D.; Zhang, M.; and Perez, E. 2024.
\newblock Towards Understanding Sycophancy in Language Models.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Shen et~al.(2024)Shen, Chen, Backes, Shen, and Zhang}]{shen2024dan}
Shen, X.; Chen, Z.; Backes, M.; Shen, Y.; and Zhang, Y. 2024.
\newblock " do anything now": Characterizing and evaluating in-the-wild jailbreak prompts on large language models.
\newblock In \emph{Proceedings of the 2024 on ACM SIGSAC Conference on Computer and Communications Security}, 1671--1685.
\bibitem[{Sheng et~al.(2019)Sheng, Chang, Natarajan, and Peng}]{sheng2019woman}
Sheng, E.; Chang, K.-W.; Natarajan, P.; and Peng, N. 2019.
\newblock The Woman Worked as a Babysitter: On Biases in Language Generation.
\newblock In Inui, K.; Jiang, J.; Ng, V.; and Wan, X., eds., \emph{Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)}, 3407--3412. Hong Kong, China: Association for Computational Linguistics.
\bibitem[{Shi et~al.(2024)Shi, Wang, Ge, Gao, Yang, Gui, Zhang, Huang, Zhao, and Lin}]{shi2024oktest}
Shi, C.; Wang, X.; Ge, Q.; Gao, S.; Yang, X.; Gui, T.; Zhang, Q.; Huang, X.; Zhao, X.; and Lin, D. 2024.
\newblock Navigating the overkill in large language models.
\newblock \emph{arXiv preprint arXiv:2401.17633}.
\bibitem[{Siddiq and Santos(2022)}]{siddiq2022securityeval}
Siddiq, M.~L.; and Santos, J. C.~S. 2022.
\newblock SecurityEval dataset: mining vulnerability examples to evaluate machine learning-based code generation techniques.
\newblock In \emph{Proceedings of the 1st International Workshop on Mining Software Repositories Applications for Privacy and Security}, MSR4P\&S 2022, 29–33. New York, NY, USA: Association for Computing Machinery.
\newblock ISBN 9781450394574.
\bibitem[{Smith et~al.(2022)Smith, Hall, Kambadur, Presani, and Williams}]{smith2022holisticbias}
Smith, E.~M.; Hall, M.; Kambadur, M.; Presani, E.; and Williams, A. 2022.
\newblock {``}{I}{'}m sorry to hear that{''}: Finding New Biases in Language Models with a Holistic Descriptor Dataset.
\newblock In Goldberg, Y.; Kozareva, Z.; and Zhang, Y., eds., \emph{Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, 9180--9211. Abu Dhabi, United Arab Emirates: Association for Computational Linguistics.
\bibitem[{Sotnikova et~al.(2021)Sotnikova, Cao, Daum{\'e}~III, and Rudinger}]{sotnikova2021hypothesisstereotypes}
Sotnikova, A.; Cao, Y.~T.; Daum{\'e}~III, H.; and Rudinger, R. 2021.
\newblock Analyzing Stereotypes in Generative Text Inference Tasks.
\newblock In Zong, C.; Xia, F.; Li, W.; and Navigli, R., eds., \emph{Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021}, 4052--4065. Online: Association for Computational Linguistics.
\bibitem[{Souly et~al.(2024)Souly, Lu, Bowen, Trinh, Hsieh, Pandey, Abbeel, Svegliato, Emmons, Watkins, and Toyer}]{souly2024strongreject}
Souly, A.; Lu, Q.; Bowen, D.; Trinh, T.; Hsieh, E.; Pandey, S.; Abbeel, P.; Svegliato, J.; Emmons, S.; Watkins, O.; and Toyer, S. 2024.
\newblock A Strong{REJECT} for Empty Jailbreaks.
\newblock In \emph{ICLR 2024 Workshop on Reliable and Responsible Foundation Models}.
\bibitem[{Srivastava et~al.(2023)Srivastava, Rastogi, Rao, Shoeb, Abid, Fisch, Brown, Santoro, Gupta, Garriga-Alonso et~al.}]{srivastava2023bigbench}
Srivastava, A.; Rastogi, A.; Rao, A.; Shoeb, A. A.~M.; Abid, A.; Fisch, A.; Brown, A.~R.; Santoro, A.; Gupta, A.; Garriga-Alonso, A.; et~al. 2023.
\newblock Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models.
\newblock \emph{Transactions on Machine Learning Research}.
\bibitem[{Sun et~al.(2023{\natexlab{a}})Sun, Abzaliev, Kotek, Klein, Xiu, and Williams}]{sun2023delphi}
Sun, D.; Abzaliev, A.; Kotek, H.; Klein, C.; Xiu, Z.; and Williams, J. 2023{\natexlab{a}}.
\newblock {DELPHI}: Data for Evaluating {LLM}s{'} Performance in Handling Controversial Issues.
\newblock In Wang, M.; and Zitouni, I., eds., \emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing: Industry Track}, 820--827. Singapore: Association for Computational Linguistics.
\bibitem[{Sun et~al.(2022)Sun, Xu, Deng, Cheng, Zheng, Zhou, Peng, Zhu, and Huang}]{sun2022diasafety}
Sun, H.; Xu, G.; Deng, J.; Cheng, J.; Zheng, C.; Zhou, H.; Peng, N.; Zhu, X.; and Huang, M. 2022.
\newblock On the Safety of Conversational Models: Taxonomy, Dataset, and Benchmark.
\newblock In Muresan, S.; Nakov, P.; and Villavicencio, A., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2022}, 3906--3923. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Sun et~al.(2023{\natexlab{b}})Sun, Zhang, Deng, Cheng, and Huang}]{sun2023safetyprompts}
Sun, H.; Zhang, Z.; Deng, J.; Cheng, J.; and Huang, M. 2023{\natexlab{b}}.
\newblock Safety assessment of chinese large language models.
\newblock \emph{arXiv preprint arXiv:2304.10436}.
\bibitem[{Sun et~al.(2024)Sun, Huang, Wang, Wu, Zhang, Gao, Huang, Lyu, Zhang, Li et~al.}]{sun2024trustllm}
Sun, L.; Huang, Y.; Wang, H.; Wu, S.; Zhang, Q.; Gao, C.; Huang, Y.; Lyu, W.; Zhang, Y.; Li, X.; et~al. 2024.
\newblock Trustllm: Trustworthiness in large language models.
\newblock \emph{arXiv preprint arXiv:2401.05561}.
\bibitem[{Tamkin et~al.(2023{\natexlab{a}})Tamkin, Askell, Lovitt, Durmus, Joseph, Kravec, Nguyen, Kaplan, and Ganguli}]{tamkin2023evaluating}
Tamkin, A.; Askell, A.; Lovitt, L.; Durmus, E.; Joseph, N.; Kravec, S.; Nguyen, K.; Kaplan, J.; and Ganguli, D. 2023{\natexlab{a}}.
\newblock Evaluating and mitigating discrimination in language model decisions.
\newblock \emph{arXiv preprint arXiv:2312.03689}.
\bibitem[{Tamkin et~al.(2023{\natexlab{b}})Tamkin, Askell, Lovitt, Durmus, Joseph, Kravec, Nguyen, Kaplan, and Ganguli}]{tamkin2023discrimeval}
Tamkin, A.; Askell, A.; Lovitt, L.; Durmus, E.; Joseph, N.; Kravec, S.; Nguyen, K.; Kaplan, J.; and Ganguli, D. 2023{\natexlab{b}}.
\newblock Evaluating and mitigating discrimination in language model decisions.
\newblock \emph{arXiv preprint arXiv:2312.03689}.
\bibitem[{Tan et~al.(2024)Tan, Hansanti, Wood, Yu, Ropers, and Costa-juss{\`a}}]{tan2024mmhb}
Tan, X.~E.; Hansanti, P.; Wood, C.; Yu, B.; Ropers, C.; and Costa-juss{\`a}, M.~R. 2024.
\newblock Towards massive multilingual holistic bias.
\newblock \emph{arXiv preprint arXiv:2407.00486}.
\bibitem[{Tedeschi et~al.(2024)Tedeschi, Friedrich, Schramowski, Kersting, Navigli, Nguyen, and Li}]{tedeschi2024alert}
Tedeschi, S.; Friedrich, F.; Schramowski, P.; Kersting, K.; Navigli, R.; Nguyen, H.; and Li, B. 2024.
\newblock ALERT: A Comprehensive Benchmark for Assessing Large Language Models' Safety through Red Teaming.
\newblock \emph{arXiv preprint arXiv:2404.08676}.
\bibitem[{Touvron et~al.(2023)Touvron, Lavril, Izacard, Martinet, Lachaux, Lacroix, Rozi{\`e}re, Goyal, Hambro, Azhar et~al.}]{touvron2023llama}
Touvron, H.; Lavril, T.; Izacard, G.; Martinet, X.; Lachaux, M.-A.; Lacroix, T.; Rozi{\`e}re, B.; Goyal, N.; Hambro, E.; Azhar, F.; et~al. 2023.
\newblock Llama: Open and efficient foundation language models.
\newblock \emph{arXiv preprint arXiv:2302.13971}.
\bibitem[{Toyer et~al.(2023)Toyer, Watkins, Mendes, Svegliato, Bailey, Wang, Ong, Elmaaroufi, Abbeel, Darrell, Ritter, and Russell}]{toyer2023tensor}
Toyer, S.; Watkins, O.; Mendes, E.; Svegliato, J.; Bailey, L.; Wang, T.; Ong, I.; Elmaaroufi, K.; Abbeel, P.; Darrell, T.; Ritter, A.; and Russell, S. 2023.
\newblock Tensor Trust: Interpretable Prompt Injection Attacks from an Online Game.
\newblock In \emph{NeurIPS 2023 Workshop on Instruction Tuning and Instruction Following}.
\bibitem[{Ung, Xu, and Boureau(2022)}]{ung2022saferdialogues}
Ung, M.; Xu, J.; and Boureau, Y.-L. 2022.
\newblock {S}a{F}e{RD}ialogues: Taking Feedback Gracefully after Conversational Safety Failures.
\newblock In Muresan, S.; Nakov, P.; and Villavicencio, A., eds., \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 6462--6481. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Vidgen et~al.(2024{\natexlab{a}})Vidgen, Agrawal, Ahmed, Akinwande, Al-Nuaimi, Alfaraj, Alhajjar, Aroyo, Bavalatti, Bartolo, Blili-Hamelin, Bollacker, Bomassani, Boston, Campos, Chakra, Chen, Coleman, Coudert, Derczynski, Dutta, Eisenberg, Ezick, Frase, Fuller, Gandikota, Gangavarapu, Gangavarapu, Gealy, Ghosh, Goel, Gohar, Goswami, Hale, Hutiri, Imperial, Jandial, Judd, Juefei-Xu, Khomh, Kailkhura, Kirk, Klyman, Knotz, Kuchnik, Kumar, Kumar, Lengerich, Li, Liao, Long, Lu, Luger, Mai, Mammen, Manyeki, McGregor, Mehta, Mohammed, Moss, Nachman, Naganna, Nikanjam, Nushi, Oala, Orr, Parrish, Patlak, Pietri, Poursabzi-Sangdeh, Presani, Puletti, Röttger, Sahay, Santos, Scherrer, Sebag, Schramowski, Shahbazi, Sharma, Shen, Sistla, Tang, Testuggine, Thangarasa, Watkins, Weiss, Welty, Wilbers, Williams, Wu, Yadav, Yang, Zeng, Zhang, Zhdanov, Zhu, Liang, Mattson, and Vanschoren}]{vidgen2024mlc05}
Vidgen, B.; Agrawal, A.; Ahmed, A.~M.; Akinwande, V.; Al-Nuaimi, N.; Alfaraj, N.; Alhajjar, E.; Aroyo, L.; Bavalatti, T.; Bartolo, M.; Blili-Hamelin, B.; Bollacker, K.; Bomassani, R.; Boston, M.~F.; Campos, S.; Chakra, K.; Chen, C.; Coleman, C.; Coudert, Z.~D.; Derczynski, L.; Dutta, D.; Eisenberg, I.; Ezick, J.; Frase, H.; Fuller, B.; Gandikota, R.; Gangavarapu, A.; Gangavarapu, A.; Gealy, J.; Ghosh, R.; Goel, J.; Gohar, U.; Goswami, S.; Hale, S.~A.; Hutiri, W.; Imperial, J.~M.; Jandial, S.; Judd, N.; Juefei-Xu, F.; Khomh, F.; Kailkhura, B.; Kirk, H.~R.; Klyman, K.; Knotz, C.; Kuchnik, M.; Kumar, S.~H.; Kumar, S.; Lengerich, C.; Li, B.; Liao, Z.; Long, E.~P.; Lu, V.; Luger, S.; Mai, Y.; Mammen, P.~M.; Manyeki, K.; McGregor, S.; Mehta, V.; Mohammed, S.; Moss, E.; Nachman, L.; Naganna, D.~J.; Nikanjam, A.; Nushi, B.; Oala, L.; Orr, I.; Parrish, A.; Patlak, C.; Pietri, W.; Poursabzi-Sangdeh, F.; Presani, E.; Puletti, F.; Röttger, P.; Sahay, S.; Santos, T.; Scherrer, N.; Sebag, A.~S.; Schramowski, P.; Shahbazi,
A.; Sharma, V.; Shen, X.; Sistla, V.; Tang, L.; Testuggine, D.; Thangarasa, V.; Watkins, E.~A.; Weiss, R.; Welty, C.; Wilbers, T.; Williams, A.; Wu, C.-J.; Yadav, P.; Yang, X.; Zeng, Y.; Zhang, W.; Zhdanov, F.; Zhu, J.; Liang, P.; Mattson, P.; and Vanschoren, J. 2024{\natexlab{a}}.
\newblock Introducing v0.5 of the AI Safety Benchmark from MLCommons.
\newblock arXiv:2404.12241.
\bibitem[{Vidgen et~al.(2024{\natexlab{b}})Vidgen, Scherrer, Kirk, Qian, Kannappan, Hale, and Röttger}]{vidgen2024simplesafetytests}
Vidgen, B.; Scherrer, N.; Kirk, H.~R.; Qian, R.; Kannappan, A.; Hale, S.~A.; and Röttger, P. 2024{\natexlab{b}}.
\newblock SimpleSafetyTests: a Test Suite for Identifying Critical Safety Risks in Large Language Models.
\newblock arXiv:2311.08370.
\bibitem[{Wang et~al.(2024{\natexlab{a}})Wang, Chen, Pei, Xie, Kang, Zhang, Xu, Xiong, Dutta, Schaeffer et~al.}]{wang2024decodingtrust}
Wang, B.; Chen, W.; Pei, H.; Xie, C.; Kang, M.; Zhang, C.; Xu, C.; Xiong, Z.; Dutta, R.; Schaeffer, R.; et~al. 2024{\natexlab{a}}.
\newblock DecodingTrust: A Comprehensive Assessment of Trustworthiness in GPT Models.
\newblock \emph{Advances in Neural Information Processing Systems}, 36.
\bibitem[{Wang et~al.(2024{\natexlab{b}})Wang, Tu, Chen, Yuan, Huang, Jiao, and Lyu}]{wang2024xsafety}
Wang, W.; Tu, Z.; Chen, C.; Yuan, Y.; Huang, J.-t.; Jiao, W.; and Lyu, M. 2024{\natexlab{b}}.
\newblock All Languages Matter: On the Multilingual Safety of {LLM}s.
\newblock In Ku, L.-W.; Martins, A.; and Srikumar, V., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2024}, 5865--5877. Bangkok, Thailand: Association for Computational Linguistics.
\bibitem[{Wang et~al.(2024{\natexlab{c}})Wang, Li, Han, Nakov, and Baldwin}]{wang2024donotanswer}
Wang, Y.; Li, H.; Han, X.; Nakov, P.; and Baldwin, T. 2024{\natexlab{c}}.
\newblock Do-Not-Answer: Evaluating Safeguards in {LLM}s.
\newblock In Graham, Y.; and Purver, M., eds., \emph{Findings of the Association for Computational Linguistics: EACL 2024}, 896--911. St. Julian{'}s, Malta: Association for Computational Linguistics.
\bibitem[{Wei, Haghtalab, and Steinhardt(2023)}]{wei2023jailbroken}
Wei, A.; Haghtalab, N.; and Steinhardt, J. 2023.
\newblock Jailbroken: How Does LLM Safety Training Fail?
\newblock \emph{arXiv preprint arXiv:2307.02483}.
\bibitem[{Wolfe et~al.(2023)Wolfe, Yang, Howe, and Caliskan}]{wolfe2023visionlanguagebias}
Wolfe, R.; Yang, Y.; Howe, B.; and Caliskan, A. 2023.
\newblock Contrastive Language-Vision AI Models Pretrained on Web-Scraped Multimodal Data Exhibit Sexual Objectification Bias.
\newblock In \emph{Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency}, FAccT '23, 1174–1185. New York, NY, USA: Association for Computing Machinery.
\newblock ISBN 9798400701924.
\bibitem[{Xie et~al.(2024)Xie, Qi, Zeng, Huang, Sehwag, Huang, He, Wei, Li, Sheng et~al.}]{xie2024sorrybench}
Xie, T.; Qi, X.; Zeng, Y.; Huang, Y.; Sehwag, U.~M.; Huang, K.; He, L.; Wei, B.; Li, D.; Sheng, Y.; et~al. 2024.
\newblock Sorry-bench: Systematically evaluating large language model safety refusal behaviors.
\newblock \emph{arXiv preprint arXiv:2406.14598}.
\bibitem[{Xu et~al.(2023)Xu, Liu, Yan, Xu, Si, Zhou, Yi, Gao, Sang, Zhang et~al.}]{xu2023cvalues}
Xu, G.; Liu, J.; Yan, M.; Xu, H.; Si, J.; Zhou, Z.; Yi, P.; Gao, X.; Sang, J.; Zhang, R.; et~al. 2023.
\newblock Cvalues: Measuring the values of chinese large language models from safety to responsibility.
\newblock \emph{arXiv preprint arXiv:2307.09705}.
\bibitem[{Xu et~al.(2021)Xu, Ju, Li, Boureau, Weston, and Dinan}]{xu2021botadversarialdialogue}
Xu, J.; Ju, D.; Li, M.; Boureau, Y.-L.; Weston, J.; and Dinan, E. 2021.
\newblock Bot-Adversarial Dialogue for Safe Conversational Agents.
\newblock In Toutanova, K.; Rumshisky, A.; Zettlemoyer, L.; Hakkani-Tur, D.; Beltagy, I.; Bethard, S.; Cotterell, R.; Chakraborty, T.; and Zhou, Y., eds., \emph{Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}, 2950--2968. Online: Association for Computational Linguistics.
\bibitem[{Yang et~al.(2024)Yang, Yang, Hui, Zheng, Yu, Zhou, Li, Li, Liu, Huang, Dong, Wei, Lin, Tang, Wang, Yang, Tu, Zhang, Ma, Yang, Xu, Zhou, Bai, He, Lin, Dang, Lu, Chen, Yang, Li, Xue, Ni, Zhang, Wang, Peng, Men, Gao, Lin, Wang, Bai, Tan, Zhu, Li, Liu, Ge, Deng, Zhou, Ren, Zhang, Wei, Ren, Liu, Fan, Yao, Zhang, Wan, Chu, Liu, Cui, Zhang, Guo, and Fan}]{yang2024qwen2}
Yang, A.; Yang, B.; Hui, B.; Zheng, B.; Yu, B.; Zhou, C.; Li, C.; Li, C.; Liu, D.; Huang, F.; Dong, G.; Wei, H.; Lin, H.; Tang, J.; Wang, J.; Yang, J.; Tu, J.; Zhang, J.; Ma, J.; Yang, J.; Xu, J.; Zhou, J.; Bai, J.; He, J.; Lin, J.; Dang, K.; Lu, K.; Chen, K.; Yang, K.; Li, M.; Xue, M.; Ni, N.; Zhang, P.; Wang, P.; Peng, R.; Men, R.; Gao, R.; Lin, R.; Wang, S.; Bai, S.; Tan, S.; Zhu, T.; Li, T.; Liu, T.; Ge, W.; Deng, X.; Zhou, X.; Ren, X.; Zhang, X.; Wei, X.; Ren, X.; Liu, X.; Fan, Y.; Yao, Y.; Zhang, Y.; Wan, Y.; Chu, Y.; Liu, Y.; Cui, Z.; Zhang, Z.; Guo, Z.; and Fan, Z. 2024.
\newblock Qwen2 Technical Report.
\newblock arXiv:2407.10671.
\bibitem[{Yu et~al.(2024{\natexlab{a}})Yu, Li, Liao, Wang, Zuchen, Mi, and Hong}]{yu2024cosafe}
Yu, E.; Li, J.; Liao, M.; Wang, S.; Zuchen, G.; Mi, F.; and Hong, L. 2024{\natexlab{a}}.
\newblock {C}o{S}afe: Evaluating Large Language Model Safety in Multi-Turn Dialogue Coreference.
\newblock In Al-Onaizan, Y.; Bansal, M.; and Chen, Y.-N., eds., \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, 17494--17508. Miami, Florida, USA: Association for Computational Linguistics.
\bibitem[{Yu, Li, and Lan(2024{\natexlab{a}})}]{yu2024beyond}
Yu, J.; Li, L.; and Lan, Z. 2024{\natexlab{a}}.
\newblock Beyond Binary Classification: A Fine-grained Safety Dataset for Large Language Models.
\newblock \emph{IEEE Access}.
\bibitem[{Yu, Li, and Lan(2024{\natexlab{b}})}]{yu2024safe}
Yu, J.; Li, L.; and Lan, Z. 2024{\natexlab{b}}.
\newblock Beyond Binary Classification: A Fine-Grained Safety Dataset for Large Language Models.
\newblock \emph{IEEE Access}, 12: 64717--64726.
\bibitem[{Yu et~al.(2024{\natexlab{b}})Yu, Lin, Yu, and Xing}]{yu2024gptfuzzer}
Yu, J.; Lin, X.; Yu, Z.; and Xing, X. 2024{\natexlab{b}}.
\newblock Gptfuzzer: Red teaming large language models with auto-generated jailbreak prompts.
\newblock \emph{arXiv preprint arXiv:2309.10253}.
\bibitem[{Yu et~al.(2024{\natexlab{c}})Yu, Leng, Huang, Wu, Liu, Ji, Zhao, Song, Cui, Cheng, Liutao, and Xiong}]{yu2024cmoraleval}
Yu, L.; Leng, Y.; Huang, Y.; Wu, S.; Liu, H.; Ji, X.; Zhao, J.; Song, J.; Cui, T.; Cheng, X.; Liutao, L.; and Xiong, D. 2024{\natexlab{c}}.
\newblock {CM}oral{E}val: A Moral Evaluation Benchmark for {C}hinese Large Language Models.
\newblock In Ku, L.-W.; Martins, A.; and Srikumar, V., eds., \emph{Findings of the Association for Computational Linguistics: ACL 2024}, 11817--11837. Bangkok, Thailand: Association for Computational Linguistics.
\bibitem[{Yuan et~al.(2024)Yuan, Li, Wang, Chen, Mao, Huang, Xue, Wang, Ren, and Wang}]{yuan2024seval}
Yuan, X.; Li, J.; Wang, D.; Chen, Y.; Mao, X.; Huang, L.; Xue, H.; Wang, W.; Ren, K.; and Wang, J. 2024.
\newblock S-Eval: Automatic and Adaptive Test Generation for Benchmarking Safety Evaluation of Large Language Models.
\newblock \emph{arXiv preprint arXiv:2405.14191}.
\bibitem[{Zhang, Pan, and Yang(2023)}]{zhang2023jade}
Zhang, M.; Pan, X.; and Yang, M. 2023.
\newblock Jade: A linguistics-based safety evaluation platform for llm.
\newblock \emph{arXiv preprint arXiv:2311.00286}.
\bibitem[{Zhang et~al.(2024{\natexlab{a}})Zhang, Lei, Liu, An, Yang, Zhao, Wang, and Lian}]{zhang2024chisafetybench}
Zhang, W.; Lei, X.; Liu, Z.; An, M.; Yang, B.; Zhao, K.; Wang, K.; and Lian, S. 2024{\natexlab{a}}.
\newblock CHiSafetyBench: A Chinese Hierarchical Safety Benchmark for Large Language Models.
\newblock \emph{arXiv preprint arXiv:2406.10311}.
\bibitem[{Zhang et~al.(2024{\natexlab{b}})Zhang, Mai, Roberts, Bommasani, Dubois, and Liang}]{zhang2024helminstruct}
Zhang, Y.; Mai, Y.; Roberts, J.~S.; Bommasani, R.; Dubois, Y.; and Liang, P. 2024{\natexlab{b}}.
\newblock HELM Instruct: A Multidimensional Instruction Following Evaluation Framework with Absolute Ratings.
\bibitem[{Zhang et~al.(2024{\natexlab{c}})Zhang, Lei, Wu, Sun, Huang, Long, Liu, Lei, Tang, and Huang}]{zhang2024safetybench}
Zhang, Z.; Lei, L.; Wu, L.; Sun, R.; Huang, Y.; Long, C.; Liu, X.; Lei, X.; Tang, J.; and Huang, M. 2024{\natexlab{c}}.
\newblock {S}afety{B}ench: Evaluating the Safety of Large Language Models.
\newblock In Ku, L.-W.; Martins, A.; and Srikumar, V., eds., \emph{Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 15537--15553. Bangkok, Thailand: Association for Computational Linguistics.
\bibitem[{Zhao, Wang, and Russakovsky(2021)}]{zhao2021understanding}
Zhao, D.; Wang, A.; and Russakovsky, O. 2021.
\newblock Understanding and evaluating racial biases in image captioning.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, 14830--14840.
\bibitem[{Zhao et~al.(2023)Zhao, Fang, Shi, Li, Chen, and Pechenizkiy}]{zhao2023chbias}
Zhao, J.; Fang, M.; Shi, Z.; Li, Y.; Chen, L.; and Pechenizkiy, M. 2023.
\newblock {CHB}ias: Bias Evaluation and Mitigation of {C}hinese Conversational Language Models.
\newblock In Rogers, A.; Boyd-Graber, J.; and Okazaki, N., eds., \emph{Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 13538--13556. Toronto, Canada: Association for Computational Linguistics.
\bibitem[{Zhao et~al.(2018)Zhao, Wang, Yatskar, Ordonez, and Chang}]{zhao2018winobias}
Zhao, J.; Wang, T.; Yatskar, M.; Ordonez, V.; and Chang, K.-W. 2018.
\newblock Gender Bias in Coreference Resolution: Evaluation and Debiasing Methods.
\newblock In Walker, M.; Ji, H.; and Stent, A., eds., \emph{Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)}, 15--20. New Orleans, Louisiana: Association for Computational Linguistics.
\bibitem[{Zhao et~al.(2024{\natexlab{a}})Zhao, Mondal, Tandon, Dillion, Gray, and Gu}]{zhao2024worldvaluesbench}
Zhao, W.; Mondal, D.; Tandon, N.; Dillion, D.; Gray, K.; and Gu, Y. 2024{\natexlab{a}}.
\newblock {W}orld{V}alues{B}ench: A Large-Scale Benchmark Dataset for Multi-Cultural Value Awareness of Language Models.
\newblock In Calzolari, N.; Kan, M.-Y.; Hoste, V.; Lenci, A.; Sakti, S.; and Xue, N., eds., \emph{Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)}, 17696--17706. Torino, Italia: ELRA and ICCL.
\bibitem[{Zhao et~al.(2024{\natexlab{b}})Zhao, Ren, Hessel, Cardie, Choi, and Deng}]{zhao2024inthewildchat}
Zhao, W.; Ren, X.; Hessel, J.; Cardie, C.; Choi, Y.; and Deng, Y. 2024{\natexlab{b}}.
\newblock (InThe)WildChat: 570K Chat{GPT} Interaction Logs In The Wild.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Zheng et~al.(2024)Zheng, Chiang, Sheng, Li, Zhuang, Wu, Zhuang, Li, Lin, Xing, Gonzalez, Stoica, and Zhang}]{zheng2024lmsys}
Zheng, L.; Chiang, W.-L.; Sheng, Y.; Li, T.; Zhuang, S.; Wu, Z.; Zhuang, Y.; Li, Z.; Lin, Z.; Xing, E.; Gonzalez, J.~E.; Stoica, I.; and Zhang, H. 2024.
\newblock LMSYS-1M: A Large-Scale Real-World {LLM} Conversation Dataset.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Zhou et~al.(2022)Zhou, Deng, Mi, Li, Wang, Huang, Jiang, Liu, and Meng}]{zhou2022cdial}
Zhou, J.; Deng, J.; Mi, F.; Li, Y.; Wang, Y.; Huang, M.; Jiang, X.; Liu, Q.; and Meng, H. 2022.
\newblock Towards Identifying Social Bias in Dialog Systems: Framework, Dataset, and Benchmark.
\newblock In Goldberg, Y.; Kozareva, Z.; and Zhang, Y., eds., \emph{Findings of the Association for Computational Linguistics: EMNLP 2022}, 3576--3591. Abu Dhabi, United Arab Emirates: Association for Computational Linguistics.
\bibitem[{Ziems et~al.(2022)Ziems, Yu, Wang, Halevy, and Yang}]{ziems2022moralintegrity}
Ziems, C.; Yu, J.; Wang, Y.-C.; Halevy, A.; and Yang, D. 2022.
\newblock The Moral Integrity Corpus: A Benchmark for Ethical Dialogue Systems.
\newblock In Muresan, S.; Nakov, P.; and Villavicencio, A., eds., \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, 3755--3773. Dublin, Ireland: Association for Computational Linguistics.
\bibitem[{Zou et~al.(2023)Zou, Wang, Carlini, Nasr, Kolter, and Fredrikson}]{zou2023advbench}
Zou, A.; Wang, Z.; Carlini, N.; Nasr, M.; Kolter, J.~Z.; and Fredrikson, M. 2023.
\newblock Universal and transferable adversarial attacks on aligned language models.
\newblock \emph{arXiv preprint arXiv:2307.15043}.
\end{thebibliography}