AMoPO: Adaptive Multi-objective Preference Optimization without Reward Models and Reference Models
Extracted source text · arXiv 2506.07165
% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.
\documentclass[11pt]{article}
% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\setlength{\textfloatsep}{3mm}
\usepackage{array}
\usepackage{booktabs} %调整表格线与上下内容的间隔
\usepackage{multirow}
\usepackage{tabularx}
\usepackage{graphicx}
\usepackage{makecell}
\usepackage{caption}
\usepackage{subcaption}
\usepackage[titletoc]{appendix}
\usepackage{textcomp}
\usepackage{xspace}
\usepackage{soul} % 导入 soul 包
\usepackage{color}
\usepackage{xcolor}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{mathrsfs}
\usepackage{bm}
\definecolor{shadecolor}{rgb}{0.92,0.92,0.92}
\usepackage{tcolorbox}
\usepackage[export]{adjustbox}
\definecolor{mygray}{RGB}{230,230,240}
\definecolor{myblue}{RGB}{175, 238, 235}
\definecolor{deepgreen}{rgb}{0.0, 0.5, 0.0}
\usepackage[linesnumbered,ruled,vlined]{algorithm2e}
\usepackage{multicol}
\usepackage{tikz}
\usetikzlibrary{shapes, positioning}
\usepackage{setspace}
\usepackage{hyperref}
\usepackage{enumitem}
\usepackage[skip=5pt]{caption}
\usepackage{xcolor} % 添加批注颜色
\usepackage{soul}
\sethlcolor{white}
\soulregister\cite7 % 针对\cite命令
\soulregister\citep7 % 针对\citep命令
\soulregister\citet7 % 针对\citet命令
\soulregister\ref7 % 针对\ref命令
\soulregister\pageref7 % 针对\pageref命令
\soulregister\eqref7 % 针对\eqref命令
\newcommand{\jqruan}[1]{\textcolor{deepgreen}{jqruan:[#1]}}
\newcommand{\todo}[1]{\textcolor{magenta}{TODO:[#1]}}
% \newcommand{\todo}[1]{\textcolor{black}{TODO:[#1]}}
\newcommand{\ourtitle}{AMoPO}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets
% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}
\usepackage{hyperref}
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.
% \title{RMoPO: Reference-free Multi-objective Preference Optimization}
\title{AMoPO: Adaptive Multi-objective Preference Optimization without Reward Models and Reference Models}
% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
% Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\ ... \\ Address line
% \And ... \And
% Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\ ... \\ Address line
% \AND
% Author 2 \\ Address line \\ ... \\ Address line \And
% Author 3 \\ Address line \\ ... \\ Address line}
% \author{First Author \\
% Affiliation / Address line 1 \\
% Affiliation / Address line 2 \\
% Affiliation / Address line 3 \\
% \texttt{email@domain} \\\And
% Second Author \\
% Affiliation / Address line 1 \\
% Affiliation / Address line 2 \\
% Affiliation / Address line 3 \\
% \texttt{email@domain} \\}
%\author{
% \textbf{First Author\textsuperscript{1}},
% \textbf{Second Author\textsuperscript{1,2}},
% \textbf{Third T. Author\textsuperscript{1}},
% \textbf{Fourth Author\textsuperscript{1}},
%\\
% \textbf{Fifth Author\textsuperscript{1,2}},
% \textbf{Sixth Author\textsuperscript{1}},
% \textbf{Seventh Author\textsuperscript{1}},
% \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
% \textbf{Ninth Author\textsuperscript{1}},
% \textbf{Tenth Author\textsuperscript{1}},
% \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
% \textbf{Twelfth Author\textsuperscript{1}},
%\\
% \textbf{Thirteenth Author\textsuperscript{3}},
% \textbf{Fourteenth F. Author\textsuperscript{2,4}},
% \textbf{Fifteenth Author\textsuperscript{1}},
% \textbf{Sixteenth Author\textsuperscript{1}},
%\\
% \textbf{Seventeenth S. Author\textsuperscript{4,5}},
% \textbf{Eighteenth Author\textsuperscript{3,4}},
% \textbf{Nineteenth N. Author\textsuperscript{2,5}},
% \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
% \textsuperscript{1}Affiliation 1,
% \textsuperscript{2}Affiliation 2,
% \textsuperscript{3}Affiliation 3,
% \textsuperscript{4}Affiliation 4,
% \textsuperscript{5}Affiliation 5
%\\
% \small{
% \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
% }
%}
\makeatletter
\renewcommand\@fnsymbol[1]{\ifcase#1\or $\dagger$ \or * \fi}
\makeatother
\author{
\textbf{Qi Liu}\textsuperscript{1},
\textbf{Jingqing Ruan}\thanks{Lead this project. \quad * Corresponding authors.}\textsuperscript{2},
\textbf{Hao Li}\textsuperscript{2},
\textbf{Haodong Zhao}\textsuperscript{2},
\textbf{Desheng Wang}\textsuperscript{2},
\textbf{Jiansong Chen}\textsuperscript{*2}, \\
\textbf{Guanglu Wan}\textsuperscript{2},
\textbf{Xunliang Cai}\textsuperscript{2},
\textbf{Zhi Zheng}\textsuperscript{1},
\textbf{Tong Xu}\textsuperscript{*1} \\
\textsuperscript{1} University of Science and Technology of China, State Key Laboratory of Cognitive Intelligence, Hefei, China \\
\textsuperscript{2} Meituan \\
\texttt{\{ruanjingqing,lihao138,zhaohaodong04,wangdesheng08,chenjiansong\}@meituan.com} \\
\texttt{liuqilq@mail.ustc.edu.cn} \\
\texttt{\{zhengzhi97, tongxu\}@ustc.edu.cn}
}
\begin{document}
\maketitle
% \begin{abstract}
% % 2个痛点：1.计算资源大 2.难以适应到多维度上
% % 我们的模型的 2个优势：1.轻量和适应性
% % 实验取得了效果宏观上36.5% ，详细分析了之后，在每个维度上都提升了质量
% The rapid advancements in Large Language Models (LLMs) require aligning their responses with human preferences to ensure the development of safe and controllable AI systems.
% However, existing methods often struggle to capture the complex and multi-dimensional nature of these preferences. This paper introduces Adaptive Multi-objective Preference Optimization (\ourtitle), a framework that enhances adaptability and efficiency in multi-objective preference optimization.
% By improving the Bradley-Terry model and using generation metrics as implicit rewards, our approach aligns LLMs with diverse preferences without additional reward models or reference models. An adaptive weight assignment mechanism models the generation space as a Gaussian distribution, allowing dynamic prioritization of preference dimensions.
% Empirical results demonstrate that {\ourtitle} outperforms state-of-the-art baselines, highlighting its effectiveness. Our code is available at xxxx.
% our approach achieves 36.5$\%$
% overall improvements over the baseline methods
% \end{abstract}
\input{latex/sections_article/0_abs_lq}
\section{Introduction}
% \input{latex/sections_article/1_introduction}
\input{latex/sections_article/1_intro_rjq}
\section{Related Work}
% \input{latex/sections_article/2_related_work}
\input{latex/sections_article/2_related_rjq}
\section{Preliminary}
% \input{latex/sections_article/3_preliminary}
\input{latex/sections_article/3_pre_rjq}
% \section{Adaptive Multi-objective Preference Optimization without
% Rewards and References(\ourtitle)}
\section{Methodology}
% \input{latex/sections_article/4_RMoPO}
\input{latex/sections_article/4_RMoPO_rjq}
\section{Experiments}
\input{latex/sections_article/5_Experiment}
\vspace{-0.2cm}
\section{Conclusions}
\input{latex/sections_article/6_con}
\section*{Acknowledgments}
\input{latex/sections_article/9_ack}
\clearpage
\section*{Limitations}
\input{latex/sections_article/7_limitations}
% \section*{Acknowledgments}
\clearpage
% \todo{@liuqi : see comments.}
% 这块如果有时间的话 麻烦把bib 全部切成统一的源格式，比如全部走dblp；
% 注意：title、作者格式、以及 期刊需包含刊号页码；会议需包含页码；
% 只有找不到期刊会议的文章，才可以用 arxiv；
\bibliography{latex/custom}
\clearpage
\appendix
\input{latex/sections_article/8_app}
\end{document}
\begin{abstract}
% 2个痛点：1.计算资源大 2.难以适应到多维度上
% 我们的模型的 2个优势：1.轻量和适应性
% 实验取得了效果宏观上36.5% ，详细分析了之后，在每个维度上都提升了质量
% The complex and multi-dimensional preference alignment is vital for large language models. However, existing multi-objective preference optimization methods often suffer from high computational overhead and difficulty adapting to multiple dimensions.
% \hl{Aligning large language models with multi-dimensional human preferences remains a critical challenge, as existing multi-objective optimization methods often fail to dynamically balance various preference dimensions and suffer from high computational overhead.}
% We introduce Adaptive Multi-objective Preference Optimization (\ourtitle), a framework that enhances adaptability and efficiency in multi-objective preference optimization.
% % 我们设计了一个新的范式，而不是提升了什么什么
% By designing a new paradigm and using generation metrics as implicit rewards, our approach aligns LLMs with diverse preferences without additional reward models or reference models.
% An adaptive weight assignment mechanism models the generation space as a Gaussian distribution, allowing dynamic prioritization of preference dimensions.
% \hl{Empirical results demonstrate that {\ourtitle} outperforms state-of-the-art baselines by 31.3$\%$, highlighting its superiority. }
% \hl{In addition, the ablation study on 7B, 14B and 32B model proves the scaling ability of {~\ourtitle}. }
% Moreover, additional analysis of multiple dimensions verifies its adaptability and effectiveness.
% % by surpassing the scores of existing baselines.
% Our codes and datasets are available at
% \href{https://anonymous.4open.science/r/AMoPO}{https://anonymous.4open.science/r/AMoPO/}.
\hl{Existing multi-objective preference alignment methods for large language models (LLMs) face limitations: (1) the inability to effectively balance various preference dimensions, and (2) reliance on auxiliary reward/reference models introduces computational complexity.
To address these challenges, we propose Adaptive Multi-objective Preference Optimization (AMoPO), a novel framework that achieves dynamic balance across preference dimensions.
By introducing the multi-objective optimization paradigm to use the dimension-aware generation metrics as implicit rewards, AMoPO aligns LLMs with diverse preferences without additional reward models or reference models.
We introduce an adaptive weight assignment mechanism that models the generation space as a Gaussian distribution, allowing dynamic prioritization of preference dimensions.
Empirical results demonstrate that {~\ourtitle} outperforms state-of-the-art baselines by 28.5$\%$, and the experiments on 7B, 14B, and 32B models reveal the scaling ability of \ourtitle.}
Moreover, additional analysis of multiple dimensions verifies its adaptability and effectiveness.
\hl{These findings validate AMoPO's capability to achieve dimension-aware preference alignment, highlighting its superiority.}
Our codes and datasets are available at
\href{https://github.com/Javkonline/AMoPO}{https://github.com/Javkonline/AMoPO}.
\end{abstract}
% Introduction
% jqruan{
% 首先，我们主打两个卖点：无参考模型(低能耗)、动态&多目标
% 直接切入MO好吗(主要实验比较也有单目标的，所以可以引一小部分)
% 大语言模型在预训练阶段已经取得了显著的进展，但在实际应用中，模型可能会输出不符合用户期望或产生有害内容。
% 因此，其能力仍需通过进一步的微调来充分发挥，尤其是在偏好对齐方面。当前主流的方案包含xxxx（单目标的主流方案可列），他们解决了什么问题，存在什么问题；
% 然而，评估回复质量是一个复杂的过程，通常需要从多个维度进行考量。这些维度包括指令的遵循性、拟人化程度以及上下文相关性等。多维度的评估对于大型语言模型的训练和优化至关重要。
% }
\hl{Recent breakthroughs in Large Language Models (LLMs)}~\citep{gpt42023openai,llama3,qwen2.5} \hl{have intensified the need for alignment with multi-dimensional human preferences, which is a critical requirement for deploying safe and controllable AI systems.}
\hl{While preference optimization methods like RLHF}~\citep{ouyang2022training} \hl{and DPO series}~\citep{DPO2023Rafailov,ethayarajh2024kto,ipo2024Azar,CPO2024xu} \hl{have advanced single-objective alignment, fundamental limitations persist when addressing the multi-dimensional nature of real-world human preferences}~\citep{helpsteer22024wang,ultrafeedback2024cui,aaai2025xuderong}.
\hl{Figure}~\ref{fig:intro} \hl{shows that different LLMs favor different criteria but struggle to balance these dimensions, highlighting the need for more comprehensive multi-objective alignment methods.}
% With the rapid advancements in Large Language Models (LLMs)~\cite{gpt42023openai,llama3,qwen2.5}, it is essential to ensure that these models produce responses and behaviors that align with human preferences, which is crucial for the development of safe and controllable AI systems.
% Preference optimization methods, such as Reinforcement Learning from Human Feedback (RLHF)~\citep{ouyang2022training,finetune2019Ziegler} and its RL-free alternative, Direct Preference Optimization (DPO)~\citep{DPO2023Rafailov}, have become active areas of research.
% % DPO is a proven technique that simplifies the complex RLHF by directly using human preferences to fine-tune LLMs through supervised learning.
% Currently, progress~\citep{ethayarajh2024kto,ipo2024Azar,CPO2024xu} has been made in developing LLMs capable of adhering to user preferences.
% However, these methods struggle to capture the true diversity of human preferences, which are inherently complex and multi-dimensional~\citep{helpsteer22024wang, ultrafeedback2024cui, rewardsoup2023Rame}.
% % % 原来的
% % For instance, Figure~\ref{fig:intro} illustrates that user 1 and user 2 have different objectives when querying LLMs about travel plans.
% % 现在的
% For instance, Figure~\ref{fig:intro} \hl{illustrates that different language models have different objectives when querying the same question about LeBron James.}
% This highlights the necessity to explore multi-objective preference optimization to better align with the diverse spectrum of human preferences.
% \todo{deeper!}
% \todo{the necessity of MO? a case or an analysis with Figure?}
% （需要进一步思考补充：为什么需要多维度？有没有什么必须要做的痛点？）
% 这里是不是要挖掘的更深入一些？
%\begin{figure}[t]
% \centering
% \includegraphics[width=0.9\linewidth]{latex/images/main/travel.pdf}
% \caption{Illustration of diverse user preferences.}
% \label{fig:intro}
% \vspace{-10pt}
%\end{figure}
\begin{figure}[t]
\centering
\includegraphics[width=1.0\linewidth]{latex/images/main/intro_lq_0215_new_final.pdf}
% \caption{\hl{The responses of different models across multiple dimensions.}}
\caption{\hl{LLM1 prefers helpfulness while LLM2 prefers correctness, yet both struggle to adapt across various dimensions.}}
\label{fig:intro}
\end{figure}
% \jqruan{
% 当前领域解决多维度偏好对齐的工作可以划分为两大类。一条线是xxx系列，包含xxx，有什么缺陷
% 为了缓解上述问题，另外一大类xxx分支被引入，但是有xx问题。
% 为了解决以上问题，我们xxx
% （TODO：当前领域内公认的缺陷有哪些。）
% 为了优化多个维度的偏好，当前的研究可以被划分为两个大的流派。
% 一类是Instruction-controlled methods，which insert control tokens into the prompt, resulting in a single controllable aligned model. But as a cost, their methods are poor in extensibility since their prompt format is pre-defined on
% existing alignment objectives and cannot extend to a new alignment objective.
% 一个是Multi-model tuning methods, which 涉及到对于每个偏好维度训练多个奖励、甚至是语言模型。
% 比如MORLHF, Reward Soups涉及到了训练多个奖励模型，MODPO将多个维度的偏好目标集成到了奖励模型计算的margin中。HM3使用model merging的方式合并多个维度的预训练模型。很明显，这类方法time-consuming and costly.
% Therefore, we attempt to reduce the number of used models further and improve the multi-objective scalability， thus propose Reference-free Multi-objective Preference Optimization (RMoPO).
% MO领域的文章：Rewarded soups,MORLHF,MODPO, RiC ,DPA,MCA,MetaAligner,HPO,,CPO,HM3,,,SPO
% \jqruan{
% Multi-model tuning methods: MORLHF, Reward Soups, RLPHF, LIMA, MODPO, DPA, HM3, HPO
% - adjusting a series of specialized models for each alignment dimension.
% Instruction-controlled methods: CPO, JANUS, RiC, MCA
% insert control tokens into the prompt
% }
% 我这里列举的很详细，这可以选几个代表性的工作cite，主要提出现有领域内MO方案的弊端，引出我们的优势
% }
% 还是要从sell 卖点的角度出发去讲当前领域内的问题，
% \todo{sell our point to be added} 无其他模型mo & adapter
\hl{Existing multi-objective alignment methods} can be categorized into two main branches.
On the one hand, instruction-controlled methods~\citep{ControlPO2024guo,JANUS2024Lee,RiC2024yang,MCA2024fu}~insert control tokens into the prompt for each preference.
However, these methods are constrained by predefined prompts and lack adaptability in aligning with various dimensions.
On the other hand, multi-model integration methods involve training multiple reward models, or even language models, for each preference dimension. For example, methods like MORLHF~\citep{zhou2023morlhf} and Reward Soups~\citep{rame2024rewarded} require training multiple reward models, while MODPO~\citep{MODPO2024zhou} integrates multiple preference objectives into the margin calculation of the reward model. HM3~\citep{zhou2024hm3} uses model merging to combine pre-trained models across different dimensions. Obviously, these methods are time-consuming and costly.
% \todo{more citation}
% \todo{Title should be changed, highlight the Reference-free and adaptive MO}
% \todo{!maybe there is a case analysis to highlight why adaptive MO is important}
To alleviate these issues, we aim to reduce the reliance on multiple models and enhance adaptability to multiple dimensions
% reduce the number of models used and improve multi-objective adaptability
by proposing Adaptive Multi-objective Preference Optimization (\ourtitle).
% Firstly, we derive a multi-objective BT optimization regime that incorporates generation metrics~\citep{simpo2024meng}~of language models as implicit rewards, which enables the LLMs to inherently grasp the meanings associated with different dimensions.
% 这里也不用enhancing了 enhancing the traditional Bradley-Terry (BT) model
Firstly, we propose a multi-objective optimization framework by designing the multi-dimensional alignment model, enabling it to focus more precisely on each dimension. By integrating generation metrics \citep{simpo2024meng} of language models as implicit rewards, our framework allows LLMs to inherently understand and align with the meaning associated with different dimensions.
% \todo{modify}
Furthermore, we develop an adaptive weight assignment mechanism that automatically determines the importance of each dimension based on the modeled output space. The contributions of this paper can be summarized as follows:
\begin{itemize}[itemsep=0pt, topsep=0pt]
% \begin{itemize}
\item \hl{We propose a multi-objective optimization framework using the Multi-objective BT model, eliminating the need for additional reward or reference models. This approach integrates a multi-dimensional paradigm with generation metrics as implicit rewards to balance various dimensions, ensuring a lightweight and efficient framework.}
% We propose a multi-objective optimization framework that eliminates the need for additional reward or reference models.
% Combining our multi-dimensional paradigm and a multi-dimensional generation metric as the implicit reward function makes our framework lightweight and highly efficient.
\item \hl{Building on the aforementioned framework, we introduce an adaptive weight assignment mechanism that models the generation space of LLMs as a Gaussian distribution. Sampling from this distribution dynamically determines the importance weight for each dimension, allowing~\ourtitle~to adaptively perceive significance levels across dimensions.}
% we propose an }adaptive weight assignment mechanism that models the generation space of LLMs as the Gaussian distribution. By sampling from this distribution, the importance weight of each specified dimension can be dynamically determined, thereby enabling our proposed~\ourtitle~to adaptively perceive the significance levels of various dimensions.
\item Empirical evaluations against multiple state-of-the-art (SOTA) baselines across several benchmarks demonstrate the effectiveness and superior performance of our proposed~\ourtitle. \hl{Further experiments on different scales and series of models confirm the effectiveness and scaling ability of~\ourtitle.}
\end{itemize}
% TODO：目前我在纠结Related work要不要只讲MO， 有三个分支：Reinforced MO alignment； Direct MO alignment； Instruction controlled MO alignment
\subsection{Preference Alignment}
% 两个分支， 一个是RLHF，一个是RL-free DPO系列的
Research on preference alignment can be categorized into two main classes. One line of research focuses on Reinforcement Learning with Human Feedback (RLHF). \citet{ouyang2022training}~demonstrated the effectiveness of RLHF for instruction-following models. Extensions~\cite{touvron2023llama2,yuan2023rrhf,wen2024language,li2024safety}~have emerged as the leading strategy to integrate human preferences via RLHF.
However, RLHF faces challenges due to the instability of Proximal Policy Optimization (PPO) and the sensitivity of reward model training.
To address these issues, another line of research has been proposed that eliminates the need for reinforcement learning by directly optimizing preferences~\cite{DPO2023Rafailov,KTO2024Ethayarajh,rdpo2024Park,ipo2024Azar,caldpo2024xiao}. Nonetheless, these methods often involve multiple models, leading to high computational costs.
\subsection{Reference-Free Preference Alignment}
% 上述的模型存在着cost问题，因此有一个分支在研究无参考模型的偏好优化
Variants of the DPO series require additional memory to deploy the reference model during training.
To address this issue, CPO~\citep{CPO2024xu}~demonstrates that the loss of DPO can be effectively approximated using a unified reference model. SimPO~\citep{simpo2024meng}~replaces the reward in DPO with a sequence generated by the policy model that approximates the maximization of the mean log-likelihood, serving as the implicit reward. ORPO~\citep{ORPO2024Hong}~explores another formulation of the implicit reward.
These methods improve the computational and memory efficiency of LLM training. However, they are primarily focused on single-dimensional preference optimization and do not sufficiently address the intrinsically diverse nature of human preferences.
\subsection{Multi-objective Preference Alignment}
% % 行业调研要完整，相关工作需要进行分类，并且说明每个类别存在的缺陷。
% 比如这里我分为两类：
% \jqruan{
% Multi-model tuning methods: MORLHF, Reward Soups, RLPHF, LIMA, MODPO, DPA, HM3, HPO - adjusting a series of specialized models for each alignment dimension.
% Instruction-controlled methods: CPO, JANUS, RiC, MCA, insert control tokens into the prompt}
% 为了满足不同用户的偏好差异， multi-objective alignment 方法近来被开发出来。
% A line of research extends RLHF as multi-objective RLHF and DPO as multi-objective DPO, which involves multiple models, whatever reward model, policy model or reference model. Extensions 被开发出来adjusting a series of specialized models for each alignment dimension. 但是这种方式计算开销大；
% As another line of work focuses on an Instruction-controlled regime， which insert user preference as a “control token” into prompt，a chieve controllability and further reduces the number of trained models to one. 但是这种方式不灵活，
% To accommodate diverse user preferences, multi-objective alignment methods have been developed in recent years.
Multi-objective alignment methods have been developed in recent years to accommodate diverse user preferences.
A series of studies have extended RLHF and DPO into multi-objective frameworks, such as MORLHF~\citep{zhou2023morlhf}~and MODPO~\citep{MODPO2024zhou}. This involves multiple models, including reward models, policy models, or reference models. Some extensions~\citep{emmerich2018tutorial, mukherjee2024multi,rewardsoup2023Rame,zhou2024hm3,RLPHF_2023jang,LIMA2023zhou,DPA2024wang,HPO2024Badrinath} also aim to adjust a series of specialized models for each alignment dimension, but these approaches incur significant computational costs.
On the other hand, another series of work~\citep{ControlPO2024guo,JANUS2024Lee,RiC2024yang,MCA2024fu}~focuses on an instruction-controlled approach. These methods incorporate user preferences as a "control token`` within the prompt, achieving controllability and reducing the number of trained models to just one. However, these approaches often lack flexibility and adaptability in aligning with various dimensions. % CPO, JANUS, RiC, MCA,
In contrast to the above methods, we employ only a single policy LLM to achieve adaptive multi-objective optimization.
% Preliminary
% 这块还是需要跟我们的主题安全契合，紧密相关的那些基础知识可以放
% 多目标偏好数据集 暂定可以加
% Multi-objective Preference Dataset
% Bradley-Terry (BT) Reward Model
% Simple Preference Optimization (SimPO)
% Multi-objective Preference Optimization
\paragraph{Bradley-Terry (BT) Reward Model.}
% 这个地方最好能同时引出 BT 以及 带奖励和参考模型的偏好对齐
% 后续可以给无参考模型的pre，再引入MO
% 衡量人类偏好的标准有很多，最常用的是BT奖励模型。因此，偏好对齐的主要目标可以定义为：xxx.其中，偏好分布可以定义为：xxx。
% In measuring human preferences, one of the most commonly used standards is the Bradley-Terry (BT) reward model.
% \begin{equation}
% \max_{\pi_{\theta}} \left[ \mathbb{E}_{x \sim \mathcal{D}, y \sim \pi_{\theta}(y \mid x)} \left[ r_{\phi}(x, y) \right] \right]
% \end{equation}
Given an LLM $\pi_{\theta}$ parameterized by $\theta$, query $x$, and response $y$, classical preference alignment methods can be formulated as:
\begin{equation}
\label{eq.1}
\small
\max_{\pi} \mathbb{E}_{\substack{x \sim \mathcal{D} \\ y \sim \pi(\cdot \mid x)}} \left[ r(y \mid x) \right] - \beta D_{\text{KL}} \left[ \pi_{\theta}(y \mid x) \| \pi_{\text{ref}}(y \mid x) \right],
\end{equation}
where $r$ measures the consistency with human preference, $\mathcal{D} = \{(x,y_w,y_l)^i\}$ denotes the preference dataset that prefers $y_w$ and disprefer $y_l$, $D_{\text{KL}} \left[ \pi(y \mid x) \| \pi_{\text{ref}}(y \mid x) \right]$ is the Kullback-Leibler divergence~\cite{rl2007jan,position2024Reizinger} between $\pi$ and reference LLM $\pi_{ref}$, and $\beta$ is a constant controlling the strength of KL.
In measuring human preferences, one of the most commonly used approximations is the Bradley-Terry (BT) reward model~\cite{bradley1952rank,sun2024rethinking,guo2024todo}. Assumed $r^*$ is the latent ground-truth reward function, the BT model defines the preference distribution $\pi^*$ as:
\begin{equation}
\small
\label{eq.2}
\pi^*(y_w \succ y_l \mid x) = \frac{\exp(r^{*}(x, y_{w}))}{\exp(r^{*}(x, y_{w})) + \exp(r^{*}(x, y_{l}))}.
\end{equation}
\paragraph{Simple Preference Optimization (SimPO).}
Compared with the primary methods~\cite{DPO2023Rafailov}, SimPO~\cite{simpo2024meng} introduces a more lightweight and straightforward approach that eliminates the need for a reference model. SimPO utilizes the generation metric~\cite{gpt42023openai,radford2018improving,touvron2023llama,touvron2023llama2,llama3}, i.e, the average log-likelihood of generating the output sequence $y$, which consists of $m$ tokens given an input sequence $x$, is computed as:
\begin{equation}
\small
\label{eq.3}
\log \pi_{\theta}(y \mid x) = \frac{1}{m} \sum_{t=1}^{m} \log \pi_{\theta}(y_{t} \mid x, y_{<t}).
\end{equation}
Thus, replacing the reward with Eq.\eqref{eq.3}, the optimization objective of SimPO can be formulated as:
\vspace{-0.2em}
% \begin{equation}
% \label{Eq.4}
% \begin{split}
% \mathcal{L}_{\text{SimPO}}&(\pi_\theta) = \\ & -\mathbb{E}_{(x, y_w, y_l) \sim \mathcal{D}} \biggl[ \log \sigma \biggl( \frac{\beta}{|y_w|} \log \pi_\theta(y_w \mid x) \\
% & \quad - \frac{\beta}{|y_l|} \log \pi_\theta(y_l \mid x) - \gamma \biggr) \biggr].
% \end{split}
% \end{equation}
\begin{equation}
\small
\label{Eq.4}
\begin{split}
\mathcal{L}_{\text{SimPO}}(\pi_\theta) = -\mathbb{E}_{(x, y_w, y_l) \sim \mathcal{D}} \biggl[ \log \sigma \biggl( \frac{\beta}{|y_w|} \log \\
\pi_\theta(y_w \mid x) - \frac{\beta}{|y_l|} \log \pi_\theta(y_l \mid x) - \gamma \biggr) \biggr], \end{split}
\end{equation}
where $\beta$ is a constant that controls the scaling of the reward difference and $\gamma$ is a target reward margin.
\paragraph{Multi-objective Preference Dataset.}
A $K$-dimensional preference dataset~\cite{ultrafeedback2024cui,helpsteer2024wang,helpsteer22024wang} can be divided into subsets \(\mathcal{D}_j\) based on objective \(j\), represented as \(\bm{\mathcal{D}} = [ \mathcal{D}_1, \mathcal{D}_2, \ldots, \mathcal{D}_K ]\). Here, each subset including $N$ samples, denoted as \(\mathcal{D}_j = \{ (x, y_w, y_l, d_j)^i \}_{i=1}^{N}\), where \({d_j}^{i}\) represents the score of the \(i\)-th sample in the dimension \(j\), such as helpfulness.
This structure facilitates the assessment of preferences across multiple dimensions by organizing data according to specific objectives.
\paragraph{Multi-objective Preference Optimization (MOPO).}
% 建议这里给一个通用的或者传统的，也可以是自己的框架模式下的MO的formulation，不用细化到MODPO，因为后续我们的方法不是condition on MODPO，所以可以不介绍这个。
Classical MOPO methods~\cite{2DDPO2024li,MODPO2024zhou} use separate reward models $\bm{r}^*={[{r_1}^*,...,{r_K}^*]}$ for multi-dimensional datasets $\bm{\mathcal{D}} = [ \mathcal{D}_1, \mathcal{D}_2,...,\mathcal{D}_K ]$, allowing customization of LLMs by adjusting reward weightings.
% 统一一下 N代表样本数，K代表维度的数目
In the context of multi-objective alignment, the primary goal is to approximate a Pareto front~\cite{mukherjee2024multi,safest2024sinha} that represents the multi-dimensional preferences for LLMs. Each preference is optimized according to a specific collective reward model $r^*_i$, with an associated known weight \(\alpha_i\).
The overall objective for MOPO is to maximize the cumulative weighted sum of rewards across all dimensions. This can be expressed as:
\begin{equation}
\small
\label{eq.5}
\max_{\pi} \mathbb{E}_{\substack{x \sim \bm{\mathcal{D}} \\ y \sim \pi(\cdot \mid x)}} \left[ R(y \mid x, d_1,...,d_K) \right] ,
\end{equation}
where \(R\) represents the dot product of the vector \(\bm{\alpha} = [\alpha_1, ..., \alpha_K]\) and $\bm{r}^*={[{r_1}^*,...,{r_K}^*]}$.
Generally, in the context of multi-objective alignment, the goal is to approximate a Pareto front of the multi-dimensional preference for LLMs.
Each preference is optimized for a specific collective reward model $r^*_i$, with an associated known weight \(\alpha_i\).
The weighted sum of these auxiliary rewards is expressed as \(R(x, y)\), represented by the dot product of the vector \(\bm{\alpha} = [\alpha_1, ..., \alpha_K]\) and $\bm{r}^*={[{r_1}^*,...,{r_K}^*]}$, denoted as follows.
\begin{equation}
\label{eq.6}
\small
R(y \mid x, d_1,...,d_K) = \sum_{j=1}^{K} \alpha_{j} r^*_{j}(x, y, d_j) = \bm{\alpha}^{\top} \mathbf{r}_{xy}.
\end{equation}
% \jqruan{see comments}
% 还是建议尽量不给读者你是拓展的谁、改写的谁、follow的谁的印象
% 这部分直接分为几个subsection，比如：
% 直敲我们的卖点：1. 怎么实现无其他模型的MO？ 2. 怎么实现adapter？
% 4.1 The Multi-Objective Derivation for~\ourtitle；
% BT拓展那块 与 Mragin-集成回去
% 从BT到loss的详细推到步骤可以放在附录
% 4.2 Adaptive Weight Assignment
% - (注意 alpha 与采样adapter的关联) ；
% 4.3 伪代码流程-Implementation？
% RMoPO
% To take into account the diversity of human preferences and address the complexity and hardness of training multiple objectives reward models, we propose Adaptive Multi-objective Preference Optimization without
% Rewards and References (\ourtitle), a more lightweight and efficient preference optimization method. In addition, we raise a weight assignment strategy based on Gaussian sampling in the generative spatial probability, which enables the Language Models to adaptively adjust the weight assignments among multiple dimensions during training rather than assigning fixed weights. The complete pipeline of our method is illustrated in Figure \ref{fig:pipeline}.
% In this section,
This section begins with the process of data preparation. We then present our~\ourtitle~by exploring the multi-objective derivation and the adaptive weight assignment mechanism. The complete pipeline of our method is illustrated in Figure~\ref{fig:pipeline}.
\begin{figure*}[!h]
\centering
\includegraphics[width=0.93\textwidth]{latex/images/main/pipeline_rjq-20241216.pdf}
% \includegraphics[width=0.9\textwidth]{latex/images/main/pipeline_rjq.pdf}
% \vspace{-1pt}
\caption{The pipeline of \ourtitle~comprises two key stages: (a) \textbf{Data Preparation}: For a prompt with two responses, $y_a$ and $y_b$, GPT-4 scores each response using the scoring criteria from three dimensions: helpfulness, correctness, and instruction following. For each dimension, a unique template is designed to ensure the model focuses on that specific dimension when generating the answer. (b) \textbf{Training}: A Gaussian-based weight assignment mechanism is employed to sample weights for each dimension. \ourtitle~is then updated using the loss $\bm{\mathcal{L}}$. }
\label{fig:pipeline}
% \vspace{-10pt}
\end{figure*}
\subsection{Data Preparation}
\label{sec:data_pre}
Figure~\ref{fig:pipeline}~(a) illustrates that for a given prompt with two responses, $y_a$ and $y_b$, GPT-4 scores each response along $K$ dimensions: helpfulness, correctness, and adherence to instructions. For the detailed scoring template, please see Appendix~\ref{prompts:3dimensions}.
By leveraging these scores $\bm{d}=[d_1....,d_K]$, we can develop a novel prompt template that integrates both the dimension and the corresponding score, a process denoted as $x^* = f(x, d)$.
This approach guides the model to focus on the specified dimension when generating a response. A preference dataset with $K$ dimensions can be partitioned into subsets \(\mathcal{D}_j\) according to objective \(j\), expressed as \(\bm{\mathcal{D}} = [ \mathcal{D}_1, \mathcal{D}_2, \ldots, \mathcal{D}_K ]\). Each subset comprises $N$ samples, represented as \(\mathcal{D}_j = \{ (x, y_a, y_b, d_j)^i \}_{i=1}^{N}\), where \({d_j}^{i}\) indicates the score of the \(i\)-th sample in dimension \(j\), such as helpfulness.
% \vspace{-10pt}
\subsection{Multi-objective Optimization Derivation}
% \subsection{The Multi-Objective Derivation for~\ourtitle}
\input{latex/images/main/amopo_algorithm}
\paragraph{Overall Optimization Objective.}
% According to weighted sum multi-objective optimization~\citep{miettinen1999nonlinear}, which considers a convex combination of different objectives, the optimal solution can be found by constructing a scalar objective function.
% Specifically, the weight vector $\bm{\alpha} = [\alpha_1, \ldots, \alpha_K]$ can be obtained with the policy $\alpha \sim \rho$, where $\sum_{k=1}^{K} \alpha_k = 1$ and $\alpha_k > 0$.
% Then the Pareto optimal solution~\cite{zhang2007moea,deb2005searching} can be determined by the following scalar optimization problem:
\hl{Here, we propose a novel paradigm based on weighted sum multi-objective optimization~\citep{miettinen1999nonlinear}, which constructs a scalar objective function to find the optimal solution. Specifically, we design a weight vector $\bm{\alpha} = [\alpha_1, \ldots, \alpha_K]$, obtained through the policy} $\alpha \sim \rho$, where $\sum_{k=1}^{K} \alpha_k = 1$ and $\alpha_k > 0$.
The Pareto solution is then determined by addressing the following scalar optimization problem:
\begin{equation}
\normalsize
\label{eq.overall}
\begin{split}
& \text{max} \ \pi(y_w \succ y_l | x, \bm{\alpha}, \bm{d}) \\
& \rightarrow \text{max} \ {\textstyle \sum\limits_{k=1}^{K} } \rho(\alpha_k|x_k^*) \log \pi^*_k(y_w \succ y_l | x_k^*),
\end{split}
\end{equation}
\hl{where we restructure} ${x}^*_k = f(x, {d_k})$ to adapt to the $k^{th}$ dimension in Section~\ref{sec:data_pre}, and $\pi^*_k$ denotes the preference alignment function for the $k^{th}$ dimension.
% \begin{equation}
% % \normalsize
% \label{eq.overall}
% \begin{split}
% \text{max} \ & P(y_w \succ y_l | x, \bm{d}) \\
% & = log {\textstyle \prod_{k=1}^{K}} \left[ p^*(y_w \succ y_l | x^*) \right]^{\rho(\alpha_k |x,y_w,y_l,\bm{d})} ,
% \end{split}
% % \end{equation}
% where ${x}^*_k = f(x, {d_k})$ restructures $x$ for the $k^{th}$ dimension in Section~\ref{sec:data_pre}, and $\pi^*_k$ denotes the preference alignment function for the $k^{th}$ dimension.
% % 总体优化目标有了，下面就是优化两个部分，1. 策略模型 2. adapter
% Thus, to enable effective optimization and balance among different objectives, we can optimize both the weight policy \(\bm{\alpha} \sim \rho\) and the preference alignment policy \(p^*\).
\hl{To optimize and balance multiple objectives effectively, dynamic adjustments are essential. Fixed weights often miss the complexities and trade-offs of different dimensions. Dynamic weight sampling allows for adaptive prioritization based on current conditions, ensuring the process remains efficient and relevant. Thus, optimizing both the weight policy $\bm{\alpha} \sim \rho$ and the preference alignment policy $p^*$ is crucial, enabling the framework to maintain balance among objectives in varying scenarios.}
\paragraph{Multi-Objective BT (MOBT) Model.}
\hl{
We introduce the Multi-Objective BT (MOBT) model to optimize the preference alignment policy, marking a significant departure from the traditional BT model described in Eq.\eqref{eq.2}. Unlike the conventional approach, which fuses scores from multiple dimensions into a singular $r(x,y)$, our model employs a dynamically weighted sum of scores across various dimensions.
This innovative strategy facilitates adaptive learning of importance weights, thereby balancing scores across multiple dimensions and achieving an adaptive multi-objective optimization framework.} This can be formulated as follows:
% To optimize the preference alignment policy, we first define the MOBT model. Unlike the traditional BT model in Eq.\eqref{eq.2}, which fuses scores from multiple dimensions into a single $r(x,y)$, we employ a weighted sum of scores across various dimensions. Additionally, this allows for adaptive learning for the importance weights, which balances the scores across multiple dimensions and achieves an adaptive multi-objective optimization framework. This can be formulated as follows:
% 建议不给读者增加负担，直接说你背后动机，然后推出一个式子，而不是把最后的这个 sigmoid^alpha 直接放在这里；
\begin{equation}
\normalsize
\label{eq.7}
\begin{split}
& \pi(y_w \succ y_l \,|\, x;\, \bm{\alpha};\, \bm{d} ) \\
& = \sum_{k = 1}^{K} \alpha_{k} \log \sigma \left( r_{k}(x^{*}_k, y_w) - r_{k}(x^{*}_k, y_l) \right) \\
& = \log \prod_{k=1}^{K} \sigma^{\alpha_k}(r_{k}(x^{*}_k, y_w) - r_{k}(x^{*}_k, y_l)) \\
& \text{s.t.} \sum_{k=1}^{K} \alpha_k = 1,~x^{*}_k = f(x, {d_k}).
\end{split}
\end{equation}
% \begin{equation}
% \label{eq.6}
% \begin{split}
% P(y_w \succ &\,y_l\,|\,x;\,\bm\alpha;\,\bm{d}) = \\
% &\prod\limits_{i=1}^{|\bm\alpha|} {\left[ \sigma^{\alpha_i}(\bm{r}(x^{*}, y_w) - \bm{r}(x^{*}, y_l)) \right]} \\
% &\text{s.t.}\: \sum_{i=1}^{|\bm\alpha|} \alpha_i = 1,\: x^{*} = f(x, \bm{d})
% \end{split}
% \end{equation}
% \vspace{0.2em}
% \vspace*{-16pt}
% \medskip
% \vspace{-6.7mm}
% \newline
% \vspace*{0.2em}
% where $\bm{\alpha} = [\alpha_1, \ldots, \alpha_K]$ denotes the weights for different dimensions, $f(x, \bm{d})$ restructures $x$ in Section~\ref{sec:data_pre}, and $\sigma(x) = \frac{1}{1 + \exp(-x)}$ is the Sigmoid function.
In our model, $\alpha$ is derived from the weight policy $ \rho$, and $\sigma(x) = \frac{1}{1 + \exp(-x)}$ is the Sigmoid function.
Evidently, the multi-objective BT Model reduces to the traditional BT Model when $K = 1, \alpha=1$.
\paragraph{{\ourtitle} Loss.}
Let the LLM policy be $\pi_\theta$ and the adaptive weight assignment policy be $\rho_{\varphi}$, we can formulate a maximum likelihood estimation objective by combining Eq.\eqref{eq.3}, Eq.\eqref{eq.7} and Eq.\eqref{eq.10} on the multi-dimensional dataset $\mathcal{D}$ as follows.
%想一下 How to claim other-model free???
\begin{equation}
\normalsize
\label{Eq.ours}
\begin{split}
\mathcal{L}&_{\text{\ourtitle}}(\pi_\theta; \bm{\alpha}; \bm{d}) =
- \mathbb{E}_{(x, y_w, y_l)\sim \mathcal{D}} \biggl[ \sum_{k=1}^{K} \alpha_k \log \sigma \\ & \biggl(\frac{\beta}{|{y_w}|} \log \pi_\theta({y_w} \mid x^{*}_k) \quad - \frac{\beta}{|{y_l}|} \log \pi_\theta({y_l} \mid x^{*}_k) \biggr) \biggr]\\
&~~~~~~~~~~~ \text{s.t.}\; x^{*}_k=f(x,{d_k}) ,
\end{split}
\end{equation}
The derivations are detailed in Appendix~\ref{math_derivation}.
\subsection{Adaptive Weight Assignment Mechanism}
To dynamically balance the contributions of each objective dimension, we utilize the language model's token generation probabilities to compute the mean $ \mu_k $ and variance $ \sigma_k^2 $ for each dimension $ k $.
These are used to parameterize a Gaussian distribution $ \rho_k $, from which the weight $ \alpha_k $ is sampled. This allows the model to adjust each dimension's significance based on confidence in token generation~\cite{shannon1948mathematical, Maximum1983Bahl}.
Formally, the mean and variance are:
\begin{equation}
\normalsize
\label{eq.10}
\mu_k = \frac{1}{T} \sum_{t=1}^{T} p_{k,t}, \ \sigma_k^2 = \frac{1}{T} \sum_{t=1}^{T} (p_{k,t} - \mu_k)^2,
\end{equation}
where $ p_{k,t} = \pi_{\theta}(y_{t} \mid x_k^*, y_{<t}) $ is the probability of the $ t^{th} $ token for dimension $ k $.
Then the weight $ \alpha_k $ is sampled as: $\alpha_k \sim \mathcal{N}(\mu_k, \sigma_k^2)$.
We normalize all the weights using the formula \(\alpha_i = \frac{e^{\alpha_i}}{\sum_{j=1}^{n} e^{\alpha_j}}\) to ensure that \(\alpha_k > 0\) and \(\sum_{k=1}^{K} \alpha_k = 1\).
% \begin{equation}
% \normalsize
% \alpha_k \sim \mathcal{N}(\mu_k, \sigma_k^2), \ \text{s.t.} \ \alpha_k > 0, \ \sum_{k=1}^{K} \alpha_k = 1.
% \end{equation}
This adaptive mechanism enhances the model’s ability to adaptively prioritize dimensions with higher confidence.
Notably, the process of generating $\alpha$ is modular and interchangeable, allowing for any desired refinements or mechanisms. This flexibility ensures that the weight vector $\bm{\alpha}$ dynamically adjusts based on model performance across dimensions, facilitating balanced optimization. The versatility of this mechanism will be further detailed in the experimental section.
\subsection{Implementation details}
% As shown in Figure \ref{fig:pipeline}, {\ourtitle} takes the multi-objective training progress into two stages.
% \textbf{Firstly}, we extend the original prompt to diversity prompts for mapping each dimension.
% We provide a set of universal templates to adapt to different dimensions, which makes LMs focus on a specific dimension when answering questions.
% And one obvious benefit is that for a specific prompt and its response, we can obtain diversity probability distribution through our method.
% \textbf{Regarding the second phase}, we transfer
As shown in Algorithm~\ref{alg:amopo},
{\ourtitle} transfers an original quadruple $<x,y_w,y_l,\bm{d}>$ into a set of quadruples $\{<x,y_w,y_l,d>\}^{d \in \bm{d}}$. And then we design a map function $f$, which changes $x$ according to different dimensions, i.e., $x^* = f(x,d)$. Subsequently, we concatenate the multi-dimensional prompts $x^*$ with the winning response $y_w$ and the losing response $y_l$ as inputs to the policy model $\pi_\theta$, obtaining the $p_d^w$ and $p_d^l$ individually.
We model the sum probabilities $p_d$ through Gaussian Sampling, deriving the weight $\alpha_d$.
By combining the weights of each dimension with the generated metrics, we can obtain the final expression for {\ourtitle}.
% We can easily obtain the all tokens probabilities $p_i^w$ of the winning responses and $p_i^l$ of the losing responses. Then, we add both responses tokens probabilities, i.e., $p_i=p_i^w+p_i^l$, in which $p_i$ means the policy model confidence in the $d_i$ dimension.
% Based on these tokens probabilities, we model $p_i$ by Gaussian Sample. And sample a probability $\alpha_i$ as the weight for the $d_i$ dimension.
% In addition, we utilize the LM generation metric to obtain the loss between the winning and losing response, which can be formulated as $l=\log\sigma(\frac{\beta}{|{y_w}|} \log \pi_\theta({y_w} \mid x^{*})-\frac{\beta}{|{y_l}|} \log \pi_\theta({y_l} \mid x^{*}))$.
% Taking into account the effect of weight $\alpha_i$, we finally obtain the loss $l=\alpha_i\log\sigma(\frac{\beta}{|{y_w}|} \log \pi_\theta({y_w} \mid x_i^{*})-\frac{\beta}{|{y_l}|} \log \pi_\theta({y_l} \mid x_i^{*}))$ in the dimension of $d_i$.
% We provide the pseudocode in Algorithm \ref{alg:amopo} in details.
\begin{algorithm*}[!htbp]
\caption{AMoPO}
\label{alg:amopo}
\setstretch{0.2}
\small{
\KwIn{Policy model $\pi_\theta$, Objectives $\bm{d}$, Mapping $f$, Epochs $epochs$, Coefficient $\beta$, Learning rate $\eta$, Dataset $\mathcal{D} = \{ (x, {y_w}, {y_l}, \bm{d})^i\}_{i=1}^{N}$}
% \KwIn{Dataset $\mathcal{D} = \{ (x, {y_w}, {y_l}, score)^i\}_{i=1}^{N}$ of samples size $N$ with different dimension scores.}
\For{epoch \textnormal{in} range($epochs$)}{
{\textbf{Initialize:} All tokens probabilities $\bm{p}= [\;]$, Weights $\bm\alpha=[\;]$, Multiple loss $\bm{\mathcal{L}} = [\;]$}\\
Sample a batch $\mathcal{D}_s = \{ (x, {y_w}, {y_l}, \bm{d})^s\}_{s=1}^{S}$ from $\mathcal{D}$;\\
\For{$d$ \textnormal{in} $\bm{d}$}{
Use the dimension $d$ and the mapping function $f$ to get the unique prompt template $x^* = f(x,d)$;\\
Merge all samples $\mathcal{D}_j^d = \{ (x^*, {y_w},{y_l}, d_{score})^s\}_{s=1}^{S} $; \\
Predict the all tokens probabilities $p_d^{w}={\{<t_i^w,p_i^w>\}}_{i=1}^{|y_w|}$ and $p_d^{l}={\{<t_i^l,p_i^l>\}}_{i=1}^{|y_l|}$ based on $\pi_\theta(y_w|x^*)$ and $\pi_\theta(y_l|x^*)$;\\
Calculate the sum probabilities $p_d = p_d^w+p_d^l$;\\
Model $p_d$ based on Gaussian Sample to get the weight $\alpha_d$;\\
Calculate current function $l=\log\sigma(\frac{\beta}{|{y_w}|} \log \pi_\theta({y_w} \mid x^{*})-\frac{\beta}{|{y_l}|} \log \pi_\theta({y_l} \mid x^{*}))$;\\
$\bm{p} \gets p_d$ ;
$\bm\alpha \gets \alpha_d$;
$\bm{\mathcal{L}} \gets l$;\\
}
Normalize different dimensions weights $\bm{\alpha}$ ;\\
$\theta \gets \theta+\eta{\nabla_\theta}\mathbb{E}_{(x, y_w, y_l,\bm{d})\sim \mathcal{D}_s} \bm{\alpha}^{T}\bm{\mathcal{L}} $
}
\KwOut{$\pi_\theta$}
}
\end{algorithm*}
% Experiment
\subsection{Experiment Setup}
% 实验设置:
% 1. training setup : 写训练数据集，已经扩展的instruction following维度和原因和使用gpt4o生成的模板（放到附录里）
% 2. Models: 使用哪些base model去做preference optimization
% 3. Benchmark: 在AlpacaEval2, Arena-Hard and Mt-bench上评估，以及每个数据集的评估模板（放到附录里）以及数据集的介绍根据 Evaluation details
% 4. Baseline:
% DPO
% ORPO
% SimPO
% Cal-DPO
% MODPO
\paragraph{Training Setup.}We perform preference optimization based on HelpSteer2~\cite{helpsteer22024wang}.
% , which is a follow-up to the popular HelpSteer~\cite{helpsteer2024wang}.
It not only includes instructions and preferred responses, but it also scores each response across multiple dimensions such as helpfulness, correctness and etc. While some researches~\cite{complexbench2024wen, ORPO2024Hong} have confirmed that instruction following is an indispensable dimension for evaluating the quality of a response, we leverage \textit{gpt4o-2024-05-13} to evaluate the instruction-following scores for each response. The template is shown in Appendix ~\ref{prompts:instruction_following}.
\paragraph{Models.}We perform preference optimization with two models from Qwen family~\cite{qwen2.5}, Qwen2.5-7B-Instruct and Qwen2.5-14B-Instruct. Both models have demonstrated robust capabilities~\cite{qwen2, qwen2.5, qwen2.5code2024hui, wu2024beyond} across various tasks, showing strong performance whether fine-tuned or used directly.
We evaluate the effectiveness of our preference optimization method on these two models with different levels of comprehension.
% \begin{table}[!htbp]
% \centering
% \caption{Evaluation details on benchmarks.}
% \label{tab:evaluation_details}
% \huge{
% \resizebox{0.48\textwidth}{!}{
% \begin{tabular}{ccccc}
% \toprule
% {\textbf{Dataset}}&Examples & baseline model & Judge Model & Metric \\
% \midrule
% AlpacaEval2.0 & 805 & GPT-4\footnotemark[1] & GPT-4o\footnotemark[3] & Win Rate \\
% ArenaHard & 500 & GPT-4\footnotemark[2] & GPT-4o\footnotemark[3]& Win Rate \\
% MT-bench& 80 & - & GPT-4o\footnotemark[3] & Rating(1-10)\\
% \bottomrule
% \end{tabular}
% \footnotetext[1]{It represents GPT-4-preview-1106.}
% \footnotetext[2]{It represents GPT-4-0314.}
% \footnotetext[3]{It represents GPT-4o-2024-05-13.}
% }
% }
% \end{table}
\begin{table}[!htbp]
\centering
\caption{Evaluation details on benchmarks.}
\label{tab:evaluation_details}
% \huge{
\resizebox{0.48\textwidth}{!}{
\begin{tabular}{ccccc}
\toprule
{\LARGE \textbf{Dataset}}& \LARGE Examples & \LARGE baseline model &\LARGE Judge Model & \LARGE Metric \\
\midrule
\LARGE AlpacaEval2.0 & \LARGE 805 & \LARGE GPT-4\footnotemark[1] & \LARGE GPT-4o\footnotemark[3] & \LARGE Win Rate \\
\LARGE ArenaHard & \LARGE 500 &\LARGE GPT-4\footnotemark[2] &\LARGE GPT-4o\footnotemark[3]& \LARGE Win Rate \\
\LARGE MT-bench&\LARGE 80 & \LARGE - &\LARGE GPT-4o\footnotemark[3] &\LARGE Rating(1-10)\\
\bottomrule
\end{tabular}
}
% }
\end{table}
\paragraph{Benchmarks.}We mainly test our method on three of the most popular and recognized instruction-following benchmarks: AlpacaEval2.0~\cite{alpaca_eval}, Arena-Hard~\cite{arenahard2024} and MT-bench~\cite{MT-bench2024zheng}. These benchmarks consist of a set of queries and their answers are under the framework of LLM-as-a-Judge~\cite{MT-bench2024zheng} (Details are in Table \ref{tab:evaluation_details}).
\paragraph{Baselines.}We divide the compared methods into two categories: single-objective and multi-objective preference optimization.
For the \textbf{{single-objective preference optimization}} methods:
% 非item形式
(1) DPO~\cite{DPO2023Rafailov} eliminates the training of additional reward functions, which directly learn a policy model from preference data.
(2) Cal-DPO~\cite{caldpo2024xiao} calibrates the implicit reward function against the ground-truth rewards based on DPO.
(3) ORPO~\cite{ORPO2024Hong} introduces a reference-free model odd ratio to compare winning and losing responses and jointly trains with the SFT stage.
(4) SimPO~\cite{simpo2024meng} proposes to leverage the generation metric with the policy model to present the implicit reward for responses, significantly simplifying the training phase.
% % item 形式
% \begin{itemize}[itemsep=0.3em]
% \item DPO~\cite{DPO2023Rafailov} eliminates the training of additional reward functions, which directly learn a policy model from preference data.
% \item Cal-DPO~\cite{caldpo2024xiao} calibrates the implicit reward function against the ground-truth rewards based on DPO.
% \item ORPO~\cite{ORPO2024Hong} introduces a reference-free model odd ratio to compare winning and losing responses and jointly trains with the SFT stage.
% \item SimPO~\cite{simpo2024meng} proposes to leverage the generation metric with the policy model to present the implicit reward for responses, significantly simplifying the training phase.
% \end{itemize}
For the \textbf{{multi-objective preference optimization}} methods:
(1) MODPO~\cite{MODPO2024zhou} divides the multi-objective training into two steps: For two dimensions, e.g., helpfulness and correctness, MODPO firstly trains a reward model in correctness dimension. Then, it introduces scores by the trained reward model when training on the helpfulness dimension.
(2) CPO~\cite{ControlPO2024guo} \hl{extends both SFT and DPO to solve the multi-objective optimization problem.}
(3) SPO~\cite{SPO2024Lou}
\hl{sequentially fine-tunes LLMs to align with multiple dimensions of human preferences.}
Appendix~\ref{imple_details} shows more details.
% \begin{itemize}[itemsep=0.2em]
% \item MODPO~\cite{MODPO2024zhou} divides the multi-objective training into two steps: For two dimensions, e.g., helpfulness and correctness, MODPO firstly trains a reward model in correctness dimension. Then, it introduces scores by the trained reward model when training on the helpfulness dimension.
% \end{itemize}
% To further elaborate, MORLHF is inherently unstable and challenging to train, and its efficacy has been shown to be inferior to that of MODPO. Therefore, it will not be considered as our baseline in this context.
% \begin{table}[h]
% \centering
% \caption{Performance comparison of different methods on Qwen2.5-14B-Instruct across various benchmarks.}
% \resizebox{0.5\textwidth}{!}{
% \begin{tabular}{lcccccc}
% \toprule
% & \multicolumn{3}{c}{Qwen2.5-14B-Instruct} \\
% \cmidrule(lr){2-4}
% \textbf{Method} & AlpacaEval 2 & Arena-Hard & MT-Bench \\
% \cmidrule(lr){2-2} \cmidrule(lr){3-3} \cmidrule(lr){4-4}
% & WR (\%) & WR (\%) & Avg.Rating \\
% \midrule
% DPO & 56.4 & 66.8 & 8.77 \\
% ORPO & 18.8 & 34.2 & 8.07 \\
% SimPO & \underline{58.6} & 67.8 & \underline{8.90} \\
% Cal-DPO & 52.8 & 65.8 & 8.68 \\
% MODPO & 55.7 & \underline{68.8} & 8.72 \\
% \ourtitle & \textbf{63.9} & \textbf{73.4} & \textbf{9.0} \\
% \bottomrule
% \end{tabular}
% }
% \label{tab:performance_comparison_14B}
% \end{table}
\footnotetext[1]{Version: GPT-4-preview-1106.}
\footnotetext[2]{Version: GPT-4-0314.}
\footnotetext[3]{Version: GPT-4o-2024-05-13.}
\subsection{Main Results}
% 分成两块写 7B-instruct (提升比较小的原因是7B的模型的理解能力比较差），AMoPO对每个维度进行设计了独特的模版和当前维度分数的解释，目的是想让模型可以根据当前维度的解释和分数以及现在这个维度的回复结果来去学习到当前维度的含义。当策略模型参数量比较小的时候，其对于维度的学习能力可能不够，模型只能够理解部分维度的含义，所以提升比较小。
As shown in Table~\ref{tab:performance_comparison}, our approach achieves 28.5$\%$ overall improvements over the baseline methods, benefiting from our multi-dimensional alignment mechanism.
% Based on the differences in the base models, our analysis can be divided into the following two parts.
\begin{table*}[h]
\centering
\resizebox{0.9\textwidth}{!}{
\begin{tabular}{llcccccccc}
\toprule
& & \multicolumn{3}{c}{Qwen2.5-7B-Instruct} & \multicolumn{3}{c}{Qwen2.5-14B-Instruct} \\
\cmidrule(lr){3-5} \cmidrule(lr){6-8}
\textbf{Objective} & \textbf{Method} & AlpacaEval 2 & Arena-Hard & MT-Bench & AlpacaEval 2 & Arena-Hard & MT-Bench \\
\cmidrule(lr){3-3} \cmidrule(lr){4-4}
\cmidrule(lr){5-5} \cmidrule(lr){6-6} \cmidrule(lr){7-7} \cmidrule(lr){8-8}
& & WR (\%) & WR (\%) & Avg.Rating & WR (\%) & WR (\%) & Avg.Rating& \\
\midrule
\multirow{4}{*}{\textbf{Single}} &
DPO &41.5 & 56.4 & 8.58 & 56.4 & 66.8 & 8.77 \\
& ORPO & 13.1 & 27.4 & 7.92 & 18.8 & 34.2 & 8.07 \\
& SimPO & \underline{56.1} & \underline{64.2} & 8.50 & 58.6 & 67.8 & \underline{8.90} \\
& Cal-DPO & 41.7 & 55.4 & \underline{8.60} & 52.8 & 65.8 & 8.68 \\
\midrule
\multirow{4}{*}{\textbf{Multiple}} &
\hl{CPO} & \hl{52.0} & \hl{62.0} & \hl{8.43} & \underline{\hl{59.0}} & {\hl{69.4}} & \hl{8.8} \\
& SPO & 41.2 & 60.4 & 8.58 & 55.9 & \underline{71.2} & 8.8 \\
& MODPO & 52.8 & 63.4 & 8.57 & 55.7 & 68.8 & 8.72 \\
& \ourtitle(ours) & \textbf{58.5} & \textbf{64.4} & \textbf{8.63} & \textbf{63.9} & \textbf{73.4} & \textbf{9.0} \\
\bottomrule
\end{tabular}
}
\caption{Performance comparison of different methods on Qwen2.5-7B-Instruct and Qwen2.5-14B-Instruct across various benchmarks.}
\label{tab:performance_comparison}
\end{table*}
For \textbf{Qwen2.5-14B-Instruct} as the base model,
% 改写一下
% {\ourtitle} surpasses the current state-of-the-art models on three datasets, and outperforms them by up to 5.3 points on AlpacaEval 2, by up to 4.5 points on Arena-Hard and 0.1 average rating improvement on MT-bench.
{\ourtitle} surpasses the current state-of-the-art methods on three datasets. \hl{It outperforms CPO by up to 4.9 points on AlpacaEval 2 and outperforms SPO by up to 2.2 points on ArenaHard.} And it also outperforms SimPO by up to 0.1 average rating improvement on MT-bench.
For single-objective preference optimization methods, e.g., SimPO, these methods do not generate responses from multidimensional perspectives.
% 考虑是否需要
% Notice that all methods are conducted on the same group of queries and responses.
\hl{For current multidimensional preference optimization methods, e.g., CPO and SPO, these methods do not incorporate the meanings of the dimensions into the model training.}
% 这一句再思考一下 应该对齐维度去说
To tackle the above issues, {\ourtitle} integrates the significance of multiple dimensions into the preference optimization and gives alignment templates to activate the model's learning.
In addition, compared with MODPO, {\ourtitle} does not require any additional training stage to alleviate extra computation overhead.
% And in terms of both the formula and the results, {\ourtitle} is more intuitive and conducive to understanding than MODPO.
% ？？要不要加上一点 说Arena-Hard比alpacaeval2要challenging 所以Arena-Hard的提升要略少
%。 3. 为什么7B 的 Arena-Hard的提升很小 通过更换base model，并且对base model进行win rate评测发现qwen系列模型在Arena-Hard模型上的初始表现就相对较高，所以提升可能就不显著。 细节见 Appendix
% \todo{see comments}
For \textbf{Qwen2.5-7B-Instruct} as the base model,
{\ourtitle} still outperforms SimPO by up to 2.4 points on AlpacaEval 2, 0.2 points on Arena-Hard, and outperforms Cal-DPO by up to 0.03 average rating on MT-bench.
% The abilities of the model vary with changes in model parameters.
% Generally speaking, the larger models possess stronger capabilities.
We observe that the improvement compared to the results of the 14B model is relatively constrained.
The reason is that the key design of {\ourtitle} is to make the policy model align different dimensions with score criteria during preference optimization.
With fewer policy model parameters, the understanding and studying capabilities are greatly weakened.
We find that the response of the trained model based on Qwen2.5-7B-Instruct will return unexpected information or hallucinated details compared to Qwen2.5-14B-Instruct.
The details are shown in Appendix~\ref{Cases}.
% 除了相较于14B提升比较微弱之外，我们也发现对于Qwen2.5-7B-Instruct 模型本身来说，关于ArenaHard数据集的提升很小，这是因为对于qwen模型来说，它本身对于ArenaHard的表现就很好，所以在此基础上再进行提升幅度会比较小，我们也更换了一个参数量大小相似的模型LLaMA做对比实验，并且报告了AMoPO对其在三个数据集上都有着显著的提升，细节见附录
% additional new appendix
\hl{In addition, we also recognize that the performance is not strong as expected on ArenaHard.
This can be explained by the already impressive performance of the Qwen2.5-7B-Instruct model, which leaves little room for further enhancement on more demanding benchmarks like ArenaHard.
To further validate the efficiency of ~\ourtitle, we extend LLaMA3.1-8B-Instruct with slightly weaker capabilities than Qwen2.5-7B-Instruct but similar parameter sizes.
AMoPO has been proven to deliver substantial gains even when applied to LLaMA3.1-8B-Instruct.}
The detailed analysis and results are shown in Appendix.~\ref{differenrt_model}.
% Nevertheless, the 7B model is still able to learn an understanding of each part of the dimensions.
%再加一段介绍两句我们也做了两两method 的评估
To more intuitively observe the performance differences, we also report the win rate between {\ourtitle} and other methods.
\hl{Experimental results demonstrate that the {\ourtitle} has better response quality, and achieves 81.5$\%$ average win rate} (Details are in Appendix.~\ref{appendix:more_results}).
% new ablation
\subsection{Different Weighted Policies}
% 验证主要是以下的方面
% 1. Gaussian Sample的有效性相对于 Fix weight
% 2. 7B->14B->32B 的scaling的结果反映了当模型越大，模型的理解能力越强的的时候，AMoPO的性能提升会更好
% New additional new appendix
\hl{To delve into the effectiveness and generalization performance of ~\ourtitle, we design three groups of experiments on three benchmarks.}
\hl{Table}~\ref{tab:performance_comparison_weighted} \hl{presents the results.
The Gaussian Sample strategy achieves 1.73 points, 1.13 points, and 0.14 rating average improvement than fixed weight with concise response, verifying the efficiency and superiority.
The decay of the challenging ArenaHard on the 7B model indicates that the lower ability model might be unable to determine the most appropriate course of action during evolution.
As the number of parameters increases, e.g., Qwen2.5-32B-Instruct, our approach achieves greater gains compared to fixed weight without generating redundant responses, demonstrating that the model possesses the ability to identify the most suitable direction for growth.
% With the increase in model size, we also observe that the average length of response becomes longer on the challenging ArenaHard, indicating that {~\ourtitle} stimulates the model's reasoning ability.}
With the increase in model size, the design of {~\ourtitle} encourages the model itself to explore more space to adaptively learn the multidimensional aspects, enhancing the capabilities.}
% \subsection{Ablation on different weight policy}
% weight policy 分类
% // 超参
% 1. fixed weight
% // 自动赋值
% 2. Top-k
% 3. Gaussian Sample
% 4. baseline : Gaussian Sample - Top-20
% Based on whether the weights can be dynamically adjusted, we divide the weight assignment policy into 2 categories: (1) Fixed weights; and (2) Adaptive weights.
% For the policy of \textbf{Fixed weights}, we assign the weights of 1:1:1:1 for the overall preference optimization, the helpfulness dimension, the correctness dimension and the instruction following dimension.
% For the \textbf{Adaptive weights} policy, we explore two different strategies: (1) Top-$K$ assignment, (2) Gaussian sample.
% The core intuition of both strategies is to utilize the overall token probabilities of the generated response from the policy model.
% For the Top-$K$ assignment, we select the top $K$\% of output tokens and average them to assign a weight for the current dimension.
% In detail, we set $K \in [20,40,60]$ for comparison analysis.
% For the Gaussian sample, we perform Gaussian modeling on output tokens and sample a probability as the current dimension weight.
% Furthermore, inspired by reinforcement learning, we combined both strategies into an additional experiment, called Baseline. Using the average probabilities of Top-$K$ tokens minus the probability of Gaussian sample to assign the current dimension weight.
% The result is shown in Table~\ref{tab:performance_comparison_weighted}. Overall, the Gaussian sampling-based approach is more effective, attributed to the variable weights of different dimensions compared to other policies, which allows the policy model to learn multidimensional aspects, and it also expands its exploration space.
% In addition, the Top-$K$ assignment is not stable
% as the policy model will neglect tokens with relatively low probabilities. This method will be not conducive to the extension of the generalization capabilities.
% Furthermore, we report the average length of the generated response of different weighted policies.
% It proves that the Gaussian sampling-based weighting strategy enhances the model capabilities without generating redundant responses.
\begin{table*}[!t]
\centering
\resizebox{0.9\textwidth}{!}{
\begin{tabular}{llcccccc}
\toprule
\multirow{2}{*}{\textbf{Model}} & \multirow{2}{*}{Weighted Policy} & \multicolumn{2}{c}{AlpacaEval 2} & \multicolumn{2}{c}{Arena-Hard} & \multicolumn{2}{c}{MT-Bench} \\
\cmidrule(lr){3-4} \cmidrule(lr){5-6} \cmidrule(lr){7-8}
& & WR (\%) & Avg.Len & WR (\%) & Avg.Len & Rating & Avg.Len \\
\midrule
\multirow{2}{*}{\hl{Qwen2.5-7B-Instruct}}
& \hl{+Fixed Weight(1:1:1:1)} & \hl{57.6(+0.9)} & \hl{1602} & \hl{\textbf{65.2}(-0.8)} & \hl{1994} & \hl{8.44(+0.19)} & \hl{\textbf{1408}} \\
% & +Top-20 & 56.5 & 1605 & \textbf{66.4} & 1994 & 8.51 & \textbf{1406} \\
% & +Top-40 & 56.8 & 1604 & 66.0 & 1994 & 8.51 & \textbf{1406} \\
% & +Top-60 & 57.3 & 1601 & 64.2 & 1990 & 8.60 & 1416 \\
% & +Baseline & 57.5 & 1602 & 63.6 & 1997 & 8.51 & 1402 \\
& \hl{+Gaussian Sample} & \hl{\textbf{58.5}} & \hl{\textbf{1600}} & \hl{64.4} & \hl{\textbf{1989}} & \hl{\textbf{8.63}} & \hl{1415} \\
\cmidrule{1-8}
\multirow{2}{*}{\hl{Qwen2.5-14B-Instruct}}
& \hl{+Fixed Weight(1:1:1:1)} & \hl{63.6(+0.3)} & \hl{1521} & \hl{72.8(+0.6)} & \hl{2014} & \hl{8.97(+0.03)} & \hl{1342} \\
% & +Top-20 & 62.6 & \textbf{1513} & \textbf{73.4} & 2019 & 8.91 & 1340 \\
% & +Top-40 & \textbf{64.4} & 1518 & 71.8 &2015 & 8.86 & 1338 \\
% & +Top-60 & 63.0 & 1520 & 71.6 & 2014 & 8.76 & 1347 \\
% & +Baseline & 64.1 & 1525 & 70.2 & 2017 & 8.78 & 1317 \\
& \hl{+Gaussian Sample} & \hl{\textbf{63.9}} & \hl{\textbf{1518}} & \hl{\textbf{73.4}} & \hl{\textbf{2013}} & \hl{\textbf{9.0}} & \hl{\textbf{1327}} \\
\cmidrule{1-8}
\multirow{2}{*}{\hl{Qwen2.5-32B-Instruct}}
& \hl{+Fixed Weight(1:1:1:1)} & \hl{65.3(+4.0)} & \hl{1649} & \hl{88.4(+3.6)} & \hl{2177} & \hl{9.1(+0.2)} & \hl{1585} \\
& \hl{+Gaussian Sample} & \hl{\textbf{69.3}} & \hl{\textbf{1634}} & \hl{\textbf{92.0}} & \hl{\textbf{2166}} & \hl{\textbf{9.3}} & \hl{\textbf{1535}} \\
\bottomrule
\end{tabular}
}
\caption{\hl{Performance comparison of different weighted policies within the MOBT framework on Qwen2.5-7B/14B/32B-Instruct models}}
% Performance comparison of different weighted policy building on our proposed MOBT framework on Qwen2.5-7B-Instruct/Qwen2.5-14B-Instruct/Qwen2.5-32B-Instruct.}
\label{tab:performance_comparison_weighted}
\end{table*}
\subsection{Detailed Analysis}
% 分两个角度，首先是三个维度的margin的变化，来验证三个维度之间的正相关性。
% To explore the relationships between multiple dimensions, we save the helpfulness, correctness and instruct following margins during the training process.
% As Figure~\ref{fig:3D} shows, the blue dots represent steps in the training process and the red arrow represents the trend of the training data.
% The x, y, and z axes represent the helpfulness, correctness and instruction following margins between the winning and losing responses, accordingly.
We analyze relationships among dimensions by tracking helpfulness, correctness, and instruction following margins during training.
% Figure~\ref{fig:3D} shows blue dots for training steps and a red arrow indicating the data trend, with the x, y, and z axes representing helpfulness, correctness, and instruction following margins, respectively.
\begin{figure}[h]
\centering
\includegraphics[width=0.36\textwidth]{latex/images/exp/3D_2.pdf}
\caption{Three-Dimensional Margin (helpfulness, correctness, instruction following margin) 3D Image.}
\label{fig:3D}
\end{figure}
% The colored surfaces represent the trend surfaces, which means that if the three dimensions are positively correlated, the trend surface will slope upwards. Conversely, if they are negatively correlated, the trend surface will slope downwards.
% 随着训练步数的增加，三个维度的margin值都在上升，说明了我们的模型在三个维度上都有了提升
% Experimental results indicate that as the number of training steps increases, the margin values for the three dimensions are all rising, indicating that our model has improved in all three dimensions, which improves the quality of the generated responses of the policy model.
Figure~\ref{fig:3D} shows that as the training steps increase, the margin values for all three dimensions rise, indicating improved quality of the generated responses from the policy model.
In addition, the three dimensions are positively correlated with the ascending trend surface during the training process,
i.e., as the margins of helpfulness and correctness increase, the margin of instruction following also increases accordingly.
We also report the comparison figures between every two dimensions, as shown in Appendix~\ref{fig:more_results}.
Compared with helpfulness and correctness, the instruction following of the initial policy model is worst (the initial margin is minimal).
However, with the {\ourtitle} policy, the instruction following margin also elevates to a considerable level, which proves that {\ourtitle} can enhance the model performance in each dimension.
% 雷达图：无论是7B还是14B的模型，在通过AMoPO的训练之后，在三个维度上的打分都有提升
\begin{figure}[h]
\centering
\includegraphics[width=0.43\textwidth]{latex/images/exp/radar_14B_new.png}
\caption{\hl{The three-dimensional scores of different methods on Qwen2.5-14B-Instruct.}}
\label{fig:qwen14B_radar}
\end{figure}
\vspace{-0.6cm}
To validate the response capability across the three dimensions on tested datasets, e.g., AlpacaEval 2, we apply \textit{gpt4o-2024-05-13} to evaluate scores across different dimensions for the responses of various methods.
The templates can be obtained in Appendix~\ref{prompts:3dimensions}.
We evaluate all responses in $[0,4]$ for each dimension, averaging them as the indicator.
As Figure~\ref{fig:qwen14B_radar} shows, regardless of the dimensions of helpfulness, correctness, and instruction following, {\ourtitle} demonstrates superior performance than other methods.
% In addition, we found that {\ourtitle} exhibit higher sensitivity to the number of model parameters.
% When the based model is transitioned from 7B to 14B, the response capability of trained model has seen a significant improvement in all dimensions.
% This indicates that as the understanding capabilities of policy model enhance, its comprehension of the three dimensions becomes increasingly clear.
% And then the policy model will generate a more comprehensive responses.
Compared with the consistent growth trend on all dimensions based on Qwen2.5-7B-Instruct (Details are shown in Appendix~\ref{appendix:more_results}), \hl{some previous methods exhibit limitations in several dimensions, e.g., the helpfulness of SimPO and the instruction following of MODPO.}
Instead, our approach achieves comprehensive improvements on all dimensions through holistic modeling and adaptively weighted assignment policy.
% Additional new appendix
\hl{We also report the responses analysis between different multi-objective preference optimization across three dimensions, i.e., helpfulness, correctness and instruct following, illustrating the effectiveness of {\ourtitle} (Details are shown in Appendix}~\ref{different_multi_object}).
In this paper, we propose an adaptive multi-objective preference optimization (\ourtitle) framework to effectively address the limitations of existing preference alignment methods in capturing the complex, multi-dimensional nature of human preferences.
% novelty
\hl{By introducing a new MOBT paradigm and using generation metrics as implicit rewards across multiple dimensions,}
% By proposing a new paradigm and utilizing generation metrics as implicit rewards for multiple dimensions
\ourtitle~achieves precise alignment with diverse preferences without the need for additional reward models or reference models, ensuring it remains lightweight and effective.
Additionally, we incorporate the adaptive weight assignment mechanism, which models the LLM generation space as a Gaussian distribution and allows for dynamic determination of the importance of each preference dimension, thereby enhancing adaptability and efficiency.
% experiments 定量百分之多少的提升；定性有哪些结论
% \todo{add quantitative and qualitative analysis}
\hl{Empirical results demonstrate that {\ourtitle} outperforms state-of-the-art baselines by 28.5$\%$ across multiple benchmarks, showcasing its superior performance.
Furthermore, detailed analysis across multiple dimensions confirms its adaptability and effectiveness.}
% Acknowledge
% \todo{add more}
This work was supported in part by the grants from National Science and Technology Major Project (No. 2023ZD0121104), and National Natural Science Foundation of China (No.62222213, 62072423).
\paragraph{Lack of Attempts on More Datasets.}In our experiment setting, we only train the policy model on Helpsteer2, lacking consideration for the impact of different training datasets, such as Ultrafeedback.
\paragraph{Lack of Consideration for Multi-turn Dialogue Preferences.}~\ourtitle~does not specifically address preference optimization that may change or evolve during multi-turn dialogues. Although~\ourtitle~is effective in handling static preference scenarios, it may require further development to manage dynamic preference changes that occur during ongoing interactions. Future work should focus on collecting more comprehensive preference datasets and integrating multiple aspects of preference optimization to train LLMs that are better aligned with evolving user preferences.
\paragraph{\hl{Lack of Consideration of All Types of Dimensions.}} \hl{In our experiment, we only consider three dimensions, including helpfulness, correctness and instruction following, so that the Gaussian distribution assumption for modeling the generation space may not generalize well to all types of preference dimensions, which could limit its adaptability in certain tasks.}
\section*{Ethics Statement}
In considering the ethical dimensions of our research, we provide the following assurances:
\begin{itemize}
\item In conducting this research, we have adhered to the highest ethical standards and guidelines. We have reported our findings with honesty and ensured clarity and accuracy in all our communications.
\item Our study strictly avoids the use of sensitive or confidential data, ensuring that all materials are appropriate for public dissemination.
\item The datasets employed in our experiments are sourced from publicly accessible and peer-reviewed scientific resources, ensuring transparency and reliability.
\item We provide a thorough account of the dataset characteristics and the hyper-parameter settings used in our experiments to maintain transparency and consistency with our results.
\item To promote transparency and facilitate further research, we commit to sharing our code on anonymous GitHub now and will open source after our paper is accepted.
\end{itemize}
\onecolumn
\section{Mathematical Derivations}
\label{math_derivation}
\subsection{Deriving the multi-objective BT model}
In this appendix, we will derive our Eq.\eqref{Eq.ours}.
We formulate the multi-objective preference optimization problem as follows:
\begin{equation}
\label{eq.overall_app}
\begin{split}
\text{max} \ \pi(y_1 \succ y_2 | x, \bm{\alpha}, \bm{d}) \rightarrow \text{max} \ {\textstyle \sum\limits_{k=1}^{K} } \rho(\alpha_k|x_k^*) \log \pi^*_k(y_1 \succ y_2 | x_k^*) ,
\end{split}
\end{equation}
where $\pi^*_k(y_w \succ y_l | x_k^*)$ and $\rho(\alpha_k|x_k^*)$ denote the distribution and weight of the $k$-th dimension, accordingly.
As shows in Eq.~\eqref{eq.2}, the distribution $\pi^*$ of the single-objective preference optimization can be obtained as:
\begin{equation}
\label{eq.11}
\begin{split}
\pi^{*}(y_1 \succ &\,y_2\,|\,x) =
{ \sigma(r^*(x, y_1) - r^*(x, y_2)) }
\end{split}
\end{equation}
We introduce dimensions $\bm{d}$, which changes the distribution of $x$ with a mapping function $f$, and then we can obtain:
\begin{equation}
\label{eq.12}
\begin{split}
\pi_1^{*}(y_1 \succ \,y_2\,|\,x,\,d_1) = &{ \sigma(r^*(x_1^{*}, y_1) - r^*(x_1^{*}, y_2)) }, x_1^* = f(x,d_1) \\
\pi_2^{*}(y_1 \succ \,y_2\,|\,x,\,d_2) = &{ \sigma(r^*(x_2^{*}, y_1) - r^*(x_2^{*}, y_2)) }, x_2^* = f(x,d_2) \\
\cdots\\
\end{split}
\end{equation}
Then, we combine the distributions of different dimensions and a set of weights $\bm\alpha$ into Eq.~\eqref{eq.overall_app}:
\begin{equation}
\label{eq.13}
\begin{split}
\pi(y_1 \succ \,y_2\,|\,x;\,\bm\alpha;\,\bm{d})
&=\ { \sum\limits_{k=1}^{K} } \alpha_k \log \pi^*_k(y_w \succ y_l | x_k^*) \\
&= \log\prod\limits_{k=1}^{K} {\left[ \sigma^{\alpha_i}(r(x_k^{*}, y_1) - r(x_k^{*}, y_2)) \right]}\;\; \text{s.t.}\: \sum_{k=1}^{K} \alpha_k = 1,\: x_k^{*} = f(x, d_k)
\end{split}
\end{equation}
Substituting Eq.\eqref{eq.3} into Eq.\eqref{eq.13} we obtain:
\begin{equation}
\label{eq.14}
\begin{split}
\pi(y_1 \succ \,y_2\,|\,x;\,\bm\alpha;\,\bm{d}) = \log\prod\limits_{k=1}^{K} {\left[ \sigma^{\alpha_k}(\log\pi_\theta(y_1|x_k^*)-\log\pi_\theta(y_2|x_k^*)) \right]}\;\; \text{s.t.}\: \sum_{k=1}^{K} \alpha_k = 1,\: x_k^{*} = f(x, d_k)
\end{split}
\end{equation}
Finally, we can calculate the {\ourtitle} loss with length normalization to obtain Eq.\eqref{Eq.ours}.
% \subsection{Pseudocode}
% To clear the {\ourtitle} process, we provide a pseudocode as follows:
% \input{latex/images/main/amopo_algorithm}
% \clearpage
% \section{Implementation Details}
% \label{imple_details}
% \subsection{Hyperparameters}
% \hl{For all compared methods, we set $\eta \in [3e-7,5e-6]$ and $epoch \in [2.0,3.0]$ for each method.
% For DPO, CPO and Cal-DPO, we set $\beta$ to 0.2.
% For SimPO, we set $\beta$ to 0.8 and $\gamma$ to 2.0.
% For ORPO, we set $\lambda \in [0.1,0.2]$.
% }
% For MODPO, due to the requirement of training a reward model beyond the primary dimension, we apply ArmoRM~\cite{ArmoRM,wang2024arithmetic}, which is a strong multi-objective reward model by training on Helpsteer2, UltraFeedback, etc. It contains helpfulness, correctness, and instruction following dimensions to fairly compare with {\ourtitle}.
% Finally, we set $\beta$ to 0.8 for {\ourtitle} training.
\clearpage
\section{Implementation Details}
\label{imple_details}
\subsection{Hyperparameters}
For all compared methods, we set $\eta \in [3e-7,5e-6]$ and $epoch \in [2.0,3.0]$ for each method.
For DPO, CPO and Cal-DPO, we set $\beta$ to 0.2.
For SimPO, we set $\beta$ to 0.8 and $\gamma$ to 2.0.
For ORPO, we set $\lambda \in [0.1,0.2]$.
For MODPO, due to the requirement of training a reward model beyond the primary dimension, we apply ArmoRM~\cite{ArmoRM,wang2024arithmetic}, which is a strong multi-objective reward model by training on Helpsteer2, UltraFeedback, etc. It contains helpfulness, correctness, and instruction following dimensions to fairly compare with {\ourtitle}.
Finally, we set $\beta$ to 0.8 for {\ourtitle} training.
Table~\ref{tab:hyperparameters} shows the detailed information on method-specific hyperparameters search ranges
for baselines
\begin{table}[!htbp]
\centering
\caption{The training details of Hyperparameters of different models}
\label{tab:hyperparameters}
\resizebox{0.9\textwidth}{!}{
\begin{tabular}{ccc}
\toprule
\textbf{Method} & \textbf{Objective} & \textbf{Hyperparameters}\\
\midrule
DPO & $-\text{log}\sigma(\beta(\text{log}\frac{\pi_{\theta}(y_w|x)}{\pi_{ref}(y_w|x)}-\text{log}\frac{\pi_{\theta}(y_l|x)}{\pi_{ref}(y_l|x)}))$ & $\beta \in [0.1,0.2]$ \\
ORPO & $-\text{log}p_{\theta}(y_w|x) - \lambda \text{log}\sigma(\text{log}\frac{p_{\theta}(y_w|x)}{1-p_{\theta}(y_w|x)}-\text{log}\frac{p_{\theta}(y_l|x)}{1-p_{\theta}(y_l|x)})$ & $\lambda \in [0.1, 0.2]$ \\
SimPO & $-\text{log}\sigma(\frac{\beta}{|y_w|}\text{log}(y_w|x)-\frac{\beta}{y_l}\text{log}(y_l|x))$ & $\beta \in [0.8, 1.0]$, $\gamma =2.0$ \\
Cal-DPO & $-\text{log}\sigma(\text{log}\frac{\pi_{\theta}(y_w|x)}{\pi_{ref}(y_w|x)}-\text{log}\frac{\pi_{\theta}(y_l|x)}{\pi_{ref}(y_l|x)})+(\text{log}\frac{\pi_{\theta}(y_w|x)}{\pi_{ref}(y_w|x)} - \frac{1}{2\beta})^2 + (\text{log}\frac{\pi_{\theta}(y_l|x)}{\pi_{ref}(y_l|x)} - \frac{1}{2\beta})^2$ & $\beta \in [0.1,0.2]$ \\
CPO & $(1) \text{log}\pi_{\theta}(y|c_1,...,c_m,x) (2) -\text{log} \sigma (\beta\text{log}\frac{\pi_{\theta}(y_w|x)}{\pi_{ref}(y_w|c,x)}- \beta\text{log}\frac{\pi_{\theta}(y_l|x)}{\pi_{ref}(y_l|c,x)})$ & $\beta \in [0.1,0.2]$ \\
MODPO & $-\text{log}\sigma(\frac{\beta}{w_k}(\text{log}\frac{\pi_{\theta}(y_w|x)}{\pi_{ref}(y_w|x)}-\text{log}\frac{\pi_{\theta}(y_l|x)}{\pi_{ref}(y_l|x)}-\frac{1}{w_k}w^T_{-k}(r_{,-k}(x,y_w)-r_{,-k}(x,y_l))))$ & $\beta = 0.2$, $w = 0.5$ , $ArmoRM$ \\
AMoPO & $-\sum_{k=1}^{K} \alpha_k \log \sigma \biggl(\frac{\beta}{|y_w|} \log \pi_\theta(y_w |x^{*}_k) - \frac{\beta}{|y_l|}\log \pi_\theta(y_l |x^{*}_k)\biggr) \text{s.t.}\; x^{*}_k=f(x,{d_k})$ & $\beta = 0.8$, $\gamma = 2.0$ \\
\bottomrule
\end{tabular}
}
\end{table}
\subsection{Training set up}
We train all the methods with 32 A100-80GB SXM GPUs. We set \textit{gradient accumulation steps} to 2, \textit{per device train batch size} to 1, and \textit{float32} used.
\section{HelpSteer2 dataset construction}
\label{prompts:3dimensions}
\subsection{Instruction Following.}
\label{prompts:instruction_following}
We present our score template for the instruction-following dimension by using \textit{gpt4o-2024-05-13}, and the template is as follows:
\begin{figure*}[!h]
\centering
\includegraphics[width=0.98\textwidth]{latex/images/appendix/instruction_following_scor.pdf}
\caption{The instruction following score prompt template.}
\label{fig:instruction_following_prompt}
% \vspace{-10pt}
\end{figure*}
\clearpage
\subsection{Helpfulness.}
\label{prompt:helpfulness}
We present our score template for the helpfulness dimension by using \textit{gpt4o-2024-05-13}, and the template is as follows:
\begin{figure*}[h]
\centering
\includegraphics[width=0.98\textwidth]{latex/images/appendix/Helpfulness_score.pdf}
\caption{The helpfulness score prompt template.}
\label{fig:helpfulness_prompt}
% \vspace{-10pt}
\end{figure*}
\clearpage
\subsection{Correctness.}
\label{prompt:correctness}
We present our score template for the correctness dimension by using \textit{gpt4o-2024-05-13}, and the template is as follows:
\begin{figure*}[!h]
\centering
\includegraphics[width=0.98\textwidth]{latex/images/appendix/Correctness_score.pdf}
\caption{The correctness score prompt template.}
\label{fig:correctness_prompt}
% \vspace{-10pt}
\end{figure*}
\clearpage
\subsection{Harmlessness.}
\label{prompt:harmlessness}
We present our jugdement template for the harmlessness dimension by using \textit{gpt4o-2024-05-13}, and the template is as follows:
\begin{figure*}[!h]
\centering
\includegraphics[width=0.98\textwidth]{latex/images/appendix/harmlessness_judgement.pdf}
\caption{The harmlessness judgement prompt template.}
\label{fig:harmlessness_prompt}
\end{figure*}
\clearpage
\subsection{Conciseness/Verbosity.}
\label{prompt:conciseness}
We also present our score template for the Verbosity dimension by using \textit{gpt4o-2024-05-13}, and the template is as follows:
\begin{figure*}[!h]
\centering
\includegraphics[width=0.98\textwidth]{latex/images/appendix/verbosity_prompt.pdf}
\caption{The verbosity score prompt template.}
\label{fig:verbosity_prompt}
\end{figure*}
\clearpage
\section{Experiments details}
\subsection{Comparison Figures}
\label{fig:more_results}
We present three comparison figures including helpfulness vs correctness, correctness vs instruction following and helpfulness vs instruction following as follows:
\begin{figure*}[!h]
\centering
\includegraphics[width=0.88\textwidth]{latex/images/exp/trainsteps.pdf}
\caption{The correctness score prompt template.}
\label{fig:figures}
% \vspace{-10pt}
\end{figure*}
% \begin{figure*}[!h]
% \centering
% \includegraphics[width=0.6\textwidth]{latex/images/exp/Helpfulness_vs_Correctness.png}
% \caption{Helpfulness Vs Correctness.}
% \label{fig:helpfulness_corrrectness}
% % \vspace{-10pt}
% \end{figure*}
% \begin{figure*}[!h]
% \centering
% \includegraphics[width=0.6\textwidth]{latex/images/exp/Correctness_vs_Instruction_Following.png}
% \caption{Correctness Vs Instruction Following.}
% \label{fig:corrrectness_instruction_following}
% % \vspace{-10pt}
% \end{figure*}
% \begin{figure*}[!h]
% \centering
% \includegraphics[width=0.6\textwidth]{latex/images/exp/Helpfulness_vs_Instruction_Following.png}
% \caption{Helpfulness Vs Instruction Following.}
% \label{fig:helpfulness_instruction_following}
% % \vspace{-10pt}
% \end{figure*}
As shown in Figure~\ref{fig:figures}, with one variable increasing, other variables tend to increase as well, indicating a potentially mutually reinforcing relationship for each figure.
In addition, the regression lines suggest that the relationships among the variables are linear, implying that changes in one variable can be used to predict changes in another variable.
Furthermore, the steepest slope between helpfulness and correctness indicates that increasing helpfulness may lead to more substantial improvements in correctness.
It is encouraging that when we find it difficult to directly increase the correctness of the responses, starting by increasing the helpfulness of the responses may be an alternative.
\clearpage
\subsection{Evaluation details}
\label{evaluation_details}
We apply \textit{gpt4o-2024-05-13} as our judging model, and the win rate template for AlpacaEval 2 and Arena-Hard is as follows:
\begin{figure*}[h]
\centering
\includegraphics[width=0.8\textwidth]{latex/images/exp/WR_Evaluation.pdf}
\caption{The evaluation of win rate prompt template.}
\label{fig:WR_prompt}
% \vspace{-10pt}
\end{figure*}
For MT-bench, we still apply \textit{gpt4o-2024-05-13} as our judging model, and the rating templates, including math and none-math category are as follows:
\begin{figure*}[h]
\centering
\includegraphics[width=0.8\textwidth]{latex/images/exp/MT_bench_math.pdf}
\caption{The evaluation of rating prompt template of math category.}
\label{fig:MT_prompt_math}
% \vspace{-10pt}
\end{figure*}
\begin{figure*}[h]
\centering
\includegraphics[width=0.8\textwidth]{latex/images/exp/MT_bench_math.pdf}
\caption{The evaluation of rating prompt template of none-math category.}
\label{fig:MT_prompt_none_math}
% \vspace{-10pt}
\end{figure*}
\clearpage
\subsection{More results}
\label{appendix:more_results}
To further explore the performance across different methods, we test on AlpacaEval 2, evaluating the win rate directly with templates provided in Appendix~\ref{evaluation_details}. Consequently, we obtain the results as follows:
\begin{table*}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{llcccccc}
\toprule
\multirow{2}{*}{\textbf{Model}} & \multirow{2}{*}{Method} & \multicolumn{5}{c}{Baseline Methods} \\
\cmidrule(lr){3-3} \cmidrule(lr){4-4} \cmidrule(lr){5-5} \cmidrule(lr){6-6} \cmidrule(lr){7-7} \cmidrule(lr){8-8}
& & DPO & Cal-DPO & ORPO & SimPO & MODPO & \hl{CPO}\\
\midrule
\multirow{1}{*}{Qwen2.5-7B-Instruct}
& {\ourtitle} & 82.0 & 83.1 & 89.2 & 76.1 & 78.4 & \hl{79.5} \\
\cmidrule{1-8}
\multirow{1}{*}{Qwen2.5-14B-Instruct}
& {\ourtitle} & 80.0 & 79.9 & 90.0 & 81.2 & 81.4 & \hl{78.1}\\
\bottomrule
\end{tabular}
}
\caption{The win rate of {\ourtitle} compared to different methods on Qwen2.5-7B-Instruct/Qwen2.5-14B-Instruct on AlpacaEval 2.}
\label{tab:performance_comparison_all_methods}
\end{table*}
We also calculate the improvement rate by comparing our method with the baseline methods, demonstrating the superiority. The details are shown as follows:
\begin{table*}[h]
\centering
\resizebox{0.98\textwidth}{!}{
\begin{tabular}{lcccccccc}
\toprule
& \multicolumn{4}{c}{Qwen2.5-7B-Instruct} & \multicolumn{4}{c}{Qwen2.5-14B-Instruct} \\
\cmidrule(lr){2-5} \cmidrule(lr){6-9}
\textbf{Method} & AlpacaEval 2 & Arena-Hard & MT-Bench &\multirow{2}{*}{Overall improvement (\%)} & AlpacaEval 2 & Arena-Hard & MT-Bench &\multirow{2}{*}{Overall improvement} \\
\cmidrule(lr){2-2} \cmidrule(lr){3-3} \cmidrule(lr){4-4}
\cmidrule(lr){6-6} \cmidrule(lr){7-7} \cmidrule(lr){8-8}
& WR (\%) & WR (\%) & Avg.Rating & & WR (\%) & WR (\%) & Avg.Rating& \\
\midrule
DPO &41.5 & 56.4 & 8.58 & 18.6 & 56.4 & 66.8 & 8.77 & 8.6 \\
ORPO & 13.1 & 27.4 & 7.92 & 163.5 & 18.8 & 34.2 & 8.07 & 122.0 \\
SimPO & \underline{56.1} & \underline{64.2} & 8.50 & 2.1 & 58.6 & 67.8 & \underline{8.90} & 6.1 \\
Cal-DPO & 41.7 & 55.4 & \underline{8.60} & 19.0 & 52.8 & 65.8 & 8.68 & 12.1 \\
\hl{CPO} & \hl{52.0} & \hl{62.0} & \hl{8.43} & \hl{6.3} & \underline{\hl{59.0}} & {\hl{69.4}} & \hl{8.8} & \hl{5.4}\\
SPO & 41.2 & 60.4 & 8.58 & 16.4 & 55.9 & \underline{71.2} & 8.8 & 6.6 \\
MODPO & 52.8 & 63.4 & 8.57 & 4.4 & 55.7 & 68.8 & 8.72 & 8.2\\
\ourtitle & \textbf{58.5} & \textbf{64.4} & \textbf{8.63} & - & \textbf{63.9} & \textbf{73.4} & \textbf{9.0} & - \\
\bottomrule
\end{tabular}
}
\caption{Performance comparison of different methods on Qwen2.5-7B-Instruct and Qwen2.5-14B-Instruct across various benchmarks.}
\label{tab:performance_comparison}
\end{table*}
Furthermore, we also evaluate the responses of {\ourtitle}, MODPO, CPO, SimPO, and DPO based on Qwen2.5-7B-Instruct, showing the quality across different dimensions.
\begin{figure}[h]
\centering
\includegraphics[width=0.6\textwidth]{latex/images/exp/radar_7B_new.png}
\caption{\hl{The three-dimensional scores of different methods on Qwen2.5-7B-Instruct.}}
\label{fig:qwen7B_radar}
\end{figure}
As shown in Figure.\ref{fig:qwen7B_radar}, \hl{although MODPO introduces a multi-dimensional reward model and CPO introduces the special tokens to incorporate multi-objective information, its effectiveness has not improved compared to single-objective optimization, e.g., SimPO.
It implies that the mechanism of MODPO and CPO is difficult to align multiple dimensions. }
Instead, our method {\ourtitle} achieves precise alignment and multidimensional improvement compared to single-objective and multi-objective preference optimization,
due to the lightweight and effective multi-objective alignment mechanism.
\clearpage
\subsection{Human validation}
We have incorporated the experiments about human validations. We have randomly sampled 20 instances from each dataset (totally have 60 samples) with 5 evaluators. The human evaluators assessed each instance from helpfulness, correctness and instruction following criteria (consistent with the paper in Appendix ~\ref{prompts:3dimensions}), and we report the human evaluations as follows. Notably, the human assessments also highlight the strengths of \ourtitle.
\begin{table*}[h]
\centering
\resizebox{0.6\textwidth}{!}{
\begin{tabular}{lccc}
\toprule
& \multicolumn{3}{c}{Qwen2.5-14B-Instruct} \\
\cmidrule(lr){2-4}
\textbf{Method} & Helpfulness & Correctness & Instruction Following \\
\cmidrule(lr){2-2} \cmidrule(lr){3-3} \cmidrule(lr){4-4}
& mean (std) & mean (std) & mean (std) \\
\midrule
CPO & $3.25_{\pm0.16}$ & $2.95_{\pm0.04}$ & $2.90_{\pm0.16}$ \\
MODPO & $3.41_{\pm0.08}$ & $3.20_{\pm0.07}$ & $3.12_{\pm0.07}$ \\
\ourtitle(ours) &\textbf{ $3.49_{\pm0.06}$ }& \textbf{$3.22_{\pm0.02}$} & \textbf{$3.19_{\pm0.04}$}\\
\bottomrule
\end{tabular}
}
\caption{The human evaluation of CPO, MODPO and {~\ourtitle} on Helpfulness, Correctness and Instruction Following.}
\label{tab:conflict_results}
\end{table*}
\subsection{More conflicts dimensions}
While the three dimensions, i.e., helpfulness, correctness and instruction following are positive, we also conducted comprehensive evaluations on Qwen2.5-7B-Instruct and Qwen2.5-14B-Instruct models specificially targeting dimensions that involve conflicts: \textit{Helpfulness-Harmlessness} and \textit{Helpfulness-Verbosity}. These evaluation template are shown in Appendix~\ref{prompts:3dimensions}.
\begin{table*}[h]
\centering
\resizebox{0.9\textwidth}{!}{
\begin{tabular}{lcccccccc}
\toprule
& \multicolumn{3}{c}{Qwen2.5-7B-Instruct} & \multicolumn{3}{c}{Qwen2.5-14B-Instruct} \\
\cmidrule(lr){2-4} \cmidrule(lr){5-7}
\textbf{Method} & Helpfulness & Harmlessness & Verbosity & Helpfulness & Harmlessness & Verbosity \\
\cmidrule(lr){2-2} \cmidrule(lr){3-3} \cmidrule(lr){4-4}
\cmidrule(lr){5-5} \cmidrule(lr){6-6} \cmidrule(lr){7-7}
& Score($\uparrow$) & pass@1($\uparrow$) & Score($\downarrow$) & Score($\uparrow$) & pass@1($\uparrow$) & Score($\downarrow$) \\
\midrule
CPO & 2.83 & 77\% & 2.95 & 3.29 & 72\% & 3.05 \\
MODPO & 2.88 & 75\% & 2.86 & 3.53 & 84\% & 2.81 \\
SPO & 2.82 & 71\% & 2.90 & 3.15 & 78\% & 2.89 \\
\ourtitle(ours) & \textbf{2.97} & \textbf{79\% }& \textbf{2.75} & \textbf{3.59} & \textbf{85\%} & \textbf{2.68} \\
\bottomrule
\end{tabular}
}
\caption{The conflicts dimensions evaluation of multi-objective preference optimization methods on AlpacaEval2 benchmark.}
\label{tab:conflict_results}
\end{table*}
Table~\ref{tab:conflict_results} indicates that {~\ourtitle} performs well not only in consistent dimensions but also in those characterized by inherent conflicts. This demonstrates the model's ability to balance competing objectives effectively, further underscoring its generalization capabilities.
\subsection{Different models}
\label{differenrt_model}
\hl{While the Gaussian sample-based method showed an average improvement of +1.41\% over the fixed weight policy on the Qwen2.5-instruct series models, we recognize that its performance was not as strong as expected on certain benchmarks. This can be attributed to the inherent strength of the Qwen2.5-7B-instruct model, which already performs exceptionally well, leaving limited room for improvement on more challenging benchmarks (such as Arena-Hard). The results of Qwen2.5-7B-Instruct and LLaMA3.1-8B-Instruct across three benchmarks are as follows:}
\begin{table}[!htbp]
\centering
\caption{The performance of different base models on benchmarks.}
\label{tab:qwen_llama_base}
\resizebox{0.9\textwidth}{!}{
\begin{tabular}{ccccc}
\toprule
\textbf{Model} & AlpacaEval2 WR(\%) & ArenaHard WR(\%) & MT-bench Rating(1-10)\\
\midrule
\hl{Qwen2.5-7B-Instruct} & \hl{45.2} & \hl{52.2} & \hl{8.40} \\
\hl{LLaMA3.1-8B-Instruct} & \hl{40.5} & \hl{33.4} & \hl{8.08}\\
\bottomrule
\end{tabular}
}
\end{table}
\hl{
To further validate our AMoPO, we extended our experiments to include the Llama3.1-8B-instruct model. These experiments demonstrated a significant improvement of +16.03\% over the fixed weight policy, indicating that Gaussian samples are more effective in enhancing models with lower baseline performance. Thus, the adaptive weight assignment mechanism introduced by our AMoPO enables it to dynamically adjust weights across various base models or preference dimensions, thereby enhancing its flexibility and effectiveness in diverse contexts.
}
\begin{table}[!htbp]
\centering
\caption{The performance of different weight policy on LLaMA3.1-8B-Instruct model.}
\label{tab:llama3.1}
\resizebox{0.9\textwidth}{!}{
\begin{tabular}{ccccc}
\toprule
\textbf{Model} & Method & AlpacaEval2 WR(\%) & ArenaHard WR(\%) & MT-bench Rating(1-10)\\
\midrule
\multirow{2}{*}{\hl{LLaMA3.1-8B-Instruct}} & \hl{+Fixed Weight(1:1:1:1)} & \hl{47.1} & \hl{43.6} & \hl{8.22}\\
& \hl{+Gaussian Sample} & \textbf{\hl{55.0}} & \textbf{\hl{56.2}} & \textbf{\hl{8.42}}\\
\bottomrule
\end{tabular}
}
\end{table}
\subsection{Ablation}
We conducted additional experiments assessing the performance of our method without length normalization and also evaluated our {\ourtitle} using a reference model.
The results, as detailed in Table~\ref{tab:ablation} suggest that length normalization contributes to more consistent and reliable outputs.
And we found that using a reference model sometimes introduced complexity and alignment issues, which could detract from optimization performance. Removing it led to efficiency gains and maintained competitive performance, highlighting the effectiveness of {\ourtitle} design.
\begin{table}[h]
\centering
\resizebox{0.98\textwidth}{!}{
\begin{tabular}{lcccccc}
\toprule
& \multicolumn{3}{c}{Qwen2.5-7B-Instruct} & \multicolumn{3}{c}{Qwen2.5-14B-Instruct} \\
\cmidrule(lr){2-4} \cmidrule(lr){5-7}
\textbf{Method} & AlpacaEval 2 & Arena-Hard & MT-Bench & AlpacaEval 2 & Arena-Hard & MT-Bench \\
\cmidrule(lr){2-2} \cmidrule(lr){3-3} \cmidrule(lr){4-4} \cmidrule(lr){5-5} \cmidrule(lr){6-6} \cmidrule(lr){7-7}
& WR (\%) & WR (\%) & Avg.Rating & WR (\%) & WR (\%) & Avg.Rating \\
\midrule
\textit{w/o} length normalization & 40.5 & 62.0 & 8.27 &50.3 & 72.0 & 8.7 \\
\textit{w/} reference model & 43.4 & 60.0 & 8.55 & 58.5 & 65.6 & 8.74 \\
\ourtitle & \textbf{58.5} & \textbf{64.4} & \textbf{8.63} & \textbf{63.9} & \textbf{73.4} & \textbf{9.0} \\
\bottomrule
\end{tabular}
}
\caption{The ablation study of reference model and length normalization.}
\label{tab:ablation}
\end{table}
\subsection{TruthfulQA validation}
To evaluate each preference objective individually, we have added experiments using the TruthfulQA~\cite{truthfulqa2022} benchmark with MC2 metric for factual accuracy assessment (like the HelpSteer2 paper). Our AMoPO framework demonstrates a 1.8\% improvement in accuracy over both CPO and MODPO baselines.
\begin{table}[htbp]
\centering
\caption{The evaluations results of TruthfulQA MC2 on Qwen2.5-14B-Instruct.}
\begin{tabular}{lc}
\toprule
Method & TruthfulQA MC2 \\
\midrule
CPO & 0.791 \\
MODPO & 0.790 \\
AMoPO(ours) & 0.810 \\
\bottomrule
\end{tabular}
\label{tab:results}
\end{table}
% \clearpage
\section{Case Study}
\label{Cases}
\subsection{7B vs 14B}
In this section, we compared Qwen2.5-7B-Instruct and Qwen2.5-14B-Instruct with the {\ourtitle} policy. As Table~\ref{tab:case_7b_14B_1} and Table~\ref{tab:case_7B_14B_2}, the fewer model will generate verbose and hallucinated responses despite using our training method.
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{6cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{Rewrite the given text and correct grammar, spelling, and punctuation errors. If you'd told me year ago that today I would finish a marathon, I would of laughed. Your support had a huge affect on me!}\\
\midrule
\textbf{\scriptsize Qwen2.5-7B-Instruct based} & \raggedright\arraybackslash\scriptsize{\textcolor{blue}{If you had told me a year ago that today I would finish a marathon, I would have laughed. Your support had a huge affect on me!} \textcolor{red}{(Note: For "affect" to be correct, it should be used as a verb. If "affect" is meant to be used as a noun, it should be "effect.}} \\
\midrule
\textbf{\scriptsize Qwen2.5-14B-Instruct based} & \raggedright\arraybackslash\scriptsize{\textcolor{blue}{If you had told me a year ago that I would finish a marathon today, I would have laughed. Your support had a huge effect on me!}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of different based models with {\ourtitle} training, the \textcolor{blue}{blue} is correct, helpful and instruction following information. Meanwhile, the \textcolor{red}{red} means wrong, harmless or disobedient content.}
\label{tab:case_7b_14B_1}
\end{table}
In the case in Table~\ref{tab:case_7b_14B_1}, the Qwen2.5-14B-Instruct can correctly answer the prompt through {\ourtitle}.
The Qwen2.5-7B-Instruct will generate redundant information after answering, e.g., \textit{Note: For "affect" to be correct, ......}, indicating the insufficient ability compared to 14B.
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{6cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{Summarize the movie in a snarky way. Try to explain the movie in just one sentence. The Shining}\\
\midrule
\textbf{\scriptsize Qwen2.5-7B-Instruct based} & \raggedright\arraybackslash\scriptsize{The Shining: A film where \textcolor{blue}{a grumpy writer takes his equally grumpy family to a haunted hotel only to spend months alone}, \textcolor{red}{resulting in his daughter becoming a talking psychic dog} while he descends into madness—because nothing says "classic horror" like a family vacation from hell.} \\
\midrule
\textbf{\scriptsize Qwen2.5-14B-Instruct based} & \raggedright\arraybackslash\scriptsize{Sure, "The Shining" is about \textcolor{blue}{a guy who thinks typing in a deserted haunted hotel while everyone around him is losing their mind} is just the remote possibility of {the century for productivity.}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of different based models with {\ourtitle} training, the \textcolor{blue}{blue} is correct, helpful and instruction following information. Meanwhile, the \textcolor{red}{red} means wrong, harmless or disobedient content. }
\label{tab:case_7B_14B_2}
\end{table}
In the case in Table~\ref{tab:case_7B_14B_2}, both models provided partly correct responses. Unlike Qwen2.5-7B-Instruct, the 14B model will not fabricate incorrect information, e.g., \textit{resulting in his daughter becoming a talking psychic dog}, which is prohibited.
\clearpage
\subsection{Fixed Weight vs Gaussian Sample}
\hl{
In this section, we supply the response of the same prompt between Gaussian Sample and fixed weight on Qwen2.5-32B-Instruct.
}
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{6cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{\hl{Is it possible to update the shell.nix to add new packages while I am already inside the shell, without restarting nix-shell?}} \\
\midrule
\textbf{\scriptsize Fixed Weight} & \raggedright\arraybackslash\scriptsize{\hl{Unfortunately, it's not possible to directly update the `shell.nix` file and have those changes take effect without restarting the `nix-shell`. When you start a `nix-shell`, it creates a new shell environment based on the dependencies specified in the `shell.nix` file. If you modify `shell.nix` to include new packages, those changes won't be reflected in the current shell session.
To apply the changes, you need to exit the current `nix-shell` and start a new one. Here are the steps:
1. Exit the current `nix-shell`:
```sh
exit
```
2. Modify your `shell.nix` file to include the new packages.
3. Start a new `nix-shell` session:
```sh
nix-shell
```
This will recreate the environment with the new dependencies specified in your updated `shell.nix` file.
If you want to avoid the overhead of restarting `nix-shell` frequently, consider using other development tools or strategies that can provide more dynamic package management within your development workflow. However, for `nix-shell`, restarting is necessary to pick up changes in `shell.nix`.}} \\
\midrule
\textbf{\scriptsize Gaussian Sample} & \raggedright\arraybackslash\scriptsize{\hl{Unfortunately, it is not possible to update the `shell.nix` file and have those changes take effect without restarting the `nix-shell`. The `nix-shell` environment is created based on the `shell.nix` file when you initially start the shell session. Any changes to `shell.nix` after the shell has been started will not be reflected until you restart the `nix-shell`.
Here are the steps you would typically follow to apply changes:
1. **Edit `shell.nix`:** Make the necessary changes to add new packages.
2. **Exit the current `nix-shell`:** You can do this by simply typing `exit`.
3. **Restart `nix-shell`:** Run `nix-shell` again to enter a new shell environment with the updated configurations.
If you frequently need to test changes to your `shell.nix`, you might consider scripting these steps to make the process quicker and more convenient.}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of different weighted assignment on Qwen2.5-32B-Instruct. }
\label{tab:case_32B_fix_gaussian}
\end{table}
\hl{
Although both models provide correct answers to the current problem, the answers generated by the model after Gaussian sample are more concise and include recommendations for the user to make future predictions. Such responses are often more helpful.
}
\clearpage
\subsection{Compared to MODPO}
In these section, we provided some cases to compare {\ourtitle} with MODPO.
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{Write a code block in Markdown containing an example of a code block in Markdown. Don't forget those quadruple backticks.}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{Here is a code block in Markdown that demonstrates how to create another code block, including the quadruple backticks and the language identifier:````markdown```markdown. This is an example of a code block in Markdown. You can see that it is formatted to look like it's not part of the regular text flow.'''} \\
\midrule
\textbf{\scriptsize MODPO} & \raggedright\arraybackslash\scriptsize{```markdown``` Example of a code block in Markdown, print("Hello, world!")```} \\
\midrule
\textbf{\scriptsize Judgment} & \raggedright\arraybackslash\scriptsize{Compared to MODPO, {[\textbf{\ourtitle}]} provides a complete example that illustrates how to create a nested code block in Markdown using quadruple backticks} \\
\bottomrule
\end{tabular}
}
\caption{Responses of MODPO and {\ourtitle} with same prompt.}
\label{tab:case_modpo_amopo_1}
\end{table}
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{Invent a new word based on a description of the word. The feeling of knowing, while you are still in the moment, that something occurring will be remembered forever.}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{\textbf{Momentrecolle} A combination of "moment" and "mémorécolle" (from the French "mémoriser," to memorize, and "coller," to stick), *momentrecolle* describes the profound awareness that the experience or moment you are living will be indelibly etched in your memory forever.} \\
\midrule
\textbf{\scriptsize MODPO} & \raggedright\arraybackslash\scriptsize{\textbf{Momenticent} A blend of "moment" and "centennial," *momenticent* captures the profound awareness that an experience or event being lived through will be etched indelibly in one\'s memory, potentially becoming a milestone that defines or significantly influences one\'s life for years to come.} \\
\midrule
\textbf{\scriptsize Judgment} & \raggedright\arraybackslash\scriptsize{\textbf{{\ourtitle}} combines aspects of both the word 'moment' and a thoughtful incorporation of French words that emphasize memory retention, making it a unique and memorable new word that captures the essence of the described feeling. {MODPO}, while also creative, uses 'centennial' which might imply a century or 100 years, potentially confusing its intended meaning. In summary, \textbf{[{\ourtitle}]}} response is better than MODPO. \\
\bottomrule
\end{tabular}
}
\caption{Responses of MODPO and {\ourtitle} with same prompt.}
\label{tab:case_modpo_amopo_2}
\end{table}
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{write an introduction of a person for resume who worked as an in-house IT for 18 years, managed team of 4 and 9 site and satellite offices with total 200 users. He's familiar with network and system infrastructure, server virtualization, cloud services and the migration.}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{\textbf{Professional Summary:} Dedicated and experienced IT professional with 18 years of hands-on experience in managing comprehensive IT operations for a diverse organization. Successfully led a team of 4 IT specialists, overseeing network and system infrastructure for 9 site and satellite offices, supporting a total of 200 users. Expert in designing, implementing, and maintaining robust network solutions, server virtualization, and cloud services. Proven track record in facilitating seamless migrations to cloud environments, ensuring high levels of system reliability and security. Committed to staying current with the latest IT trends and technologies to drive operational efficiency and innovation.} \\
\midrule
\textbf{\scriptsize MODPO} & \raggedright\arraybackslash\scriptsize{\textbf{Professional Summary} Dedicated and experienced IT professional with 18 years of hands-on experience in managing in-house IT operations for a diverse organization. Proven track record of leading a team of 4 IT professionals and overseeing the IT infrastructure for 9 site and satellite offices, serving a total of 200 users. Expert in network and system infrastructure, server virtualization, cloud services, and seamless migration processes. Committed to delivering high-quality IT solutions that drive business efficiency and innovation.} \\
\midrule
\textbf{\scriptsize Judgment} & \raggedright\arraybackslash\scriptsize{Compared to MODPO, {[\textbf{\ourtitle}]} provides a more detailed overview of the candidate's experience and skills, highlighting not only the management of IT operations and teams but also the specific expertise in network solutions, server virtualization, and cloud services. It also mentions staying current with IT trends, which adds to the candidate's credibility and commitment to professional growth. These details can better capture the attention of potential employers.} \\
\bottomrule
\end{tabular}
}
\caption{Responses of MODPO and {\ourtitle} with same prompt.}
\label{tab:case_modpo_amopo_3}
\end{table}
\clearpage
\subsection{Compared to CPO}
\hl{In this section, we provided some cases to compare {~\ourtitle} with CPO.}
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{\hl{Give the provided brand a motto to use as a part of its marketing strategies.
The Ice cream shop is a small, quaint shop located in the heart of the historic district. They offer a variety of ice cream flavors, as well as Italian ice, gelato, and sorbet. The shop has a few tables and chairs, but most of their business is to-go. They offer a loyalty program where you can get a free scoop of ice cream after 10 purchases.}}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{\hl{Certainly! Here’s a charming motto that captures the essence of your ice cream shop:
\textbf{"Indulge in History, Savor the Moment — One Scoop at a Time"}
This motto highlights the nostalgic and delightful experience your shop offers, blending the historic charm of its location with the joy of delicious treats.
}} \\
\midrule
\textbf{\scriptsize CPO} & \raggedright\arraybackslash\scriptsize{\hl{Brand Motto Suggestion: \textbf{"Indulge in History, One Scoop at a Time"}
This motto encapsulates the charm and authenticity of the historic district setting while highlighting the variety and quality of their ice cream offerings. It also invites customers to return frequently to enjoy the delightful experience, thereby encouraging participation in the loyalty program.}} \\
\midrule
\textbf{\scriptsize Judgment} & \raggedright\arraybackslash\scriptsize{\hl{Compared to CPO, \textbf{{\ourtitle}} provides a more complete and evocative motto by incorporating the phrase \textbf{'Savor the Moment' which adds a touch of nostalgia and enjoyment.} This better captures the quaint and historic essence of the shop and creates a more appealing and memorable marketing message.}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of CPO and {\ourtitle} with same prompt.}
\label{tab:case_cpo_amopo_1}
\end{table}
\hl{Compared with CPO, we observe that the responses of {\ourtitle} is more human-like and more enjoyment.}
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{\hl{Make the article available in a second language by translating it into it.
Dentistry, also known as dental medicine and oral medicine, is the branch of medicine focused on the teeth, gums, and mouth. It consists of the study, diagnosis, prevention, management, and treatment of diseases, disorders, and conditions of the mouth, most commonly focused on dentition (the development and arrangement of teeth) as well as the oral mucosa. Dentistry may also encompass other aspects of the craniofacial complex including the temporomandibular joint. The practitioner is called a dentist.
Translate to French:}}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{\hl{La dentisterie, également connue sous les noms de médecine dentaire et de médecine orale, est la branche de la médecine qui se concentre sur les dents, les gencives et la bouche. Elle englobe l'étude, le diagnostic, la prévention, la gestion et le traitement des maladies, troubles et conditions de la bouche, en se concentrant le plus souvent sur la dentition (le développement et l'arrangement des dents) ainsi que la muqueuse buccale. La dentisterie peut également englober d'autres aspects du complexe cranio-facial, y compris l'articulation temporo-mandibulaire. Le praticien est appelé dentiste.
}} \\
\midrule
\textbf{\scriptsize CPO} & \raggedright\arraybackslash\scriptsize{\hl{La dentisterie, aussi appelée médecine dentaire et médecine orale, est une branche de la médecine qui se concentre sur les dents, les gencives et la bouche. Elle inclut l'étude, le diagnostic, la prévention, la gestion et le traitement des maladies, troubles et conditions de la bouche, principalement axée sur la dentition (le développement et l'arrangement des dents) ainsi que la muqueuse buccale. La dentisterie peut également englober d'autres aspects de la structure crânio-faciale, notamment la jointure temporomandibulaire. Le praticien est appelé dentist.
Note: The French term for the practitioner is actually "dentiste" not "dental" which would mean "dental" in English. I maintained "dentist" in the last sentence to stay consistent with the original sentence structure you provided.
}} \\
\midrule
\textbf{\scriptsize Judgment} & \raggedright\arraybackslash\scriptsize{\hl{The preferred answer is {\ourtitle}, \textbf{{\ourtitle}} provides a correct and accurate translation of the given text from English to French. It maintains the original meaning and technical terminology, making it more appropriate for conveying precise medical information in French. CPO, although close, has minor inconsistencies and unnecessary explanations that detract from the straightforward nature of the request.}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of CPO and {\ourtitle} with same prompt.}
\label{tab:case_cpo_amopo_2}
\end{table}
\clearpage
\subsection{Dimensions case study}
\label{different_multi_object}
\begin{table}[h]
\centering
\resizebox{0.7\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{\hl{I want to talk about "Smooth", a song by Rob Thomas.}}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{\hl{"Smooth" is actually a song made famous by Santana featuring Rob Thomas, not solely by Rob Thomas. Released in 1999 as part of the album "Supernatural," the song became a massive hit, blending Latin rhythms with rock and pop elements. It features Rob Thomas's vocals and songwriting, with Santana providing the distinctive guitar work and Latin music influence.
The song reached number one on the US Billboard Hot 100 and won several Grammy Awards, including Record of the Year and Song of the Year in 2000. It's known for its catchy melody, smooth guitar riffs, and Thomas's soulful vocals. If you have any specific questions about the song or would like to discuss certain aspects of it, feel free to ask!}} \\
\midrule
\textbf{\scriptsize CPO} & \raggedright\arraybackslash\scriptsize{\hl{"Smooth" is a popular song written by Rob Thomas, best known as the lead singer of the rock band Matchbox Twenty, and it was a big hit when it was released as a collaborative single between Rob Thomas and latin-guitarist Santana in 1999. The song appears on Santana's album "Supernatural."
The song earned critical acclaim and commercial success, topping the charts in many countries and becoming one of the best-selling singles of all time. It won three Grammy Awards, including "Record of the Year," in 2000.
The song features Thomas's smooth vocals combined with Santana's distinctive guitar work. The lyrics tell a story of a man who speaks to a woman in Spanish but doesn't know the language, yet he seems to be able to charm her and convince her to leave with him. The song's hook features the line "I can talk a good game," which refers to the narrator's ability to charm and seemingly speak the language fluently.
"Smooth" is known for its catchy melody, memorable guitar solos, and unique fusion of rock and Latin music. The song's success helped to introduce Latin music to a broader audience and has remained a popular classic since its release.
Is there anything in particular about the song or its impact that you would like to discuss?}} \\
\midrule
\textbf{\scriptsize Helpfulness Judgment} & \raggedright\arraybackslash\scriptsize{\hl{\textbf{{\ourtitle}} is more helpful in this case as it directly addresses the user's instruction by providing essential facts about the song 'Smooth,' focusing on the key collaboration between Santana and Rob Thomas, its release details, and the song's success. \textbf{It avoids unnecessary elaboration and stays concise}, which might be preferable if the user was looking for clear and accurate information without getting into deeper analysis.}} \\
\midrule
\textbf{\scriptsize Correctness Judgment} & \raggedright\arraybackslash\scriptsize{\hl{\textbf{\ourtitle} is more correct \textbf{in terms of the factual details about the collaboration, chart success, and Grammy wins, ensuring better correctness} than \textbf{MODPO}, \textbf{which introduces inaccurate interpretations of the song's lyrics.}}} \\
\midrule
\textbf{\scriptsize Instruct Follow Judgment} & \raggedright\arraybackslash\scriptsize{\hl{\textbf{\ourtitle} \textbf{follows the instruction more closely by focusing on the song's key elements: collaboration with Santana, its release, and major achievements like chart success and Grammy Awards}. \textbf{MODPO}, while informative, diverges slightly \textbf{by adding speculative interpretations of the lyrics and the song’s broader impact on Latin music,} which goes beyond the initial request to simply talk about the song.}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of CPO and {\ourtitle} with same prompt.}
\label{tab:case_cpo_amopo_1}
\end{table}
\begin{table}[h]
\centering
\resizebox{0.8\textwidth}{!}{
\begin{tabular}{>{\centering\arraybackslash}m{2cm}>{\centering\arraybackslash}m{7cm}}
\toprule
\textbf{\scriptsize prompt} & \centering\arraybackslash\scriptsize{\hl{Find the directions between the given locations.
From: Rowan Avenue Elementary School, 600 S Rowan Ave, Los Angeles, CA 90023, United States
To: Calvary Cemetery and Mortuary, 4201 Whittier Blvd, Los Angeles, CA 90023, United States}}\\
\midrule
\textbf{\scriptsize {\ourtitle}} & \raggedright\arraybackslash\scriptsize{\hl{To find directions from Rowan Avenue Elementary School (600 S Rowan Ave, Los Angeles, CA 90023) to Calvary Cemetery and Mortuary (4201 Whittier Blvd, Los Angeles, CA 90023), you can follow these general steps:
1. Head west on S Rowan Ave toward E 62nd St for approximately 0.1 miles.
2. Turn right onto S Eastern Ave and continue for about 1.4 miles.
3. Take a left turn to merge onto Whittier Blvd.
4. Continue on Whittier Blvd for approximately 1.1 miles.
5. Calvary Cemetery and Mortuary will be on your right.
For the most accurate and up-to-date directions, it is recommended to use a GPS device or a mapping service like Google Maps, Apple Maps, or Waze, as road conditions and construction can change.}} \\
\midrule
\textbf{\scriptsize MODPO} & \raggedright\arraybackslash\scriptsize{\hl{To travel from Rowan Avenue Elementary School to Calvary Cemetery and Mortuary, you can follow these directions:
1. Start at Rowan Avenue Elementary School, located at 600 S Rowan Ave, Los Angeles, CA 90023.
2. Head west on S Rowan Ave toward E 41st St.
3. Turn left onto E 41st St.
4. Continue straight onto E 41st St, which will turn into S Gramercy Pl.
5. Turn right onto E 43rd St.
6. Turn left onto S Gramercy Pl.
7. Turn right onto S San Pedro St.
8. Continue on S San Pedro St, which will turn into Whittier Blvd.
9. Calvary Cemetery and Mortuary will be on your right at 4201 Whittier Blvd, Los Angeles, CA 90023.
Alternatively, you can use a navigation app like Google Maps or Waze to get the most accurate and up-to-date directions.}} \\
\midrule
\textbf{\scriptsize Helpfulness Judgment} & \raggedright\arraybackslash\scriptsize{\hl{\textbf{{\ourtitle}} is more helpful as it provides a clear and direct set of directions with specific landmarks and turns between Rowan Avenue Elementary School and Calvary Cemetery and Mortuary. \textbf{It also includes a helpful note recommending the use of a GPS or mapping service for the most accurate and up-to-date information, which is practical advice.} On the other hand, \textbf{MODPO} provides a more complicated route \textbf{with unnecessary detours and multiple streets} that are not as straightforward as the path suggested in {\ourtitle}. The additional routes in MODPO make it less practical for someone seeking simple and clear directions. Therefore, {\ourtitle} is more aligned with the user's need for helpful, direct, and useful information.}} \\
\midrule
\textbf{\scriptsize Correctness Judgment} & \raggedright\arraybackslash\scriptsize{\hl{\textbf{{\ourtitle}} \textbf{provides more accurate and concise directions, with specific distances and clear turn-by-turn instructions from Rowan Avenue Elementary School to Calvary Cemetery and Mortuary.} The directions follow a straightforward route, making it easier to navigate. It avoids unnecessary detours and accurately reflects the location of both destinations. \textbf{MODPO}, while containing some correct directions, \textbf{introduces unnecessary complexity and detours that make it less efficient}. Therefore, {\ourtitle} is the most correct, relevant, and practical response.}} \\
\midrule
\textbf{\scriptsize Instruct Follow Judgment} & \raggedright\arraybackslash\scriptsize{\hl{\textbf{\ourtitle} \textbf{follows the instruction closely by providing clear, concise directions from the given locations.} It meets the task goals without unnecessary complexity, ensuring full alignment with the instruction following requirements.}} \\
\bottomrule
\end{tabular}
}
\caption{Responses of MODPO and {\ourtitle} with same prompt.}
\label{tab:case_modpo_amopo_1}
\end{table}
\begin{thebibliography}{59}
\providecommand{\natexlab}[1]{#1}
\bibitem[{lla(2024)}]{llama3}
2024.
\newblock \href {https://ai.meta.com/blog/meta-llama-3/} {Introducing meta llama 3: The most capable openly available llm to date.}
\bibitem[{Azar et~al.(2024)Azar, Guo, Piot, Munos, Rowland, Valko, and Calandriello}]{ipo2024Azar}
Mohammad~Gheshlaghi Azar, Zhaohan~Daniel Guo, Bilal Piot, R{\'{e}}mi Munos, Mark Rowland, Michal Valko, and Daniele Calandriello. 2024.
\newblock \href {https://proceedings.mlr.press/v238/gheshlaghi-azar24a.html} {A general theoretical paradigm to understand learning from human preferences}.
\newblock In \emph{International Conference on Artificial Intelligence and Statistics, 2-4 May 2024, Palau de Congressos, Valencia, Spain}, volume 238 of \emph{Proceedings of Machine Learning Research}, pages 4447--4455. {PMLR}.
\bibitem[{Badrinath et~al.(2024)Badrinath, Agarwal, and Xu}]{HPO2024Badrinath}
Anirudhan Badrinath, Prabhat Agarwal, and Jiajing Xu. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2405.17956} {Hybrid preference optimization: Augmenting direct preference optimization with auxiliary objectives}.
\newblock \emph{CoRR}, abs/2405.17956.
\bibitem[{Bahl et~al.(1983)Bahl, Jelinek, and Mercer}]{Maximum1983Bahl}
Lalit~R. Bahl, Frederick Jelinek, and Robert~L. Mercer. 1983.
\newblock \href {https://doi.org/10.1109/TPAMI.1983.4767370} {A maximum likelihood approach to continuous speech recognition}.
\newblock \emph{{IEEE} Trans. Pattern Anal. Mach. Intell.}, 5(2):179--190.
\bibitem[{Bradley and Terry(1952)}]{bradley1952rank}
Ralph~Allan Bradley and Milton~E Terry. 1952.
\newblock Rank analysis of incomplete block designs: I. the method of paired comparisons.
\newblock \emph{Biometrika}, 39(3/4):324--345.
\bibitem[{Cui et~al.(2024)Cui, Yuan, Ding, Yao, He, Zhu, Ni, Xie, Xie, Lin, Liu, and Sun}]{ultrafeedback2024cui}
Ganqu Cui, Lifan Yuan, Ning Ding, Guanming Yao, Bingxiang He, Wei Zhu, Yuan Ni, Guotong Xie, Ruobing Xie, Yankai Lin, Zhiyuan Liu, and Maosong Sun. 2024.
\newblock \href {https://openreview.net/forum?id=BOorDpKHiJ} {{ULTRAFEEDBACK:} boosting language models with scaled {AI} feedback}.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net.
\bibitem[{Emmerich and Deutz(2018)}]{emmerich2018tutorial}
Michael T.~M. Emmerich and Andr{\'{e}}~H. Deutz. 2018.
\newblock \href {https://doi.org/10.1007/S11047-018-9685-Y} {A tutorial on multiobjective optimization: fundamentals and evolutionary methods}.
\newblock \emph{Nat. Comput.}, 17(3):585--609.
\bibitem[{Ethayarajh et~al.(2024{\natexlab{a}})Ethayarajh, Xu, Muennighoff, Jurafsky, and Kiela}]{ethayarajh2024kto}
Kawin Ethayarajh, Winnie Xu, Niklas Muennighoff, Dan Jurafsky, and Douwe Kiela. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2402.01306} {{KTO:} model alignment as prospect theoretic optimization}.
\newblock \emph{CoRR}, abs/2402.01306.
\bibitem[{Ethayarajh et~al.(2024{\natexlab{b}})Ethayarajh, Xu, Muennighoff, Jurafsky, and Kiela}]{KTO2024Ethayarajh}
Kawin Ethayarajh, Winnie Xu, Niklas Muennighoff, Dan Jurafsky, and Douwe Kiela. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2402.01306} {{KTO:} model alignment as prospect theoretic optimization}.
\newblock \emph{CoRR}, abs/2402.01306.
\bibitem[{Fu et~al.(2024)Fu, Hou, McAuley, and Yan}]{MCA2024fu}
Tingchen Fu, Yupeng Hou, Julian~J. McAuley, and Rui Yan. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2408.05094} {Unlocking decoding-time controllability: Gradient-free multi-objective alignment with contrastive prompts}.
\newblock \emph{CoRR}, abs/2408.05094.
\bibitem[{Guo et~al.(2024{\natexlab{a}})Guo, Cui, Yuan, Ding, Sun, Sun, Chen, Xie, Zhou, Lin, Liu, and Sun}]{ControlPO2024guo}
Yiju Guo, Ganqu Cui, Lifan Yuan, Ning Ding, Zexu Sun, Bowen Sun, Huimin Chen, Ruobing Xie, Jie Zhou, Yankai Lin, Zhiyuan Liu, and Maosong Sun. 2024{\natexlab{a}}.
\newblock \href {https://aclanthology.org/2024.emnlp-main.85} {Controllable preference optimization: Toward controllable multi-objective alignment}.
\newblock In \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, {EMNLP} 2024, Miami, FL, USA, November 12-16, 2024}, pages 1437--1454. Association for Computational Linguistics.
\bibitem[{Guo et~al.(2024{\natexlab{b}})Guo, Yin, Jiang, and Zhang}]{guo2024todo}
Yuxiang Guo, Lu~Yin, Bo~Jiang, and Jiaqi Zhang. 2024{\natexlab{b}}.
\newblock Todo: Enhancing llm alignment with ternary preferences.
\newblock \emph{arXiv preprint arXiv:2411.02442}.
\bibitem[{Hong et~al.(2024)Hong, Lee, and Thorne}]{ORPO2024Hong}
Jiwoo Hong, Noah Lee, and James Thorne. 2024.
\newblock \href {https://aclanthology.org/2024.emnlp-main.626} {{ORPO:} monolithic preference optimization without reference model}.
\newblock In \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, {EMNLP} 2024, Miami, FL, USA, November 12-16, 2024}, pages 11170--11189. Association for Computational Linguistics.
\bibitem[{Hui et~al.(2024)Hui, Yang, Cui, Yang, Liu, Zhang, Liu, Zhang, Yu, Dang, Yang, Men, Huang, Ren, Ren, Zhou, and Lin}]{qwen2.5code2024hui}
Binyuan Hui, Jian Yang, Zeyu Cui, Jiaxi Yang, Dayiheng Liu, Lei Zhang, Tianyu Liu, Jiajun Zhang, Bowen Yu, Kai Dang, An~Yang, Rui Men, Fei Huang, Xingzhang Ren, Xuancheng Ren, Jingren Zhou, and Junyang Lin. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2409.12186} {Qwen2.5-coder technical report}.
\newblock \emph{CoRR}, abs/2409.12186.
\bibitem[{Jang et~al.(2023)Jang, Kim, Lin, Wang, Hessel, Zettlemoyer, Hajishirzi, Choi, and Ammanabrolu}]{RLPHF_2023jang}
Joel Jang, Seungone Kim, Bill~Yuchen Lin, Yizhong Wang, Jack Hessel, Luke Zettlemoyer, Hannaneh Hajishirzi, Yejin Choi, and Prithviraj Ammanabrolu. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2310.11564} {Personalized soups: Personalized large language model alignment via post-hoc parameter merging}.
\newblock \emph{CoRR}, abs/2310.11564.
\bibitem[{Lee et~al.(2024)Lee, Park, Kim, and Seo}]{JANUS2024Lee}
Seongyun Lee, Sue~Hyun Park, Seungone Kim, and Minjoon Seo. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2405.17977} {Aligning to thousands of preferences via system message generalization}.
\newblock \emph{CoRR}, abs/2405.17977.
\bibitem[{Li et~al.(2024{\natexlab{a}})Li, Yao, Zhang, and Li}]{li2024safety}
Shen Li, Liuyi Yao, Lan Zhang, and Yaliang Li. 2024{\natexlab{a}}.
\newblock Safety layers in aligned large language models: The key to llm security.
\newblock \emph{arXiv preprint arXiv:2408.17003}.
\bibitem[{Li et~al.(2024{\natexlab{b}})Li, He, Huang, Bu, Liu, Guo, Wang, Gu, Su, and Zheng}]{2DDPO2024li}
Shilong Li, Yancheng He, Hui Huang, Xingyuan Bu, Jiaheng Liu, Hangyu Guo, Weixun Wang, Jihao Gu, Wenbo Su, and Bo~Zheng. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2410.19720} {2d-dpo: Scaling direct preference optimization with 2-dimensional supervision}.
\newblock \emph{CoRR}, abs/2410.19720.
\bibitem[{Li et~al.(2024{\natexlab{c}})Li, Chiang, Frick, Dunlap, Zhu, Gonzalez, and Stoica}]{arenahard2024}
Tianle Li, Wei-Lin Chiang, Evan Frick, Lisa Dunlap, Banghua Zhu, Joseph~E. Gonzalez, and Ion Stoica. 2024{\natexlab{c}}.
\newblock \href {https://lmsys.org/blog/2024-04-19-arena-hard/} {From live data to high-quality benchmarks: The arena-hard pipeline}.
\bibitem[{Li et~al.(2023)Li, Zhang, Dubois, Taori, Gulrajani, Guestrin, Liang, and Hashimoto}]{alpaca_eval}
Xuechen Li, Tianyi Zhang, Yann Dubois, Rohan Taori, Ishaan Gulrajani, Carlos Guestrin, Percy Liang, and Tatsunori~B. Hashimoto. 2023.
\newblock Alpacaeval: An automatic evaluator of instruction-following models.
\newblock \url{https://github.com/tatsu-lab/alpaca_eval}.
\bibitem[{Lin et~al.(2022)Lin, Hilton, and Evans}]{truthfulqa2022}
Stephanie Lin, Jacob Hilton, and Owain Evans. 2022.
\newblock \href {https://doi.org/10.18653/V1/2022.ACL-LONG.229} {Truthfulqa: Measuring how models mimic human falsehoods}.
\newblock In \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2022, Dublin, Ireland, May 22-27, 2022}, pages 3214--3252. Association for Computational Linguistics.
\bibitem[{Lou et~al.(2024)Lou, Zhang, Xie, Liu, Yan, and Huang}]{SPO2024Lou}
Xingzhou Lou, Junge Zhang, Jian Xie, Lifeng Liu, Dong Yan, and Kaiqi Huang. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2405.12739} {{SPO:} multi-dimensional preference sequential alignment with implicit reward modeling}.
\newblock \emph{CoRR}, abs/2405.12739.
\bibitem[{Meng et~al.(2024)Meng, Xia, and Chen}]{simpo2024meng}
Yu~Meng, Mengzhou Xia, and Danqi Chen. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2405.14734} {Simpo: Simple preference optimization with a reference-free reward}.
\newblock \emph{CoRR}, abs/2405.14734.
\bibitem[{Miettinen(1999)}]{miettinen1999nonlinear}
Kaisa Miettinen. 1999.
\newblock \emph{Nonlinear multiobjective optimization}, volume~12.
\newblock Springer Science \& Business Media.
\bibitem[{Mukherjee et~al.(2024)Mukherjee, Lalitha, Sengupta, Deshmukh, and Kveton}]{mukherjee2024multi}
Subhojyoti Mukherjee, Anusha Lalitha, Sailik Sengupta, Aniket Deshmukh, and Branislav Kveton. 2024.
\newblock Multi-objective alignment of large language models through hypervolume maximization.
\newblock \emph{arXiv preprint arXiv:2412.05469}.
\bibitem[{OpenAI(2023)}]{gpt42023openai}
OpenAI. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2303.08774} {{GPT-4} technical report}.
\newblock \emph{CoRR}, abs/2303.08774.
\bibitem[{Ouyang et~al.(2022)Ouyang, Wu, Jiang, Almeida, Wainwright, Mishkin, Zhang, Agarwal, Slama, Ray et~al.}]{ouyang2022training}
Long Ouyang, Jeffrey Wu, Xu~Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, et~al. 2022.
\newblock Training language models to follow instructions with human feedback.
\newblock \emph{Advances in neural information processing systems}, 35:27730--27744.
\bibitem[{Park et~al.(2024)Park, Rafailov, Ermon, and Finn}]{rdpo2024Park}
Ryan Park, Rafael Rafailov, Stefano Ermon, and Chelsea Finn. 2024.
\newblock \href {https://doi.org/10.18653/V1/2024.FINDINGS-ACL.297} {Disentangling length from quality in direct preference optimization}.
\newblock In \emph{Findings of the Association for Computational Linguistics, {ACL} 2024, Bangkok, Thailand and virtual meeting, August 11-16, 2024}, pages 4998--5017. Association for Computational Linguistics.
\bibitem[{Peters and Schaal(2007)}]{rl2007jan}
Jan Peters and Stefan Schaal. 2007.
\newblock \href {https://doi.org/10.1145/1273496.1273590} {Reinforcement learning by reward-weighted regression for operational space control}.
\newblock In \emph{Machine Learning, Proceedings of the Twenty-Fourth International Conference {(ICML} 2007), Corvallis, Oregon, USA, June 20-24, 2007}, volume 227 of \emph{{ACM} International Conference Proceeding Series}, pages 745--750. {ACM}.
\bibitem[{Qwen(2024)}]{qwen2.5}
Team Qwen. 2024.
\newblock \href {https://qwenlm.github.io/blog/qwen2.5/} {Qwen2.5: A party of foundation models}.
\bibitem[{Radford(2018)}]{radford2018improving}
Alec Radford. 2018.
\newblock Improving language understanding by generative pre-training.
\bibitem[{Rafailov et~al.(2023)Rafailov, Sharma, Mitchell, Manning, Ermon, and Finn}]{DPO2023Rafailov}
Rafael Rafailov, Archit Sharma, Eric Mitchell, Christopher~D. Manning, Stefano Ermon, and Chelsea Finn. 2023.
\newblock \href {http://papers.nips.cc/paper\_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html} {Direct preference optimization: Your language model is secretly a reward model}.
\newblock In \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}.
\bibitem[{Ram{\'{e}} et~al.(2023{\natexlab{a}})Ram{\'{e}}, Couairon, Dancette, Gaya, Shukor, Soulier, and Cord}]{rame2024rewarded}
Alexandre Ram{\'{e}}, Guillaume Couairon, Corentin Dancette, Jean{-}Baptiste Gaya, Mustafa Shukor, Laure Soulier, and Matthieu Cord. 2023{\natexlab{a}}.
\newblock \href {http://papers.nips.cc/paper\_files/paper/2023/hash/e12a3b98b67e8395f639fde4c2b03168-Abstract-Conference.html} {Rewarded soups: towards pareto-optimal alignment by interpolating weights fine-tuned on diverse rewards}.
\newblock In \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}.
\bibitem[{Ram{\'{e}} et~al.(2023{\natexlab{b}})Ram{\'{e}}, Couairon, Dancette, Gaya, Shukor, Soulier, and Cord}]{rewardsoup2023Rame}
Alexandre Ram{\'{e}}, Guillaume Couairon, Corentin Dancette, Jean{-}Baptiste Gaya, Mustafa Shukor, Laure Soulier, and Matthieu Cord. 2023{\natexlab{b}}.
\newblock \href {http://papers.nips.cc/paper\_files/paper/2023/hash/e12a3b98b67e8395f639fde4c2b03168-Abstract-Conference.html} {Rewarded soups: towards pareto-optimal alignment by interpolating weights fine-tuned on diverse rewards}.
\newblock In \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}.
\bibitem[{Reizinger et~al.(2024)Reizinger, Ujv{\'{a}}ry, M{\'{e}}sz{\'{a}}ros, Kerekes, Brendel, and Husz{\'{a}}r}]{position2024Reizinger}
Patrik Reizinger, Szilvia Ujv{\'{a}}ry, Anna M{\'{e}}sz{\'{a}}ros, Anna Kerekes, Wieland Brendel, and Ferenc Husz{\'{a}}r. 2024.
\newblock \href {https://openreview.net/forum?id=pVyOchWUBa} {Position: Understanding llms requires more than statistical generalization}.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net.
\bibitem[{Shannon(1948)}]{shannon1948mathematical}
Claude~Elwood Shannon. 1948.
\newblock A mathematical theory of communication.
\newblock \emph{The Bell system technical journal}, 27(3):379--423.
\bibitem[{Sinha et~al.(2024)Sinha, Cui, Das, and Zhang}]{safest2024sinha}
Ankita Sinha, Wendi Cui, Kamalika Das, and Jiaxin Zhang. 2024.
\newblock \href {https://aclanthology.org/2024.emnlp-industry.76} {Survival of the safest: Towards secure prompt optimization through interleaved multi-objective evolution}.
\newblock In \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: {EMNLP} 2024 - Industry Track, Miami, Florida, USA, November 12-16, 2024}, pages 1016--1027. Association for Computational Linguistics.
\bibitem[{Sun et~al.(2024)Sun, Shen, and Ton}]{sun2024rethinking}
Hao Sun, Yunyi Shen, and Jean-Francois Ton. 2024.
\newblock Rethinking bradley-terry models in preference-based reward modeling: Foundations, theory, and alternatives.
\newblock \emph{arXiv preprint arXiv:2411.04991}.
\bibitem[{Touvron et~al.(2023{\natexlab{a}})Touvron, Lavril, Izacard, Martinet, Lachaux, Lacroix, Rozi{\`e}re, Goyal, Hambro, Azhar et~al.}]{touvron2023llama}
Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth{\'e}e Lacroix, Baptiste Rozi{\`e}re, Naman Goyal, Eric Hambro, Faisal Azhar, et~al. 2023{\natexlab{a}}.
\newblock Llama: Open and efficient foundation language models.
\newblock \emph{arXiv preprint arXiv:2302.13971}.
\bibitem[{Touvron et~al.(2023{\natexlab{b}})Touvron, Martin, Stone, Albert, Almahairi, Babaei, Bashlykov, Batra, Bhargava, Bhosale et~al.}]{touvron2023llama2}
Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, et~al. 2023{\natexlab{b}}.
\newblock Llama 2: Open foundation and fine-tuned chat models.
\newblock \emph{arXiv preprint arXiv:2307.09288}.
\bibitem[{Wang et~al.(2024{\natexlab{a}})Wang, Lin, Xiong, Yang, Diao, Qiu, Zhao, and Zhang}]{DPA2024wang}
Haoxiang Wang, Yong Lin, Wei Xiong, Rui Yang, Shizhe Diao, Shuang Qiu, Han Zhao, and Tong Zhang. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.18653/V1/2024.ACL-LONG.468} {Arithmetic control of llms for diverse user preferences: Directional preference alignment with multi-objective rewards}.
\newblock In \emph{Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2024, Bangkok, Thailand, August 11-16, 2024}, pages 8642--8655. Association for Computational Linguistics.
\bibitem[{Wang et~al.(2024{\natexlab{b}})Wang, Lin, Xiong, Yang, Diao, Qiu, Zhao, and Zhang}]{wang2024arithmetic}
Haoxiang Wang, Yong Lin, Wei Xiong, Rui Yang, Shizhe Diao, Shuang Qiu, Han Zhao, and Tong Zhang. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.18653/V1/2024.ACL-LONG.468} {Arithmetic control of llms for diverse user preferences: Directional preference alignment with multi-objective rewards}.
\newblock In \emph{Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2024, Bangkok, Thailand, August 11-16, 2024}, pages 8642--8655. Association for Computational Linguistics.
\bibitem[{Wang et~al.(2024{\natexlab{c}})Wang, Xiong, Xie, Zhao, and Zhang}]{ArmoRM}
Haoxiang Wang, Wei Xiong, Tengyang Xie, Han Zhao, and Tong Zhang. 2024{\natexlab{c}}.
\newblock \href {https://aclanthology.org/2024.findings-emnlp.620} {Interpretable preferences via multi-objective reward modeling and mixture-of-experts}.
\newblock In \emph{Findings of the Association for Computational Linguistics: {EMNLP} 2024, Miami, Florida, USA, November 12-16, 2024}, pages 10582--10592. Association for Computational Linguistics.
\bibitem[{Wang et~al.(2024{\natexlab{d}})Wang, Dong, Delalleau, Zeng, Shen, Egert, Zhang, Sreedhar, and Kuchaiev}]{helpsteer22024wang}
Zhilin Wang, Yi~Dong, Olivier Delalleau, Jiaqi Zeng, Gerald Shen, Daniel Egert, Jimmy~J. Zhang, Makesh~Narsimhan Sreedhar, and Oleksii Kuchaiev. 2024{\natexlab{d}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2406.08673} {Helpsteer2: Open-source dataset for training top-performing reward models}.
\newblock \emph{CoRR}, abs/2406.08673.
\bibitem[{Wang et~al.(2024{\natexlab{e}})Wang, Dong, Zeng, Adams, Sreedhar, Egert, Delalleau, Scowcroft, Kant, Swope, and Kuchaiev}]{helpsteer2024wang}
Zhilin Wang, Yi~Dong, Jiaqi Zeng, Virginia Adams, Makesh~Narsimhan Sreedhar, Daniel Egert, Olivier Delalleau, Jane~Polak Scowcroft, Neel Kant, Aidan Swope, and Oleksii Kuchaiev. 2024{\natexlab{e}}.
\newblock \href {https://doi.org/10.18653/V1/2024.NAACL-LONG.185} {Helpsteer: Multi-attribute helpfulness dataset for steerlm}.
\newblock In \emph{Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers), {NAACL} 2024, Mexico City, Mexico, June 16-21, 2024}, pages 3371--3384. Association for Computational Linguistics.
\bibitem[{Wen et~al.(2024{\natexlab{a}})Wen, Ke, Gu, Wu, Huang, Zhou, Li, Hu, Gao, Xu, Liu, Tang, Wang, and Huang}]{complexbench2024wen}
Bosi Wen, Pei Ke, Xiaotao Gu, Lindong Wu, Hao Huang, Jinfeng Zhou, Wenchuang Li, Binxin Hu, Wendy Gao, Jiaxin Xu, Yiming Liu, Jie Tang, Hongning Wang, and Minlie Huang. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2407.03978} {Benchmarking complex instruction-following with multiple constraints composition}.
\newblock \emph{CoRR}, abs/2407.03978.
\bibitem[{Wen et~al.(2024{\natexlab{b}})Wen, Zhong, Khan, Perez, Steinhardt, Huang, Bowman, He, and Feng}]{wen2024language}
Jiaxin Wen, Ruiqi Zhong, Akbir Khan, Ethan Perez, Jacob Steinhardt, Minlie Huang, Samuel~R. Bowman, He~He, and Shi Feng. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2409.12822} {Language models learn to mislead humans via {RLHF}}.
\newblock \emph{CoRR}, abs/2409.12822.
\bibitem[{Wu et~al.(2024)Wu, Feng, Zhang, Che, Wen, and Tao}]{wu2024beyond}
Jinyang Wu, Mingkuan Feng, Shuai Zhang, Feihu Che, Zengqi Wen, and Jianhua Tao. 2024.
\newblock Beyond examples: High-level automated reasoning paradigm in in-context learning via mcts.
\newblock \emph{arXiv preprint arXiv:2411.18478}.
\bibitem[{Xiao et~al.(2024)Xiao, Yuan, Zhu, Li, and Honavar}]{caldpo2024xiao}
Teng Xiao, Yige Yuan, Huaisheng Zhu, Mingxiao Li, and Vasant~G Honavar. 2024.
\newblock Cal-dpo: Calibrated direct preference optimization for language model alignment.
\newblock In \emph{The Thirty-eighth Annual Conference on Neural Information Processing Systems}.
\bibitem[{Xu et~al.(2025)Xu, Li, Zhang, Lin, Zhu, Zheng, Wu, Zhao, Xu, and Chen}]{aaai2025xuderong}
Derong Xu, Xinhang Li, Ziheng Zhang, Zhenxi Lin, Zhihong Zhu, Zhi Zheng, Xian Wu, Xiangyu Zhao, Tong Xu, and Enhong Chen. 2025.
\newblock \href {https://doi.org/10.1609/AAAI.V39I24.34747} {Harnessing large language models for knowledge graph question answering via adaptive multi-aspect retrieval-augmentation}.
\newblock In \emph{AAAI-25, Sponsored by the Association for the Advancement of Artificial Intelligence, February 25 - March 4, 2025, Philadelphia, PA, {USA}}, pages 25570--25578. {AAAI} Press.
\bibitem[{Xu et~al.(2024)Xu, Sharaf, Chen, Tan, Shen, Durme, Murray, and Kim}]{CPO2024xu}
Haoran Xu, Amr Sharaf, Yunmo Chen, Weiting Tan, Lingfeng Shen, Benjamin~Van Durme, Kenton Murray, and Young~Jin Kim. 2024.
\newblock \href {https://openreview.net/forum?id=51iwkioZpn} {Contrastive preference optimization: Pushing the boundaries of {LLM} performance in machine translation}.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net.
\bibitem[{Yang et~al.(2024{\natexlab{a}})Yang, Yang, Hui, Zheng, Yu, Zhou, Li, Li, Liu, Huang, Dong, Wei, Lin, Tang, Wang, Yang, Tu, Zhang, Ma, Xu, Zhou, Bai, He, Lin, Dang, Lu, Chen, Yang, Li, Xue, Ni, Zhang, Wang, Peng, Men, Gao, Lin, Wang, Bai, Tan, Zhu, Li, Liu, Ge, Deng, Zhou, Ren, Zhang, Wei, Ren, Fan, Yao, Zhang, Wan, Chu, Liu, Cui, Zhang, and Fan}]{qwen2}
An~Yang, Baosong Yang, Binyuan Hui, Bo~Zheng, Bowen Yu, Chang Zhou, Chengpeng Li, Chengyuan Li, Dayiheng Liu, Fei Huang, Guanting Dong, Haoran Wei, Huan Lin, Jialong Tang, Jialin Wang, Jian Yang, Jianhong Tu, Jianwei Zhang, Jianxin Ma, Jin Xu, Jingren Zhou, Jinze Bai, Jinzheng He, Junyang Lin, Kai Dang, Keming Lu, Keqin Chen, Kexin Yang, Mei Li, Mingfeng Xue, Na~Ni, Pei Zhang, Peng Wang, Ru~Peng, Rui Men, Ruize Gao, Runji Lin, Shijie Wang, Shuai Bai, Sinan Tan, Tianhang Zhu, Tianhao Li, Tianyu Liu, Wenbin Ge, Xiaodong Deng, Xiaohuan Zhou, Xingzhang Ren, Xinyu Zhang, Xipin Wei, Xuancheng Ren, Yang Fan, Yang Yao, Yichang Zhang, Yu~Wan, Yunfei Chu, Yuqiong Liu, Zeyu Cui, Zhenru Zhang, and Zhihao Fan. 2024{\natexlab{a}}.
\newblock Qwen2 technical report.
\newblock \emph{arXiv preprint arXiv:2407.10671}.
\bibitem[{Yang et~al.(2024{\natexlab{b}})Yang, Pan, Luo, Qiu, Zhong, Yu, and Chen}]{RiC2024yang}
Rui Yang, Xiaoman Pan, Feng Luo, Shuang Qiu, Han Zhong, Dong Yu, and Jianshu Chen. 2024{\natexlab{b}}.
\newblock \href {https://openreview.net/forum?id=QLcBzRI3V3} {Rewards-in-context: Multi-objective alignment of foundation models with dynamic preference adjustment}.
\newblock In \emph{Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}. OpenReview.net.
\bibitem[{Yuan et~al.(2023)Yuan, Yuan, Tan, Wang, Huang, and Huang}]{yuan2023rrhf}
Zheng Yuan, Hongyi Yuan, Chuanqi Tan, Wei Wang, Songfang Huang, and Fei Huang. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2304.05302} {{RRHF:} rank responses to align language models with human feedback without tears}.
\newblock \emph{CoRR}, abs/2304.05302.
\bibitem[{Zheng et~al.(2023)Zheng, Chiang, Sheng, Zhuang, Wu, Zhuang, Lin, Li, Li, Xing, Zhang, Gonzalez, and Stoica}]{MT-bench2024zheng}
Lianmin Zheng, Wei{-}Lin Chiang, Ying Sheng, Siyuan Zhuang, Zhanghao Wu, Yonghao Zhuang, Zi~Lin, Zhuohan Li, Dacheng Li, Eric~P. Xing, Hao Zhang, Joseph~E. Gonzalez, and Ion Stoica. 2023.
\newblock \href {http://papers.nips.cc/paper\_files/paper/2023/hash/91f18a1287b398d378ef22505bf41832-Abstract-Datasets\_and\_Benchmarks.html} {Judging llm-as-a-judge with mt-bench and chatbot arena}.
\newblock In \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}.
\bibitem[{Zhou et~al.(2023{\natexlab{a}})Zhou, Liu, Xu, Iyer, Sun, Mao, Ma, Efrat, Yu, Yu, Zhang, Ghosh, Lewis, Zettlemoyer, and Levy}]{LIMA2023zhou}
Chunting Zhou, Pengfei Liu, Puxin Xu, Srinivasan Iyer, Jiao Sun, Yuning Mao, Xuezhe Ma, Avia Efrat, Ping Yu, Lili Yu, Susan Zhang, Gargi Ghosh, Mike Lewis, Luke Zettlemoyer, and Omer Levy. 2023{\natexlab{a}}.
\newblock \href {http://papers.nips.cc/paper\_files/paper/2023/hash/ac662d74829e4407ce1d126477f4a03a-Abstract-Conference.html} {{LIMA:} less is more for alignment}.
\newblock In \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}.
\bibitem[{Zhou et~al.(2024{\natexlab{a}})Zhou, Wu, Wu, Feng, and Tan}]{zhou2024hm3}
Yu~Zhou, Xingyu Wu, Jibin Wu, Liang Feng, and Kay~Chen Tan. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2409.18893} {{HM3:} hierarchical multi-objective model merging for pretrained models}.
\newblock \emph{CoRR}, abs/2409.18893.
\bibitem[{Zhou et~al.(2024{\natexlab{b}})Zhou, Liu, Shao, Yue, Yang, Ouyang, and Qiao}]{MODPO2024zhou}
Zhanhui Zhou, Jie Liu, Jing Shao, Xiangyu Yue, Chao Yang, Wanli Ouyang, and Yu~Qiao. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.18653/V1/2024.FINDINGS-ACL.630} {Beyond one-preference-fits-all alignment: Multi-objective direct preference optimization}.
\newblock In \emph{Findings of the Association for Computational Linguistics, {ACL} 2024, Bangkok, Thailand and virtual meeting, August 11-16, 2024}, pages 10586--10613. Association for Computational Linguistics.
\bibitem[{Zhou et~al.(2023{\natexlab{b}})Zhou, Liu, Yang, Shao, Liu, Yue, Ouyang, and Qiao}]{zhou2023morlhf}
Zhanhui Zhou, Jie Liu, Chao Yang, Jing Shao, Yu~Liu, Xiangyu Yue, Wanli Ouyang, and Yu~Qiao. 2023{\natexlab{b}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2310.03708} {Beyond one-preference-for-all: Multi-objective direct preference optimization for language models}.
\newblock \emph{CoRR}, abs/2310.03708.
\end{thebibliography}