Federated Data-Efficient Instruction Tuning for Large Language Models
Extracted source text · arXiv 2410.10926
\documentclass[sigconf]{acmart}
% \documentclass[sigconf,screen,review,anonymous]{acmart}
\input{definitions/paper_commands}
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
Bib\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
% \setcopyright{acmlicensed}
% \copyrightyear{2018}
% \acmYear{2018}
% \acmDOI{XXXXXXX.XXXXXXX}
% %% These commands are for a PROCEEDINGS abstract or paper.
% \acmConference[Conference acronym 'XX]{Make sure to enter the correct
% conference title from your rights confirmation emai}{June 03--05,
% 2018}{Woodstock, NY}
% %%
% %% Uncomment \acmBooktitle if the title of the proceedings is different
% %% from ``Proceedings of ...''!
% %%
% %%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
% %% June 03--05, 2018, Woodstock, NY}
% \acmISBN{978-1-4503-XXXX-X/18/06}
% \copyrightyear{2024}
% \acmYear{2024}
% \setcopyright{acmlicensed}\acmConference[WWW '24]{Proceedings of the ACM
% Web Conference 2024}{May 13--17, 2024}{Singapore, Singapore}
% \acmBooktitle{Proceedings of the ACM Web Conference 2024 (WWW '24), May
% 13--17, 2024, Singapore, Singapore}
% \acmDOI{10.1145/3589334.3645425}
% \acmISBN{979-8-4007-0171-9/24/05}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{Federated Data-Efficient Instruction Tuning for Large Language Models}
\author{Zhen Qin}
\affiliation{%
\institution{College of Computer Science and Technology\\Zhejiang University}
\city{Hangzhou}
\country{China}}
\email{zhenqin@zju.edu.cn}
\author{Zhaomin Wu}
\affiliation{%
\institution{National University of Singapore}
\city{}
\country{Singapore}}
\email{zhaomin@nus.edu.sg}
\author{Bingsheng He}
\affiliation{%
\institution{National University of Singapore}
\city{}
\country{Singapore}}
\email{hebs@comp.nus.edu.sg}
\author{Shuiguang Deng}
\affiliation{%
\institution{College of Computer Science and Technology\\Zhejiang University}
\city{Hangzhou}
\country{China}}
\email{dengsg@zju.edu.cn}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\input{sections/0-abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10010147.10010178.10010219</concept_id>
<concept_desc>Computing methodologies~Distributed artificial intelligence</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10010147.10010257.10010258</concept_id>
<concept_desc>Computing methodologies~Learning paradigms</concept_desc>
<concept_significance>500</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Computing methodologies~Distributed artificial intelligence}
\ccsdesc[500]{Computing methodologies~Learning paradigms}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{Federated Learning, Large Language Models, Data-Efficient Fine-Tuning, Data Selection, Federated Data Processing.}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
% \received{20 February 2007}
% \received[revised]{12 March 2009}
% \received[accepted]{5 June 2009}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\input{sections/1-introduction}
\input{sections/2-related}
\input{sections/3-preliminaries}
\input{sections/4-approach}
\input{sections/5-experiments}
\input{sections/6-conclusion}
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
\appendix
\input{sections/7.1-appendix-algo}
\input{sections/7.2-appendix-reproducibility}
\input{sections/7.3-appendix-additional-exp}
\end{document}
\endinput
%%
%% End of file `sample-sigconf.tex'.
% 用于url自动换行
\makeatletter
\def\UrlAlphabet{%
\do\a\do\b\do\c\do\d\do\e\do\f\do\g\do\h\do\i\do\j%
\do\k\do\l\do\m\do\n\do\o\do\p\do\q\do\r\do\s\do\t%
\do\u\do\v\do\w\do\x\do\y\do\z\do\A\do\B\do\C\do\D%
\do\E\do\F\do\G\do\H\do\I\do\J\do\K\do\L\do\M\do\N%
\do\O\do\P\do\Q\do\R\do\S\do\T\do\U\do\V\do\W\do\X%
\do\Y\do\Z}
\def\UrlDigits{\do\1\do\2\do\3\do\4\do\5\do\6\do\7\do\8\do\9\do\0}
\g@addto@macro{\UrlBreaks}{\UrlOrds}
\g@addto@macro{\UrlBreaks}{\UrlAlphabet}
\g@addto@macro{\UrlBreaks}{\UrlDigits}
% For theorems and such
% \usepackage{amsmath}
% \usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
% \usepackage{algorithm,algorithmic}
\usepackage[linesnumbered,ruled,noend]{algorithm2e}
\usepackage{tabularx}
\usepackage{multirow}
\usepackage{rotating}
\usepackage{pifont}
\usepackage{bbding} % serial number
\usepackage{enumitem}
\usepackage{enumerate}
\usepackage{xspace}
\usepackage{makecell}
\usepackage{colortbl}
\usepackage{microtype}
\usepackage{graphicx}
% \usepackage{booktabs} % for professional tables
\usepackage{amsfonts}
\usepackage{subfigure}
% \usepackage{color}
\usepackage{tikz}
\usepackage{pgfkeys}
\usepackage{soul}
\definecolor{lightgray}{gray}{0.88}
\sethlcolor{lightgray}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% THEOREMS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% % \theoremstyle{plain}
% \newtheorem{theorem}{Theorem}[section]
% \newtheorem{proposition}[theorem]{Proposition}
% \newtheorem{lemma}[theorem]{Lemma}
% \newtheorem{corollary}[theorem]{Corollary}
% % \theoremstyle{definition}
% \newtheorem{definition}[theorem]{Definition}
% \newtheorem{assumption}[theorem]{Assumption}
% \theoremstyle{remark}
% \newtheorem{remark}[theorem]{Remark}
% \theoremstyle{mythm}
\newtheorem{theorem}{Theorem}
\newtheorem{assumption}{Assumption}
\newtheorem{proposition}{Proposition}
\newtheorem{challenge}{Challenge}
\newtheorem{principle}{Principle}
% \newtheorem{proof}{Proof}
\newtheorem{lemma}{Lemma}
\newtheorem{corollary}{Corollary}
\newtheorem{claim}{Claim}
\newtheorem{note}{Note}
\newtheorem{definition}{Definition}
\newcommand{\app}{FedHDS\xspace}
\newcommand{\apppro}{FedHDS-Turbo\xspace}
% \newcommand{\appfast}{FedHDS-Fast\xspace}
\newcommand{\datani}{Natural Instructions\xspace}
\newcommand{\datadolly}{Dolly-15K\xspace}
\newcommand{\modeldatajuicer}{DataJuicer-1.3B\xspace}
% \newcommand{\modeldatajuicer}{DJ-1.3B\xspace}
\newcommand{\modelllama}{LLaMA-3B\xspace}
\newcommand{\std}[1]{{\small \!$\pm$\! {#1}}}
\newcommand{\pdata}[1]{{\small ({#1}\%)}}
\newcommand{\texl}[1]{\underline{#1}}
\newcommand{\texb}[1]{\textbf{#1}}
\newcommand{\texbl}[1]{\underline{\textbf{#1}}}
% Todo notes is useful during development; simply uncomment the next line
% and comment out the line below the next line to turn off comments
%\usepackage[disable,textsize=tiny]{todonotes}
\usepackage[textsize=tiny]{todonotes}
\newcommand{\para}[1]{\textbf{#1}}
\begin{abstract}
Instruction tuning helps improve pretrained large language models (LLMs) in terms of the responsiveness to human instructions, which is benefited from diversified instruction data.
Federated learning extends the sources of instruction data by exploiting the diversified client-side data, making it increasingly popular for tuning LLMs.
Existing approaches of federated LLM tuning typically traverse all local data during local training, bringing excessive computation overhead and posing a risk of overfitting local data.
Thus, a federated data-efficient instruction tuning approach, which consumes relatively little data from the entire dataset, is needed.
In response, this work introduces an approach of federated data-efficient instruction tuning for LLMs, \app, which utilizes a representative subset of edge-side data, coreset, to tune the LLM.
It reduces the redundancy of data samples at both intra-client and inter-client levels through a \texbl{h}ierarchical \texbl{d}ata \texbl{s}election framework performed by jointly selecting a small number of representative data samples for local training without sharing the raw data.
Extensive experiments conducted across six scenarios with various LLMs, datasets and data partitions demonstrate that \app significantly reduces the amount of data required for fine-tuning while improving the responsiveness of the instruction-tuned LLMs to unseen tasks.
\end{abstract}
\section{Introduction}
\label{sec-intro}
Large language models (LLMs) exhibit remarkable performance across a wide range of natural language tasks.
Instruction tuning \cite{wei2022flan,supernaturalinstructions} is a key technique for enabling LLMs to effectively respond to human instructions, a capability essential for many web-based LLM service providers.
Diverse high-quality data is essential for effective instruction tuning \cite{wang2024survey,wang2023self,li2024selective,liu2024coachlm}, particularly when addressing unseen tasks \cite{wei2022flan}.
As publicly available high-quality data is expected to be depleted in the near future \cite{villalobos2022will}, leveraging data directly from its sources - such as edge devices - will be critical for the continued improvement of LLMs.
However, privacy concerns and regulations such as GDPR \cite{voigt2017eu} make utilizing edge data challenging.
Federated learning (FL) \cite{mcmahan2017fl}, a privacy-preserving paradigm that leverages diverse edge-side data, is gaining popularity for tuning LLMs \cite{zhang2024fedit}, especially for generalization to unseen tasks \cite{qin2024full,bai2024federated}.
Prior works on federated instruction tuning mainly optimize communication and memory costs by parameter-efficient fine-tuning (PEFT) technologies \cite{zhang2024fedit,babakniya2023slora,zhang2023-FedPETuning,che2023FedPepTAO,kuang2024federatedscope} and zeroth-order optimization \cite{xu2024forward,qin2024full}, and have shown commendable performance in their targeted scenarios.
Nevertheless, based on our investigation, a crucial factor highly related to the \emph{computation overhead} of LLM tuning has yet to be thoroughly investigated in FL contexts, i.e., \emph{data efficiency}.
Existing approaches consume all local data \cite{qin2024full,xu2024forward,zhang2024fedit,ling2024convergence}, leading to a higher number of iterations, which suffers two issues.
(1) Efficiency: As shown in Figure \ref{pic-intro-mem}, tuning LLMs exceeds the memory capacity of popular edge GPUs.
CPU+GPU hybrid training reduces GPU memory cost but significantly increases per-step time, making the time of traversing the full dataset unbearably long (Figure \ref{pic-intro-time}).
(2) Generalization: In FL, most client-side data typically consists of a small number of task types, excessive tuning steps with redundant data may harm the generalization to unseen tasks due to overfitting.
The primary challenge to enable data-efficient instruction tuning in FL lies in how to minimize data consumption for tuning while achieving the highest possible tuning accuracy, under the constraint that edge-side data does not leave the device.
\begin{figure}[t]
\centering
\begin{minipage}[t]{0.483\linewidth}
\centering
\includegraphics[width=\linewidth]{pic/intro/memory_intro.pdf}
\vspace{-0.3cm}
\caption{GPU memory cost by backward propagation (BP) with LoRA\protect\hyperref[fn1]{$^1$} v.s. capacity of popular edge-side GPUs\protect\hyperref[fn2]{$^2$}.}
\vspace{-0.3cm}
\label{pic-intro-mem}
\end{minipage}
\hspace{0.1cm}
\begin{minipage}[t]{0.483\linewidth}
\centering
\includegraphics[width=\linewidth]{pic/intro/efficiency_intro.pdf}
\vspace{-0.3cm}
\caption{Per-step/epoch time of BP with LoRA on Natural Instructions dataset by training with CPU+GPU\protect\hyperref[fn3]{$^3$}.}
\vspace{-0.3cm}
\label{pic-intro-time}
\end{minipage}
\end{figure}
\footnotetext[1]{\label{fn1}~CPU + GPU is implemented by Deepspeed \cite{rasley2020deepspeed}. LLMs are encoded in 16-bit.}
\footnotetext[2]{\label{fn2}~The most popular desktop and laptop GPUs from Steam Hardware Survey (Sep 2024).}
\footnotetext[3]{\label{fn3}~Training with CPU+GPU is implemented based on DeepSpeed \cite{rasley2020deepspeed}.}
Some studies have explored data-efficient tuning in centralized scenarios by selecting a subset of representative data from the dataset, i.e., a coreset \cite{chen2023maybe,wu2023self,cao2024instruction}, or high-quality data \cite{zhou2023lima,li2024Superfiltering,lu2023instag,cui2023ultrafeedback} for tuning LLMs.
Among them, the coreset-based ones generally demonstrate greater advantages in reducing data volume.
However, there are two main limitations when applying these approaches to federated instruction tuning.
\textbf{L1: Suboptimal data representations for coreset selection}.
Existing approaches primarily rely on data representations extracted from the last Transformer layer \cite{chen2023maybe,wu2023self,cao2024instruction}, which may not always provide high-quality features to identify representative data samples.
\textbf{L2: Low compatibility with FL scenarios}.
Existing clustering-based approaches \cite{chen2023maybe,wu2023self,cao2024instruction} require access to all data samples at the same time for global clustering, and others rely on manual efforts \cite{zhou2023lima} or third-party LLMs \cite{li2024Superfiltering,li2024Superfiltering,lu2023instag,cui2023ultrafeedback}, both of which violates the nature of FL that the access of data is strictly confined to the client side.
Moreover, these approaches take a global perspective and thus do not support the different ratios of redundant data among clients well.
Given the lack of investigation on federated data-efficiency instruction tuning and the limitations of current centralized data-efficient tuning approaches when applied to FL, we propose a novel framework of \texbl{fed}erated \texbl{h}ierarchical \texbl{d}ata \texbl{s}election, \textbf{\app}.
To provide better data representations for coreset selection, we propose to fuse features with different abstraction levels from different Transformer layers by dimensionality reduction (L1).
This method is integrated into \app, which first individually clusters local data in each client to several groups, identifying \emph{intra-client} data redundancy, and then sends approximate centroids of these groups to the server for another clustering to identify \emph{inter-client} data redundancy.
\app takes into account the distribution of edge-side data to minimize the scale of coresets used for tuning, without directly accessing the local data (L2).
It significantly alleviates computation costs while improving generalization to unseen tasks.
The main contributions of this work are summarized below:
\begin{itemize}[leftmargin=1em]
\item We propose \app, a framework to speed up federated LLM tuning by reducing the number of data samples consumed for tuning.
It selects a coreset for each client for instruction tuning through a dual-layer clustering process, with the intra-client and inter-client data redundancy both taken into consideration.
To the best of our knowledge, this is the first study to investigate data-efficient instruction tuning for LLMs in an FL context.
\item We investigate the data features extracted from different Transformer layers relied upon coresets construction, and propose a simple yet effective method to fuse features of varying abstraction levels.
It works within \app to facilitate clustering-based coreset selection for federated instruction tuning.
\item To evaluate the effectiveness of \app, extensive experiments are conducted on two widely-used instruction datasets with different non-IID splits and LLMs, in a total of 6 scenarios.
Results show that \app relatively improves Rouge-L scores on unseen tasks over existing federated instruction tuning approaches with the full dataset by an average of 10.72\%, while consuming less than 1.5\% of that required by these approaches.
Besides, ablation studies exhibit the effectiveness of alleviating both intra-client and inter-client data redundancy by \app.
\end{itemize}
\section{Related Work}
\label{sec-related}
\subsection{Federated Fine-Tuning for LLMs}
Federated tuning for LLMs is gaining widespread attention as it enables the utilization of edge-side data without direct access.
Given the large number of parameters in LLMs, many studies have developed tuning methods based on PEFT techniques to reduce memory footprint and the communication cost of transmitting the optimized model parameters \cite{zhang2024fedit,babakniya2023slora,zhang2023-FedPETuning,che2023FedPepTAO}.
Among PEFT techniques, LoRA \cite{lora} gains significant attention \cite{zhang2024fedit}, including initialization \cite{babakniya2023slora}, partial matrix freezing for consistency between federated and local objectives \cite{sun2024lora}, and heterogeneous adapters for heterogeneous client resources \cite{bai2024federated}.
Some works are dedicated to optimizing corresponding costs, e.g. low communication costs by transmitting only loss values and fixed random seeds based on zeroth-order optimization \cite{qin2024full}, and low memory cost by training with quantization \cite{xu2024forward}.
During local training, existing works typically traverse all local data \cite{qin2024full,xu2024forward,zhang2024fedit,wu2024fedbiot,kuang2024federatedscope,sun2024lora}.
On one hand, this leads to a significant computational overhead.
On the other hand, due to the non-IID characteristics of FL, client-side data is often composed of a large number of samples from a few categories/domains.
Repeated training on this data can result in overfitting to the local data, leading to reduced generalization ability on unseen tasks.
Different from existing approaches, this work takes a data-centric perspective \cite{zha2023data}, which puts special efforts into \emph{data efficiency} by finding a small number of representative data samples for LLM tuning, aiming at better efficiency and generalization on unseen tasks.
\subsection{Data-Efficient LLM Instruction Tuning}
Existing research shows that LLMs learn knowledge primarily from pretraining, and can be taught to produce better outputs with a limited amount of instruction data \cite{zhou2023lima}.
Some works focus on using fewer data samples to achieve tuning results similar to or even better than those obtained with the full dataset \cite{survey-wang2024survey-selection}, from the perspective of data quality \cite{zhou2023lima,lu2023instag,cui2023ultrafeedback,chen2024data,li2024from,li2024Superfiltering} or representativeness \cite{chen2023maybe,wu2023self}.
Quality-oriented works either filter out low-quality samples from the training set by heuristics \cite{chen2024data}, quality indicators \cite{li2024from,chen2024data} and the assistance of third-party LLMs \cite{li2024Superfiltering}, or curating a small set of high-quality samples by manual efforts \cite{zhou2023lima} and third-party LLMs \cite{lu2023instag,cui2023ultrafeedback}.
Methods based on heuristics or quality indicators \cite{chen2024data,li2024from} may only filter out a limited number of data samples, and methods based on manual efforts \cite{zhou2023lima} or third-party LLMs \cite{li2024Superfiltering,lu2023instag,cui2023ultrafeedback} are unsuitable for cross-device FL due to potential privacy leakage.
Representativeness-based works select a few representative data with clustering algorithms for tuning, to achieve the effect of learning the knowledge from the entire dataset \cite{chen2023maybe,wu2023self}.
These methods can filter out a substantial amount of data, retaining less than 5\% of the full dataset, enhancing the efficiency of LLM tuning significantly.
However, applying coreset selection methods to federated instruction tuning still has shortcomings.
On the one hand, existing methods usually extract data features for clustering by simply using the features from the last layer of the Transformer \cite{chen2023maybe,wu2023self,li2024from}, which may lead to a suboptimal feature space to distinguish coresets.
On the other hand, the nature of FL where data does not leave the device prevents existing methods from being deeply integrated into FL, such as detecting data redundancy across clients.
\begin{figure*}[t]
\centering
\includegraphics[width=0.92\linewidth]{pic/framework}
\caption{
Overview of \app in each round of FL, where serial numbers indicate the step sequence.
\app differs from vanilla FL \cite{mcmahan2017fl} by incorporating ``I. Data Selection'' to select representative data samples before ``II. Local Training''.
Centroid $\mathbf{c}$ is derived from HDBSCAN \cite{campello2013density} and does not correspond to any specific data sample.
Data sample $\mathbf{x}_{\rightarrow \mathbf{c}_j}$ indicates the one closest to the centroid $\mathbf{c}_j$ of corresponding group $\mathcal{G}_j$.
Subset $\widetilde{\mathcal{D}}_i$ contains representative samples in local dataset $\mathcal{D}_i$ of the $i$-th client.
}
\label{pic-framework}
\end{figure*}
Our approach, \app, is designed for federated instruction tuning based on coreset selection.
It is characterized by a two-layer data selection framework that progressively eliminates the intra-client and inter-client data redundancy, based on the fused features from all Transformer layers with different levels of abstraction.
\section{Problem Formulation}
\label{sec-pre}
Assuming there are $N$ clients in an FL system, indexed by $i$, each of which holds a private dataset $\mathcal{D}_i$ containing several instances of instruction data.
Given an LLM $\mathbf{w}\in \mathbb{R}^d$ initialized by the pretrained weight $\mathbf{w}^0$, federated instruction tuning aims at optimize $\mathbf{w}$ with the private data held by clients towards the following objective,
\begin{equation}
\min_{\mathbf{w}} f(\mathbf{w}) \triangleq \sum_{i=1}^N \lambda_i \cdot \mathbb{E}_{\mathbf{x} \sim \mathcal{D}_i}\left[ \mathcal{L}(\mathbf{w}; \mathbf{x})\right],
\label{eq-fl-optimization}
\end{equation}
where $\mathcal{L}(\mathbf{w}; \mathbf{x})$ is the loss function evaluated on model $\mathbf{w}$ for any data instance $\mathbf{x}$ sampled from $\mathcal{D}_i$, and $\lambda_i$ is the weight of client $i$ which follows $\lambda_i > 0$ and $\sum_i^N \lambda_i=1$.
Symbol $\mathbf{x}$ is adopted as the batch size is set to 1 for lower memory cost on devices \cite{qin2024full,malladi2023mezo}.
To solve the objective in Eq. \eqref{eq-fl-optimization}, FL iterates multiple rounds.
In each round $r$, several clients participating in FL download the latest model parameters $\mathbf{w}^r$ from the server and perform several steps of local optimization such as stochastic gradient descent (SGD),
\begin{equation}
\mathbf{w}^r_{i, t + 1}= \mathbf{w}^r_{i, t} - \eta \cdot \nabla_{\mathbf{w}^r_{i, t}} \mathcal{L}(\mathbf{w}^r_t; \mathbf{x}), \forall \mathbf{w} \in \mathcal{D}_i,
\label{eq-local-sgd}
\end{equation}
where $\mathbf{w}^r_{i, t}$ denotes model $\mathbf{w}$ of client $i$ at the $t$-th local step in round $r$, and $\eta$ is the learning rate.
Typically, the above process iterates over the local dataset $\mathcal{D}_i$ for one or more epochs \cite{qin2024full,xu2024forward,zhang2024fedit} in a cross-device FL scenario.
After local training is finished, each active client transfers the optimized model parameters to the server.
Given the large number of parameters in LLMs, FL typically incorporates PEFT techniques \cite{zhang2023-FedPETuning}, where only a small subset of model parameters is trained and transmitted to reduce memory and communication costs.
Similarly, this work employs LoRA \cite{lora} to train and transmit only LoRA adapters as \cite{zhang2024fedit}.
Given a function of data selection $f(\mathbf{x}\mid \cdot)$ that returns 0 or 1, a coreset of $\mathcal{D}_i$ can be constructed by $\widetilde{\mathcal{D}}_i = \left\{\mathbf{x} \mid f(\mathbf{x} \mid \cdot) = 1, \forall \mathbf{x} \in \mathcal{D}_i\right\}$.
If $f$ is effective in selecting a coreset, Eq. \eqref{eq-local-sgd} can be optimized with $\widetilde{\mathcal{D}}_i$ while $\mathbf{w}$ is still on par with or even surpass that obtained with $\mathcal{D}_i$.
In this way, the computation cost of local training to traverse all the local data can be greatly reduced.
Additionally, by considering the information redundancy among local coresets, we can further reduce the iterations for each client.
Assuming an ideal distribution $\mathcal{P}^*$ that each real-world data $\mathbf{x}$ follows \cite{qin2024synergy}, and data samples in each coreset $\widetilde{\mathcal{D}}_i$ follow $\mathcal{P}_i$, where local data distributions $\left\{\mathcal{P}_1, \ldots, \mathcal{P}_N \right\}$ share mutual similarity to some extent since all the client-side data can be regarded as sampled from $\mathcal{P}^*$, so that inter-client data redundancy may exist.
Experimental results in Section \ref{subsec-exp-ablation} also illustrate this issue.
To alleviate intra-client and inter-client data redundancy, we design \app, detailed in Section \ref{sec-approach}.
\section{Approach}
\label{sec-approach}
\subsection{Overview}
\label{subsec-app-overview}
The processes of \app in each round of FL are illustrated in Figure \ref{pic-framework}.
Recall the problem formalization in Section \ref{sec-pre}, the key difference between \app and vanilla FL lies in the addition of a data selection process prior to the local training.
\app identifies the representative data samples with their latent features.
In round $r$, after downloading the global model $\mathbf{w}^r$, each client first conducts a process of data selection before local training, which contains two groups of sequential steps, i.e., intra-client selection and inter-client selection.
In intra-client selection, client $i$ takes each data sample $\mathbf{x} \in \mathcal{D}_i$ as input and gets the token-wise hidden states of $\mathbf{x}$ in each Transformer layer, where the last hidden state of each layer is concatenated into a one-dimensional vector, serving as the raw features for $\mathbf{x}$ (\ding{192} in Figure \ref{pic-framework}).
Then, a dimensionality-reduction algorithm is performed to fuse the features from different transformer layers, taking the one-dimensional features of all data samples in $\mathcal{D}_i$ into consideration (\ding{193} in Figure \ref{pic-framework}).
Next, a clustering algorithm is performed on the fused low-dimensional features to partition these data into several groups $\left\{\mathcal{G}_1, \mathcal{G}_2, \ldots\right\}$ (\ding{194} in Figure \ref{pic-framework}), where we adopt a density-based clustering, HDBSCAN \cite{campello2013density}, since it is hard to manually determine the number of clusters such as adopting K-means \cite{lloyd1982least}.
Each group $\mathcal{G}$ provides an approximate centroid $\mathbf{c}$ that does not correspond to an individual data point.
After that, the client sends these centroids $\left\{\mathbf{c}_1, \mathbf{c}_2, \ldots, \right\}$ to the server (\ding{195} in Figure \ref{pic-framework}).
Upon receiving the centroids from all active clients, the server initiates the inter-client clustering.
It applies HDBSCAN to all received centroids, resulting in several groups, denoted by $\left\{\mathcal{G}^{\text{II}}_1, \mathcal{G}^{\text{II}}_2, \ldots \right\}$, each associated with a corresponding centroid $c^{\text{II}}_j$ (\ding{196} in Figure \ref{pic-framework}).
In each $\mathcal{G}^{\text{II}}_j$, the centroid closest to $c^{\text{II}}_j$ is designated as the chosen one.
Next, the server notifies each client regarding which of their submitted centroids are selected (\ding{197} in Figure \ref{pic-framework}).
Thereafter, each client $i$ adds the data sample closest to each of the selected centroid within the corresponding group to coreset $\widetilde{\mathcal{D}}_i$ (\ding{198} in Figure \ref{pic-framework}).
After $\widetilde{\mathcal{D}}_i$ is constructed, the clients perform local training on the coresets, as in \cite{zhang2024fedit,kuang2024federatedscope}.
The above data selection processes are summarized in Algorithm \ref{algo:data-selection} of Appendix \ref{sec-appendix-algo}.
In the following subsections, we discuss the detailed design of \app.
\subsection{Intra-Client Data Selection}
\label{subsec-app-intra}
The first step in selecting data is to extract its features.
Given an LLM with $l$ Transformer layers, it can extract the data features in a manner of layer by layer and token by token, as
\begin{equation}
\begin{bmatrix}
\mathbf{h}^{1, 1}_j & \mathbf{h}^{1, 2}_j & \ldots & \mathbf{h}^{1, -1}_j \\
\mathbf{h}^{2, 1}_j & \mathbf{h}^{2, 2}_j & \ldots & \mathbf{h}^{2, -1}_j \\
\ldots & \ldots & \ldots & \ldots\\
\mathbf{h}^{l, 1}_j & \mathbf{h}^{l, 2}_j & \ldots & \mathbf{h}^{l, -1}_j \\
\end{bmatrix},
\label{eq-transformer-feature-matrix}
\end{equation}
where $\mathbf{h}^{l, b}_j \in \mathbb{R}^v$ is the hidden states of the $b$-th token from the $l$-th Transformer layer for the $j$-th data sample, and $b=-1$ corresponds to the last token.
In terms of token-level feature, \app uses the hidden state of the last token as \cite{li2024from} for two considerations:
1) since the Transformer computes the feature of the current token by considering all preceding tokens, the hidden state of the last token already contains information about all previous tokens, while performing averaging incurs additional computational costs,
and
2) when an LLM receives input, the instructions are typically embedded within a system prompt template, resulting in minimal differences in the logits of the first few tokens among data samples.
\begin{figure}[t]
\centering
\subfigure[Calinski-Harabasz $\uparrow$]{
\includegraphics[width=0.31\linewidth]{pic/exp/ch_open_llama_3b.pdf}
\label{subfig-clustering-ch}
}
\hspace{-0.1cm}
\subfigure[Silhouette Coefficient $\uparrow$]{
\includegraphics[width=0.31\linewidth]{pic/exp/sc_open_llama_3b.pdf}
\label{subfig-clustering-sc}
}
\hspace{-0.1cm}
\subfigure[F1-Score $\uparrow$]{
\includegraphics[width=0.31\linewidth]{pic/exp/f1_open_llama_3b.pdf}
\label{subfig-clustering-f1}
}
\caption{Evaluations of clustered data groups based on features from different Transformer layers, obtained in a centralized scenario with \modelllama on \datadolly.}
\label{pic-clustering-metrics-layer}
\end{figure}
As for the Transformer layer from which the features are derived, existing work typically adopts the final layer \cite{chen2023maybe,wu2023self,li2024from}.
Since different layers of the Transformer apply varying degrees of abstraction to the data, the final layer may not be a universally optimal choice.
We demonstrate this through a toy example on \datadolly dataset which contains 8 categories of instruction data, using K-means with the number of clusters aligned with that of the categories in \datadolly to partition these data.
Two internal metrics, i.e., Calinski-Harabasz Index \cite{calinski1974dendrite} and Silhouette Coefficient Index \cite{rousseeuw1987silhouettes} and one metric with category label, i.e., F1-Score are employed to evaluate the clustering results.
Figure \ref{pic-clustering-metrics-layer} shows that clustering metrics based on features from different layers yield varying results, suggesting that the final layer may not be the best for distinguishing these data, and no single layer achieves the best results across all metrics.
It is challenging to predict the optimal layer for feature extraction and calculating clustering metrics across all layers would result in significant computational costs, fusing features from all layers may serve as a feasible solution.
One naive method is to concatenate the hidden states of the last token in all Transformer layer, as
\begin{equation}
\mathbf{h}_j = \left [\mathbf{h}^{1, -1}_{j}, \mathbf{h}^{2, -1}_{j}, \ldots, \mathbf{h}^{l, -1}_{j} \right ].
\label{eq-feature-concatenate}
\end{equation}
However, as shown in Figure \ref{pic-clustering-metrics-layer}, the features from some layers may incur poor data separability, which may have a negative impact.
Generally, dimensionality reduction helps to alleviate the impact of inappropriate dimensions, e.g., PCA \cite{jolliffe2002principal} selects the most significant principal components based on the variance,
and t-SNE \cite{van2008visualizing} prioritizes preserving the relative distances between local data points during the dimensionality reduction process where the low-variance dimensions may have minimal impact on the distance calculation.
Therefore, we perform dimensionality reduction $P: \mathbb{R}^{l \times v} \to \mathbb{R}^k, k \ll l \times v$ on the concatenated features to fuse the features from different layers, as
\begin{equation}
\left\{\widetilde{\mathbf{h}}_1, \widetilde{\mathbf{h}}_2, \ldots, \widetilde{\mathbf{h}}_{\left|\mathcal{D}_i\right|} \right\} = P(\left\{\mathbf{h}_1, \mathbf{h}_2, \ldots, \mathbf{h}_{\left|\mathcal{D}_i\right|} \right\}).
\label{eq-dimension-reduction}
\end{equation}
We choose t-SNE \cite{van2008visualizing} with Barnes-hut implementation \cite{van2013barnes} for feature fusion in \app, as it is more effective in nonlinear feature spaces compared to earlier methods such as PCA.
The dimensionality $k$ of the fused features is set to 2 for better efficiency.
\begin{figure}[t]
\centering
\subfigure[\datadolly ($\alpha$=5.0) Last Layer]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly5.0-1B--1-4clients.pdf}
\label{subfig-vis-1B-dolly5.0-last}
}
\hspace{0.01cm}
\subfigure[\datadolly ($\alpha$=5.0) Feature Fusion]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly5.0-1B-tsne-4clients.pdf}
\label{subfig-vis-1B-dolly5.0-fusion}
}
\caption{Data visualization with features from the last layer and fused features obtained by \modeldatajuicer on \datadolly.}
\label{pic-visualization-feature}
\end{figure}
Figure \ref{pic-visualization-feature} provides empirical evidence for the effectiveness of feature fusion, where a randomly sampled subset of \datadolly is partitioned to 4 clients with label distribution skew.
If the hidden state of the last token from the last layer is employed, data in these clients tends to be scattered across the entire space, while the features fused from all the Transformer layers form relatively clear boundaries between clients.
Therefore, the fused feature can be used to better distinguish data samples.
When the fused features $\left\{\widetilde{\mathbf{h}}_1, \widetilde{\mathbf{h}}_2, \ldots, \widetilde{\mathbf{h}}_{\left|\mathcal{D}_i\right|} \right\}$ of local data in $\mathcal{D}_i$ are obtained, HDBSCAN \cite{campello2013density} is performed by taking them as inputs to form several groups $\left\{\mathcal{G}_1, \mathcal{G}_2, \ldots \right\}$, where an approximate representative point that does not corresponds to a real data point is provided for each group $\mathcal{G}_j$, denoted as the centroid $\mathbf{c}_j$.
These centroids are then transmitted to the server to determine whether the corresponding groups are selected for instruction tuning.
\subsection{Inter-Client Data Selection}
\label{subsec-app-inter}
As shown in Figure \ref{pic-visualization-feature}, although the data distribution across clients is non-IID, there is still some similarity in data among clients.
Thus, further selection among clients is performed so that the final data for tuning contains as little redundant information as possible.
As described in Section \ref{subsec-app-intra}, client-side data samples are partitioned into several groups, each centered around an approximate centroid.
Treating these centroids as representatives of their corresponding groups, we can cluster the centroids to filter out redundant data groups.
Thus, we applied HDBSCAN a second time, as
\begin{equation}
\mathbb{G}^{\text{II}} = \left\{\mathcal{G}^{\text{II}}_1, \mathcal{G}^{\text{II}}_2, \ldots\right\} = \text{HDBSCAN}(\left\{\mathbf{c}_1, \mathbf{c}_2, \ldots \right\})
\label{eq-layer-2-clustering}
\end{equation}
Similar to the intra-client HDBSCAN process in Section \ref{subsec-app-intra}, each $\mathcal{G}^{\text{II}}_j$ is centered around an approximate centroid $\mathbf{c}^{\text{II}}_j$.
For each group $\mathcal{G}^{\text{II}}_j$ in the second-layer selection, the server identifies the first-layer group $\mathcal{G}_s$ with the centroid $\mathbf{c}_s$ closest to $\mathbf{c}^{\text{II}}_j$ as the selected one, as
\begin{equation}
\mathbb{G}^{\text{selected}} = \left\{\mathcal{G}_s \mid \mathbf{c}_s = \underset{\mathbf{c} \in \mathcal{G}^{\text{II}}_j}{\operatorname{arg \ min}} \left \| \mathbf{c} - \mathbf{c}^{\text{II}}_j \right \| \right\}^{\mathcal{G}^{\text{II}}_j \in \mathbb{G}^{\text{II}}}.
\label{eq-layer-2-selection}
\end{equation}
Then, the selection of first-layer groups will be notified to corresponding clients.
Given the partitioned data groups $\mathbb{G}_i = \left\{\mathcal{G}_1, \mathcal{G}_2, \ldots\right\}$ of client $i$, the subset finally used for tuning $\widetilde{\mathcal{D}}_i$ is obtained as:
\begin{equation}
\widetilde{\mathcal{D}}_i = \left\{\mathbf{x} \mid q = \underset{\mathbf{x} \in \mathcal{G}_j}{\operatorname{arg \ min}} \left\| \mathbf{x} - \mathbf{c}_j\right\| \right\}^{\mathcal{G}_j \in \mathbb{G}_i, \mathcal{G}_j \in \mathbb{G}^{\text{selected}}},
\label{eq-tuning-set-construction}
\end{equation}
where in each selected group $\mathbf{G}_j$, the data sample closest to centroid $\mathbf{c}_j$ is selected in $\widetilde{\mathcal{D}}_i$, denoted by $\mathbf{x}_{\rightarrow \mathbf{c}_j}$ in Figure \ref{pic-framework}.
\subsection{Data-Efficient Federated Fine-Tuning}
\label{subsec-app-tuning}
After the coresets in clients are determined, each client $i$ performs local training only with the data in coreset $\widetilde{\mathcal{}}_i$, as
\begin{equation}
\mathbf{w}^r_{i, t + 1}= \mathbf{w}^r_{i, t} - \eta \cdot \nabla_{\mathbf{w}^r_{i, t}} \mathcal{L}(\mathbf{w}^r_t; \mathbf{x}), \forall \mathbf{x} \in \widetilde{\mathcal{D}}_i,
\label{eq-local-sgd-subset}
\end{equation}
followed by submitting the optimized $\mathbf{w}^r$ after the last local steps $\mathbf{w}^{r}_{i, -1}$ to the server.
Then, the server performs federated averaging \cite{mcmahan2017fl} on the updated parameters received from clients, as
\begin{equation}
\mathbf{w}^{r + 1} = \sum_{i \in \mathbb{M}_r} \left| \widetilde{\mathcal{D}}_i \right|\mathbf{w}^{r}_{i, -1},
\label{eq-aggregation}
\end{equation}
where $\mathbb{M}_r$ contains the indices of clients which participate in the $r$-th round of FL.
Finally, the server starts the next round.
\subsection{Enhancement for \app}
\label{subsec-app-enhancement}
\subsubsection{Enhancement on Efficiency}
\label{subsubsec-app-enhancement-efficiency}
Although \app reduces the number of data samples significantly by retaining only one sample from each data group, it still requires forward propagation across all the local data with the LLM $\mathbf{w}$ needs to be tuned, which consumes a certain amount of time when applied to edge devices.
Motivated by studies on retrieval augmented generation (RAG) which can adopt a light-weight retrieval model to extract data features for a better efficiency \cite{fan2024survey}, we replace $\mathbf{w}$ by a language model with a smaller number of parameters, e.g., GPT-2 \cite{radford2019language}, to enhance \app's feature extraction efficiency.
We term this approach as \apppro.
\subsubsection{Enhancement on Privacy}
\label{subsubsec-app-enhancement-privacy}
The local centroids are transmitted to the server after intra-client selection, which are approximate representative points that do not correspond to real data samples, and the dimensionality of the centroids is significantly lower than that of the original data features.
Nevertheless, \app offers additional information compared to vanilla FL \cite{mcmahan2017fl}, such as insights into the spatial distribution of representative samples from the clients.
To further minimize the risk of malicious exploitation of additional information, such as membership inference attacks \cite{hu2022membership}, we adopt a straightforward differential privacy approach. Specifically, we scale each dimension of the centroids, which are submitted to the server, to the range $[-1, 1]$ using the \texttt{tanh} function.
\footnote{\texttt{Tanh} is used for normalization instead of min-max normalization to maintain data distinguishability despite potential outliers.}
Gaussian noise is then added to the scaled centroids before transmission.
The differential privacy of the centroids (Theorem \ref{thm:privacy}) can be derived directly from the Gaussian mechanism \cite{dwork2014algorithmic}.
Experimental results in Section \ref{subsec-exp-privacy} demonstrate that FedHDS continues to outperform the baseline with a reasonable noise scale applied.
\begin{theorem}\label{thm:privacy}
Let $\mathbf{c} \in [-1,1]$ be the original centroid and $\mathbf{c}' = \mathbf{c} + \mathbf{z}$ be the noised centroid, where $\mathbf{z} \sim \mathcal{N}(0, \sigma^2 \mathbf{I})$ is Gaussian noise with standard deviation $\sigma$. Suppose $\varepsilon, \delta \in (0,1)$. The noisy centroid $\mathbf{c}'$ satisfies $(\varepsilon, \delta)$-differential privacy if $\sigma \geq \frac{2\sqrt{2\log(1.25/\delta)}}{\varepsilon}$.
\end{theorem}
\begin{proof}
The proof follows directly by applying the Gaussian mechanism~\cite{dwork2014algorithmic}, where the sensitivity (i.e., range of $\mathbf{c}$) is 2.
\end{proof}
\section{Evaluations}
\label{sec-exp}
\subsection{Experimental Setup}
\label{subsec-exp-setup}
\subsubsection{Baselines}
First, we introduce some federated instruction tuning approaches with full local data as baselines, including:
\begin{itemize}[leftmargin=1em]
\item FedAvg (FP) \cite{mcmahan2017fl} that tunes and transmits full LLM parameters.
It is introduced just for reference due to its high costs;
\item FedPTuning \cite{kuang2024federatedscope} that adopts with P-Tuning \cite{liu2023gpt} as the PEFT technique trained with Adam \cite{adam} optimizer;
\item FedPrompt \cite{kuang2024federatedscope} that replaces the PEFT technique in FedPTuning with Prompt Tuning \cite{lester2021power}.
\item FedIT \cite{zhang2024fedit}: a federated instruction tuning approach designed based on vanilla FL with LoRA \cite{lora}, optimized with Adam;
\item FedIT-SGD: a variation of FedIT that replaces Adam with SGD.
\end{itemize}
Given the current lack of data-efficient methods for federated LLM tuning, we develop two federated baselines that tune the LLM on a subset of local data, and introduce a centralized training approach based on coreset \cite{chen2023maybe}, including:
\begin{itemize}[leftmargin=1em]
\item Coreset-Cent \cite{chen2023maybe}: A centralized approach that extracts features with the last Transformer layer and selects a fixed ratio of central data samples from groups obtained by K-means.
\item Random: It randomly selects a certain proportion of local data to perform fine-tuning, with FedIT as the framework. Although being native, it is a strong baseline \cite{lin2024data,sachdeva2024howto}, since it preserves the distributions of the original dataset.
\item Perplexity: It selects a subset of data with the lowest perplexity scores \cite{chen2024data}, as these data samples are easy to learn by the LLM.
\end{itemize}
\subsubsection{Datasets and Evaluations}
We evaluate these approaches on two instruction datasets, \datani \cite{supernaturalinstructions} (NI) and \datadolly \cite{DatabricksBlog2023DollyV2}, which are popular in many federated instruction tuning studies \cite{qin2024full,bai2024federated,kuang2024federatedscope}.
Following \cite{qin2024full}, we employ evaluation on held-out tasks with Rouge-L as the evaluation metrics as in \cite{supernaturalinstructions,qin2024full}.
After preprocessing (detailed in Appendix \ref{subsec-appendix-hyperparameters}), NI contains 738 training tasks, each of which is assigned to a unique client, providing non-IIDness with feature skew, and the natively provided 119 test tasks are used for evaluation.
For \datadolly with 8 tasks, the last one is taken for evaluation while the rest are partitioned to 200 clients via Dirichlet distribution with $\alpha$ set to 0.5 and 5.0, respectively.
\subsubsection{Implementation}
\label{subsec-exp-implementation}
Considering this work is tailored for cross-device federated tuning, we randomly select 5\% of the clients to participate in FL in each round.
Approaches with coresets perform 60 rounds of FL on \datadolly with \modelllama, and 40 rounds for the rest scenarios.
Rouge-L scores of these approaches are evaluated with the tuned LLM after the last round of FL.
Each client performs local training on the coreset for one epoch with Adam \cite{adam} optimizer.
The hyperparameter settings of federated approaches with full data are aligned with those in \cite{qin2024full}.
For Random and Perplexity, we searched for the optimal data proportion within $\left\{0.2\%, 1.5\%, 2\%, 5\%\right\}$, where the thresholds of 0.2\% and 1.5\% are inspired by the proportion automatically obtained by \app and \apppro.
\app and \apppro adopt learning rate $\eta=3 \times 10^{-5}$ on \datadolly and $\eta=3 \times 10^{-4}$ on \datani.
Limited by space, the detailed implementation, preliminary hyperparameter search and final setups are left in Appendix \ref{subsec-appendix-hyperparameters}.
\subsection{Comparison on Accuracy}
\label{subsec-exp-main}
\input{tabs/exp_main}
\input{tabs/exp_efficiency}
We compare the performance of LLMs after instruction tuning across these approaches in Table \ref{tab-performance}.
Note that the evaluation results of FedAvg (FP), FedPTuning, FedPrompt, and FedIT in Table \ref{tab-performance} are derived from those reported in \cite{qin2024full} under the same experimental settings.
The results of other approaches are their best values obtained within the search space of hyperparameters.
From Table \ref{tab-performance}, \app and \apppro outperform federated instruction tuning approaches with full data across the six scenarios with consumed data samples less than 1.5\% of them.
Particularly, on \datani with \modeldatajuicer, \app and \apppro relatively improve the Rouge-L score over the best-performing baseline with full data, i.e., FedIT, by 19.5\% and 16.3\%, respectively.
Compared to the two federated approaches built with training on subsets, i.e., Random and Perplexity, \app and \apppro exhibit superior performance in most of the six scenarios, only on NI with \modeldatajuicer that Random slightly outperforms \apppro.
Note that in 7 out of the 12 scenarios related to the Random and Perplexity methods, the data proportions with the best Rouge-L results are inspired by the data proportion automatically acquired in \app and \apppro.
As demonstrated in \cite{cao2024instruction}, determining the optimal data proportion for the Random and Perplexity methods requires extensive experiments to be conducted manually, whereas the data proportion used in \app and \apppro is automatically determined based on the distribution of the client data.
These comparisons demonstrate the effectiveness of \app and \apppro in selecting coresets for federated data-efficient instruction tuning.
Compared to federated approaches using full data, those with coresets generally perform better on unseen tasks, especially on \datani, which has abundant instruction data.
This may result from cross-device settings, where clients' training data mainly consists of tasks from a few categories.
Tuning on complete local data causes excessive local training steps, leading the LLM to overfit local data and reducing performance on unseen tasks.
Thus, employing a data selection strategy for tuning LLMs with fewer samples in cross-device FL is promising, as it can lower computation costs while improving performance on unseen tasks.
Compared to the centralized baseline with data selection, Coreset-Cent, \app and \apppro still show a noticeable gap on the more complex dataset, \datani, indicating room for further improvement in federated tuning methods.
\subsection{Performance on Efficiency}
\label{subsec-exp-efficiency}
Table \ref{tab-speedup} presents the efficiency comparison among several selected baselines and our approaches.
It is evident that by reducing the number of local training steps (and thus the volume of consumed data samples), federated LLM tuning methods with coresets offer varying degrees of efficiency improvements compared to the best-performing federated tuning approach on the full dataset, i.e., FedIT.
When simulating BP conducted entirely on a GPU, Random typically achieves the highest speedup ratio and the shortest training time, as it does not require forward propagation to extract features.
Considering the limited GPU memory on edge devices, when hybrid training using both CPU and GPU is employed during BP to reduce peak memory usage, the forward propagation (FP) in \app and \apppro is significantly faster than the hybrid BP.
This results in a markedly lower time proportion for feature extraction.
Consequently, \apppro achieves a higher speedup ratio due to the fewer data samples required for better Rouge-L scores compared to federated tuning baselines based on coresets.
Particularly, on \datani with \modeldatajuicer, \apppro achieves a speedup ratio of 48.8$\times$ compared to FedIT.
Since \apppro is often comparable to \app in terms of Rouge-L scores as in Table \ref{tab-performance}, \apppro demonstrates greater practical value in real-world applications due to its higher efficiency.
\subsection{Ablation Studies}
\label{subsec-exp-ablation}
\subsubsection{Ablation on Two-Layer Selection}
\input{tabs/exp_ablation_dim_reduction}
As described in Section \ref{sec-approach}, \app is characterized by intra-client and inter-client selections.
To better clarify the effectiveness and contribution of both selections, we provide two approaches for ablation studies:
\begin{itemize}[leftmargin=1em]
\item FedDB: It extracts features based on the last Transformer layer, and group data samples with HDBSCAN. Then, in each group, one data sample closest to the corresponding centroid is selected.
\item \app (Intra): It implements only intra-client selection (Section \ref{subsec-app-intra}), which groups data samples with HDBSCAN based on the fused features from all layers, and selects one data sample closest to the centroid in each of the groups.
\end{itemize}
As shown in Table \ref{tab-ablation}, in some cases, the improvement of \app (Intra) over FedDB is greater than that of \app over \app (Intra), such as \datani with \modelllama.
Also in some cases, the opposite is true, such as \datadolly with \modelllama.
These indicate that the intra-client and inter-client data selection exhibit effectiveness across different scenarios.
The combination of these two mechanisms, \app, consistently outperforms FedDB in all scenarios, demonstrating the efficacy of their integration.
\subsubsection{Ablation on Methods for Feature Fusion}
During intra-client selection, the features are primarily integrated using the dimensionality reduction algorithm t-SNE.
Given the existence of other dimensionality reduction techniques, we also explored replacing t-SNE with alternative methods including PCA \cite{jolliffe2002principal} and Kernel PCA \cite{scholkopf1997kernel} to further investigate the impact of dimensionality reduction algorithms on our approach.
As presented in Table \ref{tab-ablation}, in some cases, FedHDS with PCA achieved the second-best performance.
On the relatively simpler datasets (\datadolly), replacing the dimensionality reduction algorithm did not lead to significant differences.
However, on \datani, substituting t-SNE with PCA-based dimensionality-reduction algorithms occasionally resulted in negative effects.
Therefore, in practical applications, we recommend implementing \app and \apppro using the more advanced dimensionality reduction algorithm, t-SNE, for more stable and effective coreset selection.
\subsection{Performance with Differential Privacy}
\label{subsec-exp-privacy}
\begin{figure}[t]
\centering
\begin{minipage}[t]{0.483\linewidth}
\centering
\includegraphics[width=\linewidth]{pic/exp/dp_1B_dolly.pdf}
\caption{Effects of adding DP noise to \apppro with \modeldatajuicer.}
\label{pic-exp-dp}
\end{minipage}
\hspace{0.1cm}
\begin{minipage}[t]{0.483\linewidth}
\centering
\includegraphics[width=\linewidth]{pic/exp/active_ratio_3B_instruct.pdf}
\caption{Rouge-L with different active ratio on \datani with \modelllama.}
\label{pic-exp-active-ratio}
\end{minipage}
\end{figure}
Despite
1) the feature dimensions used in our method are significantly lower compared to the original token-level hidden states, which often have thousands of dimensions, and
2) the shared centroids do not correspond to actual data points,
\app still provides additional information compared to the standard FL paradigm \cite{mcmahan2017fl,zhang2024fedit}, such as the spatial distribution of client-side centroids.
As Section \ref{subsubsec-app-enhancement-privacy}, scaling the elements of the shared centroids to the range [-1,1] and adding Gaussian noise can enhance the privacy protection of the proposed approach.
To clarify the impact of adding noise, we present example experiments by \apppro on \datadolly with \modeldatajuicer, with adjusting the power of the noise by varying its variance as \cite{shayan2020biscotti}.
As shown in Figure \ref{pic-exp-dp}, when the variance of the sampling noise is no greater than 0.1, \apppro still outperforms Random, and when the noise power continually increases, the performance of \apppro gradually degrades to a level comparable to that of Random as reported in Table \ref{tab-performance}.
\subsection{Performance in Various Federated Scenarios}
\label{subsec-exp-performance-scenrios}
The ratio of active clients in each round of FL affects the number of centroids sent to the server, potentially affecting the effectiveness of inter-client selection.
We test approaches with coresets in varying proportions of active clients in each round.
As shown in Figure \ref{pic-exp-active-ratio}, \apppro consistently outperforms Random and Perplexity across varying active client ratios.
When the active client ratio is low (1\%), all approaches perform unsatisfactorily, where the effectiveness of intra-client selection in \apppro may suffer from insufficient centroids.
With an increasing active client ratio, Random demonstrates a robust growth trend, although it still lags behind \apppro.
This improvement is likely due to when there are more active clients in each round, the randomly sampled data instances follow a distribution more aligned to the global data distribution.
Considering that clients typically participate in FL with a relatively low active ratio in each round in cross-device scenarios \cite{mcmahan2017fl,qin2024full,xu2024forward}, \apppro is more suitable for cross-device settings in terms of both accuracy and efficiency.
\begin{table*}[t]
% \renewcommand\arraystretch{0.85}
\caption{Rouge-L (\%) comparisons.
Numbers in parentheses indicate the ratio of consumed data samples compared to those used in full dataset approaches.
Each cell presents the average Rouge-L in the last round of four runs with different random seeds.
\hl{Coreset-Cent} and \hl{FedAvg (FP)} are introduced just as references since they are not suitable for cross-device FL.
Bold and underlined numbers are the best and second-best values among approaches suitable for cross-device FL, respectively.
}
\label{tab-performance}
\setlength\tabcolsep{3.3pt}
\centering
\begin{tabularx}{0.996\linewidth}{l|cc|cc|cc}
\toprule[1.0pt]
\multirow{2}{*}{Approach} & \multicolumn{2}{c|}{\textbf{\datani} (Meta Non-IID)} & \multicolumn{2}{c|}{\textbf{\datadolly ($\alpha=0.5$)}} & \multicolumn{2}{c}{\textbf{\datadolly ($\alpha=5.0$)}} \\
\cmidrule{2-7}
& \modeldatajuicer & \modelllama & \modeldatajuicer & \modelllama & \modeldatajuicer & \modelllama \\
\midrule[1.0pt]
\rowcolor{gray!20}Coreset-Cent & 31.36 \std{0.80} \pdata{1.00} & 34.81 \std{0.90} \pdata{0.01} & 33.27 \std{0.33} \pdata{0.50} & 35.48 \std{1.08} \pdata{1.00} & 33.27 \std{0.33} \pdata{0.50} & 35.48 \std{1.08} \pdata{1.00} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\rowcolor{gray!20}FedAvg (FP) & 22.08 \std{1.52} \pdata{100} & 27.88 \std{0.75} \pdata{100} & 32.30 \std{1.23} \pdata{100} & 34.27 \std{0.45} \pdata{100} & 33.38 \std{1.43} \pdata{100} & 33.95 \std{0.79} \pdata{100} \\
\midrule[1.0pt]
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
FedPTuning & 19.61 \std{2.71} \pdata{100} & 25.41 \std{1.14} \pdata{100} & 23.98 \std{3.23} \pdata{100} & 30.30 \std{1.16} \pdata{100} & 25.33 \std{2.48} \pdata{100} & 29.08 \std{1.33} \pdata{100} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
FedPrompt & \ \ 6.04 \std{0.12} \pdata{100} & \ \ 8.95 \std{2.47} \pdata{100} & 32.73 \std{0.87} \pdata{100} & 24.50 \std{4.78} \pdata{100} & 32.51 \std{1.31} \pdata{100} & 23.94 \std{4.15} \pdata{100} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
FedIT-SGD & 19.40 \std{1.83} \pdata{100} & 28.14 \std{0.85} \pdata{100} & 27.23 \std{0.68} \pdata{100} & 29.28 \std{0.50} \pdata{100} & 27.28 \std{1.35} \pdata{100} & 29.19 \std{0.89} \pdata{100} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
FedIT & 22.30 \std{0.42} \pdata{100} & 28.13 \std{0.50} \pdata{100} & 30.80 \std{0.98} \pdata{100} & 33.23 \std{1.51} \pdata{100} & 30.97 \std{0.43} \pdata{100} & 33.68 \std{1.07} \pdata{100} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\midrule[0.3pt]
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
% ✅
Random & \texl{26.20 \std{1.71}} \pdata{0.20} & 31.23 \std{1.37} \pdata{2.00} & 32.59 \std{0.10} \pdata{1.50} & 33.82 \std{0.82} \pdata{1.50} & 32.24 \std{0.43} \pdata{1.50} & 34.29 \std{0.85} \pdata{5.00} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
% ✅
Perplexity & 24.45 \std{0.77} \pdata{5.00} & 30.49 \std{0.21} \pdata{5.00} & 32.73 \std{0.15} \pdata{1.50} & 33.71 \std{0.51} \pdata{1.50} & 32.24 \std{0.22} \pdata{1.50} & 33.88 \std{0.34} \pdata{5.00} \\
\midrule[1.0pt]
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Federated Hierarchical Data Selection
% ✅
\app & \texb{26.64 \std{0.79}} \pdata{0.20} & \texl{32.32 \std{0.92}} \pdata{0.15} & \textbf{33.38 \std{0.40}} \pdata{0.82} & \textbf{35.40 \std{0.78}} \pdata{1.18} & \textbf{33.70 \std{0.19}} \pdata{0.88} & \textbf{35.79 \std{0.43}} \pdata{1.34} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
% ✅
\apppro & 25.93 \std{0.75} \pdata{0.23} & \textbf{32.93 \std{0.64}} \pdata{0.22} & \texl{33.28 \std{0.44}} \pdata{1.31} & \texl{35.01 \std{0.65}} \pdata{1.26} & \texl{33.52 \std{0.20}} \pdata{1.19} & \texl{35.42 \std{0.29}} \pdata{1.28} \\
\bottomrule[1.0pt]
\end{tabularx}
\end{table*}
\newcommand{\gpu}{\footnotesize{BP with GPU}}
\newcommand{\hybirdbp}{\footnotesize{BP with CPU+GPU}}
\newcommand{\pbar}[1]{
\begin{tikzpicture}
\draw[fill=blue!50] (0,0) rectangle (#1*0.5,0.2);
\draw (0,0) rectangle (0.5,0.2);
\end{tikzpicture}
}
\begin{table*}[t]
% \renewcommand\arraystretch{0.85}
\caption{Comparisons on per-client time (CTime), total time across all clients in all rounds (total time) \& speeding up ratio.}
\label{tab-speedup}
\setlength\tabcolsep{1.8pt}
\centering
\begin{tabularx}{0.996\linewidth}{ll|r@{\hspace{0.2cm}}rr|r@{\hspace{0.2cm}}rr|r@{\hspace{0.2cm}}rr|r@{\hspace{0.2cm}}rr}
\toprule[1.0pt]
\multicolumn{2}{c|}{\multirow{3}{*}{Approach}} & \multicolumn{6}{c|}{\textbf{\datani}} & \multicolumn{6}{c}{\textbf{\datadolly}} \\
\cmidrule{3-14}
&& \multicolumn{3}{c|}{\modeldatajuicer} & \multicolumn{3}{c|}{\modelllama} & \multicolumn{3}{c|}{\modeldatajuicer} & \multicolumn{3}{c}{\modelllama} \\
\cmidrule{3-14}
& & \multicolumn{1}{c}{\small CTime} & \multicolumn{1}{c}{\small Total Time} & \multicolumn{1}{c|}{\small Speedup} & \multicolumn{1}{c}{\small CTime} & \multicolumn{1}{c}{\small Total Time} & \multicolumn{1}{c|}{\small Speedup} & \multicolumn{1}{c}{\small CTime} & \multicolumn{1}{c}{\small Total Time} & \multicolumn{1}{c|}{\small Speedup} & \multicolumn{1}{c}{\small CTime} & \multicolumn{1}{c}{\small Total Time} & \multicolumn{1}{c}{\small Speedup} \\
\midrule[1.0pt]
\multirow{5}{*}{\rotatebox{90}{\gpu}}
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&FedIT & 66.7S & 1D3H\pbar{1.0} & 1$\times$ & 75.3S & 1D6H\pbar{1.0} & 1$\times$& 7.6S & 1H16M\pbar{1.0} & 1$\times$ & 10.6S & 1H46M\pbar{1.0} & 1$\times$\\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&Random & 1.8S & 44M16S\pbar{0.027} & 36.6$\times$ & 3.3S & 1H20M\pbar{0.044} & 22.5$\times$& 0.9S & 5M55S\pbar{0.078} & 12.86$\times$ & 2.3S & 23M24S\pbar{0.221} & 4.53$\times$ \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&Perplexity & 24.5S & 10H4M\pbar{0.373} & 2.7$\times$ & 32.4S & 13H18M\pbar{0.443} & 2.26$\times$& 3.5S & 23M19S\pbar{0.307} & 3.26$\times$ & 5.9S & 58M46S\pbar{0.554} & 1.80$\times$ \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&\app & 25.2S & 10H20M\pbar{0.383} & 2.6$\times$ & 31.8S & 13H4M\pbar{0.436} & 2.30$\times$& 3.3S & 21M46S\pbar{0.286} & 3.49$\times$ & 5.0S & 50M23S\pbar{0.475} & 2.10$\times$ \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&\apppro & 7.1S & 2H54M\pbar{0.107} & 9.3$\times$ & 8.0S & 3H16M\pbar{0.109} & 9.18$\times$& 1.2S & 8M6S\pbar{0.107} & 9.38$\times$ & 2.2S & 22M15S\pbar{0.21} & 4.76$\times$\\
\midrule[1.0pt]
\multirow{5}{*}{\rotatebox{90}{\hybirdbp}}
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&FedIT & 489S & 8D9H\pbar{1.0} & 1$\times$ & 811S & 13D21H\pbar{1.0} & 1$\times$& 56.0S & 9H20M\pbar{1.0} & 1$\times$ & 114S & 19H6M\pbar{1.0} & 1$\times$\\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&Random & 13.2S & 5H24M\pbar{0.0269} & 37.2$\times$& 35.3S & 14H29M\pbar{0.043} & 23.0$\times$& 6.5S & 43M25S\pbar{0.078} & 12.89$\times$ & 25.2S & 4H12M\pbar{0.22} & 4.54$\times$\\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&Perplexity & 46.6S & 19H9M\pbar{0.0953} & 10.5$\times$& 85S & 1D11H\pbar{0.105} & 9.51$\times$& 8.9S & 59M27S\pbar{0.106} & 9.42$\times$ & 27.5S & 4H34M\pbar{0.239} & 4.18$\times$\\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&\app & 27.5S & 11H17M\pbar{0.0561} & 17.8$\times$& 40.7S & 16H42M\pbar{0.05} & 19.9$\times$& 5.2S & 34M52S\pbar{0.062} & 16.06$\times$& 14.6S & 2H25M\pbar{0.127} & 7.90$\times$\\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
&\apppro & 10.0S & 4H7M\pbar{0.0205} & 48.8$\times$& 20.1S & 8H15M\pbar{0.025} & 40.4$\times$& 4.5S & 29M42S\pbar{0.053} & 18.86$\times$& 17.2S & 2H52M\pbar{0.15} & 6.66$\times$\\
\bottomrule[1.0pt]
\end{tabularx}
\end{table*}
\begin{table*}[t]
% \renewcommand\arraystretch{0.85}
\caption{
Rouge-L (\%) comparisons for ablation studies.
Obtaining and organization of these values are aligned with Table \ref{tab-performance}.
}
\label{tab-ablation}
\setlength\tabcolsep{3.5pt}
\centering
\begin{tabularx}{0.996\linewidth}{l|cc|cc|cc}
\toprule[1.0pt]
\multirow{2}{*}{Approach} & \multicolumn{2}{c|}{\textbf{\datani} (Meta Non-IID)} & \multicolumn{2}{c|}{\textbf{\datadolly ($\alpha=0.5$)}} & \multicolumn{2}{c}{\textbf{\datadolly ($\alpha=5.0$)}} \\
\cmidrule{2-7}
& \modeldatajuicer & \modelllama & \modeldatajuicer & \modelllama & \modeldatajuicer & \modelllama \\
\midrule[1.0pt]
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ablation app1
FedDB & 25.45 \std{0.64} \pdata{1.62}& 28.77 \std{1.91} \pdata{3.86} & 31.82 \std{0.56} \pdata{3.71} & 33.66 \std{0.48} \pdata{4.25} & 32.12 \std{1.66} \pdata{3.64} & 33.81 \std{0.87} \pdata{4.32} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ablation app2
\app (Intra) & 24.48 \std{0.78} \pdata{0.83} & 32.27 \std{0.12} \pdata{1.51} & 32.52 \std{0.65} \pdata{2.93} & 34.11 \std{0.94} \pdata{3.97} & 32.79 \std{1.15} \pdata{2.99} & 33.98 \std{1.36} \pdata{4.33} \\
\midrule[0.5pt]
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\app & \texl{26.64 \std{0.79}} \pdata{0.20} & 32.32 \std{0.92} \pdata{0.15} & \textbf{33.38 \std{0.40}} \pdata{0.82} & \textbf{35.40 \std{0.78}} \pdata{1.18} & \textbf{33.70 \std{0.19}} \pdata{0.88} & \textbf{35.79 \std{0.43}} \pdata{1.34} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\ \ \ \ w/i PCA & \texb{27.84 \std{0.99}} \pdata{0.09} & \texl{32.62 \std{0.62}} \pdata{0.08} & 33.12 \std{0.10} \pdata{0.74} & 34.90 \std{0.52} \pdata{1.86} & 33.06 \std{0.23} \pdata{0.74} & 34.60 \std{1.08} \pdata{2.12} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\ \ \ \ w/i KPCA & 26.28 \std{3.93} \pdata{0.10}& 28.95 \std{0.92} \pdata{0.08} & 33.21 \std{0.77} \pdata{0.72} & 34.80 \std{0.95} \pdata{1.62} & 33.34 \std{0.36} \pdata{0.75} & 35.11 \std{0.27} \pdata{2.08} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\midrule[0.5pt]
\apppro & 25.93 \std{0.75} \pdata{0.23} & \textbf{32.93 \std{0.64}} \pdata{0.22} & \texl{33.28 \std{0.44}} \pdata{1.31} & 35.01 \std{0.65} \pdata{1.26} & \texl{33.52 \std{0.20}} \pdata{1.19} & \texl{35.42 \std{0.29}} \pdata{1.28} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\ \ \ \ w/i PCA & 22.93 \std{2.96} \pdata{0.21} & 29.99 \std{0.78} \pdata{0.21} & 33.03 \std{0.26} \pdata{1.40} & \texl{35.35 \std{0.65}} \pdata{1.87} & 33.15 \std{0.21} \pdata{0.66} & 35.21 \std{0.54} \pdata{1.72} \\
% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
\ \ \ \ w/i KPCA & 25.80 \std{2.78} \pdata{0.03} & 28.98 \std{2.33} \pdata{0.20} & 32.86 \std{1.08} \pdata{1.86} & 35.04 \std{0.73} \pdata{1.69} & 33.40 \std{0.41} \pdata{0.71} & 34.88 \std{1.25} \pdata{1.92} \\
\bottomrule[1.0pt]
\end{tabularx}
\end{table*}
\section{Conclusion}
\label{sec-conclusion}
Existing federated instruction tuning approaches for LLMs typically train LLMs with traversing all local data during local training, which brings significant computational costs and poses a risk of overfitting to local data that may reduce the generalization capability on unseen tasks.
Considering that the demand for instruction data in instruction tuning for LLMs is not necessarily better with more data, we pioneered an exploration into selecting a small subset of core data on each client for federated LLM tuning.
To fulfill this, we propose \app, a two-layer coreset selection framework for federated data-efficient instruction tuning, which progressively solves the intra-client and inter-client data redundancy based on two sequential clustering processes.
To provide high-quality data representations for distinguishing coresets, we propose to fuse data features of varying levels of abstraction obtained from different Transformer layers with a dimensionality reduction algorithm.
Extensive experiments conducted on two real-world instruction datasets in 6 scenarios with different non-IIDness and LLMs demonstrate that \app can outperform baselines of federated instruction tuning with full data and data selection in terms of accuracy, while only consuming less than 1.5\% of the full data, significantly speeding up the federated instruction tuning of LLMs.
This work may raise new attention on data efficiency in federated instruction tuning, while enhancing the practicality of federated instruction tuning on devices by reducing the computation cost.
\section{Detailed Algorithm}
\label{sec-appendix-algo}
To facilitate a better understanding of each step of the proposed approach, we provide Algorithm \ref{algo:data-selection} to explain how our method selects the coresets, where lines \ref{line:loop-intra}-\ref{line:submit-centroids} and \ref{line:loop-coreset}-\ref{line:ret} are performed individually by each client, and lines \ref{line:initialize-centroids} and \ref{line:call-inter}-\ref{line:send-selected} are performed by the server.
\input{algo/data_selection_processes}
\begin{algorithm}
\SetAlgoLined
\DontPrintSemicolon
\SetNoFillComment
\SetKwFunction{FClientTraining}{ClientTraining}
\SetKwFunction{FIntraSelection}{IntraClientSelection}
\SetKwFunction{FInterSelection}{InterClientSelection}
\SetKwProg{Fn}{Function}{:}{end}
\setlength{\abovedisplayskip}{3pt}
\setlength{\belowdisplayskip}{3pt}
\setlength{\abovedisplayshortskip}{3pt}
\setlength{\belowdisplayshortskip}{3pt}
\caption{Processes of Data Selection in \textbf{\app}.}
\label{algo:data-selection}
\KwIn{$N$, $\left\{\mathcal{D}_1, \dots, \mathcal{D}_N\right\}$.}
\KwOut{The selected coreset $\widetilde{\mathcal{D}}_i$ for each client $i$, denoted as $\left\{\widetilde{\mathcal{D}}_1, \dots, \widetilde{\mathcal{D}}_N\right\}$.}
\vspace{0.2cm}
% initialize a list $\mathbb{C}$
Initialize a list $\mathbb{C}$ to stage the received centroids \label{line:initialize-centroids}\\
\For{$i = 1, 2, \ldots, N$}{
\label{line:loop-intra}
$\left\{\mathbf{c}_1, \mathbf{c}_2, \ldots \right\}$ = \FIntraSelection{$\mathbf{w}$, $\mathcal{D}_i$} \label{line:call-intra}
\\
$\mathbb{C} = \mathbb{C} \cup \left\{\mathbf{c}_1, \mathbf{c}_2, \ldots \right\}$ \label{line:submit-centroids} \tcp*{\ding{195} in Figure \ref{pic-framework}}
}
$\mathbb{G}^{\text{selected}}$ = \FInterSelection{$\mathbb{C}$} \label{line:call-inter}\\
send indices of selected groups to corresponding clients \label{line:send-selected}\tcp*{\ding{197} in Figure \ref{pic-framework}}
\For{$i = 1, 2, \ldots, N$}{
\label{line:loop-coreset}
$\widetilde{\mathcal{D}}_i = \left\{\mathbf{x} \mid q = \underset{\mathbf{x} \in \mathcal{G}_j}{\operatorname{arg \ min}} \left\| \mathbf{x} - \mathbf{c}_j\right\| \right\}^{\mathcal{G}_j \in \mathbb{G}_i, \mathcal{G}_j \in \mathbb{G}^{\text{selected}}}$ \label{line:inloop-coreset} \tcp*{\ding{198} in Figure \ref{pic-framework}}
}
\KwRet{
\label{line:ret}
the selected subset $\widetilde{\mathcal{D}}_i$ for each client $i$.}
\vspace{0.3cm}
\Fn{\FIntraSelection{$\mathbf{w}$, $\mathcal{D}$}}{
\For{$j = 1, 2, \ldots, |\mathcal{D}|$}{
extract features of $\mathbf{x}_j$ by each layer of $\mathbf{w}$, denoted by $\mathbf{h}_j = \left [\mathbf{h}^{1, -1}_{j}, \mathbf{h}^{2, -1}_{j}, \ldots, \mathbf{h}^{l, -1}_{j} \right ]$ \tcp*{\ding{192} in Figure \ref{pic-framework}}
}
reduce dimensionality of $\mathbb{H} = \left\{\mathbf{h}_1, \mathbf{h}_2, \ldots, \mathbf{h}_{|\mathcal{D}|}\right\}$ as $\left\{\widetilde{\mathbf{h}}_1, \widetilde{\mathbf{h}}_2, \ldots, \widetilde{\mathbf{h}}_{\left|\mathcal{D}_i\right|} \right\} = P(\left\{\mathbf{h}_1, \mathbf{h}_2, \ldots, \mathbf{h}_{\left|\mathcal{D}_i\right|} \right\})$, obtaining the fused features \tcp*{\ding{193} in Figure \ref{pic-framework}}
cluster data in $\mathcal{D}_i$ based on their fused features, as $\left\{\mathcal{G}_1, \mathcal{G}_2, \ldots \right\} = \operatorname{HDBSCAN}(\left\{\widetilde{\mathbf{h}}_1, \widetilde{\mathbf{h}}_2, \ldots, \widetilde{\mathbf{h}}_{\left|\mathcal{D}_i\right|} \right\})$, where each $\mathcal{G}_j$ corresponds to an approximate centroid $\mathbf{c}_j$ \tcp*{\ding{194} in Figure \ref{pic-framework}}
\KwRet{$\left\{\mathbf{c}_1, \mathbf{c}_2, \ldots \right\}$}
}
\vspace{0.3cm}
\Fn{\FInterSelection{$\left\{\mathbf{c}_1, \mathbf{c}_2, \ldots \right\}$}}{
perform HDBSCAN to partition received $\left\{\mathbf{c}_1, \mathbf{c}_2, \ldots \right\}$ into several groups $\left\{\mathcal{G}^{\text{II}}_1, \mathcal{G}^{\text{II}}_2, \ldots\right\}$ \tcp*{\ding{196} in Figure \ref{pic-framework}}
initialize a list $\mathbb{G}^{\text{selected}}$ \\
\For{$j = 1, 2, \ldots, \left|\left\{\mathcal{G}^{\text{II}}_1, \mathcal{G}^{\text{II}}_2, \ldots\right\}\right|$}{
$s = \underset{\mathbf{c} \in \mathcal{G}^{\text{II}}_j}{\operatorname{arg \ min}} \left \| \mathbf{c} - \mathbf{c}^{\text{II}}_j \right \| $ \\
add $\mathcal{G}_s$ into $\mathbb{G}^{\text{selected}}$ \\
}
\KwRet{$\mathbb{G}^{\text{selected}}$}
}
\end{algorithm}
\section{Reproducibility}
\label{sec-appendix-reproducibility}
\subsection{Experimental Environments}
We implement these approaches mentioned above with PyTorch \cite{paszke2019pytorch} \texttt{2.0.1}, Transformers \cite{wolf-etal-2020-transformers} \texttt{4.31.0}, scikit-learn \texttt{1.5.1}, PEFT \cite{peft} \texttt{0.4.0}, and hdbscan \texttt{0.8.37}.
Numerical experiments in Tables \ref{tab-performance} and \ref{tab-ablation} are performed on platforms equipped with NVIDIA A100 or NVIDIA A6000 GPUs, installed with Python \texttt{3.10} and CUDA \texttt{12.4}.
Efficiency results in Table \ref{tab-speedup} is obtained on a platform with an NVIDIA A6000 GPU, installed with Python \texttt{3.10}, CUDA \texttt{12.4} and DeepSpeed \texttt{0.15.2}.
\subsection{Statistics in Figures \ref{pic-intro-mem} and \ref{pic-intro-time}}
\label{subsec-appendix-mem-time}
The memory footprint and time consumption in Figures \ref{pic-intro-mem} and \ref{pic-intro-time} are measured with a maximum token list length of 1,024 where excessively long data will be truncated.
The adopted platform is equipped with an NVIDIA A6000 GPU, installed with Python \texttt{3.10}, CUDA \texttt{12.4} and DeepSpeed \texttt{0.15.2}.
For memory footprint, the 95th percentile is calculated.
The selected two GPUs are based on the most popular desktop and laptop GPUs identified from the Steam Hardware Survey (Sep 2024).
\subsection{Detailed Experimental Setups}
\label{subsec-appendix-hyperparameters}
During intra-client selection in \app and \apppro, the minimum cluster of HDBSCAN is set to the default value, i.e., 5 for \datani and 2 for \datadolly considering the relatively small size of this dataset..
During inter-client selection in \app and \apppro, the minimum cluster of HDBSCAN is uniformly set to 2.
The \texttt{rank}, \texttt{alpha} and \texttt{dropout} of LoRA adapters are set to 8, 16 and 0.05 for \app and \apppro, respectively.
As described in Section \ref{subsec-exp-implementation}, we perform preliminary hyperparameter search for for federated approaches, and adopt the advantageous settings for each approach in each scenario for large-scale experiments.
For federated approaches with coresets, we first search the learning rate in $\left\{3\times 10^{-4}, 1\times 10^{-4}, 3\times 10^{-5}\right\}$.
Then, for Random and Perplexity, we search the ratio of data samples in final selected subsets to the full data samples in $\left\{0.2\%, 1.5\%, 2\%, 5\%\right\}$.
Note that the thresholds of 0.2\% and 1.5\% are inspired by the proportion automatically obtained by \app and \apppro.
Considering the importance of an appropriate data ratio \cite{cao2024instruction}, these two baselines have benefited to some extent from the data proportion provided by \app and \apppro.
The finally adopted values for Tables \ref{tab-performance} and \ref{tab-ablation} as as follows:
All the federated approaches with coresets adopt learning rate $\eta$ of $3\times 10^{-5}$ on \datadolly.
On \datani, \app and \apppro adopt $\eta=3\times 10^{-4}$, Random adopt $\eta=3\times 10^{-4}$ with \modeldatajuicer and $\eta=3\times 10^{-5}$ with \modelllama, Perplexity adopt $\eta=3\times 10^{-5}$ with \modeldatajuicer and $\eta=3\times 10^{-4}$ with \modelllama.
For \datani, we adopt the dataset versioned by \texttt{v2.8} and the \texttt{default} split.
For \datadolly, we partition data samples to clients via Dirichlet distribution based on the \texttt{category} attribute.
This work adopts the same data preprocessing as reported in \cite{qin2024full}.
Following \cite{zhang2024fedit,qin2024full}, we adopt the prompt template from Alpaca \cite{alpaca}.
After federated tuning is finished, the Rouge-L is evaluated with the tuned LLM after the last FL round.
\section{Additional Experiments}
In Section \ref{subsec-app-inter}, we visualize the features obtained from last Transformer layer and the fused features with \modeldatajuicer on \datadolly as an example, as presented in Figure \ref{pic-visualization-feature}.
In order to illustrate the differences between fused features and the final layer features in more scenarios, we supplemented the visualizations using different LLMs in various scenario.
As shown in Figure \ref{pic-appendix-visualization-feature-aop}, the fused features create more distinct boundaries among clients.
Thus, these fused features can be utilized to more effectively distinguish between data samples.
\begin{figure}[t]
\centering
\subfigure[\datadolly ($\alpha$=0.5) Last Layer with \modelllama]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly0.5-3B--1-4clients.pdf}
\label{subfig-vis-3B-dolly0.5-last}
}
\hspace{0.01cm}
\subfigure[\datadolly ($\alpha$=0.5) Feature Fusion with \modelllama]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly0.5-3B-tsne-4clients.pdf}
\label{subfig-vis-3B-dolly0.5-fusion}
}
\subfigure[\datadolly ($\alpha$=5.0) Last Layer with \modelllama]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly5.0-3B--1-4clients.pdf}
\label{subfig-vis-3B-dolly5.0-last}
}
\hspace{0.01cm}
\subfigure[\datadolly ($\alpha$=5.0) Feature Fusion with \modelllama]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly5.0-3B-tsne-4clients.pdf}
\label{subfig-vis-3B-dolly5.0-fusion}
}
\subfigure[\datadolly ($\alpha$=0.5) Last Layer with \modeldatajuicer]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly0.5-1B--1-4clients.pdf}
\label{subfig-vis-1B-dolly0.5-last}
}
\hspace{0.01cm}
\subfigure[\datadolly ($\alpha$=0.5) Feature Fusion with \modeldatajuicer]{
\includegraphics[width=0.42\linewidth]{pic/exp/Dolly0.5-1B-tsne-4clients.pdf}
\label{subfig-vis-1B-dolly0.5-fusion}
}
\caption{Data visualization based on the last-layer feature and the fused feature of \app.}
\label{pic-appendix-visualization-feature-aop}
\end{figure}
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{58}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[\protect\citeauthoryear{Babakniya, Elkordy, Ezzeldin, Liu, Song,
El-Khamy, and Avestimehr}{Babakniya et~al\mbox{.}}{2023}]%
{babakniya2023slora}
\bibfield{author}{\bibinfo{person}{Sara Babakniya},
\bibinfo{person}{Ahmed~Roushdy Elkordy}, \bibinfo{person}{Yahya~H Ezzeldin},
\bibinfo{person}{Qingfeng Liu}, \bibinfo{person}{Kee-Bong Song},
\bibinfo{person}{Mostafa El-Khamy}, {and} \bibinfo{person}{Salman
Avestimehr}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{{SLoRA}: Federated parameter efficient fine-tuning
of language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2308.06522}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Bai, Chen, Qian, Yao, and Li}{Bai
et~al\mbox{.}}{2024}]%
{bai2024federated}
\bibfield{author}{\bibinfo{person}{Jiamu Bai}, \bibinfo{person}{Daoyuan Chen},
\bibinfo{person}{Bingchen Qian}, \bibinfo{person}{Liuyi Yao}, {and}
\bibinfo{person}{Yaliang Li}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Federated fine-tuning of large language models
under heterogeneous language tasks and client resources}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2402.11505}}
(\bibinfo{year}{2024}).
\newblock
\bibitem[\protect\citeauthoryear{Cali{\'n}ski and Harabasz}{Cali{\'n}ski and
Harabasz}{1974}]%
{calinski1974dendrite}
\bibfield{author}{\bibinfo{person}{Tadeusz Cali{\'n}ski} {and}
\bibinfo{person}{Jerzy Harabasz}.} \bibinfo{year}{1974}\natexlab{}.
\newblock \showarticletitle{A dendrite method for cluster analysis}.
\newblock \bibinfo{journal}{\emph{Communications in Statistics-theory and
Methods}} \bibinfo{volume}{3}, \bibinfo{number}{1} (\bibinfo{year}{1974}),
\bibinfo{pages}{1--27}.
\newblock
\bibitem[\protect\citeauthoryear{Campello, Moulavi, and Sander}{Campello
et~al\mbox{.}}{2013}]%
{campello2013density}
\bibfield{author}{\bibinfo{person}{Ricardo~JGB Campello},
\bibinfo{person}{Davoud Moulavi}, {and} \bibinfo{person}{J{\"o}rg Sander}.}
\bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{Density-based clustering based on hierarchical
density estimates}. In \bibinfo{booktitle}{\emph{Pacific-Asia conference on
knowledge discovery and data mining}}. Springer, \bibinfo{pages}{160--172}.
\newblock
\bibitem[\protect\citeauthoryear{Cao, Kang, Wang, and Sun}{Cao
et~al\mbox{.}}{2024}]%
{cao2024instruction}
\bibfield{author}{\bibinfo{person}{Yihan Cao}, \bibinfo{person}{Yanbin Kang},
\bibinfo{person}{Chi Wang}, {and} \bibinfo{person}{Lichao Sun}.}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Instruction Mining: Instruction Data Selection for
Tuning Large Language Models}. In \bibinfo{booktitle}{\emph{First Conference
on Language Modeling}}.
\newblock
\bibitem[\protect\citeauthoryear{Che, Liu, Zhou, Ren, Zhou, Sheng, Dai, and
Dou}{Che et~al\mbox{.}}{2023}]%
{che2023FedPepTAO}
\bibfield{author}{\bibinfo{person}{Tianshi Che}, \bibinfo{person}{Ji Liu},
\bibinfo{person}{Yang Zhou}, \bibinfo{person}{Jiaxiang Ren},
\bibinfo{person}{Jiwen Zhou}, \bibinfo{person}{Victor Sheng},
\bibinfo{person}{Huaiyu Dai}, {and} \bibinfo{person}{Dejing Dou}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Federated Learning of Large Language Models with
Parameter-Efficient Prompt Tuning and Adaptive Optimization}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2023 Conference on Empirical
Methods in Natural Language Processing}}. \bibinfo{pages}{7871--7888}.
\newblock
\bibitem[\protect\citeauthoryear{Chen, Huang, Ma, Chen, Pan, Ge, Gao, Xie, Liu,
Gao, et~al\mbox{.}}{Chen et~al\mbox{.}}{2024}]%
{chen2024data}
\bibfield{author}{\bibinfo{person}{Daoyuan Chen}, \bibinfo{person}{Yilun
Huang}, \bibinfo{person}{Zhijian Ma}, \bibinfo{person}{Hesen Chen},
\bibinfo{person}{Xuchen Pan}, \bibinfo{person}{Ce Ge}, \bibinfo{person}{Dawei
Gao}, \bibinfo{person}{Yuexiang Xie}, \bibinfo{person}{Zhaoyang Liu},
\bibinfo{person}{Jinyang Gao}, {et~al\mbox{.}}}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Data-juicer: A one-stop data processing system for
large language models}. In \bibinfo{booktitle}{\emph{Companion of the 2024
International Conference on Management of Data}}. \bibinfo{pages}{120--134}.
\newblock
\bibitem[\protect\citeauthoryear{Chen, Zhang, Zhang, Yang, Hu, Ma, Yanggong,
and Zhao}{Chen et~al\mbox{.}}{2023}]%
{chen2023maybe}
\bibfield{author}{\bibinfo{person}{Hao Chen}, \bibinfo{person}{Yiming Zhang},
\bibinfo{person}{Qi Zhang}, \bibinfo{person}{Hantao Yang},
\bibinfo{person}{Xiaomeng Hu}, \bibinfo{person}{Xuetao Ma},
\bibinfo{person}{Yifan Yanggong}, {and} \bibinfo{person}{Junbo Zhao}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Maybe only 0.5\% data is needed: A preliminary
exploration of low training data instruction tuning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2305.09246}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Conover, Hayes, Mathur, Xie, Wan, Shah,
Ghodsi, Wendell, Zaharia, and Xin}{Conover et~al\mbox{.}}{2023}]%
{DatabricksBlog2023DollyV2}
\bibfield{author}{\bibinfo{person}{Mike Conover}, \bibinfo{person}{Matt Hayes},
\bibinfo{person}{Ankit Mathur}, \bibinfo{person}{Jianwei Xie},
\bibinfo{person}{Jun Wan}, \bibinfo{person}{Sam Shah}, \bibinfo{person}{Ali
Ghodsi}, \bibinfo{person}{Patrick Wendell}, \bibinfo{person}{Matei Zaharia},
{and} \bibinfo{person}{Reynold Xin}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Free Dolly: Introducing the World's First Truly Open
Instruction-Tuned {LLM}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm}
\showURL{%
\tempurl}
\bibitem[\protect\citeauthoryear{Cui, Yuan, Ding, Yao, Zhu, Ni, Xie, Liu, and
Sun}{Cui et~al\mbox{.}}{2023}]%
{cui2023ultrafeedback}
\bibfield{author}{\bibinfo{person}{Ganqu Cui}, \bibinfo{person}{Lifan Yuan},
\bibinfo{person}{Ning Ding}, \bibinfo{person}{Guanming Yao},
\bibinfo{person}{Wei Zhu}, \bibinfo{person}{Yuan Ni},
\bibinfo{person}{Guotong Xie}, \bibinfo{person}{Zhiyuan Liu}, {and}
\bibinfo{person}{Maosong Sun}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Ultrafeedback: Boosting language models with
high-quality feedback}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2310.01377}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Dwork, Roth, et~al\mbox{.}}{Dwork
et~al\mbox{.}}{2014}]%
{dwork2014algorithmic}
\bibfield{author}{\bibinfo{person}{Cynthia Dwork}, \bibinfo{person}{Aaron
Roth}, {et~al\mbox{.}}} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{The algorithmic foundations of differential
privacy}.
\newblock \bibinfo{journal}{\emph{Foundations and Trends{\textregistered} in
Theoretical Computer Science}} \bibinfo{volume}{9}, \bibinfo{number}{3--4}
(\bibinfo{year}{2014}), \bibinfo{pages}{211--407}.
\newblock
\bibitem[\protect\citeauthoryear{Fan, Ding, Ning, Wang, Li, Yin, Chua, and
Li}{Fan et~al\mbox{.}}{2024}]%
{fan2024survey}
\bibfield{author}{\bibinfo{person}{Wenqi Fan}, \bibinfo{person}{Yujuan Ding},
\bibinfo{person}{Liangbo Ning}, \bibinfo{person}{Shijie Wang},
\bibinfo{person}{Hengyun Li}, \bibinfo{person}{Dawei Yin},
\bibinfo{person}{Tat-Seng Chua}, {and} \bibinfo{person}{Qing Li}.}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{A survey on rag meeting llms: Towards
retrieval-augmented large language models}. In
\bibinfo{booktitle}{\emph{Proceedings of the 30th ACM SIGKDD Conference on
Knowledge Discovery and Data Mining}}. \bibinfo{pages}{6491--6501}.
\newblock
\bibitem[\protect\citeauthoryear{Hu, Shen, Wallis, Allen{-}Zhu, Li, Wang, Wang,
and Chen}{Hu et~al\mbox{.}}{2022b}]%
{lora}
\bibfield{author}{\bibinfo{person}{Edward~J. Hu}, \bibinfo{person}{Yelong
Shen}, \bibinfo{person}{Phillip Wallis}, \bibinfo{person}{Zeyuan
Allen{-}Zhu}, \bibinfo{person}{Yuanzhi Li}, \bibinfo{person}{Shean Wang},
\bibinfo{person}{Lu Wang}, {and} \bibinfo{person}{Weizhu Chen}.}
\bibinfo{year}{2022}\natexlab{b}.
\newblock \showarticletitle{LoRA: Low-Rank Adaptation of Large Language
Models}. In \bibinfo{booktitle}{\emph{The Tenth International Conference on
Learning Representations, {ICLR}}}. \bibinfo{publisher}{OpenReview.net}.
\newblock
\bibitem[\protect\citeauthoryear{Hu, Salcic, Sun, Dobbie, Yu, and Zhang}{Hu
et~al\mbox{.}}{2022a}]%
{hu2022membership}
\bibfield{author}{\bibinfo{person}{Hongsheng Hu}, \bibinfo{person}{Zoran
Salcic}, \bibinfo{person}{Lichao Sun}, \bibinfo{person}{Gillian Dobbie},
\bibinfo{person}{Philip~S Yu}, {and} \bibinfo{person}{Xuyun Zhang}.}
\bibinfo{year}{2022}\natexlab{a}.
\newblock \showarticletitle{Membership inference attacks on machine learning: A
survey}.
\newblock \bibinfo{journal}{\emph{ACM Computing Surveys (CSUR)}}
\bibinfo{volume}{54}, \bibinfo{number}{11s} (\bibinfo{year}{2022}),
\bibinfo{pages}{1--37}.
\newblock
\bibitem[\protect\citeauthoryear{Jolliffe}{Jolliffe}{2002}]%
{jolliffe2002principal}
\bibfield{author}{\bibinfo{person}{Ian~T Jolliffe}.}
\bibinfo{year}{2002}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Principal component analysis for special
types of data}}.
\newblock \bibinfo{publisher}{Springer}.
\newblock
\bibitem[\protect\citeauthoryear{Kingma and Ba}{Kingma and Ba}{2015}]%
{adam}
\bibfield{author}{\bibinfo{person}{Diederik~P. Kingma} {and}
\bibinfo{person}{Jimmy Ba}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{Adam: {A} Method for Stochastic Optimization}. In
\bibinfo{booktitle}{\emph{3rd International Conference on Learning
Representations, {ICLR}}}.
\newblock
\bibitem[\protect\citeauthoryear{Kuang, Qian, Li, Chen, Gao, Pan, Xie, Li,
Ding, and Zhou}{Kuang et~al\mbox{.}}{2024}]%
{kuang2024federatedscope}
\bibfield{author}{\bibinfo{person}{Weirui Kuang}, \bibinfo{person}{Bingchen
Qian}, \bibinfo{person}{Zitao Li}, \bibinfo{person}{Daoyuan Chen},
\bibinfo{person}{Dawei Gao}, \bibinfo{person}{Xuchen Pan},
\bibinfo{person}{Yuexiang Xie}, \bibinfo{person}{Yaliang Li},
\bibinfo{person}{Bolin Ding}, {and} \bibinfo{person}{Jingren Zhou}.}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Federatedscope-llm: A comprehensive package for
fine-tuning large language models in federated learning}. In
\bibinfo{booktitle}{\emph{Proceedings of the 30th ACM SIGKDD Conference on
Knowledge Discovery and Data Mining}}. \bibinfo{pages}{5260--5271}.
\newblock
\bibitem[\protect\citeauthoryear{Lester, Al{-}Rfou, and Constant}{Lester
et~al\mbox{.}}{2021}]%
{lester2021power}
\bibfield{author}{\bibinfo{person}{Brian Lester}, \bibinfo{person}{Rami
Al{-}Rfou}, {and} \bibinfo{person}{Noah Constant}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{The Power of Scale for Parameter-Efficient Prompt
Tuning}. In \bibinfo{booktitle}{\emph{Proceedings of the 2021 Conference on
Empirical Methods in Natural Language Processing, {EMNLP}}}.
\bibinfo{pages}{3045--3059}.
\newblock
\bibitem[\protect\citeauthoryear{Li, Chen, Chen, He, Gu, and Zhou}{Li
et~al\mbox{.}}{2024a}]%
{li2024selective}
\bibfield{author}{\bibinfo{person}{Ming Li}, \bibinfo{person}{Lichang Chen},
\bibinfo{person}{Jiuhai Chen}, \bibinfo{person}{Shwai He},
\bibinfo{person}{Jiuxiang Gu}, {and} \bibinfo{person}{Tianyi Zhou}.}
\bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Selective Reflection-Tuning: Student-Selected Data
Recycling for {LLM} Instruction-Tuning}. In
\bibinfo{booktitle}{\emph{Findings of the Association for Computational
Linguistics, {ACL} 2024}}, \bibfield{editor}{\bibinfo{person}{Lun{-}Wei Ku},
\bibinfo{person}{Andre Martins}, {and} \bibinfo{person}{Vivek Srikumar}}
(Eds.). \bibinfo{pages}{16189--16211}.
\newblock
\bibitem[\protect\citeauthoryear{Li, Zhang, He, Li, Zhao, Wang, Cheng, and
Zhou}{Li et~al\mbox{.}}{2024b}]%
{li2024Superfiltering}
\bibfield{author}{\bibinfo{person}{Ming Li}, \bibinfo{person}{Yong Zhang},
\bibinfo{person}{Shwai He}, \bibinfo{person}{Zhitao Li},
\bibinfo{person}{Hongyu Zhao}, \bibinfo{person}{Jianzong Wang},
\bibinfo{person}{Ning Cheng}, {and} \bibinfo{person}{Tianyi Zhou}.}
\bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Superfiltering: Weak-to-Strong Data Filtering for
Fast Instruction-Tuning}. In \bibinfo{booktitle}{\emph{Proceedings of the
62nd Annual Meeting of the Association for Computational Linguistics (Volume
1: Long Papers), {ACL}}}, \bibfield{editor}{\bibinfo{person}{Lun{-}Wei Ku},
\bibinfo{person}{Andre Martins}, {and} \bibinfo{person}{Vivek Srikumar}}
(Eds.). \bibinfo{publisher}{Association for Computational Linguistics},
\bibinfo{pages}{14255--14273}.
\newblock
\bibitem[\protect\citeauthoryear{Li, Zhang, Li, Chen, Chen, Cheng, Wang, Zhou,
and Xiao}{Li et~al\mbox{.}}{2024c}]%
{li2024from}
\bibfield{author}{\bibinfo{person}{Ming Li}, \bibinfo{person}{Yong Zhang},
\bibinfo{person}{Zhitao Li}, \bibinfo{person}{Jiuhai Chen},
\bibinfo{person}{Lichang Chen}, \bibinfo{person}{Ning Cheng},
\bibinfo{person}{Jianzong Wang}, \bibinfo{person}{Tianyi Zhou}, {and}
\bibinfo{person}{Jing Xiao}.} \bibinfo{year}{2024}\natexlab{c}.
\newblock \showarticletitle{From Quantity to Quality: Boosting {LLM}
Performance with Self-Guided Data Selection for Instruction Tuning}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2024 Conference of the North
American Chapter of the Association for Computational Linguistics: Human
Language Technologies (Volume 1: Long Papers), {NAACL} 2024}}.
\bibinfo{publisher}{Association for Computational Linguistics},
\bibinfo{pages}{7602--7635}.
\newblock
\bibitem[\protect\citeauthoryear{Lin, Wang, Li, Yang, Feng, Wei, and Chua}{Lin
et~al\mbox{.}}{2024}]%
{lin2024data}
\bibfield{author}{\bibinfo{person}{Xinyu Lin}, \bibinfo{person}{Wenjie Wang},
\bibinfo{person}{Yongqi Li}, \bibinfo{person}{Shuo Yang},
\bibinfo{person}{Fuli Feng}, \bibinfo{person}{Yinwei Wei}, {and}
\bibinfo{person}{Tat-Seng Chua}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Data-efficient Fine-tuning for LLM-based
Recommendation}. In \bibinfo{booktitle}{\emph{Proceedings of the 47th
International ACM SIGIR Conference on Research and Development in Information
Retrieval}}. \bibinfo{pages}{365--374}.
\newblock
\bibitem[\protect\citeauthoryear{Ling, Chen, Yao, Li, and Shen}{Ling
et~al\mbox{.}}{2024}]%
{ling2024convergence}
\bibfield{author}{\bibinfo{person}{Zhenqing Ling}, \bibinfo{person}{Daoyuan
Chen}, \bibinfo{person}{Liuyi Yao}, \bibinfo{person}{Yaliang Li}, {and}
\bibinfo{person}{Ying Shen}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{On the convergence of zeroth-order federated tuning
for large language models}. In \bibinfo{booktitle}{\emph{Proceedings of the
30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}}.
\bibinfo{pages}{1827--1838}.
\newblock
\bibitem[\protect\citeauthoryear{Liu, Zheng, Du, Ding, Qian, Yang, and
Tang}{Liu et~al\mbox{.}}{2023}]%
{liu2023gpt}
\bibfield{author}{\bibinfo{person}{Xiao Liu}, \bibinfo{person}{Yanan Zheng},
\bibinfo{person}{Zhengxiao Du}, \bibinfo{person}{Ming Ding},
\bibinfo{person}{Yujie Qian}, \bibinfo{person}{Zhilin Yang}, {and}
\bibinfo{person}{Jie Tang}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{{GPT} understands, too}.
\newblock \bibinfo{journal}{\emph{AI Open}} (\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Liu, Tao, Zhao, Zhu, Ma, Zhu, Su, Hou, Zhang,
Zhang, et~al\mbox{.}}{Liu et~al\mbox{.}}{2024}]%
{liu2024coachlm}
\bibfield{author}{\bibinfo{person}{Yilun Liu}, \bibinfo{person}{Shimin Tao},
\bibinfo{person}{Xiaofeng Zhao}, \bibinfo{person}{Ming Zhu},
\bibinfo{person}{Wenbing Ma}, \bibinfo{person}{Junhao Zhu},
\bibinfo{person}{Chang Su}, \bibinfo{person}{Yutai Hou},
\bibinfo{person}{Miao Zhang}, \bibinfo{person}{Min Zhang}, {et~al\mbox{.}}}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{CoachLM: Automatic Instruction Revisions Improve
the Data Quality in LLM Instruction Tuning}. In
\bibinfo{booktitle}{\emph{2024 IEEE 40th International Conference on Data
Engineering (ICDE)}}. IEEE, \bibinfo{pages}{5184--5197}.
\newblock
\bibitem[\protect\citeauthoryear{Lloyd}{Lloyd}{1982}]%
{lloyd1982least}
\bibfield{author}{\bibinfo{person}{Stuart Lloyd}.}
\bibinfo{year}{1982}\natexlab{}.
\newblock \showarticletitle{Least squares quantization in PCM}.
\newblock \bibinfo{journal}{\emph{IEEE transactions on information theory}}
\bibinfo{volume}{28}, \bibinfo{number}{2} (\bibinfo{year}{1982}),
\bibinfo{pages}{129--137}.
\newblock
\bibitem[\protect\citeauthoryear{Lu, Yuan, Yuan, Lin, Lin, Tan, Zhou, and
Zhou}{Lu et~al\mbox{.}}{2023}]%
{lu2023instag}
\bibfield{author}{\bibinfo{person}{Keming Lu}, \bibinfo{person}{Hongyi Yuan},
\bibinfo{person}{Zheng Yuan}, \bibinfo{person}{Runji Lin},
\bibinfo{person}{Junyang Lin}, \bibinfo{person}{Chuanqi Tan},
\bibinfo{person}{Chang Zhou}, {and} \bibinfo{person}{Jingren Zhou}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{\# instag: Instruction tagging for analyzing
supervised fine-tuning of large language models}. In
\bibinfo{booktitle}{\emph{The Twelfth International Conference on Learning
Representations}}.
\newblock
\bibitem[\protect\citeauthoryear{Malladi, Gao, Nichani, Damian, Lee, Chen, and
Arora}{Malladi et~al\mbox{.}}{2023}]%
{malladi2023mezo}
\bibfield{author}{\bibinfo{person}{Sadhika Malladi}, \bibinfo{person}{Tianyu
Gao}, \bibinfo{person}{Eshaan Nichani}, \bibinfo{person}{Alex Damian},
\bibinfo{person}{Jason~D Lee}, \bibinfo{person}{Danqi Chen}, {and}
\bibinfo{person}{Sanjeev Arora}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Fine-tuning language models with just forward
passes}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing
Systems}} \bibinfo{volume}{36} (\bibinfo{year}{2023}),
\bibinfo{pages}{53038--53075}.
\newblock
\bibitem[\protect\citeauthoryear{Mangrulkar, Gugger, Debut, Belkada, Paul, and
Bossan}{Mangrulkar et~al\mbox{.}}{2022}]%
{peft}
\bibfield{author}{\bibinfo{person}{Sourab Mangrulkar}, \bibinfo{person}{Sylvain
Gugger}, \bibinfo{person}{Lysandre Debut}, \bibinfo{person}{Younes Belkada},
\bibinfo{person}{Sayak Paul}, {and} \bibinfo{person}{Benjamin Bossan}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \bibinfo{title}{{PEFT}: State-of-the-art Parameter-Efficient
Fine-Tuning methods}.
\newblock \bibinfo{howpublished}{\url{https://github.com/huggingface/peft}}.
\newblock
\bibitem[\protect\citeauthoryear{McMahan, Moore, Ramage, Hampson, and
y~Arcas}{McMahan et~al\mbox{.}}{2017}]%
{mcmahan2017fl}
\bibfield{author}{\bibinfo{person}{Brendan McMahan}, \bibinfo{person}{Eider
Moore}, \bibinfo{person}{Daniel Ramage}, \bibinfo{person}{Seth Hampson},
{and} \bibinfo{person}{Blaise~Ag{\"{u}}era y Arcas}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Communication-Efficient Learning of Deep Networks
from Decentralized Data}. In \bibinfo{booktitle}{\emph{the International
Conference on Artificial Intelligence and Statistics}},
Vol.~\bibinfo{volume}{54}. \bibinfo{pages}{1273--1282}.
\newblock
\bibitem[\protect\citeauthoryear{Paszke, Gross, Massa, Lerer, Bradbury, Chanan,
Killeen, Lin, Gimelshein, Antiga, et~al\mbox{.}}{Paszke
et~al\mbox{.}}{2019}]%
{paszke2019pytorch}
\bibfield{author}{\bibinfo{person}{Adam Paszke}, \bibinfo{person}{Sam Gross},
\bibinfo{person}{Francisco Massa}, \bibinfo{person}{Adam Lerer},
\bibinfo{person}{James Bradbury}, \bibinfo{person}{Gregory Chanan},
\bibinfo{person}{Trevor Killeen}, \bibinfo{person}{Zeming Lin},
\bibinfo{person}{Natalia Gimelshein}, \bibinfo{person}{Luca Antiga},
{et~al\mbox{.}}} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{PyTorch}: An imperative style, high-performance
deep learning library}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing
systems}} \bibinfo{volume}{32} (\bibinfo{year}{2019}).
\newblock
\bibitem[\protect\citeauthoryear{Qin, Chen, Qian, Ding, Li, and Deng}{Qin
et~al\mbox{.}}{2024a}]%
{qin2024full}
\bibfield{author}{\bibinfo{person}{Zhen Qin}, \bibinfo{person}{Daoyuan Chen},
\bibinfo{person}{Bingchen Qian}, \bibinfo{person}{Bolin Ding},
\bibinfo{person}{Yaliang Li}, {and} \bibinfo{person}{Shuiguang Deng}.}
\bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Federated Full-Parameter Tuning of Billion-Sized
Language Models with Communication Cost under 18 Kilobytes}. In
\bibinfo{booktitle}{\emph{Proceedings of the 41st International Conference on
Machine Learning, {ICML}}}, Vol.~\bibinfo{volume}{235}.
\bibinfo{pages}{41473--41497}.
\newblock
\bibitem[\protect\citeauthoryear{Qin, Chen, Zhang, Yao, Huang, Ding, Li, and
Deng}{Qin et~al\mbox{.}}{2024b}]%
{qin2024synergy}
\bibfield{author}{\bibinfo{person}{Zhen Qin}, \bibinfo{person}{Daoyuan Chen},
\bibinfo{person}{Wenhao Zhang}, \bibinfo{person}{Liuyi Yao},
\bibinfo{person}{Yilun Huang}, \bibinfo{person}{Bolin Ding},
\bibinfo{person}{Yaliang Li}, {and} \bibinfo{person}{Shuiguang Deng}.}
\bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{The Synergy between Data and Multi-Modal Large
Language Models: A Survey from Co-Development Perspective}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2407.08583}}
(\bibinfo{year}{2024}).
\newblock
\bibitem[\protect\citeauthoryear{Radford, Wu, Child, Luan, Amodei, and
Sutskever}{Radford et~al\mbox{.}}{2019}]%
{radford2019language}
\bibfield{author}{\bibinfo{person}{Alec Radford}, \bibinfo{person}{Jeff Wu},
\bibinfo{person}{Rewon Child}, \bibinfo{person}{David Luan},
\bibinfo{person}{Dario Amodei}, {and} \bibinfo{person}{Ilya Sutskever}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Language Models are Unsupervised Multitask
Learners}.
\newblock (\bibinfo{year}{2019}).
\newblock
\bibitem[\protect\citeauthoryear{Rasley, Rajbhandari, Ruwase, and He}{Rasley
et~al\mbox{.}}{2020}]%
{rasley2020deepspeed}
\bibfield{author}{\bibinfo{person}{Jeff Rasley}, \bibinfo{person}{Samyam
Rajbhandari}, \bibinfo{person}{Olatunji Ruwase}, {and}
\bibinfo{person}{Yuxiong He}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Deepspeed: System optimizations enable training
deep learning models with over 100 billion parameters}. In
\bibinfo{booktitle}{\emph{Proceedings of the 26th ACM SIGKDD International
Conference on Knowledge Discovery \& Data Mining}}.
\bibinfo{pages}{3505--3506}.
\newblock
\bibitem[\protect\citeauthoryear{Rousseeuw}{Rousseeuw}{1987}]%
{rousseeuw1987silhouettes}
\bibfield{author}{\bibinfo{person}{Peter~J Rousseeuw}.}
\bibinfo{year}{1987}\natexlab{}.
\newblock \showarticletitle{Silhouettes: a graphical aid to the interpretation
and validation of cluster analysis}.
\newblock \bibinfo{journal}{\emph{Journal of computational and applied
mathematics}} \bibinfo{volume}{20} (\bibinfo{year}{1987}),
\bibinfo{pages}{53--65}.
\newblock
\bibitem[\protect\citeauthoryear{Sachdeva, Coleman, Kang, Ni, Hong, Chi,
Caverlee, McAuley, and Cheng}{Sachdeva et~al\mbox{.}}{2024}]%
{sachdeva2024howto}
\bibfield{author}{\bibinfo{person}{Noveen Sachdeva}, \bibinfo{person}{Benjamin
Coleman}, \bibinfo{person}{Wang-Cheng Kang}, \bibinfo{person}{Jianmo Ni},
\bibinfo{person}{Lichan Hong}, \bibinfo{person}{Ed~H Chi},
\bibinfo{person}{James Caverlee}, \bibinfo{person}{Julian McAuley}, {and}
\bibinfo{person}{Derek~Zhiyuan Cheng}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{How to Train Data-Efficient LLMs}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2402.09668}}
(\bibinfo{year}{2024}).
\newblock
\bibitem[\protect\citeauthoryear{Sch{\"o}lkopf, Smola, and
M{\"u}ller}{Sch{\"o}lkopf et~al\mbox{.}}{1997}]%
{scholkopf1997kernel}
\bibfield{author}{\bibinfo{person}{Bernhard Sch{\"o}lkopf},
\bibinfo{person}{Alexander Smola}, {and} \bibinfo{person}{Klaus-Robert
M{\"u}ller}.} \bibinfo{year}{1997}\natexlab{}.
\newblock \showarticletitle{Kernel principal component analysis}. In
\bibinfo{booktitle}{\emph{International conference on artificial neural
networks}}. Springer, \bibinfo{pages}{583--588}.
\newblock
\bibitem[\protect\citeauthoryear{Shayan, Fung, Yoon, and Beschastnikh}{Shayan
et~al\mbox{.}}{2020}]%
{shayan2020biscotti}
\bibfield{author}{\bibinfo{person}{Muhammad Shayan}, \bibinfo{person}{Clement
Fung}, \bibinfo{person}{Chris~JM Yoon}, {and} \bibinfo{person}{Ivan
Beschastnikh}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Biscotti: A blockchain system for private and
secure federated learning}.
\newblock \bibinfo{journal}{\emph{IEEE Transactions on Parallel and Distributed
Systems}} \bibinfo{volume}{32}, \bibinfo{number}{7} (\bibinfo{year}{2020}),
\bibinfo{pages}{1513--1525}.
\newblock
\bibitem[\protect\citeauthoryear{Sun, Li, Li, and Ding}{Sun
et~al\mbox{.}}{2024}]%
{sun2024lora}
\bibfield{author}{\bibinfo{person}{Youbang Sun}, \bibinfo{person}{Zitao Li},
\bibinfo{person}{Yaliang Li}, {and} \bibinfo{person}{Bolin Ding}.}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Improving LoRA in Privacy-preserving Federated
Learning}. In \bibinfo{booktitle}{\emph{The Twelfth International Conference
on Learning Representations, {ICLR}}}. \bibinfo{publisher}{OpenReview.net}.
\newblock
\bibitem[\protect\citeauthoryear{Taori, Gulrajani, Zhang, Dubois, Li, Guestrin,
Liang, and Hashimoto}{Taori et~al\mbox{.}}{2023}]%
{alpaca}
\bibfield{author}{\bibinfo{person}{Rohan Taori}, \bibinfo{person}{Ishaan
Gulrajani}, \bibinfo{person}{Tianyi Zhang}, \bibinfo{person}{Yann Dubois},
\bibinfo{person}{Xuechen Li}, \bibinfo{person}{Carlos Guestrin},
\bibinfo{person}{Percy Liang}, {and} \bibinfo{person}{Tatsunori~B.
Hashimoto}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Stanford {Alpaca}: An Instruction-following LLaMA
model}.
\newblock
\bibinfo{howpublished}{\url{https://github.com/tatsu-lab/stanford_alpaca}}.
\newblock
\bibitem[\protect\citeauthoryear{Van Der~Maaten}{Van Der~Maaten}{2013}]%
{van2013barnes}
\bibfield{author}{\bibinfo{person}{Laurens Van Der~Maaten}.}
\bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{Barnes-hut-sne}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1301.3342}}
(\bibinfo{year}{2013}).
\newblock
\bibitem[\protect\citeauthoryear{Van~der Maaten and Hinton}{Van~der Maaten and
Hinton}{2008}]%
{van2008visualizing}
\bibfield{author}{\bibinfo{person}{Laurens Van~der Maaten} {and}
\bibinfo{person}{Geoffrey Hinton}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Visualizing data using t-SNE.}
\newblock \bibinfo{journal}{\emph{Journal of machine learning research}}
\bibinfo{volume}{9}, \bibinfo{number}{11} (\bibinfo{year}{2008}).
\newblock
\bibitem[\protect\citeauthoryear{Villalobos, Sevilla, Heim, Besiroglu,
Hobbhahn, and Ho}{Villalobos et~al\mbox{.}}{2022}]%
{villalobos2022will}
\bibfield{author}{\bibinfo{person}{Pablo Villalobos}, \bibinfo{person}{Jaime
Sevilla}, \bibinfo{person}{Lennart Heim}, \bibinfo{person}{Tamay Besiroglu},
\bibinfo{person}{Marius Hobbhahn}, {and} \bibinfo{person}{Anson Ho}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Will we run out of data? An analysis of the limits
of scaling datasets in Machine Learning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2211.04325}}
(\bibinfo{year}{2022}).
\newblock
\bibitem[\protect\citeauthoryear{Voigt and Von~dem Bussche}{Voigt and Von~dem
Bussche}{2017}]%
{voigt2017eu}
\bibfield{author}{\bibinfo{person}{Paul Voigt} {and} \bibinfo{person}{Axel
Von~dem Bussche}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{The eu general data protection regulation (gdpr)}.
\newblock \bibinfo{journal}{\emph{A Practical Guide, 1st Ed., Cham: Springer
International Publishing}} \bibinfo{volume}{10}, \bibinfo{number}{3152676}
(\bibinfo{year}{2017}), \bibinfo{pages}{10--5555}.
\newblock
\bibitem[\protect\citeauthoryear{Wang, Zhang, Du, Zhang, and Chu}{Wang
et~al\mbox{.}}{2024a}]%
{wang2024survey}
\bibfield{author}{\bibinfo{person}{Jiahao Wang}, \bibinfo{person}{Bolin Zhang},
\bibinfo{person}{Qianlong Du}, \bibinfo{person}{Jiajun Zhang}, {and}
\bibinfo{person}{Dianhui Chu}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{A Survey on Data Selection for LLM Instruction
Tuning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2402.05123}}
(\bibinfo{year}{2024}).
\newblock
\bibitem[\protect\citeauthoryear{Wang, Zhang, Du, Zhang, and Chu}{Wang
et~al\mbox{.}}{2024b}]%
{survey-wang2024survey-selection}
\bibfield{author}{\bibinfo{person}{Jiahao Wang}, \bibinfo{person}{Bolin Zhang},
\bibinfo{person}{Qianlong Du}, \bibinfo{person}{Jiajun Zhang}, {and}
\bibinfo{person}{Dianhui Chu}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{A Survey on Data Selection for LLM Instruction
Tuning}.
\newblock \bibinfo{journal}{\emph{arXiv:2402.05123}} (\bibinfo{year}{2024}).
\newblock
\bibitem[\protect\citeauthoryear{Wang, Kordi, Mishra, Liu, Smith, Khashabi, and
Hajishirzi}{Wang et~al\mbox{.}}{2023}]%
{wang2023self}
\bibfield{author}{\bibinfo{person}{Yizhong Wang}, \bibinfo{person}{Yeganeh
Kordi}, \bibinfo{person}{Swaroop Mishra}, \bibinfo{person}{Alisa Liu},
\bibinfo{person}{Noah~A Smith}, \bibinfo{person}{Daniel Khashabi}, {and}
\bibinfo{person}{Hannaneh Hajishirzi}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Self-Instruct: Aligning Language Models with
Self-Generated Instructions}. In \bibinfo{booktitle}{\emph{Proceedings of the
61st Annual Meeting of the Association for Computational Linguistics (Volume
1: Long Papers)}}. \bibinfo{pages}{13484--13508}.
\newblock
\bibitem[\protect\citeauthoryear{Wang, Mishra, Alipoormolabashi, Kordi,
Mirzaei, Naik, Ashok, Dhanasekaran, Arunkumar, Stap, Pathak, Karamanolakis,
Lai, Purohit, Mondal, Anderson, Kuznia, Doshi, Pal, Patel, Moradshahi,
Parmar, Purohit, Varshney, Kaza, Verma, Puri, Karia, Doshi, Sampat, Mishra,
A, Patro, Dixit, and Shen}{Wang et~al\mbox{.}}{2022}]%
{supernaturalinstructions}
\bibfield{author}{\bibinfo{person}{Yizhong Wang}, \bibinfo{person}{Swaroop
Mishra}, \bibinfo{person}{Pegah Alipoormolabashi}, \bibinfo{person}{Yeganeh
Kordi}, \bibinfo{person}{Amirreza Mirzaei}, \bibinfo{person}{Atharva Naik},
\bibinfo{person}{Arjun Ashok}, \bibinfo{person}{Arut~Selvan Dhanasekaran},
\bibinfo{person}{Anjana Arunkumar}, \bibinfo{person}{David Stap},
\bibinfo{person}{Eshaan Pathak}, \bibinfo{person}{Giannis Karamanolakis},
\bibinfo{person}{Haizhi~Gary Lai}, \bibinfo{person}{Ishan Purohit},
\bibinfo{person}{Ishani Mondal}, \bibinfo{person}{Jacob Anderson},
\bibinfo{person}{Kirby Kuznia}, \bibinfo{person}{Krima Doshi},
\bibinfo{person}{Kuntal~Kumar Pal}, \bibinfo{person}{Maitreya Patel},
\bibinfo{person}{Mehrad Moradshahi}, \bibinfo{person}{Mihir Parmar},
\bibinfo{person}{Mirali Purohit}, \bibinfo{person}{Neeraj Varshney},
\bibinfo{person}{Phani~Rohitha Kaza}, \bibinfo{person}{Pulkit Verma},
\bibinfo{person}{Ravsehaj~Singh Puri}, \bibinfo{person}{Rushang Karia},
\bibinfo{person}{Savan Doshi}, \bibinfo{person}{Shailaja~Keyur Sampat},
\bibinfo{person}{Siddhartha Mishra}, \bibinfo{person}{Sujan~Reddy A},
\bibinfo{person}{Sumanta Patro}, \bibinfo{person}{Tanay Dixit}, {and}
\bibinfo{person}{Xudong Shen}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Super-NaturalInstructions: Generalization via
Declarative Instructions on 1600+ {NLP} Tasks}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2022 Conference on Empirical
Methods in Natural Language Processing, {EMNLP}}}.
\bibinfo{publisher}{Association for Computational Linguistics},
\bibinfo{pages}{5085--5109}.
\newblock
\bibitem[\protect\citeauthoryear{Wei, Bosma, Zhao, Guu, Yu, Lester, Du, Dai,
and Le}{Wei et~al\mbox{.}}{2022}]%
{wei2022flan}
\bibfield{author}{\bibinfo{person}{Jason Wei}, \bibinfo{person}{Maarten Bosma},
\bibinfo{person}{Vincent~Y. Zhao}, \bibinfo{person}{Kelvin Guu},
\bibinfo{person}{Adams~Wei Yu}, \bibinfo{person}{Brian Lester},
\bibinfo{person}{Nan Du}, \bibinfo{person}{Andrew~M. Dai}, {and}
\bibinfo{person}{Quoc~V. Le}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Finetuned Language Models are Zero-Shot Learners}.
In \bibinfo{booktitle}{\emph{The Tenth International Conference on Learning
Representations, {ICLR}}}. \bibinfo{publisher}{OpenReview.net}.
\newblock
\bibitem[\protect\citeauthoryear{Wolf, Debut, Sanh, Chaumond, Delangue, Moi,
Cistac, Rault, Louf, Funtowicz, et~al\mbox{.}}{Wolf et~al\mbox{.}}{2020}]%
{wolf-etal-2020-transformers}
\bibfield{author}{\bibinfo{person}{Thomas Wolf}, \bibinfo{person}{Lysandre
Debut}, \bibinfo{person}{Victor Sanh}, \bibinfo{person}{Julien Chaumond},
\bibinfo{person}{Clement Delangue}, \bibinfo{person}{Anthony Moi},
\bibinfo{person}{Pierric Cistac}, \bibinfo{person}{Tim Rault},
\bibinfo{person}{R{\'e}mi Louf}, \bibinfo{person}{Morgan Funtowicz},
{et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Transformers: State-of-the-art natural language
processing}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020 conference
on empirical methods in natural language processing: system demonstrations}}.
\bibinfo{pages}{38--45}.
\newblock
\bibitem[\protect\citeauthoryear{Wu, Li, Li, Ding, and Gao}{Wu
et~al\mbox{.}}{2024}]%
{wu2024fedbiot}
\bibfield{author}{\bibinfo{person}{Feijie Wu}, \bibinfo{person}{Zitao Li},
\bibinfo{person}{Yaliang Li}, \bibinfo{person}{Bolin Ding}, {and}
\bibinfo{person}{Jing Gao}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{FedBiOT: LLM Local Fine-tuning in Federated
Learning without Full Model}. In \bibinfo{booktitle}{\emph{Proceedings of the
30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}}.
\bibinfo{pages}{3345--3355}.
\newblock
\bibitem[\protect\citeauthoryear{Wu, Lu, Xu, Lin, Su, and Zhou}{Wu
et~al\mbox{.}}{2023}]%
{wu2023self}
\bibfield{author}{\bibinfo{person}{Shengguang Wu}, \bibinfo{person}{Keming Lu},
\bibinfo{person}{Benfeng Xu}, \bibinfo{person}{Junyang Lin},
\bibinfo{person}{Qi Su}, {and} \bibinfo{person}{Chang Zhou}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Self-evolved diverse data sampling for efficient
instruction tuning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2311.08182}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Xu, Cai, Wu, Li, and Wang}{Xu
et~al\mbox{.}}{2024}]%
{xu2024forward}
\bibfield{author}{\bibinfo{person}{Mengwei Xu}, \bibinfo{person}{Dongqi Cai},
\bibinfo{person}{Yaozong Wu}, \bibinfo{person}{Xiang Li}, {and}
\bibinfo{person}{Shangguang Wang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{FwdLLM: Efficient Federated Finetuning of Large
Language Models with Perturbed Inferences}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2024 {USENIX} Annual Technical
Conference, {USENIX} {ATC}}}, \bibfield{editor}{\bibinfo{person}{Saurabh
Bagchi} {and} \bibinfo{person}{Yiying Zhang}} (Eds.).
\bibinfo{pages}{579--596}.
\newblock
\bibitem[\protect\citeauthoryear{Zha, Bhat, Lai, Yang, Jiang, Zhong, and
Hu}{Zha et~al\mbox{.}}{2023}]%
{zha2023data}
\bibfield{author}{\bibinfo{person}{Daochen Zha}, \bibinfo{person}{Zaid~Pervaiz
Bhat}, \bibinfo{person}{Kwei-Herng Lai}, \bibinfo{person}{Fan Yang},
\bibinfo{person}{Zhimeng Jiang}, \bibinfo{person}{Shaochen Zhong}, {and}
\bibinfo{person}{Xia Hu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Data-centric artificial intelligence: A survey}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2303.10158}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Zhang, Vahidian, Kuo, Li, Zhang, Yu, Wang, and
Chen}{Zhang et~al\mbox{.}}{2024}]%
{zhang2024fedit}
\bibfield{author}{\bibinfo{person}{Jianyi Zhang}, \bibinfo{person}{Saeed
Vahidian}, \bibinfo{person}{Martin Kuo}, \bibinfo{person}{Chunyuan Li},
\bibinfo{person}{Ruiyi Zhang}, \bibinfo{person}{Tong Yu},
\bibinfo{person}{Guoyin Wang}, {and} \bibinfo{person}{Yiran Chen}.}
\bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Towards building the federatedGPT: Federated
instruction tuning}. In \bibinfo{booktitle}{\emph{ICASSP 2024-2024 IEEE
International Conference on Acoustics, Speech and Signal Processing
(ICASSP)}}. IEEE, \bibinfo{pages}{6915--6919}.
\newblock
\bibitem[\protect\citeauthoryear{Zhang, Yang, Dai, Wang, Yu, Qu, and Xu}{Zhang
et~al\mbox{.}}{2023}]%
{zhang2023-FedPETuning}
\bibfield{author}{\bibinfo{person}{Zhuo Zhang}, \bibinfo{person}{Yuanhang
Yang}, \bibinfo{person}{Yong Dai}, \bibinfo{person}{Qifan Wang},
\bibinfo{person}{Yue Yu}, \bibinfo{person}{Lizhen Qu}, {and}
\bibinfo{person}{Zenglin Xu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{{FedPETuning}: When Federated Learning Meets the
Parameter-Efficient Tuning Methods of Pre-trained Language Models}. In
\bibinfo{booktitle}{\emph{Findings of the Association for Computational
Linguistics: {ACL}}}. \bibinfo{pages}{9963--9977}.
\newblock
\bibitem[\protect\citeauthoryear{Zhou, Liu, Xu, Iyer, Sun, Mao, Ma, Efrat, Yu,
Yu, et~al\mbox{.}}{Zhou et~al\mbox{.}}{2023}]%
{zhou2023lima}
\bibfield{author}{\bibinfo{person}{Chunting Zhou}, \bibinfo{person}{Pengfei
Liu}, \bibinfo{person}{Puxin Xu}, \bibinfo{person}{Srinivasan Iyer},
\bibinfo{person}{Jiao Sun}, \bibinfo{person}{Yuning Mao},
\bibinfo{person}{Xuezhe Ma}, \bibinfo{person}{Avia Efrat},
\bibinfo{person}{Ping Yu}, \bibinfo{person}{Lili Yu}, {et~al\mbox{.}}}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Lima: Less is more for alignment}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing
Systems}} \bibinfo{volume}{36} (\bibinfo{year}{2023}).
\newblock
\end{thebibliography}