Combatting Dimensional Collapse in LLM Pre-Training Data via Diversified File Selection
Extracted source text · arXiv 2504.20644
\documentclass{article} % For LaTeX2e
\usepackage{iclr2025_conference,times}
% Optional math commands from https://github.com/goodfeli/dlbook_notation.
\input{math_commands.tex}
% \usepackage{hyperref}
\usepackage{url}
\usepackage{wrapfig}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
\usepackage{microtype}
\usepackage{titletoc}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{enumitem}
\usepackage{booktabs} % for professional tables
\usepackage{multirow}
% \usepackage{amsmath}
\usepackage{amsmath}
% \usepackage{cite}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{algorithm}
% \usepackage[ruled,linesnumbered]{algorithm2e}
\usepackage{colortbl}
\usepackage{xcolor} % colors
\usepackage{algorithmic}
\newtheorem{lemma}{Lemma}
\usepackage{color}
\usepackage{bbding}
\newtheorem{assumption}{Assumption}
\let\Cross\relax
\usepackage{marvosym}
% \usepackage{marvosym}
\usepackage[pagebackref=true,breaklinks=true,colorlinks=true,bookmarks=false]{hyperref}
\definecolor{deepred}{HTML}{940000}
\hypersetup{linkcolor=[rgb]{0.6,0.25,0.25}}
\hypersetup{citecolor=[rgb]{0.0,0.3,0.6}}
% \renewcommand{\Cross}{\mwchr[134]}
\title{Combatting Dimensional Collapse in LLM Pre-Training Data via Diversified File Selection}
% Authors must not appear in the submitted version. They should be hidden
% as long as the \iclrfinalcopy macro remains commented out below.
% Non-anonymous submissions will be rejected without review.
\author{%
Ziqing Fan\textsuperscript{1,2,*}, Siyuan Du\textsuperscript{2,3,*}, Shengchao Hu\textsuperscript{1,2}, Pingjie Wang\textsuperscript{1,2},
\textbf{Li Shen}\textsuperscript{4,\Letter}, \textbf{Ya Zhang}\textsuperscript{1,2},\\
\textbf{Dacheng Tao\textsuperscript{5}}, \textbf{Yanfeng Wang\textsuperscript{1,2,\Letter}} \\
\textsuperscript{1}Shanghai Jiao Tong University, China; \textsuperscript{2}Shanghai AI Laboratory, China; \textsuperscript{3}Fudan University, China \\
\textsuperscript{4}Shenzhen Campus of Sun Yat-sen University, China; \textsuperscript{5}Nanyang Technological University, Singapore \\
% {
\small
\texttt{zqfan\_knight@sjtu.edu.cn}, \texttt{dusiyuan@pjlab.org.cn},
\texttt{charles-hu@sjtu.edu.cn}, \\
\texttt{pingjiewang@sjtu.edu.cn}, \texttt{mathshenli@gmail.com}, \texttt{ya\_zhang@sjtu.edu.cn}, \\
\texttt{dacheng.tao@ntu.edu.sg}, \texttt{wangyanfeng@sjtu.edu.cn}
% }
}
% \footnote{* denotes the equal }
% \author{Ziqing Fan, \& Amelie P. Amygdale \thanks{ Use footnote for providing further information
% about author (webpage, alternative address)---\emph{not} for acknowledging
% funding agencies. Funding acknowledgements go at the end of the paper.} \\
% Department of Computer Science\\
% Cranberry-Lemon University\\
% Pittsburgh, PA 15213, USA \\
% \texttt{\{hippo,brain,jen\}@cs.cranberry-lemon.edu} \\
% \And
% Ji Q. Ren \& Yevgeny LeNet \\
% Department of Computational Neuroscience \\
% University of the Witwatersrand \\
% Joburg, South Africa \\
% \texttt{\{robot,net\}@wits.ac.za} \\
% \AND
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email}
% }
% \author{%
% Ziqing Fan\textsuperscript{1,2}, Ruipeng Zhang\textsuperscript{1,2}, Jiangchao Yao\textsuperscript{1,2}, Bo Han\textsuperscript{3}, Ya Zhang\textsuperscript{1,2}, Yanfeng Wang\textsuperscript{1,2,\Letter} \\
% \textsuperscript{1}Cooperative Medianet Innovation Center, Shanghai Jiao Tong University, \\
% \textsuperscript{2}Shanghai AI Laboratory, \textsuperscript{3}Hong Kong Baptist University \\
% \texttt{\{zqfan\_knight, zhangrp, sunarker\}@sjtu.edu.cn} \\ \texttt{bhanml@comp.hkbu.edu.hk, \{ya\_zhang, wangyanfeng\}@sjtu.edu.cn}
% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to \LaTeX{} to determine where to break
% the lines. Using \AND forces a linebreak at that point. So, if \LaTeX{}
% puts 3 of 4 authors names on the first line, and the last on the second
% line, try using \AND instead of \And before the third author name.
\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}
\iclrfinalcopy % Uncomment for camera-ready version, but NOT for submission.
\begin{document}
\maketitle
\def\thefootnote{*}\footnotetext{These authors contributed equally to this work.}\def\thefootnote{\arabic{footnote}}
\input{contents/00_abstract}
\input{contents/01_introduction}
\input{contents/02_related}
\input{contents/03_method}
\input{contents/04_experiment}
% \input{contents/07_related_work}
\input{contents/05_conclusion}
\bibliography{main}
\bibliographystyle{iclr2025_conference}
\input{contents/06_appendix}
\end{document}
%%%%% NEW MATH DEFINITIONS %%%%%
\usepackage{amsmath,amsfonts,bm}
% Mark sections of captions for referring to divisions of figures
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
% Highlight a newly defined term
\newcommand{\newterm}[1]{{\bf #1}}
% Figure reference, lower-case.
\def\figref#1{figure~\ref{#1}}
% Figure reference, capital. For start of sentence
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
% Section reference, lower-case.
\def\secref#1{section~\ref{#1}}
% Section reference, capital.
\def\Secref#1{Section~\ref{#1}}
% Reference to two sections.
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
% Reference to three sections.
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
% Reference to an equation, lower-case.
\def\eqref#1{equation~\ref{#1}}
% Reference to an equation, upper case
\def\Eqref#1{Equation~\ref{#1}}
% A raw reference to an equation---avoid using if possible
\def\plaineqref#1{\ref{#1}}
% Reference to a chapter, lower-case.
\def\chapref#1{chapter~\ref{#1}}
% Reference to an equation, upper case.
\def\Chapref#1{Chapter~\ref{#1}}
% Reference to a range of chapters
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
% Reference to an algorithm, lower-case.
\def\algref#1{algorithm~\ref{#1}}
% Reference to an algorithm, upper case.
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
% Reference to a part, lower case
\def\partref#1{part~\ref{#1}}
% Reference to a part, upper case
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
% Random variables
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
% rm is already a command, just don't name any random variables m
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
% Random vectors
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
% Elements of random vectors
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
% Random matrices
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
% Elements of random matrices
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
% Vectors
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
% Elements of vectors
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
% Matrix
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
% Tensor
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
% Graph
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
% Sets
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
% Don't use a set called E, because this would be the same as our symbol
% for expectation.
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
% Entries of a matrix
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
% entries of a tensor
% Same font as tensor, without \bm wrapper
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
% The true underlying data generating distribution
\newcommand{\pdata}{p_{\rm{data}}}
% The empirical distribution defined by the training set
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
% The model distribution
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
% Stochastic autoencoder distributions
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} % Laplace distribution
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
% Wolfram Mathworld says $L^2$ is for function spaces and $\ell^2$ is for vectors
% But then they seem to use $L^2$ for vectors throughout the site, and so does
% wikipedia.
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} % See usage in notation.tex. Chosen to match Daphne's book.
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
\begin{abstract}
Selecting high-quality pre-training data for large language models (LLMs) is crucial for enhancing their overall performance under limited computation budget, improving both training and sample efficiency.
%largely 用 primarily；BookCorpus前加个like？
Recent advancements in file selection primarily rely on using an existing or trained proxy model to assess the similarity of samples to a target domain, such as high quality sources BookCorpus and Wikipedia.
%
However, upon revisiting these methods, the domain-similarity selection criteria demonstrates a diversity dilemma, i.e.~\textit{dimensional collapse} in the feature space, improving performance on the domain-related tasks but causing severe degradation on generic performance.
%
To prevent collapse and enhance diversity, we propose a \textbf{Di}ver\textbf{S}ified \textbf{F}ile selection algorithm~(\textbf{DiSF}), which selects the most decorrelated text files in the feature space.
%
We approach this with a classical greedy algorithm to achieve more uniform eigenvalues in the feature covariance matrix of the selected texts, analyzing its approximation to the optimal solution under a formulation of $\gamma$-weakly submodular optimization problem.
%
Empirically, we establish a benchmark and conduct extensive experiments on the TinyLlama architecture with models from 120M to 1.1B parameters.
%
Evaluating across nine tasks from the Harness framework, DiSF demonstrates a significant improvement on overall performance.
%
Specifically, DiSF \emph{saves 98.5\%} of 590M training files in SlimPajama, outperforming the full-data pre-training\footnote{Full-data pre-training in this paper refers to pre-training the LLMs on all training files in SlimPajama until the specified training budget is reached.} within a 50B training budget, and achieving about \textit{1.5x training efficiency} and \textit{5x data efficiency}. Source code
is available at:~\href{https://github.com/MediaBrain-SJTU/DiSF.git}{https://github.com/MediaBrain-SJTU/DiSF}.
\end{abstract}
\section{Introduction}
Pre-trained Large Language Models (LLMs) have demonstrated remarkable capabilities~\citep{brown2020language,chowdhery2023palm,touvron2023llama}, but their training is computationally expensive, with costs increasing as model size and training data grow~\citep{cost1,carbon,cost2}.
%
For instance, training GPT-3 with 175 billion parameters is estimated to produce 552 tons of $\text{CO}_2$ emissions and consume 1,287 MWh of energy~\citep{carbon}.
%
In practice of commercial use and common academic research, training budgets such as the number of pre-trained tokens are typically predefined, determined by available devices and training time constraints~\citep{computeoptimal}.
%
To optimize the performance of LLMs within the budget, selecting high-quality pre-training data from large text corpora is essential, boosting both training and sample efficiency.
Recent innovations of selecting files for pre-training LLMs mostly rely on using an existing or trained proxy model and designing a proxy function to access the similarity to a target domain, which is regarded as high-quality data.
%
Heuristic classification~\citep{gpt3,palm} trains a binary classifier and select similar content to text domains like Books and Wikipedia~\citep{redpajama}.
%
DSIR~\citep{dsir} also targets these domains, using a hashed n-gram extractor to measure similarity.
%
QuRating~\citep{qrating} leverages GPT-3.5-turbo to train a judge model that evaluates the quality of domains like writing and education.
%
However, as shown in Figure~\ref{fig:performance}, these methods based on specific domains lead to a diversity dilemma, known as \textit{dimensional collapse} in representation learning~\citep{dc,dc1,dc2,cwd,fanfederated}. Their feature vectors of samples span a lower-dimensional subspace, indicating less diversity, improving performance in domain related tasks, such as reading comprehension (e.g., ARC~\citep{allenai:arc} and OBQA~\citep{OpenBookQA2018}), but causing severe degradation in overall performance across diverse domains, particularly in physical world knowledge tasks like PIQA~\citep{bisk2020piqa} and HellaSwag~\citep{zellers2019hellaswag}.
\begin{figure*}[t]
\vspace{-10pt}
\centering
\subfigure[Heuristic]{
\centering
\label{fig:surface_1}
\includegraphics[width=0.185\textwidth]{figs/3d_heuristic.pdf}}
%\hspace{2mm}
\subfigure[DSIR]{
\centering
\label{fig:surface_2} \includegraphics[width=0.185\textwidth]{figs/3d_dsir.pdf}}
\subfigure[QuRating-W]{
\centering
\label{fig:surface_3}
\includegraphics[width=0.185\textwidth]{figs/3d_qurating_w.pdf}}
%\hspace{2mm}
\subfigure[D4]{
\centering
\label{fig:surface_5} \includegraphics[width=0.185\textwidth]{figs/3d_d4.pdf}}
\subfigure[DISF]{
\centering
\label{fig:surface_6}
\includegraphics[width=0.185\textwidth]{figs/3d_disf.pdf}}
\vspace{-10pt}
\caption{The t-SNE~\citep{tsne} visualization of text features (normalized to the unit sphere) selected by different methods on SlimPajama. We use Contriever~\citep{contriever} to extract features. (a) and (b) show Heuristic classification and DSIR based on the Wikipedia and Book domains, while (c) depicts QuRating based on writing judgments. We visualize top 500 text features selected by their criterion, which forms a long narrow band, indicating dimensional collapse. (d) and (e) represent D4 and our DiSF. For D4, we display 500 random samples after reducing redundancy, while for DiSF, we select samples with the highest values based on \eqref{eq:proxy}. Both methods, especially DiSF, show more uniformly scattered features, indicating improved diversity.
}\label{fig:feature}
\vspace{-20pt}
\end{figure*}
\begin{wrapfigure}{r}{0.4\textwidth}
\vspace{-13pt}
\includegraphics[width=0.4\textwidth]
{figs/intro.pdf}
\vspace{-25pt}
\caption{Commonsense reasoning ability of pre-trained TinyLlama 1B using various selection methods evaluated on seven tasks of Harness.
%
DSIR uses Wikipedia and BookCorpus as high quality source and QuRating-W selects samples with writing style score.
%
All methods select 1.5\% of training files in SlimPajama and pre-train 50B tokens.
}\label{fig:performance}
\vspace{-10pt}
\end{wrapfigure}
In our paper, we revisit these algorithms by visualizing the feature representations of their selected text samples as shown in Figure~\ref{fig:feature}, and performing eigenvalue analysis on the features' covariance matrix~(see Section~\ref{sec:rethink} for details).
%
As depicted in Figure~\ref{fig:feature} (a), (b), and (c), we observe dimensional collapse, where text samples selected based on a specific domain show dominant top eigenvalues, indicating long narrow feature spaces.
%
In contrast, text samples selected using our diversified method exhibit less dominant eigenvalues, leading to greater uniformity across feature dimensions (Figure~\ref{fig:feature} (e)).
%
Recent methods, such as D4~\citep{d4} and INGENIOUS~\citep{ingenious}, recognize the importance of diversity and select informative samples by leveraging feature distances and similarity kernels, respectively.
%
However, they fall short in achieving the level of uniform representations attained by our approach as shown in Figure~\ref{fig:feature} (d) and (e) and Figure~\ref{fig:collapse}.
To prevent collapse and enhance diversity, we propose DiSF, that selects files by minimizing the Frobenius norm of the features' covariance matrix, achieving more uniform eigenvalues.
%
We approach this with the classical greedy algorithm and analyze its approximation to the optimal solution under the formulation of $\gamma$-weakly submodular optimization problem~\citep{ratio}.
%
Empirically, we establish a benchmark on the newly released and popular TinyLlama~\citep{tinyllama} architecture with models of 120M, 560M, and 1.1B parameters.
%
Extensive experiments and ablation studies, conducted across nine tasks on the Harness framework, demonstrate the superior general performance of our method compared to baselines.
%
Specifically, out of 590M training files in SlimPajama~\citep{llama,redpajama}, our DiSF selects just 1.5\% (about 9B training tokens), outperforming full-data pre-training under 50B training budget, and achieving approximately 1.5x training efficiency and 5x data efficiency.
%
In summary, our research makes three significant contributions to the field:
\begin{itemize}[leftmargin=15pt]
\vspace{-6pt}
\item We rethink recent file selection innovations in pre-training LLMs, and identify a diversity dilemma known as dimensional collapse in feature representation learning, improving performance in domain-specific tasks but causing severe degradation in overall performance (Section \ref{sec:rethink}).
\vspace{-3pt}
\item To prevent collapse and enhance diversity, we propose DiSF, which selects the most decorrelated text files using a classical greedy algorithm, and analyze its approximation to the optimal solution under the formulation of $\gamma$-weakly submodular optimization problem~(Section \ref{sec:method}).
\vspace{-3pt}
\item We established a benchmark on TinyLlama architectures and SlimPajama text corpus with evaluation on nine tasks from Harness. Extensive experiments and ablations demonstrate the superior performance of our method, with improved training and sample efficiency~(Section \ref{sec:exp}).
\end{itemize}
% \section{Related Works}
\section{Rethinking File Selection for LLM Pre-training under Budget}\label{sec:rethink}
In this section, we first introduce the definition of file selection objective, and then revisit recent file selection innovations for pre-training LLMs, with a particular focus on the diversity dilemma.
\begin{table*}[t!]
\caption{Summary of recent innovations in file selection for LLM pre-training, categorized by their proxy model, proxy function to estimate sample importance, and whether requiring a target domain.}\label{tab:related}
\vspace{2pt}
\small
\centering
% \renewcommand\arraystretch{0.01}
\setlength{\tabcolsep}{1pt}
\scalebox{0.88}{
\begin{tabular}{ c | c c c}
\toprule[2pt] \textbf {Selection Method} & \textbf {Proxy Model ($\mathrm{M}$)}& \textbf {Proxy Function ($\mathrm{F}_{\mathrm{M}}$)}& \textbf {Domain Independent}\\ \midrule
\text { Heuristic Cls.} & Trained binary classifier& Probability to target domain& \textcolor{red}{\XSolidBrush}\\
\text { DSIR~(NeurIPS'23) } & Hashed n-gram extractor& Similarity to target distribution &\textcolor{red}{\XSolidBrush}\\
\text { QuRating~(ICML'24)}& Trained judge model via GPT-3.5& Judgement score of target ability&\textcolor{red}{\XSolidBrush}\\
\text { INGENIOUS~(Emnlp'23) } &Pre-trained LLM with warming-up& Facility Location on slimilarity matrix& \textcolor[RGB]{34,139,34}{\Checkmark}\\
\text { D4~(NeurIPS'23) } &Existing text feature extractor& Distance in feature space& \textcolor[RGB]{34,139,34}{\Checkmark}\\
\text { DiSF~(Ours) }& Existing text feature extractor& Decorrelation of feature dimensions& \textcolor[RGB]{34,139,34}{\Checkmark}\\
\bottomrule[2pt]
\end{tabular}
}
\vspace{-9pt}
\end{table*}
\subsection{Problem Statement}\label{sec:define}
Given training and selection budgets $\mathcal{T}$ and $\mathcal{S}$, our objective is to select the most valuable samples $\mathbb{V}$ from a text corpus $\mathbb{D}=\{D_1,...,D_i,...,D_N\}$ collected from various domains~$D_i$~(e.g., Wikipedia or samples with high writing qualities) to optimize model weights $\mathrm{W}$ on pre-training task $\mathcal{L}$, thereby maximizing general performance $\mathcal{A}$.
%
While $\mathcal{A}$ is challenging to verify on a given LLM, it can be inferred through diverse abilities such as commonsense and problem-solving, with evaluation tools like Harness~\citep{harness}.
%
Mathematically, $\mathbb{V}$ can be obtained through selection objective as
% \subsetneqq
\begin{equation} \label{eq:statement}
\begin{aligned}
\argmax_{\mathrm{U}\subseteq \mathbb{D}} ~ & ~ \mathcal{A}(\argmin_{\mathrm{W}}\mathcal{L}(\mathrm{W}, \mathcal{T}, \mathrm{U})), \\
\text{s.t.} ~~ & |\mathrm{U}|\leq\mathbb{\mathcal{S}}.
\end{aligned}
\end{equation}
There are various options to define $\mathcal{T}$, $\mathcal{S}$, $\mathcal{A}$, and $\mathcal{L}$.
%
In this work, we define the training budget $\mathcal{T}$ as the number of pre-trained tokens, the selection budget $\mathcal{S}$ as the number of files, the pre-training objective $\mathcal{L}$ as next-word prediction on SlimPajama~\citep{redpajama,llama}, and the LLM's generic performance $\mathcal{A}$ as the evaluation across different tasks using the Harness framework.
\subsection{Recent Selection Methods}\label{sec:recent}
Analyzing the objective defined in \eqref{eq:statement}, the inner part minimizes the pre-training objective on LLM with a predefined training budget and the selected samples, while the upper level focuses on selecting the most valuable samples to maximize the LLM's generic performance within the selection budgets.
%
Directly searching for valuable samples $\mathbb{V}$ in the full corpus $\mathbb{D}$ is extremely time-consuming and expensive.
%
To reduce this cost, recent innovations in file selection for LLM pre-training mostly rely on using an existing or trained proxy
model $\mathrm{M}$ and designing a proxy function $\mathrm{F_{\mathrm{M}}}$ based on a target domain.
%
Through linking text samples $\mathrm{x}\in \mathbb{D}$ to the generic performance $\mathcal{A}$, they transform \eqref{eq:statement} into choosing the samples with the highest values of $\mathrm{F}_{\mathrm{M}}(\mathrm{x})$, as follows:
\begin{equation}\label{eq:recent}
\mathbb{V}=\text{Top}_{S} \mathrm{F}_{\mathrm{M}}(\mathrm{x}\in \mathbb{D}).
\end{equation}
As summarized in Table~\ref{tab:related}, typical Heuristic classification~\citep{gpt3,palm} trains a binary classifier to filter web data, selecting files with probabilities to a target format above a noisy, such as BookCorpus and Wikipedia~\citep{redpajama}.
%
Similarly, DSIR \citep{dsir} improves it and also treats these two domains as high-quality domains to select files for general purpose, with a hashed n-gram feature extractor to measure the similarity between the text features and the target distribution.
%
QuRating~\citep{qrating} queries GPT-3.5-turbo and trains a judge model to assess the text samples' quality of a target style, such as writing and mathematics.
%
However, as shown in Figure~\ref{fig:performance}, selection methods based on specific domains lead to a diversity dilemma, known as \textit{dimensional collapse} in representation learning~\citep{dc,dc1,dc2,dc3}, improving performance in the domain related task but causing severe degradation in overall performance across diverse tasks.
%
The recent method D4~\citep{d4} and INGENIOUS~\citep{ingenious} reduce file redundancy by leveraging feature distances and selecting informative samples based on similarity kernels respectively, which improves diversity but can not effectively mitigate dimensional collapse as ours (see both Figure~\ref{fig:feature} and Figure~\ref{fig:collapse}).
\begin{wrapfigure}{r}{0.4\textwidth}
\vspace{-12pt}
\includegraphics[width=0.4\textwidth]
{figs/eginvaluev2.pdf}
\vspace{-20pt}
\caption{The dominance score for recent methods calculated as $\frac{\sum_{i=1}^k\lambda_i}{\sum_{j=1}^d\lambda_j}$, where
$\lambda_i$ represents the $i$-th largest eigenvalue of the feature covariance matrix, and $d$ is the dimension of the feature space. We use Contriever model to extract features. We select the top 500 text samples based on their respective selection criteria. For D4, we select 500 random samples after reducing redundancy.}\label{fig:collapse}
\vspace{-10pt}
\end{wrapfigure}
\subsection{Diversity Dilemma: Dimensional Collapse}
As shown in Figure~\ref{fig:feature}, dimensional collapse occurs in the embedding space when samples are selected based on the target domains.
%
Their embedding vectors extracted by the Contriever~\citep{contriever} span a lower-dimensional subspace, indicating less diversity.
%
To quantify this, we conduct an eigenvalue analysis on the covariance matrix of selected text features under different selection methods, and visualize the dominance score of the $\text{top}_k$ eigenvalues calculated as $\frac{\sum_{i=1}^k\lambda_i}{\sum_{j=1}^d\lambda_j},$ where $\lambda_i$ is the $i$-th large eigenvalue of the covariance matrix, and $d$ is the dimension of the feature space.
%
Smaller dominance score suggests more uniform feature dimensions and richer information~\citep{dc3,dc1,cwd,dc,dc2}.
%
As demonstrated in Figure~\ref{fig:collapse}, Heuristic classification, DSIR, and QuRating with a focus on writing style (denoted as QuRating-W) exhibit significantly higher dominance scores compared to D4, INGENIOUS, and QuRating with all styles (denoted as QuRating-A), highlighting the reduced diversity caused by target domains selection criteria.
%
To address this and enhance diversity, we propose a novel diversified file selection algorithm~(DiSF) to select text samples, spanning more uniform feature dimensions in the embedding space.
%
As red line shown in Figure~\ref{fig:collapse}, compared to D4 and QuRating-A, our DiSF further reduces the dominance score, demonstrating its effectiveness in uniforming dimensions and mitigating dimensional collapse.
\section{DiSF: Diversified File Selection}\label{sec:method}
In this section, we define the diversified selection criterion of DiSF, and the selection procedures with a classical greedy algorithm in batch level, and then analyze the function from view under $\gamma$-weakly submodular optimization theories, followed with time complexity analysis.
\subsection{Method}
\textbf{Selection criterion.}~~~~Given a set of $n$ text samples $\mathrm{U}=\{\mathrm{x}_1,...,\mathrm{x}_i,...\mathrm{x}_n\}$, their text features with a standard normalization $\mathrm{Z}=\{\mathrm{z}_1,...,\mathrm{z}_i,...\mathrm{z}_n\}$ are obtained by a text feature extractor $\mathrm{M}$ as
\begin{equation}\label{eq:feature}
\mathrm{z}_i=\frac{f(\mathrm{x}_i,\mathrm{M})-\mathrm{\mu}}{\mathrm{\sigma}},
\end{equation}
where $f$ calculates the feature representations of text samples,and $\mu$ and $\sigma$ are the mean and variance of $\{f(\mathrm{x}_1,\mathrm{M}),...,f(\mathrm{x}_i,\mathrm{M}),...f(\mathrm{x}_n,\mathrm{M})\}$, respectively.
%
Then, the covariance matrix $\mathrm{C}$ is defined as
\begin{equation}\label{covar}
\mathrm{C}(\mathrm{U},\mathrm{M})=\frac{1}{n-1}\sum_{i=1}^n\mathrm{z}_i^T\mathrm{z}_i.
\end{equation}
As discussed in Section~\ref{sec:rethink}, our goal is to prevent dimensional collapse by ensuring more uniform eigenvalues in the covariance matrix of the selected samples.
%
Directly calculating and selecting based on eigenvalues are costly, but it is feasible to optimize the Frobenius norm of the covariance matrix $\|\mathrm{C}\|_F$~\citep{dc1,dc2,cwd}, as described in Lemma~\ref{lema:frobenius}:
%
\begin{lemma}
\label{lema:frobenius}
Assuming a covariance matrix $\mathrm{C}\in\mathbf{R}^{d\times d}$ computed from the features with the standard normalization, and its eigenvalues $\{\lambda_1, \lambda_2,...,\lambda_d\}$, we will have the following equality that satisfied
\begin{equation}
\sum_{i=1}^{d}{(\lambda_i-\frac{1}{d}\sum_{j=1}^{d}{\lambda_j})^2}=\|\mathrm{C}\|^2_F - d. \nonumber
\end{equation}
\end{lemma}
We provide a detailed proof in Appendix~\ref{app:lemma1} for clarity.
%
From Lemma~\ref{lema:frobenius}, it is evident that ensuring the uniformity of the eigenvalues of the covariance matrix can be translated into minimizing the Frobenius norm of the covariance matrix.
%
Thus, given any text set $\mathrm{U}\subseteq\mathbb{D}$, we evaluate its importance as $-\|\mathrm{C}(\mathrm{U},\mathrm{M})\|_F$, and define the selection objective as
\begin{equation}\label{eq:original}
\argmax_{\mathrm{U}}-\|\mathrm{C}(\mathrm{U},\mathrm{M})\|_F, ~~\textit{s.t.}~~|\mathrm{U}|\leq \mathcal{S},
\end{equation}
where $\mathcal{S}$ is the predefined selection budget.
%
Differing from typical selection algorithms that directly define the importance of individual text samples through proxy function shown in \eqref{eq:statement}, our selection objective defined in \eqref{eq:original} is calculated on a subset of samples.
% %
% To optimize text samples for the set function, we model it as a submodular optimization problem.
\begin{wrapfigure}{r}{0.45\textwidth}
\begin{minipage}{.45\textwidth}
\small
\vspace{-20pt}
\begin{algorithm}[H]
{
\small
\caption{Selection procedure of DiSF}
\label{alg:DiSF}
\small
\textbf{Input}:$(\mathbb{D}, b, \mathcal{S}, \mathrm{M})$
\begin{algorithmic}
\STATE $\mathbb{V} \leftarrow \emptyset$.
\STATE Divide $\mathbb{D}$ into batches of $b_i$ with scale $b$.
\FOR{$i = 1,\dots,\lfloor{\frac{|\mathbb{D}|}{b}}\rfloor$}
\STATE randomly select $\mathrm{x}^* \in b_i$ and $\mathrm{U}_i\leftarrow \{\mathrm{x}^*\}$.
% \vspace{-8pt}
\WHILE{$|\mathrm{U}_i|\leq \frac{b|\mathcal{S}|}{|\mathcal{D}|}$}
\STATE $b_i\leftarrow b_i\setminus \{\mathrm{x}^*\}$.
\STATE $\mathrm{x}^*=\argmax_{\mathrm{x} \in b_i} \mathrm{F}_{\mathrm{M}}^{\text{DiSF}}(\mathrm{U}_i\cup \{\mathrm{x}\})$.
\STATE $\mathrm{U}_i\leftarrow \mathrm{U}_i\cup \{\mathrm{x}^*\}$.
\ENDWHILE
\STATE $\mathbb{V}=\mathbb{V}\cup \mathrm{U}_i$.
\ENDFOR
\end{algorithmic}
{\textbf{Output}:$\mathbb{V}$}
}
\end{algorithm}
\end{minipage}
\vspace{5pt}
\end{wrapfigure}
\textbf{Selection Procedure.}~~~~To satisfy the non-negative requirement in the later analysis, we first reformulate our proxy function into a non-negative form as
\begin{equation}\label{eq:proxy}
\mathrm{F}_{\mathrm{M}}^{\text{DiSF}}(\mathrm{U})=e^{-\|\mathrm{C}(\mathrm{U},\mathrm{M})\|_F},
\end{equation}
%
where $\mathrm{U}\subseteq \mathbb{D}$.
%
We apply the classical greedy algorithm to select the most valuable samples based on our proxy function.
%
This allows us to iteratively choose the most valuable samples as follows:
\begin{equation}\label{eq:select}
\mathrm{U}\leftarrow\mathrm{U} \cup \{\argmax_{\mathrm{x} \in \mathbb{D}\setminus \mathrm{U}} \mathrm{F}_{\mathrm{M}}^{\text{DiSF}}(\mathrm{U}\cup \{\mathrm{x}\})\}.
\end{equation}
However, directly applying the greedy algorithm as shown in \eqref{eq:select} to the entire text corpus is computationally expensive, we perform the selection at the batch scale.
%
In Section~\ref{sec:ablation}, we provide an ablation study of selection scale.
%
Given a text corpus $\mathbb{D}$, selection scale $b$, selection budget $S$, and proxy model $\mathrm{M}$, the selection process is outlined in Algorithm~\ref{alg:DiSF}.
%
We first divide the text corpus into $\lfloor\frac{|D|}{b}\rfloor$ batches, where $\lfloor\cdot\rfloor$ is the round down command.
%
In each batch $b_i$, we initialize $\mathrm{U}_i$ with a randomly selected sample and remove it from $b_i$.
%
Then, we iteratively apply $(\lfloor\frac{b|\mathcal{S}|}{|\mathbb{D}|}\rfloor-1)$ times the greedy algorithm, removing the most valuable sample from $b_i$ and adding it to $\mathrm{U}_i$ based on \eqref{eq:select}.
%
Finally, the selected samples are the combination of all $\mathrm{U}_i$.
\subsection{Analysis}
\textbf{Selection Analysis.}~~~~Diversity is often effectively modeled by submodular functions~\citep{submodular,function,fedsub,diversity1,qixing1,qixing2,qixing3}, which exhibit a property of diminishing returns, i.e., the marginal gain an element brings to a subset decreases as more elements are added to that subset.
%
Mathematically, given a set $\Omega$, a set function $\mathrm{f}: 2^N \rightarrow \mathbb{R}$ is $\gamma$-weakly submodular if and only if, for any subsets $A \subseteq B \subseteq \Omega$, and a element $\mathrm{x}\in \Omega\setminus B$, the following inequality holds:
\begin{equation}
\mathrm{f}(A \cup\{\mathrm{x}\})-\mathrm{f}(A) \geq \gamma(\mathrm{f}(B \cup\{\mathrm{x}\})-\mathrm{f}(B)),
\end{equation}
where $\gamma\in$ (0,1].
%
\begin{wrapfigure}{r}{0.4\textwidth}
% \vspace{-5pt}
\includegraphics[width=0.4\textwidth]
{figs/monotone.pdf}
\vspace{-20pt}
\caption{
Proxy value, as defined in \eqref{eq:proxy}, calculated on different methods. For each method, we randomly choose samples from their selected text files with 1.5\% selection budget. All cases generally demonstrate the property of monotonicity. Moreover, our selection method achieves significantly larger proxy values, indicating much better uniformity of feature dimensions.}\label{fig:monotone}
\vspace{-5pt}
% \vspace{-15pt}
\end{wrapfigure}
A non-negative monotone $\gamma$-weakly submodular maximization problem can be solved using the classical greedy algorithm, which guarantees a $(1-e^{-\gamma})$-approximation to the optimal solution (function value on selected samples compared to optimal value)~\citep{ratio}.
%
As demonstrated in Figure~\ref{fig:monotone}, we empirically verify the monotonicity of our proxy function by increasing the number of samples randomly chosen from the selected text files of different methods.
%
Additionally, the results in Figure~\ref{fig:monotone} indicate a significantly higher proxy value for our selection method compared to others, demonstrating the superior ability to mitigate dimensional collapse.
%
Since our proxy function also aims to capture the diversity within a given set, we try to provide some insights into how our algorithm approaches the optimal solution under this formulation.
%
To provide a theoretical guarantee of the selection, we aim to establish a lower bound for the submodular ratio under any given set where the gain is positive, as most empirical verifications in Figure~\ref{fig:monotone} suggest.
%
Since the original formulation is challenging to analyze directly, we introduce two bounds on the function value.
%
One is the upper bound $\epsilon$ on gain difference, as demonstrated in Assumption~\ref{asm:difference}, and the other one is the upper bound $\mu$ on the average utility of function value as shown in Assumption~\ref{asm:max}.
%
Therefore, we prove a lower bound of the submodular ratio of $e^{-2\mu}\frac{e^{2\mu-\epsilon}-1}{e^{2\mu}-1}$, which means given assumptions ~\ref{asm:difference}, ~\ref{asm:max}, and positive gain, our function is at least $e^{-2\mu}\frac{e^{2\mu-\epsilon}-1}{e^{2\mu}-1}$-weakly submodular function. The detailed proof is provided in the Appendix~\ref{app:subanalysis}
\textbf{Time Complexity Analysis.}~~~~Given the feature dimension $d$, the time complexity of our DiSF is less than $O(|\mathcal{S}|^2bd^2)$, and we provide a detailed calculation of the results.
%
All terms in the time complexity are at most quadratic and independent of the overall dataset size, which we consider acceptable for applying to file selection for LLM pre-training.
%
Additionally, in Figure~\ref{fig:scale} of Section~\ref{sec:ablation}, we report the exact time required to select data on the benchmark dataset. In Section~\ref{sec:efficiency} and Appendix~\ref{app:efficiency}, we compare the sampling and training efficiency of our method against the baselines.
\section{Experiment}\label{sec:exp}This part introduces the experimental setup, including dataset, evaluations, model architecture, baselines, and training details in Section~\ref{sec:setup}, the main results and efficiency analysis in Section~\ref{sec:results} and Section~\ref{sec:efficiency}, and extensive ablation studies for better understanding of our DiSF in Section~\ref{sec:ablation}.
\renewcommand\arraystretch{0.9}
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\vspace{-10pt}
\caption{
Performance on seven tasks and their average with the Harness framework. In the upper part, we pre-train TinyLlama with 120M, 560M, and 1.1B parameters from scratch, with training budget of 10B tokens and selection budget of 1.5\% of total training files. In the lower part, the training budget is increased to 50B. Results impacted by the diversity dilemma, best individual task results, and the best average performance are respectively highlighted in bold blue, black, and red.}
\vspace{2pt}
\label{tb:main}
\small
\centering
\scalebox{0.93}{
\begin{tabular}{c|l|cccccccc}
\toprule[2pt]
\multirow{2}{*}[-4pt]{Model Size} & Method
& \multicolumn{8}{c}{ Pre-training TinyLlama from scratch with 10B training budget on 1.5\% selected files} \\
\cmidrule{2-10} & \#Metric& ARC-e & ARC-c & OBQA & BoolQ & WinoGrande & HellaSwag & PIQA & Avg. \\
\midrule
\multirow{8}{*}[0pt]{120M}
& Random & 37.1& 18.2& 12.8& 61.1& 49.9& 27.0& 58.7& 37.8 \\
& DSIR & 36.6& 18.2& 13.8& \color{blue}\textbf{57.6}& 51.7& 25.1& \color{blue}\textbf{54.8}& 36.9 \\
& D4 &38.0&18.1&13.6&60.1&51.8&\textbf{27.8}&57.8&38.2 \\
& QuRating-W &37.7&\textbf{18.6}&\textbf{15.0}&60.9&51.3&\color{blue}\textbf{24.3}&\color{blue}\textbf{55.6}& 37.7\\
& QuRating-A &39.1&18.3&14.2&60.7&50.8&27.4&58.4& 38.4\\
& Doremi & 36.8& 18.2& 13.8& 60.9& \textbf{52.7}& 27.3& 58.8& 38.5 \\
&{INGENIOUS} & 39.5& 18.6& 13.4& 60.2& 50.4& 27.4& 58.7& 38.3 \\
& DiSF (Ours) & \textbf{39.9}& 17.8 & 13.8& \textbf{61.7}& 51.9& 27.6& \textbf{59.5}& \color{red}\textbf{38.9} \\
\midrule
\multirow{8}{*}{560M}
&Random& 43.2& 19.9& 15.2& 59.3& 52.9& 30.4& 62.0& 40.4 \\
& DSIR & 41.8 & 19.3& 16.8& 60.5& 50.3& \color{blue}\textbf{26.0}& \color{blue}\textbf{56.0}& 38.7 \\
& D4 & 46.5& 19.5& 16.0& 60.5& \textbf{53.2}& 29.5& 61.4& 40.9 \\
& QuRating-W &42.0&\textbf{21.3}& \textbf{17.4}&59.5&\color{blue}\textbf{49.8}&28.4&\color{blue}\textbf{58.2}& 39.5\\
& QuRating-A&46.7&19.1&17.2&\textbf{61.3}&51.1&28.9&63.4& 41.1\\
&{INGENIOUS} & 45.8& 20.6& 16.0& 58.3& 51.4& 30.8& 62.5& 40.8 \\
& Doremi & 42.7&19.3&15.8&60.9&50.4& 30.0& 63.7& 40.4 \\
& DiSF (Ours) & \textbf{47.5}&21.2&16.2&58.9&51.0& \textbf{31.1}& \textbf{64.2}& \color{red}\textbf{41.4} \\
\midrule
\multirow{8}{*}{1.1B}
& Random & 44.8 &19.0 & 16.4 & 59.9& 51.3& 30.8& 64.1 & 40.9 \\
& DSIR& 45.7& 20.3& 18.6 & 59.8& 50.4& \color{blue}\textbf{27.6}& \color{blue}\textbf{58.3}& 40.1 \\
& D4 & 46.2& 19.3& 18.8& 60.2& 51.3& 30.9& 65.4& 41.7 \\
& QuRating-W &44.4&\textbf{21.4}&17.0 & 59.6&51.4&31.0&\color{blue}\textbf{60.1}& 40.7\\
& QuRating-A &47.4&20.9&\textbf{19.8}&59.1&50.2&30.1&63.3& 41.6\\
&{INGENIOUS} & 45.3& 19.6& 19.8& 60.0& 51.2& 31.2& 64.9& 41.7 \\
& Doremi & 44.7&19.7&17.6&\textbf{61.0}&51.2& 31.1& 64.6& 41.4 \\
& DiSF (Ours) &\textbf{47.7}&19.5&18.2&59.7&\textbf{52.2}& \textbf{32.3}& \textbf{65.6}& \color{red}\textbf{42.2} \\
\midrule
\multirow{2}{*}[-4pt]{Model Size} & Method
& \multicolumn{8}{c}{ Pre-training TinyLlama from scratch with 50B training budget on 1.5\% selected files} \\
\cmidrule{2-10} & \#Metric& ARC-e & ARC-c & OBQA & BoolQ & WinoGrande & HellaSwag & PIQA & Avg. \\
\midrule
\multirow{8}{*}[0pt]{120M}
&Random& 41.5& 17.6& 16.6& 56.3& 52.2& 28.4& 59.5& 38.9 \\
& DSIR& 44.2& 18.4& 17.2& \color{blue}\textbf{53.9}& \color{blue}\textbf{49.6}& \color{blue}\textbf{25.2}& \color{blue}\textbf{56.3}&37.8 \\
& D4 &42.1&19.2&17.0&58.3&52.6&28.1&60.9& 39.7 \\
& QuRating-W &41.6&18.9&16.2&59.5&51.6&27.6&\color{blue}\textbf{56.8}& 38.9\\
& QuRating-A &\textbf{46.8}&\textbf{19.7}&15.8&60.5&51.1&27.9&58.4& 40.0\\
&{INGENIOUS} & 42.2& 19.1& 16.3& 58.7& 51.2& 28.5& 61.0& 39.6 \\
& Doremi& 40.7&18.9&16.2&60.2&52.7& 28.1& \textbf{61.4}& 39.7 \\
& DiSF (Ours) & 44.3&18.3&\textbf{17.8}&\textbf{61.1}&\textbf{53.3}& \textbf{28.7}& 61.0& \color{red}\textbf{40.6} \\
\midrule
\multirow{8}{*}{560M}
& Random & 46.0& 20.9& 16.8& 59.0& 52.5& 31.6& 64.7& 41.6 \\
& DSIR & 45.9& 22.2& 18.5& 58.1& 50.7& \color{blue}\textbf{27.8}& \color{blue}\textbf{59.3}& 40.4 \\
& D4 & 47.2& 21.7& 18.2& 58.7& 52.2& 32.4& 65.3& 42.2 \\
& QuRating-W & 46.9& \textbf{23.4}& \textbf{19.5}& \color{blue}\textbf{56.5}& 52.2& \color{blue}\textbf{28.5}& \color{blue}\textbf{61.3}& 41.2\\
& QuRating-A & \textbf{48.3}& 19.2& 18.2& 58.9& 52.1& \textbf{34.1}& \textbf{67.4}& 42.6\\
&{INGENIOUS} & 46.8& 21.6& 17.0& 59.3& 52.2& 32.8& 66.5& 42.3 \\
& Doremi & 47.2& 22.4& 18.6& 59.0& 52.6& 33.1& 66.2& 42.7 \\
& DiSF (Ours) & 47.3& 22.0& 18.8& \textbf{60.8}& \textbf{52.8}& 33.5& \textbf{67.4}& \color{red}\textbf{43.2} \\
\midrule
\multirow{8}{*}{1.1B}
& Random & 51.4& 20.9& 18.2& 56.2& 51.3& 34.7& 67.3& 42.9 \\
& DSIR & 50.2& 20.6& 20.0& 54.6& 52.4& \color{blue}\textbf{30.4}& \color{blue}\textbf{64.2}& 41.8 \\
& D4 & 53.6& 22.1& 20.9&57.3& 52.9& 35.2& 67.7& 44.2 \\
& QuRating-W & \textbf{53.9}& 23.1& 20.8& 55.0& 52.7& 35.0& \color{blue}\textbf{63.3}& 43.4\\
& QuRating-A & 53.6& \textbf{23.7}& 21.2& 60.1& 51.6& 36.5& 67.0& 44.8\\
&{INGENIOUS} & 51.7& 21.6& 22.2& 56.9& 51.7& 36.8& 67.9& 44.1 \\
& Doremi & 51.2& 21.9& \textbf{21.8}& 58.1& \textbf{54.3}& 36.6& 67.8& 44.5 \\
& DiSF (Ours) & 51.8& 22.7& 20.0& \textbf{62.0}& 53.5& \textbf{37.3}& \textbf{69.0}& \color{red}\textbf{45.2} \\
\bottomrule[2pt]
\end{tabular}
}
\vspace{-10pt}
\end{table}
\subsection{Setup}\label{sec:setup}\textbf{Dataset and evaluation.}~~~~Following many prior works~\citep{llama,tinyllama,qrating,doremi}, we employ \textbf{SlimPajama}~\citep{llama,redpajama} as the text corpus, which is specifically curated for pre-training LLMs.
%
All selections are performed on about 590M training files of SlimPajama, processed with Llama tokenizer~\citep{llama}.
%
Notably, QuRating~\citep{qrating} provides judgments on various properties of text samples in SlimPajama using a judge model trained based on GPT-3.5-turbo, which can be directly utilized in our implement.
%
To capture the diversity dilemma and evaluate generic performance of pre-trained LLMs, we use seven commonsense reasoning tasks from the popular framework \textbf{Harness}~\citep{harness}, including four reading comprehension tasks~(\textbf{ARC-e}, \textbf{ARC-c}~\citep{allenai:arc}, \textbf{OBQA}~\citep{OpenBookQA2018}, and \textbf{BoolQ}~\citep{clark2019boolq}), and three physical world knowledge tasks~(\textbf{PIQA}~\citep{bisk2020piqa}, \textbf{HellaSwag}~\citep{zellers2019hellaswag}, and \textbf{WinoGrande}~\citep{sakaguchi2021winogrande}).
%
Besides, we also employ another two tasks \textbf{MMLU}~\citep{MMLU}, and \textbf{BBH}~\citep{bbh} from Harness to evaluate the problem-solving capabilities for further clarify of our effectiveness.
%
See Section~\ref{sec:related} for detailed introduction of the dataset and tasks.
\vspace{3pt}
\textbf{Model architecture.}~~~~As for the model, we adopt \textbf{Tinyllama} architecture~\citep{tinyllama} with 120M, 560M, and 1.1B parameters.
%
Thanks to FlashAttention~\citep{flash} and Lit-GPT~\citep{litgpt}, all experiments can be conducted on NVIDIA GeForce RTX 4090 GPUs with 24GB memory, which is feasible for general academic research.
%
All experiments and selection are implemented by PyTorch~\citep{pytorch} on platforms with 8 GPUs and 64 CPUs.
%
Except for Tinyllama, we also adopt OPT~\citep{opt} and Pythia~\citep{pythia} to verify the scalability of our method on model architecture as ablation study shown in Section~\ref{sec:ablation}.
%
For feature extraction, we utilize the Contriever~\citep{contriever} with approximately 110M parameters to calculate feature representations of the text samples as defined in \eqref{eq:feature}.
%
We also experiment with other pre-trained models as feature extractors, including the text encoder of CLIP~\citep{clip} and GPT-2~\citep{gpt2}, as discussed in the ablation study of Section~\ref{sec:ablation}.
%
See Appendix~\ref{app:model} for detailed introduction of model structures, as well as their training times.
\vspace{6pt}
\textbf{Baselines.}~~~~We compare our DiSF with \textbf{Random} selection and existing file selection methods for LLM pre-training, including \textbf{DSIR}~\citep{dsir}, QuRating~\citep{qrating}, \textbf{INGENIOUS}~\citep{ingenious}, and \textbf{D4}~\citep{d4}.
%修改
Since DSIR improves on Heuristic classification~\citep{gpt3,palm}, we present only DSIR results based on Wikipedia and Books.
%
For QuRating, based on the top judgment values, we select text samples for writing style (denoted as \textbf{QuRating-W}) and uniformly select across all styles (denoted as \textbf{QuRating-A}).
%
{For INGENIOUS, we utilize Contriever to extract features, which is more efficient than warmed-up model, as analyzed in Appendix~\ref{app:INGENIOUS}.}
%
Additionally, for a comprehensive comparison, we include \textbf{Doremi}~\citep{doremi}, a recently proposed method that produces weights for pre-training on multiple text domains.
%
We use the weights as the selection ratio of text samples in different domains in our experiment.
%
Notably, in our ablation studies, we also compare our method with \textbf{Full Data} pre-training, which refers to pre-training the LLMs on all training files in SlimPajama until the specified training budget is reached.
%
See Appendix~\ref{app:baseline} for detailed implementation of the baselines.
\vspace{9pt}
\textbf{Pre-training details.}~~~~We follow all settings in TinyLlama~\citep{tinyllama}.
%
The optimizer is AdamW~\citep{adamw}, setting parameters $\beta_1$ at 0.9 and $\beta_2$ at 0.95.
%
We adopt the cosine learning rate schedule with a maximum learning rate of 4e-4 and the minimum of 4e-5, the batch size of 2M tokens, the weight decay of 0.1, and the gradient clipping threshold of 1.
%
The training budgets are 10B and 50B tokens, with 1.5\% selection budget of SlimPajama's training files.
%
Note that, we choose to report performance with a 1.5\% selection budget, since it achieves comparable performance compared to Full Data pre-training under 50B pre-training budget on TinyLlama 1.1B.
%
Unless otherwise specified, the selection scale $b$ is set to 1024.
%
See Appendix~\ref{app:model} for more details.
\vspace{15pt}
\subsection{Main Results}\label{sec:results}
\textbf{Performance on commonsense reasoning tasks.}~~~~As shown in Table~\ref{tb:main}, selection methods based on a target domain such as DSIR, D4, and QuRating-W improve performance on reading comprehension tasks like ARC-c and OBQA, but suffer significant declines on physical world knowledge tasks especially HellaSwag and PIQA (highlighted in blue), revealing the diversity dilemma.
%
In contrast, Doremi, D4, QuRating-A, and our DiSF, which select samples from multiple text domains, achieve competitive results on OBQA and ARC-c while significantly improving overall performance across the remaining tasks.
%
This highlights the critical importance of diversity in pre-training LLMs.
%
Notably, our DiSF outperforms all baselines in terms of average performance across the seven tasks, with an average improvement of 2.5\% compared to DSIR.
%
Furthermore, with increasing training budget, the improvement on DSIR becomes more pronounced, rising from 2.1\% to 3.4\% on TinyLlama 1.1B.
\begin{wraptable}{rt!}{0.4\textwidth}
\vspace{-22pt}
\setlength{\tabcolsep}{2pt}
\centering
\caption{Problem-solving performance on MMLU (5 shot) and BBH (3 shot) of TinyLlama 1.1B with 1.5\% selection budget and 50B pre-training budget.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|cc}
\toprule[2pt]
Method & MMLU(5 shot) & BBH(3 shot) \\
\midrule
DSIR & $22.9_{\pm 0.4}$&$18.5_{\pm 0.4}$\\
QuRating-W& $23.1_{\pm 0.2}$&$18.7_{\pm 0.4}$\\
QuRating-A& $24.5_{\pm 0.4}$&$20.2_{\pm 0.4}$\\
D4 & $24.3_{\pm 0.5}$&$20.8_{\pm 0.4}$\\
Doremi & $23.1_{\pm 0.5}$&$20.3_{\pm 0.4}$\\
DiSF (Ours)& $\textbf{25.4}_{\pm 0.4}$&$\textbf{21.2}_{\pm 0.4}$\\
\bottomrule[2pt]
\end{tabular}
% }
\label{tab:problemsolving}
\vspace{-10pt}
\end{wraptable}
\textbf{Scalability on model size.}~~~~To verify the scalability of selection methods across different model sizes, we use the TinyLlama architecture with models of 120M, 560M, and 1.1B parameters.
%
As shown in Table~\ref{tb:main}, our DiSF consistently outperforms all baselines, demonstrating strong scalability with increasing model size.
%
Notably, as the model scales from 120M to 1.1B, DiSF's improvement over DSIR grows from 2.8\% to 3.4\%, suggesting even greater effectiveness for larger LLMs.
\textbf{Performance on problem-solving tasks.}~~~~
As shown in Table~\ref{tab:problemsolving}, we further verify the performance of pre-trained TinyLlama 1.1B on two additional problem-solving tasks, MMLU and BBH, using various selection methods.
%
Except for better commonsense abilities, results of our DiSF shown in Table~\ref{tab:problemsolving}, demonstrate a better problem-solving capability, outperforming all other baselines.
%
Specifically, on MMLU and BBH, our DiSF respectively achieves 2.5\% and 2.7\% improvement compared to DSIR.
\begin{figure*}[t!]
\vspace{-10pt}
\centering
\subfigure{
\centering
\includegraphics[width=0.31\textwidth]{figs/sample_120M.pdf}}
\centering
\subfigure{
\centering
\includegraphics[width=0.31\textwidth]{figs/sample_560M.pdf}}
\subfigure{
\centering
\includegraphics[width=0.31\textwidth]{figs/sample_1b.pdf}}
\vspace{-0.6cm}
\caption{Average performance of TinyLlama pre-trained after 50B tokens on files selected by our DiSF with varying selection budgets compared to Full Data pre-training on SlimPajama.}
\label{fig:sample}
\vspace{-0.2cm}
\end{figure*}
\begin{figure*}[t!]
\vspace{-10pt}
\hspace{10pt}
\centering
\subfigure{
\centering
\includegraphics[width=0.31\textwidth]{figs/120m_smooth.pdf}}
\centering
\subfigure{
\centering
\includegraphics[width=0.31\textwidth]{figs/560m_smooth.pdf}}
\subfigure{
\centering
\includegraphics[width=0.31\textwidth]{figs/1b_smooth.pdf}}
\vspace{-0.4cm}
\caption{Performance of TinyLlama pre-trained under our DiSF with 1.5\% selection budget compared to Full Data pre-training on SlimPajama, as the pre-trained tokens (training budget) increases.}
\label{fig:training}
\vspace{-0.4cm}
\end{figure*}
\begin{wrapfigure}{r}{0.45\textwidth}
\vspace{-15pt}
\includegraphics[width=0.43\textwidth]
{figs/batch.pdf}
\vspace{-15pt}
\caption{
Performance of pre-trained TinyLlama under different selection scales ($\log_2{b}$). With 1.5\% selection budget and 50B training budget, we identify an ideal point with acceptable computational cost and near-optimal performance, marked by a red star.}\label{fig:scale}
\vspace{-10pt}
\end{wrapfigure}
\subsection{Efficiency Analysis}\label{sec:efficiency}
In this section, we try to analyze how many samples and computations we can save within the training budget, compared to Full Data pre-training, that is \textit{data efficiency} and \textit{training efficiency}.
%
Please note that, Full Data pre-training refers to pre-training the LLMs on all SlimPajama's training files until the specified training budget is reached.
%
In Figure~\ref{fig:sample}, we demonstrate the average commonsense performance of TinyLlama with 120M, 560M, and 1.1B parameters, pre-trained after 50B tokens on files selected by DiSF with varying selection budgets compared to Full Data.
%
It is evident that to achieve the equivalent or superior performance compared to Full Data with 50B training budget, we only need to select about 20\% of the pre-trained tokens (1.5\% of SlimPajama's training files), achieving at least \textit{5x data efficiency}.
%
In Figure~\ref{fig:training}, we show the curves of averaged commonsense reasoning performance during the pre-training under our DiSF with 1.5\% selection budget compared to Full Data.
%
It can be seen that, when achieving the performance of Full Data pre-trained with 50B tokens, our methods only need 27B, 36B, and 40B tokens, respectively on TinyLlama with 120M, 560M, and 1.1B parameters, achieving an average of \textit{1.5x training efficiency}.
\subsection{Ablation Study}\label{sec:ablation}
\begin{wraptable}{rt!}{0.45\textwidth}
\vspace{-20pt}
\setlength{\tabcolsep}{4pt}
\centering
\caption{Ablation study of our DiSF with different model architectures compared to other selection methods, respectively using TinyLlama with 1.1B parameters, Pythia with 1B parameters and OPT with 1.3B parameters.
}
\vspace{2pt}
\renewcommand\arraystretch{0.85}
% \small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|ccc}
\toprule[2pt]
Method & TinyLlama & Pythia & OPT \\
\midrule
DSIR & 41.8&41.4&43.1 \\
D4 & 44.2&43.9&44.7 \\
QuRating-A & 44.8&43.6&45.1\\
Doremi & 44.5&43.7&44.9\\
Ours & \textbf{45.2}&\textbf{44.2}&\textbf{45.5} \\
\bottomrule[2pt]
\end{tabular}
% }
\label{tab:modelsize}
\vspace{-20pt}
\end{wraptable}
\textbf{Selection budget.}~~~~This part analyzes the performance of pre-trained LLM with different selection budgets when using our DiSF.
%
We conduct the ablation on TinyLlama 120M with 50B training budget and present its average commonsense reasoning performance.
%
As shown in Figure \ref{fig:selectrate}, the performance initially rises rapidly with increasing selection ratios, reaching a peak of 41.2 with 3\% of total training files~(about 20B tokens).
%
Once the selected samples exceed 3\%, the performance begins to decline and converge to the performance of Full Data pre-training.
%
This insight may help choose selection ratios and enhance understanding of our selection algorithm.
\textbf{Model architecture.}~~~~To further verify performance across different architectures, we additionally adopt two large language models: Pythia~\citep{pythia} with 1B parameters and OPT~\citep{opt} with 1.3B parameters.
%
As shown in Table~\ref{tab:modelsize}, we compare the average commonsense reasoning performance of our DiSF with baselines, including DSIR, D4, QuRating-A, and Doremi, under the same training budget of 50B pre-trained tokens.
%
The results demonstrate that our DiSF consistently outperforms the baselines, highlighting its effectiveness and scalability across different model architectures.
\begin{wraptable}{rt!}{0.45\textwidth}
\vspace{-22pt}
\setlength{\tabcolsep}{4pt}
\centering
\caption{Ablation study of our DiSF with different feature extractor. We show results, respectively using Contriever with 110M parameters, text encoder of CLIP with 70M parameters and GPT-2 with 117M parameters.
}
\vspace{2pt}
\renewcommand\arraystretch{0.85}
% \small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|ccc}
\toprule[2pt]
TinyLlama& CLIP & Contriever& GPT-2 \\
\midrule
120 M & 39.5 & \textbf{40.6}& 40.0\\
560 M & 42.1 & \textbf{43.1}& 42.3\\
1.1 B & 43.9 & \textbf{45.2}& 44.4 \\
\bottomrule[2pt]
\end{tabular}
% }
\label{tab:extractor}
\vspace{-0.2cm}
\end{wraptable}
\textbf{Feature extractor.}~~~~In our experiments, we utilize a proxy model named Contriever~\citep{contriever}, with about 110M parameters, as feature extractor to provide the embedding space for selected text samples.
%
We also try two other pre-trained models: the text encoder of CLIP~\citep{clip} with about 70M parameters and GPT-2~\citep{gpt2} with about 117M parameters.
%
As shown in Table \ref{tab:extractor}, CLIP fails to deliver satisfactory performance, because it struggles with longer text sequences.
%
GPT-2 does not outperform the Contriever model because it is optimized for autoregressive text generation rather than producing meaningful sentence representations, whereas Contriever, specifically designed for measuring text similarity, excels at capturing text diversity.
\begin{wrapfigure}{rt!}{0.45\textwidth}
\vspace{-15pt}
\includegraphics[width=0.45\textwidth]
{figs/ratio.pdf}
\vspace{-25pt}
\caption{Average commonsense reasoning performance of TinyLlama 120M pre-trained under DiSF with different selection budget and 50B training budget. Blue and grey lines respectively denote Full Data pre-training and random selection with 1.5\% selection budget.
}\label{fig:selectrate}
\vspace{-10pt}
\end{wrapfigure}
\textbf{Selection scale.}~~~~To manage the computational cost of file selection, we apply submodular optimization with classical greedy algorithm at the batch scale, as shown in Algorithm~\ref{alg:DiSF}, introducing the hyper-parameter, selection scale $b$.
%
The computational complexity of the selection process, defined in \eqref{eq:select}, is $\mathcal{O}(\frac{b}{\mathcal{|S|}^2})$, where the larger $b$ increases the computational burden.
%
As ablation on $b$ shown in Figure~\ref{fig:scale}, a larger batch size significantly raises computational costs, while a smaller batch size fails to select sufficiently diversified files.
%
With a fixed selection budget of 1.5\%, training budget of 50B tokens, and Contriever as the feature extractor, we identified an ideal point of 1024, marked by a red star, which balances computational time and near-optimal performance.
%
This insight may provide valuable guidance for choosing an appropriate selection scale in other settings.
\section{Conclusion}
In this work, we revisit recent innovations in file selection for pre-training large language models (LLMs) and identify a diversity dilemma: dimensional collapse, where performance improves on specific tasks but degrades overall across diverse tasks.
%
To address this, we propose a novel Diversified File selection method (DiSF) which selects decorrelated text files in the embedding space to enhance diversity.
%
DiSF achieves more uniform eigenvalues of the feature covariance matrix by minimizing its Frobenius norm and solve it with a greedy algorithm.
%
We analyze its time complexity and approximation to optimal solution under $\gamma$-weakly submodular optimization, and establish a benchmark with TinyLlama architecture, evaluating performance across nine tasks from the Harness framework.
%
Extensive experiments and ablation studies demonstrate the critical role of diversity in file selection for LLM pre-training and showcase DiSF's superior effectiveness and efficiency.
\section{Acknowledgement}
The work is supported by the National Key R$\&$D Program of China (No. 2022ZD0160702), STCSM (No. 22511105700, No. 21DZ1100100), 111 plan (No. BP0719010), National Natural Science Foundation of China (No. 62306178), Shenzhen Basic Research Project (Natural Science Foundation) Basic Research Key Project (NO. JCYJ20241202124430041), CCF-Baidu Open fund (NO. CCF-Baidu202413), and the National Research Foundation, Singapore, under its NRF Professorship Award No. NRF-P2024-001. Ziqing Fan is partially supported by Wu Wen Jun Honorary Doctoral Scholarship, AI Institute, Shanghai Jiao Tong University.
\newpage
\appendix
\section{Appendix}
In the Appendix, we provide additional information including dataset, baselines, model and training details, and the proof of Lemma \ref{lema:frobenius}. As shown in the following, to help readers read, we provide a table of contents of the full paper.
\tableofcontents
\subsection{Related Work}\label{sec:related}
\textbf{Efficient File Selection}~~~~Pre-trained Large Language Models (LLMs) have exhibited exceptional performance across a wide range of downstream tasks~\citep{jiangshuyang,yerui1,yerui2,yerui3,wang1,fan1,tang}. However, their training process is computationally intensive, with costs escalating significantly as both model size and training data volume increase~\citep{cost1,carbon,cost2}.
%
To maximize the performance of LLMs within constrained budgets, it is crucial to curate high-quality pre-training data from extensive text corpora, as this enhances both training efficiency and sample effectiveness.
%
Recent advancements in pre-training file selection for LLMs primarily rely on leveraging existing or trained proxy models, coupled with proxy functions designed to assess the similarity of data to a target domain, which is typically considered high-quality. For instance, heuristic classification methods~\citep{gpt3,palm} employ binary classifiers to filter content resembling text from domains such as Books and Wikipedia~\citep{redpajama}. Similarly, DSIR~\citep{dsir} focuses on these domains by utilizing a hashed n-gram extractor to quantify similarity. Another approach, QuRating~\citep{qrating}, utilizes GPT-3.5-turbo to train a judge model that evaluates the quality of domains like writing and education.
%
Despite these innovations, methods anchored to specific domains face a diversity dilemma, akin to the phenomenon of \textit{dimensional collapse} observed in representation learning~\citep{dc,dc1,dc2,cwd,fanfederated}. These approaches often result in feature vectors that span a lower-dimensional subspace, reflecting reduced diversity. While this may enhance performance in domain-specific tasks such as reading comprehension, it leads to significant performance degradation across broader, more diverse domains, particularly in tasks requiring physical world knowledge, such as PIQA~\citep{bisk2020piqa} and HellaSwag~\citep{zellers2019hellaswag}.
%
To tackle this issue and promote greater diversity, we introduce a novel diversified file selection algorithm, termed DiSF, designed to curate text samples that span a more uniform distribution of feature dimensions within the embedding space.
\textbf{Submodular optimization.}~~~~Submodularity is a property of functions defined on a set $\Omega$ that exhibit diminishing returns.
%
Submodular functions like facility location, log determinant, and graph cut~\citep{location1,function,location2,gc,ld} are widely recognized for effectively modeling diversity~\citep{submodular,active1,fedsub,diversity1}.
%~\citep{}
Moreover, submodular optimization has achieved significant success in various fields, such as summarization~\citep{sum1,sum2}, curriculum learning~\citep{fedsub,curri}, active learning~\citep{active1,active2}, and subset selection~\citep{subset-r1,subset-r2}, where selecting diverse subsets is crucial.
%
Given a set $\Omega$, a set function $\mathrm{f}: 2^N \rightarrow \mathbb{R}$ is $\gamma$-weakly submodular if and only if, for any subsets $A \subseteq B \subseteq \Omega$, and a element $\mathrm{x}\in \Omega\setminus B$, the following inequality holds:
\begin{equation}\label{eq:ratio}
\mathrm{f}(A \cup\{\mathrm{x}\})-\mathrm{f}(A) \geq \gamma(\mathrm{f}(B \cup\{\mathrm{x}\})-\mathrm{f}(B)),
\end{equation}
where $\gamma\in$ (0,1].
%
When $\gamma=1$~\citep{pessi,ratio}, the function is submodular function.
%
A non-negative monotone $\gamma$-weakly submodular maximization problem can be solved using the classical greedy algorithm, which guarantees a $(1-e^{-\gamma})$-approximation.
%
In this work, our function aims to evaluate diversity, making it well suited to be verified under this formulation.
\subsection{Dataset:SlimPajama}\label{app:dataset}
SlimPajama is a high-quality text corpus specifically created for pre-training large language models. This corpus, derived from RedPajama \citep{redpajama}, underwent additional cleaning and deduplication processes. The original RedPajama corpus, an open-source research project, was designed to replicate the pretraining data of Llama \citep{llama} and contains over 1.2 trillion tokens. After extensive filtering to remove low-quality and duplicate content, SlimPajama retains only 50\% of the original tokens from RedPajama.
%
Notably, compared to domains like ArXiv, GitHub, and StackExchange, Heuristic classification and DSIR tend to select a higher proportion of files from Books and Wikipedia, and Qurating-W also favors the Books domain.
%
This bias toward specific domains may explain why these methods encounter dimensional collapse and performance degradation in overall task performance.
\begin{wraptable}{rt!}{0.5\textwidth}
\vspace{-15pt}
\setlength{\tabcolsep}{2pt}
\centering
\caption{Commonsense performance of INGENIOUS when using Warmed-up model (INGENIOUS-W) and Contriever (INGENIOUS-C) with 50B training budget.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|cc}
\toprule[2pt]
Method & TinyLlama-120M & TinyLlama-1.1B \\
\midrule
INGENIOUS-W & 39.0&39.6\\
INGENIOUS-C& 43.2&44.1\\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:iggenious}
\vspace{-10pt}
\end{wraptable}
\subsubsection{Evaluation}\label{app:eval}
\textbf{Harness evaluation framework.}~~~~To demonstrate the diversity dilemma and evaluate the general performance of the pre-trained LLMs, we utilize seven commonsense reasoning tasks from the widely recognized evaluation framework, Harness \citep{harness}, including four reading comprehension tasks:
\textbf{OBQA} \citep{OpenBookQA2018}: Inspired by open book exams, this task tests the ability to comprehend and apply knowledge similarly to human understanding;
%
\textbf{ARC-e and ARC-c} \citep{allenai:arc}: 7,787 multiple-choice science questions at a grade-school level, divided into an easy set (ARC-e) and a challenge set (ARC-c);
%
\textbf{BoolQ} \citep{clark2019boolq}: A reading comprehension task that focuses on naturally occurring yes/no questions,
%
and three physical world knowledge tasks:
\textbf{HellaSwag} \citep{zellers2019hellaswag}: A collection to assess physically situated commonsense reasoning capabilities;
%
\textbf{WinoGrande} \citep{sakaguchi2021winogrande}: An expansion of the Winograd Schema Challenge (WSC) with increased scale and complexity;
%
\textbf{PIQA} \citep{bisk2020piqa}: A task to measure the understanding and reasoning about physical interactions in the real world.
%
To further verify our effectiveness, we also employ two problem-solving tasks from Harness:
%
\textbf{MMLU} \citep{MMLU}: A task to measure the world knowledge and problem-solving capabilities across various subjects;
%
\textbf{BBH} \citep{bbh}: A subset of 23 challenging tasks from the BIG-Bench benchmark~\citep{bigbench} to measure the ability of complex instruction following.
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\caption{Commonsense performance when pre-trained on SlimPajama with Open-Llama 3B, 1.5\% selection ratio and 10B training budget.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|cccccc}
\toprule[2pt]
Method & Random & DSIR& QuRating-W& QuRating-A& DiSF (Ours) \\
\midrule
Commonsense ability& 43.1& 41.9& 42.6&43.6 &44.4\\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:3b}
\vspace{-10pt}
\end{table}
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\caption{Commonsense performance when pre-trained on both StarcoderData, and SlimPajama with TinyLlama 1.1B, 1.5\% selection ratio and 10B training budget.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|ccc}
\toprule[2pt]
Method & Random & DSIR&DiSF (Ours) \\
\midrule
Commonsense ability& 40.3&40.0& \textbf{41.4}\\
bbh & 12.6 & 12.5 & \textbf{13.3}\\
mmlu & 23.0 & 22.1 & \textbf{23.7}\\
code\_x\_glue & 0.80 & 0.59 & \textbf{0.86}\\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:slimstar}
\vspace{-10pt}
\end{table}
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\caption{Comparison on sample efficiency pre-trained after 50B tokens. We record selected tokens to achieve the performance of Full Data pre-trained with 50B tokens. The ablation interval is 0.5\%.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|cccccc}
\toprule[2pt]
Setting & Full-Data & DSIR& QuRating-W& QuRating-A& D4& DiSF (Ours) \\
\midrule
TinyLlama-120M& 1.00&0.60& 0.40&0.15& 0.20&0.15\\
TinyLlama-560M& 1.00&0.80& 0.45&0.20& 0.25&0.20\\
TinyLlama-1.1B& 1.00&0.80& 0.40&0.25& 0.25&0.20\\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:sample}
\vspace{-10pt}
\end{table}
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\caption{Comparison on training efficiency pre-trained with 1.5\% selection ratio. We record pre-trained tokens to achieve the performance of Full Data pre-trained with 50B tokens. - denotes the method can not reach that performance. The ablation interval is 1B.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|cccccc}
\toprule[2pt]
Setting & Full-Data & DSIR& QuRating-W& QuRating-A& D4& DiSF (Ours) \\
\midrule
TinyLlama-120M& 50B& - & - &32B& 36B&27B\\
TinyLlama-560M& 50B& - & - &46B& 47B&36B\\
TinyLlama-1.1B& 50B& - & - &52B& 56B&40B\\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:training}
\vspace{-10pt}
\end{table}
\subsection{Baselines}\label{app:baseline}
\subsubsection{DSIR.} DSIR~\citep{dsir} treats Books and Wikipedia as high-quality targets for file selection, employing a hashed n-gram feature extractor to measure the similarity between the text features and the target distribution.
%
In our experiments, following the selection procedures outlined in DSIR, we calculate importance scores using raw data (SlimPajama) and target data (Wikipedia and Books) in an n-gram feature space. The importance weights are then applied to resample a subset of the raw dataset. As for files in Wikipedia and Books domains, we proportionally integrate into the selected dataset.
\subsubsection{QuRating.} QuRating~\citep{qrating} queries GPT-3.5-turbo to train a judge model, that assess the specific quality of text samples, including four criteria: writing style, required expertise, facts \& trivia, and educational value. In this paper, for Qurating-W, we select samples with the highest scores for writing style, while for QuRating-A, we proportionally select top-scoring samples across all four criteria.
\subsubsection{Doremi.} Doremi~\citep{doremi}, a recently proposed method that produces domain weights for pre-training on multiple text domains.
%
In our experiments, we follow the domain weights calculation process of Doremi, using the domain weights from the initial data distribution as an initial reference to train a small reference model on the SlimPajama dataset.
%
We then leverage this reference model to guide the training of a small proxy model, designed to generate domain weights.
%
Finally, these domain weights are employed as the random selection ratio for text domains to construct selected dataset.
\subsubsection{INGENIOUS}\label{app:INGENIOUS}
INGENIOUS~\citep{ingenious} extracts features using a model after a warm-up phase and employs Facility Location~\citep{facility} to design a proxy function for feature importance, which measures the similarity between samples in the embedding space.
%
Notably, as shown in Table~\ref{apptab:iggenious}, we observed that INGENIOUS, when using a warmed-up model for feature extraction (INGENIOUS-W), does not achieve satisfactory performance with our selected feature extractor, Contriever (INGENIOUS-C).
%
Although INGENIOUS-C performs competitively compared to Random, DSIR, QuRating-W, and QuRating-A, our method consistently achieves the best performance across all settings.
\subsubsection{D4.} The recent method D4~\citep{d4} notices the importance of diversified selection, involving SemDeDup~\citep{semdedup} and Prototypicality~\citep{proto} to reduce file redundancy, but can not achieve satisfactory uniform representations as ours.
%
In this paper, we sequentially applied the SemDeDup and Prototypicality methods to filter the data, controlling the filtering ratios of these two steps to be $R_{\text{dedup}}=0.75$ and $R_{\text{proto}}=0.02$, respectively.
\begin{table}[ht!]
\renewcommand{\arraystretch}{1.1}
\centering
\caption{Model structure and training details of pre-training TinyLlama 120M.}
\vspace{0.1cm}
\label{tab:param120m}
\begin{tabular}{ll}
\hline
Parameter name & Value \\
\hline
Parameter number& 121,129,728\\
Hidden size& 768 \\
Intermediate Hidden Size & 2048 \\
Context Len & 2048 \\
Heads & 12 \\
Layers & 12 \\
Vocab size& 32000 \\
Minimum learning rate& 4e-5 \\
Maximum learning rate& 4e-4 \\
Optimizer& AdamW \\
$\beta_1$ of optimizer& 0.9 \\
$\beta_2$ of optimizer& 0.95 \\
Warmup steps& 2000 \\
Batch size& 2M tokens \\
Weight decay& 0.1 \\
Activation function& SwiGLU \\
Gradient clipping threshold& 1.0 \\
Platform& 8 NVIDIA GeForce RTX 4090 GPUs \\
Training times on 10B tokens& about 0.2 days \\
Training times on 50B tokens& about 1 days \\
\hline
\end{tabular}
\end{table}
\begin{table}[ht!]
\renewcommand{\arraystretch}{1.1}
\centering
\caption{Model structure and training details of pre-training TinyLlama 560M.}
\vspace{0.1cm}
\label{tab:param560m}
\begin{tabular}{ll}
\hline
Parameter name & Value \\
\hline
Parameter number& 561,072,128\\
Hidden size& 2048 \\
Intermediate Hidden Size & 2048 \\
Context Len & 2048 \\
Heads & 16 \\
Layers & 20 \\
Vocab size& 32000 \\
Minimum learning rate& 4e-5 \\
Maximum learning rate& 4e-4 \\
Optimizer& AdamW \\
$\beta_1$ of optimizer& 0.9 \\
$\beta_2$ of optimizer& 0.95 \\
Warmup steps& 2000 \\
Batch size& 2M tokens \\
Weight decay& 0.1 \\
Activation function& SwiGLU \\
Gradient clipping threshold& 1.0 \\
Platform& 8 NVIDIA GeForce RTX 4090 GPUs \\
Training times on 10B tokens& about 0.9 days \\
Training times on 50B tokens& about 4.5 days \\
\hline
\end{tabular}
\end{table}
\begin{table}[ht!]
\renewcommand{\arraystretch}{1.1}
\centering
\caption{Model structure and training details of pre-training TinyLlama 1.1B.}
\vspace{0.1cm}
\label{tab:param1.1B}
\begin{tabular}{ll}
\hline
Parameter name & Value \\
\hline
Parameter number& 1,100,048,384\\
Hidden size& 2048 \\
Intermediate Hidden Size & 5632 \\
Context Len & 2048 \\
Heads & 32 \\
Layers & 22 \\
Vocab size& 32000 \\
Minimum learning rate& 4e-5 \\
Maximum learning rate& 4e-4 \\
Optimizer& AdamW \\
$\beta_1$ of optimizer& 0.9 \\
$\beta_2$ of optimizer& 0.95 \\
Warmup steps& 2000 \\
Batch size& 2M tokens \\
Weight decay& 0.1 \\
Activation function& SwiGLU \\
Gradient clipping threshold& 1.0 \\
Platform& 8 NVIDIA GeForce RTX 4090 GPUs \\
Training times on 10B tokens& about 1.7 days \\
Training times on 50B tokens& about 8.5 days \\
\hline
\end{tabular}
\end{table}
\subsection{Model and training details}\label{app:model}
For better clarity and reproducibility, we provide the model structures and training details for pretraining TinyLlama with 120M, 560M, and 1.1B parameters, as shown in Tables~\ref{tab:param120m}, \ref{tab:param560m}, and \ref{tab:param1.1B}, respectively. These tables detail the configurations used in our experiments, facilitating easier replication of our results. Please note that our goal in this paper is not to optimize all hyper-parameters for the best LLM, but rather to compare selection methods under fair and reasonable conditions.
\subsection{More analysis}
\subsubsection{Performance on larger dataset and model scale} \label{app:larger}
To evaluate the effectiveness of our selection method on larger models and datasets, we 1) pre-train Open-Llama 3B~\citep{openllama} on SlimPajama, and 2) TinyLlama-1.1B on both SlimPajama and StarcoderData~\citep{starcoder}. StarcoderData is a dataset created for code generation, containing approximately 210M files (500GB of data). We assess the pre-trained models on common sense ability (7 tasks) and an additional code task, code\_x\_glue using harness evaluation. As shown in Table~\ref{apptab:3b}, we pre-train open-llama 3B with 10B training budget and 1.5\% selection budget with SlimPajama. We compare our method with Random, DSIR, QuRating-A, and QuRating-W. DSIR and QuRating-W meet performance degradation due to dimensional collapse, while our method achieves the best performance on all tasks.
%
As shown in Table~\ref{apptab:slimstar}, we pre-train TinyLlama-1.1B on both SlimPajama and StarcoderData with 1.5\% selection budget and 10B training budget, compared to random selection and DSIR. Results of DSIR show performance degradation in code ability. We analyze that, DSIR tends to ignore code files in StarcoderData due to the selection critetion is based on WikiPedia, which means dimensional collapse happens.
\subsubsection{Training efficiency compared to all baselines} \label{app:efficiency}
To compare the computational and data efficiency of our approach with all baselines, we present additional results under the same settings as Figures~\ref{fig:sample} and ~\ref{fig:training}, shown in Tables~\ref{apptab:sample} and \ref{apptab:training}. The results demonstrate that, to achieve equivalent performance to Full Data with a 50B training budget, our method requires the fewest samples and pre-training tokens compared to the baselines, showing promising efficiency.
%
Additionally, We also compare the cost of our method with all baselines in the process of selecting data. As reported in QuRating, annotating the data using the GPT API costs 520 NVIDIA H100 hours with additional ranking procedures of 3 hours. For DSIR, it takes more than 2 days using 48 CPUs in our platform. For DOREMI, training a 120M proxy model to provide weights for domains takes us approximately one week. In contrast, our method utilizes a public feature extractor and selects samples in about 26 hours using one GPU and 48 CPUs. Combining these facts and our complexity analysis, we believe our method is practical among these methods for larger datasets.
\subsubsection{Proof of lower bound on Submodular ratio}\label{app:subanalysis} We first recall our function, which is based on F-norm of the covariance matrix. The gain on A, and B with another sample e, can be formulated as following:
$$\Delta_A = e^{-\frac{1}{|A| + 1} \sqrt{\sum_{i=1}^d \sum_{j=1}^d \left( \sum_{x \in A} x_i x_j + e_i e_j \right)^2}} - e^{-\frac{1}{|A|} \sqrt{\sum_{i=1}^d \sum_{j=1}^d \left( \sum_{x \in A} x_i x_j \right)^2}},$$
$$\Delta_B = e^{-\frac{1}{|B| + 1} \sqrt{\sum_{i=1}^d \sum_{j=1}^d \left( \sum_{x \in B} x_i x_j + e_i e_j \right)^2}} - e^{-\frac{1}{|B|} \sqrt{\sum_{i=1}^d \sum_{j=1}^d \left( \sum_{x \in B} x_i x_j \right)^2}},$$
where $A\subset B \subset \Omega$, $e\in \Omega$, and $e \notin B$. We reformulate them as
$$\Delta_A = e^{-\frac{1}{|A|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j)^2}} (e^{\frac{1}{|A|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j)^2}-\frac{1}{|A+1|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j+e_ie_j)^2}} - 1),$$
$$\Delta_B = e^{-\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}} (e^{\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}-\frac{1}{|B+1|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j+e_ie_j)^2}} - 1).$$
We then define
$$\Delta(e|A)=\frac{1}{|A+1|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j+e_ie_j)^2}-\frac{1}{|A|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j)^2},$$
$$\Delta(e|B)=\frac{1}{|B+1|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j+e_ie_j)^2-\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}}.$$
Therefore, the original equation will be:
$$\frac{\Delta_A}{\Delta_B}=e^{\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}-\frac{1}{|A|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j)^2}} *\frac{e^{-\Delta(e|A)}-1}{e^{-\Delta(e|B)}-1}.$$
To provide a theoretical guarantee, we aim to establish a lower bound for the submodular ratio under any given set where the gain is positive, as Figure~\ref{fig:monotone} generally suggest. Since the original formulation is challenging to analyze directly, we introduce two bounds on the function applied to the text files to derive a more concrete lower bound, as follows:
\begin{assumption}[Bounded Gain Difference]
Assume the gain difference between any two sets are bounded:
\begin{equation}
|\Delta(e|A)-\Delta(e|B)|\leq \epsilon. \nonumber
\end{equation}
\label{asm:difference}
\end{assumption}
\begin{assumption}[Bounded Average Utility]
Assume average utility is $\mu$, which means $\forall U \in \Omega$, we have:
\begin{equation}
\frac{1}{|U|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in U}x_i x_j)^2}\leq \mu. \nonumber
\end{equation}
\label{asm:max}
\end{assumption}
With Assumption~\ref{asm:difference}, we have:
$$\frac{\Delta_A}{\Delta_B}\geq e^{\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}-\frac{1}{|A|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j)^2}} *\frac{e^{-\Delta(e|B)-\epsilon}-1}{e^{-\Delta(e|B)}-1}.$$
With Assumption~\ref{asm:max}, we have:
$$\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}-\frac{1}{|A|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in A}x_i x_j)^2}\geq -2\mu,$$
which means $\frac{\Delta_A}{\Delta_B}\geq e^{-2\mu} *\frac{e^{-\Delta(e|B)-\epsilon}-1}{e^{-\Delta(e|B)}-1}.$We reformulate this as
$$\frac{\Delta_A}{\Delta_B}\geq e^{-2\mu} *(e^{-\epsilon}+\frac{e^{-\epsilon}-1}{e^{-\Delta(e|B)}-1}).$$
From Assumption~\ref{asm:max}, we have $\frac{1}{|B|} \sqrt{\sum_{i}^d \sum_{j}^d (\sum_{x\in B}x_i x_j)^2}\leq \mu$. Therefore $e^{-\Delta(e|B)}\leq e^{2\mu}$, and we have:
$$\frac{\Delta_A}{\Delta_B}\geq e^{-2\mu} *(e^{-\epsilon}+\frac{e^{-\epsilon}-1}{e^{2\mu}-1}).$$
Finally, we have a lower bound as $e^{-2\mu}\frac{e^{2\mu-\epsilon}-1}{e^{2\mu}-1}$, which means given assumptions~\ref{asm:difference}, ~\ref{asm:max}, and positive gain, our function is at least $e^{-2\mu}\frac{e^{2\mu-\epsilon}-1}{e^{2\mu}-1}$-weakly submodular function.
\subsubsection{Complexity analysis} \label{app:compuana}
For a detailed analysis of time complexity, we divide it into two parts: 1) computational complexity shown in Algorithm~\ref{alg:DiSF}. In each batch, we initialize $\mathrm{U}_i$ with a randomly selected sample and remove it from the batch. Then, we iteratively apply $(\lfloor\frac{b|\mathcal{S}|}{|\mathbb{D}|}\rfloor-1)$ times the Argmax command on the batch of data with our proxy fuction. Denote the computational cost of our proxy function with k text samples as $\mathrm{F}_{|\mathrm{U}|=k}(U)=O\mathrm{F}_k)$, the computation cost will be:$$O(1+...+\frac{|\mathbb{D}|}{b})\sum_{k=1}^{\frac{b|\mathcal{S}|}{|\mathbb{D}|}}(b-k)(\mathrm{F}_{k+1}) \leq O(\frac{|\mathbb{D}|^2}{b^2})\frac{b|\mathcal{S}|}{|\mathbb{D}|}b \mathrm{F}_{k+1} =O(|\mathbb{D}|\mathcal{S}\mathrm{F}_{\frac{b|\mathcal{S}|}{|\mathbb{D}|}+1}),$$ where b is the batch scale, $|\mathbb{D}|$ is the total data scale, $|\mathcal{S}|$ is the selection budget. 2) The complexity of proxy function $O(\mathrm{F}_k)$. Given text features $z$ and their feature dimension d, Frobenius norm and $z\cdot z^T$ are both $O(d^2)$. Since our proxy function calculates k times the $z\cdot z^T$, $O(F_k)=kd^2$. Finally, the complexity of our DiSF will be: $$O(DiSF)\leq O(|\mathbb{D}|\mathcal{S}F_{\frac{b|\mathcal{S}|}{|\mathbb{D}|}+1}) =O(|S|^2bd^2)$$All terms in the time complexity are at most quadratic and independent of the overall dataset size, which we consider acceptable for applying to larger datasets. The space complexity largely depends on the stored features of all text files: $O(|\mathbb{D}|d^2)$.
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\caption{Pre-trained performance compared to DiSF-LD.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|ccccccc}
\toprule[2pt]
Setting& Random &DSIR&D4&QuRating-W&QuRating-A&DiSF-LD&DiSF (Ours) \\
\midrule
TinyLlama120M&38.9&37.8& 39.7 &38.9&40.0&39.5&40.6\\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:LD}
\vspace{-10pt}
\end{table}
\subsubsection{Comparison to submodular functions} \label{app:comparesub}
In this section, we compare our proxy function with two strictly submodular functions: facility location and log-determinant. For the facility location function, we compare with INGENIOUS. Additionally, we introduce a variant defined as $F_{LD}$=LogDet(I+C) (DiSF-LD), where $I$ is the identity matrix and $C$ is covariance matrix. We evaluate this variant alongside our original proxy function on TinyLlama-120M, using a 1.5\% selection ratio and a 50B training budget. As shown in Tables~\ref{tb:main}, and \ref{apptab:LD}, the results demonstrate that both DiSF-LD and INGENIOUS help mitigate dimensional collapse and improve the performance of pre-trained LLMs. However, neither method achieves the same level of performance as our original DiSF. This is due to their inability to directly optimize the uniformity of feature dimensions, leading to a trade-off between strict submodularity and the specific goal of optimizing dimensional uniformity.
\begin{table}[ht!]
\setlength{\tabcolsep}{4pt}
\centering
\caption{File selection of DSIR on Starcoder. We denote selected file ratios of DSIR based on Wikipedia domain under both Starcoderdata and SlimPajama as DSIR-W-SS , on Wikipedia domain under Starcoderdata as DSIR-W-S and Python domain under Starcoderdata as DSIR-P-S.
}
\vspace{4pt}
% \renewcommand\arraystretch{0.9}
\small
\centering
% \scalebox{1.00}{
\begin{tabular}{l|ccccccc}
\toprule[2pt]
Method & go & java & javascript & php& python & ruby &slimpajama \\
\midrule
Original & 0.59\% & 2.52\% & 2.45\% & 1.97\% & 1.61\% & 0.43\% & 74.07\% \\
DSIR-W-SS & 0.00\%& 0.77\% &0.54\% & 0.00\% & 0.5\% & 0.00\% & 94.0\% \\
DSIR-W-S & 0.03\% & 12.37\% & 17.15\% & 0.12\% & 7.58\% & 0.00\% & - \\
DSIR-P-S& 0.00\% & 25.04\% & 4.36\% & 2.20\% & 50.38\% & 0.00\% & - \\
\bottomrule[2pt]
\end{tabular}
% }
\label{apptab:starselect}
\vspace{-10pt}
\end{table}
\subsubsection{Observation of dimensional collapse on Starcoder} \label{app:moredimen}
{As shown in Table~\ref{apptab:starselect}, we present the selection results of DSIR on Starcoderdata, as well as on both Starcoderdata and SlimPajama, across two domains: Wikipedia and Python. The results demonstrate that DSIR, when applied to a single domain, tends to select similar files, indicating dimensional collapse. Notably, during our analysis of the scores output by QuRater, we found that the writing style scores for most files in Starcoderdata are negative. As a result, these files are unlikely to be selected when combined with SlimPajama, further highlighting the issue of dimensional collapse.}
\subsection{Proof of Lemma 1}\label{app:lemma1}
\begin{lemma}
\label{lema:intra'}
Assuming a covariance matrix $M\in\mathbf{R}^{d\times d}$ computed from the feature of each sample with the standard normalization, and its eigenvalues $\{\lambda_1, \lambda_2,...,\lambda_d\}$, we will have the following equality that satisfied
\begin{equation}
\sum_{i=1}^{d}{(\lambda_i-\frac{1}{d}\sum_{j=1}^{d}{\lambda_j})^2}=||M||^2_F - d. \nonumber
\end{equation}
\end{lemma}
\begin{proof}
Let \( M \in \mathbb{R}^{d \times d} \) be a covariance matrix computed from features that have been standardized—that is, the data has been centered (zero mean) and scaled to have unit variance along each dimension. This standard normalization implies that the trace of \( M \) equals \( d \):
\[
\operatorname{Tr}(M) = \sum_{i=1}^d \lambda_i = d.
\]
This means the average eigenvalue is:
\[
\bar{\lambda} = \frac{1}{d} \sum_{i=1}^d \lambda_i = 1.
\]
\textbf{Left-Hand Side (LHS) Calculation:}
The left-hand side involves the sum of squared deviations of the eigenvalues from their mean:
\[
\sum_{i=1}^{d} \left( \lambda_i - \frac{1}{d} \sum_{j=1}^{d} \lambda_j \right)^2=\sum_{i=1}^d \left( \lambda_i - \bar{\lambda} \right)^2 = \sum_{i=1}^d (\lambda_i - 1)^2.
\]
Expanding each term on the right, we get the following equation:
\begin{align*}
\sum_{i=1}^d (\lambda_i - 1)^2 &= \sum_{i=1}^d \left( \lambda_i^2 - 2\lambda_i + 1 \right) \\
&= \sum_{i=1}^d \lambda_i^2 - 2 \sum_{i=1}^d \lambda_i + \sum_{i=1}^d 1.
\end{align*}
Simplifying with \( \sum_{i=1}^d \lambda_i = d \) and \( \sum_{i=1}^d 1 = d \), we have:
\[
\sum_{i=1}^d \lambda_i^2 - 2d + d = \sum_{i=1}^d \lambda_i^2 - d.
\]
\textbf{Right-Hand Side (RHS) Calculation:}
First, the Frobenius norm of a matrix \( M \) is defined as
\[
\| M \|_F^2 = \sum_{i=1}^d \sum_{j=1}^d M_{ij}^2 = \operatorname{Tr}(M^\top M).
\]
Since \( M \) is symmetric (\( M = M^\top \)), this simplifies to:
\[
\| M \|_F^2 = \operatorname{Tr}(M^2).
\]
%谱分解定理
Using the spectral theorem, we have
\[
M = U \Lambda U^\top,
\]
where \( U \) is an orthogonal matrix whose columns are the eigenvectors of \( M \), and \( \Lambda = \operatorname{diag}(\lambda_1, \lambda_2, \dots, \lambda_d) \) contains the eigenvalues. Since \( U^\top U = I \), we have the following equation:
\[
M^2 = (U \Lambda U^\top)(U \Lambda U^\top) = U \Lambda U^\top U \Lambda U^\top = U \Lambda^2 U^\top,
\]
Therefore, the trace of \( M^2 \) is
\[
\operatorname{Tr}(M^2) = \operatorname{Tr}(U \Lambda^2 U^\top) = \operatorname{Tr}(\Lambda^2 U^\top U) = \operatorname{Tr}(\Lambda^2) = \sum_{i=1}^d \lambda_i^2.
\]
Thus, the right-hand side is
\[
\| M \|_F^2 - d = \left( \sum_{i=1}^d \lambda_i^2 \right) - d.
\]
\textbf{Conclusion:}
Comparing both sides:
\[
\sum_{i=1}^{d} \left( \lambda_i - \frac{1}{d} \sum_{j=1}^{d} \lambda_j \right)^2 = \sum_{i=1}^d \lambda_i^2 - d = \| M \|_F^2 - d.
\]
This completes the proof.
\end{proof}
\begin{thebibliography}{75}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Abbas et~al.(2024)Abbas, Tirumala, Simig, Ganguli, and Morcos]{semdedup}
Amro Kamal~Mohamed Abbas, Kushal Tirumala, Daniel Simig, Surya Ganguli, and Ari~S Morcos.
\newblock Semdedup: Data-efficient learning at web-scale through semantic deduplication.
\newblock In \emph{ICLR 2023 Workshop on Mathematical and Empirical Understanding of Foundation Models}, 2024.
\bibitem[Balakrishnan et~al.(2022)Balakrishnan, Li, Zhou, Himayat, Smith, and Bilmes]{fedsub}
Ravikumar Balakrishnan, Tian Li, Tianyi Zhou, Nageen Himayat, Virginia Smith, and Jeff Bilmes.
\newblock Diverse client selection for federated learning via submodular maximization.
\newblock In \emph{International Conference on Learning Representations}, 2022.
\bibitem[Bardes et~al.(2022)Bardes, Ponce, and Lecun]{dc2}
Adrien Bardes, Jean Ponce, and Yann Lecun.
\newblock Vicreg: Variance-invariance-covariance regularization for self-supervised learning.
\newblock In \emph{ICLR 2022-International Conference on Learning Representations}, 2022.
\bibitem[Biderman et~al.(2023)Biderman, Schoelkopf, Anthony, Bradley, O’Brien, Hallahan, Khan, Purohit, Prashanth, Raff, et~al.]{pythia}
Stella Biderman, Hailey Schoelkopf, Quentin~Gregory Anthony, Herbie Bradley, Kyle O’Brien, Eric Hallahan, Mohammad~Aflah Khan, Shivanshu Purohit, USVSN~Sai Prashanth, Edward Raff, et~al.
\newblock Pythia: A suite for analyzing large language models across training and scaling.
\newblock In \emph{International Conference on Machine Learning}, pp.\ 2397--2430. PMLR, 2023.
\bibitem[Bisk et~al.(2020)Bisk, Zellers, Gao, Choi, et~al.]{bisk2020piqa}
Yonatan Bisk, Rowan Zellers, Jianfeng Gao, Yejin Choi, et~al.
\newblock Piqa: Reasoning about physical commonsense in natural language.
\newblock In \emph{Proceedings of the AAAI conference on artificial intelligence}, volume~34, pp.\ 7432--7439, 2020.
\bibitem[Brown(2020{\natexlab{a}})]{brown2020language}
Tom~B Brown.
\newblock Language models are few-shot learners.
\newblock \emph{arXiv preprint arXiv:2005.14165}, 2020{\natexlab{a}}.
\bibitem[Brown(2020{\natexlab{b}})]{gpt3}
Tom~B Brown.
\newblock Language models are few-shot learners.
\newblock \emph{arXiv preprint arXiv:2005.14165}, 2020{\natexlab{b}}.
\bibitem[Chen et~al.(2020)Chen, Kornblith, Norouzi, and Hinton]{dc3}
Ting Chen, Simon Kornblith, Mohammad Norouzi, and Geoffrey Hinton.
\newblock A simple framework for contrastive learning of visual representations.
\newblock In \emph{International conference on machine learning}, pp.\ 1597--1607. PMLR, 2020.
\bibitem[Chowdhery et~al.(2023{\natexlab{a}})Chowdhery, Narang, Devlin, Bosma, Mishra, Roberts, Barham, Chung, Sutton, Gehrmann, et~al.]{chowdhery2023palm}
Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung~Won Chung, Charles Sutton, Sebastian Gehrmann, et~al.
\newblock Palm: Scaling language modeling with pathways.
\newblock \emph{Journal of Machine Learning Research}, 24\penalty0 (240):\penalty0 1--113, 2023{\natexlab{a}}.
\bibitem[Chowdhery et~al.(2023{\natexlab{b}})Chowdhery, Narang, Devlin, Bosma, Mishra, Roberts, Barham, Chung, Sutton, Gehrmann, et~al.]{palm}
Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung~Won Chung, Charles Sutton, Sebastian Gehrmann, et~al.
\newblock Palm: Scaling language modeling with pathways.
\newblock \emph{Journal of Machine Learning Research}, 24\penalty0 (240):\penalty0 1--113, 2023{\natexlab{b}}.
\bibitem[Clark et~al.(2019)Clark, Lee, Chang, Kwiatkowski, Collins, and Toutanova]{clark2019boolq}
Christopher Clark, Kenton Lee, Ming-Wei Chang, Tom Kwiatkowski, Michael Collins, and Kristina Toutanova.
\newblock Boolq: Exploring the surprising difficulty of natural yes/no questions.
\newblock \emph{arXiv preprint arXiv:1905.10044}, 2019.
\bibitem[Clark et~al.(2018)Clark, Cowhey, Etzioni, Khot, Sabharwal, Schoenick, and Tafjord]{allenai:arc}
Peter Clark, Isaac Cowhey, Oren Etzioni, Tushar Khot, Ashish Sabharwal, Carissa Schoenick, and Oyvind Tafjord.
\newblock Think you have solved question answering? try arc, the ai2 reasoning challenge.
\newblock \emph{arXiv:1803.05457v1}, 2018.
\bibitem[Computer(2023)]{redpajama}
Together Computer.
\newblock Redpajama: an open dataset for training large language models, 2023.
\newblock URL \url{https://github.com/togethercomputer/RedPajama-Data}.
\bibitem[Dao et~al.(2022)Dao, Fu, Ermon, Rudra, and R{\'e}]{flash}
Tri Dao, Dan Fu, Stefano Ermon, Atri Rudra, and Christopher R{\'e}.
\newblock Flashattention: Fast and memory-efficient exact attention with io-awareness.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 16344--16359, 2022.
\bibitem[DAS(2011)]{ratio}
A~DAS.
\newblock Submodular meets spectral: Greedy algorithms for subset selection, sparse approximation and dictionary selection.
\newblock In \emph{Proc. 28th Int. Conf. on Machine Learning (ICML'11)}, pp.\ 1057--1064, 2011.
\bibitem[Fan et~al.(2024)Fan, Yao, Zhang, Lyu, Zhang, and Wang]{fanfederated}
Ziqing Fan, Jiangchao Yao, Ruipeng Zhang, Lingjuan Lyu, Ya~Zhang, and Yanfeng Wang.
\newblock Federated learning under partially class-disjoint data via manifold reshaping.
\newblock \emph{arXiv preprint arXiv:2405.18983}, 2024.
\bibitem[Fujishige(2005)]{function}
Satoru Fujishige.
\newblock \emph{Submodular functions and optimization}.
\newblock Elsevier, 2005.
\bibitem[Gao et~al.(2024)Gao, Tow, Abbasi, Biderman, Black, DiPofi, Foster, Golding, Hsu, Le~Noac'h, Li, McDonell, Muennighoff, Ociepa, Phang, Reynolds, Schoelkopf, Skowron, Sutawika, Tang, Thite, Wang, Wang, and Zou]{harness}
Leo Gao, Jonathan Tow, Baber Abbasi, Stella Biderman, Sid Black, Anthony DiPofi, Charles Foster, Laurence Golding, Jeffrey Hsu, Alain Le~Noac'h, Haonan Li, Kyle McDonell, Niklas Muennighoff, Chris Ociepa, Jason Phang, Laria Reynolds, Hailey Schoelkopf, Aviya Skowron, Lintang Sutawika, Eric Tang, Anish Thite, Ben Wang, Kevin Wang, and Andy Zou.
\newblock A framework for few-shot language model evaluation, 07 2024.
\newblock URL \url{https://zenodo.org/records/12608602}.
\bibitem[Geng \& Liu(2023)Geng and Liu]{openllama}
Xinyang Geng and Hao Liu.
\newblock Openllama: An open reproduction of llama, May 2023.
\newblock URL \url{https://github.com/openlm-research/open_llama}.
\bibitem[Guillory \& Bilmes(2011)Guillory and Bilmes]{active2}
Andrew Guillory and Jeff~A Bilmes.
\newblock Online submodular set cover, ranking, and repeated active learning.
\newblock \emph{Advances in neural information processing systems}, 24, 2011.
\bibitem[Hendrycks et~al.(2021)Hendrycks, Burns, Basart, Zou, Mazeika, Song, and Steinhardt]{MMLU}
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt.
\newblock Measuring massive multitask language understanding.
\newblock In \emph{International Conference on Learning Representations}, 2021.
\bibitem[Hoffmann et~al.(2022)Hoffmann, Borgeaud, Mensch, Buchatskaya, Cai, Rutherford, Casas, Hendricks, Welbl, Clark, et~al.]{computeoptimal}
Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Trevor Cai, Eliza Rutherford, Diego de~Las Casas, Lisa~Anne Hendricks, Johannes Welbl, Aidan Clark, et~al.
\newblock Training compute-optimal large language models.
\newblock \emph{arXiv preprint arXiv:2203.15556}, 2022.
\bibitem[Hong et~al.(2024)Hong, Lyu, Yao, Zhang, Tsang, and Wang]{diversity1}
Feng Hong, Yueming Lyu, Jiangchao Yao, Ya~Zhang, Ivor Tsang, and Yanfeng Wang.
\newblock Diversified batch selection for training acceleration.
\newblock In \emph{Forty-first International Conference on Machine Learning}, 2024.
\bibitem[Izacard et~al.(2021)Izacard, Caron, Hosseini, Riedel, Bojanowski, Joulin, and Grave]{contriever}
Gautier Izacard, Mathilde Caron, Lucas Hosseini, Sebastian Riedel, Piotr Bojanowski, Armand Joulin, and Edouard Grave.
\newblock Unsupervised dense information retrieval with contrastive learning, 2021.
\newblock URL \url{https://arxiv.org/abs/2112.09118}.
\bibitem[Jain et~al.(2024)Jain, Nandy, Aggarwal, Tendulkar, Iyer, and De]{subset-r1}
Eeshaan Jain, Tushar Nandy, Gaurav Aggarwal, Ashish Tendulkar, Rishabh Iyer, and Abir De.
\newblock Efficient data subset selection to generalize training across models: transductive and inductive networks.
\newblock \emph{Advances in Neural Information Processing Systems}, 36, 2024.
\bibitem[Jiang et~al.(2025)Jiang, Liao, Wang, Zhang, and Wang]{jiangshuyang}
Shuyang Jiang, Yusheng Liao, Yanfeng Wang, Ya~Zhang, and Yu~Wang.
\newblock Fine-tuning with reserved majority for noise reduction.
\newblock In \emph{The Thirteenth International Conference on Learning Representations}, 2025.
\newblock URL \url{https://openreview.net/forum?id=ZV7CLf0RHK}.
\bibitem[Jing et~al.(2022)Jing, Vincent, LeCun, and Tian]{dc}
Li~Jing, Pascal Vincent, Yann LeCun, and Yuandong Tian.
\newblock Understanding dimensional collapse in contrastive self-supervised learning.
\newblock In \emph{International Conference on Learning Representations}, 2022.
\bibitem[Karanam et~al.(2022)Karanam, Killamsetty, Kokel, and Iyer]{ld}
Athresh Karanam, Krishnateja Killamsetty, Harsha Kokel, and Rishabh Iyer.
\newblock Orient: Submodular mutual information measures for data subset selection under distribution shift.
\newblock \emph{Advances in neural information processing systems}, 35:\penalty0 31796--31808, 2022.
\bibitem[Kaushal et~al.(2019)Kaushal, Iyer, Kothawade, Mahadev, Doctor, and Ramakrishnan]{gc}
Vishal Kaushal, Rishabh Iyer, Suraj Kothawade, Rohan Mahadev, Khoshrav Doctor, and Ganesh Ramakrishnan.
\newblock Learning from less data: A unified data subset selection and active learning framework for computer vision.
\newblock In \emph{2019 IEEE Winter Conference on Applications of Computer Vision (WACV)}, pp.\ 1289--1299. IEEE, 2019.
\bibitem[Kothawade et~al.(2022)Kothawade, Kaushal, Ramakrishnan, Bilmes, and Iyer]{sum1}
Suraj Kothawade, Vishal Kaushal, Ganesh Ramakrishnan, Jeff Bilmes, and Rishabh Iyer.
\newblock Prism: A rich class of parameterized submodular information measures for guided data subset selection.
\newblock In \emph{Proceedings of the AAAI Conference on Artificial Intelligence}, volume~36, pp.\ 10238--10246, 2022.
\bibitem[Krause \& Golovin(2014)Krause and Golovin]{location2}
Andreas Krause and Daniel Golovin.
\newblock Submodular function maximization.
\newblock \emph{Tractability}, 3\penalty0 (71-104):\penalty0 3, 2014.
\bibitem[Kumari et~al.(2024)Kumari, Wang, Das, Zhou, and Bilmes]{sum2}
Lilly Kumari, Shengjie Wang, Arnav Das, Tianyi Zhou, and Jeff Bilmes.
\newblock An end-to-end submodular framework for data-efficient in-context learning.
\newblock In \emph{Findings of the Association for Computational Linguistics: NAACL 2024}, pp.\ 3293--3308, 2024.
\bibitem[Lee et~al.(2021)Lee, Cao, Fan, Jiang, and Chong]{fan1}
Roy Ka-Wei Lee, Rui Cao, Ziqing Fan, Jing Jiang, and Wen-Haw Chong.
\newblock Disentangling hate in online memes.
\newblock In \emph{Proceedings of the 29th ACM international conference on multimedia}, pp.\ 5138--5147, 2021.
\bibitem[Li et~al.(2023)Li, Allal, Zi, Muennighoff, Kocetkov, Mou, Marone, Akiki, Li, Chim, et~al.]{starcoder}
Raymond Li, Loubna~Ben Allal, Yangtian Zi, Niklas Muennighoff, Denis Kocetkov, Chenghao Mou, Marc Marone, Christopher Akiki, Jia Li, Jenny Chim, et~al.
\newblock Starcoder: may the source be with you!
\newblock \emph{arXiv preprint arXiv:2305.06161}, 2023.
\bibitem[LightningAI(2023)]{litgpt}
LightningAI.
\newblock Litgpt.
\newblock \url{https://github.com/Lightning-AI/litgpt}, 2023.
\bibitem[Lin \& Bilmes(2011)Lin and Bilmes]{subset-r2}
Hui Lin and Jeff~A Bilmes.
\newblock Optimal selection of limited vocabulary speech corpora.
\newblock In \emph{INTERSPEECH}, pp.\ 1489--1492, 2011.
\bibitem[Loshchilov \& Hutter(2019)Loshchilov and Hutter]{adamw}
Ilya Loshchilov and Frank Hutter.
\newblock Decoupled weight decay regularization.
\newblock In \emph{International Conference on Learning Representations}, 2019.
\bibitem[Mihaylov et~al.(2018)Mihaylov, Clark, Khot, and Sabharwal]{OpenBookQA2018}
Todor Mihaylov, Peter Clark, Tushar Khot, and Ashish Sabharwal.
\newblock Can a suit of armor conduct electricity? a new dataset for open book question answering.
\newblock In \emph{EMNLP}, 2018.
\bibitem[Nemhauser et~al.(1978)Nemhauser, Wolsey, and Fisher]{submodular}
George~L Nemhauser, Laurence~A Wolsey, and Marshall~L Fisher.
\newblock An analysis of approximations for maximizing submodular set functions—i.
\newblock \emph{Mathematical programming}, 14:\penalty0 265--294, 1978.
\bibitem[Pang et~al.(2024)Pang, Tang, Ye, Xiong, Zhang, Wang, and Chen]{yerui2}
Xianghe Pang, Shuo Tang, Rui Ye, Yuxin Xiong, Bolun Zhang, Yanfeng Wang, and Siheng Chen.
\newblock Self-alignment of large language models via monopolylogue-based social scene simulation.
\newblock \emph{arXiv preprint arXiv:2402.05699}, 2024.
\bibitem[Paszke et~al.(2019)Paszke, Gross, Massa, Lerer, Bradbury, Chanan, Killeen, Lin, Gimelshein, Antiga, et~al.]{pytorch}
Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, et~al.
\newblock Pytorch: An imperative style, high-performance deep learning library.
\newblock \emph{Advances in neural information processing systems}, 32, 2019.
\bibitem[Patterson et~al.(2021)Patterson, Gonzalez, Le, Liang, Munguia, Rothchild, So, Texier, and Dean]{carbon}
David Patterson, Joseph Gonzalez, Quoc Le, Chen Liang, Lluis-Miquel Munguia, Daniel Rothchild, David So, Maud Texier, and Jeff Dean.
\newblock Carbon emissions and large neural network training.
\newblock \emph{arXiv preprint arXiv:2104.10350}, 2021.
\bibitem[Radford et~al.(2019)Radford, Wu, Child, Luan, Amodei, Sutskever, et~al.]{gpt2}
Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, Ilya Sutskever, et~al.
\newblock Language models are unsupervised multitask learners.
\newblock \emph{OpenAI blog}, 1\penalty0 (8):\penalty0 9, 2019.
\bibitem[Radford et~al.(2021)Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, et~al.]{clip}
Alec Radford, Jong~Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et~al.
\newblock Learning transferable visual models from natural language supervision.
\newblock In \emph{International conference on machine learning}, pp.\ 8748--8763. PMLR, 2021.
\bibitem[Rae et~al.(2021)Rae, Borgeaud, Cai, Millican, Hoffmann, Song, Aslanides, Henderson, Ring, Young, et~al.]{cost1}
Jack~W Rae, Sebastian Borgeaud, Trevor Cai, Katie Millican, Jordan Hoffmann, Francis Song, John Aslanides, Sarah Henderson, Roman Ring, Susannah Young, et~al.
\newblock Scaling language models: Methods, analysis \& insights from training gopher.
\newblock \emph{arXiv preprint arXiv:2112.11446}, 2021.
\bibitem[Renduchintala et~al.(1991)Renduchintala, Killamsetty, Bhatia, Aggarwal, Ramakrishnan, Iyer, and Krishnamurthy]{ingenious}
HSVNS~Kowndinya Renduchintala, Krishnateja Killamsetty, Sumit Bhatia, Milan Aggarwal, Ganesh Ramakrishnan, Rishabh~K Iyer, and Balaji Krishnamurthy.
\newblock Ingenious: Using informative data subsets for efficient pre-training of language models.
\newblock In \emph{The 2023 Conference on Empirical Methods in Natural Language Processing}, 1991.
\bibitem[Sakaguchi et~al.(2021)Sakaguchi, Bras, Bhagavatula, and Choi]{sakaguchi2021winogrande}
Keisuke Sakaguchi, Ronan~Le Bras, Chandra Bhagavatula, and Yejin Choi.
\newblock Winogrande: An adversarial winograd schema challenge at scale.
\newblock \emph{Communications of the ACM}, 64\penalty0 (9):\penalty0 99--106, 2021.
\bibitem[Salhi(1991{\natexlab{a}})]{facility}
Said Salhi.
\newblock Discrete location theory.
\newblock \emph{Journal of the Operational Research Society}, 42\penalty0 (12):\penalty0 1124--1125, 1991{\natexlab{a}}.
\bibitem[Salhi(1991{\natexlab{b}})]{location1}
Said Salhi.
\newblock Discrete location theory.
\newblock \emph{Journal of the Operational Research Society}, 42\penalty0 (12):\penalty0 1124--1125, 1991{\natexlab{b}}.
\bibitem[Santiago \& Yoshida(2020)Santiago and Yoshida]{pessi}
Richard Santiago and Yuichi Yoshida.
\newblock Weakly submodular function maximization using local submodularity ratio.
\newblock \emph{arXiv preprint arXiv:2004.14650}, 2020.
\bibitem[Shi et~al.(2022)Shi, Zhou, Liang, Jiang, Feng, Torr, Bai, and Tan]{cwd}
Yujun Shi, Kuangqi Zhou, Jian Liang, Zihang Jiang, Jiashi Feng, Philip~HS Torr, Song Bai, and Vincent~YF Tan.
\newblock Mimicking the oracle: An initial phase decorrelation approach for class incremental learning.
\newblock In \emph{Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pp.\ 16722--16731, 2022.
\bibitem[Sorscher et~al.(2022)Sorscher, Geirhos, Shekhar, Ganguli, and Morcos]{proto}
Ben Sorscher, Robert Geirhos, Shashank Shekhar, Surya Ganguli, and Ari Morcos.
\newblock Beyond neural scaling laws: beating power law scaling via data pruning.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 19523--19536, 2022.
\bibitem[Srivastava et~al.(2022)Srivastava, Rastogi, Rao, Shoeb, Abid, Fisch, Brown, Santoro, Gupta, Garriga-Alonso, et~al.]{bigbench}
Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal~Md Shoeb, Abubakar Abid, Adam Fisch, Adam~R Brown, Adam Santoro, Aditya Gupta, Adri{\`a} Garriga-Alonso, et~al.
\newblock Beyond the imitation game: Quantifying and extrapolating the capabilities of language models.
\newblock \emph{arXiv preprint arXiv:2206.04615}, 2022.
\bibitem[Suzgun et~al.(2022)Suzgun, Scales, Sch{\"a}rli, Gehrmann, Tay, Chung, Chowdhery, Le, Chi, Zhou, et~al.]{bbh}
Mirac Suzgun, Nathan Scales, Nathanael Sch{\"a}rli, Sebastian Gehrmann, Yi~Tay, Hyung~Won Chung, Aakanksha Chowdhery, Quoc~V Le, Ed~H Chi, Denny Zhou, et~al.
\newblock Challenging big-bench tasks and whether chain-of-thought can solve them.
\newblock \emph{arXiv preprint arXiv:2210.09261}, 2022.
\bibitem[Tang et~al.(2024)Tang, Pang, Liu, Tang, Ye, Dong, Wang, and Chen]{tang}
Shuo Tang, Xianghe Pang, Zexi Liu, Bohan Tang, Rui Ye, Xiaowen Dong, Yanfeng Wang, and Siheng Chen.
\newblock Synthesizing post-training data for llms through multi-agent simulation.
\newblock \emph{arXiv preprint arXiv:2410.14251}, 2024.
\bibitem[Thoppilan et~al.(2022)Thoppilan, De~Freitas, Hall, Shazeer, Kulshreshtha, Cheng, Jin, Bos, Baker, Du, et~al.]{cost2}
Romal Thoppilan, Daniel De~Freitas, Jamie Hall, Noam Shazeer, Apoorv Kulshreshtha, Heng-Tze Cheng, Alicia Jin, Taylor Bos, Leslie Baker, Yu~Du, et~al.
\newblock Lamda: Language models for dialog applications.
\newblock \emph{arXiv preprint arXiv:2201.08239}, 2022.
\bibitem[Tirumala et~al.(2023)Tirumala, Simig, Aghajanyan, and Morcos]{d4}
Kushal Tirumala, Daniel Simig, Armen Aghajanyan, and Ari Morcos.
\newblock D4: Improving llm pretraining via document de-duplication and diversification.
\newblock \emph{Advances in Neural Information Processing Systems}, 36, 2023.
\bibitem[Touvron et~al.(2023{\natexlab{a}})Touvron, Lavril, Izacard, Martinet, Lachaux, Lacroix, Rozi{\`e}re, Goyal, Hambro, Azhar, et~al.]{llama}
Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth{\'e}e Lacroix, Baptiste Rozi{\`e}re, Naman Goyal, Eric Hambro, Faisal Azhar, et~al.
\newblock Llama: Open and efficient foundation language models.
\newblock \emph{arXiv preprint arXiv:2302.13971}, 2023{\natexlab{a}}.
\bibitem[Touvron et~al.(2023{\natexlab{b}})Touvron, Lavril, Izacard, Martinet, Lachaux, Lacroix, Rozi{\`e}re, Goyal, Hambro, Azhar, et~al.]{touvron2023llama}
Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timoth{\'e}e Lacroix, Baptiste Rozi{\`e}re, Naman Goyal, Eric Hambro, Faisal Azhar, et~al.
\newblock Llama: Open and efficient foundation language models.
\newblock \emph{arXiv preprint arXiv:2302.13971}, 2023{\natexlab{b}}.
\bibitem[Van~der Maaten \& Hinton(2008)Van~der Maaten and Hinton]{tsne}
Laurens Van~der Maaten and Geoffrey Hinton.
\newblock Visualizing data using t-sne.
\newblock \emph{Journal of machine learning research}, 9\penalty0 (11), 2008.
\bibitem[Wang et~al.(2024)Wang, Fan, Hu, Chen, Wang, and Wang]{wang1}
Pingjie Wang, Ziqing Fan, Shengchao Hu, Zhe Chen, Yanfeng Wang, and Yu~Wang.
\newblock Reconstruct the pruned model without any retraining.
\newblock \emph{arXiv preprint arXiv:2407.13331}, 2024.
\bibitem[Wei et~al.(2015)Wei, Iyer, and Bilmes]{active1}
Kai Wei, Rishabh Iyer, and Jeff Bilmes.
\newblock Submodularity in data subset selection and active learning.
\newblock In \emph{International conference on machine learning}, pp.\ 1954--1963. PMLR, 2015.
\bibitem[Wettig et~al.(2024)Wettig, Gupta, Malik, and Chen]{qrating}
Alexander Wettig, Aatmik Gupta, Saumya Malik, and Danqi Chen.
\newblock Qurating: Selecting high-quality data for training language models.
\newblock In \emph{Forty-first International Conference on Machine Learning}, 2024.
\bibitem[Xie et~al.(2023{\natexlab{a}})Xie, Pham, Dong, Du, Liu, Lu, Liang, Le, Ma, and Yu]{doremi}
Sang~Michael Xie, Hieu Pham, Xuanyi Dong, Nan Du, Hanxiao Liu, Yifeng Lu, Percy~S Liang, Quoc~V Le, Tengyu Ma, and Adams~Wei Yu.
\newblock Doremi: Optimizing data mixtures speeds up language model pretraining.
\newblock \emph{Advances in Neural Information Processing Systems}, 36, 2023{\natexlab{a}}.
\bibitem[Xie et~al.(2023{\natexlab{b}})Xie, Santurkar, Ma, and Liang]{dsir}
Sang~Michael Xie, Shibani Santurkar, Tengyu Ma, and Percy~S Liang.
\newblock Data selection for language models via importance resampling.
\newblock \emph{Advances in Neural Information Processing Systems}, 36:\penalty0 34201--34227, 2023{\natexlab{b}}.
\bibitem[Ye et~al.(2024)Ye, Pang, Chai, Chen, Yin, Xiang, Dong, Shao, and Chen]{yerui1}
Rui Ye, Xianghe Pang, Jingyi Chai, Jiaao Chen, Zhenfei Yin, Zhen Xiang, Xiaowen Dong, Jing Shao, and Siheng Chen.
\newblock Are we there yet? revealing the risks of utilizing large language models in scholarly peer review.
\newblock \emph{arXiv preprint arXiv:2412.01708}, 2024.
\bibitem[Yi et~al.(2024)Yi, Ye, Chen, Zhu, Chen, Lian, Sun, Xie, and Wu]{yerui3}
Jingwei Yi, Rui Ye, Qisi Chen, Bin Zhu, Siheng Chen, Defu Lian, Guangzhong Sun, Xing Xie, and Fangzhao Wu.
\newblock On the vulnerability of safety alignment in open-access llms.
\newblock In \emph{Findings of the Association for Computational Linguistics ACL 2024}, pp.\ 9236--9260, 2024.
\bibitem[Zbontar et~al.(2021)Zbontar, Jing, Misra, LeCun, and Deny]{dc1}
Jure Zbontar, Li~Jing, Ishan Misra, Yann LeCun, and St{\'e}phane Deny.
\newblock Barlow twins: Self-supervised learning via redundancy reduction.
\newblock In \emph{International conference on machine learning}, pp.\ 12310--12320. PMLR, 2021.
\bibitem[Zellers et~al.(2019)Zellers, Holtzman, Bisk, Farhadi, and Choi]{zellers2019hellaswag}
Rowan Zellers, Ari Holtzman, Yonatan Bisk, Ali Farhadi, and Yejin Choi.
\newblock Hellaswag: Can a machine really finish your sentence?
\newblock \emph{arXiv preprint arXiv:1905.07830}, 2019.
\bibitem[Zhang et~al.(2024{\natexlab{a}})Zhang, Zeng, Wang, and Lu]{tinyllama}
Peiyuan Zhang, Guangtao Zeng, Tianduo Wang, and Wei Lu.
\newblock Tinyllama: An open-source small language model.
\newblock \emph{arXiv preprint arXiv:2401.02385}, 2024{\natexlab{a}}.
\bibitem[Zhang et~al.(2023)Zhang, Deng, Jian, Chen, Hu, and Yang]{qixing3}
Qixin Zhang, Zengde Deng, Xiangru Jian, Zaiyi Chen, Haoyuan Hu, and Yu~Yang.
\newblock Communication-efficient decentralized online continuous dr-submodular maximization.
\newblock In \emph{Proceedings of the 32nd ACM International Conference on Information and Knowledge Management}, pp.\ 3330--3339, 2023.
\bibitem[Zhang et~al.(2024{\natexlab{b}})Zhang, Wan, Deng, Chen, Sun, Zhang, and Yang]{qixing2}
Qixin Zhang, Zongqi Wan, Zengde Deng, Zaiyi Chen, Xiaoming Sun, Jialin Zhang, and Yu~Yang.
\newblock Boosting gradient ascent for continuous dr-submodular maximization.
\newblock \emph{arXiv preprint arXiv:2401.08330}, 2024{\natexlab{b}}.
\bibitem[Zhang et~al.(2025)Zhang, Wan, Yang, Shen, and Tao]{qixing1}
Qixin Zhang, Zongqi Wan, Yu~Yang, Li~Shen, and Dacheng Tao.
\newblock Near-optimal online learning for multi-agent submodular coordination: Tight approximation and communication efficiency.
\newblock \emph{arXiv preprint arXiv:2502.05028}, 2025.
\bibitem[Zhang et~al.(2022)Zhang, Roller, Goyal, Artetxe, Chen, Chen, Dewan, Diab, Li, Lin, et~al.]{opt}
Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen, Christopher Dewan, Mona Diab, Xian Li, Xi~Victoria Lin, et~al.
\newblock Opt: Open pre-trained transformer language models.
\newblock \emph{arXiv preprint arXiv:2205.01068}, 2022.
\bibitem[Zhou \& Bilmes(2018)Zhou and Bilmes]{curri}
Tianyi Zhou and Jeff Bilmes.
\newblock Minimax curriculum learning: Machine teaching with desirable difficulties and scheduled diversity.
\newblock In \emph{International conference on learning representations}, 2018.
\end{thebibliography}