OctoPipe: Reducing Pipeline Bubbles for Heterogeneous Models via Co-Optimizing Partitioning, Placement, and Scheduling
Extracted source text ยท arXiv 2509.23722
%% For double-blind review submission, w/o CCS and ACM Reference (max submission space)
% \documentclass[sigplan,10pt]{acmart}
% \documentclass[sigplan,review]{acmart}\settopmatter{printfolios=true,printccs=false,printacmref=false}
% \settopmatter{printfolios=true,printccs=false,printacmref=false}
\documentclass[sigplan]{acmart}\settopmatter{printfolios=true,printccs=false,printacmref=false}
\settopmatter{printfolios=true,printccs=false,printacmref=false}
%% For double-blind review submission, w/ CCS and ACM Reference
%\documentclass[sigplan,review,anonymous]{acmart}\settopmatter{printfolios=true}
%% For single-blind review submission, w/o CCS and ACM Reference (max submission space)
%\documentclass[sigplan,review]{acmart}\settopmatter{printfolios=true,printccs=false,printacmref=false}
%% For single-blind review submission, w/ CCS and ACM Reference
%\documentclass[sigplan,review]{acmart}\settopmatter{printfolios=true}
%% For final camera-ready submission, w/ required CCS and ACM Reference
% \documentclass[sigplan]{acmart}\settopmatter{}
\renewcommand\footnotetextcopyrightpermission[1]{}
\pagestyle{plain}
\settopmatter{printacmref=false} % Removes citation
\usepackage{soul}
\usepackage{xspace}
\usepackage{tikz}
\usepackage{amsmath}
\usepackage{minted}
\usepackage{xcolor}
\usepackage{pifont}
\usepackage{array}
\usepackage{makecell}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{multirow}
\usepackage{tabularx}
\usepackage{booktabs} %% For formal tables: http://ctan.org/pkg/booktabs
\usepackage{subcaption} %% For complex figures with subfigures/subcaptions http://ctan.org/pkg/subcaption
\newcommand{\etal}{\textit{et al.}}
\newcommand\encircle[1]{\tikz[baseline=(X.base)]
\node (X) [draw, shape=circle, inner sep=0pt, fill=black, text=white] {\strut #1};}
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
% \newcommand{\SysName}{\texttt{OctoPipe}}
\newcommand{\SysName}{\texttt{AdaPtis}}
\newcommand{\PPP}{Pipeline Performance Model}
\newcommand{\PG}{Pipeline Generator}
\newcommand{\PE}{Pipeline Executor}
\newcommand*\emptycirc[1][1ex]{\raisebox{-0.4ex}{\tikz\draw (0,0) circle (#1);}}
\newcommand*\halfcirc[1][1ex]{\raisebox{-0.4ex}{\hspace{-0.5ex}
\begin{tikzpicture}
\draw[fill] (0,0)-- (90:#1) arc (90:270:#1) -- cycle ;
\draw (0,0) circle (#1);
\end{tikzpicture}}}
\newcommand*\fullcirc[1][1ex]{\raisebox{-0.4ex}{\tikz\fill (0,0) circle (#1);}}
\newcommand\wcircle[1]{\raisebox{-0.2ex}{\Large\ding{\numexpr171+#1}}}
%% Conference information
%% Supplied to authors by publisher for camera-ready submission;
%% use defaults for review submission.
\acmConference[PPoPP'26]{ACM SIGPLAN Annual Symposium on Principles and Practice of Parallel Programming}{January 31--February 04, 2026}{Sydney, Australia}
\acmYear{2026}
\acmISBN{} % \acmISBN{978-x-xxxx-xxxx-x/YY/MM}
\acmDOI{} % \acmDOI{10.1145/nnnnnnn.nnnnnnn}
\startPage{1}
%% Copyright information
%% Supplied to authors (based on authors' rights management selection;
%% see authors.acm.org) by publisher for camera-ready submission;
%% use 'none' for review submission.
\setcopyright{none}
%\setcopyright{acmcopyright}
%\setcopyright{acmlicensed}
%\setcopyright{rightsretained}
%\copyrightyear{2018} %% If different from \acmYear
%% Bibliography style
\bibliographystyle{ACM-Reference-Format}
%% Citation style
%\citestyle{acmauthoryear} %% For author/year citations
%\citestyle{acmnumeric} %% For numeric citations
%\setcitestyle{nosort} %% With 'acmnumeric', to disable automatic
%% sorting of references within a single citation;
%% e.g., \cite{Smith99,Carpenter05,Baker12}
%% rendered as [14,5,2] rather than [2,5,14].
%\setcitesyle{nocompress} %% With 'acmnumeric', to disable automatic
%% compression of sequential references within a
%% single citation;
%% e.g., \cite{Baker12,Baker14,Baker16}
%% rendered as [2,3,4] rather than [2-4].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Note: Authors migrating a paper from traditional SIGPLAN
%% proceedings format to PACMPL format must update the
%% '\documentclass' and topmatter commands above; see
%% 'acmart-pacmpl-template.tex'.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
%% Title information
% \title[PPoPP Paper]{\SysName{}: Accelerating LLM Training with Adaptive Pipeline Parallelism}
% \title[PPoPP Paper]{\SysName{}: Accelerating LLM Training with Adaptive Pipeline Parallelism on Heterogeneous Models}
\title[PPoPP Paper]{\SysName{}: Reducing Pipeline Bubbles with Adaptive Pipeline Parallelism on Heterogeneous Models}
% \title[PPoPP Paper]{PPoPP Sample Paper}
%% [Short Title] is optional;
%% when present, will be used in
%% header instead of Full Title.
%\titlenote{For simple use only} %% \titlenote is optional;
%% can be repeated if necessary;
%% contents suppressed with 'anonymous'
%\subtitle{Check the format} %% \subtitle is optional
%\subtitlenote{Everything should just work} %% \subtitlenote is optional;
%% can be repeated if necessary;
%% contents suppressed with 'anonymous'
%% Author information
%% Contents and number of authors suppressed with 'anonymous'.
%% Each author should be introduced by \author, followed by
%% \authornote (optional), \orcid (optional), \affiliation, and
%% \email.
%% An author may have multiple affiliations and/or emails; repeat the
%% appropriate command.
%% Many elements are not rendered, but should be provided for metadata
%% extraction tools.
\author{} %% fill in for the camera-ready version with the format commented out below
%% Author with single affiliation.
\input{main/authors}
% %% Author with two affiliations and emails.
% \author{First2 Last2}
% \authornote{with author2 note} %% \authornote is optional;
% %% can be repeated if necessary
% \orcid{nnnn-nnnn-nnnn-nnnn} %% \orcid is optional
% \affiliation{
% \position{Position2a}
% \department{Department2a} %% \department is recommended
% \institution{Institution2a} %% \institution is required
% \streetaddress{Street2a Address2a}
% \city{City2a}
% \state{State2a}
% \postcode{Post-Code2a}
% \country{Country2a} %% \country is recommended
% }
% \email{first2.last2@inst2a.com} %% \email is recommended
% \affiliation{
% \position{Position2b}
% \department{Department2b} %% \department is recommended
% \institution{Institution2b} %% \institution is required
% \streetaddress{Street3b Address2b}
% \city{City2b}
% \state{State2b}
% \postcode{Post-Code2b}
% \country{Country2b} %% \country is recommended
% }
% \email{first2.last2@inst2b.org} %% \email is recommended
\author{} %% fill in for the camera-ready version with the format commented out below
%% Author with single affiliation.
% \author{First1 Last1}
% \authornote{with author1 note} %% \authornote is optional;
% %% can be repeated if necessary
% \orcid{nnnn-nnnn-nnnn-nnnn} %% \orcid is optional
% \affiliation{
% \position{Position1}
% \department{Department1} %% \department is recommended
% \institution{Institution1} %% \institution is required
% \streetaddress{Street1 Address1}
% \city{City1}
% \state{State1}
% \postcode{Post-Code1}
% \country{Country1} %% \country is recommended
% }
% \email{first1.last1@inst1.edu} %% \email is recommended
% %% Author with two affiliations and emails.
% \author{First2 Last2}
% \authornote{with author2 note} %% \authornote is optional;
% %% can be repeated if necessary
% \orcid{nnnn-nnnn-nnnn-nnnn} %% \orcid is optional
% \affiliation{
% \position{Position2a}
% \department{Department2a} %% \department is recommended
% \institution{Institution2a} %% \institution is required
% \streetaddress{Street2a Address2a}
% \city{City2a}
% \state{State2a}
% \postcode{Post-Code2a}
% \country{Country2a} %% \country is recommended
% }
% \email{first2.last2@inst2a.com} %% \email is recommended
% \affiliation{
% \position{Position2b}
% \department{Department2b} %% \department is recommended
% \institution{Institution2b} %% \institution is required
% \streetaddress{Street3b Address2b}
% \city{City2b}
% \state{State2b}
% \postcode{Post-Code2b}
% \country{Country2b} %% \country is recommended
% }
% \email{first2.last2@inst2b.org} %% \email is recommended
%% Remove these two lines for the camera-ready version
\fancyhead{} % clears the header fields that trigger HotCRP format checker
\renewcommand\footnotetextcopyrightpermission[1]{} % removes the ACM copyright footnote
%% Abstract
%% Note: \begin{abstract}...\end{abstract} environment must come
%% before \maketitle command
\begin{abstract}
Pipeline parallelism is widely used to train large language models (LLMs). However, increasing heterogeneity in model architectures exacerbates pipeline bubbles, thereby reducing training efficiency. Existing approaches overlook the co-optimization of model partition, model placement, and workload scheduling, resulting in limited efficiency improvement or even performance degradation. To respond, we propose \SysName{}, an LLM training system that supports adaptive pipeline parallelism. First, we develop a pipeline performance model to accurately estimate training throughput. Second, \SysName{} jointly optimizes model partition, model placement, and workload scheduling policies guided by this performance model. Third, we design a unified pipeline executor that efficiently supports the execution of diverse pipeline strategies. Extensive experiments show that \SysName{} achieves an average speedup of 1.42$\times$ (up to 2.14$\times$) over Megatron-LM I-1F1B across various LLM architectures and scales.
\end{abstract}
%% \maketitle
%% Note: \maketitle command must come after title commands, author
%% commands, abstract environment, Computing Classification System
%% environment and commands, and keywords command.
\maketitle
\input{main/1_intro}
\input{main/2_background}
\input{main/3_motivation}
\input{main/4_pipeline_unify}
\input{main/6_RUNTIME}
\input{main/7_eval}
\input{main/8_related_conclusion}
% \section{Introduction}
% The paper contents here \ldots
%% Bibliography
\bibliography{main}
\end{document}
\author{Jihu Guo}
\authornote{Equal contribution}
\affiliation{
\institution{Fudan University \& Shanghai AI Laboratory} %% \institution is required
\state{Shanghai}
\country{China} %% \country is recommended
}
\email{guojihu@pjlab.org.cn} %% \email is recommended
\author{Tenghui Ma}
\authornotemark[1]
\affiliation{
\institution{Fudan University \& Shanghai AI Laboratory} %% \institution is required
\state{Shanghai}
\country{China} %% \country is recommended
}
\email{matenghui@pjlab.org.cn} %% \email is recommended
\author{Wei Gao}
\authornote{Corresponding author}
\affiliation{
\institution{Hong Kong University of Science and Technology} %% \institution is required
\state{Hong Kong}
\country{China} %% \country is recommended
}
\email{csgaowei@ust.hk} %% \email is recommended
\author{Peng Sun}
\affiliation{
\institution{Shanghai AI Laboratory} %% \institution is required
\state{Shanghai}
\country{China} %% \country is recommended
}
\email{sunpeng@pjlab.org.cn} %% \email is recommended
\author{Jiaxing Li}
\affiliation{
\institution{Shanghai AI Laboratory} %% \institution is required
\state{Shanghai}
\country{China} %% \country is recommended
}
\email{lijiaxing@pjlab.org.cn} %% \email is recommended
\author{Xun Chen}
\affiliation{
\institution{SenseTime} %% \institution is required
\state{Shenzhen}
\country{China} %% \country is recommended
}
\email{chenxun@sensetime.com} %% \email is recommended
\author{Yuyang Jin}
\affiliation{
\institution{Tsinghua University} %% \institution is required
\state{Beijing}
\country{China} %% \country is recommended
}
\email{jinyuyang@tsinghua.edu.cn} %% \email is recommended
\author{Dahua Lin}
\affiliation{
\institution{Chinese University of Hong Kong \& Sensetime Research} %% \institution is required
\state{Hong Kong}
\country{China} %% \country is recommended
}
\email{dhlin@ie.cuhk.edu.hk} %% \email is recommended
\section{Introduction}
Large Language Models (LLMs) have demonstrated impressive performance across a broad range of domains~\cite{brown2020gpt3,dubey2024llama3,yang2025qwen3,liu2024deepseekv3,li2025minimax,team2024gemini1.5}.
However, this rapid advancement comes with a steep increase in training cost~\cite{dubey2024llama3, kaplan2020scalinglaw, wei2022emergent}, highlighting that even a modest reduction in training time (e.g., 5\%) can yield substantial cost savings~\cite{dubey2024llama3,strubell2020energy,zhu2025mist, liu2025mario}.
Pipeline Parallelism (PP)~\cite{huang2019gpipe} is critical in accelerating LLM training~\cite{narayanan2021megatron2interleaved1F1B, liu2024deepseekv3,dubey2024llama3}. A typical pipeline involves three phases. \textbf{\textit{1)} \textit{Model Partition}}: the model is split into a sequence of stages, each comprising a group of consecutive layers. \textbf{\textit{2)} \textit{Model Placement}}: these stages are mapped to devices such as GPUs, and the inter-device data dependencies, where the computation of each stage depends on the result from the preceding stage~\cite{huang2019gpipe}, are accordingly established. \textbf{\textit{3)} \textit{Workload Scheduling}}: the forward and backward passes of each micro-batch are scheduled across the assigned stages following a predefined policy, while respecting the data dependencies. Due to the inherent inter-device data dependency, pipeline bubbles (i.e., device idle time) are inevitable~\cite{huang2019gpipe}.
To reduce the bubble ratio, many PP methods~\cite{zhu2025mist, sun2024adapipe, um2024metis, zheng2022alpa, fan2021dapple, harlap2018pipedream, liu2024deepseekv3, lamy2023breadth, liu2023hanayo, narayanan2021megatron2interleaved1F1B, li2021chimera, lin2024tessel, jiang2024dynapipe, qi2024zerobubble} have been proposed to optimize the \textit{static pipeline}, which remains model partition, model placement, and workload scheduling fixed even when training configurations (e.g. micro-batch size, number of model layer, number of stages) change, such as S-1F1B~\cite{shoeybi2019megatron} and GPipe~\cite{huang2019gpipe}. These methods can be regarded as \textit{partially adaptive pipelines}, since each optimizes an individual phase of the pipeline while keeping the others static. For example, I-1F1B~\cite{narayanan2021megatron2interleaved1F1B} adapts model placement by dividing stages into smaller \textit{virtual stages}; ZB~\cite{qi2024zerobubble} tunes workload scheduling by reordering the computation of micro-batches; and Mist~\cite{zhu2025mist} adjusts model partition by changing the number of layers per stage. In all cases, only one dimension of the pipeline is adapted, while the remaining dimensions remain fixed in the style of S-1F1B~\cite{shoeybi2019megatron}. Such approaches are effective primarily on relatively homogeneous model architectures (e.g., LLaMA-2~\cite{touvron2023llama2}, GPT-3~\cite{brown2020gpt3}), where computational and memory demands are well balanced across stages.
\begin{figure}
\centering
\includegraphics[width=1\linewidth]{Figures/bubble_comparison.pdf}
\vspace{-20pt}
\caption{Bubble ratios of pipeline parallelism methods on different models. The model and training configurations are set to $L=32, P=4, T=2, G=16, nmb=16$ on 8 GPUs.}
\label{fig:bubblecomp}
\vspace{-10pt}
\end{figure}
However, modern models are increasingly heterogeneous~\cite{team2025gemma, liu2024deepseekv3, bercovich2025llama253B, blakeman2025nemotronh, minimax2025minimaxm1, lieber2024jamba}, and this heterogeneity significantly increases pipeline bubbles.
For example, Gemma~\cite{team2025gemma} has a substantially larger vocabulary than LLaMA-2~\cite{touvron2023llama2} (256K vs. 32K). Beyond vocabulary size, DeepSeek~\cite{liu2024deepseekv3} integrates both Feed-Forward Network (FFN) and Mixture-of-Experts (MoE) layers~\cite{lepikhin2020gshard}, while Nemotron-H~\cite{blakeman2025nemotronh} combines Self-Attention (SA)~\cite{vaswani2017attention} with Mamba~\cite{gu2023mamba}, leading to even greater architectural heterogeneity.
As shown in Figure~\ref{fig:bubblecomp}, existing PP methods including I-1F1B~\cite{narayanan2021megatron2interleaved1F1B}, ZB~\cite{qi2024zerobubble}, and Mist~\cite{zhu2025mist} suffer from higher bubble ratios on Gemma~\cite{team2025gemma}, DeepSeek~\cite{liu2024deepseekv3}, and Nemotron-H~\cite{blakeman2025nemotronh} compared to LLaMA-2~\cite{touvron2023llama2}.
We also observe from Figure~\ref{fig:bubblecomp} that, compared with the widely used S-1F1B~\cite{shoeybi2019megatron}, prior partially adaptive PP methods often deliver only marginal improvements and sometimes even degrade efficiency on heterogeneous models. A key reason is that they optimize only one aspect of pipeline parallelism while overlooking the co-optimization of model partition, model placement, and workload scheduling. For example, I-1F1B~\cite{narayanan2021megatron2interleaved1F1B} adapts model placement alone but leaves model partition and workload scheduling untuned. This partially adaptive pipeline shows a greater bubble ratio than the static pipeline S-1F1B~\cite{shoeybi2019megatron}, as illustrated on Nemotron-H~\cite{blakeman2025nemotronh} in Figure~\ref{fig:bubblecomp}. These results suggest that efficient training of heterogeneous models requires an adaptive pipeline parallelism that co-optimizes model partition, model placement, and workload scheduling, rather than tuning them individually (as analyzed in \S~\ref{sec:demand4adaptive}).
Nevertheless, co-optimizing model partition, placement, and workload scheduling in adaptive pipeline parallelism on heterogeneous models presents challenges: \textit{1)} co-optimization is highly complex, since the optimization of each part influences the others; \textit{2)} co-optimizing these three phases results in an exponentially large search space; and \textit{3)} the pipeline executor should be able to efficiently handle the complicated communication and computation in diverse pipelines.
To address these challenges, we propose \SysName{}, an LLM training system that enables adaptive pipeline parallelism that co-optimizes the model partition, model placement, and workload scheduling. First, we build a Pipeline Performance Model (\S\ref{sec: performance_model}) that provides accurate estimates of computation, communication, and memory costs (including detailed bubble time and overlap time) for arbitrary combinations of model partition, model placement, and workload scheduling policies. Second, \PG{} (\S\ref{sec: ada pp schedule gen}) jointly optimizes the model partition, model placement, and workload scheduling policies with the performance estimation of \PPP{}. \PG{} efficiently explores the large search space, starting with predefined settings for model partition, placement, and workload scheduling. It tunes the performance bottleneck phase per iteration to progressively optimize pipeline performance, thereby accelerating pipeline generation. Third, to support adaptive execution, \SysName{} implements a unified \PE{} (\S\ref{sec:pipeline_executor}) that carefully orchestrates computation and communication instructions based on the model partition, model placement, and workload scheduling policies generated by \PG{}, providing efficient pipeline execution.
We implement \SysName{} atop PyTorch~\cite{paszke2019pytorch} and conduct extensive experiments across different scales of Gemma~\cite{team2025gemma}, DeepSeek~\cite{liu2024deepseekv2}, and Nemotron-H~\cite{blakeman2025nemotronh}. The results show that \SysName{} consistently outperforms prior PP policies, including S-1F1B~\cite{shoeybi2019megatron}, I-1F1B~\cite{narayanan2021megatron2interleaved1F1B}, ZB~\cite{qi2024zerobubble}, and Mist~\cite{zhu2025mist}, achieving average speedups of 1.34$\times$ (up to 1.54$\times$), 1.42$\times$ (up to 2.14$\times$), 1.34$\times$ (up to 1.51$\times$), and 1.20$\times$ (up to 1.27$\times$), respectively.
\section{Background}
\subsection{Distributed Training Parallelisms}
\label{subsec:pp-explanation}
Table~\ref{tab: Symbols Table} lists relevant notations used in this paper.
\begin{table}[t]
\footnotesize
\centering
\caption{Symbols used in this paper.}
\vspace{-10pt}
\begin{tabular}{p{0.4cm}p{2.5cm}|p{0.4cm}p{3.7cm}}
\toprule
\textbf{Sym.} & \textbf{Description} & \textbf{Sym.} & \textbf{Description} \\
\midrule
$L$ & Number of layers & $D$ & {Data parallelism size} \\
$S$ & Number of stages & $T$ & {Tensor parallelism size} \\
$H$ & FFN hidden size & $P$ & {Pipeline parallelism size} \\
$V$ & Vocabulary size & $E$ & {Expert parallelism size} \\
$G$ & Global batch size & $nmb$ & Number of micro-batches \\
$F$ & Forward computation & $B$ & Input gradient computation \\
& & $W$ & Parameter gradient computation\\
\bottomrule
\end{tabular}\label{tab: Symbols Table}%
\vspace{-10pt}
\end{table}%
\noindent\textbf{Data Parallelism (DP)}~\cite{li2020pytorchddp, sergeev2018horovod} splits the input training data into smaller mini-batches. Each device maintains a copy of parameters and takes its assigned mini-batches as input. When applying sharding techniques~\cite{rajbhandari2020zero,zhao2023pytorchfsdp}, the redundant memory overhead can be reduced.
\noindent\textbf{Tensor Parallelism (TP)}~\cite{shoeybi2019megatron} partitions a layer into smaller layer shards across multiple GPUs. TP alleviates the GPU memory pressure but compensates for heavy collective communication (e.g., \texttt{all-reduce}). Hence, TP is usually adopted within a single node to avoid cross-node communication~\cite{narayanan2021megatron2interleaved1F1B}.
\noindent\textbf{Pipeline Parallelism (PP)}~\cite{huang2019gpipe} introduces point to point (P2P) communication to efficiently support cross node communication. As shown in Figure~\ref{fig: pipeline plan decouple}, PP consists of three phases, including model partition, model placement, and workload scheduling. First, the model is divided into sequential pipeline stages, each comprising a set of consecutive layers. Second, these stages are assigned to $P$ groups of devices. Intermediate results are transmitted via P2P communication between devices hosting adjacent stages. Third, the input mini-batch is further subdivided into micro-batches, which are processed in a pipelined manner across all stages to improve device utilization. We next introduce these three phases of PP.
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/pipeline_plan_decoupling.pdf}
\vspace{-20pt}
\caption{Illustrations of Model Partition, Model Placement, and Workload Scheduling in Pipeline Parallelism.}
\label{fig: pipeline plan decouple}
\vspace{-10pt}
\end{figure}
\subsection{Model Partition}
At the beginning of pipeline parallelism, the model is partitioned into multiple stages. A common strategy is to evenly allocate transformer layers across stages, with the input layer assigned to the first stage and the output layer to the final stage~\cite{shoeybi2019megatron, rajbhandari2022deepspeed, liu2023hanayo, fan2021dapple, harlap2018pipedream, narayanan2021megatron2interleaved1F1B, qi2024zerobubble, liu2024deepseekv3, jiang2024dynapipe}. With the growing heterogeneity of model architectures, this model partition scheme inevitably leads to an increasing size of bubbles. To reduce bubbles, recent approaches adjust the number of layers per stage~\cite{zhu2025mist, zheng2022alpa, um2024metis, sun2024adapipe}. These methods formulate the model partition task as an Integer Linear Programming (ILP) problem, which is solved through dynamic programming~\cite{sun2024adapipe, um2024metis} or ILP solvers~\cite{gurobi, de2008z3}. While these approaches mitigate imbalance to some extent, methods such as Mist~\cite{zhu2025mist} still exhibit substantial bubbles when training heterogeneous models like Nemotron-H~\cite{blakeman2025nemotronh}, as illustrated in Figure~\ref{fig:bubblecomp}.
\subsection{Model Placement}
Model placement refers to mapping the pipeline stages onto physical devices, which determines the inter-device data dependencies. A widely adopted strategy is to sequentially map the stages to devices~\cite{fan2021dapple, qi2024zerobubble, harlap2018pipedream, sun2024adapipe, jiang2024dynapipe, zheng2022alpa, um2024metis}. This approach assumes that the number of stages equals the pipeline parallelism size ($S = P$), with each stage assigned to a unique device. To reduce bubbles at the beginning and end of each training step, I-1F1B~\cite{narayanan2021megatron2interleaved1F1B} introduces the \textit{virtual pipeline stages}, which split pipeline stages into smaller ones. Hanayo~\cite{liu2023hanayo} builds upon this idea but applies a wave-like stage-to-device mapping policy. Other approaches, such as Chimera~\cite{li2021chimera} and DualPipe~\cite{liu2024deepseekv3}, duplicate model parameters to form multiple pipelines, allowing for the concurrent execution of multiple micro-batches across model replicas. However, the same benefits can be achieved through \textit{virtual pipeline stages} without the memory redundancy introduced by parameter duplication~\cite{liu2023hanayo}. All these methods rely on manually defined model placement policies (e.g., manually setting the number of \textit{virtual pipeline stages} in I-1F1B~\cite{narayanan2021megatron2interleaved1F1B}, pipelines in Chimera~\cite{li2021chimera}, or waves in Hanayo~\cite{liu2023hanayo}), which limits their adaptivity.
\subsection{Workload Scheduling}
Based on the inter-device data dependencies defined by model placement, the workloads, consisting of the forward pass ($F$) and the backward pass of each micro-batch, are scheduled according to a predefined execution order. When the backward splitting strategy~\cite{oh2022ooo, qi2024zerobubble, liu2024deepseekv3} is applied, the backward pass is further divided into input gradient computation ($B$) and parameter gradient computation ($W$).
The execution order can be manually designed, as in GPipe~\cite{huang2019gpipe} and S-1F1B~\cite{shoeybi2019megatron}, or automatically generated, as in ZB~\cite{qi2024zerobubble} and Tessel~\cite{lin2024tessel}. Manually designed orders are fixed; therefore, they can not adapt to configuration changes. Automatic scheduling methods model the problem as a Job Shop Scheduling Problem (JSSP), which is NP hard, and employ ILP Solvers to obtain workload scheduling policies. However, the large search space makes finding the optimal workload scheduling infeasible in practice. For example, Tessel~\cite{lin2024tessel} restricts the search space to a small set of candidate patterns, which limits flexibility and performance. ZB~\cite{qi2024zerobubble} only reschedules $W$ to fill bubbles, leaving other stages underutilized. DynaPipe~\cite{jiang2024dynapipe} focuses on scheduling micro batch execution to mitigate data heterogeneity, but it lacks mechanisms to address model architectural heterogeneity.
\section{Motivation}
\label{sec:limitation-of-existing-pp}
\subsection{Increasing Heterogeneity in Model Architectures}\label{sec:trendofheter}
Nowadays, the evolution of LLMs shows a clear trend toward increasing architectural heterogeneity~\cite{lepikhin2020gshard, blakeman2025nemotronh, minimax2025minimaxm1, bercovich2025llama253B, liu2024deepseekv3, guo2025deepseekr1, yang2025qwen3}. Instead of relying solely on standard transformer blocks~\cite{vaswani2017attention}, modern LLMs integrate diverse attention and FFN mechanisms. For example, DeepSeek~\cite{liu2024deepseekv2, guo2025deepseekr1, liu2024deepseekv3} adopts dense FFNs in the first $k$ layers (with $k$ manually defined) and sparse MoE layers in the later layers, and replaces SA~\cite{vaswani2017attention} with MLA~\cite{liu2024deepseekv3}. Nemotron-LLaMA~\cite{bercovich2025llama253B} introduces variable FFN sizes within transformer blocks, while Minimax-M1~\cite{minimax2025minimaxm1} alternates between Lightening Attention~\cite{qin2024lighteningattn} and SA. Nemotron-H~\cite{blakeman2025nemotronh} combines Mamba~\cite{gu2023mamba} and SA to form a heterogeneous attention design, and Jamba~\cite{lieber2024jamba} goes further by integrating SA, Mamba, FFN, and MoE layers into a single architecture. Beyond the hidden layers, heterogeneity also emerges in the output layer, as vocabulary sizes expand rapidly in models such as Gemma~\cite{team2025gemma}, LLaMA-3~\cite{dubey2024llama3}, and Qwen~\cite{yang2025qwen3}.
This growing heterogeneity in model architectures results in larger pipeline bubbles compared to previous models, such as LLaMA-2~\cite{touvron2023llama2}, as shown in Figure~\ref{fig:bubblecomp}. This result highlights the necessity of adaptive pipeline parallelism to mitigate bubbles and improve training efficiency.
\subsection{Demand for Adaptive Pipeline Parallelism}\label{sec:demand4adaptive}
\begin{table}[t]\centering
\caption{Taxonomy of existing pipeline parallelism approaches. \fullcirc{} : support, \emptycirc{} : not support.}\label{tab:Comparison with existing pipeline plan methods}
\vspace{-10pt}
\footnotesize
% \begin{tabular}{l@{\hspace{-8pt}}c@{\hspace{5pt}}c@{\hspace{5pt}}c@{\hspace{5pt}}c}
\begin{tabular}{lccc|c}
\toprule
&\makecell{Model\\Partition\\Tuning}
&\makecell{Workload\\Scheduling\\Tuning}
&\makecell{Model\\Placement\\Tuning}
&\makecell{Co-optimi-\\zation} \\\midrule
S-1F1B~\cite{shoeybi2019megatron} &\emptycirc &\emptycirc &\emptycirc &\emptycirc\\
Alpa~\cite{zheng2022alpa} &\fullcirc &\emptycirc &\emptycirc &\emptycirc\\
Metis~\cite{um2024metis} &\fullcirc &\emptycirc &\emptycirc &\emptycirc\\
AdaPipe~\cite{sun2024adapipe} &\fullcirc &\emptycirc &\emptycirc &\emptycirc\\
Mist~\cite{zhu2025mist} &\fullcirc &\emptycirc &\emptycirc &\emptycirc\\
ZB~\cite{qi2024zerobubble} &\emptycirc &\fullcirc &\emptycirc &\emptycirc\\
DynaPipe~\cite{jiang2024dynapipe} &\emptycirc &\fullcirc &\emptycirc &\emptycirc\\
Tessel~\cite{lin2024tessel} &\emptycirc &\fullcirc &\emptycirc &\emptycirc\\
Mario~\cite{liu2025mario} &\emptycirc &\fullcirc &\emptycirc &\emptycirc\\
I-1F1B~\cite{narayanan2021megatron2interleaved1F1B} &\emptycirc &\emptycirc &\fullcirc &\emptycirc\\
Chimera~\cite{li2021chimera} &\emptycirc &\emptycirc &\fullcirc &\emptycirc\\
Hanayo~\cite{liu2023hanayo} &\emptycirc &\emptycirc &\fullcirc &\emptycirc\\
\SysName{} (Ours) &\fullcirc &\fullcirc &\fullcirc &\fullcirc\\
\bottomrule
\end{tabular}
\vspace{-10pt}
\end{table}
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/motivation.pdf}
\vspace{-20pt}
\caption{\textbf{[Motivation].} Illustration of co-optimizing workload scheduling, model partition, and model placement for accelerating training on a heterogeneous model with a large vocabulary size (e.g. Gemma~\cite{team2025gemma}) with $L=32, P=4, nmb=4$.}
\label{fig:case}
\vspace{-10pt}
\end{figure}
Reducing pipeline bubbles in heterogeneous models calls for adaptive pipeline parallelism with co-optimization of model partition, placement, and workload scheduling. However, prior approaches typically optimize only a single aspect of the pipeline, and thus lack the co-optimization of all three dimensions, as summarized in Table~\ref{tab:Comparison with existing pipeline plan methods}.
Figure~\ref{fig:case} presents a case study of co-optimizing model partition, model placement, and workload scheduling on a Gemma-like model~\cite{team2025gemma}. We use the mainstream S-1F1B pipeline~\cite{shoeybi2019megatron} as the baseline.
First, we optimize workload scheduling under the baseline model partition and model placement by advancing $F$ and $B$, then delaying $W$ within the memory constraint, which yields a 1.28$\times$ speedup compared to the baseline.
Second, since $\text{GPU}_{4}$ exhibits longer execution time than other GPUs, we redistribute the layers to tune the model partition and mitigate the computational imbalance among devices, achieving a 1.49$\times$ speedup over the baseline.
Finally, we refine model placement to make each computation finer-grained and apply workload scheduling optimization on the new model partition and model placement. These optimizations substantially reduce pipeline bubbles, resulting in a 1.74$\times$ overall speedup compared to the baseline.
This case study demonstrates that adaptive pipeline parallelism, achieved through the co-optimization of model partition, placement, and workload scheduling, significantly contributes to reducing bubbles and improving training efficiency on heterogeneous models.
\subsection{Challenges in Pipeline Generation and Execution}\label{Lack of Unified pipeline plan Executor}
\textbf{\textit{Generation}}:
The large search space poses a fundamental challenge to the efficiency of co-optimizing model partition, model placement, and workload scheduling. As shown in Figure~\ref{fig:searching_space}, the number of cases for model partition, model placement, and workload scheduling policies grows exponentially with the number of layers, stages, and micro-batches, respectively. This explosive growth in the search space makes the DP-~\cite{zheng2022alpa,um2024metis,sun2024adapipe} or ILP-based~\cite{lin2024tessel,qi2024zerobubble,zheng2022alpa,liu2025mario} methods incur impractically long search time (Section~\S\ref{sec:pgt}). To make the search tractable, some approaches~\cite{lin2024tessel,qi2024zerobubble} manually restrict the search space. However, such restrictions risk excluding high-performance pipeline configurations.
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/searching_space.pdf}
\vspace{-20pt}
\caption{\textbf{[Motivation].} The vast search space calls for an efficient pipeline generation method.}
\label{fig:searching_space}
\vspace{-10pt}
\end{figure}
\textbf{\textit{Execution}}:
To enable adaptive pipeline parallelism, a unified executor is essential. Prior approaches typically rely on execution engines tailored to specific workload scheduling policies~\cite{shoeybi2019megatron, fan2021dapple, narayanan2021megatron2interleaved1F1B, sun2024adapipe, zhu2025mist}. Although some methods support adaptive workload scheduling~\cite{lin2024tessel, qi2024zerobubble, liu2025mario, jiang2024dynapipe}, co-optimizing the pipeline between model partition, model placement, and workload scheduling substantially increases the complexity of communication dependencies. Consequently, these partially adaptive methods are unable to adapt to diverse model partition and model placement policies. Therefore, an effective executor should be flexible enough to support diverse model partition, model placement, and workload scheduling policies, while remaining efficient in orchestrating fine-grained computation and communication.
\section{\SysName{} Design}
\subsection{Overview}
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/overview.pdf}
\vspace{-20pt}
\caption{\SysName{} system design overview.}
\label{fig: overview}
\vspace{-10pt}
\end{figure}
Figure~\ref{fig: overview} shows the workflow of \SysName{}:
\wcircle{1} \PPP{} evaluates the performance of a given pipeline under specified input parameters.
\wcircle{2} \PG{} leverages these estimates to iteratively tune model partition, model placement, and workload scheduling, and then generates the co-optimized pipeline.
\wcircle{3} \PE{} executes the co-optimized pipeline while applying communication optimizations to further improve efficiency.
\subsection{Pipeline Performance Model}\label{sec: performance_model}
To support the co-optimization of model partition, model placement, and workload scheduling, we construct a fine-grained \PPP. It takes training settings, profiled data, and pipeline configurations as input and outputs the execution time of each device (e.g., $T_d$) and its memory cost (e.g., $M_d$). These outputs contain detailed execution metrics such as $\text{BubbleTime}(d)$, $\text{OverlapTime}(d)$, $A_d$, and $G_d$, which are used to estimate pipeline performance and guide subsequent tuning (\S\ref{sec: ada pp schedule gen}). The notations used in this section are summarized in Table~\ref{tab: notations}, and Algorithm~\ref{alg: performance_model} presents the performance modelling procedure in three \textbf{Steps}.
\begin{table}[t]
\centering
\caption{Symbols used in pipeline performance model.}
\vspace{-10pt}
\begin{tabularx}{\linewidth}{@{}lX@{}}
\toprule
\textbf{Symbol} & \textbf{Description} \\
\midrule
$C_s$ & Computation cost of stage $s$. \\
$M_s$ & Memory cost of stage $s$. \\
$C_d$ & Computation cost of device $d$. \\
$M_d$ & Memory cost of device $d$. \\
$M_d^{\text{capacity}}$ & Memory capacity of device $d$.\\
$T_d$ & Runtime on device $d$. \\
$A_d$ & Activation memory on device $d$. \\
$G_d$ & Gradient memory on device $d$. \\
$\text{Layers}(s)$ & Set of layers assigned to stage $s$. \\
$\text{Stages}(d)$ & Set of stages assigned to device $d$. \\
$\text{BubbleTime}(d)$ & Total pipeline bubble time on device $d$. \\
$\text{OverlapTime}(d)$ & Time overlap on device $d$. \\
\bottomrule
\end{tabularx}\label{tab: notations}
\vspace{-10pt}
\end{table}
\textbf{Step 1}: Given a model partition, each stage is assigned a different number and type of layers, each associated with profiled computation and memory costs. The total computation and memory cost of a stage is determined by accumulating the cost of executing a specific computation type (e.g., $F$, $B$, or $W$) across all layers assigned to that stage.
\textbf{Step 2}: Given a model placement, the computation and memory cost of each device is obtained by summing the costs of all stages assigned to it. In addition, communication time and the overlapping time between communication and computation are considered, both of which are determined by the workload scheduling results.
\textbf{Step 3}: Given the model partition, model placement, and workload scheduling results, \PPP{} simulates the execution behavior of each device and identifies when and where device idle time (BubbleTime($d$)) occurs, and detects opportunities where communication can be overlapped with computation (OverlapTime($d$)). Based on the simulation results, \PPP{} provides feedback for \PG{} (\S\ref{sec: ada pp schedule gen}) to refine model partition, model placement, and workload scheduling, thereby reducing pipeline bubbles and increasing overlap time.
\begin{algorithm}[t]
\caption{Pipeline Performance Model}
\begin{algorithmic}[1]
\State \textbf{Input:} \textit{Pipeline conf., Training conf., Profiled data.}
\State \textbf{Output:} List of $T_d, M_d$
\vspace{0.5em}
\State \textbf{/* Step 1: Layer-level cost aggregation */}
\For{each stage $s$} \Comment{{\textit{Model Partition}}}
\State $C_s \gets \sum\limits_{l \in \text{Layers}(s)} \text{ProfiledCompCost}(l)$
\State $M_s \gets \sum\limits_{l \in \text{Layers}(s)} \text{ProfiledMemCost}(l)$
\EndFor
\vspace{0.5em}
\State \textbf{/* Step 2: Stage-level cost aggregation */}
\For{each device $d$} \Comment{{\textit{Model Placement}}}
\State $C_d \gets \sum\limits_{s \in \text{Stages}(d)} C_s +\text{ProfiledCommCost}(s)$
\State $M_d \gets \sum\limits_{s \in \text{Stages}(d)} M_s$
\EndFor
\vspace{0.5em}
\State \textbf{/* Step 3: Runtime and memory estimation */}
\For{each device $d$} \Comment{{\textit{Workload Scheduling}}}
\State $T_d \gets C_d + \text{BubbleTime}(d) - \text{OverlapTime}(d)$
\State $A_d \gets \text{TotalActMem}(d)$
\State $G_d \gets \text{TotalGradMem}(d)$
\State $M_d \gets M_d + A_d + G_d$
\EndFor
\end{algorithmic}\label{alg: performance_model}
\end{algorithm}
\subsection{Pipeline Generator}\label{sec: ada pp schedule gen}
\PG{} is designed to optimize the pipeline by leveraging the performance estimation of \PPP{} and then generating pipelines with co-optimized model partition, model placement, and workload scheduling.
\textbf{Optimization Objective}.
The pipeline execution time is determined by the slowest device, which has the maximum $T_d$ among all devices. In addition, the memory usage $M_d$ on each device must not exceed its memory capacity $M_d^{\text{capacity}}$. Then the pipeline optimization objective is defined as follows:
\begin{align}\label{equ: obj}
\min \quad & \max_{d} T_{d} \\
\text{s.t.} \quad & \label{equ: mem constraint}
M_{d}^{\text{capacity}} \geq M_{d} \quad \forall d
\end{align}
\textbf{Efficient Exploration of Pipelines}.
As illustrated in Figure~\ref{fig: ppgenerator}, \PG{} reduces the search space by selecting representative baseline pipelines and using them as starting points for optimization.
For model partition, we adopt the policies from S-1F1B~\cite{shoeybi2019megatron} and Mist~\cite{zhu2025mist}. For model placement, the baselines include S-1F1B~\cite{shoeybi2019megatron}, I-1F1B~\cite{narayanan2021megatron2interleaved1F1B}, and Hanayo~\cite{liu2023hanayo}. For workload scheduling, we consider S-1F1B~\cite{shoeybi2019megatron} and ZB~\cite{qi2024zerobubble}.
The performance of these baselines is estimated by the \PPP{}, which enables \PG{} to prune low-performing candidates, thereby accelerating the exploration of pipelines.
From a chosen baseline, \PG{} iteratively tunes the model partition, model placement, and workload scheduling using performance estimations from the \PPP{}. In each iteration, \PG{} identifies the bottleneck phase of the pipeline (e.g., the phase contributing the largest BubbleTime($d$)) and tunes it accordingly. This phase-by-phase tuning method avoids the combinatorial explosion of jointly searching model partition, model placement, and workload scheduling policies, while directly addressing pipeline performance bottlenecks. Since model partition or model placement changes affect execution time and inter-device dependencies, workload scheduling is adjusted in tandem. If a tuning step degrades pipeline performance, it is rolled back, and alternative adjustments are attempted. The tuning process repeats until no further improvement can be achieved, at which point \PG{} outputs the co-optimized pipeline.
\begin{figure}
\centering
\includegraphics[width=1\linewidth]{Figures/pipeline_generator.pdf}
\vspace{-20pt}
\caption{Illustrations of repeat tuning (a) model partition, (b) model placement, and (c) workload scheduling.}
\label{fig: ppgenerator}
\vspace{-10pt}
\end{figure}
\textbf{Model Partition Tuning}.
As shown in \textbf{Step 1} of Algorithm~\ref{alg: performance_model}, the model partition determines the computation and memory cost of each stage. Previous works~\cite{zhu2025mist, zheng2022alpa, um2024metis} focus on balancing $C_d$ across devices without considering other factors that affect execution time. In contrast, our method incorporates the impact of model placement and workload scheduling, since the device execution time $T_d$ depends not only on $C_d$ but also on BubbleTime($d$) and OverlapTime($d$). Specifically, as illustrated in Figure~\ref{fig: ppgenerator}(a), for each model partition policy, we perform workload scheduling, analyze BubbleTime($d$) on devices, and then tune the model partition by transferring layers from the stage with the lowest bubble ratio to the stage with the highest bubble ratio. This process continues until the difference in BubbleTime($d$) across devices is smaller than the maximum $C_s$.
\textbf{Model Placement Tuning}.
\PG{} also tunes model placement to reduce bubbles. As shown in Figure~\ref{fig: ppgenerator}(b), first, \PG{} adjusts the stage-device mapping by permuting the assignment of layers of each stage across devices, placing adjacent layers on different devices. This permutation increases the effective number of pipeline stages, resulting in more fine-grained computation tasks. Second, \PG{} performs workload scheduling on the permuted model placement while keeping the model partition fixed. If BubbleTime($d$) is reduced after permutation, the new placement is accepted as the updated baseline, and the process is repeated. Otherwise, the permutation is rolled back. To accelerate model placement tuning, \PG{} employs grouped permutations, which simultaneously permute all layers on a stage, rather than one layer at a time.
\textbf{Workload Scheduling Tuning}.
Figure~\ref{fig: ppgenerator}(c) shows three policies in tuning workload scheduling. First, given the model partition and placement, \PG{} advances the $F$ first, followed by the $B$, and finally the $W$ within the memory constraints to fill bubbles. This workload scheduling strategy is based on how data dependencies are handled. Specifically, both the $F$ pass and $B$ involve inter-device data dependencies. For instance, the $F$ on stage $i$ depends on the result from stage $i-1$, and the $B$ pass on stage $i$ depends on the result from stage $i+1$.
In contrast, $W$ has no inter-device data dependencies since it relies only on $B$ on the same device; thus, $W$ is suited for filling pipeline bubbles.
Second, \PG{} adopts an overlap-aware scheduling policy to increase OverlapTime($d$). The key idea is to avoid scheduling dependent computation tasks consecutively and instead delay certain computations to enable communication overlap. For example, consider two devices that need to exchange intermediate results of $F_i$ and $B_j$. If the subsequent computations that depend on these results are scheduled immediately, communication and computation are forced to proceed sequentially. In contrast, by first executing other independent computations and delaying the dependent ones, communication can be overlapped with computation. This overlap-aware policy not only increases OverlapTime($d$) but also reduces BubbleTime($d$).
Third, to meet the Constraint~\ref{equ: mem constraint}, \PG{} advances some $B$ and $W$ to advance the memory release time and avoid Out-of-Memory (OOM) issues. During workload scheduling, \PG{} records the workload scheduling results and identifies potential OOM time. It then advances the execution of the latest $B$ and $W$ to ahead of this time to free up memory, continuing this process until all potential OOM errors are resolved.
\section{Evaluation}
\subsection{Experimental Settings}
\def\colhspace{5}
\begin{figure*}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/e2e_performance.pdf}
\vspace{-20pt}
\caption{End-to-end training throughput of different PP methods on various model types and model sizes with input sequence length = (2K, 4K). The numbers above the bars indicate the normalized speedup over S-1F1B~\cite{shoeybi2019megatron}.}
\label{fig:e2etgs}
\vspace{-5pt}
\end{figure*}
\noindent\textbf{Implementation.} We implement \SysName{} on top of Pytorch~\cite{paszke2019pytorch} with approximately 14,000 lines of Python code.
\noindent\textbf{Testbed.} We conduct experiments on a cluster equipped with 128 NVIDIA H800 GPUs. Each node connects 8 GPUs via NVLink, while inter-node communication is handled through InfiniBand. The training performance is tested on \texttt{Gemma}~\cite{team2025gemma}, \texttt{DeepSeek}~\cite{liu2024deepseekv3}, and \texttt{Nemotron-H}~\cite{blakeman2025nemotronh}. The detailed model parameter configurations are listed in Table~\ref{tab:model_config}.
\def\modelcolhspace{5}
\begin{table}[h]
\caption{Model parameter configurations.}
\vspace{-10pt}
\centering
\footnotesize
\begin{tabular}{l@{\hspace{\modelcolhspace pt}}c@{\hspace{\modelcolhspace pt}}c@{\hspace{\modelcolhspace pt}}c@{\hspace{\modelcolhspace pt}}c@{\hspace{\modelcolhspace pt}}c@{\hspace{\modelcolhspace pt}}c}
\toprule
\textbf{Model} & \textbf{Size} & \textbf{$L$} & \textbf{$V$} & \textbf{$H$} & \textbf{FFN Type} & \textbf{Attn. Type} \\
\midrule
\multirow{3}{*}{Gemma~\cite{team2025gemma}}
& Small & 32 & 256K & 1536 & FFN & SA \\
& Medium & 64 & 512K & 1536 & FFN & SA \\
& Large & 128 & 1024K & 1536 & FFN & SA \\
\midrule
\multirow{3}{*}{DeepSeek~\cite{liu2024deepseekv3}}
& Small & 16 & 128K & 2048 & FFN+MoE & MLA \\
& Medium & 32 & 256K & 2048 & FFN+MoE & MLA \\
& Large & 64 & 512K & 2048 & FFN+MoE & MLA \\
\midrule
\multirow{3}{*}{Nemotron-H~\cite{blakeman2025nemotronh}}
& Small & 28 & 128K & 1024 & FFN & SA+Mamba \\
& Medium & 56 & 256K & 1024 & FFN & SA+Mamba \\
& Large & 112 & 512K & 1024 & FFN & SA+Mamba \\
\bottomrule
\end{tabular}
% \vspace{-10pt}
\label{tab:model_config}
\end{table}
\noindent\textbf{Baselines.} To ensure fair and reproducible comparisons, we integrate the open-source implementations of selected baselines into our framework for evaluation against \SysName{}.
\textbf{1) S-1F1B}~\cite{shoeybi2019megatron}, a widely adopted PP strategy in mainstream training frameworks such as Megatron-LM~\cite{shoeybi2019megatron} and DeepSpeed~\cite{rajbhandari2022deepspeed}.
\textbf{2) I-1F1B}~\cite{narayanan2021megatron2interleaved1F1B} optimizes the model placement of S-1F1B by leveraging \textit{virtual pipeline stages} to reduce pipeline bubbles.
\textbf{3) ZB}~\cite{qi2024zerobubble}, a pipeline approach that incorporates adaptive workload scheduling upon S-1F1B to fill pipeline bubbles.
\textbf{4) Mist}~\cite{zhu2025mist}, a state-of-the-art automatic training framework that supports adaptive model partition.
We do not include Tessel~\cite{lin2024tessel} as a baseline because its code is not publicly available. AdaPipe~\cite{sun2024adapipe} and Mario~\cite{liu2025mario} are also excluded, as their recomputation optimization techniques are orthogonal to both \SysName{} and the baselines considered. It is worth noting that recomputation~\cite{chen2016recomp} can also be incorporated into \SysName{}, which we leave for future work.
\noindent\textbf{Training Configurations.} We evaluate with $P=4,8,16$ and conduct a grid search over $D$, $T$, and $E$ to determine the optimal parallelism settings. Training throughput is measured in \underline{T}okens per \underline{S}econd (\underline{TS}). We further assess the performance of each method under varying input sequence lengths, numbers of micro-batches ($nmb$), and GPU counts.
\subsection{E2E Performance and Analysis}
As shown in Figure~\ref{fig:e2etgs}, \SysName{} consistently achieves the superior training throughput across diverse models and scales, highlighting its strength in handling model heterogeneity. Compared with S-1F1B, I-1F1B, ZB, and Mist, \SysName{} delivers average speedups of up to 1.34$\times$, 1.42$\times$, 1.34$\times$, and 1.20$\times$ across Gemma, DeepSeek, and Nemotron-H models, under both 2K and 4K sequence length settings. These gains translate to throughput improvements ranging from 11\% to over 59\%, demonstrating both the robustness and generality of \SysName{} across heterogeneous models.
In contrast, existing PP methods fall short in efficiently training heterogeneous models. ZB shows only marginal improvements (only 1.02$\times$ over S-1F1B), while I-1F1B even degrades throughput by up to 22\% on Nemotron-H (Large). These results demonstrate that optimizing a single phase of the pipeline is insufficient to address the increasing bubbles caused by model heterogeneity. By co-optimizing the model partition, model placement, and workload scheduling policies, \SysName{} achieves higher training throughput than these baselines, indicating the efficiency of co-optimization in reducing bubbles in heterogeneous models.
\subsection{Throughput with Different Sequence Lengths}\label{sec: vary_seq}
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/e2e_performance_varyseq.pdf}
\vspace{-20pt}
\caption{Throughput of \SysName{}, Mist, ZB, I-1F1B, and S-1F1B on Nemotron-H (Large) with $P=8, T=4, G=64, nmb=64$ across various input sequence lengths.}
\label{fig:vary_len}
\vspace{-10pt}
\end{figure}
Figure~\ref{fig:vary_len} shows that \SysName{} sustains high training efficiency across sequence lengths ranging from 1K to 32K.
Across all tested sequence lengths, \SysName{} consistently outperforms the baselines and achieves speedups up to 1.54$\times$, 2.14$\times$, 1.51$\times$, and 1.27$\times$, over S-1F1B, I-1F1B, ZB, and Mist, respectively. Compared with these baselines, \SysName{} obtains higher improvements as the sequence length increases from 1K to 32K. When compared with Mist, \SysName{} achieves relatively smaller but still steady improvements, with average speedups of 1.15$\times$ across all sequence lengths. These results highlight the efficiency of \SysName{} in handling training workloads across different sequence lengths.
\subsection{Ablation Study of Pipeline Co-optimization}
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/ablation_study.pdf}
\vspace{-20pt}
\caption{Ablation study of pipeline co-optimization with \raisebox{-0.2ex}{\Large\ding{172}} adaptive model placement, \raisebox{-0.2ex}{\Large\ding{173}} adaptive workload scheduling, and \raisebox{-0.2ex}{\Large\ding{174}} adaptive model partition across models.}
\label{fig:ablation_study}
\vspace{-10pt}
\end{figure}
Figure~\ref{fig:ablation_study} shows that co-optimization of adaptive workload scheduling, model partition, and model placement yields substantial improvements in training efficiency, whereas optimizing only one of them provides limited gains or may even degrade performance. Specifically, when co-optimizing model placement, workload scheduling, and model partition on the baseline method, performance improves by 1.32$\times$, 1.37$\times$, and 1.33$\times$ on Gemma, DeepSeek, and Nemotron-H, respectively. In contrast, tuning only a single phase leads to marginal benefits or performance degradation (e.g., applying model placement alone on Nemotron-H results in a 16\% slowdown, consistent with the I-1F1B results in Figure~\ref{fig:e2etgs}). Moreover,
Figure~\ref{fig:realtraces}(a), (c), and (e) show the real traces of S-1F1B, Mist, and \SysName{}. It is evident that by co-optimizing the model partition, model placement, and workload scheduling, \SysName{} reduces pipeline bubbles (i.e., the blank area in the figure) and exhibits improved training efficiency.
% \begin{figure}
% \centering
% \includegraphics[width=1\linewidth]{Figures/real traces.pdf}
% \vspace{-20pt}
% \caption{Real traces of (a) S-1F1B, (b) Mist, and (c) \SysName{} on a Nemotron-H (Large) with $P=8, T=4, G=64,nmb=64,Seqlen=4k$. Purple rectangles denote GPU runtime kernels, while white rectangles indicate bubbles.}
% \label{fig:realtraces}
% \end{figure}
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/real+sim_traces.pdf}
\vspace{-20pt}
\caption{Real ((a), (c), (e)) and \PG{} simulated ((b), (d), (f)) traces of S-1F1B, Mist, and \SysName{}, respectively, on Nemotron-H (Large) with $P=8, T=4, G=64, nmb=64, SeqLen=4K$. In real traces, purple rectangles represent GPU kernels, and white rectangles represent bubbles. In simulated traces, yellow/blue rectangles represent GPU kernels, while white rectangles indicate bubbles.}
\label{fig:realtraces}
\vspace{-10pt}
\end{figure}
\subsection{Performance Model Fidelity}\label{sec:sim acc}
As shown in Figure~\ref{fig:sim_thro_mem}, our \PPP{} achieves high modeling accuracy, with an average throughput prediction error of 2.12\% across different methods for Nemontron-H models at $Seqlen=4K$. The predicted throughput, which is normalized to S-1F1B since the modeling result is relative rather than absolute, closely matches the profiled real throughput results. It is worth noting that the maximum errors are 4.42\% for \SysName{}, 4.55\% for I-1F1B, 2.12\% for ZB, and 6.57\% for Mist. Furthermore, as shown in Figure~\ref{fig:realtraces}, the simulated pipeline traces from \PG{} closely match the real traces. These results collectively demonstrate the high accuracy of our \PPP{}.
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/model_fidelity.pdf}
\vspace{-20pt}
\caption{Pipeline performance model fidelity experimental results on Nemotron-H models with $Seqlen=4K$.}
\label{fig:sim_thro_mem}
\vspace{-10pt}
\end{figure}
\subsection{Pipeline Generation Time}\label{sec:pgt}
As shown in Figure~\ref{fig:generationtime}, \SysName{} demonstrates superior efficiency and scalability in pipeline generation compared with the ILP solver-based approaches~\cite{qi2024zerobubble,lin2024tessel,qi2024zbv}. \texttt{ILP Solver (Simple)} applies only adaptive workload scheduling, while \texttt{ILP Solver} further incorporates adaptive workload scheduling, model partition, and model placement. As the problem size increases, ILP-based methods incur rapidly growing overhead. For small cases, we measure the actual solving time (less than $10^{5}$ seconds), and for larger cases, we approximate it using \texttt{scipy.optimize.curve\_fit}, since directly solving them is impractical. In contrast, \SysName{} requires significantly less time, often completing even substantial pipeline generation problems (e.g., large models, $P$, and 256 micro-batches) within 100 seconds. Overall, \SysName{} achieves remarkable efficiency and scalability in pipeline generation.
\begin{figure}[t]
\centering
\includegraphics[width=1\linewidth]{Figures/solving_time.pdf}
\vspace{-20pt}
\caption{Pipeline generation time of ILP solver method~\cite{qi2024zerobubble, lin2024tessel, qi2024zbv} and \SysName{} across different model size (e.g. \underline{S}mall, \underline{M}edium, and \underline{L}arge) and parallelism configurations.}
\label{fig:generationtime}
\vspace{-10pt}
\end{figure}
\subsection{Scalability Experimental Results}
\textbf{Strong Scaling.}
As shown in Figure~\ref{fig:strong_scaling}, \SysName{} demonstrates superior scalability in our strong scaling experiments. We fix the sequence length, model size of the Nemotron-H (Large), pipeline parallelism size, and the number and size of micro-batches, while increasing the number of GPUs from 8 to 128. Across all configurations, \SysName{} consistently delivers the highest throughput. Furthermore, when scaling from 8 to 128 GPUs, \SysName{} achieves a scaling efficiency of 534\%, outperforming the second-best Mist, which attains 514\%. These results demonstrate that \SysName{} can efficiently accelerate the training tasks with more GPUs.
\begin{figure}[t]
\centering
\includegraphics[width=1.0\linewidth]{Figures/strong_scaling.pdf}
\vspace{-20pt}
\caption{Strong scaling experimental results of training Nemontron-H (Large) with $Seqlen$ = 4K on 128 GPUs.}
\label{fig:strong_scaling}
\vspace{-10pt}
\end{figure}
\textbf{Weak Scaling.}
As demonstrated in Figure~\ref{fig:weak_scaling}, \SysName{} maintains its computational efficiency in our weak scaling experiments. We fix the sequence length, model size of the Nemotron-H (Large), pipeline parallelism size, and the number and size of micro-batches, while increasing the number of GPUs from 8 to 128 and the global batch size from 32 to 512. Experimental results show that \SysName{} outperforms the baselines in throughput across all configurations. Moreover, when scaling from 8 to 128 GPUs, \SysName{} achieves a speedup of 519\%, demonstrating its scalability in training larger global batch sizes with larger clusters.
\begin{figure}[t]
\centering
\includegraphics[width=1.0\linewidth]{Figures/weak_scaling.pdf}
\vspace{-20pt}
\caption{Weak scaling experimental results of training Nemontron-H (Large) with $Seqlen$ = 4K on 128 GPUs.}
\label{fig:weak_scaling}
\vspace{-10pt}
\end{figure}
\section{Related Works}
\noindent\textbf{{Token-level pipeline.}}
TeraPipe~\cite{li2021terapipe} introduces token-level pipeline parallelism by splitting input sequences into finer-grained shards, which can be integrated into existing PP methods. Subsequent works further combine the existing PP methods with the token-level pipeline parallelism to mitigate pipeline bubbles~\cite{sun2024seq1f1b} and alleviate memory pressure~\cite{li2025slimpipe,chen2025SPPO}.
\noindent\textbf{Data heterogeneity.} Training dataset has different lengths of sequences, making micro-batch execution time vary~\cite{zhang2024disttrain, wang2025flexsp, jiang2024dynapipe}. Some works apply micro-batch reordering~\cite{wang2025flexsp,zhang2024disttrain,jiang2024dynapipe}, adaptive parallelism settings~\cite{wang2025flexsp}, or balanced data distribution~\cite{ge2025bytescale} to alleviate bubbles caused by data heterogeneity.
\noindent\textbf{{Memory optimizations.}}
To reduce memory overhead, \textit{recomputation}~\cite{chen2016recomp} is proposed to discard the activations and recompute them before using. \textit{ZeRO}\cite{rajbhandari2020zero,ren2021zero-offload,rajbhandari2021zero-infinity} uses GPU memory, CPU memory, or even SSD to alleviate memory pressure. These techniques are orthogonal to PP and can be combined to reduce memory usage~\cite{liu2025mario, sun2024adapipe}.
Other works focus on addressing the memory imbalance in pipelines~\cite{kim2023bpipe, yeung2024vocpara, li2025slimpipe} by adjusting the memory allocation among stages.
\noindent\textbf{Automatic parallelization.}
Many works~\cite{li2022amp, miao2022galvatron, lai2023merak, liu2024aceso} focus on automatically finding optimal parallelism combinations to improve training performance. To reduce bubbles, some works~\cite{um2024metis,zheng2022alpa,zhu2025mist} adjust the number of layers of each stage. However, the co-optimizing of model partition, model placement, and workload scheduling is not considered.
\section{Conclusion}
We propose \SysName{}, an LLM training system with adaptive pipeline parallelism. To alleviate the increasing pipeline bubbles caused by model heterogeneity, \SysName{} generates pipelines with co-optimized model partition, model placement, and workload scheduling policies, and executes these pipelines with communication optimizations. Extensive experimental results demonstrate that compared with the existing PP methods, \SysName{} achieves superior training efficiency on various types and scales of heterogeneous LLMs.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{63}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Bercovich et~al\mbox{.}(2025)]%
{bercovich2025llama253B}
\bibfield{author}{\bibinfo{person}{Akhiad Bercovich}, \bibinfo{person}{Itay Levy}, \bibinfo{person}{Izik Golan}, \bibinfo{person}{Mohammad Dabbah}, \bibinfo{person}{Ran El-Yaniv}, \bibinfo{person}{Omri Puny}, \bibinfo{person}{Ido Galil}, \bibinfo{person}{Zach Moshe}, \bibinfo{person}{Tomer Ronen}, \bibinfo{person}{Najeeb Nabwani}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Llama-nemotron: Efficient reasoning models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2505.00949}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Blakeman et~al\mbox{.}(2025)]%
{blakeman2025nemotronh}
\bibfield{author}{\bibinfo{person}{Aaron Blakeman}, \bibinfo{person}{Aarti Basant}, \bibinfo{person}{Abhinav Khattar}, \bibinfo{person}{Adithya Renduchintala}, \bibinfo{person}{Akhiad Bercovich}, \bibinfo{person}{Aleksander Ficek}, \bibinfo{person}{Alexis Bjorlin}, \bibinfo{person}{Ali Taghibakhshi}, \bibinfo{person}{Amala~Sanjay Deshmukh}, \bibinfo{person}{Ameya~Sunil Mahabaleshwarkar}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Nemotron-h: A family of accurate and efficient hybrid mamba-transformer models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2504.03624}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Brown(2020)]%
{brown2020gpt3}
\bibfield{author}{\bibinfo{person}{Tom~B Brown}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Language models are few-shot learners}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2005.14165}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Chen et~al\mbox{.}(2025)]%
{chen2025SPPO}
\bibfield{author}{\bibinfo{person}{Qiaoling Chen}, \bibinfo{person}{Shenggui Li}, \bibinfo{person}{Wei Gao}, \bibinfo{person}{Peng Sun}, \bibinfo{person}{Yonggang Wen}, {and} \bibinfo{person}{Tianwei Zhang}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{SPPO:Efficient Long-sequence LLM Training via Adaptive Sequence Pipeline Parallel Offloading}.
\newblock
\newblock
\showeprint[arxiv]{2503.10377}~[cs.DC]
\urldef\tempurl%
\url{https://arxiv.org/abs/2503.10377}
\showURL{%
\tempurl}
\bibitem[Chen et~al\mbox{.}(2016)]%
{chen2016recomp}
\bibfield{author}{\bibinfo{person}{Tianqi Chen}, \bibinfo{person}{Bing Xu}, \bibinfo{person}{Chiyuan Zhang}, {and} \bibinfo{person}{Carlos Guestrin}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Training deep nets with sublinear memory cost}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1604.06174}} (\bibinfo{year}{2016}).
\newblock
\bibitem[De~Moura and Bj{\o}rner(2008)]%
{de2008z3}
\bibfield{author}{\bibinfo{person}{Leonardo De~Moura} {and} \bibinfo{person}{Nikolaj Bj{\o}rner}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Z3: An efficient SMT solver}. In \bibinfo{booktitle}{\emph{International conference on Tools and Algorithms for the Construction and Analysis of Systems}}. Springer, \bibinfo{pages}{337--340}.
\newblock
\bibitem[Dubey et~al\mbox{.}(2024)]%
{dubey2024llama3}
\bibfield{author}{\bibinfo{person}{Abhimanyu Dubey}, \bibinfo{person}{Abhinav Jauhri}, \bibinfo{person}{Abhinav Pandey}, \bibinfo{person}{Abhishek Kadian}, \bibinfo{person}{Ahmad Al-Dahle}, \bibinfo{person}{Aiesha Letman}, \bibinfo{person}{Akhil Mathur}, \bibinfo{person}{Alan Schelten}, \bibinfo{person}{Amy Yang}, \bibinfo{person}{Angela Fan}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{The llama 3 herd of models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2407.21783}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Fan et~al\mbox{.}(2021)]%
{fan2021dapple}
\bibfield{author}{\bibinfo{person}{Shiqing Fan}, \bibinfo{person}{Yi Rong}, \bibinfo{person}{Chen Meng}, \bibinfo{person}{Zongyan Cao}, \bibinfo{person}{Siyu Wang}, \bibinfo{person}{Zhen Zheng}, \bibinfo{person}{Chuan Wu}, \bibinfo{person}{Guoping Long}, \bibinfo{person}{Jun Yang}, \bibinfo{person}{Lixue Xia}, {et~al\mbox{.}}} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{DAPPLE: A pipelined data parallel approach for training large models}. In \bibinfo{booktitle}{\emph{Proceedings of the 26th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming}}. \bibinfo{pages}{431--445}.
\newblock
\bibitem[Ge et~al\mbox{.}(2025)]%
{ge2025bytescale}
\bibfield{author}{\bibinfo{person}{Hao Ge}, \bibinfo{person}{Junda Feng}, \bibinfo{person}{Qi Huang}, \bibinfo{person}{Fangcheng Fu}, \bibinfo{person}{Xiaonan Nie}, \bibinfo{person}{Lei Zuo}, \bibinfo{person}{Haibin Lin}, \bibinfo{person}{Bin Cui}, {and} \bibinfo{person}{Xin Liu}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{ByteScale: Communication-Efficient Scaling of LLM Training with a 2048K Context Length on 16384 GPUs}. In \bibinfo{booktitle}{\emph{Proceedings of the ACM SIGCOMM 2025 Conference}}. \bibinfo{pages}{963--978}.
\newblock
\bibitem[Gu and Dao(2023)]%
{gu2023mamba}
\bibfield{author}{\bibinfo{person}{Albert Gu} {and} \bibinfo{person}{Tri Dao}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Mamba: Linear-time sequence modeling with selective state spaces}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2312.00752}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Guo et~al\mbox{.}(2025)]%
{guo2025deepseekr1}
\bibfield{author}{\bibinfo{person}{Daya Guo}, \bibinfo{person}{Dejian Yang}, \bibinfo{person}{Haowei Zhang}, \bibinfo{person}{Junxiao Song}, \bibinfo{person}{Ruoyu Zhang}, \bibinfo{person}{Runxin Xu}, \bibinfo{person}{Qihao Zhu}, \bibinfo{person}{Shirong Ma}, \bibinfo{person}{Peiyi Wang}, \bibinfo{person}{Xiao Bi}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2501.12948}} (\bibinfo{year}{2025}).
\newblock
\bibitem[{Gurobi Optimization, LLC}(2024)]%
{gurobi}
\bibfield{author}{\bibinfo{person}{{Gurobi Optimization, LLC}}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{title}{{Gurobi Optimizer Reference Manual}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.gurobi.com}
\showURL{%
\tempurl}
\bibitem[Harlap et~al\mbox{.}(2018)]%
{harlap2018pipedream}
\bibfield{author}{\bibinfo{person}{Aaron Harlap}, \bibinfo{person}{Deepak Narayanan}, \bibinfo{person}{Amar Phanishayee}, \bibinfo{person}{Vivek Seshadri}, \bibinfo{person}{Nikhil Devanur}, \bibinfo{person}{Greg Ganger}, {and} \bibinfo{person}{Phil Gibbons}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Pipedream: Fast and efficient pipeline parallel dnn training}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1806.03377}} (\bibinfo{year}{2018}).
\newblock
\bibitem[Huang et~al\mbox{.}(2019)]%
{huang2019gpipe}
\bibfield{author}{\bibinfo{person}{Yanping Huang}, \bibinfo{person}{Youlong Cheng}, \bibinfo{person}{Ankur Bapna}, \bibinfo{person}{Orhan Firat}, \bibinfo{person}{Dehao Chen}, \bibinfo{person}{Mia Chen}, \bibinfo{person}{HyoukJoong Lee}, \bibinfo{person}{Jiquan Ngiam}, \bibinfo{person}{Quoc~V Le}, \bibinfo{person}{Yonghui Wu}, {et~al\mbox{.}}} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Gpipe: Efficient training of giant neural networks using pipeline parallelism}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}} \bibinfo{volume}{32} (\bibinfo{year}{2019}).
\newblock
\bibitem[Jiang et~al\mbox{.}(2024)]%
{jiang2024dynapipe}
\bibfield{author}{\bibinfo{person}{Chenyu Jiang}, \bibinfo{person}{Zhen Jia}, \bibinfo{person}{Shuai Zheng}, \bibinfo{person}{Yida Wang}, {and} \bibinfo{person}{Chuan Wu}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{DynaPipe: Optimizing multi-task training through dynamic pipelines}. In \bibinfo{booktitle}{\emph{Proceedings of the Nineteenth European Conference on Computer Systems}}. \bibinfo{pages}{542--559}.
\newblock
\bibitem[Kaplan et~al\mbox{.}(2020)]%
{kaplan2020scalinglaw}
\bibfield{author}{\bibinfo{person}{Jared Kaplan}, \bibinfo{person}{Sam McCandlish}, \bibinfo{person}{Tom Henighan}, \bibinfo{person}{Tom~B Brown}, \bibinfo{person}{Benjamin Chess}, \bibinfo{person}{Rewon Child}, \bibinfo{person}{Scott Gray}, \bibinfo{person}{Alec Radford}, \bibinfo{person}{Jeffrey Wu}, {and} \bibinfo{person}{Dario Amodei}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Scaling laws for neural language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2001.08361}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Kim et~al\mbox{.}(2023)]%
{kim2023bpipe}
\bibfield{author}{\bibinfo{person}{Taebum Kim}, \bibinfo{person}{Hyoungjoo Kim}, \bibinfo{person}{Gyeong-In Yu}, {and} \bibinfo{person}{Byung-Gon Chun}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{BPIPE: memory-balanced pipeline parallelism for training large language models}. In \bibinfo{booktitle}{\emph{International Conference on Machine Learning}}. PMLR, \bibinfo{pages}{16639--16653}.
\newblock
\bibitem[Lai et~al\mbox{.}(2023)]%
{lai2023merak}
\bibfield{author}{\bibinfo{person}{Zhiquan Lai}, \bibinfo{person}{Shengwei Li}, \bibinfo{person}{Xudong Tang}, \bibinfo{person}{Keshi Ge}, \bibinfo{person}{Weijie Liu}, \bibinfo{person}{Yabo Duan}, \bibinfo{person}{Linbo Qiao}, {and} \bibinfo{person}{Dongsheng Li}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Merak: An efficient distributed dnn training framework with automated 3d parallelism for giant foundation models}.
\newblock \bibinfo{journal}{\emph{IEEE Transactions on Parallel and Distributed Systems}} \bibinfo{volume}{34}, \bibinfo{number}{5} (\bibinfo{year}{2023}), \bibinfo{pages}{1466--1478}.
\newblock
\bibitem[Lamy-Poirier(2023)]%
{lamy2023breadth}
\bibfield{author}{\bibinfo{person}{Joel Lamy-Poirier}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Breadth-first pipeline parallelism}.
\newblock \bibinfo{journal}{\emph{Proceedings of Machine Learning and Systems}} \bibinfo{volume}{5} (\bibinfo{year}{2023}), \bibinfo{pages}{48--67}.
\newblock
\bibitem[Lepikhin et~al\mbox{.}(2020)]%
{lepikhin2020gshard}
\bibfield{author}{\bibinfo{person}{Dmitry Lepikhin}, \bibinfo{person}{HyoukJoong Lee}, \bibinfo{person}{Yuanzhong Xu}, \bibinfo{person}{Dehao Chen}, \bibinfo{person}{Orhan Firat}, \bibinfo{person}{Yanping Huang}, \bibinfo{person}{Maxim Krikun}, \bibinfo{person}{Noam Shazeer}, {and} \bibinfo{person}{Zhifeng Chen}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Gshard: Scaling giant models with conditional computation and automatic sharding}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2006.16668}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Li et~al\mbox{.}(2025a)]%
{li2025minimax}
\bibfield{author}{\bibinfo{person}{Aonian Li}, \bibinfo{person}{Bangwei Gong}, \bibinfo{person}{Bo Yang}, \bibinfo{person}{Boji Shan}, \bibinfo{person}{Chang Liu}, \bibinfo{person}{Cheng Zhu}, \bibinfo{person}{Chunhao Zhang}, \bibinfo{person}{Congchao Guo}, \bibinfo{person}{Da Chen}, \bibinfo{person}{Dong Li}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{a}.
\newblock \showarticletitle{Minimax-01: Scaling foundation models with lightning attention}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2501.08313}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Li et~al\mbox{.}(2022)]%
{li2022amp}
\bibfield{author}{\bibinfo{person}{Dacheng Li}, \bibinfo{person}{Hongyi Wang}, \bibinfo{person}{Eric Xing}, {and} \bibinfo{person}{Hao Zhang}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Amp: Automatically finding model parallel strategies with heterogeneity awareness}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} \bibinfo{volume}{35} (\bibinfo{year}{2022}), \bibinfo{pages}{6630--6639}.
\newblock
\bibitem[Li and Hoefler(2021)]%
{li2021chimera}
\bibfield{author}{\bibinfo{person}{Shigang Li} {and} \bibinfo{person}{Torsten Hoefler}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Chimera: efficiently training large-scale neural networks with bidirectional pipelines}. In \bibinfo{booktitle}{\emph{Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis}}. \bibinfo{pages}{1--14}.
\newblock
\bibitem[Li et~al\mbox{.}(2020)]%
{li2020pytorchddp}
\bibfield{author}{\bibinfo{person}{Shen Li}, \bibinfo{person}{Yanli Zhao}, \bibinfo{person}{Rohan Varma}, \bibinfo{person}{Omkar Salpekar}, \bibinfo{person}{Pieter Noordhuis}, \bibinfo{person}{Teng Li}, \bibinfo{person}{Adam Paszke}, \bibinfo{person}{Jeff Smith}, \bibinfo{person}{Brian Vaughan}, \bibinfo{person}{Pritam Damania}, {et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Pytorch distributed: Experiences on accelerating data parallel training}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2006.15704}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Li et~al\mbox{.}(2025b)]%
{li2025slimpipe}
\bibfield{author}{\bibinfo{person}{Zhouyang Li}, \bibinfo{person}{Yuliang Liu}, \bibinfo{person}{Wei Zhang}, \bibinfo{person}{Tailing Yuan}, \bibinfo{person}{Bin Chen}, \bibinfo{person}{Chengru Song}, {and} \bibinfo{person}{Di Zhang}.} \bibinfo{year}{2025}\natexlab{b}.
\newblock \showarticletitle{SlimPipe: Memory-Thrifty and Efficient Pipeline Parallelism for Long-Context LLM Training}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2504.14519}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Li et~al\mbox{.}(2021)]%
{li2021terapipe}
\bibfield{author}{\bibinfo{person}{Zhuohan Li}, \bibinfo{person}{Siyuan Zhuang}, \bibinfo{person}{Shiyuan Guo}, \bibinfo{person}{Danyang Zhuo}, \bibinfo{person}{Hao Zhang}, \bibinfo{person}{Dawn Song}, {and} \bibinfo{person}{Ion Stoica}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Terapipe: Token-level pipeline parallelism for training large-scale language models}. In \bibinfo{booktitle}{\emph{International Conference on Machine Learning}}. PMLR, \bibinfo{pages}{6543--6552}.
\newblock
\bibitem[Lieber et~al\mbox{.}(2024)]%
{lieber2024jamba}
\bibfield{author}{\bibinfo{person}{Opher Lieber}, \bibinfo{person}{Barak Lenz}, \bibinfo{person}{Hofit Bata}, \bibinfo{person}{Gal Cohen}, \bibinfo{person}{Jhonathan Osin}, \bibinfo{person}{Itay Dalmedigos}, \bibinfo{person}{Erez Safahi}, \bibinfo{person}{Shaked Meirom}, \bibinfo{person}{Yonatan Belinkov}, \bibinfo{person}{Shai Shalev-Shwartz}, \bibinfo{person}{Omri Abend}, \bibinfo{person}{Raz Alon}, \bibinfo{person}{Tomer Asida}, \bibinfo{person}{Amir Bergman}, \bibinfo{person}{Roman Glozman}, \bibinfo{person}{Michael Gokhman}, \bibinfo{person}{Avshalom Manevich}, \bibinfo{person}{Nir Ratner}, \bibinfo{person}{Noam Rozen}, \bibinfo{person}{Erez Schwartz}, \bibinfo{person}{Mor Zusman}, {and} \bibinfo{person}{Yoav Shoham}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Jamba: A Hybrid Transformer-Mamba Language Model}.
\newblock \bibinfo{journal}{\emph{arXiv preprint}} (\bibinfo{year}{2024}).
\newblock
\urldef\tempurl%
\url{https://arxiv.org/html/2403.19887v2}
\showURL{%
\tempurl}
\newblock
\shownote{Equal contribution}.
\bibitem[Lin et~al\mbox{.}(2024)]%
{lin2024tessel}
\bibfield{author}{\bibinfo{person}{Zhiqi Lin}, \bibinfo{person}{Youshan Miao}, \bibinfo{person}{Guanbin Xu}, \bibinfo{person}{Cheng Li}, \bibinfo{person}{Olli Saarikivi}, \bibinfo{person}{Saeed Maleki}, {and} \bibinfo{person}{Fan Yang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Tessel: Boosting distributed execution of large dnn models via flexible schedule search}. In \bibinfo{booktitle}{\emph{2024 IEEE International Symposium on High-Performance Computer Architecture (HPCA)}}. IEEE, \bibinfo{pages}{803--816}.
\newblock
\bibitem[Liu et~al\mbox{.}(2024a)]%
{liu2024deepseekv2}
\bibfield{author}{\bibinfo{person}{Aixin Liu}, \bibinfo{person}{Bei Feng}, \bibinfo{person}{Bin Wang}, \bibinfo{person}{Bingxuan Wang}, \bibinfo{person}{Bo Liu}, \bibinfo{person}{Chenggang Zhao}, \bibinfo{person}{Chengqi Dengr}, \bibinfo{person}{Chong Ruan}, \bibinfo{person}{Damai Dai}, \bibinfo{person}{Daya Guo}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Deepseek-v2: A strong, economical, and efficient mixture-of-experts language model}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2405.04434}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Liu et~al\mbox{.}(2024b)]%
{liu2024deepseekv3}
\bibfield{author}{\bibinfo{person}{Aixin Liu}, \bibinfo{person}{Bei Feng}, \bibinfo{person}{Bing Xue}, \bibinfo{person}{Bingxuan Wang}, \bibinfo{person}{Bochao Wu}, \bibinfo{person}{Chengda Lu}, \bibinfo{person}{Chenggang Zhao}, \bibinfo{person}{Chengqi Deng}, \bibinfo{person}{Chenyu Zhang}, \bibinfo{person}{Chong Ruan}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Deepseek-v3 technical report}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2412.19437}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Liu et~al\mbox{.}(2024c)]%
{liu2024aceso}
\bibfield{author}{\bibinfo{person}{Guodong Liu}, \bibinfo{person}{Youshan Miao}, \bibinfo{person}{Zhiqi Lin}, \bibinfo{person}{Xiaoxiang Shi}, \bibinfo{person}{Saeed Maleki}, \bibinfo{person}{Fan Yang}, \bibinfo{person}{Yungang Bao}, {and} \bibinfo{person}{Sa Wang}.} \bibinfo{year}{2024}\natexlab{c}.
\newblock \showarticletitle{Aceso: Efficient Parallel DNN Training through Iterative Bottleneck Alleviation}. In \bibinfo{booktitle}{\emph{Proceedings of the Nineteenth European Conference on Computer Systems}}. \bibinfo{pages}{163--181}.
\newblock
\bibitem[Liu et~al\mbox{.}(2025)]%
{liu2025mario}
\bibfield{author}{\bibinfo{person}{Weijian Liu}, \bibinfo{person}{Mingzhen Li}, \bibinfo{person}{Guangming Tan}, {and} \bibinfo{person}{Weile Jia}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Mario: Near Zero-cost Activation Checkpointing in Pipeline Parallelism}. In \bibinfo{booktitle}{\emph{Proceedings of the 30th ACM SIGPLAN Annual Symposium on Principles and Practice of Parallel Programming}}. \bibinfo{pages}{197--211}.
\newblock
\bibitem[Liu et~al\mbox{.}(2023)]%
{liu2023hanayo}
\bibfield{author}{\bibinfo{person}{Ziming Liu}, \bibinfo{person}{Shenggan Cheng}, \bibinfo{person}{Haotian Zhou}, {and} \bibinfo{person}{Yang You}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Hanayo: Harnessing wave-like pipeline parallelism for enhanced large model training efficiency}. In \bibinfo{booktitle}{\emph{Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis}}. \bibinfo{pages}{1--13}.
\newblock
\bibitem[Miao et~al\mbox{.}(2022)]%
{miao2022galvatron}
\bibfield{author}{\bibinfo{person}{Xupeng Miao}, \bibinfo{person}{Yujie Wang}, \bibinfo{person}{Youhe Jiang}, \bibinfo{person}{Chunan Shi}, \bibinfo{person}{Xiaonan Nie}, \bibinfo{person}{Hailin Zhang}, {and} \bibinfo{person}{Bin Cui}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Galvatron: Efficient transformer training over multiple gpus using automatic parallelism}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2211.13878}} (\bibinfo{year}{2022}).
\newblock
\bibitem[MiniMax et~al\mbox{.}(2025)]%
{minimax2025minimaxm1}
\bibfield{author}{\bibinfo{person}{MiniMax}, \bibinfo{person}{:}, \bibinfo{person}{Aili Chen}, \bibinfo{person}{Aonian Li}, \bibinfo{person}{Bangwei Gong}, \bibinfo{person}{Binyang Jiang}, \bibinfo{person}{Bo Fei}, \bibinfo{person}{Bo Yang}, \bibinfo{person}{Boji Shan}, \bibinfo{person}{Changqing Yu}, \bibinfo{person}{Chao Wang}, \bibinfo{person}{Cheng Zhu}, \bibinfo{person}{Chengjun Xiao}, \bibinfo{person}{Chengyu Du}, \bibinfo{person}{Chi Zhang}, \bibinfo{person}{Chu Qiao}, \bibinfo{person}{Chunhao Zhang}, \bibinfo{person}{Chunhui Du}, \bibinfo{person}{Congchao Guo}, \bibinfo{person}{Da Chen}, \bibinfo{person}{Deming Ding}, \bibinfo{person}{Dianjun Sun}, \bibinfo{person}{Dong Li}, \bibinfo{person}{Enwei Jiao}, \bibinfo{person}{Haigang Zhou}, \bibinfo{person}{Haimo Zhang}, \bibinfo{person}{Han Ding}, \bibinfo{person}{Haohai Sun}, \bibinfo{person}{Haoyu Feng}, \bibinfo{person}{Huaiguang Cai}, \bibinfo{person}{Haichao Zhu}, \bibinfo{person}{Jian Sun}, \bibinfo{person}{Jiaqi Zhuang}, \bibinfo{person}{Jiaren
Cai}, \bibinfo{person}{Jiayuan Song}, \bibinfo{person}{Jin Zhu}, \bibinfo{person}{Jingyang Li}, \bibinfo{person}{Jinhao Tian}, \bibinfo{person}{Jinli Liu}, \bibinfo{person}{Junhao Xu}, \bibinfo{person}{Junjie Yan}, \bibinfo{person}{Junteng Liu}, \bibinfo{person}{Junxian He}, \bibinfo{person}{Kaiyi Feng}, \bibinfo{person}{Ke Yang}, \bibinfo{person}{Kecheng Xiao}, \bibinfo{person}{Le Han}, \bibinfo{person}{Leyang Wang}, \bibinfo{person}{Lianfei Yu}, \bibinfo{person}{Liheng Feng}, \bibinfo{person}{Lin Li}, \bibinfo{person}{Lin Zheng}, \bibinfo{person}{Linge Du}, \bibinfo{person}{Lingyu Yang}, \bibinfo{person}{Lunbin Zeng}, \bibinfo{person}{Minghui Yu}, \bibinfo{person}{Mingliang Tao}, \bibinfo{person}{Mingyuan Chi}, \bibinfo{person}{Mozhi Zhang}, \bibinfo{person}{Mujie Lin}, \bibinfo{person}{Nan Hu}, \bibinfo{person}{Nongyu Di}, \bibinfo{person}{Peng Gao}, \bibinfo{person}{Pengfei Li}, \bibinfo{person}{Pengyu Zhao}, \bibinfo{person}{Qibing Ren}, \bibinfo{person}{Qidi Xu}, \bibinfo{person}{Qile Li},
\bibinfo{person}{Qin Wang}, \bibinfo{person}{Rong Tian}, \bibinfo{person}{Ruitao Leng}, \bibinfo{person}{Shaoxiang Chen}, \bibinfo{person}{Shaoyu Chen}, \bibinfo{person}{Shengmin Shi}, \bibinfo{person}{Shitong Weng}, \bibinfo{person}{Shuchang Guan}, \bibinfo{person}{Shuqi Yu}, \bibinfo{person}{Sichen Li}, \bibinfo{person}{Songquan Zhu}, \bibinfo{person}{Tengfei Li}, \bibinfo{person}{Tianchi Cai}, \bibinfo{person}{Tianrun Liang}, \bibinfo{person}{Weiyu Cheng}, \bibinfo{person}{Weize Kong}, \bibinfo{person}{Wenkai Li}, \bibinfo{person}{Xiancai Chen}, \bibinfo{person}{Xiangjun Song}, \bibinfo{person}{Xiao Luo}, \bibinfo{person}{Xiao Su}, \bibinfo{person}{Xiaobo Li}, \bibinfo{person}{Xiaodong Han}, \bibinfo{person}{Xinzhu Hou}, \bibinfo{person}{Xuan Lu}, \bibinfo{person}{Xun Zou}, \bibinfo{person}{Xuyang Shen}, \bibinfo{person}{Yan Gong}, \bibinfo{person}{Yan Ma}, \bibinfo{person}{Yang Wang}, \bibinfo{person}{Yiqi Shi}, \bibinfo{person}{Yiran Zhong}, \bibinfo{person}{Yonghong Duan}, \bibinfo{person}{Yongxiang
Fu}, \bibinfo{person}{Yongyi Hu}, \bibinfo{person}{Yu Gao}, \bibinfo{person}{Yuanxiang Fan}, \bibinfo{person}{Yufeng Yang}, \bibinfo{person}{Yuhao Li}, \bibinfo{person}{Yulin Hu}, \bibinfo{person}{Yunan Huang}, \bibinfo{person}{Yunji Li}, \bibinfo{person}{Yunzhi Xu}, \bibinfo{person}{Yuxin Mao}, \bibinfo{person}{Yuxuan Shi}, \bibinfo{person}{Yuze Wenren}, \bibinfo{person}{Zehan Li}, \bibinfo{person}{Zelin Li}, \bibinfo{person}{Zhanxu Tian}, \bibinfo{person}{Zhengmao Zhu}, \bibinfo{person}{Zhenhua Fan}, \bibinfo{person}{Zhenzhen Wu}, \bibinfo{person}{Zhichao Xu}, \bibinfo{person}{Zhihang Yu}, \bibinfo{person}{Zhiheng Lyu}, \bibinfo{person}{Zhuo Jiang}, \bibinfo{person}{Zibo Gao}, \bibinfo{person}{Zijia Wu}, \bibinfo{person}{Zijian Song}, {and} \bibinfo{person}{Zijun Sun}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention}.
\newblock
\newblock
\showeprint[arxiv]{2506.13585}~[cs.CL]
\urldef\tempurl%
\url{https://arxiv.org/abs/2506.13585}
\showURL{%
\tempurl}
\bibitem[Narayanan et~al\mbox{.}(2021)]%
{narayanan2021megatron2interleaved1F1B}
\bibfield{author}{\bibinfo{person}{Deepak Narayanan}, \bibinfo{person}{Mohammad Shoeybi}, \bibinfo{person}{Jared Casper}, \bibinfo{person}{Patrick LeGresley}, \bibinfo{person}{Mostofa Patwary}, \bibinfo{person}{Vijay Korthikanti}, \bibinfo{person}{Dmitri Vainbrand}, \bibinfo{person}{Prethvi Kashinkunti}, \bibinfo{person}{Julie Bernauer}, \bibinfo{person}{Bryan Catanzaro}, {et~al\mbox{.}}} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Efficient large-scale language model training on gpu clusters using megatron-lm}. In \bibinfo{booktitle}{\emph{Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis}}. \bibinfo{pages}{1--15}.
\newblock
\bibitem[Oh et~al\mbox{.}(2022)]%
{oh2022ooo}
\bibfield{author}{\bibinfo{person}{Hyungjun Oh}, \bibinfo{person}{Junyeol Lee}, \bibinfo{person}{Hyeongju Kim}, {and} \bibinfo{person}{Jiwon Seo}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Out-of-order backprop: An effective scheduling technique for deep learning}. In \bibinfo{booktitle}{\emph{Proceedings of the Seventeenth European Conference on Computer Systems}}. \bibinfo{pages}{435--452}.
\newblock
\bibitem[Paszke et~al\mbox{.}(2019)]%
{paszke2019pytorch}
\bibfield{author}{\bibinfo{person}{Adam Paszke}, \bibinfo{person}{Sam Gross}, \bibinfo{person}{Francisco Massa}, \bibinfo{person}{Adam Lerer}, \bibinfo{person}{James Bradbury}, \bibinfo{person}{Gregory Chanan}, \bibinfo{person}{Trevor Killeen}, \bibinfo{person}{Zeming Lin}, \bibinfo{person}{Natalia Gimelshein}, \bibinfo{person}{Luca Antiga}, {et~al\mbox{.}}} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Pytorch: An imperative style, high-performance deep learning library}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}} \bibinfo{volume}{32} (\bibinfo{year}{2019}).
\newblock
\bibitem[Qi et~al\mbox{.}(2024a)]%
{qi2024zbv}
\bibfield{author}{\bibinfo{person}{Penghui Qi}, \bibinfo{person}{Xinyi Wan}, \bibinfo{person}{Nyamdavaa Amar}, {and} \bibinfo{person}{Min Lin}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Pipeline Parallelism with Controllable Memory}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2405.15362}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Qi et~al\mbox{.}(2024b)]%
{qi2024zerobubble}
\bibfield{author}{\bibinfo{person}{Penghui Qi}, \bibinfo{person}{Xinyi Wan}, \bibinfo{person}{Guangxing Huang}, {and} \bibinfo{person}{Min Lin}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Zero Bubble (Almost) Pipeline Parallelism}. In \bibinfo{booktitle}{\emph{The Twelfth International Conference on Learning Representations}}.
\newblock
\bibitem[Qin et~al\mbox{.}(2024)]%
{qin2024lighteningattn}
\bibfield{author}{\bibinfo{person}{Zhen Qin}, \bibinfo{person}{Weigao Sun}, \bibinfo{person}{Dong Li}, \bibinfo{person}{Xuyang Shen}, \bibinfo{person}{Weixuan Sun}, {and} \bibinfo{person}{Yiran Zhong}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Various lengths, constant speed: Efficient language modeling with lightning attention}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2405.17381}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Rajbhandari et~al\mbox{.}(2022)]%
{rajbhandari2022deepspeed}
\bibfield{author}{\bibinfo{person}{Samyam Rajbhandari}, \bibinfo{person}{Conglong Li}, \bibinfo{person}{Zhewei Yao}, \bibinfo{person}{Minjia Zhang}, \bibinfo{person}{Reza~Yazdani Aminabadi}, \bibinfo{person}{Ammar~Ahmad Awan}, \bibinfo{person}{Jeff Rasley}, {and} \bibinfo{person}{Yuxiong He}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Deepspeed-moe: Advancing mixture-of-experts inference and training to power next-generation ai scale}. In \bibinfo{booktitle}{\emph{International conference on machine learning}}. PMLR, \bibinfo{pages}{18332--18346}.
\newblock
\bibitem[Rajbhandari et~al\mbox{.}(2020)]%
{rajbhandari2020zero}
\bibfield{author}{\bibinfo{person}{Samyam Rajbhandari}, \bibinfo{person}{Jeff Rasley}, \bibinfo{person}{Olatunji Ruwase}, {and} \bibinfo{person}{Yuxiong He}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Zero: Memory optimizations toward training trillion parameter models}. In \bibinfo{booktitle}{\emph{SC20: International Conference for High Performance Computing, Networking, Storage and Analysis}}. IEEE, \bibinfo{pages}{1--16}.
\newblock
\bibitem[Rajbhandari et~al\mbox{.}(2021)]%
{rajbhandari2021zero-infinity}
\bibfield{author}{\bibinfo{person}{Samyam Rajbhandari}, \bibinfo{person}{Olatunji Ruwase}, \bibinfo{person}{Jeff Rasley}, \bibinfo{person}{Shaden Smith}, {and} \bibinfo{person}{Yuxiong He}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Zero-infinity: Breaking the gpu memory wall for extreme scale deep learning}. In \bibinfo{booktitle}{\emph{Proceedings of the international conference for high performance computing, networking, storage and analysis}}. \bibinfo{pages}{1--14}.
\newblock
\bibitem[Ren et~al\mbox{.}(2021)]%
{ren2021zero-offload}
\bibfield{author}{\bibinfo{person}{Jie Ren}, \bibinfo{person}{Samyam Rajbhandari}, \bibinfo{person}{Reza~Yazdani Aminabadi}, \bibinfo{person}{Olatunji Ruwase}, \bibinfo{person}{Shuangyan Yang}, \bibinfo{person}{Minjia Zhang}, \bibinfo{person}{Dong Li}, {and} \bibinfo{person}{Yuxiong He}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{$\{$Zero-offload$\}$: Democratizing $\{$billion-scale$\}$ model training}. In \bibinfo{booktitle}{\emph{2021 USENIX Annual Technical Conference (USENIX ATC 21)}}. \bibinfo{pages}{551--564}.
\newblock
\bibitem[Sergeev and Del~Balso(2018)]%
{sergeev2018horovod}
\bibfield{author}{\bibinfo{person}{Alexander Sergeev} {and} \bibinfo{person}{Mike Del~Balso}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Horovod: fast and easy distributed deep learning in TensorFlow}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1802.05799}} (\bibinfo{year}{2018}).
\newblock
\bibitem[Shoeybi et~al\mbox{.}(2019)]%
{shoeybi2019megatron}
\bibfield{author}{\bibinfo{person}{Mohammad Shoeybi}, \bibinfo{person}{Mostofa Patwary}, \bibinfo{person}{Raul Puri}, \bibinfo{person}{Patrick LeGresley}, \bibinfo{person}{Jared Casper}, {and} \bibinfo{person}{Bryan Catanzaro}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Megatron-lm: Training multi-billion parameter language models using model parallelism}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1909.08053}} (\bibinfo{year}{2019}).
\newblock
\bibitem[Strubell et~al\mbox{.}(2020)]%
{strubell2020energy}
\bibfield{author}{\bibinfo{person}{Emma Strubell}, \bibinfo{person}{Ananya Ganesh}, {and} \bibinfo{person}{Andrew McCallum}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Energy and policy considerations for modern deep learning research}. In \bibinfo{booktitle}{\emph{Proceedings of the AAAI conference on artificial intelligence}}, Vol.~\bibinfo{volume}{34}. \bibinfo{pages}{13693--13696}.
\newblock
\bibitem[Sun et~al\mbox{.}(2024b)]%
{sun2024seq1f1b}
\bibfield{author}{\bibinfo{person}{Ao Sun}, \bibinfo{person}{Weilin Zhao}, \bibinfo{person}{Xu Han}, \bibinfo{person}{Cheng Yang}, \bibinfo{person}{Xinrong Zhang}, \bibinfo{person}{Zhiyuan Liu}, \bibinfo{person}{Chuan Shi}, {and} \bibinfo{person}{Maosong Sun}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Seq1f1b: Efficient sequence-level pipeline parallelism for large language model training}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2406.03488}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Sun et~al\mbox{.}(2024a)]%
{sun2024adapipe}
\bibfield{author}{\bibinfo{person}{Zhenbo Sun}, \bibinfo{person}{Huanqi Cao}, \bibinfo{person}{Yuanwei Wang}, \bibinfo{person}{Guanyu Feng}, \bibinfo{person}{Shengqi Chen}, \bibinfo{person}{Haojie Wang}, {and} \bibinfo{person}{Wenguang Chen}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{AdaPipe: Optimizing Pipeline Parallelism with Adaptive Recomputation and Partitioning}. In \bibinfo{booktitle}{\emph{Proceedings of the 29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 3}}. \bibinfo{pages}{86--100}.
\newblock
\bibitem[Team et~al\mbox{.}(2024)]%
{team2024gemini1.5}
\bibfield{author}{\bibinfo{person}{Gemini Team}, \bibinfo{person}{Petko Georgiev}, \bibinfo{person}{Ving~Ian Lei}, \bibinfo{person}{Ryan Burnell}, \bibinfo{person}{Libin Bai}, \bibinfo{person}{Anmol Gulati}, \bibinfo{person}{Garrett Tanzer}, \bibinfo{person}{Damien Vincent}, \bibinfo{person}{Zhufeng Pan}, \bibinfo{person}{Shibo Wang}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2403.05530}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Team et~al\mbox{.}(2025)]%
{team2025gemma}
\bibfield{author}{\bibinfo{person}{Gemma Team}, \bibinfo{person}{Aishwarya Kamath}, \bibinfo{person}{Johan Ferret}, \bibinfo{person}{Shreya Pathak}, \bibinfo{person}{Nino Vieillard}, \bibinfo{person}{Ramona Merhej}, \bibinfo{person}{Sarah Perrin}, \bibinfo{person}{Tatiana Matejovicova}, \bibinfo{person}{Alexandre Ram{\'e}}, \bibinfo{person}{Morgane Rivi{\`e}re}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Gemma 3 technical report}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2503.19786}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Touvron et~al\mbox{.}(2023)]%
{touvron2023llama2}
\bibfield{author}{\bibinfo{person}{Hugo Touvron}, \bibinfo{person}{Louis Martin}, \bibinfo{person}{Kevin Stone}, \bibinfo{person}{Peter Albert}, \bibinfo{person}{Amjad Almahairi}, \bibinfo{person}{Yasmine Babaei}, \bibinfo{person}{Nikolay Bashlykov}, \bibinfo{person}{Soumya Batra}, \bibinfo{person}{Prajjwal Bhargava}, \bibinfo{person}{Shruti Bhosale}, {et~al\mbox{.}}} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Llama 2: Open foundation and fine-tuned chat models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2307.09288}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Um et~al\mbox{.}(2024)]%
{um2024metis}
\bibfield{author}{\bibinfo{person}{Taegeon Um}, \bibinfo{person}{Byungsoo Oh}, \bibinfo{person}{Minyoung Kang}, \bibinfo{person}{Woo-Yeon Lee}, \bibinfo{person}{Goeun Kim}, \bibinfo{person}{Dongseob Kim}, \bibinfo{person}{Youngtaek Kim}, \bibinfo{person}{Mohd Muzzammil}, {and} \bibinfo{person}{Myeongjae Jeon}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Metis: Fast Automatic Distributed Training on Heterogeneous $\{$GPUs$\}$}. In \bibinfo{booktitle}{\emph{2024 USENIX Annual Technical Conference (USENIX ATC 24)}}. \bibinfo{pages}{563--578}.
\newblock
\bibitem[Vaswani(2017)]%
{vaswani2017attention}
\bibfield{author}{\bibinfo{person}{A Vaswani}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Attention is all you need}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} (\bibinfo{year}{2017}).
\newblock
\bibitem[Wang et~al\mbox{.}(2025)]%
{wang2025flexsp}
\bibfield{author}{\bibinfo{person}{Yujie Wang}, \bibinfo{person}{Shiju Wang}, \bibinfo{person}{Shenhan Zhu}, \bibinfo{person}{Fangcheng Fu}, \bibinfo{person}{Xinyi Liu}, \bibinfo{person}{Xuefeng Xiao}, \bibinfo{person}{Huixia Li}, \bibinfo{person}{Jiashi Li}, \bibinfo{person}{Faming Wu}, {and} \bibinfo{person}{Bin Cui}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Flexsp: Accelerating large language model training via flexible sequence parallelism}. In \bibinfo{booktitle}{\emph{Proceedings of the 30th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2}}. \bibinfo{pages}{421--436}.
\newblock
\bibitem[Wei et~al\mbox{.}(2022)]%
{wei2022emergent}
\bibfield{author}{\bibinfo{person}{Jason Wei}, \bibinfo{person}{Yi Tay}, \bibinfo{person}{Rishi Bommasani}, \bibinfo{person}{Colin Raffel}, \bibinfo{person}{Barret Zoph}, \bibinfo{person}{Sebastian Borgeaud}, \bibinfo{person}{Dani Yogatama}, \bibinfo{person}{Maarten Bosma}, \bibinfo{person}{Denny Zhou}, \bibinfo{person}{Donald Metzler}, {et~al\mbox{.}}} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Emergent abilities of large language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2206.07682}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Yang et~al\mbox{.}(2025)]%
{yang2025qwen3}
\bibfield{author}{\bibinfo{person}{An Yang}, \bibinfo{person}{Anfeng Li}, \bibinfo{person}{Baosong Yang}, \bibinfo{person}{Beichen Zhang}, \bibinfo{person}{Binyuan Hui}, \bibinfo{person}{Bo Zheng}, \bibinfo{person}{Bowen Yu}, \bibinfo{person}{Chang Gao}, \bibinfo{person}{Chengen Huang}, \bibinfo{person}{Chenxu Lv}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Qwen3 technical report}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2505.09388}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Yeung et~al\mbox{.}(2024)]%
{yeung2024vocpara}
\bibfield{author}{\bibinfo{person}{Man~Tsung Yeung}, \bibinfo{person}{Penghui Qi}, \bibinfo{person}{Min Lin}, {and} \bibinfo{person}{Xinyi Wan}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Balancing Pipeline Parallelism with Vocabulary Parallelism}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2411.05288}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Zhang et~al\mbox{.}(2024)]%
{zhang2024disttrain}
\bibfield{author}{\bibinfo{person}{Zili Zhang}, \bibinfo{person}{Yinmin Zhong}, \bibinfo{person}{Ranchen Ming}, \bibinfo{person}{Hanpeng Hu}, \bibinfo{person}{Jianjian Sun}, \bibinfo{person}{Zheng Ge}, \bibinfo{person}{Yibo Zhu}, {and} \bibinfo{person}{Xin Jin}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Disttrain: Addressing model and data heterogeneity with disaggregated training for multimodal large language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2408.04275}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Zhao et~al\mbox{.}(2023)]%
{zhao2023pytorchfsdp}
\bibfield{author}{\bibinfo{person}{Yanli Zhao}, \bibinfo{person}{Andrew Gu}, \bibinfo{person}{Rohan Varma}, \bibinfo{person}{Liang Luo}, \bibinfo{person}{Chien-Chin Huang}, \bibinfo{person}{Min Xu}, \bibinfo{person}{Less Wright}, \bibinfo{person}{Hamid Shojanazeri}, \bibinfo{person}{Myle Ott}, \bibinfo{person}{Sam Shleifer}, {et~al\mbox{.}}} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Pytorch fsdp: experiences on scaling fully sharded data parallel}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2304.11277}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Zheng et~al\mbox{.}(2022)]%
{zheng2022alpa}
\bibfield{author}{\bibinfo{person}{Lianmin Zheng}, \bibinfo{person}{Zhuohan Li}, \bibinfo{person}{Hao Zhang}, \bibinfo{person}{Yonghao Zhuang}, \bibinfo{person}{Zhifeng Chen}, \bibinfo{person}{Yanping Huang}, \bibinfo{person}{Yida Wang}, \bibinfo{person}{Yuanzhong Xu}, \bibinfo{person}{Danyang Zhuo}, \bibinfo{person}{Eric~P Xing}, {et~al\mbox{.}}} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Alpa: Automating inter-and $\{$Intra-Operator$\}$ parallelism for distributed deep learning}. In \bibinfo{booktitle}{\emph{16th USENIX Symposium on Operating Systems Design and Implementation (OSDI 22)}}. \bibinfo{pages}{559--578}.
\newblock
\bibitem[Zhu et~al\mbox{.}(2025)]%
{zhu2025mist}
\bibfield{author}{\bibinfo{person}{Zhanda Zhu}, \bibinfo{person}{Christina Giannoula}, \bibinfo{person}{Muralidhar Andoorveedu}, \bibinfo{person}{Qidong Su}, \bibinfo{person}{Karttikeya Mangalam}, \bibinfo{person}{Bojian Zheng}, {and} \bibinfo{person}{Gennady Pekhimenko}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Mist: Efficient Distributed Training of Large Language Models via Memory-Parallelism Co-Optimization}. In \bibinfo{booktitle}{\emph{Proceedings of the Twentieth European Conference on Computer Systems}}. \bibinfo{pages}{1298--1316}.
\newblock
\end{thebibliography}