Diffusion Models Need Visual Priors for Image Generation
Extracted source text · arXiv 2410.08531
\documentclass{article} % For LaTeX2e
\usepackage{iclr2025_conference,times}
% Optional math commands from https://github.com/goodfeli/dlbook_notation.
\input{math_commands.tex}
\usepackage{hyperref}
\usepackage{url}
\usepackage{graphicx}
\usepackage{multirow}
\usepackage{xcolor}
\usepackage{booktabs}
\usepackage{adjustbox}
\usepackage{colortbl}
\usepackage{makecell}
% \title{Diffusion on Diffusion: \\Guiding a Diffusion Model with Itself}
\title{Diffusion Models Need Visual Priors for \\Image Generation}
% Authors must not appear in the submitted version. They should be hidden
% as long as the \iclrfinalcopy macro remains commented out below.
% Non-anonymous submissions will be rejected without review.
% \author{Xiaoyu Yue, ZiDong Wang, Zeyu Lu, Shuyang Sun, Meng Wei, Wanli Ouyang, Lei Bai, Luping Zhou \\
% University of Sydney
% }
\author{
\makecell[l]{Xiaoyu Yue$^{1,2}$ \quad Zidong Wang$^{2,3}$ \quad Zeyu Lu$^{2,4}$ \quad Shuyang Sun$^{5}$ \quad Meng Wei$^{2,6}$ \\ Wanli Ouyang$^{2,3}$ \quad Lei Bai$^{2}$ \quad Luping Zhou$^{1}$} \\
$^1$University of Sydney \quad
$^2$Shanghai AI Laboratory \quad
$^3$Chinese University of Hong Kong \\
$^4$Shanghai Jiao Tong University \quad
$^5$University of Oxford \quad
$^6$University of Hong Kong \quad
}
% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to \LaTeX{} to determine where to break
% the lines. Using \AND forces a linebreak at that point. So, if \LaTeX{}
% puts 3 of 4 authors names on the first line, and the last on the second
% line, try using \AND instead of \And before the third author name.
\def\eg{\emph{e.g.}} \def\Eg{\emph{E.g.}}
\def\ie{\emph{i.e.}} \def\Ie{\emph{I.e.}}
\def\cf{\emph{c.f.}} \def\Cf{\emph{C.f.}}
\def\etc{\emph{etc.}} \def\vs{\emph{vs.}}
\def\wrt{w.r.t.} \def\dof{d.o.f.}
\def\etal{\emph{et al.}}
% \newcommand{\fix}{\marginpar{FIX}}
% \newcommand{\new}{\marginpar{NEW}}
% \newcommand{\wzd}[1]{\textcolor{purple}{wzd:#1}}
% \newcommand{\wm}[1]{\textcolor{blue}{wm:#1}}
\definecolor{baselinecolor}{gray}{0.93}
\newcommand{\baseline}[1]{\cellcolor{baselinecolor}{#1}}
\iclrfinalcopy % Uncomment for camera-ready version, but NOT for submission.
\begin{document}
\maketitle
\input{sections/abstract}
\input{sections/Introduction}
\input{sections/related_work}
\input{sections/method}
\input{sections/experiments}
\input{sections/conclusion}
\bibliography{iclr2025_conference}
\bibliographystyle{iclr2025_conference}
\newpage
\appendix
\input{sections/appendix}
\end{document}
%%%%% NEW MATH DEFINITIONS %%%%%
\usepackage{amsmath,amsfonts,bm}
% Mark sections of captions for referring to divisions of figures
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
% Highlight a newly defined term
\newcommand{\newterm}[1]{{\bf #1}}
% Figure reference, lower-case.
\def\figref#1{figure~\ref{#1}}
% Figure reference, capital. For start of sentence
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
% Section reference, lower-case.
\def\secref#1{section~\ref{#1}}
% Section reference, capital.
\def\Secref#1{Section~\ref{#1}}
% Reference to two sections.
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
% Reference to three sections.
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
% Reference to an equation, lower-case.
\def\eqref#1{equation~\ref{#1}}
% Reference to an equation, upper case
\def\Eqref#1{Equation~\ref{#1}}
% A raw reference to an equation---avoid using if possible
\def\plaineqref#1{\ref{#1}}
% Reference to a chapter, lower-case.
\def\chapref#1{chapter~\ref{#1}}
% Reference to an equation, upper case.
\def\Chapref#1{Chapter~\ref{#1}}
% Reference to a range of chapters
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
% Reference to an algorithm, lower-case.
\def\algref#1{algorithm~\ref{#1}}
% Reference to an algorithm, upper case.
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
% Reference to a part, lower case
\def\partref#1{part~\ref{#1}}
% Reference to a part, upper case
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
% Random variables
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
% rm is already a command, just don't name any random variables m
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
% Random vectors
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
% Elements of random vectors
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
% Random matrices
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
% Elements of random matrices
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
% Vectors
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
% Elements of vectors
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
% Matrix
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
% Tensor
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
% Graph
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
% Sets
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
% Don't use a set called E, because this would be the same as our symbol
% for expectation.
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
% Entries of a matrix
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
% entries of a tensor
% Same font as tensor, without \bm wrapper
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
% The true underlying data generating distribution
\newcommand{\pdata}{p_{\rm{data}}}
% The empirical distribution defined by the training set
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
% The model distribution
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
% Stochastic autoencoder distributions
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} % Laplace distribution
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
% Wolfram Mathworld says $L^2$ is for function spaces and $\ell^2$ is for vectors
% But then they seem to use $L^2$ for vectors throughout the site, and so does
% wikipedia.
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} % See usage in notation.tex. Chosen to match Daphne's book.
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
\begin{figure}[h]
\centering
\includegraphics[width=\linewidth]{figures/teaser.pdf}
\caption{\textbf{Selected samples generated by the second stage of DoD-XL.} By training for only $1$ million steps on ImageNet-$256\times256$ dataset, DoD-XL achieves state-of-the-art image quality.}
\label{fig:teaser}
\end{figure}
\begin{abstract}
% We propose Diffusion on Diffusion (DoD), a novel iterative generation framework that enhances diffusion models by refining generated samples across multiple iterations. We observe that diffusion models typically generate images with correct semantic content, but often struggle with texture details. To address this issue, DoD takes the generated output as an additional condition, providing semantically rich guidance during the early stages of diffusion sampling, which enables the generation of higher-quality samples. Specifically, we introduce a latent embedding module that employs a compression-reconstruction approach to discard redundant detail information from the conditional samples in each iteration, retaining only the semantic information for guidance. We evaluate DoD on the popular ImageNet $256 \times 256$ dataset, achieving an FiD-50K score of 3.15 with only 190M parameters.
Conventional class-guided diffusion models generally succeed in generating images with correct semantic content, but often struggle with texture details.
This limitation stems from the usage of class priors, which only provide coarse and limited conditional information.
To address this issue, we propose Diffusion on Diffusion (DoD), an innovative multi-stage generation framework that first extracts visual priors from previously generated samples, then provides rich guidance for the diffusion model leveraging visual priors from the early stages of diffusion sampling.
Specifically, we introduce a latent embedding module that employs a compression-reconstruction approach to discard redundant detail information from the conditional samples in each stage, retaining only the semantic information for guidance. We evaluate DoD on the popular ImageNet-$256 \times 256$ dataset, reducing 7$\times$ training cost compared to SiT and DiT with even better performance in terms of the FID-50K score.
Our largest model DoD-XL achieves an FID-50K score of 1.83 with only 1 million training steps, which surpasses other state-of-the-art methods without bells and whistles during inference.
\end{abstract}
\section{Related Work}
\subsection{Diffusions and Flows}
% Denoising Diffusion Probabilistic Models (DDPMs)~\citep{ho2020denoising,saharia2022photorealistic,radford2021learning,croitoru2023diffusionsurvey,bond2021deepgenmodel} and score-based models~\citep{hyvarinen2005estimation,song2020score} have exhibited remarkable progress in the context of image generation tasks~\citep{lu2024hierarchical,ling2024diffusion,rombach2022high,saharia2022photorealistic,meng2021sdedit,ramesh2022hierarchical,ruiz2023dreambooth,poole2022dreamfusion}. The Denoising Diffusion Implicit Model (DDIM)~\citep{song2020denoising}, offers an accelerated sampling procedure. Latent Diffusion Models (LDMs)~\citep{rombach2022high} establishes a new benchmark of training deep generative models to reverse a noise process in the latent space, through the use of VAE~\citep{kingma2013auto}. Rectified flow model~\citep{liu2023flow} learns a neural ordinary differential equation (ODE) that transports between two distributions. By solving a nonlinear least squares optimization problem, rectified flow learns to map the points drawn from two distributions following the straight paths, which are the shortest paths between two points and hence yield computationally efficiency. % We follow the rectified flow framework to train our DoD models.
Denoising Diffusion Probabilistic Models (DDPMs) \citep{ho2020denoising,saharia2022photorealistic,radford2021learning,croitoru2023diffusionsurvey,bond2021deepgenmodel} and score-based models \citep{hyvarinen2005estimation,song2020score} have demonstrated significant advancements in image generation tasks \citep{lu2024hierarchical,ling2024diffusion,rombach2022high,saharia2022photorealistic,meng2021sdedit,ramesh2022hierarchical,ruiz2023dreambooth,poole2022dreamfusion}. The Denoising Diffusion Implicit Model (DDIM) \citep{song2020denoising} introduced an accelerated sampling method, while Latent Diffusion Models (LDMs) \citep{rombach2022high} set a new standard by applying deep generative models to reverse the noise process in the latent space using Variational Autoencoders (VAEs) \citep{kingma2013auto}.
Flow models ~\citep{liu2023flow,albergo2022normalizingflows,lipman2022flowmatching,albergo2023stochasticinterpolants} present an alternative approach by learning a neural ordinary differential equation (ODE) that transports between two distributions.
The rectified flow model \citep{liu2023flow} solves a nonlinear least squares optimization problem to learn mappings along straight paths, which represent the shortest distance between two points, leading to improved computational efficiency. In this work, we adopt the rectified flow as the noise scheduler to train our DoD models.
\subsection{Diffusion Transformer}
% The Transformer model~\citep{vaswani2017attention}, has successfully supplanted domain-specific architectures in a variety of fields including, but not limited to, language~\citep{brown2020language,chowdhery2023palm}, vision~\citep{dosovitskiy2020image,han2022survey_vit}, and multi-modality~\citep{team2023gemini}.
% In vision perception research, most efforts~\citep{touvron2019fixing,touvron2021training,liu2021swin,liu2022convnet} that focus on resolution are aimed at accelerating pretraining using a fixed, low resolution. On the other hand, NaViT~\citep{dehghani2023patch} implements the 'Patch n’ Pack' technique to train ViT using images at their natural, 'native' resolution.
% Notably, transformers have been also explored in the denoising diffusion probabilistic models~\citep{ho2020denoising} to synthesize images. DiT~\citep{peebles2023scalable} is the seminal work that utilizes a vision transformer as the backbone of LDMs and can serve as a strong baseline. Based on DiT architecture, MDT~\citep{gao2023masked} introduces a masked latent modeling approach, which requires two forward-runs in training and inference. U-ViT~\citep{bao2023all} treats all inputs as tokens and incorporates U-Net architectures into the ViT backbone of LDMs. DiffiT~\citep{hatamizadeh2023diffit} introduces a time-dependent self-attention module into the DiT backbone to adapt to different stages of the diffusion process. SiT~\citep{ma2024sit} utilizes the same architecture as DiT and explores different rectified flow configurations. Large-DiT and Flag-DiT~\citep{gao2024luminat2x} scale up the diffusion transformers and achieve better performance.
% SD3~\citep{esser2024scaling} introduces new noise samplers for rectified flow models and scales it to billion-level parameters. It achieves advanced text-to-image generation results.
% FiTv2 based on FiT~\citep{Lu2024FiT} achieves advanced image generation results equipped with new flexible diffusion transformer architecture and rectified flow framework.
% We follow the model architecture of FiTv2 and propose our coarse-to-fine iterative framework DoD.
The Transformer model \citep{vaswani2017attention} has successfully replaced domain-specific architectures across various fields, including language \citep{brown2020language,chowdhery2023palm}, vision \citep{dosovitskiy2020image,han2022survey_vit}, and multi-modal learning \citep{team2023gemini}. In the realm of visual perception research, numerous studies \citep{touvron2019fixing,touvron2021training,liu2021swin,liu2022convnet} have focused on accelerating pretraining by utilizing fixed, low-resolution images. Transformers have also been applied in denoising diffusion probabilistic models \citep{ho2020denoising} for image synthesis. DiT \citep{peebles2023scalable}, a pioneering work in this space, employs a vision transformer as the backbone for latent diffusion models (LDMs), serving as a strong baseline for subsequent research. MDT \citep{gao2023masked} introduces a masked latent modeling approach, requiring two forward passes during training and inference. U-ViT \citep{bao2023all} tokenizes all inputs and integrates U-Net architectures into the ViT backbone of LDMs.
% DiffiT \citep{hatamizadeh2023diffit} incorporates a time-dependent self-attention module into the DiT backbone to adapt the model to different stages of the diffusion process.
SiT \citep{ma2024sit}, utilizing the same architecture as DiT, explores various rectified flow configurations. Large-DiT and Flag-DiT \citep{gao2024luminat2x} scale up diffusion transformers to achieve improved performance. SD3 \citep{esser2024scaling} introduces novel noise samplers for rectified flow models and scales these models to billions of parameters, yielding state-of-the-art text-to-image generation results. FiTv2~\citep{wang2024fitv2}, based on FiT \citep{Lu2024FiT}, achieves advanced class-conditional image generation performance by leveraging a flexible diffusion transformer architecture within a rectified flow framework. In this work, built upon the FiTv2 architecture, we propose Diffusion on Diffusion (DoD), an innovative framework which effectively incorporates visual priors into class-guided image generation.
\section{Method}
\subsection{Preliminaries}
% \textbf{Diffusion Models.}
% Before introducing our Diffusion on Diffusion (DoD) framework, we provide a brief review of diffusion models.
% Diffusion models aim to model a distribution $p_{\theta}(z_0)$ that approximates the ground truth data distribution $q(z_0)$.
% They consist of a forward noising process and a reverse denoising process.
% In the forward noising process, Gaussian noise is recursively added to the clean sample $z_0$ over $t$ times, resulting in the noised data $z_t$.
% Leveraging the properties of Gaussian distributions, this process can be simplified as:
% \begin{equation}
% q(z_t | z_0) = \mathcal{N}(z_t; \sqrt{\bar{\alpha }_t}z_0, (1 - \bar{\alpha }_t)\mathbf{I}),
% \end{equation}
% where constants $\bar{\alpha }_t$ is a constant that defines the scaling factors of the data sample and the noise.
% Denoting the Gaussian noise at time step $t$ as $\epsilon_{t}\sim\mathcal{N}(0, \mathbf{I})$, the noised data $z_t$ can be computed using the reparameterization trick: $z_t = \sqrt{\bar{\alpha }_t} z_0 + \sqrt{1 - \bar{\alpha }_t} \epsilon_t.$
% % \begin{equation}
% % z_t = \sqrt{\bar{\alpha }_t} z_0 + \sqrt{1 - \bar{\alpha }_t} \epsilon_t.
% % \end{equation}
% In the reverse denoising process, the diffusion model is trained to gradually reverse the corruptions, \ie{}, to approximate $q(z_{t-1}|z_t)$, which can be achieved by maximizing a variational lower bound:
% \begin{equation}
% \underset{\theta}{max}\mathbb{E}_{q(z_0)}[ \log{p_{\theta}(z_0)} ] \le \underset{\theta}{max}\mathbb{E}_{q(z_0,z_1,\cdots,z_T)}[ \log{p_{\theta}(z_{0:T})} - \log{q(z_{1:T} | z_0)} ].
% \end{equation}
% DDPM \citep{ho2020denoising} models $p_{\theta}(z_{t-1} | z_t)$ as a Gaussian distribution with a trainable mean function, therefore can be trained by the simple mean squared error between the predicted noise and the ground truth noise:
% \begin{equation}
% \mathcal{L}(\epsilon_{\theta}) = \sum^T_{t=1} \underset{z_0 \sim q(z_0)}{\mathbb{E}}[ \| \epsilon_\theta (z_t) - \epsilon_t \| ^2 ],
% \end{equation}
% where $\epsilon_{\theta}$ is the noise prediction network, parameterized by $\theta$.
% Once $\epsilon_{\theta}$ is trained, the generation process starts from a Gaussian noise $z_{T} \sim \mathcal{N}(0, \mathbf{I})$, and gradually samples $z_{t-1} \sim p_{\theta}(z_{t-1} | z_t)$ until $t = 0$.
% For conditional image generation, additional information $c$ is provided to $\epsilon_{\theta}$ as a condition, and the reverse denoising process becomes $p_{\theta}(z_{t-1} | z_t, c)$.
\textbf{Diffusion and Flow Models.} Before introducing our Diffusion on Diffusion (DoD) framework, we provide a brief review of diffusion and flow models. Given the noise distribution $\epsilon \sim \mathcal{N}(0,\mathbf{I})$ and data distribution $x_0 \sim p(x)$, the models use the time-dependent forward process:
\begin{equation}
x_t = \alpha_t x_0 + \beta_t \epsilon,
\label{eq:unified_process}
\end{equation}
where $\alpha_t$ is a decreasing function of $t$ and $\beta_t$ is an increasing function of $t$.
In this unified perspective, diffusion models~\citep{ho2020denoising,song2020score,song2019ncsn,song2020ncsnv2} set $\alpha_t$ and $\beta_t$ based on stochastic differential equation (SDE) formulations, where DDPM~\citep{ho2020denoising} is equivalent to variance preserving SDE (VP-SDE) and SMLD~\citep{song2019ncsn,song2020ncsnv2} corresponds to variance exploding SDE (VE-SDE). DDIM~\citep{song2020denoising} sets $\alpha_t$ and $\beta_t$ through ordinary differential equations (ODE), which leads to fewer sampling steps but sacrifices the generation quality. Flow models~\citep{liu2023flow,lipman2022flowmatching,albergo2022normalizingflows,albergo2023stochasticinterpolants} restrict the process~\ref{eq:unified_process} on $t\in[0,1]$, and set $\alpha_0=\beta_1=1, \alpha_1=\beta_0=0$, interpolating between the two distributions through ODE formulation.
Rectified flow~\citep{liu2023flow} introduces an ODE model and transports between the data distribution and the noise distribution via a straight line path, which is the theoretically shortest route between two points.
Given empirical observations $X_0\sim p(x), X_1\sim \mathcal{N}(0,\mathbf{I})$, the forward process~\ref{eq:unified_process} is defined as: $X_t:=tX_1+(1-t)X_0$, which is the linear interpolation of $X_0$ and $X_1$.
The ODE model learns the drift at time $t\in[0,1]$:
\begin{equation}
\text{d}X_t = v(X_t, t)\text{d}t,
\end{equation}
which converts data $X_0$ from $p(x)$ to noise $X_1$ from $\pi_1$, and the drift follows the direction of $(X_1-X_0)$. In practice, a network $v_{\theta}$ is utilized to predict this drift, and the optimization target is:
\begin{equation}
\min_v\int_0^1 \mathbb E\Big[||(X_1-X_0)-v(X_t, t)||^2\Big]\text{d}t
\end{equation}
% In this work, we fully adhere to the form of diffusion models as our foundational component, and enhancing the diffusion model by adding extra conditions beyond the class label.
Previous studies~\citep{ma2024sit,esser2024sd3,wang2024fitv2,gao2024luminat2x} have demonstrated the efficiency and stability of rectified flow models. In this work, we fully adhere to the form of rectified flow as our noise scheduler and enhance the diffusion model by adding extra conditions beyond the class label. We adopt the implementation of rectified flow following SiT~\citep{ma2024sit} and use ODE sampler for image synthesis.
\textbf{FiTv2.}
DoD utilizes the state-of-the-art diffusion transformer, FiTv2~\citep{wang2024fitv2}, as the backbone.
FiTv2 is an advanced diffusion transformer on class-guided image generation, evolving from SiT~\citep{ma2024sit} and FiT~\citep{Lu2024FiT}. The key modules of FiTv2 include $2$-D Rotary Positional Embedding ($2$-D RoPE)~\citep{su2024rope}, Swish-Gated Linear Unit (SwiGLU)~\citep{shazeer2020glu}, Query-Key Vector Normalization (QK-Norm), and Adaptive Layer Normalization with Low-Rank Adaptation (AdaLN-LoRA)~\citep{hu2022lora}.
Despite these advanced modules, FiTv2 adopts the Logit-Normal sampling~\citep{esser2024sd3} strategy to accelerate the model convergence. This sampling strategy puts more attention on the middle part of the sampling process, as recent studies~\citep{karras2022edm,chen2023importance} have disclosed that the intermediate part is the most challenging part of the diffusion process.
% \textbf{$2$-D RoPE.} FiTv2 follows FiT and adopts $2$-D RoPE as its positional embedding. RoPE applys a rotary transformation to the embedding, incorporating both absolute and relative positional information into the query and key vectors. Benefiting from such property, RoPE and its high-dimensional variants have been widely adopted in current vision diffusion transformer models~\citep{Lu2024FiT,gao2024luminat2x,esser2024sd3}.
% \textbf{SwiGLU.} SwiGLU is widely used in advanced language models like LLaMA~\citep{touvron2023llama,touvron2023llama2,dubey2024llama3}. FiTv2 utilizes SwiGLU module as its Feed-forward Neural Network (FFN), rather than normal Multi-layer Perception (MLP). The SwiGLU is defined as:
% \begin{equation}
% \begin{split}
% \text{SwiGLU}(x, W, V) = \text{SiLU}(x W) \otimes (x V) \\
% \text{FFN}(x) = \text{SwiGLU}(x, W_1, W_2)W_3
% \end{split}
% \end{equation}
% \textbf{QK-Norm.} FiTv2 applies LayerNorm (LN) to the Query (Q) and Key (K) vectors before the attention calculation. This technique effectively stabilizes the training process, particularly during the mixed-precision setting, as well as slightly improves the performance.
% Formally, the attention is calculated as:
% \begin{equation}
% \text{Softmax}(\frac{1}{d_k}\text{LN}(Q_i)LN(K_i)^T).
% \end{equation}
% \textbf{AdaLN-LoRA.} FiTv2 utilizes AdaLN-LoRa to reduce the too many parameters occupied by the original AdaLN module in each transformer block. Besides, a global AdaLN module is employed to extract overlapping condition information and reduce the information redundancy in each block. Let $S^i = [ \beta_1^i, \beta_2^i, \gamma_1^i, \gamma_2^i, \alpha_1^i, \alpha_2^i ] \in \mathbb{R}^{6\times d} $ denote the tuple of all output scale and shift parameters, $\mathbf{c} \in \mathbb{R}^d$ and $\mathbf{t} \in \mathbb{R}^d$ represent the embedding for class and time step respectively. In FiTv2 blocks, $S^i$ is calculated as:
% \begin{equation}
% \begin{aligned}
% S^i &= \text{AdaLN}_{\text{global}}(\mathbf{c}+\mathbf{t}) + \text{AdaLN}_{\text{LoRA}}(\mathbf{c}+\mathbf{t}) \\
% &= W^{\text{g}}(\mathbf{c}+\mathbf{t}) + W_2^i W_1^i(\mathbf{c}+\mathbf{t}),
% \end{aligned}
% \label{eq:adaln_lora}
% \end{equation}
% where $W^{\text{g}}\in \mathbb{R}^{(6\times d)\times d}, W_2^i\in \mathbb{R}^{(6\times d)\times r}, W_1^i\in \mathbb{R}^{r\times d}$, and the bias parameters are omitted for simplicity.
% \textbf{Logit-Normal Sampling.} Despite these advanced modules, FiTv2 adopts the Logit-Normal sampling~\citep{esser2024sd3} strategy to accelerate the model convergence. Normally, rectified flow models sample times uniformly from the $[0,1]$ interval, which means each part of the noise scheduler is trained equally. FiTv2 samples timesepts from a logit-normal distribution~\citep{atchison1980lognorm}, which is formulated as:
% \begin{equation}
% u \sim \mathcal N(\mathbf 0,\mathbf 1), \quad
% t = \log(\frac{u}{1-u})
% \label{eq:logit_normal}
% \end{equation}
% where $\mathcal N(\mathbf 0,\mathbf 1)$ denotes the standard normal distribution. This sampling strategy puts more attention on the middle part of the sampling process, as recent studies~\citep{karras2022edm,chen2023importance} have disclosed that the intermediate part is the most challenging part in diffusion process.
% [VAE's latent space]
\begin{figure}[tb]
\centering
\begin{minipage}{0.59\linewidth}
\centering
\includegraphics[width=\linewidth]{figures/overview.pdf}
(a) Multi-Stage Sampling.
\end{minipage}
\hspace{0.02\linewidth}
\begin{minipage}{0.35\linewidth}
\centering
\includegraphics[width=\linewidth]{figures/lem.pdf}
(b) Latent Embedding Module.
\end{minipage}
\caption{\textbf{Illustration of (a) Multi-Stage Sampling and (b) Latent Embedding Module.} We only present the first two stages of DoD, while subsequent stages are derived from the second one.}
\label{fig:arch}
\end{figure}
\subsection{Diffusion on Diffusion with Visual Priors}
We introduce the Diffusion on Diffusion (DoD) framework, which enhances diffusion models by recurrently incorporating previously generated samples as visual priors to guide the subsequent sampling process.
% Visual priors provide more fine-grained and semantically rich guidance, effectively reducing the sampling space.
There are two core components in DoD: the Multi-Stage Sampling strategy and the Latent Embedding Module (LEM).
The multi-stage sampling enables the use of visual priors, while the LEM extracts semantic information from the samples generated in the previous stage.
\textbf{Multi-Stage Sampling.}
The sampling process of DoD consists of multiple stages, with each stage being a complete diffusion sampling process conditioned on different information.
As depicted in Figure \ref{fig:arch} (a), in the $i$-th stage, the backbone diffusion model, denoted as $F(\cdot)$, takes Gaussian noise $z_1^i \in \mathbb{R}^{H \times W \times d_{z}}$ as input and progressively denoises it to obtain the sample $z_0^i \in \mathbb{R}^{H \times W \times d_{z}}$, where $H$, $W$, and $d_z$ denote the height, width, and channels of the noised data, respectively. Let $c$ represent the conditional feature that guides the generation direction via the AdaLN-LoRA block, we have the sampling formulation as:
\begin{equation}
z_{\tau_{t-1}}^i = F(z_{\tau_{t}}^i, c), t = T, T-1, \dots, 1,
\end{equation}
where $T$ represents the total sampling steps, $\tau_t$ is a monotone increasing function of $t$ that satisfies $\tau_t\in[0,1], \tau_0=0, \tau_T=1$. The accurate form of $\{\tau_t\}_{1:T}$ is determined by the ODE sampler. The formulation of each stage closely follows that of FiTv2~\citep{wang2024fitv2}.
Typically, for class-conditional generation, the conditional feature $c$ consists only of class and time information.
In DoD, the sample generated in the previous iteration is used as additional conditioning information, as follows:
\begin{equation}
\begin{aligned}
&c = \begin{cases}
e_c + e_t + \mathbf{S} & \text{ if } i = 1 \\
e_c + e_t + e_s^{i-1} & \text{ if } i > 1
\end{cases}, \\
&e_s^{i-1} = \text{LEM}(z_0^{i-1}),
\end{aligned}
\end{equation}
where $e_c\in \mathbb{R}^{1 \times d}$ and $e_t\in \mathbb{R}^{1 \times d}$ are the embeddings for the label and time, and $e_s^{i-1} \in \mathbb{R}^{(\frac{H}{p} \times \frac{W}{p}) \times d}$ represents the embedding extracted by the latent embedding module $\text{LEM}(\cdot)$, with $p$ and $d$ denoting the patch size and hidden size of the backbone, respectively.
To maintain consistency in behavior, in the first stage, we replace $e_s$ with a trainable sample token $\mathbf{S} \in \mathbb{R}^{1 \times d}$.
% For the first iteration, to maintain consistency in behavior, we replace $e_s$ with a trainable sample token $\mathbf{S} \in \mathbb{R}^{1 \times d}$.
Note that when $i>1$, broadcast is operated on $e_c$ and $e_t$ to align their dimensions with the sample embedding $e_s$. We modify the modulation function in the AdaLN-LoRA block to accept conditional features with different dimensions accordingly.
\textbf{Latent Embedding Module (LEM).}
Since we use previously generated latents as visual priors to guide the generation process, retaining all the information from these samples would cause the diffusion model to collapse into an identity mapping, resulting in a complete reconstruction of the conditional samples.
To avoid this, we propose the Latent Embedding Module (LEM) that filters the conditional information using a compression-reconstruction approach to discard redundant details.
The architecture of LEM is illustrated in Figure \ref{fig:arch} (b), comprising an encoder and a decoder, both of which are standard vision transformers~\citep{dosovitskiy2020image}.
The encoder first patchifies the input latent $z_0 \in \mathbb{R}^{H \times W \times d_{z}}$ through a patch embedding layer into $e_p \in \mathbb{R}^{(\frac{H}{p} \times \frac{W}{p}) \times d_l}$, where $d_l$ is the hidden size of the LEM, and the patch size $p$ is consistent with that of the backbone network.
Next, $N$ learnable latent tokens $\mathbf{L} \in \mathbb{R}^{N \times d_l}$ are concatenated with $e_p$ and fed into the vision transformer.
A linear layer then reduces the channel dimension to $d_e$. Only the latent tokens are used as the output of the encoder, formally:
\begin{equation}
\hat{\mathbf{L}} = Enc((e_p + \textbf{PE}) \oplus \textbf{L}),
\end{equation}
where $\oplus$ denotes concatenation, $\textbf{PE}$ is the frozen sine-cosine positional embedding, and $Enc(\cdot)$ indicates the encoder. $\hat{\mathbf{L}} \in \mathbb{R}^{N \times d_e}$ represents the output latent tokens, providing a compressed representation of the input latent $z_0$.
The decoder seeks to map the compressed latent tokens $\hat{\mathbf{L}}$ back to the shape of the patch embedding $e_p$ to enable element-wise conditioning.
Specifically, a sequence of mask tokens $\mathbf{M} \in \mathbb{R}^{(\frac{H}{p} \times \frac{W}{p}) \times d_e}$, obtained by repeating a shared mask token, is concatenated with $\hat{\mathbf{L}}$.
The combined representation is then projected back to $d_l$ and fed into the vision transformer:
\begin{equation}
e_s = Dec((\textbf{M} + \textbf{PE}) \oplus \hat{\mathbf{L}}),
\end{equation}
where $Dec(\cdot)$ denotes the decoder, which has the same number of heads and hidden size as the encoder. The output $e_s$ is the extracted latent condition.
% Equipped with the Latent Embedding Module (LEM), DoD converts the image generation task into an image reconstruction task, similar to an autoencoder~\citep{hinton1993autoencoders}.
Equipped with the Latent Embedding Module (LEM), DoD realizes image refinement effectively through latent space reconstruction.
The encoder of LEM projects the input latents into a more compact feature space, which is then restored by the decoder and the diffusion model.
We empirically find that the compressed latent tokens $\hat{\mathbf{L}}$ primarily retain semantic information, enabling the diffusion model to focus on texture details, thereby enhancing image fidelity, as in section \ref{subsec:preliminary_experiments}.
\subsection{Configurations}
\begin{table}[t]
\caption{\textbf{Details of DoD models.} The layers of the latent embedding module are presented as {\textcolor{violet}{encoder depth}} $+$ {\textcolor{brown}{decoder depth}}. We list the number of parameters for the {\textcolor{orange}{backbone}} and the {\textcolor{cyan}{latent embedding module}} in different colors.}
\label{tab:model_size}
\begin{center}
\begin{adjustbox}{max width=0.9\textwidth}
\begin{tabular}{l|ccc|ccc|c}
\toprule[1.2pt]
\multirow{2}{*}{Model} &\multicolumn{3}{c|}{\textcolor{orange}{Backbone}} & \multicolumn{3}{c|}{\textcolor{cyan}{Latent Embedding Module}} & \multirow{2}{*}{Params (M)}\\
& Layers & Hidden size & Heads & Layers & Hidden size & Heads & \\
\midrule
DoD-S & 12 & 384 & 6 & \textcolor{violet}{8} + \textcolor{brown}{4} & 384 & 6 & \textcolor{orange}{27} + \textcolor{cyan}{21} = 48 \\
DoD-B & 12 & 768 & 12 & {\textcolor{violet}{8} + \textcolor{brown}{4}} & {768} & {12} & \textcolor{orange}{105} + \textcolor{cyan}{86} = 191 \\
DoD-XL & 28 & 1152 & 16 & {\textcolor{violet}{8} + \textcolor{brown}{4}} & {768} & {12} & \textcolor{orange}{527} + \textcolor{cyan}{86} = 613 \\
% DoD-B & 12 & 768 & 12 & \multirow{2}{*}{\textcolor{violet}{8} + \textcolor{brown}{4}} & \multirow{2}{*}{768} & \multirow{2}{*}{12} & \textcolor{orange}{105} + \textcolor{cyan}{86} \\
% DoD-XL & 28 & 1152 & 16 & & & & \textcolor{orange}{527} + \textcolor{cyan}{86} \\
\bottomrule[1.2pt]
\end{tabular}
\end{adjustbox}
\vspace{-0.3cm}
\end{center}
\end{table}
\textbf{Training.}
DoD is trained within the latent space of a pre-trained Variational Autoencoder (VAE) from Stable Diffusion~\citep{rombach2022high}.
The VAE encoder maps RGB images of shape $256 \times 256 \times 3$ to latents of shape $32 \times 32 \times 4$.
The new latents sampled by DoD are then decoded back to pixel space by the VAE decoder.
As mentioned above, the Latent Embedding Module (LEM) extracts semantic information from samples through compression and reconstruction to serve as visual priors.
We reasonably assume that the high-level semantic information extracted from generated images is similar to that obtained from real images.
This assumption allows us to use the latents of ground truth images as inputs to LDM during training, simplifying the training strategy. Such simplification allows end-to-end training of DoD on image latents and joint optimization of the backbone model and LEM.
% During the training phase, we use the latents of ground truth images, which are extracted by an off-the-shelf pre-trained VAE, as inputs to the latent embedding module.
% This simplifies the training strategy and allows DoD to be trained end-to-end on generative datasets.
Similar to training with classifier-free guidance, we randomly replace the output of the latent embedding module with a trainable sample token $\mathbf{S}$ at a probability of $p_s$, which is set to $0.5$ by default.
\textbf{Sampling.}
Classifier-free guidance (CFG)~\citep{ho2021cfg} is well-known for enhancing generation quality and improving the alignment between conditions and generated images.
For visual priors, we employ a large CFG scale to ensure the consistency of the generated samples, while we do not use CFG in the first stage of DoD.
Additionally, DoD employs the adaptive-step ODE sampler (\ie{}., dopri5) same as SiT~\citep{ma2024sit} for sample synthesis in each stage.
% We also adopt classifier-free guidance[] following FiTv2.
% [CFG] [Sampler]
\textbf{Models.}
We use three different model configurations with varying sizes, DoD-\{S, B, XL\}, as detailed in Table \ref{tab:model_size}.
For the backbone network, \ie{}, FiTv2, we closely follow the configurations outlined in their paper but reduce the model depth to match those of DiT~\citep{peebles2023scalable} and SiT~\citep{ma2024sit}.
For the latent embedding module, we use an 8-layer encoder and a 4-layer decoder by default.
Our largest model, DoD-XL, uses the same size latent embedding module as DoD-B to reduce the number of parameters.
The patch size for both the backbone network and the latent embedding module is set to $2$, and the channel dimension $d_e$ of the latent tokens in the latent embedding module is set to $16$.
\section{Experiments}
\subsection{Experimental Setup}
\textbf{Implementation Details.}
We conduct experiments on ImageNet~\citep{deng2009imagenet}, using a resolution of $256 \times 256$.
All models share the same training strategy.
% We use a constant learning rate of $1 \times 10^{-4}$ with AdamW~\citep{kingma2014adam, loshchilov2017decoupled} as the optimizer, without weight decay.
We use the AdamW~\citep{kingma2014adam, loshchilov2017decoupled} optimizer with a constant learning rate of $1 \times 10^{-4}$ and without weight decay.
Models are trained with a batch size of $256$.
Following SiT and FiTv2, we also apply an exponential moving average (EMA) with a decay factor of $0.999$ to the model weights during training, and we report the performance of the EMA checkpoints.
\textbf{Evaluation Metrics.}
We use Fréchet Inception Distance (FID)~\citep{heusel2017gans} as the primary evaluation metric. Inception Score (IS)~\citep{salimans2017inceptionscore}, sFID~\citep{nash2021sfid}, improved Precision and Recall~\citep{kynk2017precisionrecall} are also included to holistically evaluate the generation quality.
Moreover, the inference of DoD involves two processes -- \textit{sampling with class priors and sampling with visual priors}.
Since the latter process is formally similar to image reconstruction, we also adopt reconstruction-FID (rFID), PSNR, and SSIM to assess the reconstruction quality, providing a more comprehensive evaluation of our method.
\subsection{Preliminary Experiments}
\label{subsec:preliminary_experiments}
\begin{figure}[t]
\centering
\begin{minipage}{0.45\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/cfg_rfid.png}
% (a)
\end{minipage}
\begin{minipage}{0.48\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/stage_fid.png}
% (b)
\end{minipage}
\caption{\textit{Left:} \textbf{rFID score with different CFG scales.} A large CFG scale is required in DoD to effectively leverage visual priors. \textit{Right:} \textbf{FID score comparison across different stages.} Visual priors are not available in Stage 1, while both stages 2 \& 3 utilize the samples from the previous stage as visual priors. }
\label{fig:ablation_fig}
\end{figure}
% The improvement after the second stage is negligible.
% \textbf{The Upper Bound of Visual Guidance.}
\textbf{Visual Prior Drastically Improves Performance.}
We first verify how effectively visual priors can guide image generation.
% To do this, we use the latents of ground truth images as inputs for the latent embedding module and report the reconstruction-FID (rFID) of the generated images.
% Since ground truth images provide better priors than generated samples, the reconstruction results can be considered the upper bound of our method’s performance.
We use the latents of ground truth images as inputs for the latent embedding module and report the reconstruction-FID (rFID) of the generated images.
This experiment is conducted with a DoD-B model trained with $400K$ steps.
As shown in Figure~\ref{fig:ablation_fig} (\textit{Left}), classifier-free guidance (CFG) is critical for DoD, with a high CFG scale leading to significant improvements. The higher the CFG scale is, the more the model relies on the input condition, which contains the visual prior.
When the CFG scale is set to $5.5$, the model performs best, achieving an rFID score of $3.6$, which is much better than the score without CFG ($10.8$ rFID).
This encouraging result demonstrates the potential of conditioning on visual priors.
% This encouraging result demonstrates that DoD has a high upper bound.
In this work, unless otherwise stated, we default to using a CFG scale of $5.5$ when employing image priors as conditions for DoD.
We also present the generation results of DoD models trained for $400K$ steps in Figure~\ref{fig:ablation_fig} (\textit{Right}).
The visual prior is significantly effective in improving the FID-50K score, since the performance of Stage 2 (\textit{w/} visual prior) is obviously better than the performance of Stage 1 (\textit{w/o} visual prior).
% In contrast to DiT and SiT, which typically employ a smaller CFG scale (\eg{}, $1.5$), the image prior contains richer semantic information, providing clearer guidance for synthesis.
% Therefore, increasing the CFG scale allows the model to rely more on the conditions, leading to improved performance.
% Based on this, in this work, unless otherwise stated, we default to using a CFG scale of $5.5$ when employing image priors as conditions for DoD.
\begin{table}[t]
\caption{ \textbf{Effects of the number of latent tokens $N$ in LEM.} We provide a comprehensive evaluation that includes: generation in Stage 1 (S1) and Stage 2 (S2), image reconstruction, and linear probing. The results are based on DoD-B, trained for $400K$ steps.}
\label{tab:num_tokens}
\begin{center}
\begin{adjustbox}{max width=0.8\textwidth}
\begin{tabular}{c|cccc|ccc|c}
% \hline
\toprule[1.2pt]
\multirow{2}{*}{$N$} &\multicolumn{4}{c|}{Generation} & \multicolumn{3}{c|}{Reconstruction} & Linear Probe\\
& FID$_{S1}$$\downarrow$ & IS$_{S1}$$\uparrow$ & FID$_{S2}$$\downarrow$ & IS$_{S2}$$\uparrow$ & rFID$\downarrow$ & PSNR$\uparrow$ & SSIM$\uparrow$ & Top-1$\uparrow$\\
% \hline
\midrule
16 & 28.87 & 47.93 & 4.95 & \textbf{207.62} & 4.52 & 15.88 & 0.46 & \textbf{56.6}\\
32 & \textbf{28.65} & \textbf{47.98} & \textbf{4.93} & 187.14 & 3.63 & 16.17 & 0.48 & 53.2\\
64 & 31.36 & 43.61 & 9.66 & 121.13 & 2.20 & 17.69 & 0.56 & 52.4\\
256& 33.05 & 41.97 & 33.12 & 42.02 & \textbf{0.63} & \textbf{24.99} & \textbf{0.80} & 51.3\\
% \hline
\bottomrule[1.2pt]
\end{tabular}
\end{adjustbox}
\end{center}
\end{table}
\begin{table}[t]
\caption{\textbf{Effects of the joint training strategy in DoD.} We report the results based on DoD-B trained for $400K$ steps.}
\label{tab:drop_prob}
\begin{center}
\begin{adjustbox}{max width=0.7\textwidth}
\begin{tabular}{c|cccc|ccc}
% \hline
\toprule[1.2pt]
\multirow{2}{*}{$p_s$} &\multicolumn{4}{c|}{Generation} & \multicolumn{3}{c}{Reconstruction}\\
& FID$_{S1}$$\downarrow$ & IS$_{S1}$$\uparrow$ & FID$_{S2}$$\downarrow$ & IS$_{S2}$$\uparrow$ & rFID$\downarrow$ & PSNR$\uparrow$ & SSIM$\uparrow$\\
% \hline
\midrule
0.25 & 31.06 & 43.38 & 5.48 & 169.91 & \textbf{3.47} & \textbf{16.28} & \textbf{0.49} \\
0.50 & {28.65} & {47.98} & {4.93} & 187.14 & 3.63 & 16.17 & 0.48\\
0.75 & \textbf{26.60} & \textbf{52.77} & \textbf{4.31} & \textbf{222.84} & 4.22 & 15.94 & 0.48\\
% \hline
\bottomrule[1.2pt]
\end{tabular}
\end{adjustbox}
\end{center}
\end{table}
\textbf{Extracting Visual Priors Through Compression.}
In Table~\ref{tab:num_tokens}, we evaluate the effectiveness of the compression-reconstruction approach used by DoD to extract visual prior information from three perspectives: generation, reconstruction, and linear probing.
DoD controls the compression rate through the number of latent tokens $N$ in the latent embedding module, where using 256 tokens means no compression is applied.
Thanks to the generative capabilities of the diffusion model, we can reconstruct images using only $16$ latent tokens.
We draw the following conclusions:
\begin{itemize}
\item A lower compression rate (\ie{}, using more tokens) results in better reconstruction performance. This is intuitive, as a lower compression rate means that more information is retained for image restoration.
\item More semantically rich representations can be learned with fewer tokens. Drawing from self-supervised learning, we measure the quality of the representations using linear probing accuracy. The best performance is achieved with $16$ tokens, demonstrating that the latent embedding module in DoD effectively extracts global semantic information as visual priors through compression.
\end{itemize}
Interestingly, DoD with $32$ tokens performs best in generation during both the first and second stages, suggesting a trade-off between reconstruction capability and semantic richness.
By default, we utilize 32 latent tokens across all DoD variants.
% Additionally, the training of DoD is based on the assumption that images of varying fidelity can provide similar visual priors through compression.
% We validate this assumption in Figure~\ref{fig:ablation_fig} (\textit{Right}), where the output from stage 1 is used as a visual prior in stage 2, yielding more realistic images.
% However, stage 3, which relies on stage 2, does not show further improvements, indicating that the poor outputs from stage 1 and the good outputs from stage 2 provide similar prior information.
% Therefore, in our experiments, we report the performance of DoD from stage 2.
% \begin{table}[t]
% \caption{Sample table title}
% \label{tab:step_flops}
% \begin{center}
% \begin{tabular}{l|l|cccc|l}
% % \hline
% Model & Steps & FID$\downarrow$ & IS$\uparrow$ & Prec.$\uparrow$ & Rec.$\uparrow$ & Total GFLOPs$\downarrow$ \\
% \hline
% \hline
% FiTv2-B & 60 & 19.67 & 73.34 & 0.61 & \textbf{0.65} & 1638$_{(=27.3\times60)}$ \\
% \hline
% FiTv2-B$_{cfg=1.5}$ & 60 & 5.35 & 163.65 & 0.77 & 0.56 & 3276$_{(=27.3\times2\times60)}$ \\
% \hline
% \multirow{2}{*}{DoD-B} & 30 & 23.43 & 65.85 & 0.59 & 0.65 & 795$_{(=26.5\times30)}$ \\
% & 60 & \textbf{3.35} & \textbf{262.61} & \textbf{0.84} & 0.51 & {2409.6}$_{(=795+24.6+26.5\times2\times30)}$ \\
% \hline
% \hline
% FiTv2-B & 240 & 18.66 & 73.96 & 0.61 & \textbf{0.66} & 6552$_{(=27.3\times240)}$ \\
% \hline
% FiTv2-B$_{cfg=1.5}$ & 240 & 5.03 & 165.41 & 0.77 & 0.57 & 13104$_{(=27.3\times2\times240)}$ \\
% \hline
% \multirow{2}{*}{DoD-B} & 120 & 20.81 & 67.67 & 0.60 & 0.67 & 3180$_{(=26.5\times120)}$ \\
% & 240 & \textbf{3.13} & \textbf{254.34} & \textbf{0.82} & 0.53 & {9564.6}$_{(=3180+24.6+26.5\times2\times120)}$ \\
% \hline
% \end{tabular}
% \end{center}
% \end{table}
\begin{table}[t]
\caption{\textbf{Comparison between DoD and FiTv2.} Both of the models are trained with 1.5M steps. We report the performance of DoD in Stage 1 (S1) and Stage 2 (S2) separately. With the same sampling steps and less computation, our DoD demonstrates better FID performance.}
\label{tab:step_flops}
% \begin{center}
\begin{adjustbox}{max width=1.\textwidth}
\centering
\begin{tabular}{l|l|ccccc|l}
% \hline
\toprule[1.2pt]
Model & Steps & FID$\downarrow$ & sFID$\downarrow$ & IS$\uparrow$ & Prec.$\uparrow$ & Rec.$\uparrow$ & Sampling Compute GFLOPs$\downarrow$ \\
% \hline
% \hline
\midrule
FiTv2-B & 60 & 19.67 & 6.45 & 73.34 & 0.61 & \textbf{0.66} & 1638$_{(=27.3\times60)}$ \\
% \hline
FiTv2-B-G (cfg=1.5) & 60 & 5.35 & \textbf{4.82} & 163.65 & 0.77 & 0.56 & 3276$_{(=27.3\times2\times60)}$ \\
% \hline
\baseline{DoD-B (S1)} & \baseline{30} & \baseline{23.43} & \baseline{7.36} & \baseline{65.85} & \baseline{0.59} & \baseline{0.65} & \baseline{795$_{(=26.5\times30)}$} \\
\baseline{DoD-B-G (S2, cfg=5.5)} & \baseline{60$_{=(30+30)}$} & \baseline{\textbf{3.35}} & \baseline{4.87} & \baseline{\textbf{262.61}} & \baseline{\textbf{0.84}} & \baseline{0.51} & \baseline{{2409.6}$_{(=795+24.6+26.5\times2\times30)}$} \\
% \multirow{2}{*}{DoD-B} & 30 & 23.43 & 65.85 & 795$_{(=26.5\times30)}$ \\
% & 60$_{=(30+30)}$ & \textbf{3.35} & \textbf{262.61} & {2409.6}$_{(=795+24.6+26.5\times2\times30)}$ \\
% \hline
% \hline
\midrule
FiTv2-B & 240 & 18.66 & 6.15 & 73.96 & 0.61 & 0.66 & 6552$_{(=27.3\times240)}$ \\
% \hline
FiTv2-B-G (cfg=1.5) & 240 & 5.03 & \textbf{4.91} & 165.41 & 0.77 & 0.57& 13104$_{(=27.3\times2\times240)}$ \\
% \hline
\baseline{DoD-B (S1) }& \baseline{120} & \baseline{20.81} & \baseline{6.39} & \baseline{67.67} & \baseline{0.60} & \baseline{\textbf{0.67}} & \baseline{3180$_{(=26.5\times120)}$} \\
\baseline{DoD-B-G (S2, cfg=5.5)} & \baseline{240$_{=(120+120)}$} & \baseline{\textbf{3.13}} & \baseline{5.44} & \baseline{\textbf{254.34}} & \baseline{\textbf{0.82}} & \baseline{0.53} &\baseline{{9564.6}$_{(=3180+24.6+26.5\times2\times120)}$} \\
% \multirow{2}{*}{DoD-B} & 120 & 20.81 & 67.67 & 3180$_{(=26.5\times120)}$ \\
% & 240$_{=(120+120)}$ & \textbf{3.13} & \textbf{254.34} & {9564.6}$_{(=3180+24.6+26.5\times2\times120)}$ \\
% \hline
\bottomrule[1.2pt]
\end{tabular}
\end{adjustbox}
% \end{center}
\end{table}
\begin{table}[t]
\caption{\textbf{Benchmarking class-conditional image generation on ImageNet $256\times256$.} ``-G'' indicates results with classifier-free guidance, ``S2'' denotes results from the second stage of DoD.}
\label{tab:sota}
\begin{center}
\begin{adjustbox}{max width=.8\textwidth}
\begin{tabular}{l|cc|ccccc}
\toprule[1.2pt]
Model & Images & Params & FID$\downarrow$ & sFID$\downarrow$ & IS$\uparrow$ & Prec.$\uparrow$ & Rec.$\uparrow$ \\
% \hline
\midrule
BigGAN-deep & - & - & 6.95 & 7.36 & 171.40 & \textbf{0.87} & 0.28 \\
StyleGAN-XL & - & - & 2.30 & \textbf{4.02} & 265.12 & 0.78 & 0.53 \\
MaskGIT & 355M & - &6.18 &- &182.10 &0.80 &0.51 \\
CDM & - & - & 4.88 & - & 158.71 & - & - \\
ADM-G,U & 507M & 673M & 3.94 & 6.14 & 215.84 & 0.83 & 0.53 \\
LDM-4-G (cfg=1.5) & 214M & 395M & 3.60 & 5.12 & 247.67 & \textbf{0.87} & 0.48 \\
U-ViT-H-G (cfg=1.4) & 512M & 501M & 2.35 &5.68&265.02&0.82&0.57 \\
Efficient-DiT-G (cfg=1.5) & - & 675M & 2.01 & 4.49 & 271.04 & 0.82 & 0.60 \\
Flag-DiT-G & 256M & 4.23B & 1.96 & 4.43 & \textbf{284.80} & 0.82 & 0.61 \\
% DiT-XL/2 & 1792M & 675M & 9.62 & 6.85 & 121.50 & 0.67 & 0.67 \\
DiT-XL-G (cfg=1.5) & 1792M & 675M & 2.27 & 4.60 & 278.24 & 0.83 & 0.57 \\
SiT-XL-G (cfg=1.5) & 1792M & 675M & 2.15 & 4.50 & 258.09 & 0.81 & 0.60 \\
FiT-XL-G (cfg=1.5) & 512M & 824M & 4.21 & 10.01 & 254.87 & 0.84 & 0.51 \\
FiTv2-XL-G (cfg=1.5) & 512M & 671M & 2.26 & 4.53 & 260.95 & 0.81 & 0.59 \\
FiTv2-3B-G (cfg=1.5) & 256M & 3B
&2.15&4.49&276.32&0.82&0.59\\
% \hline
% \hline
\midrule
DoD-S-G (S2, cfg=5.5) & 102.4M & 48M & 19.70 & 8.08 & 74.94 & 0.67 & 0.48 \\
& 256M & 48M & 11.97 & 6.23 & 108.95 & 0.74 & 0.48 \\
& 384M & 48M & 9.71 & 6.16 & 123.63 & 0.75 & 0.49 \\
% \hline
% \hline
\midrule
DoD-B-G (S2, cfg=5.5) & 102.4M & 191M & 4.93 & 5.61 & 187.14 & 0.81 & 0.48 \\
& 256M & 191M & 3.42 & 5.65 & 236.59 & 0.82 & 0.52 \\
& 384M & 191M & 3.15 & 5.74 & 248.48 & 0.82 & 0.54 \\
% \hline
% \hline
\midrule
DoD-XL-G (S2, cfg=3.5) & 102.4M & 613M & 2.34 & 5.00 & 228.42 & 0.78 & 0.61 \\
& 256M & 613M & \textbf{1.83} & 5.00 & 263.69 & 0.77 & \textbf{0.65} \\
% & 384M & 613M & - & - & - & - & - \\
% \hline
\bottomrule[1.2pt]
\end{tabular}
\end{adjustbox}
\end{center}
\end{table}
\textbf{Joint Training in DoD.}
In DoD, we implement a joint training strategy that optimizes both the backbone diffusion model and the latent embedding module concurrently.
The training process is controlled by a hyperparameter $p_s$, which dictates the probability that the ground truth image priors are not utilized.
The results of generation and reconstruction are detailed in Table~\ref{tab:drop_prob}.
A higher value of $p_s$ leads to improved generation performance, while a lower value is more beneficial for reconstruction.
As this work aims to validate the importance of visual priors rather than pursuing state-of-the-art performance, we simply set $p_s = 0.5$ as the default value.
Additionally, the joint training of DoD is based on the assumption that images of varying fidelity can provide similar visual priors through compression.
We validate this assumption in Figure~\ref{fig:ablation_fig} (\textit{Right}), where the output from stage 1 is used as a visual prior in stage 2, yielding more realistic images.
However, stage 3, which relies on stage 2, does not show further improvements, indicating that the poor outputs from stage 1 and the good outputs from stage 2 provide similar prior information.
Therefore, in our experiments, we report the performance of DoD from stage 2.
% \textbf{Fair Comparison with Baseline Model.}
\textbf{Visual Prior Leads to Efficient Sampling in DoD.}
One potential drawback of DoD is that it involves a multi-stage sampling process, which may result in a substantial computational overhead.
Therefore, in Table 4, we demonstrate that \textit{DoD outperforms the baseline model (i.e., FiTv2) even with fewer inference GFLOPs.}
We employ the Euler sampler and fixed the number of sampling steps.
The total GFLOPs is calculated as GFLOPs multiplied by the number of sampling steps.
For methods utilizing classifier-free guidance (CFG), the total GFLOPs are doubled.
For DoD, we separately present the performance of two sampling stages, where the total GFLOPs for stage 2 are added to those of stage 1.
Our observations are as follows:
\begin{itemize}
\item Increasing the sampling steps from 60 to 240 reduces the FID score across all cases.
\item Due to the use of a shallower backbone, DoD at Stage 1 performs worse than FiTv2. However, in Stage 2, with the same number of sampling steps and fewer GFLOPs, DoD fully surpasses the baseline FiTv2. This further demonstrates the efficacy of the visual prior.
\end{itemize}
This experiment highlights the efficiency of DoD and the effectiveness of leveraging visual priors.
\subsection{Main Experiments}
The comparison against state-of-the-art class-conditional generation methods is shown in Table~\ref{tab:sota}.
To ensure fair comparisons, we use the total number of training images (denoted as ``Images'' in the table) as a measure of the training cost.
% For fair comparisons, we use the total number of training images, listed as ``Images'' in the table, to measure the training cost.
The total number of training images is calculated as $\text{training steps} \times \text{batch size}$.
\textbf{DoD achieves better FID scores with lower training costs and much fewer parameters.}
Specifically, using only $191M$ parameters, DoD-B achieves an FID score of $3.15$, surpassing both ADM ($3.94$ FID with $673M$ parameters) and LDM-4 ($3.60$ FID with $395M$ parameters).
% This encouraging result demonstrates that DoD significantly enhances the upper bound of diffusion models while effectively improving training efficiency.
When scaling up the model, DoD-XL further improves performance to a FID of $1.83$, using only $613M$ parameters and $1M$ training steps, outperforming all previous diffusion models. Qualitative results are shown in Figure~\ref{fig:teaser} and Figure~\ref{fig:qualitative_results}.
\begin{figure}[t]
\centering
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/1/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/1/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/1/3.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/2/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/2/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/2/3.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/12/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/12/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/12/3.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/13/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/13/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/13/3.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/3/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/3/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/3/3.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/4/0.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/4/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/4/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/5/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/5/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/5/3.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/6/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/6/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/6/3.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/7/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/7/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/7/3.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/8/1.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/8/2.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/8/3.png}
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/11/1.png}
Stage 1
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/11/2.png}
Stage 2
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/11/3.png}
Stage 3
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/10/1.png}
Stage 1
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/10/2.png}
Stage 2
\end{minipage}
\begin{minipage}{0.155\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/three_stage/10/3.png}
Stage 3
\end{minipage}
\caption{\textbf{Qualitative Results.} We present images generated by DoD-XL with $1M$ training steps. Across all stages, semantic information is preserved, and image quality improves progressively.}
\label{fig:qualitative_results}
\end{figure}
% \subsection{Ablation Studies}
% \textbf{Sampler.}
% \textbf{dropout prob.}
% \textbf{Linear probing.}
\section{Conclusion}
In this work, we propose using visual priors to further enhance diffusion models. To achieve this, we introduce Diffusion on Diffusion (DoD), a novel multi-stage generation framework that provides rich guidance for the diffusion model by leveraging visual priors from previously generated samples.
% In this work, we introduce Diffusion on Diffusion (DoD), a novel multi-stage generation framework that leverages visual priors to enhance diffusion models.
The latent embedding module in DoD effectively extracts semantic information from generated samples through a compression-reconstruction approach.
DoD demonstrates remarkable training and sampling efficiency while being easy to reproduce.
We conduct extensive experiments to validate the potential of injecting visual priors and explore the design space of the model.
We hope our work will inspire the community to further investigate the use of visual priors in image generation.
\section{Appendix}
\subsection{Technologies adopted by FiTv2}
FiTv2~\cite{wang2024fitv2} is an advanced diffusion transformer on class-guided image generation, evolving from SiT~\citep{ma2024sit} and FiT~\citep{Lu2024FiT}. The key modules of FiTv2 include $2$-D Rotary Positional Embedding ($2$-D RoPE)~\citep{su2024rope}, Swish-Gated Linear Unit (SwiGLU)~\citep{shazeer2020glu}, Query-Key Vector Normalization (QK-Norm), and Adaptive Layer Normalization with Low-Rank Adaptation (AdaLN-LoRA)~\citep{hu2022lora}.
\textbf{$2$-D RoPE.} FiTv2 follows FiT and adopts $2$-D RoPE as its positional embedding. RoPE applys a rotary transformation to the embedding, incorporating both absolute and relative positional information into the query and key vectors. Benefiting from such property, RoPE and its high-dimensional variants have been widely adopted in current vision diffusion transformer models~\citep{Lu2024FiT,gao2024luminat2x,esser2024sd3}.
\textbf{SwiGLU.} SwiGLU is widely used in advanced language models like LLaMA~\citep{touvron2023llama,touvron2023llama2,dubey2024llama3}. FiTv2 utilizes SwiGLU module as its Feed-forward Neural Network (FFN), rather than normal Multi-layer Perception (MLP). The SwiGLU is defined as:
\begin{equation}
\begin{split}
\text{SwiGLU}(x, W, V) = \text{SiLU}(x W) \otimes (x V) \\
\text{FFN}(x) = \text{SwiGLU}(x, W_1, W_2)W_3
\end{split}
\end{equation}
\textbf{QK-Norm.} FiTv2 applies LayerNorm (LN) to the Query (Q) and Key (K) vectors before the attention calculation. This technique effectively stabilizes the training process, particularly during the mixed-precision setting, as well as slightly improves the performance.
Formally, the attention is calculated as:
\begin{equation}
\text{Softmax}(\frac{1}{d_k}\text{LN}(Q_i)LN(K_i)^T).
\end{equation}
\textbf{AdaLN-LoRA.} FiTv2 utilizes AdaLN-LoRa to reduce the too many parameters occupied by the original AdaLN module in each transformer block. Besides, a global AdaLN module is employed to extract overlapping condition information and reduce the information redundancy in each block. Let $S^i = [ \beta_1^i, \beta_2^i, \gamma_1^i, \gamma_2^i, \alpha_1^i, \alpha_2^i ] \in \mathbb{R}^{6\times d} $ denote the tuple of all output scale and shift parameters, $\mathbf{c} \in \mathbb{R}^d$ and $\mathbf{t} \in \mathbb{R}^d$ represent the embedding for class and time step respectively. In FiTv2 blocks, $S^i$ is calculated as:
\begin{equation}
\begin{aligned}
S^i &= \text{AdaLN}_{\text{global}}(\mathbf{c}+\mathbf{t}) + \text{AdaLN}_{\text{LoRA}}(\mathbf{c}+\mathbf{t}) \\
&= W^{\text{g}}(\mathbf{c}+\mathbf{t}) + W_2^i W_1^i(\mathbf{c}+\mathbf{t}),
\end{aligned}
\label{eq:adaln_lora}
\end{equation}
where $W^{\text{g}}\in \mathbb{R}^{(6\times d)\times d}, W_2^i\in \mathbb{R}^{(6\times d)\times r}, W_1^i\in \mathbb{R}^{r\times d}$, and the bias parameters are omitted for simplicity.
\textbf{Logit-Normal Sampling.} Despite these advanced modules, FiTv2 adopts the Logit-Normal sampling~\citep{esser2024sd3} strategy to accelerate the model convergence. Normally, rectified flow models sample times uniformly from the $[0,1]$ interval, which means each part of the noise scheduler is trained equally. FiTv2 samples timesepts from a logit-normal distribution~\citep{atchison1980lognorm}, which is formulated as:
\begin{equation}
u \sim \mathcal N(\mathbf 0,\mathbf 1), \quad
t = \log(\frac{u}{1-u})
\label{eq:logit_normal}
\end{equation}
where $\mathcal N(\mathbf 0,\mathbf 1)$ denotes the standard normal distribution. This sampling strategy puts more attention on the middle part of the sampling process, as recent studies~\citep{karras2022edm,chen2023importance} have disclosed that the intermediate part is the most challenging part in diffusion process.
\subsection{More Qualitative Results}
We present additional qualitative results generated by DoD-B and DoD-XL, displayed in Figure~\ref{fig:viz_b} and Figure~\ref{fig:viz_xl}, respectively.
\begin{figure}[t]
\centering
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/1/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/1/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/2/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/2/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/3/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/3/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/4/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/4/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/5/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/5/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/6/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/6/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/7/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/7/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/8/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/8/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/9/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/9/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/14/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/14/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/11/1.png}
Stage 1
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/11/2.png}
Stage 2
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/13/1.png}
Stage 1
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/B/13/2.png}
Stage 2
\end{minipage}
\caption{\textbf{Images generated by DoD-B.} }
\label{fig:viz_b}
\end{figure}
\begin{figure}[t]
\centering
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/15/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/15/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/2/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/2/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/3/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/3/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/17/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/17/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/5/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/5/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/6/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/6/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/7/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/7/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/8/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/8/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/9/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/9/2.png}
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/14/1.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/14/2.png}
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/11/1.png}
Stage 1
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/11/2.png}
Stage 2
\end{minipage}
\hspace{0.01\textwidth}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/16/1.png}
Stage 1
\end{minipage}
\begin{minipage}{0.24\textwidth}
\centering
\includegraphics[width=\linewidth]{figures/viz/XL/16/2.png}
Stage 2
\end{minipage}
\caption{\textbf{Images generated by DoD-XL.} }
\label{fig:viz_xl}
\end{figure}
\begin{thebibliography}{64}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Albergo \& Vanden-Eijnden(2022)Albergo and Vanden-Eijnden]{albergo2022normalizingflows}
Michael~S Albergo and Eric Vanden-Eijnden.
\newblock Building normalizing flows with stochastic interpolants.
\newblock \emph{arXiv preprint arXiv:2209.15571}, 2022.
\bibitem[Albergo et~al.(2023)Albergo, Boffi, and Vanden-Eijnden]{albergo2023stochasticinterpolants}
Michael~S Albergo, Nicholas~M Boffi, and Eric Vanden-Eijnden.
\newblock Stochastic interpolants: A unifying framework for flows and diffusions.
\newblock \emph{arXiv preprint arXiv:2303.08797}, 2023.
\bibitem[Atchison \& Shen(1980)Atchison and Shen]{atchison1980lognorm}
Jhon Atchison and Sheng~M Shen.
\newblock Logistic-normal distributions: Some properties and uses.
\newblock \emph{Biometrika}, 1980.
\bibitem[Bao et~al.(2023)Bao, Nie, Xue, Cao, Li, Su, and Zhu]{bao2023all}
Fan Bao, Shen Nie, Kaiwen Xue, Yue Cao, Chongxuan Li, Hang Su, and Jun Zhu.
\newblock All are worth words: A vit backbone for diffusion models.
\newblock In \emph{IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2023.
\bibitem[Bond-Taylor et~al.(2021)Bond-Taylor, Leach, Long, and Willcocks]{bond2021deepgenmodel}
Sam Bond-Taylor, Adam Leach, Yang Long, and Chris~G Willcocks.
\newblock Deep generative modelling: A comparative review of vaes, gans, normalizing flows, energy-based and autoregressive models.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence}, 2021.
\bibitem[Brown et~al.(2020)Brown, Mann, Ryder, Subbiah, Kaplan, Dhariwal, Neelakantan, Shyam, Sastry, Askell, et~al.]{brown2020language}
Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared~D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et~al.
\newblock Language models are few-shot learners.
\newblock \emph{Advances in Neural Information Processing Systems}, 2020.
\bibitem[Chen(2023)]{chen2023importance}
Ting Chen.
\newblock On the importance of noise scheduling for diffusion models.
\newblock \emph{arXiv preprint arXiv:2301.10972}, 2023.
\bibitem[Chowdhery et~al.(2023)Chowdhery, Narang, Devlin, Bosma, Mishra, Roberts, Barham, Chung, Sutton, Gehrmann, et~al.]{chowdhery2023palm}
Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung~Won Chung, Charles Sutton, Sebastian Gehrmann, et~al.
\newblock Palm: Scaling language modeling with pathways.
\newblock \emph{Journal of Machine Learning Research}, 2023.
\bibitem[Croitoru et~al.(2023)Croitoru, Hondru, Ionescu, and Shah]{croitoru2023diffusionsurvey}
Florinel-Alin Croitoru, Vlad Hondru, Radu~Tudor Ionescu, and Mubarak Shah.
\newblock Diffusion models in vision: A survey.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence}, 2023.
\bibitem[Deng et~al.(2009)Deng, Dong, Socher, Li, Li, and Fei-Fei]{deng2009imagenet}
Jia Deng, Wei Dong, Richard Socher, Li-Jia Li, Kai Li, and Li~Fei-Fei.
\newblock Imagenet: A large-scale hierarchical image database.
\newblock In \emph{IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2009.
\bibitem[Dosovitskiy et~al.(2020)Dosovitskiy, Beyer, Kolesnikov, Weissenborn, Zhai, Unterthiner, Dehghani, Minderer, Heigold, Gelly, et~al.]{dosovitskiy2020image}
Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, et~al.
\newblock An image is worth 16x16 words: Transformers for image recognition at scale.
\newblock \emph{arXiv preprint arXiv:2010.11929}, 2020.
\bibitem[Dubey et~al.(2024)Dubey, Jauhri, Pandey, Kadian, Al-Dahle, Letman, Mathur, Schelten, Yang, Fan, et~al.]{dubey2024llama3}
Abhimanyu Dubey, Abhinav Jauhri, Abhinav Pandey, Abhishek Kadian, Ahmad Al-Dahle, Aiesha Letman, Akhil Mathur, Alan Schelten, Amy Yang, Angela Fan, et~al.
\newblock The llama 3 herd of models.
\newblock \emph{arXiv preprint arXiv:2407.21783}, 2024.
\bibitem[Esser et~al.(2024{\natexlab{a}})Esser, Kulal, Blattmann, Entezari, M{\"u}ller, Saini, Levi, Lorenz, Sauer, Boesel, et~al.]{esser2024scaling}
Patrick Esser, Sumith Kulal, Andreas Blattmann, Rahim Entezari, Jonas M{\"u}ller, Harry Saini, Yam Levi, Dominik Lorenz, Axel Sauer, Frederic Boesel, et~al.
\newblock Scaling rectified flow transformers for high-resolution image synthesis.
\newblock In \emph{International Conference on Machine Learning}, 2024{\natexlab{a}}.
\bibitem[Esser et~al.(2024{\natexlab{b}})Esser, Kulal, Blattmann, Entezari, M{\"u}ller, Saini, Levi, Lorenz, Sauer, Boesel, et~al.]{esser2024sd3}
Patrick Esser, Sumith Kulal, Andreas Blattmann, Rahim Entezari, Jonas M{\"u}ller, Harry Saini, Yam Levi, Dominik Lorenz, Axel Sauer, Frederic Boesel, et~al.
\newblock Scaling rectified flow transformers for high-resolution image synthesis.
\newblock In \emph{International Conference on Machine Learning}, 2024{\natexlab{b}}.
\bibitem[Gao et~al.(2024)Gao, Zhuo, Lin, Liu, Chen, Du, Xie, Luo, Qiu, Zhang, et~al.]{gao2024luminat2x}
Peng Gao, Le~Zhuo, Ziyi Lin, Chris Liu, Junsong Chen, Ruoyi Du, Enze Xie, Xu~Luo, Longtian Qiu, Yuhang Zhang, et~al.
\newblock Lumina-t2x: Transforming text into any modality, resolution, and duration via flow-based large diffusion transformers.
\newblock \emph{arXiv preprint arXiv:2405.05945}, 2024.
\bibitem[Gao et~al.(2023)Gao, Zhou, Cheng, and Yan]{gao2023masked}
Shanghua Gao, Pan Zhou, Ming-Ming Cheng, and Shuicheng Yan.
\newblock Masked diffusion transformer is a strong image synthesizer.
\newblock \emph{arXiv preprint arXiv:2303.14389}, 2023.
\bibitem[Goodfellow et~al.(2014)Goodfellow, Pouget-Abadie, Mirza, Xu, Warde-Farley, Ozair, Courville, and Bengio]{goodfellow2014gan}
Ian Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair, Aaron Courville, and Yoshua Bengio.
\newblock Generative adversarial nets.
\newblock \emph{Advances in Neural Information Processing Systems}, 2014.
\bibitem[Goodfellow et~al.(2020)Goodfellow, Pouget-Abadie, Mirza, Xu, Warde-Farley, Ozair, Courville, and Bengio]{goodfellow2020gan}
Ian Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair, Aaron Courville, and Yoshua Bengio.
\newblock Generative adversarial networks.
\newblock \emph{Communications of the ACM}, 2020.
\bibitem[Han et~al.(2022)Han, Wang, Chen, Chen, Guo, Liu, Tang, Xiao, Xu, Xu, et~al.]{han2022survey_vit}
Kai Han, Yunhe Wang, Hanting Chen, Xinghao Chen, Jianyuan Guo, Zhenhua Liu, Yehui Tang, An~Xiao, Chunjing Xu, Yixing Xu, et~al.
\newblock A survey on vision transformer.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence}, 2022.
\bibitem[Heusel et~al.(2017)Heusel, Ramsauer, Unterthiner, Nessler, and Hochreiter]{heusel2017gans}
Martin Heusel, Hubert Ramsauer, Thomas Unterthiner, Bernhard Nessler, and Sepp Hochreiter.
\newblock Gans trained by a two time-scale update rule converge to a local nash equilibrium.
\newblock \emph{Advances in Neural Information Processing Systems}, 2017.
\bibitem[Ho \& Salimans(2021)Ho and Salimans]{ho2021cfg}
J.~Ho and T~Salimans.
\newblock Classifier-free diffusion guidance.
\newblock In \emph{NeurIPS 2021 Workshop on Deep Generative Models and Downstream Applications}, 2021.
\bibitem[Ho et~al.(2020)Ho, Jain, and Abbeel]{ho2020denoising}
Jonathan Ho, Ajay Jain, and Pieter Abbeel.
\newblock Denoising diffusion probabilistic models.
\newblock \emph{Advances in Neural Information Processing Systems}, 2020.
\bibitem[Hu et~al.(2022)Hu, Wallis, Allen-Zhu, Li, Wang, Wang, Chen, et~al.]{hu2022lora}
Edward~J Hu, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu~Wang, Weizhu Chen, et~al.
\newblock Lora: Low-rank adaptation of large language models.
\newblock In \emph{International Conference on Learning Representations}, 2022.
\bibitem[Hyv{\"a}rinen \& Dayan(2005)Hyv{\"a}rinen and Dayan]{hyvarinen2005estimation}
Aapo Hyv{\"a}rinen and Peter Dayan.
\newblock Estimation of non-normalized statistical models by score matching.
\newblock \emph{Journal of Machine Learning Research}, 2005.
\bibitem[Karras et~al.(2022)Karras, Aittala, Aila, and Laine]{karras2022edm}
Tero Karras, Miika Aittala, Timo Aila, and Samuli Laine.
\newblock Elucidating the design space of diffusion-based generative models.
\newblock \emph{Advances in Neural Information Processing Systems}, 2022.
\bibitem[Kingma(2014)]{kingma2014adam}
Diederik~P Kingma.
\newblock Adam: A method for stochastic optimization.
\newblock \emph{arXiv preprint arXiv:1412.6980}, 2014.
\bibitem[Kingma \& Welling(2013)Kingma and Welling]{kingma2013auto}
Diederik~P Kingma and Max Welling.
\newblock Auto-encoding variational bayes.
\newblock \emph{arXiv preprint arXiv:1312.6114}, 2013.
\bibitem[Kynkäänniemi et~al.(2019)Kynkäänniemi, Karras, Laine, and Lehtinen]{kynk2017precisionrecall}
T.~Kynkäänniemi, T.~Karras, S.~Laine, and T~Lehtinen, J.and~Aila.
\newblock Improved precision and recall metric for assessing generative models.
\newblock \emph{Advances in Neural Information Processing Systems}, 2019.
\bibitem[Ling et~al.(2024)Ling, Lu, Luo, Bai, Behera, Jin, Pan, Jiang, and Yamagata]{ling2024diffusion}
Fenghua Ling, Zeyu Lu, Jing-Jia Luo, Lei Bai, Swadhin~K Behera, Dachao Jin, Baoxiang Pan, Huidong Jiang, and Toshio Yamagata.
\newblock Diffusion model-based probabilistic downscaling for 180-year east asian climate reconstruction.
\newblock \emph{npj Climate and Atmospheric Science}, 2024.
\bibitem[Lipman et~al.(2022)Lipman, Chen, Ben-Hamu, Nickel, and Le]{lipman2022flowmatching}
Yaron Lipman, Ricky~TQ Chen, Heli Ben-Hamu, Maximilian Nickel, and Matt Le.
\newblock Flow matching for generative modeling.
\newblock \emph{arXiv preprint arXiv:2210.02747}, 2022.
\bibitem[Liu et~al.(2023)Liu, Gong, and qiang liu]{liu2023flow}
Xingchao Liu, Chengyue Gong, and qiang liu.
\newblock Flow straight and fast: Learning to generate and transfer data with rectified flow.
\newblock In \emph{International Conference on Learning Representations}, 2023.
\bibitem[Liu et~al.(2021)Liu, Lin, Cao, Hu, Wei, Zhang, Lin, and Guo]{liu2021swin}
Ze~Liu, Yutong Lin, Yue Cao, Han Hu, Yixuan Wei, Zheng Zhang, Stephen Lin, and Baining Guo.
\newblock Swin transformer: Hierarchical vision transformer using shifted windows.
\newblock In \emph{IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2021.
\bibitem[Liu et~al.(2022)Liu, Mao, Wu, Feichtenhofer, Darrell, and Xie]{liu2022convnet}
Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, and Saining Xie.
\newblock A convnet for the 2020s.
\newblock In \emph{IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2022.
\bibitem[Loshchilov \& Hutter(2017)Loshchilov and Hutter]{loshchilov2017decoupled}
Ilya Loshchilov and Frank Hutter.
\newblock Decoupled weight decay regularization.
\newblock \emph{arXiv preprint arXiv:1711.05101}, 2017.
\bibitem[Lu et~al.(2024{\natexlab{a}})Lu, Wang, Huang, Wu, Liu, Ouyang, and BAI]{Lu2024FiT}
Zeyu Lu, ZiDong Wang, Di~Huang, Chengyue Wu, Xihui Liu, Wanli Ouyang, and LEI BAI.
\newblock Fit: Flexible vision transformer for diffusion model.
\newblock In \emph{International Conference on Machine Learning}, 2024{\natexlab{a}}.
\bibitem[Lu et~al.(2024{\natexlab{b}})Lu, Wu, Chen, Wang, Bai, Qiao, and Liu]{lu2024hierarchical}
Zeyu Lu, Chengyue Wu, Xinyuan Chen, Yaohui Wang, Lei Bai, Yu~Qiao, and Xihui Liu.
\newblock Hierarchical diffusion autoencoders and disentangled image manipulation.
\newblock In \emph{IEEE/CVF Winter Conference on Applications of Computer Vision}, 2024{\natexlab{b}}.
\bibitem[Ma et~al.(2024)Ma, Goldstein, Albergo, Boffi, Vanden-Eijnden, and Xie]{ma2024sit}
Nanye Ma, Mark Goldstein, Michael~S Albergo, Nicholas~M Boffi, Eric Vanden-Eijnden, and Saining Xie.
\newblock Sit: Exploring flow and diffusion-based generative models with scalable interpolant transformers.
\newblock \emph{arXiv preprint arXiv:2401.08740}, 2024.
\bibitem[Meng et~al.(2021)Meng, He, Song, Song, Wu, Zhu, and Ermon]{meng2021sdedit}
Chenlin Meng, Yutong He, Yang Song, Jiaming Song, Jiajun Wu, Jun-Yan Zhu, and Stefano Ermon.
\newblock Sdedit: Guided image synthesis and editing with stochastic differential equations.
\newblock \emph{arXiv preprint arXiv:2108.01073}, 2021.
\bibitem[Nash et~al.(2021)Nash, Menick, Dieleman, and Battaglia]{nash2021sfid}
C.~Nash, J.~Menick, S.~Dieleman, and P.~W Battaglia.
\newblock Generating images with sparse representations.
\newblock \emph{arXiv preprint arXiv:2103.03841}, 2021.
\bibitem[Peebles \& Xie(2023)Peebles and Xie]{peebles2023scalable}
William Peebles and Saining Xie.
\newblock Scalable diffusion models with transformers.
\newblock In \emph{IEEE/CVF International Conference on Computer Vision}, 2023.
\bibitem[Podell et~al.(2023)Podell, English, Lacey, Blattmann, Dockhorn, M{\"u}ller, Penna, and Rombach]{podell2023sdxl}
Dustin Podell, Zion English, Kyle Lacey, Andreas Blattmann, Tim Dockhorn, Jonas M{\"u}ller, Joe Penna, and Robin Rombach.
\newblock Sdxl: Improving latent diffusion models for high-resolution image synthesis.
\newblock \emph{arXiv preprint arXiv:2307.01952}, 2023.
\bibitem[Poole et~al.(2022)Poole, Jain, Barron, and Mildenhall]{poole2022dreamfusion}
Ben Poole, Ajay Jain, Jonathan~T Barron, and Ben Mildenhall.
\newblock Dreamfusion: Text-to-3d using 2d diffusion.
\newblock \emph{arXiv preprint arXiv:2209.14988}, 2022.
\bibitem[Radford et~al.(2021)Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, et~al.]{radford2021learning}
Alec Radford, Jong~Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et~al.
\newblock Learning transferable visual models from natural language supervision.
\newblock In \emph{International Conference on Machine Learning}, 2021.
\bibitem[Ramesh et~al.(2022)Ramesh, Dhariwal, Nichol, Chu, and Chen]{ramesh2022hierarchical}
Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, and Mark Chen.
\newblock Hierarchical text-conditional image generation with clip latents.
\newblock \emph{arXiv preprint arXiv:2204.06125}, 2022.
\bibitem[Ren et~al.(2024)Ren, Li, Chen, Pei, Shao, Guo, Peng, Song, and Zhu]{ren2024ultrapixel}
Jingjing Ren, Wenbo Li, Haoyu Chen, Renjing Pei, Bin Shao, Yong Guo, Long Peng, Fenglong Song, and Lei Zhu.
\newblock Ultrapixel: Advancing ultra-high-resolution image synthesis to new peaks.
\newblock \emph{arXiv preprint arXiv:2407.02158}, 2024.
\bibitem[Rombach et~al.(2022)Rombach, Blattmann, Lorenz, Esser, and Ommer]{rombach2022high}
Robin Rombach, Andreas Blattmann, Dominik Lorenz, Patrick Esser, and Bj{\"o}rn Ommer.
\newblock High-resolution image synthesis with latent diffusion models.
\newblock In \emph{IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2022.
\bibitem[Ruiz et~al.(2023)Ruiz, Li, Jampani, Pritch, Rubinstein, and Aberman]{ruiz2023dreambooth}
Nataniel Ruiz, Yuanzhen Li, Varun Jampani, Yael Pritch, Michael Rubinstein, and Kfir Aberman.
\newblock Dreambooth: Fine tuning text-to-image diffusion models for subject-driven generation.
\newblock In \emph{IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2023.
\bibitem[Saharia et~al.(2022)Saharia, Chan, Saxena, Li, Whang, Denton, Ghasemipour, Gontijo~Lopes, Karagol~Ayan, Salimans, et~al.]{saharia2022photorealistic}
Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily~L Denton, Kamyar Ghasemipour, Raphael Gontijo~Lopes, Burcu Karagol~Ayan, Tim Salimans, et~al.
\newblock Photorealistic text-to-image diffusion models with deep language understanding.
\newblock \emph{Advances in Neural Information Processing Systems}, 2022.
\bibitem[Salimans et~al.(2016)Salimans, Goodfellow, Zaremba, Cheung, Radford, and Chen]{salimans2017inceptionscore}
T.~Salimans, I.~Goodfellow, W.~Zaremba, V.~Cheung, A.~Radford, and X~Chen.
\newblock Improved techniques for training gans.
\newblock \emph{Advances in Neural Information Processing Systems}, 2016.
\bibitem[Sauer et~al.(2022)Sauer, Schwarz, and Geiger]{sauer2022stylegan}
Axel Sauer, Katja Schwarz, and Andreas Geiger.
\newblock Stylegan-xl: Scaling stylegan to large diverse datasets.
\newblock In \emph{ACM SIGGRAPH 2022 conference proceedings}, 2022.
\bibitem[Shazeer(2020)]{shazeer2020glu}
Noam Shazeer.
\newblock Glu variants improve transformer.
\newblock \emph{arXiv preprint arXiv:2002.05202}, 2020.
\bibitem[Song et~al.(2020{\natexlab{a}})Song, Meng, and Ermon]{song2020denoising}
Jiaming Song, Chenlin Meng, and Stefano Ermon.
\newblock Denoising diffusion implicit models.
\newblock \emph{arXiv preprint arXiv:2010.02502}, 2020{\natexlab{a}}.
\bibitem[Song \& Ermon(2019)Song and Ermon]{song2019ncsn}
Yang Song and Stefano Ermon.
\newblock Generative modeling by estimating gradients of the data distribution.
\newblock \emph{Advances in Neural Information Processing Systems}, 2019.
\bibitem[Song \& Ermon(2020)Song and Ermon]{song2020ncsnv2}
Yang Song and Stefano Ermon.
\newblock Improved techniques for training score-based generative models.
\newblock \emph{Advances in Neural Information Processing Systems}, 2020.
\bibitem[Song et~al.(2020{\natexlab{b}})Song, Sohl-Dickstein, Kingma, Kumar, Ermon, and Poole]{song2020score}
Yang Song, Jascha Sohl-Dickstein, Diederik~P Kingma, Abhishek Kumar, Stefano Ermon, and Ben Poole.
\newblock Score-based generative modeling through stochastic differential equations.
\newblock \emph{arXiv preprint arXiv:2011.13456}, 2020{\natexlab{b}}.
\bibitem[Su et~al.(2024)Su, Ahmed, Lu, Pan, Bo, and Liu]{su2024rope}
Jianlin Su, Murtadha Ahmed, Yu~Lu, Shengfeng Pan, Wen Bo, and Yunfeng Liu.
\newblock Roformer: Enhanced transformer with rotary position embedding.
\newblock \emph{Neurocomputing}, 2024.
\bibitem[Team et~al.(2023)Team, Anil, Borgeaud, Wu, Alayrac, Yu, Soricut, Schalkwyk, Dai, Hauth, et~al.]{team2023gemini}
Gemini Team, Rohan Anil, Sebastian Borgeaud, Yonghui Wu, Jean-Baptiste Alayrac, Jiahui Yu, Radu Soricut, Johan Schalkwyk, Andrew~M Dai, Anja Hauth, et~al.
\newblock Gemini: a family of highly capable multimodal models.
\newblock \emph{arXiv preprint arXiv:2312.11805}, 2023.
\bibitem[Touvron et~al.(2019)Touvron, Vedaldi, Douze, and J{\'e}gou]{touvron2019fixing}
Hugo Touvron, Andrea Vedaldi, Matthijs Douze, and Herv{\'e} J{\'e}gou.
\newblock Fixing the train-test resolution discrepancy.
\newblock \emph{Advances in Neural Information Processing Systems}, 2019.
\bibitem[Touvron et~al.(2021)Touvron, Cord, Douze, Massa, Sablayrolles, and J{\'e}gou]{touvron2021training}
Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, and Herv{\'e} J{\'e}gou.
\newblock Training data-efficient image transformers \& distillation through attention.
\newblock In \emph{International Conference on Machine Learning}, 2021.
\bibitem[Touvron et~al.(2023a)Touvron, Lavril, Izacard, Martinet, Lachaux, Lacroix, and et~al]{touvron2023llama}
Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, and Baptiste~Rozière et~al.
\newblock Llama: Open and efficient foundation language models.
\newblock \emph{arXiv preprint arXiv:2302.13971}, 2023a.
\bibitem[Touvron et~al.(2023b)Touvron, Martin, Stone, Albert, Almahairi, Babaei, and et~al]{touvron2023llama2}
Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, and Nikolay~Bashlykov et~al.
\newblock Llama 2: Open foundation and fine-tuned chat models.
\newblock \emph{arXiv preprint arXiv:2307.09288}, 2023b.
\bibitem[Vaswani et~al.(2017)Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez, Kaiser, and Polosukhin]{vaswani2017attention}
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan~N Gomez, {\L}ukasz Kaiser, and Illia Polosukhin.
\newblock Attention is all you need.
\newblock \emph{Advances in Neural Information Processing Systems}, 2017.
\bibitem[Wang et~al.(2024)Wang, Lu, Huang, Zhou, Ouyang, and BAI]{wang2024fitv2}
ZiDong Wang, Zeyu Lu, Di~Huang, Cai Zhou, Wanli Ouyang, and LEI BAI.
\newblock Fitv2: Scalable and improved flexible vision transformer for diffusion model.
\newblock \emph{arXiv preprint arXiv:2402.12376}, 2024.
\bibitem[Yang et~al.(2024)Yang, Jiang, Hong, Teng, Zheng, Dong, Ding, and Tang]{yang2024infdit}
Zhuoyi Yang, Heyang Jiang, Wenyi Hong, Jiayan Teng, Wendi Zheng, Yuxiao Dong, Ming Ding, and Jie Tang.
\newblock Inf-dit: Upsampling any-resolution image with memory-efficient diffusion transformer.
\newblock \emph{arXiv preprint arXiv:2405.04312}, 2024.
\end{thebibliography}