DeepSpeed-VisualChat: Multi-Round Multi-Image Interleave Chat via Multi-Modal Causal Attention
Extracted source text · arXiv 2309.14327
\documentclass{article} % For LaTeX2e
% \usepackage{iclr2023_conference,times}
\usepackage{fullpage}
\usepackage{wrapfig,lipsum,booktabs}
\input packages.tex
\input macros.tex
\input{math_notation_for_this_paper}
\usepackage{subfig}
\newcommand{\llms}{LLMs\xspace}
\newcommand{\lvlms}{LVLMs\xspace}
\newcommand{\qwen}{QWen-VL\xspace}
\newcommand{\sparkles}{SparklesChat\xspace}
\newcommand{\llava}{LLaVA\xspace}
\newcommand{\otter}{Otter\xspace}
\newcommand{\mrsi}{MrSi\xspace}
\newcommand{\mrmi}{MrMi\xspace}
\newcommand{\ca}{CA\xspace}
\newcommand{\cra}{CrA\xspace}
\newcommand{\mmca}{MMCA\xspace}
\newcommand{\llama}{LLaMA-2\xspace}
\newcommand{\ourssingle}{DeepSpeed-VisualChat-Single\xspace}
\newcommand{\ours}{DeepSpeed-VisualChat\xspace}
\usepackage{pifont}% http://ctan.org/pkg/pifont
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
\usepackage{arydshln}
\usepackage{xcolor}
\usepackage{listings}
\definecolor{myblue}{rgb}{0,0.2,0.8}
\hypersetup{ %
pdftitle={},
pdfauthor={},
pdfsubject={},
pdfkeywords={},
pdfborder=0 0 0,
pdfpagemode=UseNone,
colorlinks=true,
linkcolor=myblue,
citecolor=myblue,
filecolor=myblue,
urlcolor=myblue,
pdfview=FitH}
\makeatletter
\def\adl@drawiv#1#2#3{%
\hskip.5\tabcolsep
\xleaders#3{#2.5\@tempdimb #1{1}#2.5\@tempdimb}%
#2\z@ plus1fil minus1fil\relax
\hskip.5\tabcolsep}
\newcommand{\cdashlinelr}[1]{%
\noalign{\vskip\aboverulesep
\global\let\@dashdrawstore\adl@draw
\global\let\adl@draw\adl@drawiv}
\cdashline{#1}
\noalign{\global\let\adl@draw\@dashdrawstore
\vskip\belowrulesep}}
\makeatother
\definecolor{upforestgreen}{rgb}{0.6, 0.8, 0.2}
\newcommand {\xiaoxia}[1]{{\color{blue}\sf{[Xiaoxia: #1]}}}
\newcommand {\minjia}[1]{{\color{orange}\sf{[Minjia: #1]}}}
\newcommand {\zhewei}[1]{{\color{upforestgreen}\sf{[Zhewei: #1]}}}
\newcommand {\conglong}[1]{{\color{purple}\sf{[conglong: #1]}}}
\usepackage{chngcntr}
\usepackage{adjustbox}
\usepackage{wrapfig}
\usepackage{arydshln}
\usepackage{tcolorbox}
\def\httilde{\mbox{\tt\raisebox{-.5ex}{\symbol{126}}}}
\newcommand*\samethanks[1][\value{footnote}]{\footnotemark[#1]}
\newcommand\atsign{@}
\begin{document}
\title{\ours: Multi-Round Multi-Image Interleave Chat via Multi-Modal Causal Attention}
\author{
Zhewei Yao, Xiaoxia Wu, Conglong Li, Minjia Zhang, Heyang Qin\\
Olatunji Ruwase, Ammar Ahmad Awan, Samyam Rajbhandari, Yuxiong He
\\
\\ DeepSpeed of Microsoft
}
\date{}
\maketitle
%%%%%%%% BODY TEXT
\input{_s0_abstract.tex}
\input{_s1_intro.tex}
\input{_s2_related_work.tex}
\input{_s3_method.tex}
\input{_s4_results.tex}
\input{_s5_discussion.tex}
\input{_s6_conclusion.tex}
\section*{Contributions}
\label{sec:contribution}
\noindent \textbf{ZY}: Full engagement and project lead.
\noindent \textbf{XW}: Training/evaluation pipeline development and writing.
\noindent \textbf{CL}: Data support.
\noindent \textbf{MZ}: Training pipeline support.
\noindent \textbf{QH}: DeepSpeed ZeRO feature adaptation.
\noindent \textbf{OR}: DeepSpeed ZeRO feature adaptation.
\noindent \textbf{AAA}: Software support.
\noindent \textbf{SR}: Consulting.
\noindent \textbf{YH}: Team lead.
\section*{Acknowledgment}
We thank the entire DeepSpeed team for their support.{
% \bibliographystyle{plainnat}
% \bibliographystyle{alpha}
\bibliographystyle{plain}
\bibliography{ref.bib}
}
% \clearpage
% \onecolumn
\input _s7_appendix.tex
\end{document}
% \usepackage[square,sort&compress,comma,numbers]{natbib}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{url} % simple URL typesetting
\usepackage{booktabs} % professional-quality tables
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{soul}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{outlines}
\usepackage{xurl}
\usepackage[colorlinks=true,
citecolor=blue,
filecolor=black,
linkcolor=blue,
urlcolor=blue]{hyperref}
% Optional math commands from https://github.com/goodfeli/dlbook_notation.
\input{math_commands.tex}
\usepackage{multirow}
\usepackage{paralist}
\usepackage{multicol}
\usepackage{diagbox}
\usepackage[ruled,noend]{algorithm2e}
\newcommand{\theHalgorithm}{\arabic{algorithm}}
\newcommand\mycommfont[1]{\scriptsize\ttfamily\textcolor{blue}{#1}}
\SetCommentSty{mycommfont}
\usepackage{here}
\usepackage{amsmath,amssymb,amsfonts,amsbsy,amsfonts,latexsym}
\usepackage{makecell}
\usepackage{xcolor}
\usepackage{colortbl}
%table
\usepackage{tabularx,colortbl,xcolor}
\usepackage[normalem]{ulem}
\useunder{\uline}{\ul}{}
\newcommand\hy{\textrm{-}}
\usepackage{enumitem}
\usepackage{xparse}% http://ctan.org/pkg/xparse
% \DeclareGraphicsExtensions{.pdf,.png}
\SetKwInput{KwInput}{Input}
\SetKwInput{KwRequire}{Require}
\usepackage{longtable}
% \usepackage{pgfplots}
%%%%% NEW MATH DEFINITIONS %%%%%
\usepackage{amsmath,amsfonts,bm}
% Mark sections of captions for referring to divisions of figures
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
% Highlight a newly defined term
\newcommand{\newterm}[1]{{\bf #1}}
% Figure reference, lower-case.
\def\figref#1{figure~\ref{#1}}
% Figure reference, capital. For start of sentence
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
% Section reference, lower-case.
\def\secref#1{section~\ref{#1}}
% Section reference, capital.
\def\Secref#1{Section~\ref{#1}}
% Reference to two sections.
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
% Reference to three sections.
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
% Reference to an equation, lower-case.
\def\eqref#1{equation~\ref{#1}}
% Reference to an equation, upper case
\def\Eqref#1{Equation~\ref{#1}}
% A raw reference to an equation---avoid using if possible
\def\plaineqref#1{\ref{#1}}
% Reference to a chapter, lower-case.
\def\chapref#1{chapter~\ref{#1}}
% Reference to an equation, upper case.
\def\Chapref#1{Chapter~\ref{#1}}
% Reference to a range of chapters
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
% Reference to an algorithm, lower-case.
\def\algref#1{algorithm~\ref{#1}}
% Reference to an algorithm, upper case.
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
% Reference to a part, lower case
\def\partref#1{part~\ref{#1}}
% Reference to a part, upper case
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
% Random variables
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
% rm is already a command, just don't name any random variables m
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
% Random vectors
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
% Elements of random vectors
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
% Random matrices
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
% Elements of random matrices
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
% Vectors
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
% Elements of vectors
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
% Matrix
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
% Tensor
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
% Graph
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
% Sets
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
% Don't use a set called E, because this would be the same as our symbol
% for expectation.
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
% Entries of a matrix
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
% entries of a tensor
% Same font as tensor, without \bm wrapper
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
% The true underlying data generating distribution
\newcommand{\pdata}{p_{\rm{data}}}
% The empirical distribution defined by the training set
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
% The model distribution
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
% Stochastic autoencoder distributions
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} % Laplace distribution
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
% Wolfram Mathworld says $L^2$ is for function spaces and $\ell^2$ is for vectors
% But then they seem to use $L^2$ for vectors throughout the site, and so does
% wikipedia.
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} % See usage in notation.tex. Chosen to match Daphne's book.
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
\def\gap{\hspace*{.2in}}
\newcommand{\sketchplan}[1]{[\emph{#1}]}
\NewDocumentCommand{\var}{O{s} m O{}}{%
\ensuremath{#1_{#2}^{#3}}% add \vphantom{<bizarre sup>}
}
\usepackage{siunitx}
\newcommand\accnum[1]{\num[round-precision=3]{#1}}
\newcommand\pernum[1]{\num[round-precision=1]{#1}}
\newcommand\scnum[1]{\num[scientific-notation=true,round-precision=1]{#1}}
\newcommand\numt[1]{\num[round-precision=2,round-mode=figures,scientific-notation=true]{#1}}
\newcommand\numo[1]{\num[round-precision=1,round-mode=figures,scientific-notation=true]{#1}}
%\newcommand{\numo}[1]{
%\pgfmathparse{#1}\pgfmathprintnumberto[precision=1, assume math mode=true]{\pgfmathresult}{\roundednumber}\roundednumber
%}
%\newcommand{\numt}[1]{
%\pgfmathparse{#1}\pgfmathprintnumberto[precision=2, assume math mode=true]{\pgfmathresult}{\roundednumber}\roundednumber
%}
\newcommand{\pref}[1]{page~\pageref{#1}}
\newcommand{\vect}[1]{\boldsymbol{#1}} % vector
\newcommand{\mat}[1]{\boldsymbol{#1}} % matrix
\newcommand{\defeq}{\ensuremath{\mathrel{\mathop:}=}}
\newcommand{\eqdef}{\ensuremath{=\mathrel{\mathop:}}}
\newcommand{\ipoint}[1]{\textit{\textbf{\color{darkgray}#1}}}
\newcommand{\todo}[1]{{\color{red}todo: #1}}
%\newcommand{\comment}[1]{{\color{blue}comment: #1}}
\newcommand{\commentout}[1]{}
\renewcommand{\b}[1]{{#1}}
\newcommand{\grbf}[1]{\mbox{\boldmath${#1}$\unboldmath}} %\renewcommand{\grbf}[1] {\mathbf{#1}}
\newcommand{\ns}[1]{\ensuremath{\mathbb{#1}}}
\newcommand{\fs}[1]{\ensuremath{\mathcal{#1}}}
\newcommand{\idiv}{\ensuremath{\nabla\cdot}}
\newcommand{\igrad}{\ensuremath{\nabla}}
\newcommand{\ilap}{\rotatebox[origin=c]{180}{$\nabla$}}
\newcommand{\icurl}{\ensuremath{\nabla \times}}
\newcommand{\tr}{\ensuremath{\operatorname{tr}}}
\newcommand{\id}{\ensuremath{\operatorname{id}}}
\newcommand{\ts}{\textsuperscript}
\newcommand{\half}[1]{\frac{#1}{2}}
\renewcommand{\d}[1]{\mathop{}\!\mathrm{d}#1}
\newcommand{\dt}{\d{t}}
\newcommand{\dx}{\d{\vect{x}}}
\newcommand{\p} {\partial}
\newcommand{\op}[1]{\ensuremath{\mathcal{#1}}}
\newcommand{\di}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\MA}[1]{{\mathcal #1}}
\newcommand{\reals}{\mathbb{R}}
\newcommand{\codenm}[1]{{\tt #1}}
\newcommand{\zapspace}{\topsep=0pt\partopsep=0pt\itemsep=0pt\parskip=0pt}
\newcommand{\msection}[1]{\vspace*{-5pt}\section{#1}\vspace{-4pt}}
\newcommand{\msubsection}[1]{\vspace*{-5pt}\subsection{#1}\vspace{-4pt}}
%\newcommand{\msection}[1]{\section{#1}}
%\newcommand{\msubsection}[1]{\subsection{#1}}
\definecolor{light-gray}{gray}{0.80}
%\renewcommand{\algorithmiccomment}[1]{#1}
\newcommand{\mcol}[2]{\multicolumn{#1}{r}{#2}}
\newcommand{\mrow}[2]{\multirow{#1}{*}{#2}}
\newcommand{\mrowrot}[2]{
\parbox[t]{2mm}{\multirow{#1}{*}{\rotatebox[origin=c]{90}{#2}}}
}
%\newcommand{\algcolor}[2]{\hspace*{-\fboxsep}\colorbox{#1}{\parbox{\linewidth}{#2}}}
\newcommand{\algcolor}[2]{\colorbox{#1}{\parbox{\linewidth}{#2}}}
\newcommand{\algemph}[1]{\algcolor{light-gray}{#1}}
\newcommand{\algcmt}[1]{\hfill {\footnotesize\ttfamily\textcolor{blue}{/* {#1} */}}}
%\newcommand{\algcmt}[1]{{\footnotesize\ttfamily\textcolor{blue}{\dotfill #1}}}
\newcommand{\algcc}[1]{\hfill {\footnotesize\ttfamily\textcolor{blue}{//{#1}}}}
% \let\oldparagraph\paragraph
% \renewcommand\paragraph{\subsubsection*}
\newcommand{\bfeval}[1]{
\textcolor{black}{\textbf{\eval{#1}}}
}
\newcommand\aref{Algorithm~\ref}
\newcommand\appref{Appendix~\ref}
\newcommand\eref{Eq.~\ref}
\newcommand\fref{Figure~\ref}
\newcommand\tref{Table~\ref}
% \newcommand\sref{\textsection~\ref}
\newcommand\sref{Section~\ref}
\newcommand\cmt[1]{\tcp*[r]{\scriptsize \color{gray!80!black}#1}}
\newcommand{\red}[1]{{\color{red} #1}}
\newcommand{\blue}[1]{{\color{blue} #1}}
\newcommand{\bluem}[1]{{\bf\color{blue} #1}}
\newcommand\ha{ \rowcolor{orange!0}}
\newcommand\hb{ \rowcolor{orange!15}}
\newcommand\hc{ \rowcolor{orange!40}}
\newcommand\hd{ \rowcolor{orange!45}}
\newcommand\he{ \rowcolor{orange!60}}
\newcommand\hf{ \rowcolor{orange!75}}
\newcommand\hg{ \rowcolor{orange!90}}
\newcommand\cha{\cellcolor{orange!0}}
\newcommand\chb{\cellcolor{orange!15}}
\newcommand\chc{\cellcolor{orange!40}}
\newcommand\ga{ \rowcolor{gray!0}}
\newcommand\gb{ \rowcolor{gray!15}}
\newcommand\gc{ \rowcolor{gray!40}}
\newcommand\gd{ \rowcolor{gray!45}}
% notation simplification
\newcommand\MS{{\mathcal{M}}}
\def\x{{\bf x}}
\def\H{{\bf H}}
\def\g{{\bf g}}
\def\Hg{{\bf Hg}}
% \def\0{{\bf 0}}
\def\s{{\bf s}}
\def\t{{\bf t}}
\def\p{{\bf p}}
\def\vv{{\bf v}}
\def\R{{\mathbb R}}
\def\AS{{\mathcal{A}}}
\def\V{{\mathbb{V}}}
\let\proof\relax
\let\endproof\relax
\usepackage{amsthm}
\let\lemma\relax
\let\assumption\relax
\let\remk\relax
\let\theorem\relax
\usepackage{amsthm}
\newtheorem{mytheorem}{Theorem}
\newtheorem{assumption}[mytheorem]{Assumption}
\newtheorem{lemma}[mytheorem]{Lemma}
\newtheorem{remk}[mytheorem]{Remark}
\newcommand{\es}[1]{{\color{red} #1}}
\newcommand{\MP}{\xspace\tiny{MP}}
\newcommand{\mixsolver}{\xspace{MixSolver}}
% \newcommand{\textapprox}{\raisebox{0.5ex}{\texttildelow}}
%\newtheorem{thm}{Theorem}
%\newtheorem{lem}[thm]{Lemma}
%\newtheorem{cor}[thm]{Corollary}
%\newtheorem{rem}[thm]{Remark}
%\newtheorem{remark}[thm]{Remark}
%\newtheorem{conj}[thm]{Conjecture}
\def\wq{\rmW_q}
\def\wk{\rmW_k}
\def\wv{\rmW_v}
\def\wo{\rmW_o}
\def\wqh{\rmW_{q, h}}
\def\wkh{\rmW_{k, h}}
\def\wvh{\rmW_{v, h}}
\def\woh{\rmW_{o, h}}
\def\wfcf{\rmW_{fc1}}
\def\wfcs{\rmW_{fc2}}
\def\loss{\mathcal{L}}
\def\lossreg{\loss_{reg}}
\def\lossobj{\loss_{obj}}
\def\lambdareg{\lambda_{reg}}
\def\lambdamax{\lambda_{max}}
\def\lambdamin{\lambda_{min}}
\def\att{\text{Att}\xspace}
\def\fc{\text{FC}}
\def\topk{\text{Top}_k\xspace}
\def\sigmoid{\text{Sigmoid}\xspace}
\def\q{{\mathbf{q}}}
% \newcommand{\OURS}{random-LTD\xspace}
% \newcommand{\tokenbypass}{TokenBypass\xspace}
% \newcommand{\ltd}{LTD\xspace}
% \newcommand{\rtd}{RTD\xspace}
% \newcommand{\mha}{MHA\xspace}
% \newcommand{\ffc}{FFC\xspace}
% \newcommand{\bert}{BERT\xspace}
% \newcommand{\vit}{ViT\xspace}
% \newcommand{\gpt}{GPT\xspace}
% \newcommand{\pslg}{MSLG\xspace}
% \newcommand{\layertoken}{LayerToken\xspace}
% \newcommand{\layertokens}{LayerTokens\xspace}
% \newcommand{\layertokenlr}{LayerToken LR\xspace}
% \def\bertbase{BERT$_{\text{base}}$\xspace}
% \def\bertlarge{BERT$_{\text{large}}$\xspace}
% \newcommand{\gptm}{GPT-3$_{\text{350M}}$\xspace}
% \newcommand{\gptb}{GPT-3$_{\text{1.3B}}$\xspace}
% \newcommand{\gpthf}{GPT-2$_{\text{350M}}$\xspace}
\begin{abstract}
% Current multi-modal models are constrained by their inability to efficiently process interleaved image-and-text inputs in multi-round, multi-image conversations and are further hampered by limitations in financial resources for training, access to training data, and exploration of innovative architectures, affecting their adaptability and scalability in diverse interactive scenarios.
% We introduce a framework, Deepspeed-VisualChat, to optimize the capabilities of Large Language Models (LLMs) by integrating multi-modal functionalities. The primary objective is to enhance Large Vision and Language Models' (LVLMS) proficiency in processing interleaved image-and-text inputs. We unveil the DeepSpeed Multi Round and Multi Images Chat framework, which stands out for its open-source support for multi-round and multi-image dialogues. An innovative multi-modal causal attention mechanism is proposed, ensuring efficient attention weight computations across different modalities. Our model incorporates data blending techniques on extant datasets, ensuring seamless multi-round, multi-image conversations. Compared to contemporaneous models such as Qwen and Sparkles, our approach exhibits superior scalability and performance. This research marks a significant step in multi-modal language model advancement, laying the foundation for future explorations.
Most of the existing multi-modal models, hindered by their incapacity to adeptly manage interleaved image-and-text inputs in multi-image, multi-round dialogues, face substantial constraints in resource allocation for training and data accessibility, impacting their adaptability and scalability across varied interaction realms.
To address this, we present the \ours framework, designed to optimize Large Language Models (LLMs) by incorporating multi-modal capabilities, with a focus on enhancing the proficiency of Large Vision and Language Models in handling interleaved inputs.
Our framework is notable for (1) its open-source support for multi-round and multi-image dialogues, (2) introducing an innovative multi-modal causal attention mechanism, and (3) utilizing data blending techniques on existing datasets to assure seamless interactions in multi-round, multi-image conversations.
Compared to existing frameworks, \ours shows superior scalability up to 70B parameter language model size, representing a significant advancement in multi-modal language models and setting a solid foundation for future explorations.\footnote{Code will be released soon as a part of \url{https://github.com/microsoft/DeepSpeedExamples}}
\end{abstract}
\section{Introduction}
\label{sec:introduction}
State-of-the-art large language models (\llms) like GPT~\cite{brown2020language,OpenAI2023GPT4TR} have showcased exceptional prowess in myriad text generation and comprehension tasks, especially when subjected to zero-/few-shot learning. Once these models undergo supervised fine-tuning or reinforcement learning combined with human feedback, their proficiency in versatile interactive challenges—ranging from coding tasks~\cite{copilot} to quantitative reasoning \cite{lewkowycz2022solving}, mathematical proofs~\cite{imani-etal-2023-mathprompter,welleck2022naturalprover}, and AI chatbot interactions~\cite{chatgpt,chatllama,ouyang2022training,yao2023dschat}—becomes comparable to human experts.
% In order to break the text-only limitation of \llms, researchers and practitioners extend \llms capability from text-only modality to multiple modalities, including image, audio, and video, through feature alignment and/or model modification~\cite{girdhar2023imagebind, gong2023multimodal,zhu2023minigpt,liu2023visual,li2023otter, bai2023qwen,huang2023sparkles}.
% Among different multi-modal combinations, large vision and language models (\lvlms) attract one of the most attentions~\cite{zhu2023minigpt, liu2023visual} due to the appreciation to achieve visual-and-textual understanding capability.
Seeking to transcend the bounds of text-only processing inherent to \llms, numerous researchers have made strides in endowing these models with multi-modal capabilities. These advances span across various modalities such as images, audios, and videos, often achieved via feature alignment and model alterations~\cite{girdhar2023imagebind, gong2023multimodal,zhu2023minigpt,liu2023visual,li2023otter, bai2023qwen,huang2023sparkles}. Notably, among these multi-modal endeavors, large vision and language models (\lvlms) have garnered significant interest~\cite{zhu2023minigpt, liu2023visual}, mainly owing to their potential in facilitating comprehensive visual-textual understanding.
% However, existing frameworks and/or works mainly focus on either (1) single-image based tasks, e.g., single-image visual question answering and captioning~\cite{liu2023visual}, or (2) multiple images but need all of them to be input in the same time~\cite{li2023otter}.
% Both lack the capability of taking interleave image-and-text inputs.
% More recently, \qwen~\cite{bai2023qwen} extends \llava~\cite{liu2023visual} framework to accept interleave image-and-text inputs but it also exhibits two crucial problems: (1) the training cost of \qwen is beyond normal research labs budget; and (2) it produces unsatisfying results for multi-image cases even with the huge amount of training cost, see \zhewei{Give an comparison examples, the two cat example between us and qwen should be good enough}.
% Presently, the majority of prevailing frameworks or studies either (1) predominantly concentrate on tasks associated with single images, such as visual question answering and captioning for individual images~\cite{liu2023visual}, or (2) accommodate multiple images, albeit with a prerequisite for simultaneous input~\cite{li2023otter}. Both approaches fall short in handling interleaved image-and-text inputs. In a progressive stride, the \qwen framework~\cite{bai2023qwen} builds upon the \llava architecture~\cite{liu2023visual}, introducing capabilities to process interleaved image-and-text inputs. Nevertheless, \qwen confronts two significant impediments: (1) the financial outlay for training \qwen surpasses what most research labs can afford; and (2) it does not release its training data.
Current frameworks and studies largely focus on either (1) tasks related to individual images, like visual question answering and captioning~\cite{liu2023visual}, or (2) handling multiple images but requiring concurrent input~\cite{li2023otter}. Neither approach adeptly manages interleaved image-and-text inputs. The \qwen framework~\cite{bai2023qwen}, an extension of the \llava architecture~\cite{liu2023visual}, makes progress in this direction. However, its training costs prove prohibitive for many research labs, and it withholds its training data. In parallel, the \sparkles model~\cite{huang2023sparkles}, annotated by GPT4, relies on continuous training with MiniGPT4~\cite{zhu2023minigpt} due to its limited dataset. Both \qwen and \sparkles adopt prevailing \lvlms designs without innovative architectural exploration. Additionally, in multi-image contexts, their performance is found lacking, even with significant training investments~\footnote{We count the pretraining cost of MiniGPT4 as a part of~\sparkles.}, as shown in our comparisons~\fref{tab:compare1}.
% Another concurrent work, \sparkles~\cite{huang2023sparkles} released its synthetic, which is annotated by GPT4. However, its released model has to be continually trained using MiniGPT4~\cite{zhu2023minigpt} due to the small annotated dataset (6.5K vision-text pairs).
% Also, for both works~\cite{bai2023qwen,huang2023sparkles}, they mainly follow the main-stream of the \lvlms' design without new model architecture exploration.
% Meanwhile, in scenarios entailing multiple images, their output is subpar despite the extensive training resources invested (we count the pretraining cost of MiniGPT4 as a part of~\sparkles), as illustrated by our comparison examples see~\fref{tab:compare1}.
While larger language models typically demonstrate superior generation abilities compared to their smaller counterparts, prevailing frameworks~\cite{li2023otter,liu2023visual,zhu2023minigpt,bai2023qwen} predominantly concentrate their efforts on \llms with 7 or 13 billion parameters (e.g., \llama-7B or \llama-13B~\cite{touvron2023llama}). This focus restricts the exploration of the extensive capabilities inherent to larger \llms.
% Meanwhile, larger language models generally exhibit better generation capability than smaller ones.
% However, existing frameworks~\cite{li2023otter,liu2023visual,zhu2023minigpt,bai2023qwen} still focus their \llms in 7B (e.g., \llama-7B~\cite{touvron2023llama}) or 13B (e.g., \llama-13B) scales,
% which cannot fully investigate the exceptional prowess of \llms.
\begin{figure}
\centering
\includegraphics[width=0.83\textwidth]{figs/banner.png}
\caption{An example of \ours.
}
\label{fig:banner}
\end{figure}
To address the aforementioned challenges, we introduce the DeepSpeed multi-round and multi-image Chat framework (\ours), offering several key contributions:
\begin{itemize}
\item \textbf{Fully Open-Sourced Multi-round Multi-image Framework:} \ours, one of the pioneering fully open-sourced frameworks, enables multi-round and multi-image dialogues, accommodating interleaved text-and-image inputs, as visualized in~\fref{fig:banner}.
% supports multi-round and multi-image conversations with interleaving text-and-image inputs (as shown in~\fref{fig:banner}) , which is, to the best of our knowledge, one of the first few fully open-sourced frameworks/models that can achieve this capability.
\item \textbf{Multi-Modal Causal Attention (MMCA):} We devise a novel \mmca for multi-modal models that independently computes attention weights across various modalities. \mmca attains objectives analogous to conventional cross-attention mechanisms~\cite{li2023otter}, yet offers enhanced causal attention interpretations for generative tasks, eliminating the need for additional modules or parameters, and it presents superior training data efficiency compared to standard causal attention~\cite{zhu2023minigpt,liu2023visual}.
% Multi-Modal causal Attention Mechanism
% We introduce a new multi-modal causal attention mechanism (\mmca) for multi-modal models by separately computing attention weight across different modalities.
% \mmca achieves (1) similar goal as normal cross attention mechanism~\cite{li2023otter} but with better causal attention explanation for generation tasks and without any extra module/parameter injection, and (2) better training data efficiency as compared to pure causal attention used in~\cite{zhu2023minigpt,liu2023visual}.
\item \textbf{Data Blending for Interleaved Inputs: } To facilitate conversations with interleaved modalities, \ours employs assorted data blending techniques on existing datasets, overcoming the shortage of interleaved text-and-image inputs in most available open-sourced datasets.
% Most existing open-sourced datasets do not have sufficient interleaving text-and-image inputs.
% In order to achieve multi-round multi-image conversation with interleaving modalities inputs, \ours utilizes several different data blending techniques on existing datasets.
\item \textbf{Unprecedented Scalability:} We leverage the DeepSpeed framework~\cite{rasley2020deepspeed} to amplify our training with a 2B visual encoder from~\cite{ilharco_gabriel_2021_5143773} and a 70B language decoder from \llama~\cite{touvron2023llama}, illustrating the remarkable scalability of our framework.
% We scale up our training to with 2B visual encoder from~\cite{ilharco_gabriel_2021_5143773} and 70B language decoder from \llama~\cite{touvron2023llama} using DeepSpeed framework~\cite{rasley2020deepspeed}.
% This demonstrates the incredible scalability of our training framework.
\end{itemize}
These innovations demonstrate our commitment to progressing multi-modal conversational AI models, ensuring enhanced interoperability, attention mechanism sophistication, scalable solutions, and comprehensive dataset utilization.
\section{Related Work}
\label{sec:related_work}
Multi-modal models, especially those focusing on vision-language integrations, typically fall into two distinct categories: dual-encoder-based models~\cite{lxmert,vilbert,vl-bert,vinvl,clip,oscar,vilt,albef,simvlm,vlmo,ofa,coca,vlbeit,blip,pali,clip,align,vilclip,klite,florence,flava,liu2023learning,shen2023scaling}, and models comprising visual encoders and textual decoders~\cite{flamingo,li2023blip2,liu2023visual,li2023otter,zhu2023minigpt,gong2023multimodal,su2023pandagpt,bai2023qwen,zhao2023mmicl,awadalla2023openflamingo,huang2023sparkles}. Our work is associated with the latter, often referred to as Large Visual Language Models (\lvlms), thus our discussion predominantly revolves around \lvlms.
% Multi-modal models, particularly vision-language models, can be broadly categorized to two branches, dual-encoder based models~\cite{lxmert,vilbert,vl-bert,vinvl,clip,oscar,vilt,albef,simvlm,vlmo,ofa,coca,vlbeit,blip,pali,clip,align,vilclip,klite,florence,flava,liu2023learning,shen2023scaling} and visual encoder and textual decoder models~\cite{flamingo,li2023blip2,liu2023visual,li2023otter,zhu2023minigpt,gong2023multimodal,su2023pandagpt,bai2023qwen,awadalla2023openflamingo,huang2023sparkles}.
% Our work belongs to the second category, a.k.a., Large Visual Language Models (\lvlms).
% Therefore, our discussion will focus on \lvlms.
Most implementations of \lvlms deploy one of two architecture styles: (1) The Flamingo design~\cite{flamingo,li2023otter,awadalla2023openflamingo,zhao2023mmicl} incorporates cross-attention, introducing new parameters to \llms to interlink visual and textual elements. (2) MiniGPT4~\cite{zhu2023minigpt,liu2023visual,bai2023qwen,huang2023sparkles} employs causal-attention design, integrating visual tokens as textual embeddings. Although both designs effectively assimilate visual information and generate textual content, their advantages and drawbacks are manifold. The Flamingo design consumes significant training/inference memory and less data due to the introduction of a relatively larger set of new parameters. Conversely, the MiniGPT4 design, while less memory-intensive, is more data-dependent to effectively align visual and textual features. Consequently, an emerging query is whether a novel architecture can harmonize the introduction of fewer new parameters with data efficiency.
% Most works of \lvlms use two different model architectures: (1) Flamingo~\cite{flamingo,li2023otter,awadalla2023openflamingo} cross-attention design, where a newly introduced cross attention is added to \llms and used to connect the visual and textual information;
% and (2) MiniGPT4~\cite{zhu2023minigpt,liu2023visual,bai2023qwen,huang2023sparkles} causal-attention design, where visual tokens are projected as textual embedding and then used as normal textual features.
% Although both architectures can capture the vision information and generate textual content, the pros and cons between the two approaches can be high-levelly summarized as:
% (1) The first Flamingo design usually introduces non-trivial amount of new parameters.
% Thus it requires more training/inference memory but fewer data.
% While the MiniGPT4 design has less memory pressure, it requires more data hungry to well align the visual and textual features.
% Therefore, a natural question here is if we can find a new architecture that can benefit for both few new parameters and data efficiency.
Despite the substantial advancements achieved by existing \lvlms, certain aspects, particularly multi-round multi-image conversation involving interleaved image and text input, remain unaddressed. Works like~\cite{liu2023visual} predominantly concentrate on single image conversation, and~\cite{li2023otter} necessitate simultaneous input of all images, limiting their applicability in conventional conversational applications.
% Besides the great success of existing \lvlms, there is one common and attracting scenario that are not being well addressed.
% The multi-round multi-image conversation with interleaving image and text input.
% For instance, \cite{liu2023visual} main focuses on single image conversation, and~\cite{li2023otter} needs to feed all images at once.
% As such, they can hardly be used for the normal conversation application.
\begin{wrapfigure}{r}{0.45\textwidth}
\centering
%\vspace{-1.5cm}
\includegraphics[width=0.4\textwidth]{figs/model.png}
\caption{Model Structure. A pre-trained vision encoder encodes an image which is then projected through a linear layer to align with the hidden dimension of the text embedding layer's output. These different inputs are subsequently merged and forwarded to language models like \llama powered by our new Multi-Modal Causal Attention (MMCA) mechanism. Both the vision encoder and the language model are frozen, depicted with an icon of a blue snowflake. T-token\# (I-token\#) means Image (Text) tokens.
}
\label{fig:model}
\vspace{-0.8cm}
\end{wrapfigure}
The paucity of data pertinent to these scenarios has led researchers to explore available data to facilitate new applications. A contemporary work, \sparkles~\cite{huang2023sparkles}, exploits the GPT-4~\cite{gpt4} API to synthesize several thousands of multi-round multi-image data in a unique format. However, \sparkles does not innovate any new architectural modifications and relies on the pre-trained model, MiniGPT4~\cite{zhu2023minigpt}, thereby incurring additional training costs and complexity. Moreover, \sparkles does not exhibit any superior performance in unseen multi-image tasks compared to \ours, even without utilizing multi-image interleaved chat data. Refer to~\fref{tab:compare1} and~\ref{tab:compare3} for detailed examples.
% One of the main reasons why researchers and practitioners have begun to study this scenario is the lack of data.
% Therefore, a natural question here is if we can use existing data to enable this new application.
% A concurrent work \sparkles~\cite{huang2023sparkles} utilizes GPT-4~\cite{gpt4} API to generate a couple of thousands of multi-round multi-image data using a specific format.
% However, \sparkles does not introduce any new model architecture changes and its training inherits the pretrained model, MiniGPT4~\cite{zhu2023minigpt}, which introduces extra training cost and complex.
% Furthermore, for unseen multi-image tasks, the performance of \sparkles has no advantage over \ours even though we did not really use multi-image interleave chat data.
% Please see~\fref{tab:compare1} and~\ref{tab:compare3} for examples.
% % \zhewei{The image caption and order might be wrong. Need double check.}
\section{Method}
Our model architecture is built on the structure of MiniGPT4~\cite{zhu2023minigpt,liu2023visual}, as depicted in~\fref{fig:model}. Specifically, we maintain the entirety of the visual encoder and the whole language model, with the exception of the embedding layer, in a frozen state.
Thus, the only trainable parameters within our model are the visual feature projection layer (a linear layer) and the language model's embedding.
In total, our set of trainable parameters ranges around $O(10M)$ to $O(100M)$, primarily contingent on the size of the embedding layer.\footnote{
The linear layer's learnable parameters are calculated by multiplying the dimensions of language and vision embeddings. Meanwhile, the learnable parameters in the language model embedding layer are determined by multiplying the vocabulary size with the hidden dimension of the word embeddings.}
% Our model architecture follows MiniGPT4~\cite{zhu2023minigpt,liu2023visual} structure as illustrated in~\fref{fig:model}. Particularly, we freeze the entire visual encoder, and the entire language model except the embedding layer.
% Therefore, the only trained parameters in our model is the visual feature projection layer (a linear layer) and the embedding of the language model.
% In total, our trainable parameters is around $O(10M)$ to $O(100M)$, which mainly depends on the embedding layer size.
Diverging from the previous MiniGPT4 architecture, we substitute the conventional causal attention mechanism with our proposed multi-modal causal attention mechanism (refer to~\sref{sec:mmca}). This modification solely alters the computation of causal attention and does not incorporate any new parameters.
% Different than previous MiniGPT4 architecture, we replace the normal causal attention mechanism with our proposed multi-modal causal attention mechanism (see~\sref{sec:mmca}), which only changes the computation of causal attention but does not introduce any new parameters.
% We use an unified instruction tuning format for all of our experiments, and the template can be found in~\fref{fig:ift_template}.
% Particularly, please note that we do not use any special token as our prefix, e.g., ``\#\#\# Image i'' is not a special token and the tokenizer just decodes it as normal string.
We adopt a unified instruction tuning format for all experiments, and the template is shown in~\fref{fig:ift_template}.
It is crucial to note that we do not employ any special tokens as our prefix; for example, “\#\#\# Image i” is not a special token, and the tokenizer interprets it as a regular string.
\begin{figure}
\center{
\begin{lstlisting}
<System Instruction> % You are a powerful vision-language assistant.
### Image 1: <image> % some image, e.g., cat-1.png
### Question: <question> % please describe the image.
### Answer: <answer> % It's a cute black cat.
### Image 2: <image> % some image, e.g., cat-2.png
### Image 3: <image> % some image, e.g., cat-3.png
### Question: <question> % What's the difference between the three cats?
### Answer: <answer> % The colors of the three cats are different.
...
\end{lstlisting}
}
\caption{Here <System Instruction>, <question>, <answer> can be simply replaced by text, and <image> can be replaced by real image tokens.
The content after ``\%'' is an example.}
\label{fig:ift_template}
\end{figure}
In alignment with the recent trend of instruction fine-tuning, the final loss of our model is calculated solely based on “<answer>”, as illustrated in~\fref{fig:ift_template}. If multiple conversations are present, we compute the loss for all corresponding “<answer>” instances.
Throughout the paper, unless specifically mentioned, we employ the \llama family as our language model and utilize the extracted (and frozen) visual encoder from \qwen~\cite{bai2023qwen} as our visual encoder, which accepts 448x448 images and produces 256 image tokens per image. The rationale for opting for \qwen’s encoder over the typically utilized CLIP~\cite{clip} is elaborated in~\sref{sec:si_learning}. The sequence length for training \llama is capped at 4096. When referring to our model as \ours-xB (e.g., \ours-13B), the size refers exclusively to the language model components (\llama-13B).
% Following the recent trend of instruction finetuning, the final loss of our model is computed only on ``<answer>'' as shown in~\fref{fig:ift_template}.
% If there are multiple conversations, we compute the loss for all such ``<answer>''s.
% Throughout the paper, if there is no specific mention, we use \llama family as our language and use the extracted visual encoder (frozen) from \qwen~\cite{bai2023qwen} as our visual encoder, which accepts 448x448 images and outputs 256 image tokens per image.
% The reason to use \qwen's encoder instead of normally-used CLIP~\cite{clip} is discussed in~\sref{sec:si_learning}.
% The sequence length to train \llama is constrained to be 4096.
% When we reference to our model as \ours-xB (e.g., \ours-13B), the size solely depends on the language model parts (\llama-13B).
% \section{Methodology}
% \label{sec:methodology}
% In this section, we begin with multi-round single-image (\mrsi) conversation using our new proposed text-image causal attention in~\sref{sec:multi-round-single-image} and extend it to multi-round multi-image (\mrmi) chat using an effective data blending technique in~\sref{sec:multi-round-multi-image}.
\section{Single-Image Exploration}
\label{sec:multi-round-single-image}
\subsection{Multi-Modal Causal Attention}
\label{sec:mmca}
% There are two common attention mechanisms used to connect visual-and-textual component in a multi-modal model, i.e., causal attention used in~\cite{zhu2023minigpt,bai2023qwen} and cross attention used in~\cite{li2023otter,flamingo}.
There are two common attention mechanisms used to connect the visual and textual components in a multi-modal model: causal attention, as used in~\cite{zhu2023minigpt, bai2023qwen}, and cross attention, as used in~\cite{li2023otter, flamingo}.
\begin{figure}
\centering
\includegraphics[width=1.03\textwidth]{figs/attention.png}
% \includegraphics[width=0.33\textwidth]{figs/attention1/standard.png}
% \includegraphics[width=0.28\textwidth]{figs/attention1/cra.png}
% \includegraphics[width=0.33\textwidth]{figs/attention1/ours.png}
\caption{Different Attention Mechanisms: Examine the differing attention mechanisms using an input sentence "User: Please describe the image." coupled with three \textbf{Image} tokens (I-token1, I-token2, I-token3). On the left, we demonstrate standard causal attention~\cite{zhu2023minigpt, bai2023qwen}, treating image tokens as text ones. In the middle, we present cross attention applied to images~\cite{li2023otter, flamingo}, while maintaining standard causal attention for text tokens. On the right, we illustrate our innovative multi-modal attention proposal where image tokens only perform self-attention, and text tokens attend to text/image tokens independently, highlighted with an orange mask. This mechanism is defined by: $\text{softmax}(QK^T \odot M_1)+\text{softmax}(QK^T\odot M_2)$ with $Q$ and $K$ as query and key, the mask $M_1=[M==1]$ and the mask $M_2=[M==2]$. $M\in\mathbf{R}^{d\times d}$ is defined by the dimension $d$, the length of (image and text) tokens. In the case of the above figure, $d=10$.}
\label{fig:attention}
\end{figure}
% \zhewei{probably need a figure for all three attentions}
% \textbf{Causal Attention (\ca)}: The \ca-based method simply projects visual features (i.e., the features from the output of the final visual encoder layer) into textual features and combines them with the normal textual features after the textual embedding layer to feed into \llms. The benefit of \ca is that it's a natural extension of the original attention mechanism in \llms, and as such, it doesn't introduce any extra modules or parameters. However, this approach raises some intuitive problems: (1) For a visual token, it attends to previous visual and textual tokens, even though visual tokens are already fully encoded in a bidirectional manner and don't need further attention from other visual tokens or the beginning of textual tokens.
\textbf{Causal Attention (\ca).} The \ca-based method simply projects visual features (i.e., the features from the output of the final visual encoder layer) into textual features and combines them with the normal textual features after the textual embedding layer to feed into \llms. The benefit of \ca is that it is a natural extension of the original attention mechanism in \llms, and as such, it does not introduce any extra modules or parameters. However, this approach raises some intuitive problems:
(1) For a visual token, it attends to previous visual and textual tokens, even though visual tokens are already fully encoded in a bidirectional manner and do not need further attention from other visual tokens or the beginning of textual tokens.
(2) For a textual token, it needs to distribute its attention between its previous textual and image tokens. Due to these issues, we found that the data efficiency of \ca in \lvlms is often problematic. To address this, \llava and \qwen require visual-language pretraining to fully align visual features with textual features. We also test and compare it with our proposed \mmca in~\sref{sec:result_of_mrsi}.
% \textbf{causal attention (\ca)} \ca-based method simply projects the visual features (aka the feature from the output of the final visual encoder layer) to textual features, and combines it with the normal textual feature after textual embedding layer to feed into \llms.
% The benefit of \ca is that it is a natural extension of the original attention mechanism of \llms and as such, there is no extra module/parameter introduced.
% However, this leads to other intuitive problems:
% (1) For a visual token, it will pay its attention to previous visual and textual tokens.
% But the visual tokens are already fully encoded in a bidirectional way, which means they do not need to further attend to other visual tokens and/or begin to attend textual tokens;
% (2) For a textual token, it needs to learn how to split its attention weights to its previous textual and image tokens.
% Due to those issues, we found out that the data efficiency of \ca in \lvlms is usually problematic and in order to overcome this, \llava and \qwen need to do visual-language pretraining to fully align the visual features to textual features.
% We also test it and compare it with our proposed \mmca in~\sref{sec:result_of_mrsi}.
\textbf{Cross Attention (\cra).} The alternative, cross attention (\cra) for visual tokens, along with \ca for text tokens, exhibits better data efficiency but also comes with a few drawbacks:
(1) It introduces new parameters to the model. For example, \otter has more than 1.5 billion trained parameters compared to the millions of trained parameters in \llava. This significantly increases the training cost and memory requirements.
(2) It requires careful design if an image is introduced in the middle of a conversation during training, as previous text tokens should not be able to attend to the image.
% \textbf{Cross attention (\cra)}. The alternative, cross attention (\cra) along with \ca, shows better data efficiency but it also has a few drawbacks: (1) it has to introduce new parameters to the model. For instance, \otter has more than 1.5B trained parameters as compared to millions of trained parameters in \llava. This significantly increases the training cost and/or training memory pressure.
% (2) it needs careful design if the image is shown up in the middle of conversation during training, since previous text tokens should not be able to attend the image.
% \textbf{Multi-modal causal attention mechanism (\mmca).}To overcome those issues, we propose a new multi-modal causal attention mechanism (\mmca).
% The overall idea is as follows:
% (1) for visual tokens, it will not attend to any other tokens except itself as visual token is fully encoded by visual encoder;
% (2) for textual tokens, it will attend all its previous tokens. However, it will have two separate attention weight matrices to its previous textual tokens and image tokens.
% The intuition of the second point of \mmca is that the attention weight to one modality may affect its to another modality.
% For instance, textual token may pay more attention to textual information other than visual information.
% As such, if the attention weight matrix is normalized across both modalities, the attention score for visual tokens might be very small. See \fref{fig:attention} for the a visualization of the three attention mechanisms.
\textbf{Our proposed method: Multi-Modal Causal Attention Mechanism (\mmca).} To overcome these issues, we propose a new multi-modal causal attention mechanism (\mmca). The overall idea is as follows:
(1) For visual tokens, they only attend to themselves, as visual tokens are encoded by the visual encoder.
(2) For textual tokens, they attend to all their previous tokens. However, they have two separate attention weight matrices for their previous textual tokens and image tokens.
The intuition behind the second point of \mmca is that the attention weight for one modality may affect the other modality. For instance, a textual token may pay more attention to textual information than visual information. Therefore, if the attention weight matrix is normalized across both modalities, the attention score for visual tokens might be very small. Refer to \fref{fig:attention} for a visualization of the three attention mechanisms.
\subsection{Result}
\label{sec:result_of_mrsi}
\subsubsection{Comparison between Different Attentions}
\label{sec:comparison_between_attn}
\paragraph{Experimental Setting}
% We use \llama-7B as the language model and \qwen-visual-encoder as our visual encoder.
% We use a simple linear projection layer to connect the two models.
% We trained our model on 2 LLaVa datasets , as summarized in the first two rows in~\tref{tab:data-summary}.
% We trained all models for 5 epochs and training batch size as 128 and evaluation them mainly on single-image caption and single-image VQA.
% The peak learning rate is set to be 1e-3 for the projection linear layer and the embedding layer, and the optimizer is AdamW~\cite{loshchilov2017decoupled} with first-/second- order coefficient as $(0.0, 0.95)$.
% We split the training/validation dataset as 0.9/0.1 for the entire dataset, and we use 10\% of the entire training iteration as warm-up steps.
% We use DeepSpeed~\cite{rasley2020deepspeed} as our training framework and FP16 training to speedup the training process.
We employ the \llama-7B language model in conjunction with the \qwen-visual-encoder as our visual encoder. These two models are connected via a straightforward linear projection layer. Our model underwent training on two LLaVa datasets, as outlined in the initial two rows of~\tref{tab:data-summary}.
During training, all models were run for 5 epochs with a training batch size of 128. Our primary evaluation focused on single-image captioning and single-image Visual Question Answering (VQA). The peak learning rate was set to 1e-3 for both the projection linear layer and the embedding layer, and we employed the AdamW optimizer~\cite{loshchilov2017decoupled} with first- and second-order coefficients set to (0.0, 0.95).
For dataset splitting, we divided the training and validation datasets in a 90/10 ratio across the entire dataset. Additionally, we incorporated 10\% of the total training iterations as warm-up steps. Our training framework of choice was DeepSpeed~\cite{rasley2020deepspeed}, and we utilized FP16 training to expedite the training process.
\begin{table}[t]
\centering
\footnotesize
\vspace{-0.3cm}
\caption{Training datasets summary. Due to context length limitation, for otter\_mimicit\_sn, otter\_mimicit\_tvc, and otter\_mimicit\_vst datasets we only used the samples with $\leq$ 8 images.}\label{tab:data-summary}
\vspace{-0.3cm}
\begin{tabular}{@{}lrl@{}}
\toprule
Name & Num. samples & Description \\
\midrule
(1) llava & 49924 & The detail description and complex reasoning data used by the \llava model~\cite{liu2023visual}. \\
& & Randomly concatenate 1 to 3 samples into one sample. Details in~\sref{sec:data-blending}. \\
(2) llava\_dial & 37818 & The conversation data used by the \llava model~\cite{liu2023visual}. \\
& & Randomly concatenate 1 to 2 samples into one sample. Details in~\sref{sec:data-blending}. \\
(3) otter\_mimicit\_cgd & 70940 & The COCO (General) data used by the \otter model~\cite{li2023otter}. \\
(4) otter\_mimicit\_sd & 8006 & The SD (Surveillance) data used by the \otter model~\cite{li2023otter}. \\
(5) otter\_mimicit\_sn & 487 & The SN (Indoor Ego.) data used by the \otter model~\cite{li2023otter}. \\
(6) otter\_mimicit\_tvc & 2 & The TVC (TV) data used by the \otter model~\cite{li2023otter}. \\
(7) otter\_mimicit\_vst & 115 & The VIST (Story) data used by the \otter model~\cite{li2023otter}. \\
(8) llava\_otter\_blend & 48869 & Data blended from llava, llava\_dial, otter\_mimicit\_cgd. Details in~\sref{sec:data-blending}. \\
(9) sparkles\_dialogue & 6520 & The SparklesDialogue data used by the \sparkles model~\cite{huang2023sparkles}. \\
\midrule
Total & 222681 & \\
\bottomrule
\end{tabular}
\vspace{-0.3cm}
\end{table}
Throughout this work, we mainly compare the generation capability of different models on certain examples without comprehensively testing models on existing benchmark.
Please see more details in~\sref{sec:conclusions} for limitations of our work.
\paragraph{Demo results.} We begin by showcasing various examples that highlight the capabilities of \ours in single-image visual language conversations, employing different attention mechanisms. As demonstrated in~\fref{tab:attention-comparison-cat}, \fref{tab:attention-comparison-quirrel}, and~\fref{tab:attention-comparison-lake}, \ours, when coupled with \mmca, effectively discerns visual details in images and furnishes coherent responses to user queries.
Furthermore, \ours exhibits a more comprehensive and precise grasp of image details compared to alternative attention mechanisms, such as the use of combined masks from both causal attention and cross attention. It is also evident that, in contrast to the combination of \cra and \ca, as well as \mmca, \ca alone may exhibit slightly more errors (\fref{tab:attention-comparison-cat}) and capture a lower degree of reasoning capability (\fref{tab:attention-comparison-lake}).
% We first present several examples that demonstrate the capabilities of \ours in single-image visual language conversations using different attention mechanisms.
% As illustrated in~\fref{tab:attention-comparison-cat}, \ref{tab:attention-comparison-quirrel}, and~\ref{tab:attention-comparison-lake}, \ours with \mmca can successfully identify visual details in images and provide reasonable responses to users' queries.
% Additionally, \ours can capture details in images more comprehensively and accurately than alternative attention mechanisms such as using combined masks from both causal attention and cross attention.
% Also, it can be seen that as compared to \cra plus \ca and \mmca, \ca alone may have slightly more errors (\fref{tab:attention-comparison-cat}) and capture less reasoning capability (\fref{tab:attention-comparison-lake}).
% We observe that the use of \mmca can occasionally degrade the quality of \ourssingle's output, as shown in~t\ref{tab:attention-comparison-ironing}.
% We suspect while \mmca helps improve the multi-round multi-image generation, it may undermine the mode's capabilities in the single-image generation because \mmca requires more data to learn both causal attention and cross-attention for both single-image generation and multi-image multi-round generation.
% \minjia{@Zhewei, I chose these figures because (1) \ours shows better performance than the normal cross-attention + causal attention on these ones, (2) the response from \ours are reasonable and without noticeable mistakes, (3) they tend to fall into different categories (avoided adding too many cat images), and (4) no out-of-distribution images. There are other images that show better than the baseline, yet both ours and the baseline have mistakes, so they are not included here. }
% \paragraph{Comparison}
%% Editing the image here https://microsoft-my.sharepoint.com/:p:/p/xiaoxiawu/EZCwF8OLqKBHutxusvFJiDABj3gaG8nSHH_JQGPjhejc_g?e=csMprU
\begin{figure}
\centering
\includegraphics[width=1.0\textwidth]{figs/single/cat.png}
\caption{
Example visual and language inputs that demonstrate the output comparison between (1) the standard causal attention (\textbf{\ca}) (2) the standard causal attention combined with cross-attention (\textbf{\ca+ \cra}) and (3) the special multi-modal causal attention (\textbf{\mmca}) in \ourssingle. The LLM here is \llama-7B.
% \zhewei{@xiaoxia, the first sentence of CA, "the image is of green", what does this mean? Also, "looking at the sky" is not properly right.
% }
}\label{tab:attention-comparison-cat}
\end{figure}
\begin{figure}
\centering
\includegraphics[width=0.8\textwidth]{figs/single/squirrel.png}
\caption{
\ourssingle accurately identifies the squirrel and camera in the image, while the \ca+\cra model mistakenly includes “standing next to a tree”. The LLM here is \llama-7B.
}
\label{tab:attention-comparison-quirrel}
\end{figure}
\begin{figure}
\centering
\includegraphics[width=0.8\textwidth]{figs/single/lake.png}
\caption{
\ourssingle accurately identifies the scene as a beautiful lake and offers a set of plausible suggestions. In contrast, \ca (\ca+\cra) misinterprets the image as containing “dock with a boat ramp” (pier with the boats). The LLM here is \llama-7B. } \label{tab:attention-comparison-lake}
\end{figure}
% \begin{table}[!ht]
% \caption{
% This example illustrates that the use of \mmca can occasionally degrade the quality of \ourssingle’s output. While the integration of cross-attention and causal-attention successfully identifies the ironing man standing on a car, \mmca mistakenly interprets the man as holding a suitcase.
% }
% \label{tab:attention-comparison-ironing}
% % \small
% \huge
% \begin{adjustbox}{width=0.999\linewidth}
% \centering
% \begin{tabular}{p{4in}p{8in}}
% \toprule
% \multicolumn{2}{l}{Image and Language Inputs Example: Ironing Man} \\ \midrule
% \begin{minipage}{.95\linewidth}
% \includegraphics[width=1\linewidth]{./figs/ironing.png} \\
% \end{minipage} & {User: What is unusual about this image?} \\ \midrule
% \ca & The unusual aspect of this image is that a man is standing on top of an ironing board, which is placed on the back of a taxi. This is not a common sight, as ironing boards are typically used indoors, and placing one on top of a moving vehicle is not a typical practice. The man is also wearing a yellow shirt, which adds to the unusual nature of the scene. \\ \midrule
% \ca + \cra & The unusual aspect of this image is that a man is ironing clothes while standing on top of a car parked on a busy street. This is an unconventional way of ironing clothes, as it is typically done inside a home or a laundry room. The man is also wearing a yellow shirt, which adds to the unusual nature of the scene. \\ \midrule
% \mmca & In this image, a man is standing on a busy street holding a suitcase and a parking meter.
% While it is not uncommon to see people carrying suitcases, it is unusual to see someone holding a parking meter. This suggests that the man may be trying to retrieve his car from a parking spot, which is not a typical activity on a busy street. The presence of the parking meter adds an element of humor to the scene, as it is not typically seen in such a context. \\
% \bottomrule
% \end{tabular}
% \end{adjustbox}
% \end{table}
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \zhewei{@Minjia, please select several examples for~\tref{tab:attention-comparison} and add some text.}.
\subsubsection{Result of Scaling \ourssingle}
\paragraph{Experimental Setting}
% All settings are the same as~\sref{sec:comparison_between_attn} except we increase the language model size from \llama-7B to \llama-13B and the training epoch from 5 to 10.
All settings remain consistent with those outlined in~\sref{sec:comparison_between_attn}, with the exception of two modifications: an increase in the language model size from \llama-7B to \llama-13B and an extension of the training epoch count from 5 to 10.
\paragraph{Demo results}
% By increasing the language model sizes from 7B to 13B, we observe the model can better recognize the image and capture the reasoning between questions and images.
% Also, from the upper example in~\fref{tab:7b-13b-scupture}, besides the wrong description issue, it can be seen that \ourssingle-7B has repeating context but this problem is not frequently observed for \ourssingle-13B due to the better generation capability for larger language models.
% This observation aligns with the better exceptional prowess of larger language models.
Upon increasing the language model size from 7B to 13B, we observe a marked improvement in the model's ability to recognize images and capture the logical connections between questions and images.
Additionally, referring to the upper example in~\fref{tab:7b-13b-scupture}, aside from the issue of incorrect descriptions, it becomes apparent that \ourssingle-7B occasionally exhibits repetitive context. However, this problem is less frequent with \ourssingle-13B, owing to the enhanced generative capabilities of larger language models. This observation aligns with the superior performance typically associated with larger language models.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}
\centering
\includegraphics[width=0.8\textwidth]{figs/single/sculpture.png}
% \caption{
% This example illustrates the difference between \ourssingle-13B and \ourssingle-7B.
% }
% \label{tab:7b-13b-scupture}
% \end{figure}
% \begin{figure}
\centering
\includegraphics[width=0.8\textwidth]{figs/single/iron.png}
\caption{
The above two examples illustrate the difference between \ourssingle-13B and \ourssingle-7B.
% \zhewei{@xiaoxia, please change the name in the image as well. Make them be the same as cpaiton.}
}
\label{tab:7b-13b-scupture}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Other Learning}
\label{sec:si_learning}
Throughout the training process of \ourssingle, we accumulated several additional lessons. Note that most of these observations lack sufficient evidence and require further exploration. We present them here to assist others, but they should not be considered final conclusions.
% Through the training process of \ourssingle, we also learned a lot of other lessons.
% Most observations we made here are probably without sufficient evidence and/or lack of exploration.
% We share them here to help others instead of making the final conclusion.
\begin{itemize}
\item \textbf{Better Visual Encoder:} Commonly, the CLIP visual encoder is used in \lvlms. However, the CLIP encoder's resolution is limited to 224x224, which restricts the level of detail in the images. In our testing, we discovered that using the newly released visual encoder from \qwen significantly improves the final model quality due to its higher input resolution (448x448) and larger encoder size (2B parameters).
% CLIP visual encoder is commonly used in \lvlms.
% However, the resolution of CLIP encoder is only 224x224, which cannot provide high detailed images.
% In our testing, we find that using the newly released visual encoder from \qwen can clearly improve the final model quality as it has higher input resolution, 448x448, and larger encoder sizes (2B parameters).
\item \textbf{Overfitting or Not:} Typically, we select the best evaluation checkpoint or one close to it for final testing. However, during \ourssingle training, we found that the final checkpoint, even if it appears overfitted, often delivers better testing results compared to middle checkpoints. Does this imply that we should intentionally overfit our model? The answer is no. We experimented with 5, 10, and 20 epochs for \ourssingle-13B and observed that 10-epoch training typically yields superior final model quality.
% Normally, we pick the best evaluation checkpoint or its neighborhood for final testing.
% However, for \ourssingle training, we found out that the final over-fitted checkpoint usually provides better testing results as compred to middle checkpoint.
% Does this mean we should significantly overfit our model?
% The answer is \textbf{No}.
% We tried 5/10/20 epochs for \ourssingle-13B, and observed that 10-epochs training usually provides better final model quality.
\item \textbf{Adding LoRA to Visual Encoder or Language Decoder:} We attempted to introduce LoRA-based training to enhance model quality. However, applying LoRA to either module did not yield any significant benefits.
% We tried to incorporate LoRA-based training to improve the model quality.
% However, applying LoRA to either module did not yield any additional benefits.
\item \textbf{Lowering the Learning Rate for Pretrained Components:} We experimented with a smaller learning rate for language embedding since it is already pretrained. However, our results indicated that there is no significant difference when using a separate lower learning rate.
% We tried to use a smaller learning rate for language embedding as it is already pretrained.
% However, our results indicates that there is no obvious difference of using a separate lower learning rate.
\item \textbf{Using Chat-/Non-Chat-Based Models:} We explored both chat-based and non-chat-based LLama-2 models. Our findings suggest that when using the chat-based model, strict adherence to the chat-based model's instruction tuning format is crucial. Failing to do so resulted in even worse model quality than the non-chat-based model.
% We naively tried both chat-based and non-chat-based LLama-2 models.
% Our finding is that in order to use the chat-based model, you need to strictly follow the chat-based model's instruction tuning format.
% Otherwise, the quality of the trained model is even worse than non-chat-based model.
\item \textbf{Inserting New Special Tokens or Not:} As illustrated in~\fref{fig:ift_template}, a few tokens can be replaced by new inserted special tokens, such as encoding "\#\#\#Human: " as a new special token. However, our testing revealed that it is better not to incorporate them as special tokens. Introducing them as special tokens significantly worsened our generation performance compared to the previous approach.
% As shown in~\fref{fig:ift_template}, a few tokens can be replaced by a new inserted speicial tokens, like encoding ``\#\#\#Human: '' as a new special token.
% However, in our testing, we found out that it is better to not incorporate them as special tokens.
% After introducing them as special tokens, our generation performance is significantly worse than before.
\end{itemize}
% \begin{table}[!ht]
% \caption{
% The Qwen-VL’s visual encoder significantly enhances the model’s ability to recognize and understand images in detail. In contrast, the baseline model that utilizes OpenAI’s CLIP model struggles to discern the number of individuals and their specific characteristics. With the help of Qwen-VL’s visual encoder, the model can accurately determine that the post contains five superheroes.
% }
% \label{tab:attention-comparison-superhero}
% % \small
% \huge
% \begin{adjustbox}{width=0.999\linewidth}
% \centering
% \begin{tabular}{p{4in}p{8in}}
% \toprule
% \multicolumn{2}{l}{Image and Language Inputs Example: Super Hero} \\ \midrule
% \begin{minipage}{.95\linewidth}
% \includegraphics[width=1\linewidth]{./figs/superhero.png} \\
% \end{minipage} & {User: Introduce the movie character in this picture.} \\ \midrule
% OpenAI-Clip (ViT-large-patch14) & The movie is "The Adventures of Tintin" and the character is Tintin. \\ \midrule
% LAION-CLIP (ViT-bigG-patch14) & The image features a group of five characters from the movie "The Avengers." The characters are Captain America, Iron Man, Thor, Hulk, and Black Widow.
% The group is standing in a line, with Captain America standing in the center and Iron Man, Thor, Hulk, and Black Widow standing to his left and right respectively. The characters are wearing their iconic costumes and are posing in a dynamic manner, with their arms raised and their faces looking towards the camera.
% The image captures the teamwork and strength of the Avengers, as they stand together to face any challenge. The characters' costumes and poses also showcase their individual personalities and abilities, making them stand out in the scene. \\
% \bottomrule
% \end{tabular}
% \end{adjustbox}
% \end{table}
\section{Multi-Round Multi-Image Exploration}
\label{sec:multi-round-multi-image}
\subsection{Data Blending}
\label{sec:data-blending}
% One key missing piece to enable multi-round and multi-image conversation is data. The only multi-round multi-image data we found is the SparklesDialogue data~\cite{huang2023sparkles}, which only has 6520 samples. In order to address the issue, we used two methods to synthesize multi-round multi-image data from existing single image or single round data: simple data concatenation and \llava-\otter data blending.
One critical missing element for enabling multi-round and multi-image conversations is data. The sole source of multi-round multi-image data we located is the SparklesDialogue dataset~\cite{huang2023sparkles}, which contains a mere 6520 samples. To address this limitation, we employed two methods to synthesize multi-round multi-image data from existing single-image or single-round data: simple data concatenation and \llava-\otter data blending.
\subsubsection{Simple data concatenation}
% For the llava and llava\_dial datasets used by the \llava model, each sample includes a single/multi-round conversations for a single image. To simulate the situation where a user may ask questions on multiple images sequentially, for these two datasets we performed a simple data postprocessing: we will randomly concatenate different number of samples as a single sample. For llava we concatenate 1 to 3 samples, and for llava\_dial we concatenate 1 to 2 samples (\tref{tab:data-summary}).
For the "llava" and "llava\_dial" datasets utilized by the \llava model, each sample comprises single/multi-round conversations for a single image. To simulate scenarios where a user sequentially asks questions about multiple images, we conducted straightforward data post-processing for these two datasets. Specifically, we randomly concatenated different numbers of samples into a single sample. In the case of "llava," we concatenated 1 to 3 samples, while for "llava\_dial," we concatenated 1 to 2 samples (refer to \tref{tab:data-summary}).
\subsubsection{\llava-\otter data blending}
We noticed that the llava and llava\_dial datasets used by \llava model and the otter\_mimicit\_cgd dataset used by the \otter model all use the COCO train2017 images. For the llava and llava\_dial datasets, each sample includes a single/multi-round conversations for a single image. For the otter\_mimicit\_cgd dataset, each sample includes a single-round conversation for a pair of images. This enables us to build a synthesized multi-round multi-image data llava\_otter\_blend as a more natural blending: for each sample in the otter\_mimicit\_cgd dataset, we look for llava and llava\_dial samples that use the same image, and then build a new sample in a "llava/llava\_dial conversations then otter\_mimicit\_cgd conversation" fashion (as shown in \tref{tab:data-summary}).
\subsection{Results}
\subsubsection{Comparison with \qwen and \sparkles}
\paragraph{Experimental Setting}
% We use data (1) to (8) as shown in~\tref{tab:data-summary}.
% We do not include \sparkles's dialogue data to better explore if the new proposed data blending method can enable the multi-image multi-round interleave chat without introducing new data. We use \llama-13B as our language model and use 6 epochs to train the model. All rest settings are the same as~\sref{sec:result_of_mrsi}.
We utilize the datasets (1) to (8) as illustrated in \tref{tab:data-summary}. We deliberately exclude the dialogue data from \sparkles to thoroughly investigate whether our newly proposed data blending technique can facilitate the interweaving of multi-image multi-round chats without the incorporation of new data. \llama-13B is employed as our language model, and the model is trained over 6 epochs. All other settings remain consistent with those outlined in \sref{sec:result_of_mrsi}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[H]
\centering
\includegraphics[width=1.0\textwidth]{figs/compare-qwen-sparkles/two.png}
\caption{
The above example (the conversation is from left to right panel) illustrates the difference among \ours-13B, \qwen, and \sparkles.
\qwen provides considerable short and accurate answers on describing the individual image but fails to remember the first image at the penultimate question.
While \sparkles is good at interpreting the difference between images but provides inaccurate information when describing individual images.
}
\label{tab:compare1}
\end{figure}
\begin{figure}
\centering
\includegraphics[width=1.02\textwidth]{figs/compare-qwen-sparkles/zootopiaCompare.png}
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/a.pdf}
% % \centering
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/b.pdf}
\caption{
The given example, displayed in a left-to-right panel conversation format, demonstrates the disparities among \ours-13B, \qwen, and \sparkles.
\qwen excels in delivering succinct and accurate responses when describing individual images. Conversely, \sparkles elaborates on the first two images but inaccurately identifies the scene as being from "Winnie" although later correctly attributing it to "Zootopia".
When it comes to narrative skills, both \qwen and \sparkles exhibit proficiency in story-telling, a skill that our model lacks.
This deficiency can be attributed to the absence of narrative-centric content within the training data utilized for this model.
In~\fref{tab:compare}, we will compare the model trained with and without narrative-centric content.}
\label{tab:compare2}
\end{figure}
% \paragraph{Demo Results}
% We compare \ours-13B with \qwen and \sparkles in~\fref{tab:compare1} and~\ref{tab:compare2}.
% For~\fref{tab:compare1}, it is an unseen task for all trained models.
% As we can see, \ours-13B achieves better answer quality as compared to the other models.
% Particularly, \qwen provides considerable short and accurate answers on describing the individual image but fails to remember the first/second images at later
% questions.
% While \sparkles is good at interpreting the difference between images but provides inaccurate
% information when describing individual images.
% For~\fref{tab:compare2}, it is a narrative task that is the main training of \sparkles and probably included in \qwen's training data (the data is close source), but not included in the training of \ours (aka data (1) to (8) in~\tref{tab:data-summary}).
% \ours still exhibits good single image description and some capability of narrative.
% Meanwhile, \qwen and \sparkles show better narrative power.
% However, all models show flaws for certain question.
% Please note that the training cost of \ours is much less than that of \qwen and \sparkles (\sparkles used the pre-training checkpoint of MiniGPT4)
\paragraph{Demo Results}
We compare \ours-13B with \qwen and \sparkles as illustrated in \fref{tab:compare1} and \fref{tab:compare2}. The tasks presented in \fref{tab:compare1} are unseen to all the trained models. Notably, \ours-13B outperforms in terms of answer quality when compared to the other models. Specifically, while \qwen excels at offering succinct and accurate descriptions of individual images, it struggles to recall the first or second images during subsequent questions. On the other hand, \sparkles excels at discerning differences between images, yet occasionally provides imprecise descriptions of individual images.
The tasks in \fref{tab:compare2} center around narratives. Narratives are the primary training focus of \sparkles and might be a part of \qwen's training data (as its data is proprietary), but they were not part of \ours's training (i.e., datasets (1) to (8) as mentioned in \tref{tab:data-summary}). Despite this, \ours continues to provide commendable descriptions of individual images and exhibits some narrative skills. In contrast, both \qwen and \sparkles demonstrate superior narrative abilities. Nonetheless, each model has its own set of limitations for specific questions.
It is worth noting that the training expenditure for \ours is significantly lower than that for \qwen and \sparkles, with the latter having utilized the pre-training checkpoint of MiniGPT4.
\subsubsection{Result of \ours}
\label{sec:small_scale}
\begin{figure}
\centering
\includegraphics[width=1.0\textwidth]{figs/compare-qwen-sparkles/zootopia.png}
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/a.pdf}
% % \centering
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/b.pdf}
\caption{
The above example (the conversation is from left to right panel) illustrates the differences between \ours-13B-Set1 and \ours-13b-Set2. We see that Set1 is better at describing the individual images (for example, Set1 can recognize the fox and "next window") but does not have the ability to tell a story.
While Set2 lost some ability to focus on the details of the images but can tell a story based on the given images.
}
\label{tab:compare}
\end{figure}
\paragraph{Experimental Setting}
The setting remains the same as mentioned above, with the addition of 6.5K more examples from \sparkles.
\paragraph{Demo result}
% We compare \ours-13B with and without data from \sparkles.
% For simplicity, we refer \ours-13B without (with) \sparkles's data as \ours-13B-Set1 (-Set2).
% First of all, by incorporating \sparkles's data, \ours exhibits better narrative capability.
% Similar to \sparkles, the new trained model also lost some ability to focus on the details of individual images.
% Besides above, we find another interesting outcome by introducing more data.
% \ours-13B-Set2 becomes more sensitive to prompt tuning as compared to the previous version.
% Particularly, in~\fref{tab:compare5}, by slightly changing the prompt (highlighted in \red{red} text) without chaning the meaning of the question, \ours-13B-Set2 provides different answers.
% However, the original \ours-13B-Set1 gives more or less similar answers.
% We conjecture that the increasing of sensitivity is from some unbalance question format/template introduced by \sparkles's dataset.
% Please also see~\fref{tab:compare3} and~\ref{tab:compare4} for cross-comparison between \ours and \qwen/\sparkles.
We perform comparisons between \ours-13B with and without incorporating data from \sparkles. For clarity, we will refer to \ours-13B without (and with) \sparkles's data as \ours-13B-Set1 (-Set2). First and foremost, by integrating \sparkles's data, \ours demonstrates enhanced narrative capability. Similar to \sparkles, the newly trained model also displays a reduced ability to concentrate on the details of individual images.
Beyond the aforementioned, the introduction of additional data yields another intriguing observation. \ours-13B-Set2 exhibits increased sensitivity to prompt tuning compared to its predecessor. Specifically, as shown in \fref{tab:compare5}, a slight alteration to the prompt (highlighted in \red{red} text) without changing the question's meaning, leads \ours-13B-Set2 to provide disparate answers. Conversely, the original \ours-13B-Set1 tends to offer more congruent responses. We hypothesize that this heightened sensitivity results from an imbalance in question formats/templates introduced by \sparkles's dataset.
For cross-comparison between \ours and \qwen/\sparkles, please refer to \fref{tab:compare3} and \fref{tab:compare4}.
% \subsubsection{\ours with \llama-70B}
% We successfully launch the training with \llama-70B using the same training setting as~\sref{sec:small_scale}.
% However, the final model is not well trained.
% We suspect the hyper-parameters used for \llama-13B is not optimal for \llama-70B, e.g., the learning rate is too large and the training epoch is not sufficient.
% We leave it as the future work to well train \ours-70B.
\subsubsection{\ours with \llama-70B}
We have initiated training with \llama-70B, maintaining the same training settings as outlined in~\sref{sec:small_scale}. However, the resulting model is not adequately trained. We conjecture that the hyper-parameters optimal for \llama-13B may not be suitable for \llama-70B; for instance, the learning rate might be excessive, and the number of training epochs insufficient. Perfecting the training of \ours-70B is earmarked for future work.
\subsubsection{Other Learning}
% \begin{itemize}
% % \item Adding LoRA to language part causes very bad hallucination issues \zhewei{Give a few examples}
% \item \textbf{Projection layer exploration:} we tried two different projection layers used to connect visual encoders and \llms, i.e., the single linear layer and a Vision Transformer layer.
% We did not find any advantage of the Vision Transformer appoarch in the initial phase, so we did not further explore this.
% \item \textbf{More complicated data blending:} We tried even more compliated data blending methods, e.g., shuffling the image ID of \otter and \llava dataset. For instance, the paired images in \otter are latter referenced as to the first and third image by introducing another image as the second image.
% However, our results showed degradation performance with incomplete sentence and wrong reference.
% We investigate the data and conjecture it is probably caused by wrong reference in the training data when we do the data blending.
% \end{itemize}
\begin{itemize}
% \item Incorporating LoRA into the language component resulted in severe hallucination issues \zhewei{Provide a few instances}.
\item \textbf{Exploration of Projection Layers:} We experimented with two different projection layers to bridge visual encoders and \llms: a single linear layer and a Vision Transformer layer. We did not observe any benefits from the Vision Transformer approach in the preliminary phase, so we decided not to pursue this route further.
\item \textbf{Advanced Data Blending Techniques:} We explored more intricate data blending methods, such as shuffling the image ID of the \otter and \llava datasets. For example, in the \otter dataset, the paired images were later referenced as the first and third images by inserting another image as the second one. However, our experiments led to deteriorated performance, characterized by incomplete sentences and incorrect references. Upon reviewing the data, we hypothesized that these issues were probably due to incorrect references in the training data during the data blending process.
\end{itemize}
\begin{figure}
\centering
\includegraphics[width=1.0\textwidth]{figs/compare-qwen-sparkles/robustness.png}
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/a.pdf}
% % \centering
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/b.pdf}
\caption{
The given example, displayed in a left-to-right panel conversation format, demonstrates the disparities between \ours-13B-Set1 and \ours-13B-Set2, given a slightly different prompt series.}
\label{tab:compare5}
\end{figure}
% \section{\ours with \llama-70B}
% \label{sec:discussion}
% \section{Limitations and Conclusions}
% \label{sec:conclusions}
% \paragraph{Limitations}
% Since the benchmarking evaluation is not the focus of this work, we did not include any such benchmarking result.
% This probably leads to the demos illustrated in the paper bias and not comprehensive.
% Also, we found that the data is the key component to get a high quality \lvlms but we cannot provide such datasets due to resource constraint.
% We notice larger language model can provide better model quality but failed to train a \llama-70B based model.
% We tried to train with the \llama-70B model but the final training result is still optimal.
% We also observed the hallucination issue of \ours since it inherits from \llama family.
% \paragraph{Conclusions}
% % In this work, we propose \ours, which \zhewei{Probably use GPT-4 to make a conclusion.}
% In conclusion, We propose \ours, the Multi Round and Multi Images Chat framework, a pioneering solution offering open-source support for multi-image dialogues and featuring an innovative multi-modal causal attention mechanism and efficient data blending techniques. Our approach surpasses contemporaneous models in scalability enabling enhanced adaptability in diverse interactive scenarios, without incurring additional training costs or complexity. This breakthrough facilitates unprecedented advancements in large vision and language models, laying a robust foundation for the seamless integration of visual and textual information in future multi-modal models.
\section{Limitations and Conclusions}
\label{sec:conclusions}
\paragraph{Limitations}
Given that the focus of this work is not on benchmarking evaluations, we abstained from incorporating any such results. This might have resulted in the demonstrations illustrated in the paper appearing biased and not being comprehensive. Additionally, we have observed that data is a pivotal component to achieve high-quality \lvlms, but we were unable to provide such datasets due to constraints on resources. We acknowledge that larger language models can potentially offer superior model quality, but we encountered difficulties in training a model based on \llama-70B. We successfully trained the \llama-70B model, but the end results were suboptimal. We also noted the hallucination issue with \ours, a problem inherited from the \llama family.
\paragraph{Conclusions}
In conclusion, we propose \ours, the multi-round and multi-image chat framework, a pioneering solution offering open-source support for multi-image dialogues and featuring an innovative multi-modal causal attention mechanism and efficient data blending techniques. Our approach surpasses contemporaneous models in scalability enabling enhanced adaptability in diverse interactive scenarios, without incurring additional training costs or complexity. This breakthrough facilitates unprecedented advancements in large vision and language models, laying a robust foundation for the seamless integration of visual and textual information in future multi-modal models.
\appendix
%%%%%%%%%%%%%%%%%%%
% Re-count the Figure/Algorithm/Tables after this point.
%%%%%%%%%%%%%%%%%%%
\counterwithin{figure}{section}
\counterwithin{table}{section}
\section{More examples}
\begin{figure}
\centering
\includegraphics[width=1.0\textwidth]{figs/compare-qwen-sparkles/counting.png}
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/a.pdf}
% % \centering
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/b.pdf}
\caption{
The given example, displayed in a left-to-right panel conversation format, demonstrates the disparities among \ours-13B-Set1 and \ours-13B-Set2, \qwen, and \sparkles.}
\label{tab:compare3}
\end{figure}
\begin{figure}
\centering
\includegraphics[width=1.0\textwidth]{figs/compare-qwen-sparkles/ceos.png}
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/a.pdf}
% % \centering
% \includegraphics[width=0.51\textwidth]{figs/compare-qwen-sparkles/b.pdf}
\caption{
The given example, displayed in a left-to-right panel conversation format, demonstrates the disparities among \ours-13B-Set1 and \ours-13B-Set2, \qwen, and \sparkles.}
\label{tab:compare4}
\end{figure}
\begin{thebibliography}{10}
\bibitem{gpt4}
Open AI.
\newblock Gpt-4 technical report.
\newblock {\em arXiv preprint arXiv:2303.08774}, 2023.
\bibitem{flamingo}
Jean{-}Baptiste Alayrac, Jeff Donahue, Pauline Luc, Antoine Miech, Iain Barr,
Yana Hasson, Karel Lenc, Arthur Mensch, Katie Millican, Malcolm Reynolds,
Roman Ring, Eliza Rutherford, Serkan Cabi, Tengda Han, Zhitao Gong, Sina
Samangooei, Marianne Monteiro, Jacob Menick, Sebastian Borgeaud, Andrew
Brock, Aida Nematzadeh, Sahand Sharifzadeh, Mikolaj Binkowski, Ricardo
Barreira, Oriol Vinyals, Andrew Zisserman, and Karen Simonyan.
\newblock Flamingo: a visual language model for few-shot learning.
\newblock {\em CoRR}, abs/2204.14198, 2022.
\bibitem{chatllama}
ChatLLaMa Authors.
\newblock Chatllama.
\newblock \url{https://github.com/juncongmoo/chatllama}, 2023.
\bibitem{awadalla2023openflamingo}
Anas Awadalla, Irena Gao, Josh Gardner, Jack Hessel, Yusuf Hanafy, Wanrong Zhu,
Kalyani Marathe, Yonatan Bitton, Samir Gadre, Shiori Sagawa, et~al.
\newblock Openflamingo: An open-source framework for training large
autoregressive vision-language models.
\newblock {\em arXiv preprint arXiv:2308.01390}, 2023.
\bibitem{bai2023qwen}
Jinze Bai, Shuai Bai, Shusheng Yang, Shijie Wang, Sinan Tan, Peng Wang, Junyang
Lin, Chang Zhou, and Jingren Zhou.
\newblock Qwen-vl: A frontier large vision-language model with versatile
abilities.
\newblock {\em arXiv preprint arXiv:2308.12966}, 2023.
\bibitem{vlmo}
Hangbo Bao, Wenhui Wang, Li~Dong, Qiang Liu, Owais~Khan Mohammed, Kriti
Aggarwal, Subhojit Som, Songhao Piao, and Furu Wei.
\newblock Vlmo: Unified vision-language pre-training with
mixture-of-modality-experts.
\newblock In {\em Advances in Neural Information Processing Systems}, 2022.
\bibitem{brown2020language}
Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared~D Kaplan, Prafulla
Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell,
et~al.
\newblock Language models are few-shot learners.
\newblock {\em Advances in neural information processing systems},
33:1877--1901, 2020.
\bibitem{pali}
Xi~Chen, Xiao Wang, Soravit Changpinyo, AJ~Piergiovanni, Piotr Padlewski,
Daniel Salz, Sebastian Goodman, Adam Grycner, Basil Mustafa, Lucas Beyer,
et~al.
\newblock Pali: A jointly-scaled multilingual language-image model.
\newblock {\em arXiv preprint arXiv:2209.06794}, 2022.
\bibitem{girdhar2023imagebind}
Rohit Girdhar, Alaaeldin El-Nouby, Zhuang Liu, Mannat Singh, Kalyan~Vasudev
Alwala, Armand Joulin, and Ishan Misra.
\newblock Imagebind: One embedding space to bind them all.
\newblock In {\em CVPR}, 2023.
\bibitem{copilot}
GitHub.
\newblock Github copilot.
\newblock \url{https://github.com/features/copilot/}, 2021.
\bibitem{gong2023multimodal}
Tao Gong, Chengqi Lyu, Shilong Zhang, Yudong Wang, Miao Zheng, Qian Zhao,
Kuikun Liu, Wenwei Zhang, Ping Luo, and Kai Chen.
\newblock Multimodal-gpt: A vision and language model for dialogue with humans.
\newblock {\em arXiv preprint arXiv:2305.04790}, 2023.
\bibitem{huang2023sparkles}
Yupan Huang, Zaiqiao Meng, Fangyu Liu, Yixuan Su, Nigel Collier, and Yutong Lu.
\newblock Sparkles: Unlocking chats across multiple images for multimodal
instruction-following models.
\newblock {\em arXiv preprint arXiv:2308.16463}, 2023.
\bibitem{ilharco_gabriel_2021_5143773}
Gabriel Ilharco, Mitchell Wortsman, Ross Wightman, Cade Gordon, Nicholas
Carlini, Rohan Taori, Achal Dave, Vaishaal Shankar, Hongseok Namkoong, John
Miller, Hannaneh Hajishirzi, Ali Farhadi, and Ludwig Schmidt.
\newblock Openclip, July 2021.
\newblock If you use this software, please cite it as below.
\bibitem{imani-etal-2023-mathprompter}
Shima Imani, Liang Du, and Harsh Shrivastava.
\newblock {M}ath{P}rompter: Mathematical reasoning using large language models.
\newblock In {\em Proceedings of the 61st Annual Meeting of the Association for
Computational Linguistics (Volume 5: Industry Track)}, pages 37--42, Toronto,
Canada, July 2023. Association for Computational Linguistics.
\bibitem{align}
Chao Jia, Yinfei Yang, Ye~Xia, Yi{-}Ting Chen, Zarana Parekh, Hieu Pham,
Quoc~V. Le, Yun{-}Hsuan Sung, Zhen Li, and Tom Duerig.
\newblock Scaling up visual and vision-language representation learning with
noisy text supervision.
\newblock In Marina Meila and Tong Zhang, editors, {\em Proceedings of the 38th
International Conference on Machine Learning, {ICML} 2021, 18-24 July 2021,
Virtual Event}, volume 139 of {\em Proceedings of Machine Learning Research},
pages 4904--4916. {PMLR}, 2021.
\bibitem{vilt}
Wonjae Kim, Bokyung Son, and Ildoo Kim.
\newblock {ViLT}: Vision-and-language transformer without convolution or region
supervision.
\newblock In Marina Meila and Tong Zhang, editors, {\em Proceedings of the 38th
International Conference on Machine Learning, {ICML} 2021, 18-24 July 2021,
Virtual Event}, volume 139 of {\em Proceedings of Machine Learning Research},
pages 5583--5594. {PMLR}, 2021.
\bibitem{lewkowycz2022solving}
Aitor Lewkowycz, Anders Andreassen, David Dohan, Ethan Dyer, Henryk
Michalewski, Vinay Ramasesh, Ambrose Slone, Cem Anil, Imanol Schlag, Theo
Gutman-Solo, et~al.
\newblock Solving quantitative reasoning problems with language models.
\newblock {\em Advances in Neural Information Processing Systems},
35:3843--3857, 2022.
\bibitem{li2023otter}
Bo~Li, Yuanhan Zhang, Liangyu Chen, Jinghao Wang, Fanyi Pu, Jingkang Yang,
Chunyuan Li, and Ziwei Liu.
\newblock Mimic-it: Multi-modal in-context instruction tuning.
\newblock {\em arXiv preprint arXiv:2306.05425}, 2023.
\bibitem{li2023blip2}
Junnan Li, Dongxu Li, Silvio Savarese, and Steven Hoi.
\newblock Blip-2: Bootstrapping language-image pre-training with frozen image
encoders and large language models.
\newblock {\em arXiv preprint arXiv:2301.12597}, 2023.
\bibitem{blip}
Junnan Li, Dongxu Li, Caiming Xiong, and Steven Hoi.
\newblock Blip: Bootstrapping language-image pre-training for unified
vision-language understanding and generation.
\newblock In {\em ICML}, pages 12888--12900. PMLR, 2022.
\bibitem{albef}
Junnan Li, Ramprasaath Selvaraju, Akhilesh Gotmare, Shafiq Joty, Caiming Xiong,
and Steven Chu~Hong Hoi.
\newblock Align before fuse: Vision and language representation learning with
momentum distillation.
\newblock In {\em Advances in neural information processing systems},
volume~34, pages 9694--9705, 2021.
\bibitem{oscar}
Xiujun Li, Xi~Yin, Chunyuan Li, Pengchuan Zhang, Xiaowei Hu, Lei Zhang, Lijuan
Wang, Houdong Hu, Li~Dong, Furu Wei, Yejin Choi, and Jianfeng Gao.
\newblock Oscar: Object-semantics aligned pre-training for vision-language
tasks.
\newblock In Andrea Vedaldi, Horst Bischof, Thomas Brox, and Jan{-}Michael
Frahm, editors, {\em Computer Vision - {ECCV} 2020 - 16th European
Conference, Glasgow, UK, August 23-28, 2020, Proceedings, Part {XXX}}, volume
12375 of {\em Lecture Notes in Computer Science}, pages 121--137. Springer,
2020.
\bibitem{liu2023visual}
Haotian Liu, Chunyuan Li, Qingyang Wu, and Yong~Jae Lee.
\newblock Visual instruction tuning.
\newblock {\em arXiv preprint arXiv:2304.08485}, 2023.
\bibitem{liu2023learning}
Haotian Liu, Kilho Son, Jianwei Yang, Ce~Liu, Jianfeng Gao, Yong~Jae Lee, and
Chunyuan Li.
\newblock Learning customized visual models with retrieval-augmented knowledge.
\newblock {\em CVPR}, 2023.
\bibitem{loshchilov2017decoupled}
Ilya Loshchilov and Frank Hutter.
\newblock Decoupled weight decay regularization.
\newblock {\em arXiv preprint arXiv:1711.05101}, 2017.
\bibitem{vilbert}
Jiasen Lu, Dhruv Batra, Devi Parikh, and Stefan Lee.
\newblock {ViLBERT}: Pretraining task-agnostic visiolinguistic representations
for vision-and-language tasks.
\newblock In {\em Advances in Neural Information Processing Systems 32: Annual
Conference on Neural Information Processing Systems 2019, NeurIPS 2019,
December 8-14, 2019, Vancouver, BC, Canada}, pages 13--23, 2019.
\bibitem{chatgpt}
OpenAI.
\newblock Chatgpt.
\newblock \url{https://openai.com/blog/chatgpt}, 2022.
\bibitem{OpenAI2023GPT4TR}
OpenAI.
\newblock Gpt-4 technical report.
\newblock {\em ArXiv}, abs/2303.08774, 2023.
\bibitem{ouyang2022training}
Long Ouyang, Jeffrey Wu, Xu~Jiang, Diogo Almeida, Carroll Wainwright, Pamela
Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, et~al.
\newblock Training language models to follow instructions with human feedback.
\newblock {\em Advances in Neural Information Processing Systems},
35:27730--27744, 2022.
\bibitem{clip}
Alec Radford, Jong~Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh,
Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark,
Gretchen Krueger, and Ilya Sutskever.
\newblock Learning transferable visual models from natural language
supervision.
\newblock In Marina Meila and Tong Zhang, editors, {\em Proceedings of the 38th
International Conference on Machine Learning, {ICML} 2021, 18-24 July 2021,
Virtual Event}, volume 139 of {\em Proceedings of Machine Learning Research},
pages 8748--8763. {PMLR}, 2021.
\bibitem{rasley2020deepspeed}
Jeff Rasley, Samyam Rajbhandari, Olatunji Ruwase, and Yuxiong He.
\newblock Deepspeed: System optimizations enable training deep learning models
with over 100 billion parameters.
\newblock In {\em Proceedings of the 26th ACM SIGKDD International Conference
on Knowledge Discovery \& Data Mining}, pages 3505--3506, 2020.
\bibitem{klite}
Sheng Shen, Chunyuan Li, Xiaowei Hu, Yujia Xie, Jianwei Yang, Pengchuan Zhang,
Zhe Gan, Lijuan Wang, Lu~Yuan, Ce~Liu, et~al.
\newblock K-lite: Learning transferable visual models with external knowledge.
\newblock In {\em Advances in Neural Information Processing Systems}, 2022.
\bibitem{vilclip}
Sheng Shen, Liunian~Harold Li, Hao Tan, Mohit Bansal, Anna Rohrbach, Kai-Wei
Chang, Zhewei Yao, and Kurt Keutzer.
\newblock How much can clip benefit vision-and-language tasks?
\newblock In {\em ICLR}, 2022.
\bibitem{shen2023scaling}
Sheng Shen, Zhewei Yao, Chunyuan Li, Trevor Darrell, Kurt Keutzer, and Yuxiong
He.
\newblock Scaling vision-language models with sparse mixture of experts.
\newblock {\em arXiv preprint arXiv:2303.07226}, 2023.
\bibitem{flava}
Amanpreet Singh, Ronghang Hu, Vedanuj Goswami, Guillaume Couairon, Wojciech
Galuba, Marcus Rohrbach, and Douwe Kiela.
\newblock {FLAVA:} {A} foundational language and vision alignment model.
\newblock {\em CoRR}, abs/2112.04482, 2021.
\bibitem{vl-bert}
Weijie Su, Xizhou Zhu, Yue Cao, Bin Li, Lewei Lu, Furu Wei, and Jifeng Dai.
\newblock {VL-BERT:} pre-training of generic visual-linguistic representations.
\newblock In {\em 8th International Conference on Learning Representations,
{ICLR} 2020, Addis Ababa, Ethiopia, April 26-30, 2020}. OpenReview.net, 2020.
\bibitem{su2023pandagpt}
Yixuan Su, Tian Lan, Huayang Li, Jialu Xu, Yan Wang, and Deng Cai.
\newblock Pandagpt: One model to instruction-follow them all.
\newblock {\em arXiv preprint arXiv:2305.16355}, 2023.
\bibitem{lxmert}
Hao Tan and Mohit Bansal.
\newblock {LXMERT}: Learning cross-modality encoder representations from
transformers.
\newblock In Kentaro Inui, Jing Jiang, Vincent Ng, and Xiaojun Wan, editors,
{\em Proceedings of the 2019 Conference on Empirical Methods in Natural
Language Processing and the 9th International Joint Conference on Natural
Language Processing, {EMNLP-IJCNLP} 2019, Hong Kong, China, November 3-7,
2019}, pages 5099--5110. Association for Computational Linguistics, 2019.
\bibitem{touvron2023llama}
Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine
Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale,
et~al.
\newblock Llama 2: Open foundation and fine-tuned chat models.
\newblock {\em arXiv preprint arXiv:2307.09288}, 2023.
\bibitem{ofa}
Peng Wang, An~Yang, Rui Men, Junyang Lin, Shuai Bai, Zhikang Li, Jianxin Ma,
Chang Zhou, Jingren Zhou, and Hongxia Yang.
\newblock Unifying architectures, tasks, and modalities through a simple
sequence-to-sequence learning framework.
\newblock {\em CoRR}, abs/2202.03052, 2022.
\bibitem{vlbeit}
Wenhui Wang, Hangbo Bao, Li~Dong, Johan Bjorck, Zhiliang Peng, Qiang Liu, Kriti
Aggarwal, Owais~Khan Mohammed, Saksham Singhal, Subhojit Som, et~al.
\newblock Image as a foreign language: Beit pretraining for all vision and
vision-language tasks.
\newblock {\em arXiv preprint arXiv:2208.10442}, 2022.
\bibitem{simvlm}
Zirui Wang, Jiahui Yu, Adams~Wei Yu, Zihang Dai, Yulia Tsvetkov, and Yuan Cao.
\newblock {SimVLM}: Simple visual language model pretraining with weak
supervision.
\newblock In {\em ICLR}, 2022.
\bibitem{welleck2022naturalprover}
Sean Welleck, Jiacheng Liu, Ximing Lu, Hannaneh Hajishirzi, and Yejin Choi.
\newblock Naturalprover: Grounded mathematical proof generation with language
models.
\newblock {\em Advances in Neural Information Processing Systems},
35:4913--4927, 2022.
\bibitem{yao2023dschat}
Zhewei Yao, Reza~Yazdani Aminabadi, Olatunji Ruwase, Samyam Rajbhandari,
Xiaoxia Wu, Ammar~Ahmad Awan, Jeff Rasley, Minjia Zhang, Conglong Li, Connor
Holmes, Zhongzhu Zhou, Michael Wyatt, Molly Smith, Lev Kurilenko, Heyang Qin,
Masahiro Tanaka, Shuai Che, Shuaiwen~Leon Song, and Yuxiong He.
\newblock {DeepSpeed-Chat: Easy, Fast and Affordable RLHF Training of
ChatGPT-like Models at All Scales}.
\newblock {\em arXiv preprint arXiv:2308.01320}, 2023.
\bibitem{coca}
Jiahui Yu, Zirui Wang, Vijay Vasudevan, Legg Yeung, Mojtaba Seyedhosseini, and
Yonghui Wu.
\newblock Coca: Contrastive captioners are image-text foundation models.
\newblock {\em CoRR}, abs/2205.01917, 2022.
\bibitem{florence}
Lu~Yuan, Dongdong Chen, Yi{-}Ling Chen, Noel Codella, Xiyang Dai, Jianfeng Gao,
Houdong Hu, Xuedong Huang, Boxin Li, Chunyuan Li, Ce~Liu, Mengchen Liu,
Zicheng Liu, Yumao Lu, Yu~Shi, Lijuan Wang, Jianfeng Wang, Bin Xiao, Zhen
Xiao, Jianwei Yang, Michael Zeng, Luowei Zhou, and Pengchuan Zhang.
\newblock Florence: {A} new foundation model for computer vision.
\newblock {\em CoRR}, abs/2111.11432, 2021.
\bibitem{vinvl}
Pengchuan Zhang, Xiujun Li, Xiaowei Hu, Jianwei Yang, Lei Zhang, Lijuan Wang,
Yejin Choi, and Jianfeng Gao.
\newblock {VinVL}: Revisiting visual representations in vision-language models.
\newblock In {\em {IEEE} Conference on Computer Vision and Pattern Recognition,
{CVPR} 2021, virtual, June 19-25, 2021}, pages 5579--5588. Computer Vision
Foundation / {IEEE}, 2021.
\bibitem{zhao2023mmicl}
Haozhe Zhao, Zefan Cai, Shuzheng Si, Xiaojian Ma, Kaikai An, Liang Chen, Zixuan
Liu, Sheng Wang, Wenjuan Han, and Baobao Chang.
\newblock Mmicl: Empowering vision-language model with multi-modal in-context
learning.
\newblock {\em arXiv preprint arXiv:2309.07915}, 2023.
\bibitem{zhu2023minigpt}
Deyao Zhu, Jun Chen, Xiaoqian Shen, Xiang Li, and Mohamed Elhoseiny.
\newblock Minigpt-4: Enhancing vision-language understanding with advanced
large language models.
\newblock {\em arXiv preprint arXiv:2304.10592}, 2023.
\end{thebibliography}