When Large Language Models Meet Speech: A Survey on Integration Approaches
Extracted source text · arXiv 2502.19548
% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.
\documentclass[11pt]{article}
% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets
% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
\usepackage{enumitem}
% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
\usepackage{tikz}
\usepackage{booktabs}
\usepackage[table]{xcolor}
\usepackage{multicol,multirow}
\usepackage{makecell}
\usepackage[edges]{forest}
%\usepackage{pgf-pie}
\usepackage{pgfplots}
\pgfplotsset{compat=1.18}
\usepackage{hyperref}
\usepackage[framemethod=tikz]{mdframed}
\usepackage{subcaption}
\usepackage{amsmath}
\definecolor{lgreen}{rgb}{0.89,0.94,0.85}
\definecolor{lred}{rgb}{0.98, 0.90, 0.84}
\definecolor{lyellow}{rgb}{1.00, 0.95, 0.80}
\definecolor{lblue}{rgb}{0.85, 0.89, 0.95}
\definecolor{hidden-draw}{RGB}{20,68,106}
\definecolor{hidden-pink}{RGB}{255,245,247}
\definecolor{bblue}{rgb}{0.95,0.97,0.98}
\definecolor{blueref}{rgb}{0.005,0.0,0.478}
\definecolor{la}{RGB}{128,208,195}
\definecolor{lb}{RGB}{158,170,196}
\definecolor{lc}{RGB}{128,208,195}
\definecolor{ld}{RGB}{158,170,196}
\definecolor{le}{RGB}{128,208,195}
\definecolor{bk}{RGB}{158,170,196}
% \newcommand{\cref}[1]{\textcolor{blue}{\ref{#1}}}
\newcommand{\revised}[1]{\textcolor{black}{#1}}
%\newcommand{\revised}[1]{\textcolor{violet}{#1}}
\newcommand{\needrevise}[1]{\textcolor{red}{#1}}
\tikzset{%
parent/.style = {align=center,text width=0.7cm, rounded corners=2pt, line width=0.8mm, fill=white!0, draw=white!90},
child/.style = {align=center,text width=1.4cm,rounded corners=2pt, fill=blue!10,draw=blue!90,line width=0.3mm},
% Dataset style
T1/.style = {align=center,text width=1.8cm,rounded corners=3pt, fill=lblue!100, draw=black,line width=0.2mm},
T1_end/.style = {align=left, text width=5cm,rounded corners=5pt, fill=lblue!100,draw=black!0,line width=0.3mm},
T2/.style = {align=center,text width=1.8cm,rounded corners=3pt, fill=lred!100, draw=black,line width=0.2mm},
T2_end/.style = {align=left, text width=5cm,rounded corners=5pt, fill=lred!100,draw=black!0,line width=0.3mm},
T3/.style = {align=center,text width=1.8cm,rounded corners=3pt, fill=lyellow!100, draw=black,line width=0.2mm},
T3_end/.style = {align=left, text width=5cm,rounded corners=5pt, fill=lyellow!100,draw=black!0,line width=0.3mm},
T4/.style = {align=center,text width=1.8cm,rounded corners=3pt, fill=lgreen!100, draw=black,line width=0.2mm},
T4_end/.style = {align=left, text width=5cm,rounded corners=5pt, fill=lgreen!100,draw=black!0,line width=0.3mm}
}
% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.
\title{When Large Language Models Meet Speech:\\A Survey on Integration Approaches}
% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
% Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\ ... \\ Address line
% \And ... \And
% Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\ ... \\ Address line
% \AND
% Author 2 \\ Address line \\ ... \\ Address line \And
% Author 3 \\ Address line \\ ... \\ Address line}
\author{
\revised{Zhengdong Yang, Shuichiro Shimizu, Yahan Yu, Chenhui Chu}\\
\revised{Kyoto University}\\
\revised{\texttt{\{zd-yang, sshimizu, yahan\}@nlp.ist.i.kyoto-u.ac.jp} \quad \texttt{chu@i.kyoto-u.ac.jp}}
}
%\author{
% \textbf{First Author\textsuperscript{1}},
% \textbf{Second Author\textsuperscript{1,2}},
% \textbf{Third T. Author\textsuperscript{1}},
% \textbf{Fourth Author\textsuperscript{1}},
%\\
% \textbf{Fifth Author\textsuperscript{1,2}},
% \textbf{Sixth Author\textsuperscript{1}},
% \textbf{Seventh Author\textsuperscript{1}},
% \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
% \textbf{Ninth Author\textsuperscript{1}},
% \textbf{Tenth Author\textsuperscript{1}},
% \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
% \textbf{Twelfth Author\textsuperscript{1}},
%\\
% \textbf{Thirteenth Author\textsuperscript{3}},
% \textbf{Fourteenth F. Author\textsuperscript{2,4}},
% \textbf{Fifteenth Author\textsuperscript{1}},
% \textbf{Sixteenth Author\textsuperscript{1}},
%\\
% \textbf{Seventeenth S. Author\textsuperscript{4,5}},
% \textbf{Eighteenth Author\textsuperscript{3,4}},
% \textbf{Nineteenth N. Author\textsuperscript{2,5}},
% \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
% \textsuperscript{1}Affiliation 1,
% \textsuperscript{2}Affiliation 2,
% \textsuperscript{3}Affiliation 3,
% \textsuperscript{4}Affiliation 4,
% \textsuperscript{5}Affiliation 5
%\\
% \small{
% \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
% }
%}
\begin{document}
\maketitle
\begin{abstract}
Recent advancements in large language models (LLMs) have spurred interest in expanding their application beyond text-based tasks. A large number of studies have explored integrating other modalities with LLMs, notably speech modality, which is naturally related to text.
This paper surveys the integration of speech with LLMs, categorizing the methodologies into three primary approaches: text-based, latent-representation-based, and audio-token-based integration.
%Each approach leverages the linguistic and generative capabilities of LLMs to enhance various aspects of speech processing, from transcription to real-time interaction.
We also demonstrate how these methods are applied across various speech-related applications and highlight the challenges in this field to offer inspiration for future research.
%promising direction of employing decoder-only LLM architectures in conjunction with specialized language modeling techniques tailored for speech. Our survey underscores the potential of LLMs to revolutionize speech processing applications, suggesting a future where speech and text modalities are seamlessly integrated, enhancing both the accessibility and effectiveness of human-computer interactions.
\end{abstract}
\section{Introduction}
In recent years, the field of natural language processing (NLP) has been greatly reshaped by the development of large language models (LLMs)~\cite{brown-etal-2020,touvron-etal-2023-llama,geminiteam-2024-gemini,bai-2023-qwen,deepseekai-2024-deepseek}.
These models have not only shown excellent ability in understanding and generating text but have also sparked interest in their potential applicability across other modalities, including speech.
The integration of speech and LLMs offers a wide range of potential applications, including speech translation, conversational chatbots, and enhanced human-computer interaction in robotics.
%As the first form of human language naturally related to text, speech is a prominent modality that presents a fertile ground for the expansion of LLM capabilities beyond the written form.
%Recent studies have begun to explore the integration of speech and LLMs.
Survey papers have reviewed speech language models \citep{peng-etal-2024-speechllmsurvey,cui-etal-2025-speechlmsurvey}, as well as audio language models~\cite{latif-2023-large-audio} and multimodal language models~\cite{ghosh-2024-vision-language-model,zhang-2024-mmllm}.
However, there still lacks a survey specifically on the integration approaches of speech and LLMs, posing a challenge for researchers seeking to address this complex problem.
Distinct from other survey papers regarding speech and LLMs, this paper provides insight into the problem by specifically surveying the integration approaches of speech and LLMs.
Studies on LLM tokenization~\citep{chai-etal-2024-tokenization,tao-etal-2024-scalingvocab} suggests that tokenization methods can affect the performance of LLMs.
On the other hand, studies on tokenization methods for speech language modeling~\citep{gat-etal-2023-discreteslm,borsos-etal-2023-audiolm} also show that speech tokenization methods can affect the performance of speech language models.
In contrast to text processing, speech--LLM integration approaches are not limited to discrete tokenization, as presented in this paper.
Therefore, studying the integration between speech and LLMs can be a key to innovations.
% To the best of our knowledge, there is one survey paper related to this topic~\cite{latif-2023-large-audio}, where they considered a broader range of audio that includes speech. However, they primarily focused on individual models handling audio inputs and outputs, without considering the integration of multiple models, such as utilizing an LLM to rescore hypotheses generated by automatic speech recognition (ASR) models. They also only focused on the models' training data and different tasks they are capable of, without a systematic taxonomy of methodologies.
% \needrevise{Add comparisons with \citep{peng-etal-2024-speechllmsurvey,cui-etal-2025-speechlmsurvey,liu-niehues-2024-sthighlights}}
\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{fig_overview_6.pdf}
\caption{Overview of three fundamental approaches for speech--LLM integration.}
\label{fig:overview}
\end{figure}
In this paper, we systematically categorize a substantial body of research on speech--LLM integration,\footnote{One challenge that affects the scope of studies is the lack of standard definition for LLMs. In this paper, we adopt the loose definition by \citet{zhao-etal-2023-survey-llm}, focusing on models with over 10 billion parameters, while also including notable studies with smaller models.}
and provide a clear taxonomy of the integration approaches.
We broadly categorize the integration into the following three types:
(a) \textbf{Text-based integration}: LLMs process textual data, integrated with speech-to-text and/or text-to-speech models;
(b) \textbf{Latent-representation-based integration}: Latent vector representations that encode speech data are utilized, mainly as inputs to LLMs;
(c) \textbf{Audio-token-based integration}: Speech tokens, such as semantic tokens and/or acoustic tokens, are used as the inputs/outputs for LLMs.
The overview of these approaches is illustrated in Figure \ref{fig:overview}, and the detailed taxonomy with representative studies is presented in Figure \ref{fig:taxonomy}.
\iffalse
The rest of the paper is organized as follows.
Section \ref{sec:background} provides backgrounds on language modeling and speech representations.
Sections \ref{sec:text-based-integration}, \ref{sec:latent-representation-based-integration}, and \ref{sec:audio-token-based-integration} offers literature review based on our taxonomy.
Section \ref{sec:comparative-analysis} provides a comparison of the methods.
Section \ref{sec:challenges} discusses potential future work and Section \ref{sec:conclusion} concludes the paper.
\fi
\input{taxonomy}
\section{Background} \label{sec:background}
\subsection{Language Modeling}
Language modeling dates back to statistical models like $N$-gram models~\citep{brown-etal-1992-ngram}, which were central to early NLP and automatic speech recognition (ASR) systems~\citep{bahl-etal-1983}.
Neural-network-based language modeling was introduced by~\citet{bengio-etal-2000}, formulating the probability
\[
P(w_{1:T}) = \prod_{t=1}^T P(w_t | w_{1:t-1}),
\]
where $w_t$ is the $t$-th token (text subwords or speech tokens) and $w_{i:j} = (w_i, w_{i + 1}, \cdots, w_j)$ is the subsequence from $i$-th token to $j$-th token.
Advances in hardware enabled neural models to scale, leading to innovations such as sequence-to-sequence learning~\citep{sutskever-etal-2014-seq2seq} and the attention mechanism~\citep{bahdanau-etal-2015}.
The Transformer architecture~\citep{vaswani-etal-2017} revolutionalizes language modeling using self-attention by efficiently modeling long-range dependencies utilizing parallelized computation.
Decoder-only Transformer~\citep{liu-etal-2018} gains prominence through the multi-task learning paradigm with generative pre-training and discrinative fine-tuning~\citep{radford-etal-2018}.
Scaling these models~\citep{radford-etal-2019,brown-etal-2020} shows generalization to multiple tasks without explicit supervision and achieves comparative performance against task-specific models, which leads to the advent of LLM era.
Techniques such as instruction tuning~\citep{wei-etal-2022} and alignment to human preference via reinforcement learning~\citep{ouyang-etal-2022} further advance LLM capabilities.
For a comprehensive overview of LLMs, we refer the readers to LLM survey papers~\citep{zhao-etal-2023-survey-llm,minaee-etal-2024}.
\subsection{Speech Representations} \label{sec:background-speech-representations}
Speech is captured as waveform signals, sampled at a specific sampling rate and quantization value, and represented as a sequence of amplitude values.
For deep learning applications, they are often converted to speech representations with shorter sequence lengths, such as log Mel filterbanks, which we refer to as \textbf{filterbanks}.
%\revised{
%Speech is captured as waveform signals, sampled at a specific sampling rate and quantization value, and represented as a sequence of amplitude values. For deep learning applications, they are often converted to speech representations with shorter sequence lengths. The most commonly used representations are log Mel filterbanks, obtained by applying the Fourier transform, followed by the Mel scaling and logarithm. We refer to these as \textbf{filterbanks}.
Following the success of the unsupervised pre-training approach in other fields, self-supervised speech models (S3Ms)~\citep{baevski-etal-2020-wav2vec2,hsu-etal-2021-hubert,chen-etal-2022-wavlm} were introduced.
These models predict masked frames to learn representations that capture phonetic information \citep{choi-etal-2024}, along with a wide range of speech characteristics, such as speaker identity, paralinguistic information, and word-level information \citep{pasad-etal-2024}.
We refer to these contextualized frame-level features as \textbf{latent representations}.
To integrate speech (which is naturally continuous) with LLMs (which are designed to handle discrete tokens), \textbf{audio tokens} that are more discrete than previous representations have been studied recently.
There are two mainstream approaches to modeling speech as audio tokens:\footnote{Different studies use different terms for these tokens
(e.g., \citet{lakhotia-etal-2021-gslm} used the term ``units'' for semantic tokens).
We use the terms semantic tokens and acoustic tokens as in \citet{borsos-etal-2023-audiolm} and the term audio tokens to represent either or both of them similar to \citet{defossez-etal-2024-moshi}.} \textbf{semantic tokens}
and \textbf{acoustic tokens}.\footnote{We note that there are other efforts than semantic or acoustic tokens to discretize speech signals \citep{bai-etal-2024-dmel}.}
Semantic tokens can be obtained from S3Ms by discretizing latent representations using $k$-means.
Acoustic tokens have been studied in the line of audio codecs (e.g., MP3 \citep{iso11172-3}, Opus \cite{rfc6716}, etc.), which aim to compress audio signals.
Neural audio codecs \citep{zeghidour-etal-2021-soundstream,defossez-etal-2023-encodec,kumar-etal-2023-dac} have recently gained traction as powerful tools to improve coding efficiency as well as perceptual quality.
Because acoustic tokens exhibit better quality in producing output speech signals \cite{borsos-etal-2023-audiolm} but fail to capture semantic information like semantic tokens do, recent studies combine both to obtain better representations for speech-language modeling \citep{wang-etal-2023-viola,zhang-etal-2024-speechtokenizer,defossez-etal-2024-moshi}.
\section{Text-based Integration} \label{sec:text-based-integration}
%straightforward: cascaded, for many task
%when considering tasks like ASR, LLM can take hypotheses instead of direct recognition result
%Text-based integration represents a straightforward method to incorporate speech modality into LLMs.
This approach primarily utilizes text as both input and output for LLMs.
%making it highly adaptable to various applications.
One of the most direct implementations of this method is through \textbf{cascaded integration}, which is highly adaptable to various tasks.
%where speech is first converted into text via ASR, then processed by LLMs, and optionally converted back to speech.
%This setup is particularly versatile, supporting a wide range of tasks
%from transcription enhancements to interactive voice responses.
Within tasks like ASR, LLMs are not limited to working with the final recognition results but can also operate on intermediate hypotheses. This allows for sophisticated operations like \textbf{LLM rescoring},
%where an LLM re-evaluates and adjusts intermediate hypotheses to improve accuracy,
and \textbf{LLM GER} (Generative Error Correction).
%where an LLM generates an entirely new transcription hypothesis from initial hypotheses, potentially surpassing the original predictions in quality and relevance (Figure \ref{fig:rescore}).
\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{fig_rescore_2.pdf}
\caption{Two text-based approaches using hypotheses for improving speech-to-text tasks.}
\label{fig:rescore}
\end{figure}
\subsection{Cascaded Integration} \label{sec:cascaded-integration}
Cascaded integration is the most simple and straightforward approach for integrating speech with LLMs.
This involves the backbone LLM being supported by ASR and Text-to-Speech (TTS) interfaces for handling speech inputs and outputs, as well as the LLM invoking external models to solve speech-related tasks.
This approach has been employed in models such as AudioGPT~\cite{huang-2024-audio-gpt} and HuggingGPT~\cite{shen-2023-hugginggpt} to expand the applications of LLMs.
The primary advantage of cascaded integration is its ease of implementation, allowing various tasks to be integrated into a single model with minimal effort.
However, this method suffers from accuracy problems due to error propagation and efficiency problems due to the latency of processing through multiple steps.
\subsection{LLM Rescoring} \label{sec:llm-rescoring}
Before research on LLMs explosively expanded, ASR models had already begun using rescoring mechanisms with external language models to improve transcription accuracy~\cite{mcdermott-2019-lm-fusion,shin-2019-sentence-scoring,salazar-2020-mlm-scoring,xu-2022-rescorebert}.
Recently, there have been efforts towards utilizing LLMs for rescoring, and it has been proven to be effective~\cite{chen-2023-longform,udagawa-2022-rescoring}.
As shown in Figure \ref{fig:rescore} (a), this process involves generating a list of $n$-best hypotheses $\mathcal{H} = \{Y_1,Y_2,...,Y_n\}$ from the speech input $X$ through the initial ASR decoding, which is then re-evaluated using an LLM to find the hypothesis with higher linguistic coherence to improve accuracy.
The rescoring process can be expressed as
\begin{align*}
Y^* &= \text{argmax}_{Y_i \in \mathcal{H}} \ [(1\!-\!\lambda) \log p_{\text{AM}}(Y_i|X) \\
&\quad + \lambda \log p_{\text{LLM}}(Y_i)]
\end{align*}
where $p_{\text{AM}}$ and $p_{\text{LLM}}$ is the probability according to the ASR model (Acoustic Modle, AM) and the LLM, and $\lambda$ is a weight balancing the contributions of two models.
%A method closely related to rescoring is shallow fusion~\cite{hu-2023-shallow-fusion}. Instead of re-evaluating the hypotheses after the initial decoding process, shallow fusion occurs during the beam search decoding process.
%Specifically, at the timestep $t$, with the predicted token sequence denoted as $Y_{1:t-1}$, the next token $y_t$ is predicted based on both models:
%\begin{align*}
%y_t^* &= \text{argmax}_{y_{t}} \ [(1\!-\!\lambda) \log p_{\text{AM}}(y_{t}|X,Y_{1:t-1}) \\
%&\quad + \lambda \log p_{\text{LLM}}(y_{t}|Y_{1:t-1})]
%\end{align*}
\subsection{LLM Generative Error Correction} \label{sec:llm-ger}
Based on the concept of rescoring, LLM GER represents a new approach that fully utilizes the generative capability of LLMs.
As shown in Figure \ref{fig:rescore} (b), the method utilizes an instructional prompt together with the n-best hypotheses list to guide the LLM in generating transcription predictions.
~\citet{chen-2023-hyporadise} described this task performed by the LLM as hypotheses-to-transcription (H2T), which they explore under both fine-tuning and few-shot settings.
%In the fine-tuning setting, using Low-Rank Adaptation (LoRA)~\cite{xu-2024-lora} for Parameter Efficient Fine-Tuning (PEFT), demonstrate superior accuracy with fewer trainable parameters compared to full fine-tuning. In the few-shot setting, in-context learning has proven effective for this H2T task.
Further studies have explored this direction with PEFT~\cite{hu-2024-noise-robust} or in-context learning~\cite{yang-2023-tap,ma-2023-error-correction}.
%The investigation of this direction started by using the instruction to direct LLMs to perform the aforementioned ASR rescoring in a generative fashion~\cite{yang-2023-tap}.
%Building on this, further studies have explored LLMs' potential for error correction,
Instead of merely selecting the best hypothesis from a pre-existing set,
this method allows the LLM to generate a new transcription based on the hypotheses.
By doing so, LLMs can potentially produce a result with a quality better than all initial hypotheses.
Such an approach has proven to be well-suited for other tasks such as speech-to-text translation (S2TT), which require more generative capabilities of the model~\cite{hu-etal-2024-gentranslate}.
\citet{lin-etal-2024-neko} adopted the mixture-of-experts architecture \citep{fedus-etal-2022,jiang-etal-2024-mixtral} to let different experts handle different types of generative errors produced by task-specific models including ASR and ST models.
\begin{figure*}[t]
\includegraphics[width=\linewidth]{fig_adapter_3.pdf}
\caption{Different strategies for modality adaptation.}
\label{fig:adapter}
\end{figure*}
\section{Latent-representation-based Integration} \label{sec:latent-representation-based-integration}
In this integration approach,
%as illustrated in Figure \ref{fig:overview},
a \textbf{speech encoder} is used to process the speech input and generate latent representations that are directly fed into the LLM, bypassing the embedding layer.
The speech encoder is usually a Transformer-based model, which can be pre-trained on large-scale speech data or trained from scratch for the speech--LLM integration.
A critical issue of this approach is the sequence length gap between the speech and text modalities. Speech features, often sampled at rates of 50 to 100 frames per second, result in longer sequences compared to text tokens. Consequently, some form of \textbf{modality adaptation} mechanism is necessary to bridge these modalities, ensuring that the latent representations align with the LLM’s embedding space.
\subsection{Speech Encoder}
\label{sec:speech-encoder}
%The speech representations can be either traditional acoustic features like Mel-filter bank or representations from a pre-trained speech encoder.
For the speech encoder, various pre-trained models can be utilized to provide representations learned from large-scale speech data.
This includes S3Ms (e.g. HuBERT~\cite{hsu-etal-2021-hubert}) as well as the encoder of pre-trained ASR models (e.g. Whisper~\cite{radford-etal-2023-whisper}).
To connect the pre-trained speech encoder and the LLM, an \emph{adapter} (also referred to as a \emph{bridge network} or a \emph{module connector}) is usually adopted.
%that involve training models on tasks where the supervision signal is derived from the data itself, rather than relying on external annotations. This approach allows models to exploit the inherent structure of audio data to discover predictive patterns and features that are valuable for various downstream tasks such as speech recognition, speaker identification, and more.
An alternative approach is to train a speech encoder from scratch, specifically for the LLM integration.
The structure of the speech encoder is usually a multi-layer Transformer~\cite{vaswani-etal-2017} encoder or Conformer~\cite{gulati-2020-conformer}.
%using Mel-filter bank features, a traditional acoustic features used in speech processing, as inputs. They are derived from the Fourier transform of audio signals, focusing on capturing the energy distribution across mel-scaled frequency bands. This scaling mimics the human ear's perception of sound, making these features particularly useful for speech recognition tasks. F-bank features are often used as input for various machine learning models in speech recognition systems, providing a time-frequency representation of the speech signal.
\subsection{Modality Adaptation}
\label{sec:modality-adaptation}
The modality adaptation mechanism is adopted for the speech encoder to map original frame-wise representations to token-wise representations.
This allows the LLM to process the speech sequence in a manner similar to how it processes the text sequence, without having to deal with overly long sequences caused by long-form speech inputs.
The adaptation is usually accomplished by the adapter between pre-trained models, which is introduced in \ref{sec:speech-encoder}, or any part of the self-designed trained-from-scratch speech encoder.
To this end, various adaptation methods have been proposed. Apart from some rarely-used strategies such as random downsampling~\cite{wang-2023-slm}, most of the methods can be categorized into three groups, as shown in Figure \ref{fig:adapter}: convolutional downsampling, CTC compression, and Q-former.
According to comparisons conducted by ~\citet{hono-2023-integration} and ~\citet{yu-2024-connecting}, the Q-former generally outperforms convolutional downsampling, which in turn outperforms CTC compression.
Next, we will detail these strategies in the following sections.
\subsubsection{Convolutional Downsampling} \label{sec:downsampling}
A basic strategy where the speech representation sequence is downsampled from its original length using convolutional layers~\cite{hono-2023-integration}.
When using the same number for kernel size and stride, it becomes equivalent to simply stacking multiple representation vectors together, which may only benefit parameter efficiency~\cite{fathullah-2024-prompting}.
For better alignment with the LLM's embeddings,
some variants add a fully connected layer or a multi-head Transformer network after the convolutional layers~\citep{yu-2024-connecting}.
%Network layers where each neuron is connected to every neuron in the previous layer, facilitating complex pattern recognition through the weighted summation of inputs.
%A key component in Transformers, where multiple attention mechanisms (heads) are applied in parallel to process different subsets of a query sequence against a different sequence (keys and values), allowing the model to capture various aspects and dependencies across the sequences more effectively and diversely.
\subsubsection{CTC Compression} \label{sec:ctc-compression}
CTC compression involves two steps: (i) Training the speech encoder, or a part of it, on the ASR task based on Connectionist Temporal Classification (CTC)~\cite{graves-2013-ctc}; (ii) Compressing the representation sequence based on the results of CTC predictions. Several specific strategies for the step (ii) include:
\paragraph{Blank-removal}
Discarding all the frames predicted as blanks~\cite{hono-2023-integration,wu-2023-decoder-only}.
%simply discard all the frames that predicted the blank symbol according to the distribution of the CTC posteriors
\paragraph{Frame-averaging}
Averaging the latent representations of consecutive frames whose CTC predictions are the same label~\cite{hono-2023-integration,wu-2023-decoder-only}.
\paragraph{Blank-probability}
Discarding any frames with a CTC probability for blank exceeding a threshold~\cite{ling-2024-fully-formatted}.
%The compression model can be the same as / separate from the adapter.
%CTC (Connectionist Temporal Classification): Method for aligning longer input sequences (audio frames) with shorter output sequences (text). Frame-Level Classifications + Dynamic Programming + Objective Function
\subsubsection{Q-Former} \label{sec:qformer}
Many studies~\cite{yu-2024-connecting,tang-etal-2024-salmonn,pan-2023-cosmic} adopt Q-Former for the modality adaptation. Q-Former is a Transformer-based module converting variable-length input sequences into fixed-length output query representations. It is initially proposed for vision--text modality alignment~\cite{li-2023-blip}.
%Fine-tuning pre-trained speech models without a modality adapter also works. Perhaps it is only needed when the speech model is frozen. The tuning strategy still needs investigation.
\subsection{Training Strategy}
The optimal approach is to train the whole model consisting of the speech encoder and the LLM. However, fully fine-tuning the LLM is computationally expensive. Consequently, PEFT methods such as LoRA~\cite{xu-2024-lora}, are often adopted to reduce resource consumption.
In scenarios where a pre-trained speech encoder is utilized, the primary focus is training the adapter connecting two pre-trained models. Whether to fine-tune the speech encoder or the LLM with PEFT seems optional as the decision differs across studies.
On this topic, several studies~\cite{hono-2023-integration,pham-2024-comprehensive} have conducted systematic investigations by comparing the effects of fine-tuning versus freezing different modules.
%Although speech encoders are generally smaller than LLMs and thus easier to fine-tune,
%\citet{hono-2023-integration} indicated that fine-tuning the pre-trained speech encoder typically does not yield as significant an impact as fine-tuning the LLM.
As for training the speech encoder from scratch, it is commonly done jointly with fine-tuning LLM with PEFT.
\citet{wu-2023-decoder-only} proposed employing a two-stage training process where PEFT is not initiated until the speech encoder has undergone some initial training, ensuring a more stable training progression.
%For F-bank, usually the modality adapter is trained first, then the whole system with LoRA.
%\subsection{Model Fusion}
%Model fusion integrates a language model directly into the ASR decoding process, thereby creating a more intertwined and dynamic interaction between the ASR system and the LLM. This method involves combining scores from both the ASR model and the LLM in real-time, adjusting and improving transcription as the decoding process unfolds. Such integration allows the strengths of both models to be utilized simultaneously: the ASR model's acoustic decoding capabilities and the LLM's superior linguistic modeling. This synergistic approach not only enhances the accuracy of the transcribed text but also improves the system's ability to handle complex linguistic structures and varied speech nuances, making it highly effective in real-world applications where diverse accents and dialects are present.
%\subsubsection{Shallow Fusion}
%Shallow fusion is a method used during the decoding phase of speech recognition. In this approach, the scores from the ASR model and a separate language model are combined linearly. The language model typically operates as an external component, and its scores are incorporated at the beam search stage of decoding.~\cite{hu-2023-shallow-fusion} This allows the language model to influence the selection of word sequences during the decoding process without altering the internal dynamics of the ASR model.
%\subsubsection{Cold Fusion}
%Cold fusion~\cite{cold-fusion} integrates the language model more deeply into the ASR system compared to shallow fusion. In cold fusion, the outputs of a pre-trained language model are fed directly into the ASR model during training. This technique involves training the ASR system to utilize both its own acoustic predictions and the insights provided by the language model. The integration happens at a feature-level rather than at the final decision level, allowing the ASR model to leverage semantic and syntactic knowledge from the language model throughout its decision-making process.
%\subsubsection{Deep Fusion}
%Deep fusion~\cite{deep-fusion}, also known as tight integration, involves a more intricate merging of the language model and the ASR system. In this setup, the language model and the ASR model are jointly trained from the start, allowing both models to influence each other’s learning process significantly. This method aims to create a unified model where the distinction between the acoustic model and the language model becomes less defined, leading to potentially better performance as the models can optimize their parameters in concert based on both acoustic and linguistic data.
%\subsection{Large Speech Language Models？}
\section{Audio-token-based Integration} \label{sec:audio-token-based-integration}
As explained in Section \ref{sec:background-speech-representations}, there are two types of audio tokens: semantic tokens and acoustic tokens.
Studies use either or both, treating them as independent tokens like text or converting them back to latent representations.
When semantic tokens are used for the output, a separate model to convert them into filterbanks and to waveform signals similar to TTS models is often used.
\subsection{Speech Language Models without LLMs}
\subsubsection{Semantic Token} \label{sec:semantic-token}
\citet{lakhotia-etal-2021-gslm} introduced the concept of generative spoken language modeling, which uses semantic tokens as both the input and output of a language model.
\citet{polyak-etal-2021} and \citet{kharitonov-etal-2022-pgslm} extended the idea by incorporating prosodic information.
These models only used audio, and although they could output temporarily coherent contents, semantic understanding ability, and audio quality remained as challenges.
\subsubsection{Acoustic Token} \label{sec:acoustic-token}
The use of acoustic tokens for language modeling was popularized in the task of TTS \citep{wang-etal-2023-valle,chen-etal-2024-valle2}, where a few seconds of sample speech is used to generate coherent speech as those samples given text.
\citet{wang-etal-2023-viola} extended the idea to cover multi-tasking of ASR, machine translation (MT), and TTS.
\subsubsection{Integration of Semantic and Acoustic tokens} \label{sec:semantic-and-acoustic}
AudioLM \citep{borsos-etal-2023-audiolm} was introduced to incorporate both semantic and acoustic tokens.
They adopted a two-stage approach, where the model first predicts semantic tokens and then subsequently predicts acoustic tokens.
\citet{zhang-etal-2024-speechtokenizer} employed distillation from semantic representation to the first quantizer to combine semantic and acoustic tokens.
Their generation approach is similar to \citet{wang-etal-2023-valle}, where an autoregressive model produces the first-layer token and a non-autoregressive model produces the rest.
\input{comparison_table}
\subsection{Integration of LLMs into Speech Language Models}
The use of LLMs as an underlying model of speech language models has become more popular following the success of LLMs.
\citet{hassid-etal-2023-twist} showed that training speech language models using underlying pretrained text language models could improve performance.
VoxtLM \citep{maiti-etal-2024-voxtlm} uses OPT \citep{zhang-etal-2022-opt} as an underlying LLM and conducts multi-task fintuning it on ASR, TTS, and language modeling on text and semantic tokens.
AudioPaLM \citep{rubenstein-etal-2023-audiopalm} adopts a similar approach to \citet{borsos-etal-2023-audiolm} but also enables text input/output, leveraging PaLM \citep{chowdhery-etal-2023-palm} as the underlying LLM.
TWIST and SpeechGPT \cite{zhang-etal-2023-speechgpt} use Llama \cite{touvron-etal-2023-llama} as the underlying model and use semantic tokens as both input and output.
Spirit-LM \citep{nguyen-etal-2025-spiritlm} uses Llama 2 \citep{touvron-etal-2023-llama2} as the underlying model and uses a mixture of semantic and text tokens for both input and output.
\citet{defossez-etal-2024-moshi} introduced a new architecture for speech language modeling.
Similarly to previous studies, they trained a text LLM and a codec model similar to \citet{zhang-etal-2024-speechtokenizer}.
Their architecture enables duplex ability of listening and generating speech by using a hierarchical autoregressive model consisting of two Transformer modules, namely Temporal Transformer and Depth Transformer \citep{lee-etal-2022-rqtransformer}.
This architecture is also shown to be effective for speech-to-speech translation (S2ST)~\cite{labiausse-etal-2025-hibiki}.
\section{Comparative Analysis} \label{sec:comparative-analysis}
% \needrevise{This section should provide a comparison of methods based on the list of studies in Sections \ref{sec:text-based-integration}, \ref{sec:latent-representation-based-integration}, and \ref{sec:audio-token-based-integration}. I tentatively moved \ref{sec:application} under this section, but focusing on application is just one aspect of it and this section requires significant revisions.}
%\subsection{Quantitative comparison} \label{sec:application}
%Focus on Benchmark?
%\section{Tasks and Evaluations}
%\revised{We provide a comprehensive performance comparison as shown in Table~\ref{tab:benchmarks}. Given the numerous applications available, we focus on comparing different models based on ASR, S2TT, S2ST, TTS, and speech emotion recognition (SER) tasks.}
\subsection{Advantages and Disadvantages} \label{sec:proscons}
It is important to understand the advantages and disadvantages of the three integration approaches when employing them for different applications. We summarize their pros and cons as follows:
\begin{itemize}[left=0pt,itemsep=-2pt]
\item \textbf{Degree of integration:} Latent-representation-based $>$ Audio-token-based $>$ Text-based.
\item \textbf{Interpretability:} Text-based $>$ Audio-token-based $>$ Latent-representation-based.
\item \textbf{Speech generation ability:} Text-based and audio-token-based approach can generate speech, whereas latent-representation-based approaches typically cannot.
\end{itemize}
Based on these characteristics, different approaches excel in different scenarios:
\begin{itemize}[left=0pt,itemsep=-2pt]
\item Latent-representation-based and audio-token-based approaches are better than text-based approach, in scenarios where sufficient resources (data, computational power, and time) are available or when real-time processing is required. Deeper integration reduces error propagation and latency but demands more extensive resources.
\item Text-based approach is better than latent-representation-based and audio-token-based approaches, in scenarios where resources are limited or when greater interpretability is required.
\item Latent-representation-based approach is better than audio-token-based, in scenarios where speech is only considered as input. Latent-representation-based methods provide the deepest integration when generating textual or other downstream outputs.
\item Audio-token-based is better than latent-representation-based, in scenarios where speech is also considered as output. Generating speech from latent representations remains challenging, making audio-token-based approaches more suitable in these cases.
\end{itemize}
\subsection{Quantitative Comparison} \label{sec:quantitative}
\revised{We provide a comprehensive performance comparison in Table~\ref{tab:benchmarks}, comparing studies with different integration approaches across multiple tasks: ASR, S2TT, S2ST, and TTS.
% , and speech emotion recognition (SER) tasks.
The table highlights key metrics such as word error rate (WER) and BLEU, alongside each model's integration method and backbone components.}
While Table~\ref{tab:benchmarks} summarizes many state-of-the-art results, direct comparisons can be challenging due to variations in backbone LLMs, acoustic front-ends, and training protocols. For instance, although BLEU scores on CoVoST~2~\cite{wang-etal-2021-covost2} indicate that deeper integration methods (e.g., latent-representation-based) often yield improved performance, model size and training resources also play a significant role. Future work should focus on more uniform experimental conditions to isolate the impact of each approach.
Nonetheless, the metrics in Table~\ref{tab:benchmarks} serve as a starting point for evaluating how well each method handles different tasks. The results suggest that deeper integration (e.g., latent-representation-based) can be beneficial when sufficient computational resources
%\footnote{\revised{Further discussions are in Appendix~\ref{app:computational}}}
and training data are available, whereas text-based methods often offer greater interpretability and simpler pipelines.
\subsection{\revised{Computational Trade-offs}}
\revised{As different methods have different computational requirements, it is difficult to have direct comparisons through a literature review. For numerical comparisons, comparing different approaches with a standard setting is required, which is one future direction. That said, we provide an overview of the computational requirements per category below:}
\revised{\begin{itemize}[left=0pt,itemsep=-2pt]
\item For text-based integration, the computational cost is equivalent to that of running separate models for ASR, LLM, and TTS. Rescoring based on $N$-best list increases the inference compute by a constant factor. For generative error correction, studies have explored full fine-tuning, parameter-efficient fine-tuning, and mixture-of-experts of LLMs. Parameter-efficient fine-tuning can provide better performance compared to full fine-tuning~\cite{chen-2023-hyporadise}. Mixture-of-experts requires high memory and computation for training, but shows good performance in handling multiple tasks with light inference.
\item For latent-representation-based integration, speech encoders are connected to LLMs through adapters. These adapters are lightweight, and studies have explored full fine-tuning and parameter-efficient fine-tuning, including freezing a part of the models. \citet{pham-2024-comprehensive} showed that applying LoRA to the LLM significantly enhances the performance, and for the encoder module, full fine-tuning yields the best performance, followed by partial fine-tuning, while partial fine-tuning is more cost-effective.
\item For audio-token-based integration, it involves audio token encoding/decoding and language modeling with audio tokens. For semantic tokens, feature extraction from S3Ms and training/assignment steps of k-means is required. They are lightweight compared to language modeling (the largest S3Ms have about 2B parameters), but efficient implementation is required for large-scale data. If the output is speech, a vocoder to convert semantic tokens to waveforms is required, which is also relatively lightweight compared to language modeling. For acoustic tokens, encoding and decoding are done using the same model, and the inference is lightweight. Computational cost for language modeling is similar to that of text-based ones.
\end{itemize}}
\revised{Overall, we can argue that the computational cost is bottlenecked by the size of the LLM that is used. Text-based integration and audio-token-based integration would require roughly the same computational bottleneck by LLMs, while latent-representation-based integration requires handling both the speech encoder and LLMs at the same time.}
\iffalse
\subsubsection{Automatic Speech Recognition}
ASR is a foundational task in speech processing, and its integration with LLMs has been extensively studied.
%This integration aims to leverage the advanced linguistic capabilities of LLMs to enhance the accuracy and flexibility of ASR systems in various contexts.
Various approaches introduced in previous sections have been explored for this integration, including LLM rescoring~\cite{udagawa-2022-rescoring}, LLM GER~\cite{chen-2023-hyporadise,yang-2023-tap,lin-etal-2024-neko}, as well as using LLM as a decoder with latent representation~\cite{fathullah-2024-prompting,hono-2023-integration,yu-2024-connecting} or audio token as inputs~\cite{rubenstein-etal-2023-audiopalm,du-etal-2024-lauragpt}.
LLMs have demonstrated their effectiveness across different data scenarios, notably in processing long-form data~\cite{chen-2023-longform} and enhancing performance in low-resource settings.
Some investigation has been conducted for the adaptation of LLMs for fully formatted ASR, which involves converting speech signals into complete transcriptions that include punctuation, capitalization, numbers, and other formatting details~\cite{ling-2024-fully-formatted}.
%By leveraging the direct output of LLMs, this approach circumvents the need for extensive format post-processing typically required in conventional ASR systems.
Domain adaptation is another crucial aspect where LLMs offer significant advantages. Utilizing LLMs can simplify the adaptation of ASR systems to new domains because collecting target domain text is generally easier than obtaining audio--text pairs specific to that domain~\cite{li-2023-asr-domain}. Key benefits provided by LLMs in this area include zero-shot adaptation, flexible prompt design, and leveraging the LLM community.
%By integrating LLMs with ASR, researchers and developers can not only improve the transcription quality in various settings but also expand the applicability of ASR technologies to more complex and diverse applications.
\needrevise{As a quantitative comparison, word error rate on Librispeech \cite{panayotov-etal-2015-librispeech} test sets is provided in the Appendix Table XXX. However, we note that it does not provide a fair comparison due to variations in the backbone LLMs and acoustic models used. Fair comparison among different integration methods can be a future research topic.}
\subsubsection{Speech-to-text Translation}
ST is the task of translating spoken language from one language directly into text in another language.
%often requiring both recognition of the spoken source and its translation.
%While cascaded ST involves sequentially performing ASR followed by Machine Translation (MT), end-to-end ST offers a more integrated approach by directly translating speech into text, bypassing intermediate transcription.
%This section focuses on end-to-end ST.
%as cascaded ST primarily combines existing ASR and MT technologies without fundamentally innovating the translation process.
%End-to-end ST
It shares similarities with ASR in that both are speech-to-text tasks, which suggests that methodologies developed for ASR are applicable to ST.
Many latent-representation-based~\cite{wu-2023-decoder-only,pan-2023-cosmic} and audio-token-based~\cite{rubenstein-etal-2023-audiopalm,du-etal-2024-lauragpt} speech--LLM models can be adapted for both ASR and ST.
For text-based integration, compared to LLM rescoring, ST is better suited for LLM GER that leverages the generative capabilities of LLMs~\cite{hu-etal-2024-gentranslate}, as ST is a task that involves more generation complexity than ASR.
%due to the need to not only recognize speech but also translate it into a different language.
%Furthermore, the advent of multilingual LLMs~\cite{xue-2021-mt5} has opened new avenues for enhancing multilingual ST. These models, trained across multiple languages, can understand and generate text in various linguistic contexts, making them exceptionally well-suited for ST applications.
%Some recent studies highlight the potential of multilingual LLMs in ST, pointing towards their ability to improve translation accuracy and fluency across diverse languages. This capability not only expands the scope of ST applications but also enhances the quality of translation, underscoring the significant potential of LLMs to revolutionize speech translation tasks.
%By fully utilizing the generative abilities of LLMs, end-to-end ST can achieve superior performance and flexibility, positioning LLMs as pivotal in advancing the field of speech translation. This approach not only simplifies the translation pipeline but also maximizes the linguistic proficiency inherent in sophisticated language models, promising significant improvements in real-world multilingual communication scenarios.
\needrevise{As a quantitative comparison, BLEU score \cite{papineni-etal-2002-bleu} on CoVoST 2 \cite{wang-etal-2021-covost2} is provided in Appendix Table YYY. Again, we note that this does not provide a fair comparison.}
%\subsection{S2ST}
\subsubsection{Text-to-speech}
TTS, also known as speech synthesis, is a task that involves converting written text into corresponding speech.
%a process integral to developing interactive and user-friendly applications like virtual assistants and reading tools.
Utilizing LLMs for TTS often involves generating audio tokens with the LLM, which are then decoded into actual speech using a vocoder.
TTS often serves as one of the tasks for training audio-token-based speech--LLM models~\cite{maiti-etal-2024-voxtlm,du-etal-2024-lauragpt}.
There are a few studies that specifically focus on leveraging LLMs for TTS.
An empirical study~\cite{hao-2024-boosting} indicates that simply fine-tuning LLMs does not achieve the performance level of specialized pre-trained TTS models such as VALL-E\revised{~\cite{wang-etal-2023-valle}}. Instead, LLMs being utilized solely as text encoders for TTS yields better results.
GPT-Talker~\cite{liu-2024-gpt-talker} showcases the effectiveness of LLM for conversational speech synthesis. In this study, the LLM, specifically a GPT model, is used for context modeling, predicting sequences of semantic tokens with style knowledge based on multi-turn conversations.
%Generally, this direction is still underexplored and needs further investigation.
%to fully understand and harness its capabilities in TTS technologies.
\subsection{Evaluation of Basic Speech Understanding}
Speech has different granularity of understanding levels (e.g., phonetic, lexical, syntactic).
The Zero Resource Speech Challenge \citep{dunbar-etal-2021-zrsc21} summarized metrics for these different levels of understanding.
To measure phonetic understanding, the ABX \citep{schatz-etal-2013-abx} metric evaluates the model's ability to distinguish between minimal pairs of triphones that only differ by one phoneme (e.g., `aba' and `apa').
For lexical understanding, spot-the-word metric uses pairs of an existing word and a matching non-word (e.g., ``brick'' and ``$^*$blick''). The sWUGGY dataset is commonly used for this task.
For syntactic understanding, pairs of phrases are used for acceptability judgment, one being grammatical and the other being nongrammatical are used (e.g., ``dogs eat meat'' and ``$^*$dogs eats meat'').
The sBLIMP dataset is commonly used for this task.
\subsection{Other Tasks}
Besides the tasks above, there are many other speech-related tasks that LLMs can possibly perform, such as speech-to-speech translation, speech emotion recognition, speech question answering, speaker verification, speech keyword extraction, speech enhancement, etc.
Some studies~\cite{tang-etal-2024-salmonn,pan-2023-cosmic} conducted multitask learning with LLM and observed the model’s ability to solve unseen tasks.
\fi
\section{Challenges} \label{sec:challenges}
% \needrevise{
% Because this paper is about the integration methods of speech and LLMs, discussed future work should also focus on that aspect. The whole section requires significant revisions.
% }
%\revised{In this section, we discuss challenges and future works based on speech--LLM integration methods.
%More discussions are in Appendix~\ref{app:challenge}.}
\subsection{Text-based Integration}
Text-based integration preserves the original LLM input modality, which is text. Because the model is already optimized for textual inputs, this approach typically requires the least adaptation from the LLM. However, transforming speech into text before feeding it into the LLM inevitably introduces a layer of abstraction and potential information loss such as prosody and emotion, which limits the downstream performance.
Therefore, the main challenge is how to convey the rich information of source speech through text. In current practice, the intermediate text is often generated based on the highest probability~\cite{chen-2023-hyporadise, yang-2023-tap, radhakrishnan2023whispering}. Although this strategy maximizes intermediate accuracy, it may not be optimal for the LLM and its final outputs. One possible direction is to inject controlled randomness during decoding to ensure additional diversity. However, degrading accuracy could also have a negative effect, creating a trade-off. How to balance diversity and accuracy remains an open research question that requires future investigation.
\subsection{Latent-representation-based Integration}
In contrast to text-based integration, latent-representation-based integration employs an intermediate representation \revised{closer to the source speech but more distant from the LLM’s native input space.} Consequently, the primary challenge lies in aligning these acoustic representations with the textual embedding space of the LLM. Section~\ref{sec:modality-adaptation} introduces multiple modality adaptation mechanisms to address this and enable LLMs to handle speech more effectively.
While the primary focus of most research introduced in this paper has been on adapting LLMs to \revised{handle speech data better}, there is comparatively less attention on how these adaptations affect performance on text-based tasks. The alignment of text and speech data, along with their latent representations, remains underexplored, with only a few studies focusing on this issue~\cite{wang2023blsp,lu-2024-desta}.
A promising solution for improving speech-text alignment is the generation of high-quality synthetic speech data to supplement real datasets, thus closing the gap between the speech modality and text-based LLMs. More thorough investigations into how to align speech and text representations in complex languages (beyond English) could also strengthen the multimodal capabilities of LLMs.
\subsection{Audio-token-based Integration}
As stated in Sections \ref{sec:background-speech-representations} and \ref{sec:audio-token-based-integration},
S3Ms (and their derived semantic tokens) mainly capture phonetic information~\cite{choi-etal-2024}, while acoustic tokens offer higher fidelity in generating speech signals~\cite{borsos-etal-2023-audiolm}.
Integrating both representations, as presented in Section \ref{sec:semantic-and-acoustic}, is one approach that requires further investigation.
While this approach achieves strong performance across various downstream tasks (Table~\ref{tab:benchmarks}), there are still limited studies on this approach, where the models are English-centric models that rely on millions of hours of speech data.
Minimizing computational requirements is essential to extend this approach to languages lacking the vast resources available for English.
Some studies suggest that latent-representation-based integration can outperform audio-token-based integration~\cite{wang-etal-2024-comparativestudydiscretespeech}. However, it is possible that this is due to the suboptimal representation of audio tokens~\cite{gat-etal-2023-discreteslm}, requiring further improvement of tokenization methods and comparison with other integration approaches.
\subsection{Fair Comparison Across Integration Approaches}
Beyond these integration-approach-specific challenges, there is a notable gap in comparing the different integration approaches under a unified setting.
Most existing works focus on one approach, making it difficult to assess their relative merits consistently.
A fair comparison among these integration methods could clarify how different factors affect performance.
Developing standardized benchmarks, protocols, and reporting practices for speech--LLM research would help future work isolate the core differences between these approaches.
\subsection{Multilingualism}
\revised{
The challenge of multilingualism in speech--LLM integration is also significant.
Most existing speech--LLM models are designed to handle only English, which limits their range of applicability.
Although there are studies focusing on multilingual capabilities~\cite{hu-etal-2024-gentranslate,hu-2023-shallow-fusion,huang-2024-fusion-comprehensive}, the majority still rely on English-centric LLMs.
Only a small fraction of research efforts~\cite{denisov-2024-multilingual} explicitly consider the multilingualism of pre-trained LLMs.
However, the effectiveness of a speech--LLM model is often constrained by the quality of the text-based backbone LLM.
This dependency implies that any limitations in the backbone LLMs, especially in terms of handling multiple languages, can directly impact the performance of speech LLMs. }
%This issue is particularly pronounced for low-resource languages, where high-quality text LLMs may not be available, thereby restricting the potential improvements that can be achieved with speech LLMs in these languages.
\subsection{Real-time Processing}
\revised{
Real-time processing presents another substantial challenge in the field of speech--LLM integration.
Compared to text-based applications, speech-based applications such as ST more often require real-time processing to be practical in everyday use.
However, the large size of LLMs typically leads to increased latency, which can neutralize some of the benefits gained from improved accuracy.
While most speech LLMs are capable of handling tasks offline, there is a need for more innovative methods that can facilitate online processing.
Deeper integration with latent representations or audio tokens tends to result in lower latency compared to text-based approaches.
Recent models that focus on real-time applications, such as Moshi~\cite{defossez-etal-2024-moshi}, show promising starts in addressing this challenge.
Developing such methods would significantly enhance the utility of LLMs in real-time speech applications, making them more viable for real-world usage.}
% \subsection{Comparison across different integration methods}
% \iffalse
% \begin{itemize}
% \item Comprehensive benchmark: Most of the models are only evaluated on certain tasks (ASR, ST, etc.) although they could possibly more tasks. We can even think of new task/benchmarks, like something that involves both semantic understanding and acoustic richness like expressive ST, dialogue, etc.
% \item Simplifying architecture: I think one of LLM's success attributes to the simplicity of the model, because it's important for scaling.
% \item Audio tokenization: How to align text/speech/latent reps well is understudied. Most of the models use English but it is typographically not so complex, how the method can perform in typographically complex languages like Japanese is understudied.
% \item Instruction tuning
% \item Reducing data size: Currently good models are consuming more than what humans consume in a lifetime.
% \item Reducing model size: Only those with abundant computational resources can train a good model. We should be more clever and devise method that does not require that much compute. This is also important in speech because speech applications more often require real-time processing than text.
% \item Real-time processing: Most of the models can handle things offline but more clever methods could be devised for online processing.
% \item Multilinguality: Most of the models only handles English. Some requires good underlying text LMs, which means the performance of speech LMs are possibly bounded by the performance of text LMs. Low-resource language is also a concern.
% \item Handling multichannel audio: Most of the models only handle single-channel audio. Multichannel audio can capture spatial information which can be useful to identifying speaker etc.
% \item Duplex ability of hearing and speaking: There are only few models that have this ability. There could be other designs that can be considered.
% \item Online learning: In text we had to predefine characters and tokens. Speech frees us from this constraint. Possibly the model could learn a new language only from audio without any predefined human knowledge.
% \end{itemize}
% \fi
\section{Conclusion} \label{sec:conclusion}
In this survey, we systematically categorize the integration of LLMs with speech modality, outlining three primary approaches: text-based, latent-representation-based, and audio-token-based integration.
Each method demonstrates unique strengths in performing different speech-related tasks.
However, numerous challenges remain in this evolving field, presenting significant opportunities for future research.
% , such as speech-text alignment, multilingualism, and real-time processing.
%Particularly promising is the use of decoder-only LLM architectures, which align well with language modeling to enhance speech processing tasks. As this field evolves, these integrations suggest a future where LLMs significantly advance the efficiency and effectiveness of human-computer speech interactions.
%\clearpage
%\section*{Acknowledgments}
%\newpage
\section*{Limitations}
In our survey of the integration of speech with LLMs, we have covered the significant developments within this field.
Despite our comprehensive approach, we acknowledge that the rapid pace of development in speech processing and LLM means that some recent advances or discussions might have escaped our attention and are not fully addressed within this paper.
\revised{
\section*{Acknowledgment}
This work was supported by the SPRING program of Kyoto University and JSPS KAKENHI Grant Number JP23K28144 and JP23KJ1356.
}
% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\input{paper.bbl}
%\appendix
%\revised{\section{Other Challenges}
%\label{app:challenge}}
% \paragraph{Speech-Text Alignment}
%A critical challenge in integrating LLMs with speech modalities lies in the area of speech-text alignment.
%While the primary focus of most research introduced in this paper has been on adapting LLMs to better handle speech data, there is often less attention on how these adaptations impact the performance on text-based tasks. The alignment of text and speech data, along with their latent representations, remains an underexplored area with only a few studies focusing on it~\cite{wang2023blsp,lu-2024-desta}.
%This issue is further compounded when dealing with languages that have complex typographies, such as Chinese and Japanese, as the majority of existing models are primarily focusing on English, a language with relatively simple typography.
%\section{Example Appendix}
%\label{sec:appendix}
%This is an appendix.
\end{document}
\tikzstyle{my-box}=[
rectangle,
draw=hidden-draw,
rounded corners,
text opacity=1,
minimum height=1.5em,
minimum width=5em,
inner sep=2pt,
align=center,
fill opacity=.5,
line width=0.8pt,
]
\tikzstyle{leaf}=[my-box, minimum height=1.5em,
fill=bblue!40, text=black, align=left,font=\normalsize,
inner xsep=2pt,
inner ysep=4pt,
line width=0.8pt,
draw=black,
]
\begin{figure*}[t!]
\centering
\resizebox{1.0\textwidth}{!}{
\begin{forest}
forked edges,
for tree={
grow'=0,
draw,
reversed=true,
anchor=base west,
parent anchor=east,
child anchor=west,
base=left,
font=\large,
rectangle,
rounded corners,
align=left,
minimum width=4em,
edge+={darkgray, line width=1pt},
s sep=3pt,
inner xsep=2pt,
inner ysep=3pt,
line width=0.8pt,
ver/.style={rotate=90, child anchor=north, parent anchor=south, anchor=center},
},
where level=1{text width=11.5em,font=\normalsize,}{},
where level=2{text width=18em,font=\normalsize,}{},
where level=3{text width=8em,font=\normalsize,}{},
where level=4{text width=5em,font=\normalsize,}{},
[
Speech-LLM Integration, ver
[
Text-based \textcolor{blueref}{(Sec.~\ref{sec:text-based-integration})}
[
Cascaded Integration \textcolor{blueref}{(Sec.~\ref{sec:cascaded-integration})}
[
AudioGPT \cite{huang-2024-audio-gpt}{, } HuggingGPT \cite{shen-2023-hugginggpt}{, }\citet{dighe-2024-uncertainty}
, leaf, text width=46em
]
]
[
LLM Rescoring \textcolor{blueref}{(Sec.~\ref{sec:llm-rescoring})}
[
\citet{chen-2023-longform}{, }\citet{udagawa-2022-rescoring}{, }\citet{li-2023-asr-domain}{, }\citet{hu-2023-shallow-fusion}{, }\citet{huang-2024-fusion-comprehensive}
, leaf, text width=46em
]
]
[
LLM GER \textcolor{blueref}{(Sec.~\ref{sec:llm-ger})}
[
HyPoradise \cite{chen-2023-hyporadise}{, }\citet{yang-2023-tap}{, }GenTranslate \cite{hu-etal-2024-gentranslate}{, }\citet{ma-2023-error-correction}{, }\\\citet{hu-2024-noise-robust}{, }Whispering Llama \cite{radhakrishnan2023whispering}{, }MMGER \cite{mu2024mmger}{, }\\\citet{yang2024large}{, }\citet{ko2024benchmarking}{, }LipGER~\cite{ghosh2024lipger}{, }\citet{li2024investigating}{, }\citet{tang2024contextualized} {, }\\ \citet{chen2024s} {, }\citet{dighe2024leveraging}
, leaf, text width=46em
]
]
]
[
Latent-representation-based\\ \textcolor{blueref}{(Sec.~\ref{sec:latent-representation-based-integration})}
[
Convolutional Downsampling \textcolor{blueref}{(Sec.~\ref{sec:downsampling})}
[
\citet{hono-2023-integration}{, }\citet{yu-2024-connecting}{, }\citet{fathullah-2024-prompting}{, }LTU \cite{gong-2024-ltu}{, }\citet{pham-2024-comprehensive}{, }\\BLSP \cite{wang2023blsp}{, }SALM \cite{chen-2024-salm}{, }WavLLM \cite{hu-2024-wavllm}{, }DeSTA \cite{lu-2024-desta}{, }\\Llama-Omni \cite{fang-etal-2024-llamaomni}{, }Seed-ASR \cite{bai2024seed}{, }SpeechVerse \cite{das2024speechverse}{, }\\SLAM-ASR \cite{ma2024embarrassingly}{, }\citet{denisov2024teaching} {, }MaLa-ASR~\cite{yang2024mala}{, }\\WHISMA~\cite{li2024whisma}{, }\citet{lakomkin2024end}{, }\citet{noroozi2024instruction}{, }\citet{kang2024prompting}{, }\\ \citet{li2024using} {, }\citet{chen2024s}
, leaf, text width=46em
]
]
[
CTC Compression \textcolor{blueref}{(Sec.~\ref{sec:ctc-compression})}
[
\citet{hono-2023-integration}{, }Speech-Llama \cite{wu-2023-decoder-only}{, }\citet{ling-2024-fully-formatted}{, } \citet{fathullah2023towards}
, leaf, text width=46em
]
]
[
Q-Former \textcolor{blueref}{(Sec.~\ref{sec:qformer})}
[
\citet{yu-2024-connecting}{, }SALMONN \cite{tang-etal-2024-salmonn}{, }COSMIC \cite{pan-2023-cosmic}{, }XLLM \cite{chen-2023-xllm}{, }\\DeSTA \cite{lu-2024-desta}{, }Secap \cite{xu2024secap}{, }BLSP-KD \cite{wang2024blsp}{, } CoT-ST~\cite{du2024cot}
, leaf, text width=46em
]
]
[
Other Adaptation Methods
[
SLM \cite{wang-2023-slm}{, } Qwen-Audio \cite{chu-2023-qwen-audio}{, } Qwen2-Audio \cite{chu-2024-qwen2-audio}{, } \\LauraGPT \cite{du-etal-2024-lauragpt}{, } LLaST \cite{chen2024llast}{, } BESTOW \cite{chen2024bestow}{, }\\ SALSA \cite{mittal2024salsa}{, } Ideal-LLM~\cite{xue2024ideal} {, }\citet{xu2024comparing}{, }\citet{chen2024s}
, leaf, text width=46em
]
]
]
[
Audio-token-based \textcolor{blueref}{(Sec.~\ref{sec:audio-token-based-integration})}
[
Semantic Tokens \textcolor{blueref}{(Sec.~\ref{sec:semantic-token})}
[
VoxtLM \cite{maiti-etal-2024-voxtlm}{, }SpeechGPT \cite{zhang-etal-2023-speechgpt}{, }TWIST \cite{hassid-etal-2023-twist}{, }\\Spirit-LM \cite{nguyen-etal-2025-spiritlm}{, } GPT-Talker \cite{liu-2024-gpt-talker}{, } BASE TTS \cite{lajszczak2024base}{, }\\Llama-Omni \cite{fang-etal-2024-llamaomni}{, }SpeechGPT-Gen \cite{zhang-etal-2024-speechgptgen}{, } DiscreteSLU \cite{shon2024discreteslu}{, }\\ PSLM~\cite{mitsui2024pslm}{, }\citet{cheng2024task}{, }VSP-LLM~\cite{yeo2024visual}{, }\citet{xu2024comparing}
, leaf, text width=46em
]
]
[
Acoustic Tokens \textcolor{blueref}{(Sec.~\ref{sec:acoustic-token})}
[
\citet{hao-2024-boosting}{, } LauraGPT \cite{du-etal-2024-lauragpt}{, }\citet{neekhara2024improving}{, }Uniaudio~\cite{yang2023uniaudio}
, leaf, text width=46em
]
]
[
Semantic and Acoustic Tokens \textcolor{blueref}{(Sec.~\ref{sec:semantic-and-acoustic})}
[
AudioPaLM \cite{rubenstein-etal-2023-audiopalm}{, }Moshi \cite{defossez-etal-2024-moshi}{, }Hibiki~\cite{labiausse-etal-2025-hibiki}{, }\\ Emova~\cite{chen2024emova}{, }\citet{shen2024get}
, leaf, text width=46em
]
]
]
]
\end{forest}
}
\caption{Taxonomy for speech-LLM integration. ``Latent-representation-based'' integration is categorized by different modality adaptation strategies. ``Other Adaptation Methods'' also includes studies that do not explicitly mention their modality adaptation methods. }
\label{fig:taxonomy}
\end{figure*}
\begin{table*}[t]
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{cccccc}
\toprule
\textbf{Task} & \makecell[c]{\textbf{Dataset}} & \textbf{Metrics} & \textbf{Model} & \textbf{Integration Method} & \textbf{Performance} \\
\midrule
\multirow{22}{*}{ASR} & \multirow{12}{*}{\makecell[c]{\textbf{Librispeech}\\ \small \textit{dev-clean} | \textit{dev-other} | \\ \small \textit{test-clean} | \textit{test-other}}} & \multirow{12}{*}{WER $\downarrow$} & Whisper-large-v2 \cite{radford-etal-2023-whisper} & Non-LLM & --- | --- | 2.7 | 5.2 \\
&&& HyPoradise \cite{chen-2023-hyporadise} & Text-based $\rightarrow$ LLM GER & --- | --- | 1.8 | 3.7 \\
&&& BLSP \cite{wang2023blsp} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & --- | --- | 10.4 | --- \\
&&& SpeechVerse \cite{das2024speechverse} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & --- | --- | 2.1 | 4.4 \\
&&& Seed-ASR \cite{bai2024seed} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & --- | --- | \textbf{1.5} | \textbf{2.8} \\
&&& SALMONN \cite{tang-etal-2024-salmonn} & Latent-representation-based $\rightarrow$ Q-Former & --- | --- | 2.1 | 4.9\\
&&& SLM \cite{wang-2023-slm} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | --- | 2.6 | 5.0 \\
&&& Qwen-Audio \cite{chu-2023-qwen-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & 1.8 | 4.0 | 2.0 | 4.2 \\
&&& Qwen2-Audio \cite{chu-2024-qwen2-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & \textbf{1.3} | \textbf{3.4} | 1.6 | 3.6\\
&&& LauraGPT \cite{du-etal-2024-lauragpt} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | --- | 4.4 | 7.7 \\
&&& SpeechGPT-Gen \cite{zhang-etal-2024-speechgptgen} & Audio-token-based $\rightarrow$ Semantic Token & --- | --- | 2.4 | --- \\
&&& SLAM-ASR \cite{ma2024embarrassingly} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & --- | --- | 1.9 | 3.8 \\
&&& BESTOW \cite{chen2024bestow} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | --- | --- | 3.2 \\
&&& \citet{hao-2024-boosting} & Audio-token-based $\rightarrow$ Acoustic Tokens & 3.7 | 6.6 | 3.4 | 7.1 \\
\cline{2-6}
& \multirow{4}{*}{\makecell[c]{\textbf{Fleurs}\\ \small \textit{zh} | \textit{en}}} & \multirow{4}{*}{WER $\downarrow$} & Whisper-large-v2 \cite{radford-etal-2023-whisper} & Non-LLM & 4.2 | 14.7 \\
&&& \citet{huang-2024-fusion-comprehensive} w/ PaLM2 & Text-based $\rightarrow$ LLM Rescoring & 13.1 | --- \\
&&& Seed-ASR \cite{bai2024seed} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & \textbf{3.43} | --- \\
&&& Qwen2-Audio \cite{chu-2024-qwen2-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | \textbf{7.5} \\
&&& DiscreteSLU \cite{shon2024discreteslu} & Audio-token-based $\rightarrow$ Semantic Token & 12.6 | --- \\
\cline{2-6}
& \multirow{4}{*}{\makecell[c]{\textbf{AISHELL-2} \\ \small \textit{Mic | iOS | Android}}} & \multirow{4}{*}{WER $\downarrow$} & Seed-ASR \cite{bai2024seed} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & \textbf{2.2} | \textbf{2.2} | \textbf{2.2} \\
&&& Qwen-Audio \cite{chu-2023-qwen-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & 3.3 | 3.1 | 3.3 \\
&&& Qwen2-Audio \cite{chu-2024-qwen2-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & 3.0 | 3.0 | 2.9 \\
&&& LauraGPT \cite{du-etal-2024-lauragpt} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | 3.2 | --- \\
\cline{2-6}
& \multirow{2}{*}{\makecell[c]{\textbf{VoxPopoli}\\ \small \textit{All}}} & \multirow{2}{*}{WER $\downarrow$} & SpeechVerse \cite{das2024speechverse} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & \textbf{6.5} \\
&&& AudioPaLM \cite{rubenstein-etal-2023-audiopalm} & Audio-token-based $\rightarrow$ Semantic and Acoustic Tokens & 9.8 \\
\hline
\multirow{11}{*}{S2TT} & \multirow{11}{*}{\makecell[c]{\textbf{CoVoST2}\\ \small \textit{en-de} | \textit{de-en} |\\ \small \textit{en-zh} | \textit{zh-en} |\\ \small \textit{es-en} | \textit{fr-en} |\\ \small \textit{it-en} | \textit{ja-en}}} & \multirow{11}{*}{BLEU $\uparrow$}
& Whisper-large-v2 \cite{radford-etal-2023-whisper} & Non-LLM & --- | 36.3 | --- | 18.0 | 40.1 | 36.4 | 30.9 | 26.1 \\
&&& GenTranslate \cite{hu-etal-2024-gentranslate} & Text-based $\rightarrow$ LLM GER & --- | 39.2 | --- | 21.6 | 42.0 | 41.7 | --- | 22.9 \\
&&& GenTranslate-V2 \cite{hu-etal-2024-gentranslate} & Text-based $\rightarrow$ LLM GER & --- | 40.6 | --- | 23.3 | 43.6 | 42.7 | --- | 25.4 \\
&&& BLSP \cite{wang2023blsp} & Latent-representation-based $\rightarrow$ Convolutional Downsampling & 24.4 | --- | 41.3 | --- | --- | --- | --- | --- \\
&&& Speech-Llama \cite{wu-2023-decoder-only} & Latent-representation-based $\rightarrow$ CTC Compression & --- | 27.1 | --- | 12.3 | 27.9 | 25.2 | 25.9 | 19.9 \\
&&& SALMONN \cite{tang-etal-2024-salmonn} & Latent-representation-based $\rightarrow$ Q-Former & 18.6 | --- | 33.1 | --- | --- | --- | --- | --- \\
&&& Qwen-Audio \cite{chu-2023-qwen-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & 25.1 | 33.9 | 41.5 | 15.7 | 39.7 | 38.5 | 36.0 | --- \\
&&& Qwen2-Audio \cite{chu-2024-qwen2-audio} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & \textbf{29.9} | 35.2 | \textbf{45.2} | 24.4 | 40.0 | 38.5 | 36.3 | --- \\
&&& LauraGPT \cite{du-etal-2024-lauragpt} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | --- | 38.5 | --- | --- | --- | --- | --- \\
&&& LLaST \cite{chen2024llast} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & --- | 41.2 | --- | 24.8 | \textbf{46.1} | \textbf{45.1} | \textbf{43.0} | \textbf{28.8} \\
&&& AudioPaLM \cite{rubenstein-etal-2023-audiopalm} & Audio-token-based $\rightarrow$ Semantic and Acoustic Tokens & --- | \textbf{43.4} | --- | \textbf{25.5} | 44.2 | 44.8 | --- | 25.9 \\
&&& Ideal-LLM \cite{xue2024ideal} & Latent-representation-based $\rightarrow$ Other Adaptation Methods & 25.9 | 38.5 | --- | --- | 41.5 | 40.0 | 38.0 | --- \\
\hline
\multirow{4}{*}{S2ST} & \multirow{4}{*}{\makecell[c]{\textbf{CVSS S2ST}\\ \small \textit{de-en} | \textit{zh-en} |\\ \small \textit{es-en} | \textit{fr-en} |\\ \small \textit{it-en} | \textit{ja-en}}} & \multirow{4}{*}{ASR-BLEU $\uparrow$} & Translatotron 2 + pretraining & & \\
&&& + TTS aug \cite{jia-etal-2022-s2st} & \multirow{-2}{*}{Non-LLM} & \multirow{-2}{*}{33.6 | 13.1 | 38.5 | 36.5 | 35.7 | 8.5} \\
&&& & & \\
&&& \multirow{-2}{*}{AudioPaLM \cite{rubenstein-etal-2023-audiopalm}} & \multirow{-2}{*}{Audio-token-based $\rightarrow$ Semantic and Acoustic Tokens} & \multirow{-2}{*}{\textbf{37.2} | \textbf{20.0} | \textbf{40.4} | \textbf{38.3} | \textbf{39.4} | \textbf{20.9}} \\
\hline
\multirow{18}{*}{TTS} & \multirow{3}{*}{\makecell[c]{\textbf{AISHELL-1}}} & \multirow{3}{*}{\makecell[c]{CER $\downarrow$ | SECS $\uparrow$ |\\ MOSNet $\uparrow$}} & VALL-E Phone \cite{wang-etal-2023-valle} & Non-LLM & \textbf{4.75} | \textbf{0.91} | \textbf{3.22} \\
&&& VALL-E Token \cite{wang-etal-2023-valle} & Non-LLM & 6.52 | \textbf{0.91} | 3.19 \\
&&& LauraGPT \cite{du-etal-2024-lauragpt} & Audio-token-based $\rightarrow$ Acoustic Tokens & 6.91 | 0.90 | 3.14 \\
\cline{2-6}
& \multirow{5}{*}{\makecell[c]{\textbf{LibriTTS}}} & \multirow{5}{*}{\makecell[c]{WER $\downarrow$ | SECS $\uparrow$ |\\ MOSNet $\uparrow$}} & VALL-E Phone \cite{wang-etal-2023-valle} & Non-LLM & \textbf{4.30} | 0.92 | 3.28 \\
&&& VALL-E Token \cite{wang-etal-2023-valle} & Non-LLM & 6.57 | \textbf{0.93} | 3.28 \\
&&& SpeechGPT-Gen (zero-shot) \cite{zhang-etal-2024-speechgptgen} & Audio-token-based $\rightarrow$ Semantic Token & 3.10 | 0.63 | 3.63 \\
&&& VoxtLM \cite{maiti-etal-2024-voxtlm} & Audio-token-based $\rightarrow$ Semantic Token & --- | --- | \textbf{4.36} \\
&&& LauraGPT \cite{du-etal-2024-lauragpt} & Audio-token-based $\rightarrow$ Acoustic Tokens & 8.62 | 0.91 | 3.26 \\
\cline{2-6}
& \multirow{5}{*}{\makecell[c]{\textbf{Topic-StoryCloze}}} & \multirow{5}{*}{\makecell[c]{TSC $\uparrow$}} & Spirit-LM \cite{nguyen-etal-2025-spiritlm} & Audio-token-based $\rightarrow$ Semantic Token & 82.9 \\
&&& TWIST-1.3B \cite{hassid-etal-2023-twist} & Audio-token-based $\rightarrow$ Semantic Token & 70.6 \\
&&& TWIST-7B \cite{hassid-etal-2023-twist} & Audio-token-based $\rightarrow$ Semantic Token & 74.1 \\
&&& TWIST-13B \cite{hassid-etal-2023-twist} & Audio-token-based $\rightarrow$ Semantic Token & 76.4 \\
&&& Moshi \cite{defossez-etal-2024-moshi} & Audio-token-based $\rightarrow$ Semantic and Acoustic Tokens & \textbf{83.6} \\
\cline{2-6}
& \multirow{5}{*}{\makecell[c]{\textbf{StoryCloze}}} & \multirow{5}{*}{\makecell[c]{SSC $\uparrow$}} & Spirit-LM \cite{nguyen-etal-2025-spiritlm} & Audio-token-based $\rightarrow$ Semantic Token & 61.0 \\
&&& TWIST-1.3B \cite{hassid-etal-2023-twist} & Audio-token-based $\rightarrow$ Semantic Token & 52.4 \\
&&& TWIST-7B \cite{hassid-etal-2023-twist} & Audio-token-based $\rightarrow$ Semantic Token & 55.3 \\
&&& TWIST-13B \cite{hassid-etal-2023-twist} & Audio-token-based $\rightarrow$ Semantic Token & 55.4 \\
&&& Moshi \cite{defossez-etal-2024-moshi} & Audio-token-based $\rightarrow$ Semantic and Acoustic Tokens & \textbf{62.7} \\
\bottomrule
\end{tabular}
}
\caption{Quantitative comparison based on applications. The \textbf{Bold} denotes the best result.}
\label{tab:benchmarks}
\end{table*}
\begin{thebibliography}{144}
\providecommand{\natexlab}[1]{#1}
\bibitem[{Baevski et~al.(2020)Baevski, Zhou, Mohamed, and Auli}]{baevski-etal-2020-wav2vec2}
Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, and Michael Auli. 2020.
\newblock wav2vec 2.0: A framework for self-supervised learning of speech representations.
\newblock In \emph{Thirty-fourth Conference on Neural Information Processing Systems}.
\bibitem[{Bahdanau et~al.(2015)Bahdanau, Cho, and Bengio}]{bahdanau-etal-2015}
Dzmitry Bahdanau, Kyunghyun Cho, and Yoshua Bengio. 2015.
\newblock Neural machine translation by jointly learning to align and translate.
\newblock In \emph{3rd International Conference on Learning Representations, {ICLR} 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings}.
\bibitem[{Bahl et~al.(1983)Bahl, Jelinek, and Mercer}]{bahl-etal-1983}
Lalit~R. Bahl, Frederick Jelinek, and Robert~L. Mercer. 1983.
\newblock \href {https://doi.org/10.1109/TPAMI.1983.4767370} {A maximum likelihood approach to continuous speech recognition}.
\newblock \emph{IEEE Transactions on Pattern Analysis and Machine Intelligence}, PAMI-5(2):179--190.
\bibitem[{Bai et~al.(2024{\natexlab{a}})Bai, Likhomanenko, Zhang, Gu, Aldeneh, and Jaitly}]{bai-etal-2024-dmel}
He~Bai, Tatiana Likhomanenko, Ruixiang Zhang, Zijin Gu, Zakaria Aldeneh, and Navdeep Jaitly. 2024{\natexlab{a}}.
\newblock \href {https://arxiv.org/abs/2407.15835} {dmel: Speech tokenization made simple}.
\newblock \emph{Preprint}, arXiv:2407.15835.
\bibitem[{Bai et~al.(2023)Bai, Bai, Chu, Cui, Dang, Deng, Fan, Ge, Han, Huang, Hui, Ji, Li, Lin, Lin, Liu, Liu, Lu, Lu, Ma, Men, Ren, Ren, Tan, Tan, Tu, Wang, Wang, Wang, Wu, Xu, Xu, Yang, Yang, Yang, Yang, Yao, Yu, Yuan, Yuan, Zhang, Zhang, Zhang, Zhang, Zhou, Zhou, Zhou, and Zhu}]{bai-2023-qwen}
Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu~Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An~Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou, and Tianhang Zhu. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2309.16609} {Qwen technical report}.
\newblock \emph{CoRR}, abs/2309.16609.
\bibitem[{Bai et~al.(2024{\natexlab{b}})Bai, Chen, Chen, Chen, Chen, Ding, Dong, Dong, Du, Gao et~al.}]{bai2024seed}
Ye~Bai, Jingping Chen, Jitong Chen, Wei Chen, Zhuo Chen, Chuang Ding, Linhao Dong, Qianqian Dong, Yujiao Du, Kepan Gao, et~al. 2024{\natexlab{b}}.
\newblock Seed-asr: Understanding diverse speech and contexts with llm-based speech recognition.
\newblock \emph{arXiv preprint arXiv:2407.04675}.
\bibitem[{Bengio et~al.(2000)Bengio, Ducharme, and Vincent}]{bengio-etal-2000}
Yoshua Bengio, R\'{e}jean Ducharme, and Pascal Vincent. 2000.
\newblock \href {https://proceedings.neurips.cc/paper_files/paper/2000/file/728f206c2a01bf572b5940d7d9a8fa4c-Paper.pdf} {A neural probabilistic language model}.
\newblock In \emph{Advances in Neural Information Processing Systems}, volume~13. MIT Press.
\bibitem[{Borsos et~al.(2023)Borsos, Marinier, Vincent, Kharitonov, Pietquin, Sharifi, Roblek, Teboul, Grangier, Tagliasacchi, and Zeghidour}]{borsos-etal-2023-audiolm}
Zal\'{a}n Borsos, Rapha\"{e}l Marinier, Damien Vincent, Eugene Kharitonov, Olivier Pietquin, Matt Sharifi, Dominik Roblek, Olivier Teboul, David Grangier, Marco Tagliasacchi, and Neil Zeghidour. 2023.
\newblock \href {https://doi.org/10.1109/TASLP.2023.3288409} {Audiolm: A language modeling approach to audio generation}.
\newblock \emph{IEEE/ACM Trans. Audio, Speech and Lang. Proc.}, 31:2523–2533.
\bibitem[{Brown et~al.(1992)Brown, Della~Pietra, deSouza, Lai, and Mercer}]{brown-etal-1992-ngram}
Peter~F. Brown, Vincent~J. Della~Pietra, Peter~V. deSouza, Jenifer~C. Lai, and Robert~L. Mercer. 1992.
\newblock \href {https://aclanthology.org/J92-4003} {Class-based \textit{n}-gram models of natural language}.
\newblock \emph{Computational Linguistics}, 18(4):467--480.
\bibitem[{Brown et~al.(2020)Brown, Mann, Ryder, Subbiah, Kaplan, Dhariwal, Neelakantan, Shyam, Sastry, Askell, Agarwal, Herbert-Voss, Krueger, Henighan, Child, Ramesh, Ziegler, Wu, Winter, Hesse, Chen, Sigler, Litwin, Gray, Chess, Clark, Berner, McCandlish, Radford, Sutskever, and Amodei}]{brown-etal-2020}
Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared~D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, Sandhini Agarwal, Ariel Herbert-Voss, Gretchen Krueger, Tom Henighan, Rewon Child, Aditya Ramesh, Daniel Ziegler, Jeffrey Wu, Clemens Winter, Chris Hesse, Mark Chen, Eric Sigler, Mateusz Litwin, Scott Gray, Benjamin Chess, Jack Clark, Christopher Berner, Sam McCandlish, Alec Radford, Ilya Sutskever, and Dario Amodei. 2020.
\newblock \href {https://proceedings.neurips.cc/paper_files/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf} {Language models are few-shot learners}.
\newblock In \emph{Advances in Neural Information Processing Systems}, volume~33, pages 1877--1901. Curran Associates, Inc.
\bibitem[{Chai et~al.(2024)Chai, Fang, Peng, and Li}]{chai-etal-2024-tokenization}
Yekun Chai, Yewei Fang, Qiwei Peng, and Xuhong Li. 2024.
\newblock \href {https://doi.org/10.18653/v1/2024.findings-emnlp.86} {Tokenization falling short: On subword robustness in large language models}.
\newblock In \emph{Findings of the Association for Computational Linguistics: EMNLP 2024}, pages 1582--1599, Miami, Florida, USA. Association for Computational Linguistics.
\bibitem[{Chen et~al.(2023{\natexlab{a}})Chen, Hu, Yang, Siniscalchi, Chen, and Siong}]{chen-2023-hyporadise}
Chen Chen, Yuchen Hu, Chao{-}Han~Huck Yang, Sabato~Marco Siniscalchi, Pin{-}Yu Chen, and Chng~Eng Siong. 2023{\natexlab{a}}.
\newblock \href {http://papers.nips.cc/paper\_files/paper/2023/hash/6492267465a7ac507be1f9fd1174e78d-Abstract-Datasets\_and\_Benchmarks.html} {Hyporadise: An open baseline for generative speech recognition with large language models}.
\newblock In \emph{Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023}.
\bibitem[{Chen et~al.(2024{\natexlab{a}})Chen, Li, Hu, Siniscalchi, Chen, Chng, and Yang}]{chen2024s}
Chen Chen, Ruizhe Li, Yuchen Hu, Sabato~Marco Siniscalchi, Pin-Yu Chen, Ensiong Chng, and Chao-Han~Huck Yang. 2024{\natexlab{a}}.
\newblock It's never too late: Fusing acoustic information into large language models for automatic speech recognition.
\newblock \emph{arXiv preprint arXiv:2402.05457}.
\bibitem[{Chen et~al.(2023{\natexlab{b}})Chen, Han, Zhao, Zhang, Shi, Xu, and Xu}]{chen-2023-xllm}
Feilong Chen, Minglun Han, Haozhi Zhao, Qingyang Zhang, Jing Shi, Shuang Xu, and Bo~Xu. 2023{\natexlab{b}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2305.04160} {{X-LLM:} bootstrapping advanced large language models by treating multi-modalities as foreign languages}.
\newblock \emph{CoRR}, abs/2305.04160.
\bibitem[{Chen et~al.(2024{\natexlab{b}})Chen, Gou, Huang, Liu, Tan, Xu, Wang, Zhu, Zeng, Yang et~al.}]{chen2024emova}
Kai Chen, Yunhao Gou, Runhui Huang, Zhili Liu, Daxin Tan, Jing Xu, Chunwei Wang, Yi~Zhu, Yihan Zeng, Kuo Yang, et~al. 2024{\natexlab{b}}.
\newblock Emova: Empowering language models to see, hear and speak with vivid emotions.
\newblock \emph{arXiv preprint arXiv:2409.18042}.
\bibitem[{Chen et~al.(2024{\natexlab{c}})Chen, Liu, Zhou, Liu, Tan, Li, Zhao, Qian, and Wei}]{chen-etal-2024-valle2}
Sanyuan Chen, Shujie Liu, Long Zhou, Yanqing Liu, Xu~Tan, Jinyu Li, Sheng Zhao, Yao Qian, and Furu Wei. 2024{\natexlab{c}}.
\newblock \href {https://arxiv.org/abs/2406.05370} {Vall-e 2: Neural codec language models are human parity zero-shot text to speech synthesizers}.
\newblock \emph{Preprint}, arXiv:2406.05370.
\bibitem[{Chen et~al.(2022)Chen, Wang, Chen, Wu, Liu, Chen, Li, Kanda, Yoshioka, Xiao, Wu, Zhou, Ren, Qian, Qian, Wu, Zeng, Yu, and Wei}]{chen-etal-2022-wavlm}
Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu~Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Xiangzhan Yu, and Furu Wei. 2022.
\newblock \href {https://doi.org/10.1109/JSTSP.2022.3188113} {Wavlm: Large-scale self-supervised pre-training for full stack speech processing}.
\newblock \emph{IEEE Journal of Selected Topics in Signal Processing}, 16(6):1505--1518.
\bibitem[{Chen et~al.(2023{\natexlab{c}})Chen, Allauzen, Huang, Park, Rybach, Huang, Cabrera, Audhkhasi, Ramabhadran, Moreno, and Riley}]{chen-2023-longform}
Tongzhou Chen, Cyril Allauzen, Yinghui Huang, Daniel~S. Park, David Rybach, W.~Ronny Huang, Rodrigo Cabrera, Kartik Audhkhasi, Bhuvana Ramabhadran, Pedro~J. Moreno, and Michael Riley. 2023{\natexlab{c}}.
\newblock \href {https://doi.org/10.1109/ICASSP49357.2023.10096429} {Large-scale language model rescoring on long-form data}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing {ICASSP} 2023, Rhodes Island, Greece, June 4-10, 2023}, pages 1--5. {IEEE}.
\bibitem[{Chen et~al.(2024{\natexlab{d}})Chen, Zhang, Bai, Chen, and Nakamura}]{chen2024llast}
Xi~Chen, Songyang Zhang, Qibing Bai, Kai Chen, and Satoshi Nakamura. 2024{\natexlab{d}}.
\newblock Llast: Improved end-to-end speech translation system leveraged by large language models.
\newblock \emph{arXiv preprint arXiv:2407.15415}.
\bibitem[{Chen et~al.(2024{\natexlab{e}})Chen, Huang, Andrusenko, Hrinchuk, Puvvada, Li, Ghosh, Balam, and Ginsburg}]{chen-2024-salm}
Zhehuai Chen, He~Huang, Andrei Andrusenko, Oleksii Hrinchuk, Krishna~C. Puvvada, Jason Li, Subhankar Ghosh, Jagadeesh Balam, and Boris Ginsburg. 2024{\natexlab{e}}.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10447553} {{SALM:} speech-augmented language model with in-context learning for speech recognition and translation}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2024, Seoul, Republic of Korea, April 14-19, 2024}, pages 13521--13525. {IEEE}.
\bibitem[{Chen et~al.(2024{\natexlab{f}})Chen, Huang, Hrinchuk, Puvvada, Koluguri, {\.Z}elasko, Balam, and Ginsburg}]{chen2024bestow}
Zhehuai Chen, He~Huang, Oleksii Hrinchuk, Krishna~C Puvvada, Nithin~Rao Koluguri, Piotr {\.Z}elasko, Jagadeesh Balam, and Boris Ginsburg. 2024{\natexlab{f}}.
\newblock Bestow: Efficient and streamable speech language model with the best of two worlds in gpt and t5.
\newblock In \emph{2024 IEEE Spoken Language Technology Workshop (SLT)}, pages 147--154. IEEE.
\bibitem[{Cheng et~al.(2024)Cheng, Futami, Kashiwagi, Tsunoo, Teo, Arora, and Watanabe}]{cheng2024task}
Yao-Fei Cheng, Hayato Futami, Yosuke Kashiwagi, Emiru Tsunoo, Wen~Shen Teo, Siddhant Arora, and Shinji Watanabe. 2024.
\newblock Task arithmetic for language expansion in speech translation.
\newblock \emph{arXiv preprint arXiv:2409.11274}.
\bibitem[{Choi et~al.(2024)Choi, Pasad, Nakamura, Fukayama, Livescu, and Watanabe}]{choi-etal-2024}
Kwanghee Choi, Ankita Pasad, Tomohiko Nakamura, Satoru Fukayama, Karen Livescu, and Shinji Watanabe. 2024.
\newblock \href {https://doi.org/10.21437/Interspeech.2024-1157} {Self-supervised speech representations are more phonetic than semantic}.
\newblock In \emph{Interspeech 2024}, pages 4578--4582.
\bibitem[{Chowdhery et~al.(2023)Chowdhery, Narang, Devlin, Bosma, Mishra, Roberts, Barham, Chung, Sutton, Gehrmann, Schuh, Shi, Tsvyashchenko, Maynez, Rao, Barnes, Tay, Shazeer, Prabhakaran, Reif, Du, Hutchinson, Pope, Bradbury, Austin, Isard, Gur{-}Ari, Yin, Duke, Levskaya, Ghemawat, Dev, Michalewski, Garcia, Misra, Robinson, Fedus, Zhou, Ippolito, Luan, Lim, Zoph, Spiridonov, Sepassi, Dohan, Agrawal, Omernick, Dai, Pillai, Pellat, Lewkowycz, Moreira, Child, Polozov, Lee, Zhou, Wang, Saeta, Diaz, Firat, Catasta, Wei, Meier{-}Hellstern, Eck, Dean, Petrov, and Fiedel}]{chowdhery-etal-2023-palm}
Aakanksha Chowdhery, Sharan Narang, Jacob Devlin, Maarten Bosma, Gaurav Mishra, Adam Roberts, Paul Barham, Hyung~Won Chung, Charles Sutton, Sebastian Gehrmann, Parker Schuh, Kensen Shi, Sasha Tsvyashchenko, Joshua Maynez, Abhishek Rao, Parker Barnes, Yi~Tay, Noam Shazeer, Vinodkumar Prabhakaran, Emily Reif, Nan Du, Ben Hutchinson, Reiner Pope, James Bradbury, Jacob Austin, Michael Isard, Guy Gur{-}Ari, Pengcheng Yin, Toju Duke, Anselm Levskaya, Sanjay Ghemawat, Sunipa Dev, Henryk Michalewski, Xavier Garcia, Vedant Misra, Kevin Robinson, Liam Fedus, Denny Zhou, Daphne Ippolito, David Luan, Hyeontaek Lim, Barret Zoph, Alexander Spiridonov, Ryan Sepassi, David Dohan, Shivani Agrawal, Mark Omernick, Andrew~M. Dai, Thanumalayan~Sankaranarayana Pillai, Marie Pellat, Aitor Lewkowycz, Erica Moreira, Rewon Child, Oleksandr Polozov, Katherine Lee, Zongwei Zhou, Xuezhi Wang, Brennan Saeta, Mark Diaz, Orhan Firat, Michele Catasta, Jason Wei, Kathy Meier{-}Hellstern, Douglas Eck, Jeff Dean, Slav Petrov, and Noah Fiedel.
2023.
\newblock \href {https://jmlr.org/papers/v24/22-1144.html} {Palm: Scaling language modeling with pathways}.
\newblock \emph{J. Mach. Learn. Res.}, 24:240:1--240:113.
\bibitem[{Chu et~al.(2024)Chu, Xu, Yang, Wei, Wei, Guo, Leng, Lv, He, Lin, Zhou, and Zhou}]{chu-2024-qwen2-audio}
Yunfei Chu, Jin Xu, Qian Yang, Haojie Wei, Xipin Wei, Zhifang Guo, Yichong Leng, Yuanjun Lv, Jinzheng He, Junyang Lin, Chang Zhou, and Jingren Zhou. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2407.10759} {Qwen2-audio technical report}.
\newblock \emph{CoRR}, abs/2407.10759.
\bibitem[{Chu et~al.(2023)Chu, Xu, Zhou, Yang, Zhang, Yan, Zhou, and Zhou}]{chu-2023-qwen-audio}
Yunfei Chu, Jin Xu, Xiaohuan Zhou, Qian Yang, Shiliang Zhang, Zhijie Yan, Chang Zhou, and Jingren Zhou. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2311.07919} {Qwen-audio: Advancing universal audio understanding via unified large-scale audio-language models}.
\newblock \emph{CoRR}, abs/2311.07919.
\bibitem[{Cui et~al.(2025)Cui, Yu, Jiao, Meng, Zhang, Wang, Guo, and King}]{cui-etal-2025-speechlmsurvey}
Wenqian Cui, Dianzhi Yu, Xiaoqi Jiao, Ziqiao Meng, Guangyan Zhang, Qichao Wang, Yiwen Guo, and Irwin King. 2025.
\newblock \href {https://arxiv.org/abs/2410.03751} {Recent advances in speech language models: A survey}.
\newblock \emph{Preprint}, arXiv:2410.03751.
\bibitem[{Das et~al.(2024)Das, Dingliwal, Ronanki, Paturi, Huang, Mathur, Yuan, Bekal, Niu, Jayanthi et~al.}]{das2024speechverse}
Nilaksh Das, Saket Dingliwal, Srikanth Ronanki, Rohit Paturi, Zhaocheng Huang, Prashant Mathur, Jie Yuan, Dhanush Bekal, Xing Niu, Sai~Muralidhar Jayanthi, et~al. 2024.
\newblock Speechverse: A large-scale generalizable audio language model.
\newblock \emph{arXiv preprint arXiv:2405.08295}.
\bibitem[{DeepSeek-AI(2024)}]{deepseekai-2024-deepseek}
DeepSeek-AI. 2024.
\newblock \href {https://arxiv.org/abs/2401.02954} {Deepseek llm: Scaling open-source language models with longtermism}.
\newblock \emph{Preprint}, arXiv:2401.02954.
\bibitem[{D{\'e}fossez et~al.(2023)D{\'e}fossez, Copet, Synnaeve, and Adi}]{defossez-etal-2023-encodec}
Alexandre D{\'e}fossez, Jade Copet, Gabriel Synnaeve, and Yossi Adi. 2023.
\newblock \href {https://openreview.net/forum?id=ivCd8z8zR2} {High fidelity neural audio compression}.
\newblock \emph{Transactions on Machine Learning Research}.
\newblock Featured Certification, Reproducibility Certification.
\bibitem[{Denisov and Vu(2024{\natexlab{a}})}]{denisov2024teaching}
Pavel Denisov and Thang Vu. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.18653/V1/2024.FINDINGS-NAACL.52} {Teaching a multilingual large language model to understand multilingual speech via multi-instructional training}.
\newblock In \emph{Findings of the Association for Computational Linguistics: {NAACL} 2024, Mexico City, Mexico, June 16-21, 2024}, pages 814--834. Association for Computational Linguistics.
\bibitem[{Denisov and Vu(2024{\natexlab{b}})}]{denisov-2024-multilingual}
Pavel Denisov and Thang Vu. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.18653/V1/2024.FINDINGS-NAACL.52} {Teaching a multilingual large language model to understand multilingual speech via multi-instructional training}.
\newblock In \emph{Findings of the Association for Computational Linguistics: {NAACL} 2024, Mexico City, Mexico, June 16-21, 2024}, pages 814--834. Association for Computational Linguistics.
\bibitem[{Dighe et~al.(2024{\natexlab{a}})Dighe, Su, Zheng, Liu, Garg, Niu, and Tewfik}]{dighe2024leveraging}
Pranay Dighe, Yi~Su, Shangshang Zheng, Yunshu Liu, Vineet Garg, Xiaochuan Niu, and Ahmed Tewfik. 2024{\natexlab{a}}.
\newblock Leveraging large language models for exploiting asr uncertainty.
\newblock In \emph{ICASSP 2024-2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, pages 12231--12235. IEEE.
\bibitem[{Dighe et~al.(2024{\natexlab{b}})Dighe, Su, Zheng, Liu, Garg, Niu, and Tewfik}]{dighe-2024-uncertainty}
Pranay Dighe, Yi~Su, Shangshang Zheng, Yunshu Liu, Vineet Garg, Xiaochuan Niu, and Ahmed~H. Tewfik. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10446132} {Leveraging large language models for exploiting {ASR} uncertainty}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2024, Seoul, Republic of Korea, April 14-19, 2024}, pages 12231--12235. {IEEE}.
\bibitem[{Du et~al.(2024{\natexlab{a}})Du, Ma, Yang, Deng, Chen, Yang, Xiang, Liu, and Qin}]{du2024cot}
Yexing Du, Ziyang Ma, Yifan Yang, Keqi Deng, Xie Chen, Bo~Yang, Yang Xiang, Ming Liu, and Bing Qin. 2024{\natexlab{a}}.
\newblock Cot-st: Enhancing llm-based speech translation with multimodal chain-of-thought.
\newblock \emph{arXiv preprint arXiv:2409.19510}.
\bibitem[{Du et~al.(2024{\natexlab{b}})Du, Wang, Chen, Chu, Gao, Li, Hu, Zhou, Xu, Ma, Wang, Zheng, Zhou, Yan, and Zhang}]{du-etal-2024-lauragpt}
Zhihao Du, Jiaming Wang, Qian Chen, Yunfei Chu, Zhifu Gao, Zerui Li, Kai Hu, Xiaohuan Zhou, Jin Xu, Ziyang Ma, Wen Wang, Siqi Zheng, Chang Zhou, Zhijie Yan, and Shiliang Zhang. 2024{\natexlab{b}}.
\newblock \href {https://arxiv.org/abs/2310.04673} {Lauragpt: Listen, attend, understand, and regenerate audio with gpt}.
\newblock \emph{Preprint}, arXiv:2310.04673.
\bibitem[{Défossez et~al.(2024)Défossez, Mazaré, Orsini, Royer, Pérez, Jégou, Grave, and Zeghidour}]{defossez-etal-2024-moshi}
Alexandre Défossez, Laurent Mazaré, Manu Orsini, Amélie Royer, Patrick Pérez, Hervé Jégou, Edouard Grave, and Neil Zeghidour. 2024.
\newblock \href {https://arxiv.org/abs/2410.00037} {Moshi: a speech-text foundation model for real-time dialogue}.
\newblock \emph{Preprint}, arXiv:2410.00037.
\bibitem[{Fang et~al.(2024)Fang, Guo, Zhou, Ma, Zhang, and Feng}]{fang-etal-2024-llamaomni}
Qingkai Fang, Shoutao Guo, Yan Zhou, Zhengrui Ma, Shaolei Zhang, and Yang Feng. 2024.
\newblock Llama-omni: Seamless speech interaction with large language models.
\newblock \emph{arXiv preprint arXiv:2409.06666}.
\bibitem[{Fathullah et~al.(2024)Fathullah, Wu, Lakomkin, Jia, Shangguan, Li, Guo, Xiong, Mahadeokar, Kalinli, Fuegen, and Seltzer}]{fathullah-2024-prompting}
Yassir Fathullah, Chunyang Wu, Egor Lakomkin, Junteng Jia, Yuan Shangguan, Ke~Li, Jinxi Guo, Wenhan Xiong, Jay Mahadeokar, Ozlem Kalinli, Christian Fuegen, and Mike Seltzer. 2024.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10447605} {Prompting large language models with speech recognition abilities}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2024, Seoul, Republic of Korea, April 14-19, 2024}, pages 13351--13355. {IEEE}.
\bibitem[{Fathullah et~al.(2023)Fathullah, Wu, Lakomkin, Jia, Shangguan, Mahadeokar, Kalinli, Fuegen, and Seltzer}]{fathullah2023towards}
Yassir Fathullah, Chunyang Wu, Egor Lakomkin, Junteng Jia, Yuan Shangguan, Jay Mahadeokar, Ozlem Kalinli, Christian Fuegen, and Mike Seltzer. 2023.
\newblock Towards general-purpose speech abilities for large language models using unpaired data.
\newblock \emph{arXiv preprint arXiv:2311.06753}.
\bibitem[{Fedus et~al.(2022)Fedus, Dean, and Zoph}]{fedus-etal-2022}
William Fedus, Jeff Dean, and Barret Zoph. 2022.
\newblock \href {https://arxiv.org/abs/2209.01667} {A review of sparse expert models in deep learning}.
\newblock \emph{Preprint}, arXiv:2209.01667.
\bibitem[{Gat et~al.(2023)Gat, Kreuk, Anh~Nguyen, Lee, Copet, Synnaeve, Dupoux, and Adi}]{gat-etal-2023-discreteslm}
Itai Gat, Felix Kreuk, Tu~Anh~Nguyen, Ann Lee, Jade Copet, Gabriel Synnaeve, Emmanuel Dupoux, and Yossi Adi. 2023.
\newblock \href {https://doi.org/10.18653/v1/2023.iwslt-1.46} {Augmentation invariant discrete representation for generative spoken language modeling}.
\newblock In \emph{Proceedings of the 20th International Conference on Spoken Language Translation (IWSLT 2023)}, pages 465--477, Toronto, Canada (in-person and online). Association for Computational Linguistics.
\bibitem[{Ghosh et~al.(2024{\natexlab{a}})Ghosh, Acharya, Saha, Jain, and Chadha}]{ghosh-2024-vision-language-model}
Akash Ghosh, Arkadeep Acharya, Sriparna Saha, Vinija Jain, and Aman Chadha. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2404.07214} {Exploring the frontier of vision-language models: {A} survey of current methodologies and future directions}.
\newblock \emph{CoRR}, abs/2404.07214.
\bibitem[{Ghosh et~al.(2024{\natexlab{b}})Ghosh, Kumar, Seth, Chiniya, Tyagi, Duraiswami, and Manocha}]{ghosh2024lipger}
Sreyan Ghosh, Sonal Kumar, Ashish Seth, Purva Chiniya, Utkarsh Tyagi, Ramani Duraiswami, and Dinesh Manocha. 2024{\natexlab{b}}.
\newblock Lipger: Visually-conditioned generative error correction for robust automatic speech recognition.
\newblock \emph{arXiv preprint arXiv:2406.04432}.
\bibitem[{Gong et~al.(2024)Gong, Luo, Liu, Karlinsky, and Glass}]{gong-2024-ltu}
Yuan Gong, Hongyin Luo, Alexander~H. Liu, Leonid Karlinsky, and James~R. Glass. 2024.
\newblock \href {https://openreview.net/forum?id=nBZBPXdJlC} {Listen, think, and understand}.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net.
\bibitem[{Google(2024)}]{geminiteam-2024-gemini}
Gemini~Team Google. 2024.
\newblock \href {https://arxiv.org/abs/2312.11805} {Gemini: A family of highly capable multimodal models}.
\newblock \emph{Preprint}, arXiv:2312.11805.
\bibitem[{Graves et~al.(2013)Graves, Mohamed, and Hinton}]{graves-2013-ctc}
Alex Graves, Abdel{-}rahman Mohamed, and Geoffrey~E. Hinton. 2013.
\newblock \href {https://doi.org/10.1109/ICASSP.2013.6638947} {Speech recognition with deep recurrent neural networks}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2013, Vancouver, BC, Canada, May 26-31, 2013}, pages 6645--6649. {IEEE}.
\bibitem[{Gulati et~al.(2020)Gulati, Qin, Chiu, Parmar, Zhang, Yu, Han, Wang, Zhang, Wu, and Pang}]{gulati-2020-conformer}
Anmol Gulati, James Qin, Chung{-}Cheng Chiu, Niki Parmar, Yu~Zhang, Jiahui Yu, Wei Han, Shibo Wang, Zhengdong Zhang, Yonghui Wu, and Ruoming Pang. 2020.
\newblock \href {https://doi.org/10.21437/INTERSPEECH.2020-3015} {Conformer: Convolution-augmented transformer for speech recognition}.
\newblock In \emph{21st Annual Conference of the International Speech Communication Association, Interspeech 2020, Virtual Event, Shanghai, China, October 25-29, 2020}, pages 5036--5040. {ISCA}.
\bibitem[{Hao et~al.(2024)Hao, Zhou, Liu, Li, Hu, Wang, and Wei}]{hao-2024-boosting}
Hongkun Hao, Long Zhou, Shujie Liu, Jinyu Li, Shujie Hu, Rui Wang, and Furu Wei. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2401.00246} {Boosting large language model for speech synthesis: An empirical study}.
\newblock \emph{CoRR}, abs/2401.00246.
\bibitem[{Hassid et~al.(2023)Hassid, Remez, Nguyen, Gat, Conneau, Kreuk, Copet, D{\'e}fossez, Synnaeve, Dupoux, Schwartz, and Adi}]{hassid-etal-2023-twist}
Michael Hassid, Tal Remez, Tu~Anh Nguyen, Itai Gat, Alexis Conneau, Felix Kreuk, Jade Copet, Alexandre D{\'e}fossez, Gabriel Synnaeve, Emmanuel Dupoux, Roy Schwartz, and Yossi Adi. 2023.
\newblock \href {https://openreview.net/forum?id=UlHueVjAKr} {Textually pretrained speech language models}.
\newblock In \emph{Thirty-seventh Conference on Neural Information Processing Systems}.
\bibitem[{Hono et~al.(2023)Hono, Mitsuda, Zhao, Mitsui, Wakatsuki, and Sawada}]{hono-2023-integration}
Yukiya Hono, Koh Mitsuda, Tianyu Zhao, Kentaro Mitsui, Toshiaki Wakatsuki, and Kei Sawada. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2312.03668} {An integration of pre-trained speech and language models for end-to-end speech recognition}.
\newblock \emph{CoRR}, abs/2312.03668.
\bibitem[{Hsu et~al.(2021)Hsu, Bolte, Tsai, Lakhotia, Salakhutdinov, and Mohamed}]{hsu-etal-2021-hubert}
Wei-Ning Hsu, Benjamin Bolte, Yao-Hung~Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, and Abdelrahman Mohamed. 2021.
\newblock \href {https://doi.org/10.1109/TASLP.2021.3122291} {Hubert: Self-supervised speech representation learning by masked prediction of hidden units}.
\newblock \emph{IEEE/ACM Trans. Audio, Speech and Lang. Proc.}, 29:3451–3460.
\bibitem[{Hu et~al.(2023)Hu, Sainath, Li, Du, Huang, Dai, Zhang, Cabrera, Chen, and Strohman}]{hu-2023-shallow-fusion}
Ke~Hu, Tara~N. Sainath, Bo~Li, Nan Du, Yanping Huang, Andrew~M. Dai, Yu~Zhang, Rodrigo Cabrera, Zhifeng Chen, and Trevor Strohman. 2023.
\newblock \href {https://doi.org/10.1109/ICASSP49357.2023.10094796} {Massively multilingual shallow fusion with large language models}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing {ICASSP} 2023, Rhodes Island, Greece, June 4-10, 2023}, pages 1--5. {IEEE}.
\bibitem[{Hu et~al.(2024{\natexlab{a}})Hu, Zhou, Liu, Chen, Hao, Pan, Liu, Li, Sivasankaran, Liu, and Wei}]{hu-2024-wavllm}
Shujie Hu, Long Zhou, Shujie Liu, Sanyuan Chen, Hongkun Hao, Jing Pan, Xunying Liu, Jinyu Li, Sunit Sivasankaran, Linquan Liu, and Furu Wei. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2404.00656} {Wavllm: Towards robust and adaptive speech large language model}.
\newblock \emph{CoRR}, abs/2404.00656.
\bibitem[{Hu et~al.(2024{\natexlab{b}})Hu, Chen, Yang, Li, Zhang, Chen, and Chng}]{hu-2024-noise-robust}
Yuchen Hu, Chen Chen, Chao{-}Han~Huck Yang, Ruizhe Li, Chao Zhang, Pin{-}Yu Chen, and Engsiong Chng. 2024{\natexlab{b}}.
\newblock \href {https://openreview.net/forum?id=ceATjGPTUD} {Large language models are efficient learners of noise-robust speech recognition}.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net.
\bibitem[{Hu et~al.(2024{\natexlab{c}})Hu, Chen, Yang, Li, Zhang, Chen, and Chng}]{hu-etal-2024-gentranslate}
Yuchen Hu, Chen Chen, Chao{-}Han~Huck Yang, Ruizhe Li, Dong Zhang, Zhehuai Chen, and EngSiong Chng. 2024{\natexlab{c}}.
\newblock \href {https://aclanthology.org/2024.acl-long.5} {Gentranslate: Large language models are generative multilingual speech and machine translators}.
\newblock In \emph{Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2024, Bangkok, Thailand, August 11-16, 2024}, pages 74--90. Association for Computational Linguistics.
\bibitem[{Huang et~al.(2024{\natexlab{a}})Huang, Li, Yang, Shi, Chang, Ye, Wu, Hong, Huang, Liu, Ren, Zou, Zhao, and Watanabe}]{huang-2024-audio-gpt}
Rongjie Huang, Mingze Li, Dongchao Yang, Jiatong Shi, Xuankai Chang, Zhenhui Ye, Yuning Wu, Zhiqing Hong, Jiawei Huang, Jinglin Liu, Yi~Ren, Yuexian Zou, Zhou Zhao, and Shinji Watanabe. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.1609/AAAI.V38I21.30570} {Audiogpt: Understanding and generating speech, music, sound, and talking head}.
\newblock In \emph{Thirty-Eighth {AAAI} Conference on Artificial Intelligence, {AAAI} 2024, Thirty-Sixth Conference on Innovative Applications of Artificial Intelligence, {IAAI} 2024, Fourteenth Symposium on Educational Advances in Artificial Intelligence, {EAAI} 2014, February 20-27, 2024, Vancouver, Canada}, pages 23802--23804. {AAAI} Press.
\bibitem[{Huang et~al.(2024{\natexlab{b}})Huang, Allauzen, Chen, Gupta, Hu, Qin, Zhang, Wang, Chang, and Sainath}]{huang-2024-fusion-comprehensive}
W.~Ronny Huang, Cyril Allauzen, Tongzhou Chen, Kilol Gupta, Ke~Hu, James Qin, Yu~Zhang, Yongqiang Wang, Shuo{-}Yiin Chang, and Tara~N. Sainath. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10448361} {Multilingual and fully non-autoregressive {ASR} with large language model fusion: {A} comprehensive study}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2024, Seoul, Republic of Korea, April 14-19, 2024}, pages 13306--13310. {IEEE}.
\bibitem[{ISO(1993)}]{iso11172-3}
ISO. 1993.
\newblock {ISO/IEC 11172-3:1993} information technology — coding of moving pictures and associated audio for digital storage media at up to about 1,5 mbit/s — part 3: Audio.
\bibitem[{Jia et~al.(2022)Jia, Ding, Bapna, Cherry, Zhang, Conneau, and Morioka}]{jia-etal-2022-s2st}
Ye~Jia, Yifan Ding, Ankur Bapna, Colin Cherry, Yu~Zhang, Alexis Conneau, and Nobu Morioka. 2022.
\newblock \href {https://doi.org/10.21437/Interspeech.2022-10938} {Leveraging unsupervised and weakly-supervised data to improve direct speech-to-speech translation}.
\newblock In \emph{Interspeech 2022}, pages 1721--1725.
\bibitem[{Jiang et~al.(2024)Jiang, Sablayrolles, Roux, Mensch, Savary, Bamford, Chaplot, de~las Casas, Hanna, Bressand, Lengyel, Bour, Lample, Lavaud, Saulnier, Lachaux, Stock, Subramanian, Yang, Antoniak, Scao, Gervet, Lavril, Wang, Lacroix, and Sayed}]{jiang-etal-2024-mixtral}
Albert~Q. Jiang, Alexandre Sablayrolles, Antoine Roux, Arthur Mensch, Blanche Savary, Chris Bamford, Devendra~Singh Chaplot, Diego de~las Casas, Emma~Bou Hanna, Florian Bressand, Gianna Lengyel, Guillaume Bour, Guillaume Lample, Lélio~Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Sandeep Subramanian, Sophia Yang, Szymon Antoniak, Teven~Le Scao, Théophile Gervet, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William~El Sayed. 2024.
\newblock \href {https://arxiv.org/abs/2401.04088} {Mixtral of experts}.
\newblock \emph{Preprint}, arXiv:2401.04088.
\bibitem[{Kang and Roy(2024)}]{kang2024prompting}
Wonjune Kang and Deb Roy. 2024.
\newblock Prompting large language models with audio for general-purpose speech summarization.
\newblock \emph{arXiv preprint arXiv:2406.05968}.
\bibitem[{Kharitonov et~al.(2022)Kharitonov, Lee, Polyak, Adi, Copet, Lakhotia, Nguyen, Riviere, Mohamed, Dupoux, and Hsu}]{kharitonov-etal-2022-pgslm}
Eugene Kharitonov, Ann Lee, Adam Polyak, Yossi Adi, Jade Copet, Kushal Lakhotia, Tu~Anh Nguyen, Morgane Riviere, Abdelrahman Mohamed, Emmanuel Dupoux, and Wei-Ning Hsu. 2022.
\newblock \href {https://doi.org/10.18653/v1/2022.acl-long.593} {Text-free prosody-aware generative spoken language modeling}.
\newblock In \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, pages 8666--8681, Dublin, Ireland. Association for Computational Linguistics.
\bibitem[{Ko et~al.(2024)Ko, Li, Yang, and Kawahara}]{ko2024benchmarking}
Yuka Ko, Sheng Li, Chao-Han~Huck Yang, and Tatsuya Kawahara. 2024.
\newblock Benchmarking japanese speech recognition on asr-llm setups with multi-pass augmented generative error correction.
\newblock \emph{arXiv preprint arXiv:2408.16180}.
\bibitem[{Kumar et~al.(2023)Kumar, Seetharaman, Luebs, Kumar, and Kumar}]{kumar-etal-2023-dac}
Rithesh Kumar, Prem Seetharaman, Alejandro Luebs, Ishaan Kumar, and Kundan Kumar. 2023.
\newblock \href {https://openreview.net/forum?id=qjnl1QUnFA} {High-fidelity audio compression with improved {RVQGAN}}.
\newblock In \emph{Thirty-seventh Conference on Neural Information Processing Systems}.
\bibitem[{Labiausse et~al.(2025)Labiausse, Mazaré, Grave, Pérez, Défossez, and Zeghidour}]{labiausse-etal-2025-hibiki}
Tom Labiausse, Laurent Mazaré, Edouard Grave, Patrick Pérez, Alexandre Défossez, and Neil Zeghidour. 2025.
\newblock \href {https://arxiv.org/abs/2502.03382} {High-fidelity simultaneous speech-to-speech translation}.
\newblock \emph{Preprint}, arXiv:2502.03382.
\bibitem[{{\L}ajszczak et~al.(2024){\L}ajszczak, C{\'a}mbara, Li, Beyhan, van Korlaar, Yang, Joly, Mart{\'\i}n-Cortinas, Abbas, Michalski et~al.}]{lajszczak2024base}
Mateusz {\L}ajszczak, Guillermo C{\'a}mbara, Yang Li, Fatih Beyhan, Arent van Korlaar, Fan Yang, Arnaud Joly, {\'A}lvaro Mart{\'\i}n-Cortinas, Ammar Abbas, Adam Michalski, et~al. 2024.
\newblock Base tts: Lessons from building a billion-parameter text-to-speech model on 100k hours of data.
\newblock \emph{arXiv preprint arXiv:2402.08093}.
\bibitem[{Lakhotia et~al.(2021)Lakhotia, Kharitonov, Hsu, Adi, Polyak, Bolte, Nguyen, Copet, Baevski, Mohamed, and Dupoux}]{lakhotia-etal-2021-gslm}
Kushal Lakhotia, Eugene Kharitonov, Wei-Ning Hsu, Yossi Adi, Adam Polyak, Benjamin Bolte, Tu-Anh Nguyen, Jade Copet, Alexei Baevski, Abdelrahman Mohamed, and Emmanuel Dupoux. 2021.
\newblock \href {https://doi.org/10.1162/tacl_a_00430} {On generative spoken language modeling from raw audio}.
\newblock \emph{Transactions of the Association for Computational Linguistics}, 9:1336--1354.
\bibitem[{Lakomkin et~al.(2024)Lakomkin, Wu, Fathullah, Kalinli, Seltzer, and Fuegen}]{lakomkin2024end}
Egor Lakomkin, Chunyang Wu, Yassir Fathullah, Ozlem Kalinli, Michael~L Seltzer, and Christian Fuegen. 2024.
\newblock End-to-end speech recognition contextualization with large language models.
\newblock In \emph{ICASSP 2024-2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, pages 12406--12410. IEEE.
\bibitem[{Latif et~al.(2023)Latif, Shoukat, Shamshad, Usama, Cuay{\'{a}}huitl, and Schuller}]{latif-2023-large-audio}
Siddique Latif, Moazzam Shoukat, Fahad Shamshad, Muhammad Usama, Heriberto Cuay{\'{a}}huitl, and Bj{\"{o}}rn~W. Schuller. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2308.12792} {Sparks of large audio models: {A} survey and outlook}.
\newblock \emph{CoRR}, abs/2308.12792.
\bibitem[{Lee et~al.(2022)Lee, Kim, Kim, Cho, and Han}]{lee-etal-2022-rqtransformer}
Doyup Lee, Chiheon Kim, Saehoon Kim, Minsu Cho, and Wook-Shin Han. 2022.
\newblock \href {https://doi.org/10.1109/CVPR52688.2022.01123} {Autoregressive image generation using residual quantization}.
\newblock In \emph{2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 11513--11522.
\bibitem[{Li et~al.(2023{\natexlab{a}})Li, Li, Savarese, and Hoi}]{li-2023-blip}
Junnan Li, Dongxu Li, Silvio Savarese, and Steven C.~H. Hoi. 2023{\natexlab{a}}.
\newblock \href {https://proceedings.mlr.press/v202/li23q.html} {{BLIP-2:} bootstrapping language-image pre-training with frozen image encoders and large language models}.
\newblock In \emph{International Conference on Machine Learning, {ICML} 2023, 23-29 July 2023, Honolulu, Hawaii, {USA}}, volume 202 of \emph{Proceedings of Machine Learning Research}, pages 19730--19742. {PMLR}.
\bibitem[{Li et~al.(2024{\natexlab{a}})Li, Do, Keizer, Farag, Stoyanchev, and Doddipatla}]{li2024whisma}
Mohan Li, Cong-Thanh Do, Simon Keizer, Youmna Farag, Svetlana Stoyanchev, and Rama Doddipatla. 2024{\natexlab{a}}.
\newblock Whisma: A speech-llm to perform zero-shot spoken language understanding.
\newblock In \emph{2024 IEEE Spoken Language Technology Workshop (SLT)}, pages 1115--1122. IEEE.
\bibitem[{Li et~al.(2024{\natexlab{b}})Li, Chen, Kwok, Chu, Chng, and Kawai}]{li2024investigating}
Sheng Li, Chen Chen, Chin~Yuen Kwok, Chenhui Chu, Eng~Siong Chng, and Hisashi Kawai. 2024{\natexlab{b}}.
\newblock Investigating asr error correction with large language model and multilingual 1-best hypotheses.
\newblock In \emph{Proc. Interspeech}, pages 1315--1319.
\bibitem[{Li et~al.(2023{\natexlab{b}})Li, Wu, Li, and Liu}]{li-2023-asr-domain}
Yuang Li, Yu~Wu, Jinyu Li, and Shujie Liu. 2023{\natexlab{b}}.
\newblock \href {https://doi.org/10.1109/ASRU57964.2023.10389732} {Prompting large language models for zero-shot domain adaptation in speech recognition}.
\newblock In \emph{{IEEE} Automatic Speech Recognition and Understanding Workshop, {ASRU} 2023, Taipei, Taiwan, December 16-20, 2023}, pages 1--8. {IEEE}.
\bibitem[{Li et~al.(2024{\natexlab{c}})Li, Yu, Zhang, Ren, Zhao, Zhao, Tao, Su, and Yang}]{li2024using}
Yuang Li, Jiawei Yu, Min Zhang, Mengxin Ren, Yanqing Zhao, Xiaofeng Zhao, Shimin Tao, Jinsong Su, and Hao Yang. 2024{\natexlab{c}}.
\newblock Using large language model for end-to-end chinese asr and ner.
\newblock \emph{arXiv preprint arXiv:2401.11382}.
\bibitem[{Lin et~al.(2024)Lin, Yang, Chen, Zelasko, Yang, Chen, Puvvada, Fu, Hu, Chiu, Balam, Ginsburg, and Wang}]{lin-etal-2024-neko}
Yen-Ting Lin, Chao-Han~Huck Yang, Zhehuai Chen, Piotr Zelasko, Xuesong Yang, Zih-Ching Chen, Krishna~C Puvvada, Szu-Wei Fu, Ke~Hu, Jun~Wei Chiu, Jagadeesh Balam, Boris Ginsburg, and Yu-Chiang~Frank Wang. 2024.
\newblock \href {https://arxiv.org/abs/2411.05945} {Neko: Toward post recognition generative correction large language models with task-oriented experts}.
\newblock \emph{Preprint}, arXiv:2411.05945.
\bibitem[{Ling et~al.(2024)Ling, Hu, Qian, Ye, Qian, Gong, Lin, and Zeng}]{ling-2024-fully-formatted}
Shaoshi Ling, Yuxuan Hu, Shuangbei Qian, Guoli Ye, Yao Qian, Yifan Gong, Ed~Lin, and Michael Zeng. 2024.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10448204} {Adapting large language model with speech for fully formatted end-to-end speech recognition}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2024, Seoul, Republic of Korea, April 14-19, 2024}, pages 11046--11050. {IEEE}.
\bibitem[{Liu et~al.(2018)Liu, Saleh, Pot, Goodrich, Sepassi, Kaiser, and Shazeer}]{liu-etal-2018}
Peter~J. Liu, Mohammad Saleh, Etienne Pot, Ben Goodrich, Ryan Sepassi, Lukasz Kaiser, and Noam Shazeer. 2018.
\newblock \href {https://openreview.net/forum?id=Hyg0vbWC-} {Generating wikipedia by summarizing long sequences}.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Liu et~al.(2024)Liu, Hu, Ren, Yin, and Li}]{liu-2024-gpt-talker}
Rui Liu, Yifan Hu, Yi~Ren, Xiang Yin, and Haizhou Li. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2407.21491} {Generative expressive conversational speech synthesis}.
\newblock \emph{CoRR}, abs/2407.21491.
\bibitem[{Lu et~al.(2024)Lu, Chen, Fu, Huang, Ginsburg, Wang, and Lee}]{lu-2024-desta}
Ke{-}Han Lu, Zhehuai Chen, Szu{-}Wei Fu, He~Huang, Boris Ginsburg, Yu{-}Chiang~Frank Wang, and Hung{-}yi Lee. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2406.18871} {Desta: Enhancing speech language models through descriptive speech-text alignment}.
\newblock \emph{CoRR}, abs/2406.18871.
\bibitem[{Ma et~al.(2023)Ma, Qian, Manakul, Gales, and Knill}]{ma-2023-error-correction}
Rao Ma, Mengjie Qian, Potsawee Manakul, Mark J.~F. Gales, and Kate~M. Knill. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2307.04172} {Can generative large language models perform {ASR} error correction?}
\newblock \emph{CoRR}, abs/2307.04172.
\bibitem[{Ma et~al.(2024)Ma, Yang, Yang, Gao, Wang, Du, Yu, Chen, Zheng, Zhang et~al.}]{ma2024embarrassingly}
Ziyang Ma, Guanrou Yang, Yifan Yang, Zhifu Gao, Jiaming Wang, Zhihao Du, Fan Yu, Qian Chen, Siqi Zheng, Shiliang Zhang, et~al. 2024.
\newblock An embarrassingly simple approach for llm with strong asr capacity.
\newblock \emph{arXiv preprint arXiv:2402.08846}.
\bibitem[{Maiti et~al.(2024)Maiti, Peng, Choi, Jung, Chang, and Watanabe}]{maiti-etal-2024-voxtlm}
Soumi Maiti, Yifan Peng, Shukjae Choi, Jee-Weon Jung, Xuankai Chang, and Shinji Watanabe. 2024.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10447112} {Voxtlm: Unified decoder-only models for consolidating speech recognition, synthesis and speech, text continuation tasks}.
\newblock In \emph{ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, pages 13326--13330.
\bibitem[{McDermott et~al.(2019)McDermott, Sak, and Variani}]{mcdermott-2019-lm-fusion}
Erik McDermott, Hasim Sak, and Ehsan Variani. 2019.
\newblock \href {https://doi.org/10.1109/ASRU46091.2019.9003790} {A density ratio approach to language model fusion in end-to-end automatic speech recognition}.
\newblock In \emph{{IEEE} Automatic Speech Recognition and Understanding Workshop, {ASRU} 2019, Singapore, December 14-18, 2019}, pages 434--441. {IEEE}.
\bibitem[{Minaee et~al.(2024)Minaee, Mikolov, Nikzad, Chenaghlu, Socher, Amatriain, and Gao}]{minaee-etal-2024}
Shervin Minaee, Tomas Mikolov, Narjes Nikzad, Meysam Chenaghlu, Richard Socher, Xavier Amatriain, and Jianfeng Gao. 2024.
\newblock \href {https://arxiv.org/abs/2402.06196} {Large language models: A survey}.
\newblock \emph{Preprint}, arXiv:2402.06196.
\bibitem[{Mitsui et~al.(2024)Mitsui, Mitsuda, Wakatsuki, Hono, and Sawada}]{mitsui2024pslm}
Kentaro Mitsui, Koh Mitsuda, Toshiaki Wakatsuki, Yukiya Hono, and Kei Sawada. 2024.
\newblock \href {https://aclanthology.org/2024.findings-emnlp.151} {{PSLM:} parallel generation of text and speech with llms for low-latency spoken dialogue systems}.
\newblock In \emph{Findings of the Association for Computational Linguistics: {EMNLP} 2024, Miami, Florida, USA, November 12-16, 2024}, pages 2692--2700. Association for Computational Linguistics.
\bibitem[{Mittal et~al.(2024)Mittal, Prabhu, Sarawagi, and Jyothi}]{mittal2024salsa}
Ashish Mittal, Darshan Prabhu, Sunita Sarawagi, and Preethi Jyothi. 2024.
\newblock Salsa: Speedy asr-llm synchronous aggregation.
\newblock \emph{arXiv preprint arXiv:2408.16542}.
\bibitem[{Mu et~al.(2024)Mu, Li, Shao, Wei, Wan, Zheng, Zhou, and Xie}]{mu2024mmger}
Bingshen Mu, Yangze Li, Qijie Shao, Kun Wei, Xucheng Wan, Naijun Zheng, Huan Zhou, and Lei Xie. 2024.
\newblock Mmger: Multi-modal and multi-granularity generative error correction with llm for joint accent and speech recognition.
\newblock \emph{arXiv preprint arXiv:2405.03152}.
\bibitem[{Neekhara et~al.(2024)Neekhara, Hussain, Ghosh, Li, Valle, Badlani, and Ginsburg}]{neekhara2024improving}
Paarth Neekhara, Shehzeen Hussain, Subhankar Ghosh, Jason Li, Rafael Valle, Rohan Badlani, and Boris Ginsburg. 2024.
\newblock Improving robustness of llm-based speech synthesis by learning monotonic alignment.
\newblock \emph{arXiv preprint arXiv:2406.17957}.
\bibitem[{Nguyen et~al.(2025)Nguyen, Muller, Yu, Costa-Jussa, Elbayad, Popuri, Ropers, Duquenne, Algayres, Mavlyutov et~al.}]{nguyen-etal-2025-spiritlm}
Tu~Anh Nguyen, Benjamin Muller, Bokai Yu, Marta~R Costa-Jussa, Maha Elbayad, Sravya Popuri, Christophe Ropers, Paul-Ambroise Duquenne, Robin Algayres, Ruslan Mavlyutov, et~al. 2025.
\newblock Spirit-lm: Interleaved spoken and written language model.
\newblock \emph{Transactions of the Association for Computational Linguistics}, 13:30--52.
\bibitem[{Noroozi et~al.(2024)Noroozi, Chen, Majumdar, Huang, Balam, and Ginsburg}]{noroozi2024instruction}
Vahid Noroozi, Zhehuai Chen, Somshubra Majumdar, Steve Huang, Jagadeesh Balam, and Boris Ginsburg. 2024.
\newblock Instruction data generation and unsupervised adaptation for speech language models.
\newblock \emph{arXiv preprint arXiv:2406.12946}.
\bibitem[{Ouyang et~al.(2022)Ouyang, Wu, Jiang, Almeida, Wainwright, Mishkin, Zhang, Agarwal, Slama, Ray, Schulman, Hilton, Kelton, Miller, Simens, Askell, Welinder, Christiano, Leike, and Lowe}]{ouyang-etal-2022}
Long Ouyang, Jeffrey Wu, Xu~Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul~F Christiano, Jan Leike, and Ryan Lowe. 2022.
\newblock \href {https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf} {Training language models to follow instructions with human feedback}.
\newblock In \emph{Advances in Neural Information Processing Systems}, volume~35, pages 27730--27744. Curran Associates, Inc.
\bibitem[{Pan et~al.(2023)Pan, Wu, Gaur, Sivasankaran, Chen, Liu, and Li}]{pan-2023-cosmic}
Jing Pan, Jian Wu, Yashesh Gaur, Sunit Sivasankaran, Zhuo Chen, Shujie Liu, and Jinyu Li. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2311.02248} {{COSMIC:} data efficient instruction-tuning for speech in-context learning}.
\newblock \emph{CoRR}, abs/2311.02248.
\bibitem[{Pasad et~al.(2024)Pasad, Chien, Settle, and Livescu}]{pasad-etal-2024}
Ankita Pasad, Chung-Ming Chien, Shane Settle, and Karen Livescu. 2024.
\newblock \href {https://doi.org/10.1162/tacl_a_00656} {{What Do Self-Supervised Speech Models Know About Words?}}
\newblock \emph{Transactions of the Association for Computational Linguistics}, 12:372--391.
\bibitem[{Peng et~al.(2024)Peng, Wang, Xi, Li, Zhang, and Yu}]{peng-etal-2024-speechllmsurvey}
Jing Peng, Yucheng Wang, Yu~Xi, Xu~Li, Xizhuo Zhang, and Kai Yu. 2024.
\newblock \href {https://arxiv.org/abs/2410.18908} {A survey on speech large language models}.
\newblock \emph{Preprint}, arXiv:2410.18908.
\bibitem[{Pham et~al.(2024)Pham, Lin, Han, Li, Zhang, Lu, and Wang}]{pham-2024-comprehensive}
Van~Tung Pham, Yist~Y. Lin, Tao Han, Wei Li, Jun Zhang, Lu~Lu, and Yuxuan Wang. 2024.
\newblock \href {https://doi.org/10.48550/ARXIV.2406.17272} {A comprehensive solution to connect speech encoder and large language model for {ASR}}.
\newblock \emph{CoRR}, abs/2406.17272.
\bibitem[{Polyak et~al.(2021)Polyak, Adi, Copet, Kharitonov, Lakhotia, Hsu, Mohamed, and Dupoux}]{polyak-etal-2021}
Adam Polyak, Yossi Adi, Jade Copet, Eugene Kharitonov, Kushal Lakhotia, Wei-Ning Hsu, Abdelrahman Mohamed, and Emmanuel Dupoux. 2021.
\newblock \href {https://doi.org/10.21437/Interspeech.2021-475} {Speech resynthesis from discrete disentangled self-supervised representations}.
\newblock In \emph{Interspeech 2021}, pages 3615--3619.
\bibitem[{Radford et~al.(2023)Radford, Kim, Xu, Brockman, McLeavey, and Sutskever}]{radford-etal-2023-whisper}
Alec Radford, Jong~Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, and Ilya Sutskever. 2023.
\newblock \href {https://proceedings.mlr.press/v202/radford23a.html} {Robust speech recognition via large-scale weak supervision}.
\newblock In \emph{International Conference on Machine Learning, {ICML} 2023, 23-29 July 2023, Honolulu, Hawaii, {USA}}, volume 202 of \emph{Proceedings of Machine Learning Research}, pages 28492--28518. {PMLR}.
\bibitem[{Radford et~al.(2018)Radford, Narasimhan, Salimans, and Sutskever}]{radford-etal-2018}
Alec Radford, Karthik Narasimhan, Tim Salimans, and Ilya Sutskever. 2018.
\newblock \href {https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf} {Improving language understanding by generative pre-training}.
\bibitem[{Radford et~al.(2019)Radford, Wu, Child, Luan, Amodei, and Sutskever}]{radford-etal-2019}
Alec Radford, Jeff Wu, Rewon Child, David Luan, Dario Amodei, and Ilya Sutskever. 2019.
\newblock \href {https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf} {Language models are unsupervised multitask learners}.
\bibitem[{Radhakrishnan et~al.(2023)Radhakrishnan, Yang, Khan, Kumar, Kiani, Gomez-Cabrero, and Tegner}]{radhakrishnan2023whispering}
Srijith Radhakrishnan, Chao-Han~Huck Yang, Sumeer~Ahmad Khan, Rohit Kumar, Narsis~A Kiani, David Gomez-Cabrero, and Jesper~N Tegner. 2023.
\newblock Whispering llama: A cross-modal generative error correction framework for speech recognition.
\newblock \emph{arXiv preprint arXiv:2310.06434}.
\bibitem[{Rubenstein et~al.(2023)Rubenstein, Asawaroengchai, Nguyen, Bapna, Borsos, de~Chaumont~Quitry, Chen, Badawy, Han, Kharitonov, Muckenhirn, Padfield, Qin, Rozenberg, Sainath, Schalkwyk, Sharifi, Ramanovich, Tagliasacchi, Tudor, Velimirović, Vincent, Yu, Wang, Zayats, Zeghidour, Zhang, Zhang, Zilka, and Frank}]{rubenstein-etal-2023-audiopalm}
Paul~K. Rubenstein, Chulayuth Asawaroengchai, Duc~Dung Nguyen, Ankur Bapna, Zalán Borsos, Félix de~Chaumont~Quitry, Peter Chen, Dalia~El Badawy, Wei Han, Eugene Kharitonov, Hannah Muckenhirn, Dirk Padfield, James Qin, Danny Rozenberg, Tara Sainath, Johan Schalkwyk, Matt Sharifi, Michelle~Tadmor Ramanovich, Marco Tagliasacchi, Alexandru Tudor, Mihajlo Velimirović, Damien Vincent, Jiahui Yu, Yongqiang Wang, Vicky Zayats, Neil Zeghidour, Yu~Zhang, Zhishuai Zhang, Lukas Zilka, and Christian Frank. 2023.
\newblock \href {https://arxiv.org/abs/2306.12925} {Audiopalm: A large language model that can speak and listen}.
\newblock \emph{Preprint}, arXiv:2306.12925.
\bibitem[{Salazar et~al.(2020)Salazar, Liang, Nguyen, and Kirchhoff}]{salazar-2020-mlm-scoring}
Julian Salazar, Davis Liang, Toan~Q. Nguyen, and Katrin Kirchhoff. 2020.
\newblock \href {https://doi.org/10.18653/V1/2020.ACL-MAIN.240} {Masked language model scoring}.
\newblock In \emph{Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, {ACL} 2020, Online, July 5-10, 2020}, pages 2699--2712. Association for Computational Linguistics.
\bibitem[{Shen et~al.(2024)Shen, Zhang, Wu, Xiu, AlBadawy, Lu, Seltzer, and He}]{shen2024get}
Maohao Shen, Shun Zhang, Jilong Wu, Zhiping Xiu, Ehab AlBadawy, Yiting Lu, Mike Seltzer, and Qing He. 2024.
\newblock Get large language models ready to speak: A late-fusion approach for speech generation.
\newblock \emph{arXiv preprint arXiv:2410.20336}.
\bibitem[{Shen et~al.(2023)Shen, Song, Tan, Li, Lu, and Zhuang}]{shen-2023-hugginggpt}
Yongliang Shen, Kaitao Song, Xu~Tan, Dongsheng Li, Weiming Lu, and Yueting Zhuang. 2023.
\newblock \href {https://doi.org/10.48550/ARXIV.2303.17580} {Hugginggpt: Solving {AI} tasks with chatgpt and its friends in huggingface}.
\newblock \emph{CoRR}, abs/2303.17580.
\bibitem[{Shin et~al.(2019)Shin, Lee, and Jung}]{shin-2019-sentence-scoring}
Joonbo Shin, Yoonhyung Lee, and Kyomin Jung. 2019.
\newblock \href {http://proceedings.mlr.press/v101/shin19a.html} {Effective sentence scoring method using {BERT} for speech recognition}.
\newblock In \emph{Proceedings of The 11th Asian Conference on Machine Learning, {ACML} 2019, 17-19 November 2019, Nagoya, Japan}, volume 101 of \emph{Proceedings of Machine Learning Research}, pages 1081--1093. {PMLR}.
\bibitem[{Shon et~al.(2024)Shon, Kim, Hsu, Sridhar, Watanabe, and Livescu}]{shon2024discreteslu}
Suwon Shon, Kwangyoun Kim, Yi-Te Hsu, Prashant Sridhar, Shinji Watanabe, and Karen Livescu. 2024.
\newblock Discreteslu: A large language model with self-supervised discrete speech units for spoken language understanding.
\newblock \emph{arXiv preprint arXiv:2406.09345}.
\bibitem[{Sutskever et~al.(2014)Sutskever, Vinyals, and Le}]{sutskever-etal-2014-seq2seq}
Ilya Sutskever, Oriol Vinyals, and Quoc~V. Le. 2014.
\newblock Sequence to sequence learning with neural networks.
\newblock In \emph{Proceedings of the 27th International Conference on Neural Information Processing Systems - Volume 2}, NIPS'14, page 3104–3112, Cambridge, MA, USA. MIT Press.
\bibitem[{Tang et~al.(2024)Tang, Yu, Sun, Chen, Tan, Li, Lu, Ma, and Zhang}]{tang-etal-2024-salmonn}
Changli Tang, Wenyi Yu, Guangzhi Sun, Xianzhao Chen, Tian Tan, Wei Li, Lu~Lu, Zejun Ma, and Chao Zhang. 2024.
\newblock \href {https://openreview.net/forum?id=14rn7HpKVk} {{SALMONN:} towards generic hearing abilities for large language models}.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net.
\bibitem[{Tang and Tung(2024)}]{tang2024contextualized}
Yixuan Tang and Anthony~KH Tung. 2024.
\newblock Contextualized speech recognition: rethinking second-pass rescoring with generative large language models.
\newblock In \emph{Proceedings of the Thirty-Third International Joint Conference on Artificial Intelligence}, pages 6478--6485.
\bibitem[{Tao et~al.(2024)Tao, Liu, Dou, Muennighoff, Wan, Luo, Lin, and Wong}]{tao-etal-2024-scalingvocab}
Chaofan Tao, Qian Liu, Longxu Dou, Niklas Muennighoff, Zhongwei Wan, Ping Luo, Min Lin, and Ngai Wong. 2024.
\newblock \href {https://openreview.net/forum?id=sKCKPr8cRL} {Scaling laws with vocabulary: Larger models deserve larger vocabularies}.
\newblock In \emph{The Thirty-eighth Annual Conference on Neural Information Processing Systems}.
\bibitem[{Touvron et~al.(2023{\natexlab{a}})Touvron, Lavril, Izacard, Martinet, Lachaux, Lacroix, Rozi{\`{e}}re, Goyal, Hambro, Azhar, Rodriguez, Joulin, Grave, and Lample}]{touvron-etal-2023-llama}
Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie{-}Anne Lachaux, Timoth{\'{e}}e Lacroix, Baptiste Rozi{\`{e}}re, Naman Goyal, Eric Hambro, Faisal Azhar, Aur{\'{e}}lien Rodriguez, Armand Joulin, Edouard Grave, and Guillaume Lample. 2023{\natexlab{a}}.
\newblock \href {https://doi.org/10.48550/ARXIV.2302.13971} {Llama: Open and efficient foundation language models}.
\newblock \emph{CoRR}, abs/2302.13971.
\bibitem[{Touvron et~al.(2023{\natexlab{b}})Touvron, Martin, Stone, Albert, Almahairi, Babaei, Bashlykov, Batra, Bhargava, Bhosale, Bikel, Blecher, Ferrer, Chen, Cucurull, Esiobu, Fernandes, Fu, Fu, Fuller, Gao, Goswami, Goyal, Hartshorn, Hosseini, Hou, Inan, Kardas, Kerkez, Khabsa, Kloumann, Korenev, Koura, Lachaux, Lavril, Lee, Liskovich, Lu, Mao, Martinet, Mihaylov, Mishra, Molybog, Nie, Poulton, Reizenstein, Rungta, Saladi, Schelten, Silva, Smith, Subramanian, Tan, Tang, Taylor, Williams, Kuan, Xu, Yan, Zarov, Zhang, Fan, Kambadur, Narang, Rodriguez, Stojnic, Edunov, and Scialom}]{touvron-etal-2023-llama2}
Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian~Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez, Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit~Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushkar Mishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric~Michael Smith, Ranjan Subramanian, Xiaoqing~Ellen Tan, Binh Tang, Ross Taylor, Adina Williams, Jian~Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, and Thomas
Scialom. 2023{\natexlab{b}}.
\newblock \href {https://arxiv.org/abs/2307.09288} {Llama 2: Open foundation and fine-tuned chat models}.
\newblock \emph{Preprint}, arXiv:2307.09288.
\bibitem[{Udagawa et~al.(2022)Udagawa, Suzuki, Kurata, Itoh, and Saon}]{udagawa-2022-rescoring}
Takuma Udagawa, Masayuki Suzuki, Gakuto Kurata, Nobuyasu Itoh, and George Saon. 2022.
\newblock \href {https://doi.org/10.21437/INTERSPEECH.2022-11123} {Effect and analysis of large-scale language model rescoring on competitive {ASR} systems}.
\newblock In \emph{23rd Annual Conference of the International Speech Communication Association, Interspeech 2022, Incheon, Korea, September 18-22, 2022}, pages 3919--3923. {ISCA}.
\bibitem[{Valin et~al.(2012)Valin, Vos, and Terriberry}]{rfc6716}
Jean-Marc Valin, Koen Vos, and Timothy~B. Terriberry. 2012.
\newblock {Definition of the Opus Audio Codec}.
\newblock {RFC 6716}.
\newblock Internet Engineering Task Force (IETF).
\bibitem[{Vaswani et~al.(2017)Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez, Kaiser, and Polosukhin}]{vaswani-etal-2017}
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan~N. Gomez, \L{}ukasz Kaiser, and Illia Polosukhin. 2017.
\newblock Attention is all you need.
\newblock In \emph{Proceedings of the 31st International Conference on Neural Information Processing Systems}, NIPS'17, page 6000–6010, Red Hook, NY, USA. Curran Associates Inc.
\bibitem[{Wang et~al.(2021)Wang, Wu, Gu, and Pino}]{wang-etal-2021-covost2}
Changhan Wang, Anne Wu, Jiatao Gu, and Juan Pino. 2021.
\newblock \href {https://doi.org/10.21437/Interspeech.2021-2027} {Covost 2 and massively multilingual speech translation}.
\newblock In \emph{Interspeech 2021}, pages 2247--2251.
\bibitem[{Wang et~al.(2023{\natexlab{a}})Wang, Liao, Huang, Lu, Wu, Liu, Zong, and Zhang}]{wang2023blsp}
Chen Wang, Minpeng Liao, Zhongqiang Huang, Jinliang Lu, Junhong Wu, Yuchen Liu, Chengqing Zong, and Jiajun Zhang. 2023{\natexlab{a}}.
\newblock Blsp: Bootstrapping language-speech pre-training via behavior alignment of continuation writing.
\newblock \emph{arXiv preprint arXiv:2309.00916}.
\bibitem[{Wang et~al.(2024{\natexlab{a}})Wang, Liao, Huang, and Zhang}]{wang2024blsp}
Chen Wang, Minpeng Liao, Zhongqiang Huang, and Jiajun Zhang. 2024{\natexlab{a}}.
\newblock Blsp-kd: Bootstrapping language-speech pre-training via knowledge distillation.
\newblock \emph{arXiv preprint arXiv:2405.19041}.
\bibitem[{Wang et~al.(2023{\natexlab{b}})Wang, Chen, Wu, Zhang, Zhou, Liu, Chen, Liu, Wang, Li, He, Zhao, and Wei}]{wang-etal-2023-valle}
Chengyi Wang, Sanyuan Chen, Yu~Wu, Ziqiang Zhang, Long Zhou, Shujie Liu, Zhuo Chen, Yanqing Liu, Huaming Wang, Jinyu Li, Lei He, Sheng Zhao, and Furu Wei. 2023{\natexlab{b}}.
\newblock \href {https://arxiv.org/abs/2301.02111} {Neural codec language models are zero-shot text to speech synthesizers}.
\newblock \emph{Preprint}, arXiv:2301.02111.
\bibitem[{Wang et~al.(2024{\natexlab{b}})Wang, Cui, Yang, Chen, and Meng}]{wang-etal-2024-comparativestudydiscretespeech}
Dingdong Wang, Mingyu Cui, Dongchao Yang, Xueyuan Chen, and Helen Meng. 2024{\natexlab{b}}.
\newblock \href {https://arxiv.org/abs/2411.08742} {A comparative study of discrete speech tokens for semantic-related tasks with large language models}.
\newblock \emph{Preprint}, arXiv:2411.08742.
\bibitem[{Wang et~al.(2023{\natexlab{c}})Wang, Han, Shafran, Wu, Chiu, Cao, Chen, Zhang, Soltau, Rubenstein, Zilka, Yu, Pundak, Siddhartha, Schalkwyk, and Wu}]{wang-2023-slm}
Mingqiu Wang, Wei Han, Izhak Shafran, Zelin Wu, Chung{-}Cheng Chiu, Yuan Cao, Nanxin Chen, Yu~Zhang, Hagen Soltau, Paul~K. Rubenstein, Lukas Zilka, Dian Yu, Golan Pundak, Nikhil Siddhartha, Johan Schalkwyk, and Yonghui Wu. 2023{\natexlab{c}}.
\newblock \href {https://doi.org/10.1109/ASRU57964.2023.10389703} {{SLM:} bridge the thin gap between speech and text foundation models}.
\newblock In \emph{{IEEE} Automatic Speech Recognition and Understanding Workshop, {ASRU} 2023, Taipei, Taiwan, December 16-20, 2023}, pages 1--8. {IEEE}.
\bibitem[{Wang et~al.(2023{\natexlab{d}})Wang, Zhou, Zhang, Wu, Liu, Gaur, Chen, Li, and Wei}]{wang-etal-2023-viola}
Tianrui Wang, Long Zhou, Ziqiang Zhang, Yu~Wu, Shujie Liu, Yashesh Gaur, Zhuo Chen, Jinyu Li, and Furu Wei. 2023{\natexlab{d}}.
\newblock \href {https://arxiv.org/abs/2305.16107} {Viola: Unified codec language models for speech recognition, synthesis, and translation}.
\newblock \emph{Preprint}, arXiv:2305.16107.
\bibitem[{Wei et~al.(2022)Wei, Bosma, Zhao, Guu, Yu, Lester, Du, Dai, and Le}]{wei-etal-2022}
Jason Wei, Maarten Bosma, Vincent Zhao, Kelvin Guu, Adams~Wei Yu, Brian Lester, Nan Du, Andrew~M. Dai, and Quoc~V Le. 2022.
\newblock \href {https://openreview.net/forum?id=gEZrGCozdqR} {Finetuned language models are zero-shot learners}.
\newblock In \emph{International Conference on Learning Representations}.
\bibitem[{Wu et~al.(2023)Wu, Gaur, Chen, Zhou, Zhu, Wang, Li, Liu, Ren, Liu, and Wu}]{wu-2023-decoder-only}
Jian Wu, Yashesh Gaur, Zhuo Chen, Long Zhou, Yimeng Zhu, Tianrui Wang, Jinyu Li, Shujie Liu, Bo~Ren, Linquan Liu, and Yu~Wu. 2023.
\newblock \href {https://doi.org/10.1109/ASRU57964.2023.10389705} {On decoder-only architecture for speech-to-text and large language model integration}.
\newblock In \emph{{IEEE} Automatic Speech Recognition and Understanding Workshop, {ASRU} 2023, Taipei, Taiwan, December 16-20, 2023}, pages 1--8. {IEEE}.
\bibitem[{Xu et~al.(2022)Xu, Gu, Kolehmainen, Khan, Gandhe, Rastrow, Stolcke, and Bulyko}]{xu-2022-rescorebert}
Liyan Xu, Yile Gu, Jari Kolehmainen, Haidar Khan, Ankur Gandhe, Ariya Rastrow, Andreas Stolcke, and Ivan Bulyko. 2022.
\newblock \href {https://doi.org/10.1109/ICASSP43922.2022.9747118} {Rescorebert: Discriminative speech recognition rescoring with bert}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2022, Virtual and Singapore, 23-27 May 2022}, pages 6117--6121. {IEEE}.
\bibitem[{Xu et~al.(2024{\natexlab{a}})Xu, Chen, Yu, Huang, Wu, Zhang, Li, Luo, and Gu}]{xu2024secap}
Yaoxun Xu, Hangting Chen, Jianwei Yu, Qiaochu Huang, Zhiyong Wu, Shi-Xiong Zhang, Guangzhi Li, Yi~Luo, and Rongzhi Gu. 2024{\natexlab{a}}.
\newblock \href {https://doi.org/10.1609/aaai.v38i17.29902} {Secap: speech emotion captioning with large language model}.
\newblock In \emph{Proceedings of the Thirty-Eighth AAAI Conference on Artificial Intelligence and Thirty-Sixth Conference on Innovative Applications of Artificial Intelligence and Fourteenth Symposium on Educational Advances in Artificial Intelligence}, AAAI'24/IAAI'24/EAAI'24. AAAI Press.
\bibitem[{Xu et~al.(2024{\natexlab{b}})Xu, Zhang, Yu, Wu, and Yu}]{xu2024comparing}
Yaoxun Xu, Shi-Xiong Zhang, Jianwei Yu, Zhiyong Wu, and Dong Yu. 2024{\natexlab{b}}.
\newblock Comparing discrete and continuous space llms for speech recognition.
\newblock \emph{arXiv preprint arXiv:2409.00800}.
\bibitem[{Xu et~al.(2024{\natexlab{c}})Xu, Xie, Gu, Chen, Chang, Zhang, Chen, Zhang, and Tian}]{xu-2024-lora}
Yuhui Xu, Lingxi Xie, Xiaotao Gu, Xin Chen, Heng Chang, Hengheng Zhang, Zhengsu Chen, Xiaopeng Zhang, and Qi~Tian. 2024{\natexlab{c}}.
\newblock \href {https://openreview.net/forum?id=WvFoJccpo8} {Qa-lora: Quantization-aware low-rank adaptation of large language models}.
\newblock In \emph{The Twelfth International Conference on Learning Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}. OpenReview.net.
\bibitem[{Xue et~al.(2024)Xue, Ren, Geng, Wei, Li, Shao, Yang, Diao, and Xie}]{xue2024ideal}
Hongfei Xue, Wei Ren, Xuelong Geng, Kun Wei, Longhao Li, Qijie Shao, Linju Yang, Kai Diao, and Lei Xie. 2024.
\newblock Ideal-llm: Integrating dual encoders and language-adapted llm for multilingual speech-to-text.
\newblock \emph{arXiv preprint arXiv:2409.11214}.
\bibitem[{Yang et~al.(2023{\natexlab{a}})Yang, Gu, Liu, Ghosh, Bulyko, and Stolcke}]{yang-2023-tap}
Chao{-}Han~Huck Yang, Yile Gu, Yi{-}Chieh Liu, Shalini Ghosh, Ivan Bulyko, and Andreas Stolcke. 2023{\natexlab{a}}.
\newblock \href {https://doi.org/10.1109/ASRU57964.2023.10389673} {Generative speech recognition error correction with large language models and task-activating prompting}.
\newblock In \emph{{IEEE} Automatic Speech Recognition and Understanding Workshop, {ASRU} 2023, Taipei, Taiwan, December 16-20, 2023}, pages 1--8. {IEEE}.
\bibitem[{Yang et~al.(2024{\natexlab{a}})Yang, Park, Gong, Li, Chen, Lin, Chen, Hu, Dhawan, {\.Z}elasko et~al.}]{yang2024large}
Chao-Han~Huck Yang, Taejin Park, Yuan Gong, Yuanchao Li, Zhehuai Chen, Yen-Ting Lin, Chen Chen, Yuchen Hu, Kunal Dhawan, Piotr {\.Z}elasko, et~al. 2024{\natexlab{a}}.
\newblock Large language model based generative error correction: A challenge and baselines forspeech recognition, speaker tagging, and emotion recognition.
\newblock \emph{arXiv preprint arXiv:2409.09785}.
\bibitem[{Yang et~al.(2023{\natexlab{b}})Yang, Tian, Tan, Huang, Liu, Chang, Shi, Zhao, Bian, Wu et~al.}]{yang2023uniaudio}
Dongchao Yang, Jinchuan Tian, Xu~Tan, Rongjie Huang, Songxiang Liu, Xuankai Chang, Jiatong Shi, Sheng Zhao, Jiang Bian, Xixin Wu, et~al. 2023{\natexlab{b}}.
\newblock Uniaudio: An audio foundation model toward universal audio generation.
\newblock \emph{arXiv preprint arXiv:2310.00704}.
\bibitem[{Yang et~al.(2024{\natexlab{b}})Yang, Ma, Yu, Gao, Zhang, and Chen}]{yang2024mala}
Guanrou Yang, Ziyang Ma, Fan Yu, Zhifu Gao, Shiliang Zhang, and Xie Chen. 2024{\natexlab{b}}.
\newblock Mala-asr: Multimedia-assisted llm-based asr.
\newblock \emph{arXiv preprint arXiv:2406.05839}.
\bibitem[{Yeo et~al.(2024)Yeo, Han, Kim, and Ro}]{yeo2024visual}
Jeong~Hun Yeo, Seunghee Han, Minsu Kim, and Yong~Man Ro. 2024.
\newblock \href {https://aclanthology.org/2024.findings-emnlp.666} {Where visual speech meets language: {VSP-LLM} framework for efficient and context-aware visual speech processing}.
\newblock In \emph{Findings of the Association for Computational Linguistics: {EMNLP} 2024, Miami, Florida, USA, November 12-16, 2024}, pages 11391--11406. Association for Computational Linguistics.
\bibitem[{Yu et~al.(2024)Yu, Tang, Sun, Chen, Tan, Li, Lu, Ma, and Zhang}]{yu-2024-connecting}
Wenyi Yu, Changli Tang, Guangzhi Sun, Xianzhao Chen, Tian Tan, Wei Li, Lu~Lu, Zejun Ma, and Chao Zhang. 2024.
\newblock \href {https://doi.org/10.1109/ICASSP48485.2024.10445874} {Connecting speech encoder and large language model for {ASR}}.
\newblock In \emph{{IEEE} International Conference on Acoustics, Speech and Signal Processing, {ICASSP} 2024, Seoul, Republic of Korea, April 14-19, 2024}, pages 12637--12641. {IEEE}.
\bibitem[{Zeghidour et~al.(2021)Zeghidour, Luebs, Omran, Skoglund, and Tagliasacchi}]{zeghidour-etal-2021-soundstream}
Neil Zeghidour, Alejandro Luebs, Ahmed Omran, Jan Skoglund, and Marco Tagliasacchi. 2021.
\newblock \href {https://arxiv.org/abs/2107.03312} {Soundstream: An end-to-end neural audio codec}.
\newblock \emph{Preprint}, arXiv:2107.03312.
\bibitem[{Zhang et~al.(2023)Zhang, Li, Zhang, Zhan, Wang, Zhou, and Qiu}]{zhang-etal-2023-speechgpt}
Dong Zhang, Shimin Li, Xin Zhang, Jun Zhan, Pengyu Wang, Yaqian Zhou, and Xipeng Qiu. 2023.
\newblock \href {https://doi.org/10.18653/v1/2023.findings-emnlp.1055} {{S}peech{GPT}: Empowering large language models with intrinsic cross-modal conversational abilities}.
\newblock In \emph{Findings of the Association for Computational Linguistics: EMNLP 2023}, pages 15757--15773, Singapore. Association for Computational Linguistics.
\bibitem[{Zhang et~al.(2024{\natexlab{a}})Zhang, Zhang, Zhan, Li, Zhou, and Qiu}]{zhang-etal-2024-speechgptgen}
Dong Zhang, Xin Zhang, Jun Zhan, Shimin Li, Yaqian Zhou, and Xipeng Qiu. 2024{\natexlab{a}}.
\newblock Speechgpt-gen: Scaling chain-of-information speech generation.
\newblock \emph{arXiv preprint arXiv:2401.13527}.
\bibitem[{Zhang et~al.(2024{\natexlab{b}})Zhang, Yu, Dong, Li, Su, Chu, and Yu}]{zhang-2024-mmllm}
Duzhen Zhang, Yahan Yu, Jiahua Dong, Chenxing Li, Dan Su, Chenhui Chu, and Dong Yu. 2024{\natexlab{b}}.
\newblock \href {https://doi.org/10.18653/V1/2024.FINDINGS-ACL.738} {Mm-llms: Recent advances in multimodal large language models}.
\newblock In \emph{Findings of the Association for Computational Linguistics, {ACL} 2024, Bangkok, Thailand and virtual meeting, August 11-16, 2024}, pages 12401--12430. Association for Computational Linguistics.
\bibitem[{Zhang et~al.(2022)Zhang, Roller, Goyal, Artetxe, Chen, Chen, Dewan, Diab, Li, Lin, Mihaylov, Ott, Shleifer, Shuster, Simig, Koura, Sridhar, Wang, and Zettlemoyer}]{zhang-etal-2022-opt}
Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen, Christopher Dewan, Mona Diab, Xian Li, Xi~Victoria Lin, Todor Mihaylov, Myle Ott, Sam Shleifer, Kurt Shuster, Daniel Simig, Punit~Singh Koura, Anjali Sridhar, Tianlu Wang, and Luke Zettlemoyer. 2022.
\newblock \href {https://arxiv.org/abs/2205.01068} {Opt: Open pre-trained transformer language models}.
\newblock \emph{Preprint}, arXiv:2205.01068.
\bibitem[{Zhang et~al.(2024{\natexlab{c}})Zhang, Zhang, Li, Zhou, and Qiu}]{zhang-etal-2024-speechtokenizer}
Xin Zhang, Dong Zhang, Shimin Li, Yaqian Zhou, and Xipeng Qiu. 2024{\natexlab{c}}.
\newblock \href {https://openreview.net/forum?id=AF9Q8Vip84} {Speechtokenizer: Unified speech tokenizer for speech language models}.
\newblock In \emph{The Twelfth International Conference on Learning Representations}.
\bibitem[{Zhao et~al.(2023)Zhao, Zhou, Li, Tang, Wang, Hou, Min, Zhang, Zhang, Dong, Du, Yang, Chen, Chen, Jiang, Ren, Li, Tang, Liu, Liu, Nie, and Wen}]{zhao-etal-2023-survey-llm}
Wayne~Xin Zhao, Kun Zhou, Junyi Li, Tianyi Tang, Xiaolei Wang, Yupeng Hou, Yingqian Min, Beichen Zhang, Junjie Zhang, Zican Dong, Yifan Du, Chen Yang, Yushuo Chen, Zhipeng Chen, Jinhao Jiang, Ruiyang Ren, Yifan Li, Xinyu Tang, Zikang Liu, Peiyu Liu, Jian-Yun Nie, and Ji-Rong Wen. 2023.
\newblock \href {https://arxiv.org/abs/2303.18223} {A survey of large language models}.
\newblock \emph{Preprint}, arXiv:2303.18223.
\end{thebibliography}