MISSRec: Pre-training and Transferring Multi-modal Interest-aware Sequence Representation for Recommendation
Extracted source text · arXiv 2308.11175
\PassOptionsToPackage{table,xcdraw}{xcolor}
\documentclass[sigconf]{acmart}
% \documentclass[sigconf,anonymous,review]{acmart}
%%%% TODO: removed in camera-ready (below)
% \settopmatter{printacmref=true}
\settopmatter{printacmref=false} % Removes citation information below abstract\renewcommand\footnotetextcopyrightpermission[1]{} % removes footnote with conference information in first column
\pagestyle{plain} % removes running headers
%%%% TODO: removed in camera-ready (above)
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
\usepackage{makecell}
\usepackage{cleveref}
\usepackage{amsthm}
\usepackage{bm}
\usepackage[table,xcdraw]{xcolor}
\usepackage{multicol}
\usepackage{colortbl}
\usepackage{amsmath}
\usepackage{multirow}
% \usepackage[normalem]{ulem}
% \useunder{\uline}{\ul}{}
% \usepackage{hyperref}
% \usepackage{graphicx}
\usepackage{subfigure}
\usepackage{enumitem}
%\usepackage{stfloats}
% \usepackage{threeparttable}
\usepackage{booktabs}
\usepackage{bbding}
\usepackage{color}
\usepackage{xcolor}
% \usepackage{balance}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{algorithm}
\usepackage{listings}
\usepackage{natbib}
\usepackage[misc]{ifsym} % \Letter
\newcommand{\black}[1]{\textcolor{black}{#1}}
% Copy from MoCo
%----------------------------------------------------------
% this is for adding footnote after algorithm
\usepackage{etoolbox}
\makeatletter
\AfterEndEnvironment{algorithm}{\let\@algcomment\relax}
\AtEndEnvironment{algorithm}{\kern2pt\hrule\relax\vskip3pt\@algcomment}
\let\@algcomment\relax
\newcommand\algcomment[1]{\def\@algcomment{\footnotesize#1}}
\renewcommand\fs@ruled{\def\@fs@cfont{\bfseries}\let\@fs@capt\floatc@ruled
\def\@fs@pre{\hrule height.8pt depth0pt \kern2pt}%
\def\@fs@post{}%
\def\@fs@mid{\kern2pt\hrule\kern2pt}%
\let\@fs@iftopcapt\iftrue}
\makeatother
%----------------------------------------------------------
% Copyright
% \setcopyright{none}
% \setcopyright{acmcopyright}
% \copyrightyear{2018}
% \acmYear{2018}
% \acmDOI{10.1145/1122445.1122456}
%% These commands are for a PROCEEDINGS abstract or paper.
% \acmConference[Woodstock '18]{Woodstock '18: ACM Symposium on Neural
% Gaze Detection}{June 03--05, 2018}{Woodstock, NY}
% \acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
% June 03--05, 2018, Woodstock, NY}
% \acmPrice{15.00}
% \acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
\acmSubmissionID{1218}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
Bib\TeX}}}
\copyrightyear{2023}
\acmYear{2023}
\setcopyright{rightsretained}
\acmConference[MM '23]{Proceedings of the 31st ACM International Conference on Multimedia}{October 29-November 3, 2023}{Ottawa, ON, Canada}
\acmBooktitle{Proceedings of the 31st ACM International Conference on Multimedia (MM '23), October 29-November 3, 2023, Ottawa, ON, Canada}
\acmDOI{10.1145/3581783.3611967}
\acmISBN{979-8-4007-0108-5/23/10}
% The following includes the CC license icon appropriate for your paper.
% Download the image from www.scomminc.com/pp/acmsig/4ACM-CC-by-88x31.eps
% and place within your figs or figures folder
\makeatletter
\gdef\@copyrightpermission{
\begin{minipage}{0.3\columnwidth}
\href{https://creativecommons.org/licenses/by/4.0/}{\includegraphics[width=0.90\textwidth]{imgs/4ACM-CC-by-88x31.pdf}}
\end{minipage}\hfill
\begin{minipage}{0.7\columnwidth}
\href{https://creativecommons.org/licenses/by/4.0/}{This work is licensed under a Creative Commons Attribution International 4.0 License.}
\end{minipage}
\vspace{5pt}
}
\makeatother
% 1 Authors, replace the red X's with your assigned DOI string during the rightsreview eform process.
% 2 Your DOI link will become active when the proceedings appears in the DL.
% 3 Retain the DOI string between the curly braces for uploading your presentation video.
\settopmatter{printacmref=true}
\begin{document}
% \fancyhead{}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title[MISSRec: Pre-training and Transferring Multi-modal Interest-aware Sequence Representation \\ for Recommendation]{MISSRec: Pre-training and Transferring Multi-modal Interest-aware Sequence Representation for Recommendation}
\author{Jinpeng Wang}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{wjp20@mails.tsinghua.edu.cn}
\author{Ziyun Zeng}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{zengzy21@mails.tsinghua.edu.cn}
\author{Yunxiao Wang}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{wang-yx20@mails.tsinghua.edu.cn}
\author{Yuting Wang}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{wangyt22@mails.tsinghua.edu.cn}
\author{Xingyu Lu}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{luxy22@mails.tsinghua.edu.cn}
\author{Tianxiang Li}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{litx21@mails.tsinghua.edu.cn}
\author{Jun Yuan}
\affiliation{%
\institution{Huawei Noah’s Ark Lab}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{yuanjun25@huawei.com}
% \author{Jieming Zhu}
% \affiliation{%
% \institution{Huawei Noah’s Ark Lab}
% % \streetaddress{P.O. Box 1212}
% \city{Shenzhen}
% % \state{Ohio}
% \country{China}
% % \postcode{43017-6221}
% }
% \email{jiemingzhu@ieee.org}
\author{Rui Zhang$^\text{\Letter}$}
\affiliation{%
% \institution{Huawei Noah’s Ark Lab}
% % \streetaddress{P.O. Box 1212}
% \city{Shenzhen}
% % \state{Ohio}
% \country{China}
% % \postcode{43017-6221}
\country{www.ruizhang.info}
}
\email{rayteam@yeah.net}
\author{Hai-Tao Zheng}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\affiliation{%
\institution{Peng Cheng Laboratory}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{zheng.haitao@sz.tsinghua.edu.cn}
\author{Shu-Tao Xia$^\text{\Letter}$}
\affiliation{%
\institution{Tsinghua Shenzhen International Graduate School, Tsinghua University}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\affiliation{%
\institution{Research Center of Artificial Intelligence, Peng Cheng Laboratory}
% \streetaddress{P.O. Box 1212}
\city{Shenzhen}
% \state{Ohio}
\country{China}
% \postcode{43017-6221}
}
\email{xiast@sz.tsinghua.edu.cn}
\thanks{\Letter\ Corresponding authors.}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Jinpeng Wang et al.}
%% No italics and no comma
%% If needed use a foot or author note to identify equal contribution
\input{command}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\input{sections/Abstract}
%
% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
% Please copy and paste the code instead of the example below.
%
%
% The code below should be generated by the tool at
% http://dl.acm.org/ccs.cfm
% Please copy and paste the code instead of the example below.
%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10002951.10003317.10003347.10003350</concept_id>
<concept_desc>Information systems~Recommender systems</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10002951.10003227.10003251</concept_id>
<concept_desc>Information systems~Multimedia information systems</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10002951.10003260.10003261.10003271</concept_id>
<concept_desc>Information systems~Personalization</concept_desc>
<concept_significance>500</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Information systems~Recommender systems}
\ccsdesc[500]{Information systems~Multimedia information systems}
\ccsdesc[500]{Information systems~Personalization}
%
% Keywords. The author(s) should pick words that accurately describe
% the work being presented. Separate the keywords with commas.
\keywords{multi-modal sequential recommendation, pre-training, parameter-efficient fine-tuning, interest-aware sequence representation}
\maketitle
\input{sections/Introduction}
\input{sections/Related_Work}
\input{sections/Method}
\input{sections/Experiments}
\input{sections/Conclusions}
\bibliographystyle{ACM-Reference-Format}
\balance
\bibliography{main}
\end{document}
\endinput
%%
%% End of file `sample-authordraft.tex'.
\newcommand{\dui}{\textcolor{teal}{\CheckmarkBold}}
\newcommand{\cuo}{\textcolor{purple}{\XSolidBrush}}
\newcommand{\ie}{\emph{i.e.},~}
\newcommand{\eg}{\emph{e.g.},~}
\newcommand{\wrt}{\emph{w.r.t.}~}
\newcommand{\aka}{\emph{aka}~}
\renewcommand{\paragraph}[1]{\medskip\noindent\textbf{#1.~}}
\newcommand{\todo}{\color{red}{\textbf{TODO.}}~}
\newcommand{\xiaok}[1]{\left(#1\right)}
\newcommand{\zhongk}[1]{\left[#1\right]}
\newcommand{\dak}[1]{\left\{#1\right\}}
\newcommand{\jiaok}[1]{\left<#1\right>}
\newcommand{\shuk}[1]{\left\lVert#1\right\rVert}
\newcommand{\shuks}[1]{\left\lVert#1\right\rVert^2}
\newcommand{\shangk}[1]{\left\lceil #1 \right\rceil}
\newcommand{\xiak}[1]{\left\lfloor #1 \right\rfloor}
\newcommand{\argmax}[1]{{\mathop{\arg\mathrm{max}}_{#1}\,}}
\newcommand{\argmin}[1]{{\mathop{\arg\mathrm{min}}_{#1}\,}}
\newcommand{\pfrac}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\prob}[2]{p\xiaok{#1 \mid #2}}
\newcommand{\T}{\top}
\newcommand{\dif}{\mathop{}\!\mathrm{d}}
\newcommand{\biset}[1]{\{0,1\}^{#1}}
\newcommand{\ReLU}{\mathrm{ReLU}}
\newcommand{\relu}{\mathrm{ReLU}}
\newcommand{\sign}{\mathrm{sign}}
\newcommand\softmax{\mathrm{softmax}}
\newcommand\KL{D_{\mathrm{KL}}}
\newcommand\Var{\mathrm{Var}}
\newcommand\Cov{\mathrm{Cov}}
\newcommand{\Tr}{\mathrm{Tr}}
\newcommand{\tr}{\mathrm{tr}}
\newcommand{\dist}{\mathrm{dist}}
\newcommand{\concat}{\mathrm{concat}}
\newcommand{\mean}{\mathrm{mean}}
\newcommand{\diag}{\mathrm{diag}}
\newcommand{\cov}{\mathrm{cov}}
\newcommand{\range}[1]{{0,1,\cdots,#1}} % 0, 1, ..., N
\newcommand{\Range}[1]{{1,2,\cdots,#1}} % 1, 2, ..., N
\newcommand{\opseq}[3]{{#1_1 #3 #1_2 #3 \cdots #3 #1_{#2}}}
\newcommand{\seq}[2]{\opseq{#1}{#2}{,}}
\newcommand{\xseq}[2]{\opseq{#1}{#2}{\times}}
\newcommand{\bma}{\bm{a}}
\newcommand{\bmb}{\bm{b}}
\newcommand{\bmc}{\bm{c}}
\newcommand{\bmd}{\bm{d}}
\newcommand{\bme}{\bm{e}}
\newcommand{\bmf}{\bm{f}}
\newcommand{\bmg}{\bm{g}}
\newcommand{\bmh}{\bm{h}}
\newcommand{\bmi}{\bm{i}}
\newcommand{\bmj}{\bm{j}}
\newcommand{\bmk}{\bm{k}}
\newcommand{\bml}{\bm{l}}
\newcommand{\bmm}{\bm{m}}
\newcommand{\bmn}{\bm{n}}
\newcommand{\bmo}{\bm{o}}
\newcommand{\bmp}{\bm{p}}
\newcommand{\bmq}{\bm{q}}
\newcommand{\bmr}{\bm{r}}
\newcommand{\bms}{\bm{s}}
\newcommand{\bmt}{\bm{t}}
\newcommand{\bmu}{\bm{u}}
\newcommand{\bmv}{\bm{v}}
\newcommand{\bmw}{\bm{w}}
\newcommand{\bmx}{\bm{x}}
\newcommand{\bmy}{\bm{y}}
\newcommand{\bmz}{\bm{z}}
\newcommand{\bmzero}{\bm{0}}
\newcommand{\bmone}{\bm{1}}
\newcommand{\bmalpha}{\bm{\alpha}}
\newcommand{\bmbeta}{\bm{\beta}}
\newcommand{\bmgamma}{\bm{\gamma}}
\newcommand{\bmdelta}{\bm{\delta}}
\newcommand{\bmepsilon}{\bm{\epsilon}}
\newcommand{\bmtheta}{\bm{\theta}}
\newcommand{\bmiota}{\bm{\iota}}
\newcommand{\bmkappa}{\bm{\kappa}}
\newcommand{\bmlambda}{\bm{\lambda}}
\newcommand{\bmmu}{\bm{\mu}}
\newcommand{\bmnu}{\bm{\nu}}
\newcommand{\bmxi}{\bm{\xi}}
\newcommand{\bmpi}{\bm{\pi}}
\newcommand{\bmrho}{\bm{\rho}}
\newcommand{\bmsigma}{\bm{\sigma}}
\newcommand{\bmtau}{\bm{\tau}}
\newcommand{\bmupsilon}{\bm{\upsilon}}
\newcommand{\bmphi}{\bm{\phi}}
\newcommand{\bmchi}{\bm{\chi}}
\newcommand{\bmpsi}{\bm{\psi}}
\newcommand{\bmomega}{\bm{\omega}}
\newcommand{\bmA}{\bm{A}}
\newcommand{\bmB}{\bm{B}}
\newcommand{\bmC}{\bm{C}}
\newcommand{\bmD}{\bm{D}}
\newcommand{\bmE}{\bm{E}}
\newcommand{\bmF}{\bm{F}}
\newcommand{\bmG}{\bm{G}}
\newcommand{\bmH}{\bm{H}}
\newcommand{\bmI}{\bm{I}}
\newcommand{\bmJ}{\bm{J}}
\newcommand{\bmK}{\bm{K}}
\newcommand{\bmL}{\bm{L}}
\newcommand{\bmM}{\bm{M}}
\newcommand{\bmN}{\bm{N}}
\newcommand{\bmO}{\bm{O}}
\newcommand{\bmP}{\bm{P}}
\newcommand{\bmQ}{\bm{Q}}
\newcommand{\bmR}{\bm{R}}
\newcommand{\bmS}{\bm{S}}
\newcommand{\bmT}{\bm{T}}
\newcommand{\bmU}{\bm{U}}
\newcommand{\bmV}{\bm{V}}
\newcommand{\bmW}{\bm{W}}
\newcommand{\bmX}{\bm{X}}
\newcommand{\bmY}{\bm{Y}}
\newcommand{\bmZ}{\bm{Z}}
\newcommand{\bmGamma}{\bm{\Gamma}}
\newcommand{\bmDelta}{\bm{\Delta}}
\newcommand{\bmTheta}{\bm{\Theta}}
\newcommand{\bmLambda}{\bm{\Lambda}}
\newcommand{\bmXi}{\bm{\Xi}}
\newcommand{\bmPi}{\bm{\Pi}}
\newcommand{\bmSigma}{\bm{\Sigma}}
\newcommand{\bmUpsilon}{\bm{\Upsilon}}
\newcommand{\bmPhi}{\bm{\Phi}}
\newcommand{\bmPsi}{\bm{\Psi}}
\newcommand{\bmOmega}{\bm{\Omega}}
\newcommand{\calA}{\mathcal{A}}
\newcommand{\calB}{\mathcal{B}}
\newcommand{\calC}{\mathcal{C}}
\newcommand{\calD}{\mathcal{D}}
\newcommand{\calE}{\mathcal{E}}
\newcommand{\calF}{\mathcal{F}}
\newcommand{\calG}{\mathcal{G}}
\newcommand{\calH}{\mathcal{H}}
\newcommand{\calI}{\mathcal{I}}
\newcommand{\calJ}{\mathcal{J}}
\newcommand{\calK}{\mathcal{K}}
\newcommand{\calL}{\mathcal{L}}
\newcommand{\calM}{\mathcal{M}}
\newcommand{\calN}{\mathcal{N}}
\newcommand{\calO}{\mathcal{O}}
\newcommand{\calP}{\mathcal{P}}
\newcommand{\calQ}{\mathcal{Q}}
\newcommand{\calR}{\mathcal{R}}
\newcommand{\calS}{\mathcal{S}}
\newcommand{\calT}{\mathcal{T}}
\newcommand{\calU}{\mathcal{U}}
\newcommand{\calV}{\mathcal{V}}
\newcommand{\calW}{\mathcal{W}}
\newcommand{\calX}{\mathcal{X}}
\newcommand{\calY}{\mathcal{Y}}
\newcommand{\calZ}{\mathcal{Z}}
\newcommand{\bbC}{\mathbb{C}}
\newcommand{\bbE}{\mathbb{E}}
\newcommand{\bbI}{\mathbb{I}}
\newcommand{\bbN}{\mathbb{N}}
\newcommand{\bbQ}{\mathbb{Q}}
\newcommand{\bbR}{\mathbb{R}}
\newcommand{\bbZ}{\mathbb{Z}}
\newcommand{\tabincell}[2]{\begin{tabular}{@{}#1@{}}#2\end{tabular}}
\newcommand{\modelname}{\textsf{MISSRec}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{abstract}
The goal of sequential recommendation (SR) is to predict a user's potential interested items based on her/his historical interaction sequences.
Most existing sequential recommenders are developed based on ID features, which, despite their widespread use, often underperform with sparse IDs and struggle with the cold-start problem.
Besides, inconsistent ID mappings hinder the model's transferability, isolating similar recommendation domains that could have been co-optimized.
This paper aims to address these issues by exploring the potential of multi-modal information in learning robust and generalizable sequence representations.
We propose \textbf{\modelname{}}, a multi-modal pre-training and transfer learning framework for SR.
On the user side, we design a Transformer-based encoder-decoder model, where the contextual encoder learns to capture the
sequence-level multi-modal user interests % synergy
while a novel interest-aware decoder is developed to grasp item-modality-interest relations for better sequence representation.
On the candidate item side, we adopt a dynamic fusion module to produce user-adaptive item representation, providing more precise matching between users and items.
We pre-train the model with contrastive learning objectives and fine-tune it in an efficient manner.
Extensive experiments demonstrate the effectiveness and flexibility of \modelname{}, promising a practical solution for real-world recommendation scenarios.
Data and code are available on \url{https://github.com/gimpong/MM23-MISSRec}.
\end{abstract}
\section{Introduction}
\label{sec: introduction}
\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth]{imgs/intro_final.pdf}
\caption{Challenges of multi-modal sequential recommendation.
(a) The multi-modal synergy within each item can be user-dependent and dynamic.
Users may interact with the same item focusing on different modalities.
(b) Redundant interaction may overwhelm sparse but important interests. Modeling interactions equally will lead to biased predictions.
}
\label{fig:intro}
\end{figure}
Recommendation systems~\cite{RecSys,LightGCN,su2021detecting,su2022detecting} are an important component in various application domains, from e-commerce to content platforms, improving user engagement and satisfaction.
Sequential recommendation (SR)~\cite{SR_survey1,SR_survey2}, a popular methodology of recommendation, aims to predict users' potential interested items according to their historical interaction sequences.
Typically, this task is formulated as a representation learning problem, where user embeddings are learned to capture underlying preferences behind the interacted item sequence and to match with suitable item embeddings.
Current SR approaches~\cite{SASRec,BERT4Rec,S3-Rec,ICLRec,DFAR} predominantly rely on ID features as input, which, despite their widespread uses in practice, exhibit two primary shortcomings.
(\textbf{i}) The robustness and scalability largely depend on the distribution of user-item interaction.
Due to the sparsity of interaction data, it is challenging to learn accurate item and sequence representations based on ID information, let alone the cold-start problem~\cite{coldstart_problem} for new items.
Meanwhile, ID-based approaches often exhibit a popularity bias toward popular IDs~\cite{PopBias1,PopBias2}, posing the fairness issue for the majority of infrequent IDs.
(\textbf{ii}) Transferring knowledge to new scenarios is hard because of inconsistent ID mappings. It isolates similar domains that can be co-optimized, limiting the applicability of ID-based models.
Given the ubiquity of text and images in real-world scenarios, leveraging multi-modal data is promising to remedy the above shortcomings~\cite{yang2021unimf}.
Particularly, both text and visual information play crucial roles in attracting user attention -- compelling titles can boost user engagement, while visual elements like colors and shapes, can influence user decisions.
Considering remarkable progress in computer vision~\cite{ViT_Survey1,ViT_Survey2} and natural language understanding~\cite{Prompt_survey,LLM_survey}, they offer robust and generalizable semantic extraction capability to item content.
Therefore, we believe it necessary and also feasible to harness multi-modal content understanding to the SR.
However, such investigation remains in a fledgling stage, as our literature review (\S\ref{sec:related_work}) will show.
\emph{How does multi-modal information impact SR?
How to make better use of such information in sequence modeling?}
These two questions drive our study in this paper.
In general, utilizing multi-modal information in SR poses
two challenges.
(\textbf{i}) The multi-modal synergy within each item is user-dependent and dynamic.
Users may interact with the same item for various reasons, and different modalities contribute unequally to user interest in an item.
Even for the same user-item pair, such patterns can be time- and context-varying, making it difficult to design effective multi-modal fusion.
(\textbf{ii}) Information redundancy can overwhelm essential user interests.
Interaction sequences typically exhibit imbalanced interest distribution.
A user's sequence usually contains lots of homogeneous items, \eg daily necessities, while other informative items, \eg sports goods, may sparsely appear.
If we treat all interactions equally in user behavior modeling, it will result in overemphasis on specific kinds of items and insufficient focus on others.
We give an illustration of these challenges in \cref{fig:intro}.
To tackle these challenges, we propose \textbf{\modelname{}} (\underline{M}ulti-modal \underline{I}ntere\underline{S}t-aware \underline{S}equential \underline{Rec}ommendation), a multi-modal pre-training and transferring framework for SR.
In general, \modelname{} takes interaction sequences with multi-modal item information as input, learns ID-agnostic representations with an interest-aware encoder-decoder model via self-supervised pre-training, and can be efficiently adapted for multiple domains to enhance recommendation.
Specifically, (\textbf{i}) to bridge the semantic gap between general domains and recommendation, we incorporate multi-modal feature adapters~\cite{Adapter1,Adapter2} to distill personalized semantics of general multi-modal features in a parameter-efficient manner.
(\textbf{ii}) To explore the multi-modal synergy in the user-item interaction process,
% we design a Transformer-based contextual encoder for multi-modal sequence processing.
% Compared to item-level static fusion (\eg vector addition), our approach can adaptively capture useful modality cues in each item for personalization.
% on the candidate item side,
we
% also
introduce a lightweight dynamic fusion strategy to produce user-specific item representations.
(\textbf{iii}) To mitigate the adverse effect of information redundancy on sequence modeling, we introduce an interest discovery module to mine global multi-modal interests among users.
By associating items with relevant interests via adaptive clustering, we convert multi-modal sequences into a series of interest tokens.
(\textbf{iv}) We design a novel Transformer-based encoder-decoder architecture for interest-aware multi-modal sequence modeling, where the contextual encoder captures the relations among multi-modal user interests and the decoder grasps essential item-modality-interest patterns for comprehensive sequence representation.
We conduct detailed experiments and show the effectiveness of \modelname{} in leveraging multi-modal data to learn robust and transferable sequence representation.
In particular, it exhibits better generalizability than ID-based and text-based approaches and further alleviates the item cold-start issue.
To summarize, we make the following contributions in this paper.
\setlist{nolistsep}
\begin{itemize}[leftmargin=1.5em]
\item We highlight the significance and challenges of exploiting multi-modal information for SR and propose an effective pre-training and efficient transfer learning framework for it.
\item To capture the dynamic multi-modal synergy in the user-item interaction process, we
% design a Transformer-based contextual encoder for multi-modal sequential modeling and
adopt a lightweight dynamic fusion module to produce user-adaptive candidate item representation.
\item We introduce a multi-modal interest discovery module, based on which we construct an interest-aware decoder to model item-modality-interest relations for better sequence representation.
\item Extensive experiments of pre-training and downstream domain adaptation justify the comprehensive merits of our approach.
\end{itemize}
\section{Related Works}
\label{sec:related_work}
\subsection{Sequential Recommendation}
Sequential recommendation (SR) aims to predict the next suitable item for a given user based on her/his interacted item sequence~\cite{SR_survey1,SR_survey2}.
Early SR approaches~\cite{Fossil,FPMC} exploited the Markov chain mechanism for sequential modeling, while deep SR approaches explored CNN-~\cite{Caser,NextINet}, RNN-~\cite{DUPN,GRU4Rec}, MLP-~\cite{FMLP-Rec,MMMLP} and attention-based~\cite{SASRec,HGN,FDSA,BERT4Rec} architectures to model the transition patterns in the interaction sequences.
Besides, interest modeling~\cite{DIN,MIND,ComiRec,DFAR,Re4} has been another popular methodology for SR, where user interests were usually implemented by attention or clustering.
From other perspectives, extensive efforts have been devoted to designing effective learning strategies for SR, including temporal-aware learning tasks~\cite{TiSASRec,TCPSRec,RESETBERT} and contrastive learning objectives~\cite{CL4SRec,ICLRec,CL4SRec,S3-Rec,DuoRec,ContraRec}.
Note that most existing SR approaches are designed with ID features, \eg item IDs or attribute IDs.
They usually suffer from the cold-start problem and fall short of transferability.
Several SR approaches have leveraged multi-modal item contents to mitigate these issues.
For instance, for each item, CSAN~\cite{CSAN} aggregated multiple information including multi-modal features and projected the result to obtain an embedding, providing better item representation than the ID embedding alone.
MM-Rec~\cite{MM-Rec} adopted pre-trained VL-BERT~\cite{VL-BERT} as the multi-modal item encoder and built a single-tower model for SR.
Inspired by the success of MLP-Mixer~\cite{MLP-Mixer} in CV, MMMLP~\cite{MMMLP} adapted the architecture to multi-modal SR.
On the basis of these works, we further focus on learning universal sequence representations with multi-modal item content.
Our \modelname{} contributes a pre-training and efficient transfer framework for multi-modal SR, which can benefit real-world practice.
\subsection{Pre-training and Transfer Learning in Recommendation}
The paradigm of ``pre-train and transfer'' has become increasingly popular in recommendation.
Compared to cross-domain recommendation~\cite{CDR_Survey}, this paradigm is more general as it does not require cross-domain correspondence, \eg overlapped items.
CLUE~\cite{CLUE} and PeterRec~\cite{PETERec} designed ID-based sequence models and adopted contrastive learning objectives~\cite{MoCo,SimCLR} for model learning, where PeterRec further made the subsequent transfer parameter-efficient.
UPRec~\cite{UPRec} leveraged user profiles and social information to construct auxiliary pre-text tasks.
The assumption that both pre-training and target domains share ID vocabulary restricts their application scopes.
By contrast, modality-based methods are more flexible.
For instance, text-based SR pre-training approaches~\cite{UniSRec,VQRec} utilized a pre-trained language model (PLM, \eg BERT~\cite{BERT}) as the frozen feature extractor in item representation modules, showing efficacy and efficiency.
\citet{MoRec} designed text-based and image-based SR models under different scenarios, demonstrating the improvement from the tunable feature extractors.
\citet{TransRec} considered mixed modality sequences where each item is either a text or an image. They also chose to optimize the SR model and feature extractors jointly, improving the item representation but largely downgrading learning efficiency.
Furthermore, to capture the multi-modal synergy within each item, two latest works~\cite{MSM4SR,MMSRec} have focused on SR pre-training and transfer learning with multi-modal item representations.
Our \modelname{} also belongs to this setting, but we have fulfilled
two % three
improvements.
(\textbf{i}) To our best knowledge, we are the first to design interest-aware modeling for multi-modal SR.
(\textbf{ii}) We design user-adaptive candidate item fusion to model users' dynamic attention to different modalities.
\subsection{Multi-modal Recommendation}
Leveraging multi-modal information (\eg text and images) has been shown to improve the recommendation efficacy in many applications, \eg news feed~\cite{UNBERT,IMRec,liu2022boosting,MINER}, short-video feed~\cite{SEMI,jiang2020aspect,cai2021heterogeneous}, fashion e-commerce~\cite{VBPR,song2023mm}, and can effectively alleviate cold-start issues~\cite{MML,MTPR}.
Therefore, multi-modal recommendation (MMR) has become an active research topic~\cite{MMRec_Survey1,MMRec_Survey2,MMRec_Survey3}.
Early approaches explored MMR with matrix factorization models~\cite{VBPR,ACF}, while the latter ones mainly developed GNN-based models~\cite{MMGCN,GRCN,MKGAT} to demonstrate the efficacy.
On this basis, recent works have delved into self-supervised learning strategies~\cite{BM3,LATTICE,SLMRec,MMSSL,MMGCL,LCD} to enhance in-domain robustness or pre-training strategies~\cite{U-BERT,PMGT,MMCPR,CHEST,CPT-HG,HyperCTR} to improve cross-domain generalization.
Despite the remarkable progress in MMR, existing solutions were predominantly designed for non-sequential scenarios, \eg collaborative filtering~\cite{SimpleX}, which are not very suitable for SR.
Fortunately, our work uncovers some challenges in multi-modal SR (see \S\ref{sec: introduction} and \cref{fig:intro}) and fulfills effective solutions. We hope it can provide a research basis for future work.
\begin{figure*}[t]
\centering
\includegraphics[width=\textwidth]{imgs/arc_new.pdf}
\vspace{-2em}
\caption{\modelname{} consists of two stages.
In the first stage, it extracts multi-modal features from item content and transforms them into a token sequence.
Meanwhile, it adaptively converts the token sequence into multi-modal interest tokens.
Then, it encodes the mined interest sequence to capture contextual cues for personalization.
Next, by developing an interest-aware decoding mechanism, it produces a comprehensive sequence embedding.
Finally, it adopts sequence-item and sequence-sequence contrastive objectives for pre-training.
In the second stage, it uses the sequence-item objective for fine-tuning.
Both transductive (\ie w/ item IDs) and inductive (\ie ID-agnostic) tuning settings are supported.
The downstream domain adaptation is parameter-efficient as only the modality-specific adapters need to be tuned.
\emph{This figure is best viewed in color.}}
\label{fig:arc}
\end{figure*}
\section{Method}
\label{sec:method}
\subsection{Problem Formulation and Method Overview}
\label{subsec:overview}
Given the historical interaction sequence of the $i$-th user, $S_{i}=[ I_1, I_2, \cdots, I_{T_i} ]$, ordered by timestamps, sequential recommendation (SR) aims to predict the next item $I_{T_i+1}$ that the user may interact with by learning representations for $S_{i}$ and candidate items.
Under the multi-modal settings, each item contains a unique item ID and multi-modal content.
Without loss of generality, we study leveraging text and image modalities to improve SR in this paper.
We propose \textbf{\modelname{}}, a pre-training and transferring framework, as illustrated in \cref{fig:arc}.
\modelname{} consists of 7 main components:
(\textbf{i}, \S\ref{subsubsec:feat_extr}) Pre-trained, frozen text and image encoders for extracting multi-modal features.
(\textbf{ii}, \S\ref{subsubsec:adapter}) A pair of modality-specific adapters to transform multi-modal features into input tokens, which bridge the semantic gap between general features and personalization.
In the fine-tuning stage, they help with efficient model adaptation.
(\textbf{iii}, \S\ref{subsubsec:candidate_item_enc}) A dynamic fusion module for generating user-adaptive candidate item representations.
(\textbf{iv}, \S\ref{subsubsec:cluster_interests}) A multi-modal interest discovery module that generates user interest tokens for the decoder.
(\textbf{v}, \S\ref{subsubsec:mm_enc}) A contextual Transformer encoder for the interest sequence that dynamically grasps important cues for personalization.
(\textbf{vi}, \S\ref{subsubsec:mm_dec}) An interest-aware Transformer decoder to capture the item-modality-interest relation for better sequence representation.
(\textbf{vii}, \S\ref{subsec:pretrain} and \S\ref{subsec:fine_tuning}) The pre-training and downstream fine-tuning tasks to achieve transferable recommendation.
\subsection{Universal Multi-modal Item Representation}
\label{subsec:item_rep}
\subsubsection{Multi-modal Feature Extraction with Pre-trained Transformers}
\label{subsubsec:feat_extr}
\modelname{} utilizes multi-modal features to achieve universal representations.
Concretely, we take advantage of the pre-trained transformers for item content understanding.
Specifically, we use BERT~\cite{BERT} and ViT~\cite{ViT} for text and visual feature extraction, respectively.
We adopt the cross-modal pre-trained version~\cite{CLIP} following \citet{MSM4SR}.
For an item $I$, we obtain its text feature vector by $\bmf^\text{t}=\phi^\text{t}(I)$ and the image feature vector by $\bmf^\text{v}=\phi^\text{v}(I)$, where $\phi^\text{t}$ and $\phi^\text{v}$ denotes the text encoder the visual encoder, respectively.
\subsubsection{Feature Augmentation}
\label{subsubsec:feat_aug}
To enhance the robustness of the sequence representation model, we apply dropout~\cite{Dropout} as the feature augmentation.
Given the text features $\bmF^\text{t}_i=[\seq{\bmf^\text{t}}{T_i}]$, we apply twice the feature augmentation and subsequently obtain $\tilde{\bmF}_i^\text{t}=[\seq{\tilde{\bmf}^\text{t}}{T_i}]$ and
$\tilde{\bmF}_i^{\text{t}'}=[\seq{\tilde{\bmf}^{\text{t}'}}{T_i}]$ as the augmented text features.
They serve as two positive views \wrt $\bmF^\text{t}_i$.
The image features $\bmF^\text{v}_i$ follow analogous augmentation procedure.
\subsubsection{Bridge Domain Gap with Modality-specific Adapters}
\label{subsubsec:adapter}
We adopt modality-specific adapters~\cite{Adapter1,Adapter2} to reduce irrelevant semantics in multi-modal features and enhance the personalization.
We transform the augmented features and fuse the results as item tokens, for example, $\bmX_i=[\seq{\bmx}{L_i}]=[\psi^\text{t}(\tilde{F}^\text{t}_i);\psi^\text{v}(\tilde{F}^\text{v}_i)]$. $\psi^\text{t}$ and $\psi^\text{v}$ denote the text and visual adapters, respectively.
In contrast to tunable feature extractors, adapters with frozen extractors exhibits two strengths.
(\textbf{i}) It allows (pre-) training of the sequence model based on pre-extracted features, reducing time and memory overhead.
(\textbf{ii}) Adapters enable parameter-efficient transfer in downstream domains, which is preferable to fine-tuning.
\subsubsection{Candidate Item Representation with Dynamic Fusion}
\label{subsubsec:candidate_item_enc}
To model users' dynamic attention to different modalities of candidate items, we design a lightweight fusion module to generate user-adaptive item representations.
Given a candidate item $I_k$, we first encode it with modality encoders and adapters to obtain modality embeddings, \ie $\bmx_k^\text{t}$ and $\bmx_k^\text{v}$.
For the $i$-th user, we compute its sequence representation $\bmu_i$ according to the procedures in \S\ref{subsec:mid_module} and \S\ref{subsec:mm_enc_dec}.
Then, the adaptive representation of $I_k$ is defined by the weighted fusion of modality embeddings, namely
\begin{gather}\label{equ:user-adaptive-item-rep}
\bmv_k =
\frac{
e^{\alpha\cdot\langle \bmu_i,\bmx_k^\text{t} \rangle}\cdot\bmx_k^\text{t}+
e^{\alpha\cdot\langle \bmu_i,\bmx_k^\text{v} \rangle}\cdot\bmx_k^\text{v}
}{
e^{\alpha\cdot\langle \bmu_i,\bmx_k^\text{t} \rangle}+
e^{\alpha\cdot\langle \bmu_i,\bmx_k^\text{v} \rangle}
}.
\end{gather}
$\langle\cdot,\cdot\rangle$ denotes the inner product.
$\alpha\ge0$ is a concentration factor to balance two modalities.
Besides the efficacy, the fusion keeps efficiency as well, because it is factorizable when computing matching scores.
Specifically, let us define $s^\text{t}_{i,k}=\langle\bmu_i,\bmv_j^\text{t}\rangle$ and $s^\text{v}_{i,k}=\langle\bmu_i,\bmv_j^\text{v}\rangle$, the matching score between $\bmu_i$ and $\bmv_k$ can be compute by
\begin{gather}\label{equ:factorizable-ui-match-score}
\langle\bmu_i,\bmv_k\rangle=\frac{s^\text{t}_{i,k}\cdot e^{\alpha\cdot s^\text{t}_{i,k}}+s^\text{v}_{i,k}\cdot e^{\alpha\cdot s^\text{v}_{i,k}}
}
{e^{\alpha\cdot s^\text{t}_{i,k}}+e^{\alpha\cdot s^\text{v}_{i,k}}}.
\end{gather}
We can equivalently pre-compute modality-specific scores and fuse them to get the overall score.
In particular, dynamic fusion exhibits a transition between mean and max poolings:
$\langle\bmu_i,\bmv_k\rangle=(s^\text{t}_{i,k}+s^\text{v}_{i,k})/2$ when $\alpha=0$, while $\langle\bmu_i,\bmv_k\rangle=\max(s^\text{t}_{i,k},s^\text{v}_{i,k})$ when $\alpha\rightarrow+\infty$.
\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth]{imgs/mid_module.pdf}
\vspace{-2em}
\caption{Convert item tokens to interest tokens via the multi-modal interest discovery (MID) module. An item can be converted to different interests according to multiple modalities.}
\label{fig:mid_module}
\end{figure}
\subsection{Discover Multi-modal User Interests}
\label{subsec:mid_module}
Typically, item sequences reflect users' implicit interests that are important for SR.
Interest modeling can reduce the negative impact of information redundancy, helping to capture personalized semantics for precise representation.
In this sub-section, we introduce a multi-modal interest discovery (MID) module to assist this goal.
\subsubsection{Excavate and Index Interests via Adaptive Clustering}
\label{subsubsec:cluster_interests}
% TODO: 分析实验，用时对比，放在附录里
During the training process, item tokens are expected to convey both transferable multi-modal information and personalized semantics.
Hence, we adopt a variant of $k$-nearest neighbor-based density peaks clustering algorithm (DPC-KNN)~\cite{DPC-KNN} on the whole item token set $\calX=\{\bmx_i\}_{i=1}^{N_I}$, to explore the distribution of user interests before every training epoch starts.
First, for the $i$-th token, we compute its local density by considering its $k$-nearest neighbors:
\begin{gather}
\rho_i=\exp\xiaok{-\frac{1}{k}\sum_{\bmx_j\in k\text{NN}(\bmx_i)}\shuk{\bmx_i-\bmx_j}_2^2}.
\end{gather}
Then, for the $i$-th token, we compute its minimum distance to any other token with higher local density, namely
\begin{gather}
\delta_i=
\begin{cases}
\min_{j:\rho_j>\rho_i}\shuk{\bmx_i-\bmx_j}_2, & \text{if }\exists j\text{ s.t. }\rho_j>\rho_i, \\
\max_{j}\shuk{\bmx_i-\bmx_j}_2, &\text{otherwise}.
\end{cases}
\end{gather}
Next, we select $K_\text{c}$ cluster centroids with the highest $\rho_i\times\delta_i$ scores as the interest prototypes.
Finally, we index each token by assigning it to the nearest centroid (\ie the interest prototype).
\subsubsection{Translate Multi-modal Item Tokens into Interest Tokens}
\label{subsubsec:transform_interest}
As shown in \cref{fig:arc,fig:mid_module}, the MID module receives item tokens, finds the nearest interest prototype for each token, and returns irredundant interest tokens.
To accelerate token conversion, we
% maintain
can build
the interest index of each item token after clustering for fast lookup.
\subsection{Multi-modal Interest-aware Sequence Model}
\label{subsec:mm_enc_dec}
We design an encoder-decoder model for sequence representation, which unifies sequence modeling and multi-modal fusion.
\subsubsection{Transformer-based Multi-modal Context Encoder}
\label{subsubsec:mm_enc}
To pursue personalization, we expect the model to adaptively capture crucial cues from the given context, \ie discovered interest tokens (\S\ref{subsubsec:transform_interest}), denoted by $\seq{\bmbeta}{M_i}$.
We construct an encoder with $K_\text{e}$ Transformer encoder blocks, each of which follows the standard design~\cite{attention}.
The context encoder processes the input and generates an encoded token sequence $\seq{\bmxi}{M_i}$ for next-step decoding.
\subsubsection{Transformer-based Interest-aware Decoder}
\label{subsubsec:mm_dec}
After obtaining the encoded interest tokens $\seq{\bmxi}{M_i}$, we design an interest-aware decoding process to model item-modality-interest relations for precise and comprehensive sequence representation.
Specifically, we construct a decoder with $K_\text{d}$ Transformer decoder blocks.
For the multi-head cross-attention module in each decoder block, we take $\seq{\bmxi}{M_i}$ as the key and value.
We add item-wise positional embeddings $\bmP_i=[\seq{\bmp}{L_i}]$ to the item tokens $\bmX_i=[\seq{\bmx}{L_i}]$ and form the decoding queries.
Let us denote $\bmY_i=[\seq{\bmy}{L_i}]$ as the decoded output \wrt $\bmX_i$, then the embedding (\ie the sequence representation) for the $i$-th user is produced by the output of the last query token, namely $\bmu_i=\bmy_{L_i}$.
\subsection{Self-supervised Contrastive Pre-training}
\label{subsec:pretrain}
To pursue universal representation for multi-modal interaction sequences, we design two self-supervised contrastive learning (CL) tasks for pre-training the model.
\subsubsection{Sequence-item Contrastive Learning}
\label{subsubsec:seq_item_cl}
To capture the correspondence between interaction sequences and candidate items, we take sequence-item CL as a pre-training task. The objective \wrt $\bmu_i$:
\begin{equation}
\label{equ:SI-CL}
\ell_i^{S-I}=-\log\frac
{\exp({\langle\bmu_i,\bmv_{T_i+1}\rangle/\tau})}
{\sum_{j=1}^B\exp({\langle\bmu_i,\bmv_{T_j+1}\rangle/\tau})},
\end{equation}
where $B$ denotes the size of mini-batch. $\tau>0$ us a temperature factor.
The user-item matching scores are computed by \cref{equ:factorizable-ui-match-score}.
\subsubsection{Sequence-sequence Contrastive Learning}
\label{subsubsec:seq_seq_cl}
Apart from sequence-item contrast, we leverage semantic invariance as a self-supervising signal to enhance the robustness of the sequence model. Given $\bmu_i,\bmu_i'$ as the representations of two augmented sequences (see \S\ref{subsubsec:feat_aug}) \wrt the $i$-th user, we define the sequence-sequence CL by
\begin{equation}
\label{equ:SS-CL}
\ell_i^{S-S}=-
% \!
\log
% \!
\frac
{\exp({\langle\bmu_i,\bmu_i'\rangle/\tau})}
{\sum_{j=1}^B
% \!
% (
\exp({\langle\bmu_i,\bmu_j\rangle/\tau}) + %\! + \!
\exp({\langle\bmu_i,\bmu_j'\rangle/\tau})
% )
}.
\end{equation}
\subsubsection{Overall Objectives}
\label{subsubsec:overall_objs}
We sum up pre-train objectives as follows
\begin{gather}
\label{equ:pre-train}
\calL_\text{pre-train}=\frac1B\sum_{i=1}^B\zhongk{\ell_i^{S-I}+\lambda\cdot \ell_i^{S-S} + \frac{\gamma}{M_i^2} \sum_{m,m'=1}^{M_i}\langle\bmxi_m,\bmxi_{m'}\rangle}.
\end{gather}
The last term is an orthogonal regularization to diversify interest-aware decoded results.
$\lambda, \gamma>0$ are the weights for different terms.
\subsection{Efficient Fine-tuning}
\label{subsec:fine_tuning}
In order to transfer the recommendation knowledge to downstream domains and enhance their performance, we further study efficient fine-tuning based on the pre-trained sequence model.
For the tuning objectives, we adopt sequence-item CL and orthogonal regularization to directly optimize the next-item prediction, namely,
\begin{gather}
\label{equ:fine-tune}
\calL_\text{fine-tune}=\frac1B\sum_{i=1}^B\zhongk{\ell_i^{S-I}+ \frac{\gamma}{M_i^2} \sum_{m,m'=1}^{M_i}\langle\bmxi_m,\bmxi_{m'}\rangle}.
\end{gather}
As shown in the right block of \cref{fig:arc}, we disable the feature augmentation and the sequence-sequence contrast, so the sequence model only encodes once per user, which is computation-efficient.
Besides, only the item ID embedding table (if included) and modality-specific adapters need to be tuned while other modules keep frozen, which is also parameter-efficient~\cite{UnifiedPETL}.
Following ~\citet{UniSRec}, we support \emph{inductive} and \emph{transductive} transfer settings for \modelname{}.
\subsubsection{Inductive Transfer}
\label{subsubsec:inductive_transfer}
Given a target domain with lots of cold items, making robust predictions is tricky for ID-based recommenders.
We support an ID-agnostic solution to deal with extreme sparsity, where the model predicts next item for the $i$-th user by
\begin{gather}
\text{Pr}_\text{ind}(I_{T_i+1}\mid \seq{I}{T_i})=\text{softmax}(\langle\bmu_i,\bmv_{T_i+1}\rangle).
\end{gather}
\subsubsection{Transductive Transfer}
\label{subsubsec:transductive_transfer}
Given a target domain of mostly warm items, we pursue more precise recommendations by utilizing item IDs.
Specifically, the model predicts next item for the $i$-th user by
\begin{gather}
\text{Pr}_\text{trd}(I_{T_i+1}\mid \seq{I}{T_i})=\text{softmax}(\langle\bar{\bmu}_i,\bmv_{T_i+1}+\bmz_{T_i+1}\rangle),
\end{gather}
where $\bmz_{T_i+1}$ is the ID embeddings of $I_{T_i+1}$. $\bar{\bmu}_i$ denotes the encoded sequence representation with item IDs (see the right block in \cref{fig:arc}).
\section{Experiments}
\label{sec:experiments}
\subsection{Research Questions}
\label{subsec:research_questions}
We evaluate the proposed method by conducting experiments on three datasets. We aim to answer the following research
questions:
% \setlist{nolistsep}
\begin{itemize}%[leftmargin=1.75em]
\item[\textbf{RQ1:}] Compared with state-of-the-art sequential recommendation models using various types of information, can \modelname{} achieve competitive performance in downstream domains?
\item[\textbf{RQ2:}] Is \modelname{} better than other modality-based baselines in utilizing different modalities? How much impact do multi-modal information and pre-training have on its efficacy?
\item[\textbf{RQ3:}] How do different designs contribute to \modelname{}'s efficacy?
\end{itemize}
\begin{table}[t] %
\caption{Statistics of Pre-processed Datasets. ``Cover.'' denotes the image coverage among the item set. ``Avg. SL'' denotes the average length of interaction sequences.}
% \vspace{-1em}
\label{tab:dataset}
\resizebox{\columnwidth}{!}{
\begin{tabular}{l *{5}{r}}
\toprule
Datasets & \#Users & \#Items & \#Img. (Cover./\%) & \#Inters. & Avg. SL.\\
\hline
\midrule
\emph{Pre-trained} & 1,361,408 & 446,975 & 94,151 (21.06\%) & 14,029,229 & 13.51 \\
- Food & 115,349 & 39,670 & 29,990 (75.60\%) & 1,027,413 & 8.91 \\
- CDs & 94,010 & 64,439 & 21,166 (32.85\%) & 1,118,563 & 12.64 \\
- Kindle & 138,436 & 98,111 & 0 (0\%) & 2,204,596 & 15.93 \\
- Movies & 281,700 & 59.203 & 8,675 (14.65\%) & 3,226,731 & 11.45 \\
- Home & 731,913 & 185,552 & 34,320 (18.50\%) & 6,451,926 & 8.82 \\
\midrule
Scientific & 8,442 & 4,385 & 1,585 (36.15\%) & 59,427 & 7.04 \\
Pantry & 13,101 & 4,898 & 4,587 (93.65\%) & 126,962 & 9.69 \\
Instruments & 24,962 & 9,964 & 6,289 (63.12\%) & 208,926 & 8.37 \\
Arts & 45,486 & 21,019 & 9,437 (44.90\%) & 395,150 & 8.69 \\
Office & 87,436 & 25,986 & 16,628 (63.99\%) & 684,837 & 7.84 \\
\bottomrule
\end{tabular}
}
\end{table}
\subsection{Experimental Setup}
\label{subsec:setup}
\subsubsection{Datasets}
\label{subsubsec:dataset_setup}
We adopt 10 domains including ``\emph{Grocery and Gourmet Food}'', ``\emph{Home and Kitchen}'', ``\emph{CDs and Vinyl}'', ``\emph{Kindle Store}'', ``\emph{Movies and TV}'', ``\emph{Prime Pantry}'', ``\emph{Industrial and Scientific}'', ``\emph{Musical Instruments}'', ``\emph{Arts, Crafts and Sewing}'', and ``\emph{Office Products}'', from the standard benchmark dataset, \textbf{Amazon Review}~\cite{AmazonReview}.
To provide extensive evaluations of the transferability, We divide the former 5 and the latter 5 into pre-training and downstream target domains, respectively.
We follow \citet{UniSRec} to pre-process for interaction data and retrieve text information including \emph{title}, \emph{categories} and \emph{brand} from the metadata.
To support multi-modal inputs, we further crawl item images according to the URLs in the metadata.
The dataset statistics are reported in \cref{tab:dataset}.
While texts are fully available in the metadata, there are many missing images in the dataset due to expired products or URLs.
We still retain the items without images in our experiments to keep a fair comparison with previous works~\cite{UniSRec}.
As we will show below, our \modelname{} can exhibit robust improvement on incomplete multi-modal datasets.
\subsubsection{Metrics}
\label{subsubsec:metrics_setup}
Following previous works~\cite{UniSRec,S3-Rec}, we adopt two standard metrics, \ie \textbf{Recall} (\textbf{R@$K$}) and \textbf{NDCG} (\textbf{N@$K$}), to evaluate the retrieval performance.
We set $K$ to 10 and 50 for showcases.
\subsubsection{Baselines}
\label{subsubsec:baselines_setup}
We compare our method with 7 state-of-the-art sequential recommenders, including
(\textbf{i}) 2 pure ID-based recommenders: SASRec~\cite{SASRec} and
BERT4Rec~\cite{BERT4Rec};
(\textbf{ii}) 1 attribute-aware recommender,
S$^3$-Rec~\cite{S3-Rec};
(\textbf{iii}) 4 text-based recommenders,
FDSA~\cite{FDSA},
ZESRec~\cite{ZESRec}, and
UniSRec~\cite{UniSRec}.
We derive an extra text-feature-based variant from SASRec regarding its simplicity and effectiveness.
For S$^3$-Rec, we inherit previous works~\cite{UniSRec} to embed the tags with textual features so as to support universal representation.
We adopt both transductive and inductive variants of UniSRec for comparison.
Therefore, we have 9 baselines in total.
\begin{table*}[t]
\centering
\caption{Comparisons on different target datasets.
``T'' and ``V'' stands for text and visual features.
``\emph{Improv}.'' denotes the relative improvement of \modelname{} to the best baselines.
The best and second-best results are in bold and underlined, respectively.}
% \vspace{-1em}
\setlength{\tabcolsep}{0.5em}{
\resizebox{1.0\textwidth}{!}{
\begin{tabular}{lclccccccrlccccr}
\toprule
\multicolumn{2}{l}{\makecell{Input Type \& Model $\rightarrow$}} & & \multicolumn{2}{c}{ID} & \multicolumn{3}{c}{T+ID} & T+V+ID & \multirow{2}{*}{\makecell{\emph{Improv.} \\ w/ ID}} & & \multicolumn{3}{c}{T} & T+V & \multirow{2}{*}{\makecell{\emph{Improv.} \\ w/o ID}} \\
\cmidrule(l){1-2} \cmidrule(l){4-5} \cmidrule(l){6-8} \cmidrule(l){9-9} \cmidrule(l){12-14} \cmidrule(l){15-15}
Dataset & Metric & & {\small SASRec} & {\footnotesize BERT4Rec} & FDSA & S$^3$-Rec & {\small UniSRec} & {\small\modelname{}} & & & {\small SASRec} & {\small ZESRec} & {\small UniSRec} & {\small\modelname{}} & \\
\hline
\midrule
\multirow{4}{*}{Scientific} & R@10 & & 0.1080 & 0.0488 & 0.0899 & 0.0525 & \underline{0.1235} & \textbf{0.1360} & \cellcolor[HTML]{E9EAFF}10.12\% & & 0.0994 & 0.0851 & \underline{0.1188} & \textbf{0.1278} & \cellcolor[HTML]{EFFBEC}7.58\% \\
& N@10 & & 0.0553 & 0.0243 & 0.0580 & 0.0275 & \underline{0.0634} & \textbf{0.0753} & \cellcolor[HTML]{E9EAFF}18.77\% & & 0.0561 & 0.0475 & \underline{0.0641} & \textbf{0.0658} & \cellcolor[HTML]{EFFBEC}2.65\% \\
& R@50 & & 0.2042 & 0.1185 & 0.1732 & 0.1418 & \textbf{0.2473} & \underline{0.2431} & \cellcolor[HTML]{E9EAFF}- & & 0.2162 & 0.1746 & \textbf{0.2394} & \underline{0.2375} & \cellcolor[HTML]{EFFBEC}- \\
& N@50 & & 0.0760 & 0.0393 & 0.0759 & 0.0468 & \underline{0.0904} & \textbf{0.0983} & \cellcolor[HTML]{E9EAFF}8.74\% & & 0.0815 & 0.0670 & \textbf{0.0903} & \underline{0.0893} & \cellcolor[HTML]{EFFBEC}- \\
\midrule
\multirow{4}{*}{Pantry} & R@10 & & 0.0501 & 0.0308 & 0.0395 & 0.0444 & \underline{0.0693} & \textbf{0.0779} & \cellcolor[HTML]{E9EAFF}12.41\% & & 0.0585 & 0.0454 & \underline{0.0636} & \textbf{0.0771} & \cellcolor[HTML]{EFFBEC}21.23\% \\
& N@10 & & 0.0218 & 0.0152 & 0.0209 & 0.0214 & \underline{0.0311} & \textbf{0.0365} & \cellcolor[HTML]{E9EAFF}17.36\% & & 0.0285 & 0.0230 & \underline{0.0306} & \textbf{0.0345} & \cellcolor[HTML]{EFFBEC}12.75\% \\
& R@50 & & 0.1322 & 0.1030 & 0.1151 & 0.1315 & \underline{0.1827} & \textbf{0.1875} & \cellcolor[HTML]{E9EAFF}2.63\% & & 0.1647 & 0.1141 & \underline{0.1658} & \textbf{0.1833} & \cellcolor[HTML]{EFFBEC}10.55\% \\
& N@50 & & 0.0394 & 0.0305 & 0.0370 & 0.0400 & \underline{0.0556} & \textbf{0.0598} & \cellcolor[HTML]{E9EAFF}7.55\% & & 0.0523 & 0.0378 & \underline{0.0527} & \textbf{0.0571} & \cellcolor[HTML]{EFFBEC}8.35\% \\
\midrule
\multirow{4}{*}{Instruments} & R@10 & & 0.1118 & 0.0813 & 0.1070 & 0.1056 & \underline{0.1267} & \textbf{0.1300} & \cellcolor[HTML]{E9EAFF}2.60\% & & 0.1127 & 0.0783 & \underline{0.1189} & \textbf{0.1201} & \cellcolor[HTML]{EFFBEC}1.01\% \\
& N@10 & & 0.0612 & 0.0620 & \underline{0.0796} & 0.0713 & 0.0748 & \textbf{0.0843} & \cellcolor[HTML]{E9EAFF}5.90\% & & 0.0661 & 0.0497 & \underline{0.0680} & \textbf{0.0771} & \cellcolor[HTML]{EFFBEC}13.38\% \\
& R@50 & & 0.2106 & 0.1454 & 0.1890 & 0.1927 & \textbf{0.2387} & \underline{0.2370} & \cellcolor[HTML]{E9EAFF}- & & 0.2104 & 0.1387 & \textbf{0.2255} & \underline{0.2218} & \cellcolor[HTML]{EFFBEC}- \\
& N@50 & & 0.0826 & 0.0756 & 0.0972 & 0.0901 & \underline{0.0991} & \textbf{0.1071} & \cellcolor[HTML]{E9EAFF}8.07\% & & 0.0873 & 0.0627 & \underline{0.0912} & \textbf{0.0988} & \cellcolor[HTML]{EFFBEC}8.33\% \\
\midrule
\multirow{4}{*}{Arts} & R@10 & & 0.1108 & 0.0722 & 0.1002 & 0.1003 & \underline{0.1239} & \textbf{0.1314} & \cellcolor[HTML]{E9EAFF}6.05\% & & 0.0977 & 0.0664 & \underline{0.1066} & \textbf{0.1119} & \cellcolor[HTML]{EFFBEC}4.97\% \\
& N@10 & & 0.0587 & 0.0479 & \underline{0.0714} & 0.0601 & 0.0712 & \textbf{0.0767} & \cellcolor[HTML]{E9EAFF}7.42\% & & 0.0562 & 0.0375 & \underline{0.0586} & \textbf{0.0625} & \cellcolor[HTML]{EFFBEC}6.66\% \\
& R@50 & & 0.2030 & 0.1367 & 0.1779 & 0.1888 & \underline{0.2347} & \textbf{0.2410} & \cellcolor[HTML]{E9EAFF}2.68\% & & 0.1916 & 0.1323 & \underline{0.2049} & \textbf{0.2100} & \cellcolor[HTML]{EFFBEC}2.49\% \\
& N@50 & & 0.0788 & 0.0619 & 0.0883 & 0.0793 & \underline{0.0955} & \textbf{0.1002} & \cellcolor[HTML]{E9EAFF}4.92\% & & 0.0766 & 0.0518 & \underline{0.0799} & \textbf{0.0836} & \cellcolor[HTML]{EFFBEC}4.63\% \\
\midrule
\multirow{4}{*}{Office} & R@10 & & 0.1056 & 0.0825 & 0.1118 & 0.1030 & \textbf{0.1280} & \underline{0.1275} & \cellcolor[HTML]{E9EAFF}- & & 0.0929 & 0.0641 & \underline{0.1013} & \textbf{0.1038} & \cellcolor[HTML]{EFFBEC}2.47\% \\
& N@10 & & 0.0710 & 0.0634 & \textbf{0.0868} & 0.0653 & 0.0831 & \underline{0.0856} & \cellcolor[HTML]{E9EAFF}- & & 0.0582 & 0.0391 & \underline{0.0619} & \textbf{0.0666} & \cellcolor[HTML]{EFFBEC}7.59\% \\
& R@50 & & 0.1627 & 0.1227 & 0.1665 & 0.1613 & \textbf{0.2016} & \underline{0.2005} & \cellcolor[HTML]{E9EAFF}- & & 0.1580 & 0.1113 & \textbf{0.1702} & \underline{0.1701} & \cellcolor[HTML]{EFFBEC}- \\
& N@50 & & 0.0835 & 0.0721 & 0.0987 & 0.0780 & \underline{0.0991} & \textbf{0.1012} & \cellcolor[HTML]{E9EAFF}2.12\% & & 0.0723 & 0.0493 & \underline{0.0769} & \textbf{0.0808} & \cellcolor[HTML]{EFFBEC}5.07\% \\
\hline
\bottomrule
\end{tabular}
}
}
\label{tab:main_results}
\end{table*}
\subsubsection{Implementation Details}
\label{subsubsec:baselines_setup}
% The training batch size is 2048.
We take Adam as the common optimizer in our comparison, and we carefully choose suitable learning rates from $\{3e^{-4}, 1e^{-3}, 3e^{-3}, 1e^{-2}\}$ for different baselines.
We take 100 epochs for pre-training.
And for fine-tuning, we implement early stopping with a patience of 10 to prevent over-fitting.
We adopt CLIP-B/32~\cite{CLIP,HuggingHash} as the basic feature encoder to extract
% 512-d
\texttt{[CLS]} features for each text and image.
The modality adapters then transform the features into the 300-d latent space for sequence representation.
In the MID module (\S\ref{subsec:mid_module}), the number of interest prototypes, $K_\text{c}$, is set to 2\% of the item amount during pre-training, and is set to 50\% of the item amount during fine-tuning.
To keep a consistent parameter scale with baselines, we build our model with 1 Transformer encoder block and 1 decoder block, \ie $K_\text{e}=K_\text{d}=1$.
The loss weights in \cref{equ:pre-train,equ:fine-tune} are set $\lambda=1e^{-3}$ and $\gamma=1e^{-4}$, respectively.
The temperature factor for contrastive learning objectives, \eg \cref{equ:SI-CL,equ:SS-CL}, is set $\tau=0.07$.
The concentration factor $\alpha$ in \S\ref{subsubsec:candidate_item_enc} is set to a learnable variable.
The data and code have been released on \url{https://github.com/gimpong/MM23-MISSRec}.
\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth]{imgs/cold_start.pdf}
% \vspace{-2em}
\caption{Performance comparison on long-tailed items.
The histogram represents the number of test samples (\ie interacted items) of different popularities.
The line chart shows the relative improvement to SASRec$_\text{ID}$ on Recall@10.}
\label{fig:cold_start_analysis}
\end{figure}
\subsection{Comparasion with State-of-the-arts (RQ1)}
\label{subsec:sota}
\subsubsection{Overall Performance}
\label{subsubsec:ovearll_sota}
The overall comparison results on five different downstream datasets are shown in \cref{tab:main_results}, from which we obtain three findings.
(\textbf{i}) Text features can be exploited as an effective alternative or supplement to ID features, and pre-training can further enhance their efficacy.
Comparing the SASRec variants with different types of input, the text-based variant achieves competitive performance and outperforms the ID-based variant on ``Pantry'' and ``Instruments'' datasets.
Besides, text-enhanced approaches FDSA and S$^3$-Rec outperform ID-based BERT4Rec by large margins.
By adopting pre-training, UniSRec achieves further improvement.
(\textbf{ii}) ID information is still important for personalization.
Transductive methods generally perform better than inductive ones.
(\textbf{iii}) Under both transductive and inductive settings, our \modelname{} outperforms state-of-the-art baselines in most cases, and the margins are considerable.
We attribute the performance gain to three factors.
First, multi-modal information reflects more precise and comprehensive user preferences, benefiting personalization.
Second, the synergy of different designs in \modelname{} contributes to the efficacy of multi-modal sequence representation.
Third, pre-training further strengthened the above advantages.
\subsubsection{Performance \wrt Different Item Frequencies}
\label{subsubsec:frequency_sota}
In this sub-section, we select the best performers from \cref{tab:main_results} and compare their efficacy \wrt long-tailed items.
As shown in \cref{fig:cold_start_analysis}, all modality-based methods demonstrate effectiveness to ID-based SASRec on the least frequent items, while the performance gains begin to vanish and even turn to degradation as the items warm up.
Compared with other baselines, \modelname{} with multi-modal learning shows the best few-short performance and also keeps robust improvement with more frequent items, suggesting its feasibility in addressing long-tailed and cold-start recommendation issues.
\subsection{Multi-modal \& Pre-training Analyses (RQ2)}
\label{subsec:mm_pretrain_analyses}
\begin{table}[t] %
\caption{Model comparison \wrt various input types on the full-modality data subset. We can see the superior capability of \modelname{} in leveraging modality features.}
% \vspace{-1em}
\setlength{\tabcolsep}{0.3em}{
\resizebox{\columnwidth}{!}{
\begin{tabular}{cccccccccccc}
\toprule
& & & \multicolumn{4}{c}{Scientific} & & \multicolumn{4}{c}{Office} \\
\cmidrule(l){4-7} \cmidrule{9-12}
Input Type & Variant & & R@10 & R@50 & N@10 & N@50 & & R@10 & R@50 & N@10 & N@50 \\
\hline
\midrule
\multirow{2}{*}{ID+T} & UniSRec & & 0.1676 & 0.3200 & 0.0872 & 0.1208 & & 0.1376 & 0.2152 & 0.0937 & 0.1104 \\
& \modelname{} & & \textbf{0.1697} & \textbf{0.3249} & \textbf{0.0890} & \textbf{0.1227} & & \textbf{0.1398} & \textbf{0.2195} & \textbf{0.0953} & \textbf{0.1122} \\
\midrule
\multirow{2}{*}{ID+V} & UniSRec & & 0.1631 & 0.3112 & 0.0857 & 0.1188 & & 0.1324 & 0.2071 & 0.0921 & 0.1086 \\
& \modelname{} & & \textbf{0.1657} & \textbf{0.3165} & \textbf{0.0869} & \textbf{0.1198} & & \textbf{0.1375} & \textbf{0.2152} & \textbf{0.0946} & \textbf{0.1113} \\
\midrule
\multirow{2}{*}{ID+T+V} & UniSRec & & 0.1680 & 0.3209 & 0.0865 & 0.1194 & & 0.1407 & 0.2203 & 0.0957 & 0.1133 \\
& \modelname{} & & \textbf{0.1711} & \textbf{0.3265} & \textbf{0.0890} & \textbf{0.1227} & & \textbf{0.1421} & \textbf{0.2223} & \textbf{0.0966} & \textbf{0.1138} \\
\midrule
\multirow{2}{*}{T} & UniSRec & & 0.1602 & 0.3202 & 0.0806 & 0.1148 & & 0.1183 & 0.1893 & 0.0773 & 0.0926 \\
& \modelname{} & & \textbf{0.1619} & \textbf{0.3239} & \textbf{0.0814} & \textbf{0.1165} & & \textbf{0.1186} & \textbf{0.1899} & \textbf{0.0787} & \textbf{0.0943} \\
\midrule
\multirow{2}{*}{V} & UniSRec & & 0.1568 & 0.3135 & 0.0782 & 0.1118 & & 0.1140 & 0.1823 & 0.0763 & 0.0910 \\
& \modelname{} & & \textbf{0.1575} & \textbf{0.3144} & \textbf{0.0789} & \textbf{0.1118} & & \textbf{0.1152} & \textbf{0.1848} & \textbf{0.0771} & \textbf{0.0923} \\
\midrule
\multirow{2}{*}{T+V} & UniSRec & & 0.1609 & 0.3216 & 0.0808 & 0.1141 & & 0.1205 & 0.1929 & 0.0792 & 0.0945 \\
& \modelname{} & & \textbf{0.1633} & \textbf{0.3265} & \textbf{0.0812} & \textbf{0.1160} & & \textbf{0.1209} & \textbf{0.1936} & \textbf{0.0798} & \textbf{0.0953} \\
\bottomrule
\end{tabular}
}
}
\label{tab:modality_analysis}
\end{table}
\subsubsection{Comparison with Different Input Modalities}
\label{subsubsec:compare_with_diff_modality}
In this sub-section, we investigate the capacities of different modality-based sequence models in leveraging multi-modal information.
Specifically, we filter out the items without images in ``Scientific'' and ``Office'' datasets.
Then we conduct experiments on the filtered subsets to enable horizontal comparison among different settings, \eg visual-only \emph{vs} text-only.
To ensure objective evaluation, we adopt the same CLIP-B/32 feature extractors for all comparison methods.
We report the results in \cref{tab:modality_analysis}.
Interestingly, although utilizing visual modality alone does not lead to satisfactory results, the multi-modal synergy creates an effect of ``1+1>2'', which highlights the significance of multi-modal sequential modeling.
Besides, \modelname{} demonstrates robust and competitive performance under various settings.
\begin{table}[t] %
\caption{Analysis of the effect of pre-training.}
% \vspace{-1em}
\setlength{\tabcolsep}{0.2em}{
\resizebox{\columnwidth}{!}{
\begin{tabular}{cclcccclcccc}
\toprule
\multicolumn{2}{c}{\underline{Model: \modelname{}}} & & \multicolumn{4}{c}{Scientific} & & \multicolumn{4}{c}{Office} \\
\cmidrule(l){4-7} \cmidrule(l){9-12}
w/ ID? & Pre-train? & & R@10 & R@50 & N@10 & N@50 & & R@10 & R@50 & N@10 & N@50 \\
\hline
\midrule
\multirow{2}{*}{\dui} & \dui & & 0.1360 & 0.2431 & 0.0753 & 0.0983 & & 0.1275 & 0.2005 & 0.0856 & 0.1012 \\
& \cuo & & 0.1282 & 0.2376 & 0.0711 & 0.0946 & & 0.1269 & 0.2001 & 0.0848 & 0.1005 \\
\multicolumn{2}{c}{\cellcolor[HTML]{E9EAFF}\emph{Improv.} w/ ID} && \cellcolor[HTML]{E9EAFF}6.08\% & \cellcolor[HTML]{E9EAFF}2.31\% & \cellcolor[HTML]{E9EAFF}5.91\% & \cellcolor[HTML]{E9EAFF}3.91\% & & \cellcolor[HTML]{E9EAFF}0.47\% & \cellcolor[HTML]{E9EAFF}0.20\% & \cellcolor[HTML]{E9EAFF}0.94\% & \cellcolor[HTML]{E9EAFF}0.70\% \\
\midrule
\multirow{2}{*}{\cuo} & \dui & & 0.1278 & 0.2375 & 0.0658 & 0.0893 & & 0.1038 & 0.1701 & 0.0666 & 0.0808 \\
& \cuo & & 0.1269 & 0.2354 & 0.0659 & 0.0891 & & 0.1072 & 0.1726 & 0.0694 & 0.0834 \\
\multicolumn{2}{c}{\cellcolor[HTML]{EFFBEC}\emph{Improv.} w/o ID} && \cellcolor[HTML]{EFFBEC}0.71\% & \cellcolor[HTML]{EFFBEC}0.89\% & \cellcolor[HTML]{EFFBEC}-0.15\% & \cellcolor[HTML]{EFFBEC}0.22\% & \cellcolor[HTML]{EFFBEC} & \cellcolor[HTML]{EFFBEC}-3.17\% & \cellcolor[HTML]{EFFBEC}-1.45\% & \cellcolor[HTML]{EFFBEC}-4.03\% & \cellcolor[HTML]{EFFBEC}-3.12\% \\
\bottomrule
\end{tabular}
}
}
\label{tab:pretrain_analysis}
\end{table}
\subsubsection{Pre-training Analysis}
\label{subsubsec:pretraining_effect}
Here we examine the effectiveness of pre-training to \modelname{}.
As shown in \cref{tab:pretrain_analysis}, we can see that pre-training enhances recommendation performance in target domains under transductive settings.
However, we observe the negative transfer under inductive settings, especially on the larger dataset, Office.
A possible reason for the phenomenon is
that the model can not adapt well to the target data with a large domain gap accompanied by quite limited tunable parameters (\ie adapters).
\subsection{Model Analyses (RQ3)}
\label{subsec:model_analyses}
To investigate the efficacy of different components, we construct 5 variants of \modelname{}$_\text{T+V+ID}$ for comparison.
Specifically, as presented by the leftmost column in \cref{tab:ablation_study}, variant (1) removes the sequence-sequence contrastive task (\S\ref{subsubsec:seq_seq_cl}) from pre-training. Variant (2) removes the modality-specific adapters (\S\ref{subsubsec:adapter}) so that the sequence model directly consumes 512-d CLIP features.
Variant (3) replaces the interest-aware decoder (\S\ref{subsubsec:mm_dec}) layer with another encoder layer and builds a 2-layer contextual encoder that only processes interest tokens.
The sequence embedding is produced by mean-pooling the output token sequence.
Variant (4) only keeps the decoding structure and builds a 2-layer decoder-only model.
It also removes the cross-attention module at each decoder layer due to the absence of encoder output.
Variant (5) removes the orthogonal regularization in learning objectives (\cref{equ:pre-train,equ:fine-tune}).
\subsubsection{Effects of Modality Adapters}
\label{subsubsec:analyze_adapters}
Comparing variants (2) with (0), we can see significant performance decays, owing to the semantic gap between general multi-modal features and user interests.
As a result, multi-modal features themselves as item representations could not provide sufficient personalization for recommendation, and modality adapters are required.
On the other hand, although end-to-end optimizing feature extractors with the recommender also helps to mitigate the gap, as revealed by \cite{MoRec,TransRec}, it will consume much more computation resources.
Anyway, parameter-efficient tuning with modality adapters can be preferable.
\subsubsection{Effects of Learning Objectives}% and Negative Sample Mining}
\label{subsubsec:analyze_loss}
Comparing (1) and (0), we learn that $\ell_i^\text{S-S}$ in the pre-training stage can enhance the representation robustness, as it helps to improve downstream recommendation.
Comparing (5) and (0), we find that the orthogonal regularization leads to slightly better results on the Scientific dataset but shows a negative impact on the Office dataset.
We can infer that the orthogonal regularization is sensitive to the domain or the weighting factor, which requires careful choosing according to specific scenarios.
\subsubsection{Effects of Interest-aware Decoding and Contextual Encoding}
\label{subsubsec:analyze_enc_dec}
We can learn that variant (3) exhibits poor performance among the variants, suggesting the necessity of the collaboration between context encoding and interest-aware decoding.
Besides, by comparing variants (4) and (0), variant (4) without accessing encoded interest information also shows performance degradation, which justifies the efficacy of interest mining and contextual modeling.
\begin{table}[t] %
\caption{Ablation study with \modelname{}.}
% \vspace{-1em}
\setlength{\tabcolsep}{0.2em}{
\resizebox{\columnwidth}{!}{
\begin{tabular}{llcccclcccc}
\toprule
& & \multicolumn{4}{c}{Scientific} & & \multicolumn{4}{c}{Office} \\
\cmidrule(l){3-6} \cmidrule(l){8-11}
Variant & & R@10 & R@50 & N@10 & N@50 & & R@10 & R@50 & N@10 & N@50 \\
\hline
\midrule
(0) \modelname{} && \textbf{0.1360} & \textbf{0.2431} & \textbf{0.0753} & \textbf{0.0983} && \underline{0.1275} & \underline{0.2005} & \underline{0.0856} & \underline{0.1012} \\
\cmidrule(l){1-1}
(1) \phantom{-} w/o $\ell^{S-S}_i$ in Eq.(\ref{equ:pre-train}) && 0.1337 & 0.2390 & 0.0710 & 0.0926 && 0.1273 & 0.2000 & 0.0847 & 0.1002 \\
(2) \phantom{-} w/o Modality Adapters && 0.1058 & 0.1896 & 0.0582 & 0.0759 && 0.1029 & 0.1617 & 0.0708 & 0.0838 \\
(3) \phantom{-} 2-Layer Encoder Only && 0.1220 & 0.2186 & 0.0680 & 0.0890 && 0.1175 & 0.1849 & 0.0823 & 0.0974 \\
(4) \phantom{-} 2-Layer Decoder Only && 0.1172 & 0.2092 & 0.0647 & 0.0844 && 0.1003 & 0.1575 & 0.0696 & 0.0821 \\
(5) \phantom{-} w/o Oth. Reg. in Eqs.(\ref{equ:pre-train}-\ref{equ:fine-tune}) && \underline{0.1355} & \underline{0.2403} & \underline{0.0745} & \underline{0.0974} && \textbf{0.1286} & \textbf{0.2019} & \textbf{0.0873} & \textbf{0.1034} \\
\bottomrule
\end{tabular}
}
}
\label{tab:ablation_study}
\end{table}
\section{Conclusions}
\label{sec:conclusion}
This paper addresses the limitations of existing sequential recommendation models that rely heavily on ID features by exploring the potential of multi-modal information.
We propose \modelname{}, a novel multi-modal pre-training and transfer learning framework for sequential recommendation, which effectively tackles the cold-start problem and allows for efficient domain adaptation.
By utilizing a transformer-based contextual encoder, an interest-aware decoder, a lightweight dynamic fusion module,
\modelname{} demonstrates improved performance and generalizability compared to existing methods.
Extensive experiments also showcase the compatibility and robustness of \modelname{} in handling incomplete or missing modalities, reinforcing its pragmatic value for real-world scenarios.
More importantly, our paper suggests a promising direction for future research in leveraging multi-modal information for sequential recommendation.
We hope \modelname{} can provide some inspiration to further explorations.
\begin{acks}
We want to thank the anonymous reviewers and the meta-reviewer for their valuable comments and suggestions.
This research is supported by National Natural Science Foundation of China (Grant No.62171248 and Grant No.62276154), the Natural Science Foundation of Guangdong Province (Grant No. 2023A1515012914), Shenzhen Science and Technology Program (JCYJ20220818101012025), Basic Research Fund of Shenzhen City (Grant No. JCYJ20210324120012033 and JSGG20210802154402007), the Major Key Project of PCL for Experiments and Applications (PCL2021A07, and PCL2021A06), and Overseas Cooperation Research Fund of Tsinghua Shenzhen International
Graduate School (HW2021008).
We gratefully acknowledge the support of MindSpore\footnote{https://www.mindspore.cn}, which is a new deep learning framework used for this research.
\end{acks}
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{98}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Abdollahpouri et~al\mbox{.}(2020)]%
{PopBias1}
\bibfield{author}{\bibinfo{person}{Himan Abdollahpouri},
\bibinfo{person}{Masoud Mansoury}, \bibinfo{person}{Robin Burke}, {and}
\bibinfo{person}{Bamshad Mobasher}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{The connection between popularity bias,
calibration, and fairness in recommendation}. In
\bibinfo{booktitle}{\emph{RecSys}}.
\newblock
\bibitem[Bapna and Firat(2019)]%
{Adapter2}
\bibfield{author}{\bibinfo{person}{Ankur Bapna} {and} \bibinfo{person}{Orhan
Firat}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Simple, Scalable Adaptation for Neural Machine
Translation}. In \bibinfo{booktitle}{\emph{EMNLP}}.
\newblock
\bibitem[Cai et~al\mbox{.}(2021)]%
{cai2021heterogeneous}
\bibfield{author}{\bibinfo{person}{Desheng Cai}, \bibinfo{person}{Shengsheng
Qian}, \bibinfo{person}{Quan Fang}, {and} \bibinfo{person}{Changsheng Xu}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Heterogeneous hierarchical feature aggregation
network for personalized micro-video recommendation}.
\newblock \bibinfo{journal}{\emph{IEEE TMM}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Cen et~al\mbox{.}(2020)]%
{ComiRec}
\bibfield{author}{\bibinfo{person}{Yukuo Cen}, \bibinfo{person}{Jianwei Zhang},
\bibinfo{person}{Xu Zou}, \bibinfo{person}{Chang Zhou},
\bibinfo{person}{Hongxia Yang}, {and} \bibinfo{person}{Jie Tang}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Controllable multi-interest framework for
recommendation}. In \bibinfo{booktitle}{\emph{KDD}}.
\newblock
\bibitem[Chen et~al\mbox{.}(2017)]%
{ACF}
\bibfield{author}{\bibinfo{person}{Jingyuan Chen}, \bibinfo{person}{Hanwang
Zhang}, \bibinfo{person}{Xiangnan He}, \bibinfo{person}{Liqiang Nie},
\bibinfo{person}{Wei Liu}, {and} \bibinfo{person}{Tat-Seng Chua}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Attentive collaborative filtering: Multimedia
recommendation with item-and component-level attention}. In
\bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Chen et~al\mbox{.}(2020)]%
{SimCLR}
\bibfield{author}{\bibinfo{person}{Ting Chen}, \bibinfo{person}{Simon
Kornblith}, \bibinfo{person}{Mohammad Norouzi}, {and}
\bibinfo{person}{Geoffrey Hinton}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{A simple framework for contrastive learning of
visual representations}. In \bibinfo{booktitle}{\emph{ICML}}.
\newblock
\bibitem[Chen et~al\mbox{.}(2022)]%
{ICLRec}
\bibfield{author}{\bibinfo{person}{Yongjun Chen}, \bibinfo{person}{Zhiwei Liu},
\bibinfo{person}{Jia Li}, \bibinfo{person}{Julian McAuley}, {and}
\bibinfo{person}{Caiming Xiong}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Intent contrastive learning for sequential
recommendation}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Cheng et~al\mbox{.}(2021)]%
{CLUE}
\bibfield{author}{\bibinfo{person}{Mingyue Cheng}, \bibinfo{person}{Fajie
Yuan}, \bibinfo{person}{Qi Liu}, \bibinfo{person}{Xin Xin}, {and}
\bibinfo{person}{Enhong Chen}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Learning transferable user representations with
sequential behaviors via contrastive pre-training}. In
\bibinfo{booktitle}{\emph{ICDM}}.
\newblock
\bibitem[Dai et~al\mbox{.}(2022)]%
{LCD}
\bibfield{author}{\bibinfo{person}{Quanyu Dai}, \bibinfo{person}{Yalei Lv},
\bibinfo{person}{Jieming Zhu}, \bibinfo{person}{Junjie Ye},
\bibinfo{person}{Zhenhua Dong}, \bibinfo{person}{Rui Zhang},
\bibinfo{person}{Shu-Tao Xia}, {and} \bibinfo{person}{Ruiming Tang}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{LCD: Adaptive Label Correction for Denoising Music
Recommendation}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Deldjoo et~al\mbox{.}(2020)]%
{MMRec_Survey3}
\bibfield{author}{\bibinfo{person}{Yashar Deldjoo}, \bibinfo{person}{Markus
Schedl}, \bibinfo{person}{Paolo Cremonesi}, {and} \bibinfo{person}{Gabriella
Pasi}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Recommender systems leveraging multimedia content}.
\newblock \bibinfo{journal}{\emph{ACM CSUR}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Deldjoo et~al\mbox{.}(2021)]%
{MMRec_Survey2}
\bibfield{author}{\bibinfo{person}{Yashar Deldjoo}, \bibinfo{person}{Markus
Schedl}, \bibinfo{person}{Bal{\'a}zs Hidasi}, \bibinfo{person}{Yinwei Wei},
{and} \bibinfo{person}{Xiangnan He}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Multimedia recommender systems: Algorithms and
challenges}.
\newblock In \bibinfo{booktitle}{\emph{Recommender systems handbook}}.
\newblock
\bibitem[Devlin et~al\mbox{.}(2019)]%
{BERT}
\bibfield{author}{\bibinfo{person}{Jacob Devlin}, \bibinfo{person}{Ming-Wei
Chang}, \bibinfo{person}{Kenton Lee}, {and} \bibinfo{person}{Kristina
Toutanova}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{BERT: Pre-training of Deep Bidirectional
Transformers for Language Understanding}. In
\bibinfo{booktitle}{\emph{NAACL}}.
\newblock
\bibitem[Ding et~al\mbox{.}(2022)]%
{ZESRec}
\bibfield{author}{\bibinfo{person}{Hao Ding}, \bibinfo{person}{Yifei Ma},
\bibinfo{person}{Anoop Deoras}, \bibinfo{person}{Yuyang Wang}, {and}
\bibinfo{person}{Hao Wang}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Zero-Shot Recommender Systems}. In
\bibinfo{booktitle}{\emph{ICLR Workshops}}.
\newblock
\bibitem[Dosovitskiy et~al\mbox{.}(2020)]%
{ViT}
\bibfield{author}{\bibinfo{person}{Alexey Dosovitskiy}, \bibinfo{person}{Lucas
Beyer}, \bibinfo{person}{Alexander Kolesnikov}, \bibinfo{person}{Dirk
Weissenborn}, \bibinfo{person}{Xiaohua Zhai}, \bibinfo{person}{Thomas
Unterthiner}, \bibinfo{person}{Mostafa Dehghani}, \bibinfo{person}{Matthias
Minderer}, \bibinfo{person}{Georg Heigold}, \bibinfo{person}{Sylvain Gelly},
{et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{An Image is Worth 16x16 Words: Transformers for
Image Recognition at Scale}. In \bibinfo{booktitle}{\emph{ICLR}}.
\newblock
\bibitem[Du et~al\mbox{.}(2016)]%
{DPC-KNN}
\bibfield{author}{\bibinfo{person}{Mingjing Du}, \bibinfo{person}{Shifei Ding},
{and} \bibinfo{person}{Hongjie Jia}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Study on density peaks clustering based on
k-nearest neighbors and principal component analysis}.
\newblock \bibinfo{journal}{\emph{Elsevier KBS}} (\bibinfo{year}{2016}).
\newblock
\bibitem[Du et~al\mbox{.}(2020)]%
{MTPR}
\bibfield{author}{\bibinfo{person}{Xiaoyu Du}, \bibinfo{person}{Xiang Wang},
\bibinfo{person}{Xiangnan He}, \bibinfo{person}{Zechao Li},
\bibinfo{person}{Jinhui Tang}, {and} \bibinfo{person}{Tat-Seng Chua}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{How to learn item representation for cold-start
multimedia recommendation?}. In \bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Han et~al\mbox{.}(2022)]%
{ViT_Survey2}
\bibfield{author}{\bibinfo{person}{Kai Han}, \bibinfo{person}{Yunhe Wang},
\bibinfo{person}{Hanting Chen}, \bibinfo{person}{Xinghao Chen},
\bibinfo{person}{Jianyuan Guo}, \bibinfo{person}{Zhenhua Liu},
\bibinfo{person}{Yehui Tang}, \bibinfo{person}{An Xiao},
\bibinfo{person}{Chunjing Xu}, \bibinfo{person}{Yixing Xu}, {et~al\mbox{.}}}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{A survey on vision transformer}.
\newblock \bibinfo{journal}{\emph{IEEE TPAMI}} (\bibinfo{year}{2022}).
\newblock
\bibitem[He et~al\mbox{.}(2022)]%
{UnifiedPETL}
\bibfield{author}{\bibinfo{person}{Junxian He}, \bibinfo{person}{Chunting
Zhou}, \bibinfo{person}{Xuezhe Ma}, \bibinfo{person}{Taylor
Berg-Kirkpatrick}, {and} \bibinfo{person}{Graham Neubig}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Towards a Unified View of Parameter-Efficient
Transfer Learning}. In \bibinfo{booktitle}{\emph{ICLR}}.
\newblock
\bibitem[He et~al\mbox{.}(2020b)]%
{MoCo}
\bibfield{author}{\bibinfo{person}{Kaiming He}, \bibinfo{person}{Haoqi Fan},
\bibinfo{person}{Yuxin Wu}, \bibinfo{person}{Saining Xie}, {and}
\bibinfo{person}{Ross Girshick}.} \bibinfo{year}{2020}\natexlab{b}.
\newblock \showarticletitle{Momentum contrast for unsupervised visual
representation learning}. In \bibinfo{booktitle}{\emph{CVPR}}.
\newblock
\bibitem[He et~al\mbox{.}(2021)]%
{HyperCTR}
\bibfield{author}{\bibinfo{person}{Li He}, \bibinfo{person}{Hongxu Chen},
\bibinfo{person}{Dingxian Wang}, \bibinfo{person}{Shoaib Jameel},
\bibinfo{person}{Philip Yu}, {and} \bibinfo{person}{Guandong Xu}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Click-through rate prediction with multi-modal
hypergraphs}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[He and McAuley(2016a)]%
{Fossil}
\bibfield{author}{\bibinfo{person}{Ruining He} {and} \bibinfo{person}{Julian
McAuley}.} \bibinfo{year}{2016}\natexlab{a}.
\newblock \showarticletitle{Fusing similarity models with markov chains for
sparse sequential recommendation}. In \bibinfo{booktitle}{\emph{ICDM}}.
\newblock
\bibitem[He and McAuley(2016b)]%
{VBPR}
\bibfield{author}{\bibinfo{person}{Ruining He} {and} \bibinfo{person}{Julian
McAuley}.} \bibinfo{year}{2016}\natexlab{b}.
\newblock \showarticletitle{VBPR: visual bayesian personalized ranking from
implicit feedback}. In \bibinfo{booktitle}{\emph{AAAI}}.
\newblock
\bibitem[He et~al\mbox{.}(2020a)]%
{LightGCN}
\bibfield{author}{\bibinfo{person}{Xiangnan He}, \bibinfo{person}{Kuan Deng},
\bibinfo{person}{Xiang Wang}, \bibinfo{person}{Yan Li},
\bibinfo{person}{Yongdong Zhang}, {and} \bibinfo{person}{Meng Wang}.}
\bibinfo{year}{2020}\natexlab{a}.
\newblock \showarticletitle{Lightgcn: Simplifying and powering graph
convolution network for recommendation}. In
\bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Hidasi et~al\mbox{.}(2016)]%
{GRU4Rec}
\bibfield{author}{\bibinfo{person}{Bal{\'a}zs Hidasi},
\bibinfo{person}{Alexandros Karatzoglou}, \bibinfo{person}{Linas Baltrunas},
{and} \bibinfo{person}{Domonkos Tikk}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Session-based recommendations with recurrent neural
networks}. In \bibinfo{booktitle}{\emph{ICLR}}.
\newblock
\bibitem[Hou et~al\mbox{.}(2023)]%
{VQRec}
\bibfield{author}{\bibinfo{person}{Yupeng Hou}, \bibinfo{person}{Zhankui He},
\bibinfo{person}{Julian McAuley}, {and} \bibinfo{person}{Wayne~Xin Zhao}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Learning vector-quantized item representation for
transferable sequential recommenders}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Hou et~al\mbox{.}(2022)]%
{UniSRec}
\bibfield{author}{\bibinfo{person}{Yupeng Hou}, \bibinfo{person}{Shanlei Mu},
\bibinfo{person}{Wayne~Xin Zhao}, \bibinfo{person}{Yaliang Li},
\bibinfo{person}{Bolin Ding}, {and} \bibinfo{person}{Ji-Rong Wen}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Towards Universal Sequence Representation Learning
for Recommender Systems}. In \bibinfo{booktitle}{\emph{KDD}}.
\newblock
\bibitem[Houlsby et~al\mbox{.}(2019)]%
{Adapter1}
\bibfield{author}{\bibinfo{person}{Neil Houlsby}, \bibinfo{person}{Andrei
Giurgiu}, \bibinfo{person}{Stanislaw Jastrzebski}, \bibinfo{person}{Bruna
Morrone}, \bibinfo{person}{Quentin De~Laroussilhe}, \bibinfo{person}{Andrea
Gesmundo}, \bibinfo{person}{Mona Attariyan}, {and} \bibinfo{person}{Sylvain
Gelly}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Parameter-efficient transfer learning for NLP}. In
\bibinfo{booktitle}{\emph{ICML}}.
\newblock
\bibitem[Huang et~al\mbox{.}(2018)]%
{CSAN}
\bibfield{author}{\bibinfo{person}{Xiaowen Huang}, \bibinfo{person}{Shengsheng
Qian}, \bibinfo{person}{Quan Fang}, \bibinfo{person}{Jitao Sang}, {and}
\bibinfo{person}{Changsheng Xu}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Csan: Contextual self-attention network for user
sequential recommendation}. In \bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Jiang et~al\mbox{.}(2020)]%
{jiang2020aspect}
\bibfield{author}{\bibinfo{person}{Hao Jiang}, \bibinfo{person}{Wenjie Wang},
\bibinfo{person}{Yinwei Wei}, \bibinfo{person}{Zan Gao},
\bibinfo{person}{Yinglong Wang}, {and} \bibinfo{person}{Liqiang Nie}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{What aspect do you like: Multi-scale time-aware
user interest modeling for micro-video recommendation}. In
\bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Jiang et~al\mbox{.}(2021)]%
{CPT-HG}
\bibfield{author}{\bibinfo{person}{Xunqiang Jiang}, \bibinfo{person}{Yuanfu
Lu}, \bibinfo{person}{Yuan Fang}, {and} \bibinfo{person}{Chuan Shi}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Contrastive pre-training of gnns on heterogeneous
graphs}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Kang and McAuley(2018)]%
{SASRec}
\bibfield{author}{\bibinfo{person}{Wang-Cheng Kang} {and}
\bibinfo{person}{Julian McAuley}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Self-attentive sequential recommendation}. In
\bibinfo{booktitle}{\emph{ICDM}}.
\newblock
\bibitem[Khan et~al\mbox{.}(2022)]%
{ViT_Survey1}
\bibfield{author}{\bibinfo{person}{Salman Khan}, \bibinfo{person}{Muzammal
Naseer}, \bibinfo{person}{Munawar Hayat}, \bibinfo{person}{Syed~Waqas Zamir},
\bibinfo{person}{Fahad~Shahbaz Khan}, {and} \bibinfo{person}{Mubarak Shah}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Transformers in vision: A survey}.
\newblock \bibinfo{journal}{\emph{ACM CSUR}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Lei et~al\mbox{.}(2021)]%
{SEMI}
\bibfield{author}{\bibinfo{person}{Chenyi Lei}, \bibinfo{person}{Yong Liu},
\bibinfo{person}{Lingzi Zhang}, \bibinfo{person}{Guoxin Wang},
\bibinfo{person}{Haihong Tang}, \bibinfo{person}{Houqiang Li}, {and}
\bibinfo{person}{Chunyan Miao}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Semi: A sequential multi-modal information transfer
network for e-commerce micro-video recommendations}. In
\bibinfo{booktitle}{\emph{KDD}}. \bibinfo{pages}{3161--3171}.
\newblock
\bibitem[Li et~al\mbox{.}(2019)]%
{MIND}
\bibfield{author}{\bibinfo{person}{Chao Li}, \bibinfo{person}{Zhiyuan Liu},
\bibinfo{person}{Mengmeng Wu}, \bibinfo{person}{Yuchi Xu},
\bibinfo{person}{Huan Zhao}, \bibinfo{person}{Pipei Huang},
\bibinfo{person}{Guoliang Kang}, \bibinfo{person}{Qiwei Chen},
\bibinfo{person}{Wei Li}, {and} \bibinfo{person}{Dik~Lun Lee}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Multi-interest network with dynamic routing for
recommendation at Tmall}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Li et~al\mbox{.}(2020)]%
{TiSASRec}
\bibfield{author}{\bibinfo{person}{Jiacheng Li}, \bibinfo{person}{Yujie Wang},
{and} \bibinfo{person}{Julian McAuley}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Time interval aware self-attention for sequential
recommendation}. In \bibinfo{booktitle}{\emph{WSDM}}.
\newblock
\bibitem[Li et~al\mbox{.}(2022)]%
{MINER}
\bibfield{author}{\bibinfo{person}{Jian Li}, \bibinfo{person}{Jieming Zhu},
\bibinfo{person}{Qiwei Bi}, \bibinfo{person}{Guohao Cai},
\bibinfo{person}{Lifeng Shang}, \bibinfo{person}{Zhenhua Dong},
\bibinfo{person}{Xin Jiang}, {and} \bibinfo{person}{Qun Liu}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{MINER: multi-interest matching network for news
recommendation}. In \bibinfo{booktitle}{\emph{ACL Findings}}.
\newblock
\bibitem[Liang et~al\mbox{.}(2023)]%
{MMMLP}
\bibfield{author}{\bibinfo{person}{Jiahao Liang}, \bibinfo{person}{Xiangyu
Zhao}, \bibinfo{person}{Muyang Li}, \bibinfo{person}{Zijian Zhang},
\bibinfo{person}{Wanyu Wang}, \bibinfo{person}{Haochen Liu}, {and}
\bibinfo{person}{Zitao Liu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{MMMLP: Multi-modal Multilayer Perceptron for
Sequential Recommendations}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Lin et~al\mbox{.}(2023)]%
{DFAR}
\bibfield{author}{\bibinfo{person}{Guanyu Lin}, \bibinfo{person}{Chen Gao},
\bibinfo{person}{Yu Zheng}, \bibinfo{person}{Jianxin Chang},
\bibinfo{person}{Yanan Niu}, \bibinfo{person}{Yang Song},
\bibinfo{person}{Zhiheng Li}, \bibinfo{person}{Depeng Jin}, {and}
\bibinfo{person}{Yong Li}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Dual-interest Factorization-heads Attention for
Sequential Recommendation}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Liu et~al\mbox{.}(2023)]%
{Prompt_survey}
\bibfield{author}{\bibinfo{person}{Pengfei Liu}, \bibinfo{person}{Weizhe Yuan},
\bibinfo{person}{Jinlan Fu}, \bibinfo{person}{Zhengbao Jiang},
\bibinfo{person}{Hiroaki Hayashi}, {and} \bibinfo{person}{Graham Neubig}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Pre-train, prompt, and predict: A systematic survey
of prompting methods in natural language processing}.
\newblock \bibinfo{journal}{\emph{ACM CSUR}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Liu et~al\mbox{.}(2022b)]%
{liu2022boosting}
\bibfield{author}{\bibinfo{person}{Qijiong Liu}, \bibinfo{person}{Jieming Zhu},
\bibinfo{person}{Quanyu Dai}, {and} \bibinfo{person}{Xiaoming Wu}.}
\bibinfo{year}{2022}\natexlab{b}.
\newblock \showarticletitle{Boosting deep ctr prediction with a plug-and-play
pre-trainer for news recommendation}. In \bibinfo{booktitle}{\emph{COLING}}.
\newblock
\bibitem[Liu et~al\mbox{.}(2021)]%
{PMGT}
\bibfield{author}{\bibinfo{person}{Yong Liu}, \bibinfo{person}{Susen Yang},
\bibinfo{person}{Chenyi Lei}, \bibinfo{person}{Guoxin Wang},
\bibinfo{person}{Haihong Tang}, \bibinfo{person}{Juyong Zhang},
\bibinfo{person}{Aixin Sun}, {and} \bibinfo{person}{Chunyan Miao}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Pre-training graph transformer with multimodal side
information for recommendation}. In \bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Liu et~al\mbox{.}(2022a)]%
{MMCPR}
\bibfield{author}{\bibinfo{person}{Zhuang Liu}, \bibinfo{person}{Yunpu Ma},
\bibinfo{person}{Matthias Schubert}, \bibinfo{person}{Yuanxin Ouyang}, {and}
\bibinfo{person}{Zhang Xiong}.} \bibinfo{year}{2022}\natexlab{a}.
\newblock \showarticletitle{Multi-Modal Contrastive Pre-training for
Recommendation}. In \bibinfo{booktitle}{\emph{ICMR}}.
\newblock
\bibitem[Ma et~al\mbox{.}(2019)]%
{HGN}
\bibfield{author}{\bibinfo{person}{Chen Ma}, \bibinfo{person}{Peng Kang}, {and}
\bibinfo{person}{Xue Liu}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Hierarchical gating networks for sequential
recommendation}. In \bibinfo{booktitle}{\emph{KDD}}.
\newblock
\bibitem[Mao et~al\mbox{.}(2021)]%
{SimpleX}
\bibfield{author}{\bibinfo{person}{Kelong Mao}, \bibinfo{person}{Jieming Zhu},
\bibinfo{person}{Jinpeng Wang}, \bibinfo{person}{Quanyu Dai},
\bibinfo{person}{Zhenhua Dong}, \bibinfo{person}{Xi Xiao}, {and}
\bibinfo{person}{Xiuqiang He}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{SimpleX: A simple and strong baseline for
collaborative filtering}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Ni et~al\mbox{.}(2019)]%
{AmazonReview}
\bibfield{author}{\bibinfo{person}{Jianmo Ni}, \bibinfo{person}{Jiacheng Li},
{and} \bibinfo{person}{Julian McAuley}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Justifying recommendations using distantly-labeled
reviews and fine-grained aspects}. In \bibinfo{booktitle}{\emph{EMNLP}}.
\newblock
\bibitem[Ni et~al\mbox{.}(2018)]%
{DUPN}
\bibfield{author}{\bibinfo{person}{Yabo Ni}, \bibinfo{person}{Dan Ou},
\bibinfo{person}{Shichen Liu}, \bibinfo{person}{Xiang Li},
\bibinfo{person}{Wenwu Ou}, \bibinfo{person}{Anxiang Zeng}, {and}
\bibinfo{person}{Luo Si}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Perceive your users in depth: Learning universal
user representations from multiple e-commerce tasks}. In
\bibinfo{booktitle}{\emph{KDD}}.
\newblock
\bibitem[Pan et~al\mbox{.}(2022)]%
{MML}
\bibfield{author}{\bibinfo{person}{Xingyu Pan}, \bibinfo{person}{Yushuo Chen},
\bibinfo{person}{Changxin Tian}, \bibinfo{person}{Zihan Lin},
\bibinfo{person}{Jinpeng Wang}, \bibinfo{person}{He Hu}, {and}
\bibinfo{person}{Wayne~Xin Zhao}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Multimodal Meta-Learning for Cold-Start Sequential
Recommendation}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Qiu et~al\mbox{.}(2022)]%
{DuoRec}
\bibfield{author}{\bibinfo{person}{Ruihong Qiu}, \bibinfo{person}{Zi Huang},
\bibinfo{person}{Hongzhi Yin}, {and} \bibinfo{person}{Zijian Wang}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Contrastive learning for representation
degeneration problem in sequential recommendation}. In
\bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Qiu et~al\mbox{.}(2021)]%
{U-BERT}
\bibfield{author}{\bibinfo{person}{Zhaopeng Qiu}, \bibinfo{person}{Xian Wu},
\bibinfo{person}{Jingyue Gao}, {and} \bibinfo{person}{Wei Fan}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{U-BERT: Pre-training user representations for
improved recommendation}. In \bibinfo{booktitle}{\emph{AAAI}}.
\newblock
\bibitem[Quadrana et~al\mbox{.}(2018)]%
{SR_survey2}
\bibfield{author}{\bibinfo{person}{Massimo Quadrana}, \bibinfo{person}{Paolo
Cremonesi}, {and} \bibinfo{person}{Dietmar Jannach}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Sequence-aware recommender systems}.
\newblock \bibinfo{journal}{\emph{ACM CSUR}} (\bibinfo{year}{2018}).
\newblock
\bibitem[Radford et~al\mbox{.}(2021)]%
{CLIP}
\bibfield{author}{\bibinfo{person}{Alec Radford}, \bibinfo{person}{Jong~Wook
Kim}, \bibinfo{person}{Chris Hallacy}, \bibinfo{person}{Aditya Ramesh},
\bibinfo{person}{Gabriel Goh}, \bibinfo{person}{Sandhini Agarwal},
\bibinfo{person}{Girish Sastry}, \bibinfo{person}{Amanda Askell},
\bibinfo{person}{Pamela Mishkin}, \bibinfo{person}{Jack Clark},
{et~al\mbox{.}}} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Learning transferable visual models from natural
language supervision}. In \bibinfo{booktitle}{\emph{ICML}}.
\newblock
\bibitem[Rendle et~al\mbox{.}(2010)]%
{FPMC}
\bibfield{author}{\bibinfo{person}{Steffen Rendle}, \bibinfo{person}{Christoph
Freudenthaler}, {and} \bibinfo{person}{Lars Schmidt-Thieme}.}
\bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{Factorizing personalized markov chains for
next-basket recommendation}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Resnick and Varian(1997)]%
{RecSys}
\bibfield{author}{\bibinfo{person}{Paul Resnick} {and} \bibinfo{person}{Hal~R
Varian}.} \bibinfo{year}{1997}\natexlab{}.
\newblock \showarticletitle{Recommender systems}.
\newblock \bibinfo{journal}{\emph{Commun. ACM}} (\bibinfo{year}{1997}).
\newblock
\bibitem[Schein et~al\mbox{.}(2002)]%
{coldstart_problem}
\bibfield{author}{\bibinfo{person}{Andrew~I Schein},
\bibinfo{person}{Alexandrin Popescul}, \bibinfo{person}{Lyle~H Ungar}, {and}
\bibinfo{person}{David~M Pennock}.} \bibinfo{year}{2002}\natexlab{}.
\newblock \showarticletitle{Methods and metrics for cold-start
recommendations}. In \bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Song et~al\mbox{.}(2023a)]%
{MMSRec}
\bibfield{author}{\bibinfo{person}{Kunzhe Song}, \bibinfo{person}{Qingfeng
Sun}, \bibinfo{person}{Can Xu}, \bibinfo{person}{Kai Zheng}, {and}
\bibinfo{person}{Yaming Yang}.} \bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{Self-Supervised Multi-Modal Sequential
Recommendation}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2304.13277}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[Song et~al\mbox{.}(2023b)]%
{song2023mm}
\bibfield{author}{\bibinfo{person}{Xuemeng Song}, \bibinfo{person}{Chun Wang},
\bibinfo{person}{Changchang Sun}, \bibinfo{person}{Shanshan Feng},
\bibinfo{person}{Min Zhou}, {and} \bibinfo{person}{Liqiang Nie}.}
\bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{MM-FRec: Multi-Modal Enhanced Fashion Item
Recommendation}.
\newblock \bibinfo{journal}{\emph{IEEE TKDE}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Srivastava et~al\mbox{.}(2014)]%
{Dropout}
\bibfield{author}{\bibinfo{person}{Nitish Srivastava},
\bibinfo{person}{Geoffrey Hinton}, \bibinfo{person}{Alex Krizhevsky},
\bibinfo{person}{Ilya Sutskever}, {and} \bibinfo{person}{Ruslan
Salakhutdinov}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Dropout: a simple way to prevent neural networks
from overfitting}.
\newblock \bibinfo{journal}{\emph{JMLR}} (\bibinfo{year}{2014}).
\newblock
\bibitem[Su et~al\mbox{.}(2020)]%
{VL-BERT}
\bibfield{author}{\bibinfo{person}{Weijie Su}, \bibinfo{person}{Xizhou Zhu},
\bibinfo{person}{Yue Cao}, \bibinfo{person}{Bin Li}, \bibinfo{person}{Lewei
Lu}, \bibinfo{person}{Furu Wei}, {and} \bibinfo{person}{Jifeng Dai}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{VL-BERT: Pre-training of Generic Visual-Linguistic
Representations}. In \bibinfo{booktitle}{\emph{ICLR}}.
\newblock
\bibitem[Su et~al\mbox{.}(2021)]%
{su2021detecting}
\bibfield{author}{\bibinfo{person}{Yixin Su}, \bibinfo{person}{Rui Zhang},
\bibinfo{person}{Sarah Erfani}, {and} \bibinfo{person}{Zhenghua Xu}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Detecting beneficial feature interactions for
recommender systems}. In \bibinfo{booktitle}{\emph{AAAI}}.
\newblock
\bibitem[Su et~al\mbox{.}(2022)]%
{su2022detecting}
\bibfield{author}{\bibinfo{person}{Yixin Su}, \bibinfo{person}{Yunxiang Zhao},
\bibinfo{person}{Sarah Erfani}, \bibinfo{person}{Junhao Gan}, {and}
\bibinfo{person}{Rui Zhang}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Detecting arbitrary order beneficial feature
interactions for recommender systems}. In \bibinfo{booktitle}{\emph{KDD}}.
\newblock
\bibitem[Sun et~al\mbox{.}(2019)]%
{BERT4Rec}
\bibfield{author}{\bibinfo{person}{Fei Sun}, \bibinfo{person}{Jun Liu},
\bibinfo{person}{Jian Wu}, \bibinfo{person}{Changhua Pei},
\bibinfo{person}{Xiao Lin}, \bibinfo{person}{Wenwu Ou}, {and}
\bibinfo{person}{Peng Jiang}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{BERT4Rec: Sequential recommendation with
bidirectional encoder representations from transformer}. In
\bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Sun et~al\mbox{.}(2020)]%
{MKGAT}
\bibfield{author}{\bibinfo{person}{Rui Sun}, \bibinfo{person}{Xuezhi Cao},
\bibinfo{person}{Yan Zhao}, \bibinfo{person}{Junchen Wan},
\bibinfo{person}{Kun Zhou}, \bibinfo{person}{Fuzheng Zhang},
\bibinfo{person}{Zhongyuan Wang}, {and} \bibinfo{person}{Kai Zheng}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Multi-modal knowledge graphs for recommender
systems}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Tang and Wang(2018)]%
{Caser}
\bibfield{author}{\bibinfo{person}{Jiaxi Tang} {and} \bibinfo{person}{Ke
Wang}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Personalized top-n sequential recommendation via
convolutional sequence embedding}. In \bibinfo{booktitle}{\emph{WSDM}}.
\newblock
\bibitem[Tao et~al\mbox{.}(2022)]%
{SLMRec}
\bibfield{author}{\bibinfo{person}{Zhulin Tao}, \bibinfo{person}{Xiaohao Liu},
\bibinfo{person}{Yewei Xia}, \bibinfo{person}{Xiang Wang},
\bibinfo{person}{Lifang Yang}, \bibinfo{person}{Xianglin Huang}, {and}
\bibinfo{person}{Tat-Seng Chua}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Self-supervised learning for multimedia
recommendation}.
\newblock \bibinfo{journal}{\emph{IEEE TMM}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Tian et~al\mbox{.}(2022)]%
{TCPSRec}
\bibfield{author}{\bibinfo{person}{Changxin Tian}, \bibinfo{person}{Zihan Lin},
\bibinfo{person}{Shuqing Bian}, \bibinfo{person}{Jinpeng Wang}, {and}
\bibinfo{person}{Wayne~Xin Zhao}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Temporal Contrastive Pre-Training for Sequential
Recommendation}. In \bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Tolstikhin et~al\mbox{.}(2021)]%
{MLP-Mixer}
\bibfield{author}{\bibinfo{person}{Ilya~O Tolstikhin}, \bibinfo{person}{Neil
Houlsby}, \bibinfo{person}{Alexander Kolesnikov}, \bibinfo{person}{Lucas
Beyer}, \bibinfo{person}{Xiaohua Zhai}, \bibinfo{person}{Thomas Unterthiner},
\bibinfo{person}{Jessica Yung}, \bibinfo{person}{Andreas Steiner},
\bibinfo{person}{Daniel Keysers}, \bibinfo{person}{Jakob Uszkoreit},
{et~al\mbox{.}}} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Mlp-mixer: An all-mlp architecture for vision}.
\newblock \bibinfo{journal}{\emph{NIPS}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Vaswani et~al\mbox{.}(2017)]%
{attention}
\bibfield{author}{\bibinfo{person}{Ashish Vaswani}, \bibinfo{person}{Noam
Shazeer}, \bibinfo{person}{Niki Parmar}, \bibinfo{person}{Jakob Uszkoreit},
\bibinfo{person}{Llion Jones}, \bibinfo{person}{Aidan~N Gomez},
\bibinfo{person}{{\L}ukasz Kaiser}, {and} \bibinfo{person}{Illia
Polosukhin}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Attention is all you need}. In
\bibinfo{booktitle}{\emph{NIPS}}.
\newblock
\bibitem[Wang et~al\mbox{.}(2023a)]%
{ContraRec}
\bibfield{author}{\bibinfo{person}{Chenyang Wang}, \bibinfo{person}{Weizhi Ma},
\bibinfo{person}{Chong Chen}, \bibinfo{person}{Min Zhang},
\bibinfo{person}{Yiqun Liu}, {and} \bibinfo{person}{Shaoping Ma}.}
\bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{Sequential recommendation with multiple contrast
signals}.
\newblock \bibinfo{journal}{\emph{ACM TOIS}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Wang et~al\mbox{.}(2023b)]%
{CHEST}
\bibfield{author}{\bibinfo{person}{Hui Wang}, \bibinfo{person}{Kun Zhou},
\bibinfo{person}{Xin Zhao}, \bibinfo{person}{Jingyuan Wang}, {and}
\bibinfo{person}{Ji-Rong Wen}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{Curriculum Pre-Training Heterogeneous Subgraph
Transformer for Top-N Recommendation}.
\newblock \bibinfo{journal}{\emph{ACM TOIS}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Wang et~al\mbox{.}(2022a)]%
{TransRec}
\bibfield{author}{\bibinfo{person}{Jie Wang}, \bibinfo{person}{Fajie Yuan},
\bibinfo{person}{Mingyue Cheng}, \bibinfo{person}{Joemon~M Jose},
\bibinfo{person}{Chenyun Yu}, \bibinfo{person}{Beibei Kong},
\bibinfo{person}{Zhijin Wang}, \bibinfo{person}{Bo Hu}, {and}
\bibinfo{person}{Zang Li}.} \bibinfo{year}{2022}\natexlab{a}.
\newblock \showarticletitle{TransRec: Learning Transferable Recommendation from
Mixture-of-Modality Feedback}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2206.06190}}
(\bibinfo{year}{2022}).
\newblock
\bibitem[Wang et~al\mbox{.}(2022b)]%
{HuggingHash}
\bibfield{author}{\bibinfo{person}{Jinpeng Wang}, \bibinfo{person}{Ziyun Zeng},
\bibinfo{person}{Bin Chen}, \bibinfo{person}{Yuting Wang},
\bibinfo{person}{Dongliang Liao}, \bibinfo{person}{Gongfu Li},
\bibinfo{person}{Yiru Wang}, \bibinfo{person}{Shu-Tao Xia}, {and}
\bibinfo{person}{Peng~Cheng Intelligence}.} \bibinfo{year}{2022}\natexlab{b}.
\newblock \showarticletitle{Hugs Are Better Than Handshakes: Unsupervised
Cross-Modal Transformer Hashing with Multi-granularity Alignment}. In
\bibinfo{booktitle}{\emph{BMVC}}.
\newblock
\bibitem[Wang et~al\mbox{.}(2019)]%
{SR_survey1}
\bibfield{author}{\bibinfo{person}{Shoujin Wang}, \bibinfo{person}{Liang Hu},
\bibinfo{person}{Yan Wang}, \bibinfo{person}{Longbing Cao},
\bibinfo{person}{Quan~Z. Sheng}, {and} \bibinfo{person}{Mehmet Orgun}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Sequential Recommender Systems: Challenges,
Progress and Prospects}. In \bibinfo{booktitle}{\emph{IJCAI}}.
\newblock
\bibitem[Wei et~al\mbox{.}(2023)]%
{MMSSL}
\bibfield{author}{\bibinfo{person}{Wei Wei}, \bibinfo{person}{Chao Huang},
\bibinfo{person}{Lianghao Xia}, {and} \bibinfo{person}{Chuxu Zhang}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Multi-Modal Self-Supervised Learning for
Recommendation}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2302.10632}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[Wei et~al\mbox{.}(2020)]%
{GRCN}
\bibfield{author}{\bibinfo{person}{Yinwei Wei}, \bibinfo{person}{Xiang Wang},
\bibinfo{person}{Liqiang Nie}, \bibinfo{person}{Xiangnan He}, {and}
\bibinfo{person}{Tat-Seng Chua}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Graph-refined convolutional network for multimedia
recommendation with implicit feedback}. In \bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Wei et~al\mbox{.}(2019)]%
{MMGCN}
\bibfield{author}{\bibinfo{person}{Yinwei Wei}, \bibinfo{person}{Xiang Wang},
\bibinfo{person}{Liqiang Nie}, \bibinfo{person}{Xiangnan He},
\bibinfo{person}{Richang Hong}, {and} \bibinfo{person}{Tat-Seng Chua}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{MMGCN: Multi-modal graph convolution network for
personalized recommendation of micro-video}. In
\bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Wu et~al\mbox{.}(2022)]%
{MM-Rec}
\bibfield{author}{\bibinfo{person}{Chuhan Wu}, \bibinfo{person}{Fangzhao Wu},
\bibinfo{person}{Tao Qi}, \bibinfo{person}{Chao Zhang},
\bibinfo{person}{Yongfeng Huang}, {and} \bibinfo{person}{Tong Xu}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{MM-Rec: Visiolinguistic Model Empowered Multimodal
News Recommendation}. In \bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Xiao et~al\mbox{.}(2021)]%
{UPRec}
\bibfield{author}{\bibinfo{person}{Chaojun Xiao}, \bibinfo{person}{Ruobing
Xie}, \bibinfo{person}{Yuan Yao}, \bibinfo{person}{Zhiyuan Liu},
\bibinfo{person}{Maosong Sun}, \bibinfo{person}{Xu Zhang}, {and}
\bibinfo{person}{Leyu Lin}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{UPRec: User-aware Pre-training for Recommender
Systems}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2102.10989}}
(\bibinfo{year}{2021}).
\newblock
\bibitem[Xie et~al\mbox{.}(2022)]%
{CL4SRec}
\bibfield{author}{\bibinfo{person}{Xu Xie}, \bibinfo{person}{Fei Sun},
\bibinfo{person}{Zhaoyang Liu}, \bibinfo{person}{Shiwen Wu},
\bibinfo{person}{Jinyang Gao}, \bibinfo{person}{Jiandong Zhang},
\bibinfo{person}{Bolin Ding}, {and} \bibinfo{person}{Bin Cui}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Contrastive learning for sequential
recommendation}. In \bibinfo{booktitle}{\emph{ICDE}}.
\newblock
\bibitem[Xun et~al\mbox{.}(2021)]%
{IMRec}
\bibfield{author}{\bibinfo{person}{Jiahao Xun}, \bibinfo{person}{Shengyu
Zhang}, \bibinfo{person}{Zhou Zhao}, \bibinfo{person}{Jieming Zhu},
\bibinfo{person}{Qi Zhang}, \bibinfo{person}{Jingjie Li},
\bibinfo{person}{Xiuqiang He}, \bibinfo{person}{Xiaofei He},
\bibinfo{person}{Tat-Seng Chua}, {and} \bibinfo{person}{Fei Wu}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Why do we click: visual impression-aware news
recommendation}. In \bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Yang et~al\mbox{.}(2021)]%
{yang2021unimf}
\bibfield{author}{\bibinfo{person}{Shiquan Yang}, \bibinfo{person}{Rui Zhang},
\bibinfo{person}{Sarah~M Erfani}, {and} \bibinfo{person}{Jey~Han Lau}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{UniMF: A Unified Framework to Incorporate
Multimodal Knowledge Bases into End-to-End Task-Oriented Dialogue Systems.}.
In \bibinfo{booktitle}{\emph{IJCAI}}.
\newblock
\bibitem[Yi et~al\mbox{.}(2022)]%
{MMGCL}
\bibfield{author}{\bibinfo{person}{Zixuan Yi}, \bibinfo{person}{Xi Wang},
\bibinfo{person}{Iadh Ounis}, {and} \bibinfo{person}{Craig Macdonald}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Multi-modal graph contrastive learning for
micro-video recommendation}. In \bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Yuan et~al\mbox{.}(2020)]%
{PETERec}
\bibfield{author}{\bibinfo{person}{Fajie Yuan}, \bibinfo{person}{Xiangnan He},
\bibinfo{person}{Alexandros Karatzoglou}, {and} \bibinfo{person}{Liguang
Zhang}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Parameter-efficient transfer from sequential
behaviors for user modeling and recommendation}. In
\bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Yuan et~al\mbox{.}(2019)]%
{NextINet}
\bibfield{author}{\bibinfo{person}{Fajie Yuan}, \bibinfo{person}{Alexandros
Karatzoglou}, \bibinfo{person}{Ioannis Arapakis}, \bibinfo{person}{Joemon~M
Jose}, {and} \bibinfo{person}{Xiangnan He}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{A simple convolutional generative network for next
item recommendation}. In \bibinfo{booktitle}{\emph{WSDM}}.
\newblock
\bibitem[Yuan et~al\mbox{.}(2023)]%
{MoRec}
\bibfield{author}{\bibinfo{person}{Zheng Yuan}, \bibinfo{person}{Fajie Yuan},
\bibinfo{person}{Yu Song}, \bibinfo{person}{Youhua Li},
\bibinfo{person}{Junchen Fu}, \bibinfo{person}{Fei Yang},
\bibinfo{person}{Yunzhu Pan}, {and} \bibinfo{person}{Yongxin Ni}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Where to Go Next for Recommender Systems? ID-vs.
Modality-based recommender models revisited}. In
\bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2021c)]%
{LATTICE}
\bibfield{author}{\bibinfo{person}{Jinghao Zhang}, \bibinfo{person}{Yanqiao
Zhu}, \bibinfo{person}{Qiang Liu}, \bibinfo{person}{Shu Wu},
\bibinfo{person}{Shuhui Wang}, {and} \bibinfo{person}{Liang Wang}.}
\bibinfo{year}{2021}\natexlab{c}.
\newblock \showarticletitle{Mining latent structures for multimedia
recommendation}. In \bibinfo{booktitle}{\emph{MM}}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2023)]%
{MSM4SR}
\bibfield{author}{\bibinfo{person}{Lingzi Zhang}, \bibinfo{person}{Xin Zhou},
{and} \bibinfo{person}{Zhiqi Shen}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Multimodal Pre-training Framework for Sequential
Recommendation via Contrastive Learning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2303.11879}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[Zhang et~al\mbox{.}(2021b)]%
{UNBERT}
\bibfield{author}{\bibinfo{person}{Qi Zhang}, \bibinfo{person}{Jingjie Li},
\bibinfo{person}{Qinglin Jia}, \bibinfo{person}{Chuyuan Wang},
\bibinfo{person}{Jieming Zhu}, \bibinfo{person}{Zhaowei Wang}, {and}
\bibinfo{person}{Xiuqiang He}.} \bibinfo{year}{2021}\natexlab{b}.
\newblock \showarticletitle{UNBERT: User-News Matching BERT for News
Recommendation.}. In \bibinfo{booktitle}{\emph{IJCAI}}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2022)]%
{Re4}
\bibfield{author}{\bibinfo{person}{Shengyu Zhang}, \bibinfo{person}{Lingxiao
Yang}, \bibinfo{person}{Dong Yao}, \bibinfo{person}{Yujie Lu},
\bibinfo{person}{Fuli Feng}, \bibinfo{person}{Zhou Zhao},
\bibinfo{person}{Tat-Seng Chua}, {and} \bibinfo{person}{Fei Wu}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Re4: Learning to Re-contrast, Re-attend,
Re-construct for Multi-interest Recommendation}. In
\bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2019)]%
{FDSA}
\bibfield{author}{\bibinfo{person}{Tingting Zhang}, \bibinfo{person}{Pengpeng
Zhao}, \bibinfo{person}{Yanchi Liu}, \bibinfo{person}{Victor~S Sheng},
\bibinfo{person}{Jiajie Xu}, \bibinfo{person}{Deqing Wang},
\bibinfo{person}{Guanfeng Liu}, \bibinfo{person}{Xiaofang Zhou},
{et~al\mbox{.}}} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Feature-level Deeper Self-Attention Network for
Sequential Recommendation.}. In \bibinfo{booktitle}{\emph{IJCAI}}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2021a)]%
{PopBias2}
\bibfield{author}{\bibinfo{person}{Yang Zhang}, \bibinfo{person}{Fuli Feng},
\bibinfo{person}{Xiangnan He}, \bibinfo{person}{Tianxin Wei},
\bibinfo{person}{Chonggang Song}, \bibinfo{person}{Guohui Ling}, {and}
\bibinfo{person}{Yongdong Zhang}.} \bibinfo{year}{2021}\natexlab{a}.
\newblock \showarticletitle{Causal intervention for leveraging popularity bias
in recommendation}. In \bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Zhao(2022)]%
{RESETBERT}
\bibfield{author}{\bibinfo{person}{Qihang Zhao}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{RESETBERT4Rec: A pre-training model integrating
time and user historical behavior for sequential recommendation}. In
\bibinfo{booktitle}{\emph{SIGIR}}.
\newblock
\bibitem[Zhao et~al\mbox{.}(2023)]%
{LLM_survey}
\bibfield{author}{\bibinfo{person}{Wayne~Xin Zhao}, \bibinfo{person}{Kun Zhou},
\bibinfo{person}{Junyi Li}, \bibinfo{person}{Tianyi Tang},
\bibinfo{person}{Xiaolei Wang}, \bibinfo{person}{Yupeng Hou},
\bibinfo{person}{Yingqian Min}, \bibinfo{person}{Beichen Zhang},
\bibinfo{person}{Junjie Zhang}, \bibinfo{person}{Zican Dong},
{et~al\mbox{.}}} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{A survey of large language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2303.18223}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[Zhou et~al\mbox{.}(2018)]%
{DIN}
\bibfield{author}{\bibinfo{person}{Guorui Zhou}, \bibinfo{person}{Xiaoqiang
Zhu}, \bibinfo{person}{Chenru Song}, \bibinfo{person}{Ying Fan},
\bibinfo{person}{Han Zhu}, \bibinfo{person}{Xiao Ma},
\bibinfo{person}{Yanghui Yan}, \bibinfo{person}{Junqi Jin},
\bibinfo{person}{Han Li}, {and} \bibinfo{person}{Kun Gai}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Deep interest network for click-through rate
prediction}. In \bibinfo{booktitle}{\emph{KDD}}.
\newblock
\bibitem[Zhou et~al\mbox{.}(2023b)]%
{MMRec_Survey1}
\bibfield{author}{\bibinfo{person}{Hongyu Zhou}, \bibinfo{person}{Xin Zhou},
\bibinfo{person}{Zhiwei Zeng}, \bibinfo{person}{Lingzi Zhang}, {and}
\bibinfo{person}{Zhiqi Shen}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{A Comprehensive Survey on Multimodal Recommender
Systems: Taxonomy, Evaluation, and Future Directions}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2302.04473}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[Zhou et~al\mbox{.}(2020)]%
{S3-Rec}
\bibfield{author}{\bibinfo{person}{Kun Zhou}, \bibinfo{person}{Hui Wang},
\bibinfo{person}{Wayne~Xin Zhao}, \bibinfo{person}{Yutao Zhu},
\bibinfo{person}{Sirui Wang}, \bibinfo{person}{Fuzheng Zhang},
\bibinfo{person}{Zhongyuan Wang}, {and} \bibinfo{person}{Ji-Rong Wen}.}
\bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{S3-Rec: Self-supervised Learning for Sequential
Recommendation with Mutual Information Maximization}. In
\bibinfo{booktitle}{\emph{CIKM}}.
\newblock
\bibitem[Zhou et~al\mbox{.}(2022)]%
{FMLP-Rec}
\bibfield{author}{\bibinfo{person}{Kun Zhou}, \bibinfo{person}{Hui Yu},
\bibinfo{person}{Wayne~Xin Zhao}, {and} \bibinfo{person}{Ji-Rong Wen}.}
\bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Filter-enhanced MLP is all you need for sequential
recommendation}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Zhou et~al\mbox{.}(2023a)]%
{BM3}
\bibfield{author}{\bibinfo{person}{Xin Zhou}, \bibinfo{person}{Hongyu Zhou},
\bibinfo{person}{Yong Liu}, \bibinfo{person}{Zhiwei Zeng},
\bibinfo{person}{Chunyan Miao}, \bibinfo{person}{Pengwei Wang},
\bibinfo{person}{Yuan You}, {and} \bibinfo{person}{Feijun Jiang}.}
\bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{Bootstrap Latent Representations for Multi-Modal
Recommendation}. In \bibinfo{booktitle}{\emph{WWW}}.
\newblock
\bibitem[Zhu et~al\mbox{.}(2021)]%
{CDR_Survey}
\bibfield{author}{\bibinfo{person}{Feng Zhu}, \bibinfo{person}{Yan Wang},
\bibinfo{person}{Chaochao Chen}, \bibinfo{person}{Jun Zhou},
\bibinfo{person}{Longfei Li}, {and} \bibinfo{person}{Guanfeng Liu}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Cross-domain recommendation: challenges, progress,
and prospects}. In \bibinfo{booktitle}{\emph{IJCAI}}.
\newblock
\end{thebibliography}