Knowledge Graph-enhanced Sampling for Conversational Recommender System
Extracted source text · arXiv 2110.06637
% https://zhuanlan.zhihu.com/p/145009285
\documentclass[10pt,journal,compsoc]{IEEEtran}\usepackage{bm}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{enumitem}
\usepackage{bbm}
\usepackage{tabularx,booktabs}
\usepackage[section]{placeins}
\usepackage{float}
\usepackage{ifpdf}
\ifpdf
% pdf code
\else
% dvi code
\fi
% *** CITATION PACKAGES ***
%
\ifCLASSOPTIONcompsoc
% The IEEE Computer Society needs nocompress option
% requires cite.sty v4.0 or later (November 2003)
\usepackage[nocompress]{cite}
\else
% normal IEEE
\usepackage{cite}
\fi
% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
\usepackage[pdftex]{graphicx}
% \usepackage{svg}
% \svgsetup{
% inkscapepath=i/svg-inkscape/
% }
% \svgpath{{pictures/}}
% declare the path(s) where your graphic files are
% \graphicspath{{../pdf/}{../jpeg/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
% or other class option (dvipsone, dvipdf, if not using dvips). graphicx
% will default to the driver specified in the system graphics.cfg if no
% driver is specified.
% \usepackage[dvips]{graphicx}
% declare the path(s) where your graphic files are
% \graphicspath{{../eps/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation
% can be obtained at:
% http://www.ctan.org/pkg/graphicx
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found at:
% http://www.ctan.org/pkg/epslatex
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). The IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex
% *** MATH PACKAGES ***
%
\usepackage{amsmath}
\usepackage{amssymb}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics.
%
% Note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/amsmath
% *** SPECIALIZED LIST PACKAGES ***
%\usepackage{acronym}
% acronym.sty was written by Tobias Oetiker. This package provides tools for
% managing documents with large numbers of acronyms. (You don't *have* to
% use this package - unless you have a lot of acronyms, you may feel that
% such package management of them is bit of an overkill.)
% Do note that the acronym environment (which lists acronyms) will have a
% problem when used under IEEEtran.cls because acronym.sty relies on the
% description list environment - which IEEEtran.cls has customized for
% producing IEEE style lists. A workaround is to declared the longest
% label width via the IEEEtran.cls \IEEEiedlistdecl global control:
%
% \renewcommand{\IEEEiedlistdecl}{\IEEEsetlabelwidth{SONET}}
% \begin{acronym}
%
% \end{acronym}
% \renewcommand{\IEEEiedlistdecl}{\relax}% remember to reset \IEEEiedlistdecl
%
% instead of using the acronym environment's optional argument.
% The latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/acronym
\newenvironment{sequation}{\small\begin{equation}}{\end{equation}}
\newenvironment{tequation}{\tiny\begin{equation}}{\end{equation}}
\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as the IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/pkg/algorithms
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/pkg/algorithmicx
\usepackage{algorithm}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
% *** ALIGNMENT PACKAGES ***
%
\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/array
%\usepackage{mdwmath}
%\usepackage{mdwtab}
% Also highly recommended is Mark Wooding's extremely powerful MDW tools,
% especially mdwmath.sty and mdwtab.sty which are used to format equations
% and tables, respectively. The MDWtools set is already installed on most
% LaTeX systems. The lastest version and documentation is available at:
% http://www.ctan.org/pkg/mdwtools
% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.
%\usepackage{eqparbox}
% Also of notable interest is Scott Pakin's eqparbox package for creating
% (automatically sized) equal width boxes - aka "natural width parboxes".
% Available at:
% http://www.ctan.org/pkg/eqparbox
% *** SUBFIGURE PACKAGES ***
\ifCLASSOPTIONcompsoc
\usepackage[caption=false,font=footnotesize,labelfont=sf,textfont=sf]{subfig}
\else
\usepackage[caption=false,font=footnotesize]{subfig}
\fi
% subfig.sty, written by Steven Douglas Cochran, is the modern replacement
% for subfigure.sty, the latter of which is no longer maintained and is
% incompatible with some LaTeX packages including fixltx2e. However,
% subfig.sty requires and automatically loads Axel Sommerfeldt's caption.sty
% which will override IEEEtran.cls' handling of captions and this will result
% in non-IEEE style figure/table captions. To prevent this problem, be sure
% and invoke subfig.sty's "caption=false" package option (available since
% subfig.sty version 1.3, 2005/06/28) as this is will preserve IEEEtran.cls
% handling of captions.
% Note that the Computer Society format requires a sans serif font rather
% than the serif font used in traditional IEEE formatting and thus the need
% to invoke different subfig.sty package options depending on whether
% compsoc mode has been enabled.
%
% The latest version and documentation of subfig.sty can be obtained at:
% http://www.ctan.org/pkg/subfig
% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure.
% Be aware that LaTeX2e kernels dated 2015 and later have fixltx2e.sty's
% corrections already built into the system in which case a warning will
% be issued if an attempt is made to load fixltx2e.sty as it is no longer
% needed.
% The latest version and documentation can be found at:
% http://www.ctan.org/pkg/fixltx2e
%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/stfloats
% Do not use the stfloats baselinefloat ability as the IEEE does not allow
% \baselineskip to stretch. Authors submitting work to the IEEE should note
% that the IEEE rarely uses double column equations and that authors should try
% to avoid such use. Do not be tempted to use the cuted.sty or midfloat.sty
% packages (also by Sigitas Tolusis) as the IEEE does not format its papers in
% such ways.
% Do not attempt to use stfloats with fixltx2e as they are incompatible.
% Instead, use Morten Hogholm'a dblfloatfix which combines the features
% of both fixltx2e and stfloats:
%
% \usepackage{dblfloatfix}
% The latest version can be found at:
% http://www.ctan.org/pkg/dblfloatfix
%\ifCLASSOPTIONcaptionsoff
% \usepackage[nomarkers]{endfloat}
% \let\MYoriglatexcaption\caption
% \renewcommand{\caption}[2][\relax]{\MYoriglatexcaption[#2]{#2}}
%\fi
% endfloat.sty was written by James Darrell McCauley, Jeff Goldberg and
% Axel Sommerfeldt. This package may be useful when used in conjunction with
% IEEEtran.cls' captionsoff option. Some IEEE journals/societies require that
% submissions have lists of figures/tables at the end of the paper and that
% figures/tables without any captions are placed on a page by themselves at
% the end of the document. If needed, the draftcls IEEEtran class option or
% \CLASSINPUTbaselinestretch interface can be used to increase the line
% spacing as well. Be sure and use the nomarkers option of endfloat to
% prevent endfloat from "marking" where the figures would have been placed
% in the text. The two hack lines of code above are a slight modification of
% that suggested by in the endfloat docs (section 8.4.1) to ensure that
% the full captions always appear in the list of figures/tables - even if
% the user used the short optional argument of \caption[]{}.
% IEEE papers do not typically make use of \caption[]'s optional argument,
% so this should not be an issue. A similar trick can be used to disable
% captions of packages such as subfig.sty that lack options to turn off
% the subcaptions:
% For subfig.sty:
% \let\MYorigsubfloat\subfloat
% \renewcommand{\subfloat}[2][\relax]{\MYorigsubfloat[]{#2}}
% However, the above trick will not work if both optional arguments of
% the \subfloat command are used. Furthermore, there needs to be a
% description of each subfigure *somewhere* and endfloat does not add
% subfigure captions to its list of figures. Thus, the best approach is to
% avoid the use of subfigure captions (many IEEE journals avoid them anyway)
% and instead reference/explain all the subfigures within the main caption.
% The latest version of endfloat.sty and its documentation can obtained at:
% http://www.ctan.org/pkg/endfloat
%
% The IEEEtran \ifCLASSOPTIONcaptionsoff conditional can also be used
% later in the document, say, to conditionally put the References on a
% page by themselves.
% *** PDF, URL AND HYPERLINK PACKAGES ***
%
\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/url
% Basically, \url{my_url_here}.
% NOTE: PDF thumbnail features are not required in IEEE papers
% and their use requires extra complexity and work.
%\ifCLASSINFOpdf
% \usepackage[pdftex]{thumbpdf}
%\else
% \usepackage[dvips]{thumbpdf}
%\fi
% thumbpdf.sty and its companion Perl utility were written by Heiko Oberdiek.
% It allows the user a way to produce PDF documents that contain fancy
% thumbnail images of each of the pages (which tools like acrobat reader can
% utilize). This is possible even when using dvi->ps->pdf workflow if the
% correct thumbpdf driver options are used. thumbpdf.sty incorporates the
% file containing the PDF thumbnail information (filename.tpm is used with
% dvips, filename.tpt is used with pdftex, where filename is the base name of
% your tex document) into the final ps or pdf output document. An external
% utility, the thumbpdf *Perl script* is needed to make these .tpm or .tpt
% thumbnail files from a .ps or .pdf version of the document (which obviously
% does not yet contain pdf thumbnails). Thus, one does a:
%
% thumbpdf filename.pdf
%
% to make a filename.tpt, and:
%
% thumbpdf --mode dvips filename.ps
%
% to make a filename.tpm which will then be loaded into the document by
% thumbpdf.sty the NEXT time the document is compiled (by pdflatex or
% latex->dvips->ps2pdf). Users must be careful to regenerate the .tpt and/or
% .tpm files if the main document changes and then to recompile the
% document to incorporate the revised thumbnails to ensure that thumbnails
% match the actual pages. It is easy to forget to do this!
%
% Unix systems come with a Perl interpreter. However, MS Windows users
% will usually have to install a Perl interpreter so that the thumbpdf
% script can be run. The Ghostscript PS/PDF interpreter is also required.
% See the thumbpdf docs for details. The latest version and documentation
% can be obtained at.
% http://www.ctan.org/pkg/thumbpdf
% NOTE: PDF hyperlink and bookmark features are not required in IEEE
% papers and their use requires extra complexity and work.
% *** IF USING HYPERREF BE SURE AND CHANGE THE EXAMPLE PDF ***
% *** TITLE/SUBJECT/AUTHOR/KEYWORDS INFO BELOW!! ***
\newcommand\MYhyperrefoptions{bookmarks=true,bookmarksnumbered=true,
pdfpagemode={UseOutlines},plainpages=false,pdfpagelabels=true,
colorlinks=true,linkcolor={black},citecolor={black},urlcolor={black},
pdftitle={Knowledge Graph-enhanced Sampling for Conversational Recommender System},%<!CHANGE!
pdfsubject={Conversational Recommender System},%<!CHANGE!
pdfauthor={Mengyuan Zhao, Xiaowen Huang*, Jitao Sang},%<!CHANGE!
pdfkeywords={Conversational Recommender System, Knowledge Graph, Negative Sampling, Active Learning, Reinforcement Learning}}%<^!CHANGE!
\ifCLASSINFOpdf
\usepackage[\MYhyperrefoptions,pdftex]{hyperref}
\else
\usepackage[\MYhyperrefoptions,breaklinks=true,dvips]{hyperref}
\usepackage{breakurl}
\fi
% One significant drawback of using hyperref under DVI output is that the
% LaTeX compiler cannot break URLs across lines or pages as can be done
% under pdfLaTeX's PDF output via the hyperref pdftex driver. This is
% probably the single most important capability distinction between the
% DVI and PDF output. Perhaps surprisingly, all the other PDF features
% (PDF bookmarks, thumbnails, etc.) can be preserved in
% .tex->.dvi->.ps->.pdf workflow if the respective packages/scripts are
% loaded/invoked with the correct driver options (dvips, etc.).
% As most IEEE papers use URLs sparingly (mainly in the references), this
% may not be as big an issue as with other publications.
%
% That said, Vilar Camara Neto created his breakurl.sty package which
% permits hyperref to easily break URLs even in dvi mode.
% Note that breakurl, unlike most other packages, must be loaded
% AFTER hyperref. The latest version of breakurl and its documentation can
% be obtained at:
% http://www.ctan.org/pkg/breakurl
% breakurl.sty is not for use under pdflatex pdf mode.
%
% The advanced features offer by hyperref.sty are not required for IEEE
% submission, so users should weigh these features against the added
% complexity of use.
% The package options above demonstrate how to enable PDF bookmarks
% (a type of table of contents viewable in Acrobat Reader) as well as
% PDF document information (title, subject, author and keywords) that is
% viewable in Acrobat reader's Document_Properties menu. PDF document
% information is also used extensively to automate the cataloging of PDF
% documents. The above set of options ensures that hyperlinks will not be
% colored in the text and thus will not be visible in the printed page,
% but will be active on "mouse over". USING COLORS OR OTHER HIGHLIGHTING
% OF HYPERLINKS CAN RESULT IN DOCUMENT REJECTION BY THE IEEE, especially if
% these appear on the "printed" page. IF IN DOUBT, ASK THE RELEVANT
% SUBMISSION EDITOR. You may need to add the option hypertexnames=false if
% you used duplicate equation numbers, etc., but this should not be needed
% in normal IEEE work.
% The latest version of hyperref and its documentation can be obtained at:
% http://www.ctan.org/pkg/hyperref
% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex). ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course.)
% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
%
% paper title
% Titles are generally capitalized except for words such as a, an, and, as,
% at, but, by, for, in, nor, of, on, or, the, to and up, which are usually
% not capitalized unless they are the first or last word of the title.
% Linebreaks \\ can be used within to get better formatting as desired.
% Do not put math or special symbols in the title.
\title{Knowledge Graph-enhanced Sampling for Conversational Recommendation System}
%
%
% author names and IEEE memberships
% note positions of commas and nonbreaking spaces (~) LaTeX will not break
% a structure at a ~ so this keeps an author's name from being broken across
% two lines.
% use \thanks{} to gain access to the first footnote area
% a separate \thanks must be used for each paragraph as LaTeX2e's \thanks
% was not built to handle multiple paragraphs
%
%
%\IEEEcompsocitemizethanks is a special \thanks that produces the bulleted
% lists the Computer Society journals use for "first footnote" author
% affiliations. Use \IEEEcompsocthanksitem which works much like \item
% for each affiliation group. When not in compsoc mode,
% \IEEEcompsocitemizethanks becomes like \thanks and
% \IEEEcompsocthanksitem becomes a line break with idention. This
% facilitates dual compilation, although admittedly the differences in the
% desired content of \author between the different types of papers makes a
% one-size-fits-all approach a daunting prospect. For instance, compsoc
% journal papers have the author affiliations above the "Manuscript
% received ..." text while in non-compsoc journals this is reversed. Sigh.
\author{Mengyuan Zhao, % <-this % stops a space,
Xiaowen Huang*,
Lixi Zhu,
Jitao Sang,
Jian Yu
\IEEEcompsocitemizethanks{\IEEEcompsocthanksitem Mengyuan Zhao is with the School of Computer and Information Technology \& Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing 100044, China.\protect\\
% note need leading \protect in front of \\ to get a newline within \thanks as
% \\ is fragile and will error, could use \hfil\break instead.
E-mail: 19120453@bjtu.edu.cn.
%\IEEEcompsocthanksitem J. Doe and J. Doe are with Anonymous University.}% <-this % stops a space
\IEEEcompsocthanksitem Xiaowen Huang is with the School of Computer and Information Technology \& Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing 100044, China.\protect\\
E-mail: xwhuang@bjtu.edu.cn.
\IEEEcompsocthanksitem Lixi Zhu is with the School of Computer and Information Technology \& Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing 100044, China.\protect\\
E-mail: zlxxlz1026@gmail.com.
\IEEEcompsocthanksitem Jitao Sang is with the School of Computer and Information Technology \& Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing 100044, China.\protect\\
E-mail: jtsang@bjtu.edu.cn.
\IEEEcompsocthanksitem Jian Yu is with the School of Computer and Information Technology \& Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing 100044, China.\protect\\
E-mail: jianyu@bjtu.edu.cn.}
\thanks{Manuscript received July 1st, 2021; revised September 1st, 2021.\protect\\
(Corresponding author: Xiaowen Huang.)}
}
% note the % following the last \IEEEmembership and also \thanks -
% these prevent an unwanted space from occurring between the last author name
% and the end of the author line. i.e., if you had this:
%
% \author{....lastname \thanks{...} \thanks{...} }
% ^------------^------------^----Do not want these spaces!
%
% a space would be appended to the last name and could cause every name on that
% line to be shifted left slightly. This is one of those "LaTeX things". For
% instance, "\textbf{A} \textbf{B}" will typeset as "A B" not "AB". To get
% "AB" then you have to do: "\textbf{A}\textbf{B}"
% \thanks is no different in this regard, so shield the last } of each \thanks
% that ends a line with a % and do not let a space in before the next \thanks.
% Spaces after \IEEEmembership other than the last one are OK (and needed) as
% you are supposed to have spaces between the names. For what it is worth,
% this is a minor point as most people would not even notice if the said evil
% space somehow managed to creep in.
% The paper headers
\markboth{IEEE TRANSACTIONS ON KNOWLEDGE AND DATA ENGINEERING, VOL. xx, NO. x, JULY 2021}%
{Zhao \MakeLowercase{\textit{et al.}}: Knowledge Graph-enhanced Sampling for Conversational Recommendation System}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the twoside option.
%
% *** Note that you probably will NOT want to include the author's ***
% *** name in the headers of peer review papers. ***
% You can use \ifCLASSOPTIONpeerreview for conditional compilation here if
% you desire.
% The publisher's ID mark at the bottom of the page is less important with
% Computer Society journal papers as those publications place the marks
% outside of the main text columns and, therefore, unlike regular IEEE
% journals, the available text space is not reduced by their presence.
% If you want to put a publisher's ID mark on the page you can do it like
% this:
%\IEEEpubid{0000--0000/00\$00.00~\copyright~2015 IEEE}
% or like this to get the Computer Society new two part style.
%\IEEEpubid{\makebox[\columnwidth]{\hfill 0000--0000/00/\$00.00~\copyright~2015 IEEE}%
%\hspace{\columnsep}\makebox[\columnwidth]{Published by the IEEE Computer Society\hfill}}
% Remember, if you use this you must call \IEEEpubidadjcol in the second
% column for its text to clear the IEEEpubid mark (Computer Society journal
% papers don't need this extra clearance.)
% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}
% for Computer Society papers, we must declare the abstract and index terms
% PRIOR to the title within the \IEEEtitleabstractindextext IEEEtran
% command as these need to go into the title area created by \maketitle.
% As a general rule, do not put math, special symbols or citations
% in the abstract or keywords.
\IEEEtitleabstractindextext{%
\begin{abstract}
The traditional recommendation systems mainly use offline user data to train offline models, and then recommend items for online users, thus suffering from the unreliable estimation of user preferences based on sparse and noisy historical data. Conversational Recommendation System (CRS) uses the interactive form of the dialogue systems to solve the intrinsic problems of traditional recommendation systems. However, due to the lack of contextual information modeling, the existing CRS models are unable to deal with the exploitation and exploration (E\&E) problem well, resulting in the heavy burden on users. To address the aforementioned issue, this work proposes a contextual information enhancement model tailored for CRS, called Knowledge Graph-enhanced Sampling (KGenSam). KGenSam integrates the dynamic graph of user interaction data with the external knowledge into one heterogeneous Knowledge Graph (KG) as the contextual information environment. Then, two samplers are designed to enhance knowledge by sampling fuzzy samples with high uncertainty for obtaining user preferences and reliable negative samples for updating recommender to achieve efficient acquisition of user preferences and model updating, and thus provide a powerful solution for CRS to deal with E\&E problem. Experimental results on two real-world datasets demonstrate the superiority of KGenSam with significant improvements over state-of-the-art methods.
% The traditional recommendation systems mainly use offline user data to train offline models, and then recommend items for online users. This results in suffering from the unreliable estimation of user preferences based on sparse and noisy historical data. Since the dialogue systems focus on the user’s real-time feedback data and obtains the user’s current interaction intentions, it inspires the Conversational Recommendation System (CRS) to use the interactive form to solve the remaining problems of traditional recommendation systems. However, due to the lack of contextual information modeling, the existing CRS models are unable to deal with the exploitation and exploration (E\&E) problem well, resulting in the heavy burden on users. To address the aforementioned issue, this work proposes a contextual information enhancement model tailored for CRS, called Knowledge Graph-enhanced Sampling (KGenSam). KGenSam integrates the dynamic graph of user interaction data with the external knowledge into one heterogeneous Knowledge Graph (KG) as the contextual information environment. Then, two samplers are designed to enhance knowledge by sampling fuzzy samples with high uncertainty for obtaining user preferences and reliable negative samples for updating recommender to achieve efficient acquisition of user preferences and model updating, and thus provide a powerful solution for CRS to deal with E\&E problem. Experimental results on two real-world datasets demonstrate the superiority of KGenSam with significant improvements over state-of-the-art methods.
\end{abstract}
% Note that keywords are not normally used for peerreview papers.
\begin{IEEEkeywords}
Conversational recommendation system, knowledge graph, negative sampling, active learning, reinforcement learning.
\end{IEEEkeywords}}
% make the title area
\maketitle
% To allow for easy dual compilation without having to reenter the
% abstract/keywords data, the \IEEEtitleabstractindextext text will
% not be used in maketitle, but will appear (i.e., to be "transported")
% here as \IEEEdisplaynontitleabstractindextext when compsoc mode
% is not selected <OR> if conference mode is selected - because compsoc
% conference papers position the abstract like regular (non-compsoc)
% papers do!
\IEEEdisplaynontitleabstractindextext
% \IEEEdisplaynontitleabstractindextext has no effect when using
% compsoc under a non-conference mode.
% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle
% Computer Society journal (but not conference!) papers do something unusual
% with the very first section heading (almost always called "Introduction").
% They place it ABOVE the main text! IEEEtran.cls does not automatically do
% this for you, but you can achieve this effect with the provided
% \IEEEraisesectionheading{} command. Note the need to keep any \label that
% is to refer to the section immediately after \section in the above as
% \IEEEraisesectionheading puts \section within a raised box.
% The very first letter is a 2 line initial drop letter followed
% by the rest of the first word in caps (small caps for compsoc).
%
% form to use if the first word consists of a single letter:
% \IEEEPARstart{A}{demo} file is ....
%
% form to use if you need the single drop letter followed by
% normal text (unknown if ever used by the IEEE):
% \IEEEPARstart{A}{}demo file is ....
%
% Some journals put the first two words in caps:
% \IEEEPARstart{T}{his demo} file is ....
%
% Here we have the typical use of a "T" for an initial drop letter
% and "HIS" in caps to complete the first word.
% \IEEEPARstart{T}{his} demo file is intended to serve as a ``starter file''
% for IEEE Computer Society journal papers produced under \LaTeX\ using
% IEEEtran.cls version 1.8b and later.
% You must have at least 2 lines in the paragraph with the drop letter
% (should never be an issue)
% I wish you the best of success.
% \hfill mds
% \hfill August 26, 2015
% An example of a floating figure using the graphicx package.
% Note that \label must occur AFTER (or within) \caption.
% For figures, \caption should occur after the \includegraphics.
% Note that IEEEtran v1.7 and later has special internal code that
% is designed to preserve the operation of \label within \caption
% even when the captionsoff option is in effect. However, because
% of issues like this, it may be the safest practice to put all your
% \label just after \caption rather than within \caption{}.
%
% Reminder: the "draftcls" or "draftclsnofoot", not "draft", class
% option should be used if it is desired that the figures are to be
% displayed while in draft mode.
%
%\begin{figure}[!t]
%\centering
%\includegraphics[width=2.5in]{myfigure}
% where an .eps filename suffix will be assumed under latex,
% and a .pdf suffix will be assumed for pdflatex; or what has been declared
% via \DeclareGraphicsExtensions.
%\caption{Simulation results for the network.}
%\label{fig_sim}
%\end{figure}
% Note that the IEEE typically puts floats only at the top, even when this
% results in a large percentage of a column being occupied by floats.
% However, the Computer Society has been known to put floats at the bottom.
% An example of a double column floating figure using two subfigures.
% (The subfig.sty package must be loaded for this to work.)
% The subfigure \label commands are set within each subfloat command,
% and the \label for the overall figure must come after \caption.
% \hfil is used as a separator to get equal spacing.
% Watch out that the combined width of all the subfigures on a
% line do not exceed the text width or a line break will occur.
%
%\begin{figure*}[!t]
%\centering
%\subfloat[Case I]{\includegraphics[width=2.5in]{box}%
%\label{fig_first_case}}
%\hfil
%\subfloat[Case II]{\includegraphics[width=2.5in]{box}%
%\label{fig_second_case}}
%\caption{Simulation results for the network.}
%\label{fig_sim}
%\end{figure*}
%
% Note that often IEEE papers with subfigures do not employ subfigure
% captions (using the optional argument to \subfloat[]), but instead will
% reference/describe all of them (a), (b), etc., within the main caption.
% Be aware that for subfig.sty to generate the (a), (b), etc., subfigure
% labels, the optional argument to \subfloat must be present. If a
% subcaption is not desired, just leave its contents blank,
% e.g., \subfloat[].
% An example of a floating table. Note that, for IEEE style tables, the
% \caption command should come BEFORE the table and, given that table
% captions serve much like titles, are usually capitalized except for words
% such as a, an, and, as, at, but, by, for, in, nor, of, on, or, the, to
% and up, which are usually not capitalized unless they are the first or
% last word of the caption. Table text will default to \footnotesize as
% the IEEE normally uses this smaller font for tables.
% The \label must come after \caption as always.
%
%\begin{table}[!t]
%% increase table row spacing, adjust to taste
\renewcommand{\arraystretch}{1.3}
% if using array.sty, it might be a good idea to tweak the value of
% \extrarowheight as needed to properly center the text within the cells
%\caption{An Example of a Table}
%\label{table_example}
%\centering
%% Some packages, such as MDW tools, offer better commands for making tables
%% than the plain LaTeX2e tabular which is used here.
%\begin{tabular}{|c||c|}
%\hline
%One & Two\\
%\hline
%Three & Four\\
%\hline
%\end{tabular}
%\end{table}
% Note that the IEEE does not put floats in the very first column
% - or typically anywhere on the first page for that matter. Also,
% in-text middle ("here") positioning is typically not used, but it
% is allowed and encouraged for Computer Society conferences (but
% not Computer Society journals). Most IEEE journals/conferences use
% top floats exclusively.
% Note that, LaTeX2e, unlike IEEE journals/conferences, places
% footnotes above bottom floats. This can be corrected via the
% \fnbelowfloat command of the stfloats package.
\ifCLASSOPTIONcompsoc
\IEEEraisesectionheading{\section{Introduction}\label{sec:introduction}}
\else
\section{Introduction}
\label{sec:introduction}
\fi
\IEEEPARstart{R}{ecommender} systems are the information filtering systems that help users filter a large number of invalid information to obtain information or items by estimating their interests and preferences. The mainstream traditional recommendation systems mainly use offline and historical user data to continuously train and optimize offline models, and then recommend items for online users. There are three main problems: the unrealistic estimation of user preferences based on sparse and noisy historical data\cite{2020WangDenoising,2020WangClicknotLike,2019LeeMeLU}, the ignorance of online contextual factors that affect user behavior\cite{2020CenControllableMultiInterest,2019GaoBLOMA,2019JagermanOffPolicyEvaluation}, and the unreliable assumption that users are aware of their preferences by default\cite{2013WangContingencyApproachInteractionModes}. Inspired by the dialogue systems, which focus on the user's real-time feedback data and obtains the user's current interaction intentions, a new recommendation method, Conversational Recommendation System (CRS), is trying to combine the interactive form of the dialogue systems with the recommendation task and becomes an effective means to solve the traditional recommendation problem. Through online interactive methods, CRS can guide and capture user' current preferences and interests by asking user whether he/she likes an item attribute or not, and utilize the user’s preferred attributes or rejected attributes to provide recommendation results that most fit users' current preferences in time.
The core research difficulty of CRS is exploitation and exploration (E\&E) problem. Specifically, at each turn of the conversational session, CRS may choose to utilize the user preference information obtained during the previous interaction to output the current best recommended items, which is the exploitation of E\&E; or choose to continue to interact with users to obtain more real-time user preference information, which is the exploration of E\&E. The difficulty of solving CRS's E\&E problem lies in achieving a special Nash equilibrium in the field of recommendation systems: if CRS is too inclined to choose exploitation action ($recommend$) without understanding the current user preferences, there will be a lot of failed recommendations which poor user experience; in contrast, if CRS adopts excessive exploration action ($ask$), the interaction burden on users will be increased and the risk of user churn will become serious. At present, CRS researchers mostly use Deep Reinforcement Learning (DRL) to learn a conversational recommendation policy to solve the E\&E problem. Because DRL has the ability of sequential decision-making, and it naturally fits with the interaction form of CRS. From the experimental results of the previous work, the CRS based on DRL achieves much better results than other methods, and the state-of-the-art CRS methods are all based on DRL. However, the existing works of conversational recommendation fail to put forward an effective solution to E\&E problem and build an efficient and accurate CRS, due to the lack of information utilization of the interactive environment.
In this paper, we believe that sufficient modeling of environment information is the key point to solving the E\&E problem. Therefore, our goal is to build a CRS that can make good use of the information of contextual environment at each turn of conversational recommendations. From the perspective of data enhancement, we propose a novel solution named Knowledge Graph-enhanced Sampling (KGenSam), which solves the insufficient information modeling problem in CRS. In KGenSam, we introduce Knowledge Graph (KG) into the basic CRS framework as its interactive environment, and design two sampler modules to model and enhance the information of KG environment. Hence, CRS will learn to make E\&E decisions based on enhanced knowledge.
KGenSam adopts the sampling method of KG nodes to enhance knowledge in CRS. The design of the two samplers in KGenSam is inspired by solving the following two specific problems which prevent CRS from making the optimal decisions of E\&E:
\textbf{(1) What attributes should CRS ask to help the recommender fit the current user's preferences efficiently?}\cite{lei2020estimation}
In a conversational recommendation session, CRS needs to obtain sufficient user preference information as much as possible to achieve a successful recommendation that fits the current user preference. Obviously, on the premise of successful recommendation in the final round, the fewer attributes the CRS asks, the shorter the interaction rounds, and the higher the efficiency of CRS. We believe that every asking opportunity of CRS is extremely valuable, and CRS needs to obtain as much user preference information as possible at every $ask$ opportunity. In the design of this work, we naturally refer to the core idea of Active Learning, that is, when CRS chooses $ask$ action, high-uncertainty attribute candidates that can not be judged by the current model need to be given priority attention. Because these attribute samples with high uncertainty will greatly improve the recommendation confidence after being annotated by users and will make great contributions to achieving the goal of obtaining sufficient user preferences with as little user interaction burden as possible. Therefore, to screen out the fuzzy samples with high uncertainty, we design an Active Sampler, which takes the sample distribution state in KG and conversation state as input and learns the features of fuzzy samples via active reinforcement learning.
\textbf{(2) How can CRS update the recommender precisely to fit the implicit preference information in user feedback?}
To train an accurate personalized recommender, both positive and negative samples of user feedback are required\cite{2012RendleBPR,2017HeFastMF}.The user interaction form of CRS naturally solves the one-class problem\cite{2008PanOneClass} of traditional recommendation that most user data is positive feedback. In CRS, besides the user's positive feedback data, the user's explicit negative feedback data can also be directly obtained. But CRS also presents a new research challenge. The number of conversation turns is often limited to more than a dozen, which causes the amount of user feedback data in a conversation recommendation session is also very limited. Generally, sparse user feedback and huge candidate item space require a lot of training data to train an effective recommender. How to update the recommender by using rare online feedback data has become a new challenge for researchers. Inspired by the idea of solving one-class problems by negative sampling in traditional recommendation systems, we try to supplement sparse interactive data by enhancing the positive and negative sample pairs in KG. We design a Negative Sampler, which refers to the hard negative samples mining method in image recognition. The Negative Sampler takes the sample distribution state in KG and conversation state as input, learns the features of hard negative samples via reinforcement learning, and constructs the positive and negative sample pairs enhanced from the user feedback data to accurately update the recommender.
To evaluate the effectiveness of KGenSam, we conducted experiments on the Yelp and LastFM datasets. The experimental results show that KGenSam significantly outperforms the state-of-the-art CRS methods, indicating that KGenSam has the ability to enhance the user experience of the whole CRS. We release the codes, datasets and demo of the KGenSam-based online CRS at \href{https://github.com/ORZisemoji/KGenSam}{https://github.com/ORZisemoji/KGenSam}.
In a nutshell, this work makes the following main contributions:
\begin{itemize}
\item We propose the KGenSam model, which provides a new angle for building CRS. By sampling in KG, KGenSam enhances the contextual knowledge of CRS and solves the E\&E problem caused by insufficient information modeling of CRS. To the best of our knowledge, it is the first time to introduce the knowledge graph-enhanced sampling method into the conversational recommendation.
\item We introduce the active learning and the hard-negative sample mining methods into CRS. By incorporating the fuzzy samples and hard-negative samples in the CRS task, we facilitate the utilization of KG for CRS, which contributes to mine the user preference information in the KG environment to solve the data sparsity problem of the user feedback.
\item Extensive experiments have been conducted on two benchmark datasets, demonstrating that KGenSam-based CRS is able to achieve better performance than state-of-the-arts with much fewer conversation turns, which indicates high sample efficiency.
\item We implemented an online CRS to verify the effectiveness of our method in real application scenarios.
\end{itemize}
\section{Related work}
% \subsection{Conversational Recommender System}
Recommender systems are the important applications of artificial intelligence, which can help users find the information that fits their preferences in the huge amount of data when there is no clear demand or the environment information is overloaded.
At present, the mainstream recommendation systems are based on offline, historical user data, constantly optimize the offline performance, train the model, and then recommend items for online users. This offline update mode makes the recommender systems have inherent weakness \cite{2020WangDenoising,2020WangClicknotLike,2019LeeMeLU}, that is, it is difficult to fit the user's online behavior, and the user's real-time feedback will not be received in time\cite{2020CenControllableMultiInterest,2019GaoBLOMA,2019JagermanOffPolicyEvaluation,2013WangContingencyApproachInteractionModes}, so it is difficult for the systems to know the user's current intention and interest when using the recommender systems.
The development of dialogue technology in dialogue systems provides a new solution to the problems existing in traditional recommendation systems, which is called conversational recommendation. Conversational recommendation breaks the barrier of information asymmetry between systems and user in static recommendation systems through rich interaction behavior, and allows recommendation systems to capture user preferences dynamically in the interactive conversation with users. Based on the idea of conversational recommendation, CRS is a task-oriented and multi-rounds recommendation system, on the one hand, it explores users' current interest preferences, guides users to find new interest points, which realizes users' long-term retention; on the other hand, it also can receive feedback from users and update the recommender in real-time, which realizes dynamic learning of preferences.
Recently years, researchers have proposed different CRS models designed from different perspectives. To define a standard form for conversation recommendation, Zhang et al.\cite{zhang2018towards} proposed a conversation search engine called SAUR. To solve the problem of user cold-start in the movie recommendation scenario, Li et al.\cite{li2018towards} designed a
CRS that can interact with users through natural language. To obtain both long-term and real-time user preferences, Christakopoulou et al.\cite{christakopoulou2018q} designed a CRS that can take user interaction history as part of input data. After that, Sun et al.\cite{sun2018conversational} considered multi-rounds interaction with the user and designed a CRS that can obtain user attribute-level preferences multiple times. Lei et al.\cite{lei2020estimation} proposed a multi-rounds CRS framework named EAR, which can obtain both item-level and attribute-level feedback during the interaction. In order to take full advantage of conversational recommendation, Lei et al.\cite{lei2020interactive} introduced KG into CRS and proposed a new CRS framework called CPR. In order to make full use of user feedback information, Xu et al.\cite{xv2021FPAN} proposed FPAN framework, in which Gate Mechanism is used to dynamically modify the representations of
items and users based on the feedback information.
Overall, the research of CRS is committed to improving the satisfaction of online recommendations and the long-term retention rate of users in the system, and DRL has been widely used in CRS for learning conversational recommendation policy. However, the standard DRL model can not meet the needs of CRS, researchers are faced with the difficult convergence problem of CRS's training and low effectiveness problem of CRS's interaction. The main reason for these problems is the lack of clear rules in the contextual environment in CRS which leads to the lack of clear reward function and clear action space in the training of the reinforcement model. This can be summarized as the problem of insufficient environmental information modeling, which delays the updating of parameters in the training interaction, so CRS needs more interaction rounds to get user preference information and achieve a successful recommendation.
% \subsection{Active Learning in Recommendation}
% Subsection text here.
% needed in second column of first page if using \IEEEpubid
%\IEEEpubidadjcol
% \subsection{Negative Sampling for Recommendation}
% Negative sampling method in recommended system\cite{2020WangKGPolicy}
\section{Preliminaries}
\subsection{Problem Formulation}
%MDP
% desc KG DATA
In this subsection, We formally present interaction data and KG in our Task.
\textbf{Interaction Data.} $U$, $I$ and $A$ denote the set of users, items and attributes, respectively. Each item $i\in I$ is associated with a set of attributes $P_i\subseteq A$.
For each user, we view each observed user-item interaction as a positive instance, while sampling a negative item to pair the user. And training interactive data is structured into two pairwise data sets, $O_I$ and $O_A$. $O_I$ and $O_A$ represent the user feedback on items and attributes, respectively:
\begin{equation}
O_I = \{(u,i^+,i^-)|u\in U,i^+\in I^+,i^-\in I^-\}
\end{equation}
where $I^+$ denotes the sets of positive items, each of which has at least one historical interaction record with the user, and $I^-$ denote the sets of negative items, most of which are implicit negative feedback data of the user, and a few items are explicit negative feedback directly obtained by CRS.
\begin{equation}
O_A = \{(u,p^+,p^-)|u\in U,p^+\in A^+,i^-\in A^-\}
\end{equation}
where $A^+$ denotes the sets of positive attributes that are the attributes of positive items or the user preferences $P_u$ acquired during the conversational recommendation session, and $A^-$ denote the sets of negative attributes that are the attributes of negative items or the $ask$ attributes rejected by the user during the conversational recommendation session.
\textbf{Knowledge Graph.} As prior efforts\cite{2018ZhaoKB4Rec} show, items in user-item interaction data can be aligned with the corresponding entities in external KG. With such alignments, the external knowledge of item attributes as well as the interaction data are organized in the form of heterogeneous KG\cite{2019CaoUnifying}.
%\subsection{Background: multi-round CRS}
\subsection{Background settings of CRS in Our Task}
% \FloatBarrier
% LaTeX 中如何取消表格和图片浮动？
% https://www.zhihu.com/question/25082703
\begin{figure}[htb]
\centering
\includegraphics[width=3.7in]{pictures/CRSframework.pdf}
\caption{The basic framework and workflow of multi-round CRS.}
\label{fig:CRSframework}
\end{figure}
As shown in Figure \ref{fig:CRSframework}, the user is the interaction object of CRS, and CRS mainly consists of a recommender module and an interactive policy module: the recommender module is responsible for generating a recommendation list based on the user preference information; the interactive policy module is responsible for deciding whether to ask users for more preference information or output recommended items at current conversational turn. As shown in Figure \ref{fig:CRSframework}, a conversational recommendation session starts when the user enters the system, and then CRS chooses the $recommend$ action or the $ask$ action according to the current understanding of the current user’s preferences. If the $recommend$ action is selected, the system will output the recommended items ranked to the top by the recommender to the user, and if the $ask$ action is selected, the system will output the asked item attributes selected by the interactive policy to the user. After that, back to the user side, the user gives binary feedback (accept or reject) on the recommended items or the asked item attributes. The whole interactive session ends when a successful recommendation is achieved or the set maximum round $T$ is reached.
\subsubsection{User Simulator}
User simulator is used to replace real users to interact with the system, making CRS tasks more suitable for offline academic research. We use the same user simulator design as \cite{sun2018conversational,lei2020estimation}.
\subsubsection{Recommender}
In the existing research work of CRS, researchers tend to choose the simple but effective recommendation model, Factorization Machine(FM), as the recommender of CRS. Because of its effectiveness and agility, FM can meet the recommendation requirements of CRS without increasing the complexity of CRS algorithm. Given user $u$ and his/her preferred attributes $P_u$ in the conversation, FM predict how likely $u$ will like current item $i$ and current attribute $p$ in the conversation session as:
\begin{equation}
\tilde{f}(i\mid u,P_u) = \boldsymbol{u}^T\boldsymbol{i}+\sum_{{p_k}\in{P_u}}\boldsymbol{i}^T\boldsymbol{p_k}
\end{equation}
\begin{equation}
\tilde{f}(p\mid u,P_u) = \boldsymbol{u}^T\boldsymbol{p}+\sum_{{p_k}\in{P_u}}\boldsymbol{p}^T\boldsymbol{p_k}
\end{equation}
The FM is trained by optimizing the pairwise Bayesian Personalized Ranking (BPR)\cite{2012RendleBPR} objective which assumes that the ground truth sample (item or attribute) should be ranked higher than other samples, and the training losses of the FM model are as follows:
\begin{sequation}
\begin{split}
{Loss}_{item} =\sum_{{(u,i^+,i^-)}\in{O_I}}&
-\ln\sigma (\tilde{f}(i^+\mid u,P_u)-\tilde{f}(i^-\mid u,P_u))\\
& + \lambda_{\Theta_{FM}} \|{\Theta_{FM}}\|^2
\end{split}
\end{sequation}
\begin{sequation}
\begin{split}
{Loss}_{attri} =\sum_{{(u,p^+,p^-)}\in{O_A}}& -\ln\sigma (\tilde{f}(p^+\mid u,P_u)-\tilde{f}(p^-\mid u,P_u))\\
& + \lambda_{\Theta_{FM}} \|{\Theta_{FM}}\|^2
\end{split}
\end{sequation}
\subsubsection{Interact Policy Network}
We use a two-layer feed forward neural network as the interact policy network and use the standard Deep Q-learning\cite{2019VolodymyrHuman} for optimization.
The policy network takes the current session state $state_his$ as input and outputs the values $Q(s, a)$ for the two actions, $a_{ask}$ and $a_{rec}$. CRS will choose the action with higher estimated reward $reward$, which is composed of $reward_{rec\_suc}$, $reward_{rec\_fail}$, $reward_{ask\_suc}$, $reward_{ask\_fail}$, $reward_{reach\_max\_turn}$. The value setting of reward function is shown in section \ref{sec:section 5.2.3}.
\section{Methodology}
% \FloatBarrier
\subsection{Overview}
\begin{figure*}[htb]%[!t]
\centering
\includegraphics[width=7in]{pictures/framework.pdf}
% \includegraphics{pictures/framework.png}
\caption{KGenSam-based CRS framework overview. The key design is two sampler modules: (1) Negative Sampler outputs high-quality negative item samples and constructs positive and negative sample pairs to the effective update of the recommender at each turn of conversation; (2) Active Sampler outputs fuzzy item attribute samples with high uncertainty as the user preference information to be asked if CRS takes the $ask$ action.}
\label{fig:framework}
\end{figure*}
Figure \ref{fig:framework} illustrates the architecture of the proposed KGenSam, a general solution framework for knowledge-enhanced conversational recommendation. The middle subgraph in Figure \ref{fig:framework} shows the main body of a typical CRS. The workflow of the KGenSam CRS framework is as follows:
(1) Firstly, we construct the KG of user interaction data, and integrate the external KG to enrich the context information of interaction environment, and use knowledge to assist CRS interaction agent to make the decisions of actions.
(2) On the basis of the KG environment, we use the KG sampling methods\cite{2020HuGPA,2020WangKGPolicy} to enhance the knowledge and design two samplers based on reinforcement learning for CRS. Two samplers sample fuzzy samples and negative samples in KG respectively. Specifically, first, the \textbf{Active Sampler} focuses on the item attribute sample nodes representing user preferences in the KG, outputs fuzzy samples with large uncertainty through active reinforcement learning, and uses fuzzy samples for $ask$ action of CRS, so as to effectively improve the confidence of the model and improve the interaction efficiency of the system; second, the \textbf{Negative Sampler} focuses on the item sample nodes in the KG, outputs high-quality negative samples through reinforcement learning, supplements sparse online user data with negative samples, and updates the recommender with positive and negative sample pairs, so as to improve the update efficiency of the recommendation model. The two samplers work together to achieve the efficient acquisition of user preferences and reduce the interaction rounds, which reduce the user interaction burden.
The details of these two samplers modules will be discussed in section \ref{sec:section 4.2} and section \ref{sec:section 4.3} respectively.
(3) Finally, with the help of two samplers, the interactive policy network module learns the conversational strategy based on the enhanced knowledge, outputs the action ($ask$ or $recommend$) to be taken by the system, which guides the conversational direction, and realizes the acquisition of online user preferences and the successful recommendation of items that fit the user's current preferences at the last turn.
\subsection{Active Sampler via Active Reinforced Learning on KG}
\label{sec:section 4.2}
%ALdemopic
\begin{figure}[htb]%[!t]
\centering
\includegraphics[width=3.5in]{pictures/ALdemopic.pdf}
\caption{Illustrations of the item attribute prediction score distribution of user preference output by the recommender. (a) When the recommender is not fully trained, the recommender's score on the item attribute is mostly 0.5, meaning that the user's preference for the attribute is uncertain. (b) When the recommender is fully trained, the prediction scores of item attributes are mostly close to 1.0 or 0.0, and the distribution curve of the sample is similar to $U$, and different users have different $U$, such as (b1) or (b2).}
\label{fig:ALdemopic}
\end{figure}
As discussed before, one of the problems in CRS research works is that "What attributes to ask ?", of which the specific research content is to help the recommender fit the current user's preferences efficiently. Corresponding to tackling this problem, the task of the Active Sampler is to choose fuzzy samples of item attributes to ask so as to shorten the conversation, as shown in Figure \ref{fig:ALdemopic}. In order to define the active learning process more easily, we regard the recommender as a two-classifier. For each sample, the recommender predicts whether belongs to user preferences, and the output result is between 0 and 1. Further, we cast active sampling as a Markov Decision Process (MDP) and design a policy network to learn the optimal $ask$ strategy.
\subsubsection{MDP of Active Sampling in KG}
In KG environment, there are 33 attributes in LastFM, 29 first-level attributes and 590 second-level attributes in Yelp. Considering that there are not many item attribute nodes, we put all attribute nodes that are not labeled by users into the candidate $\boldsymbol{Pool_{p^*}}$ (it should be noted here that it may be a better method to collect the attribute nodes with one or two hops of the current user node as the candidate pool for the dataset with too many attributes, which is worth trying in the future). Intuitively, given the condition of the current $state$ in KG, the Active Sampler takes an action by selecting the next fuzzy attribute node to $ask$. It is then rewarded by the performance gain of the FM trained with the updated set of labeled nodes. Formally, the key elements of MDP (MDP = \{$S, A, P, R$\}) are defined as follows.
\textbf{State.} The $\boldsymbol{state}$ vector is a concatenation of four component vectors that encode environmental information from different perspectives:
\begin{equation}
\begin{split}
state=&{state}_{entropy}\oplus{state}_{degree}\oplus{state}_{KL}\\
&\oplus{state}_{selected}\oplus{state}_{adj}
\end{split}
\end{equation}
% \begin{itemize}[leftmargin=*]
\begin{itemize}
\item $\bm{{State}_{entropy}.}$ Information theory often uses entropy to measure the degree of chaos and uncertainty of a system. The larger the entropy is, the more chaotic and uncertain the information of the system is. On the contrary, the smaller the entropy is, the more stable and deterministic the system is. In this binary classification scenario, the entropy of candidate samples is an important basis for the Active Sampler to sample. Therefore, we calculate the entropy of the candidate attribute sample nodes to calculate the uncertainty of current environmental information, which is defined as follows :
\begin{equation}
-(\hat{y}_{\boldsymbol{Pool_{p^*}}}\log{\hat{y}_{\boldsymbol{Pool}}} +(1 - \hat{y}_{\boldsymbol{Pool_{p^*}}})\log{(1 - \hat{y}_{\boldsymbol{Pool_{p^*}}})})
\end{equation}
\item $\bm{{State}_{degree}.}$When calibrating an important node in the network, the relevant graph node centrality dimension is often used to define and measure the importance of the node\cite{2006KissEvaluatingCentrality}. Consequently, we use degree centrality, the most common graph node centrality dimension, to measure the importance of attribute sample nodes in KG. The definition is as follows :
\begin{equation}
\min{(degree(\boldsymbol{Pool_{p^*}})/\gamma,1)}
\end{equation}
where the hyperparameter $\gamma$ (set to 20) scale node degree and clip ${State}_{degree}$ to 1.
\item $\bm{{State}_{KL}.}$ Kullback–Leibler (KL) divergence can measure the "distance" (asymmetric) between two probabilities, so KL divergence can be used to calculate those data samples with large deviation\cite{1990HurvichKL,2005SmithKL}. We compute the average KL divergence of the sample predicted label distribution between the attribute sample node and its neighbors , which can help the Active Sampler better identify potential clusters and decision boundaries in KG. The ${State}_{KL}$ measures local similarity as:
\begin{equation}
\begin{split}
\frac{1}{\#neighbors}&
(D_{KL}(\hat{y}_{\boldsymbol{Pool_{p^*}}}\|\hat{y}_{\boldsymbol{neighbors}})+\\
& D_{KL}(\hat{y}_{\boldsymbol{neighbors}}\|\hat{y}_{\boldsymbol{Pool_{p^*}}}))
\end{split}
\end{equation}
\item $\bm{{State}_{selected}.}$ For $p\in \boldsymbol{Pool_{p^*}}$, if $p$ has been asked and responded, $\boldsymbol{{state}_{selected}}(p) = 1$; if $p$ has not been chosen yet, $\boldsymbol{{state}_{selected}}(p) = 0$.
\item $\bm{{State}_{adj}.}$ In addition to the content information of the candidate attribute nodes, we also take the structural information of the current subgraph in KG as part of the $\boldsymbol{state}$. We construct the adjacency list dictionary of $\boldsymbol{Pool_{p^*}}$ as $\boldsymbol{{state}_{adj}}$ to take advantage of the KG structure.
\end{itemize}
\textbf{Action. }The space of actions is $\boldsymbol{Pool_{p^*}}$, and Active Sampler chooses an action $a^t$ from $\boldsymbol{Pool_{p^*}}$ based on the output results of active policy network, $\pi{(\cdot |state^{t-1})}$. As the state is changing during the interaction, the action space $\boldsymbol{Pool_{p^*}}$ is dynamic.
\textbf{Reward. } The $reward$ measures the quality of the $ask$ attribute sample at each turn. However, there is no ground truth in the selection of the best fuzzy samples in a active learning model, $reward$ rely on the feedback from the recommender. Consequently, we use the increase of the FM model's performance score from $t$ to $t+1$ turn to define the soft reward function and design the $reward$ as:
\begin{equation}
reward^t=Gain\_Score(\hat{y}_{valid}^{t+1},\hat{y}_{valid}^{t})
\end{equation}
where $\hat{y}_{valid}^{t}$ is the evaluation score of the FM model at turn $t$ and $Gain\_Score$ calculates the increase of the score, where we use AUC as the evaluation metric.
\textbf{State Transition Dynamic. }Given an action $a^t$ at state $state^t$ , the transition to the next state $state^{t+1}$ is determined as:
\begin{equation}
P(state^{t+1}=(u,a^t)|state^t,a^t)=1
\end{equation}
\textbf{Objective Function. }Towards learning a stochastic policy $\pi$, we maximize the expected cumulative discounted reward. The objective function with respect to the Active Sampler parameters $\Theta_{AS}$ is as follows:
\begin{equation}
\max_{\Theta_{AS}} \sum_{{(u,p^+,p^-)}\in{O_A}} \mathbbm{E}_{\pi}[\sum_{t=1}^T \lambda_{AS}^{t-1}reward^t]
\end{equation}
where $\lambda_{AS}$ is the decay factor.
\subsubsection{Active Policy Network}
% \textbf{Architecture}
As the right subgraph in Figure \ref{fig:framework} shows, to effectively propagate useful information over the KG and thereby better measure node informativeness, we first parameterize the policy network $\pi$ as a one-layer Graph Convolutional Network (GCN)\cite{2016gcn}. Then We use ReLU as the activation function $\sigma$. On top of the GCN, we apply a linear layer to output the final output embedding and get the score for each node. These scores are then normalized by a softmax layer to generate a probability distribution over all candidate attributes $p$ in $\boldsymbol{Pool_{p^*}}$.
% \textbf{Training Algorithm}
By maximizing its long-term returns with policy gradient\cite{1999SuttonPolicyGradient}, active policy network can effectively learn to optimize the long-term performance of the recommender in an end-to-end fashion.
%算法流程图参考
%https://my.oschina.net/u/4318809/blog/4411390
\subsection{Negative Sampler via Reinforcement Learning on KG}
\label{sec:section 4.3}
After solving the problem that CRS asks what attributes to make each opportunity of $ask$ the most efficient, the next problem naturally arises, that is, how can the feedback signal of users for fuzzy samples be accurately learned by the recommender ? In previous CRS work, all unobserved interactions are regarded as negative samples, so it is easy to produce low-quality negative samples, and it is difficult to update the model parameters of the recommender. Considering that CRS has very few session turns, the update of recommender need to be accurate and rapid.
In recommendation systems, users often directly ignore the uninterested items, which will not produce users' explicit negative feedback behavior and leads to the common One-Class Problem of recommender training data. To solve the negative sample missing problem caused by One-Class data, traditional recommendation systems often construct effective positive and negative sample pairs by negative sampling to learn user preferences. Drawing on the same idea, we design a Negative Sampler to help the recommender learn preference information as efficiently and timely as the Active Sampler's acquisition of user feedback information. The collected high-quality negative samples supplement the rare online negative feedback and user history positive samples. Similar to the active sampling, we cast negative sampling as a MDP and design a policy network to learn the features of high quality negative samples in conversational recommendation.
\subsubsection{MDP of Negative Sampling in KG}
%$M$ = \{$S, A, P, R$\}
In KG environment, different from the Active Sampler, the Negative Sampler needs to focus on the item nodes, so the candidate set to been sampled is very large. Since the Negative Sampler requires labor-intensive negative discovering and is memory-consuming and time-consuming to distill useful signals in large-scale KG. Thus, we design the Negative Sampler to navigate from a positive item over the KG structure like $i^+\to p \to i$ and yield two-hops neighbors as candidate negative items . Conditioned on the navigated local subgraph $state$, the Negative Sampler takes an action by selecting a batch of possible negative samples from this two-hops candidate items and construct training pairs with ground-truth positive items. Similar to the reward design of Active Sampler, the reward of Negative Sampler is the performance score of the FM trained with the constructed positive and negative pairs. Formally, the key elements of MDP (MDP = \{$S, A, P, R$\}) are defined as follows.
\textbf{State. }Given user $u$, the $state^t$ at turn $t$ is defined as subgraph paths $(u\to \cdots \to i^t)$, where $i^t$ is the node the Negative Sampler visits currently.
\textbf{Action. }The space of actions is $\boldsymbol{Pool_{i^-}}$, which is the candidate sets of two hop neighbors.
\textbf{Reward. }The design goal of reward is to measure the quality of negative samples. Similar to the concept of hard negative in the field of Image Detection, we need to find the discriminative negative samples which are easily misjudged by the recommender in the CRS task. Inspired by the negative sampling model (KGPolicy) proposed by Wang et al.\cite{2020WangKGPolicy}, we apply the hypothesis and definition of "Real Negative Sample" to the negative sampling task scenario of CRS. The definition of high-quality negative sample has two characteristics: it is similar to the user representation\cite{park2018Adversarial,wang2017IRGAN}; it is close to the real positive sample. The negative samples with these two characteristics can represent the real negative samples and provide a larger gradient for the parameter update of the recommender. From this point of view, we define the reward of negative sampler as follows.
\begin{equation}
\begin{split}
reward^t=(\boldsymbol{{Pool_{i^-}^t}})\mathsf{T} \cdot \boldsymbol{u}+(\boldsymbol{Pool_{i^-}^t})\mathsf{T} \cdot \boldsymbol{i^+}
\end{split}
\end{equation}
\textbf{State Transition Dynamic. }Given an action $a^t$ at state $state^t$ , the transition to the next state $state^{t+1}$ is determined as:
\begin{equation}
P(state^{t+1}=(u,a^t)|state^t,a^t)=1
\end{equation}
\textbf{Objective Function. }To maximize the sum of expected rewards obtained from following policy $\pi$ over the training graphs, the objective function with respect to the negative policy network parameters $\Theta_{NS}$ is:
\begin{equation}
\max_{\Theta_{NS}} \sum_{{(u,i^+,i^-)}\in{O_I}} \mathbbm{E}_{\pi}[\sum_{t=1}^T \lambda_{NS}^{t-1}reward^t]
\end{equation}
where $\lambda_{NS}$ is the decay factor.
\subsubsection{Negative Policy Network}
% \textbf{Architecture}
As shown in the left side of Figure \ref{fig:framework}, the Negative Sampler first obtains the initial representation of the nodes through one layer of GCN, which captures the knowledge-aware negative signals, and then output a batch of negative item nodes with high weight through the attention model constructed by two-layer activation function.
% \textbf{Training Algorithm}
%算法流程图参考
%https://my.oschina.net/u/4318809/blog/4411390
\section{Experiments and Analysis}
\subsection{Experimental Setup}
\subsubsection{Datasets}
As summarized in Table \ref{table:datasets}, we use two publicly available datasets: LastFM and Yelp. Each dataset is composed of two components, the user-item interactions and KG derived from external data sources, Freebase for LastFM and local business information network for Yelp. The cut ratio of training, validation and testing sets is 7: 1: 2. The yelp dataset we use here is slightly different from EAR\cite{lei2020estimation} and CPR, using 590 second-level attributes instead of 29 first-level attributes. Thus, for real application scenarios, LastFM represents the dataset with fewer attributes, Yelp represents the dataset with more attributes.
\begin{table}[!t]
\caption{Dataset Statictics of LastFM and Yelp}
\label{table:datasets}
\centering
% Some packages, such as MDW tools, offer better commands for making tables
% than the plain LaTeX2e tabular which is used here.
\begin{tabular}{c|c|c|c}
\toprule[1pt]
& &\textbf{LastFM} &\textbf{Yelp} \\ \midrule[0.5pt]
\multirow{4}*{User-Item Interaction}
~& \#Users & 1,801 &27,675\\
~& \#Items & 7,432 &70,311\\
~& \#attributes & \textbf{33} &\textbf{590}\\
~& \#Interactions & 76,693 &1,368,606\\
\hline
\multirow{2}*{Knowledge Graph}
~& \#Entities & 9,266 &98,576\\
~& \#Triplets & 138,217 &2,533,827\\
\bottomrule[1pt]
\end{tabular}
\end{table}
\subsubsection{Parameter Setting}
The maximum turn $T$ is set as 15.
All hyperparameters for offline training of FM remains the same as EAR\cite{lei2020estimation}. The detailed rewards and the hyperparameters of interact policy net remain the same as CPR\cite{lei2020interactive}. All the hyperparameters related samplers training are tuned according to the validation set.
\subsubsection{Training Steps}
The training process is made up of three stages.
\textbf{Stage 1:} Pretraining of recommender for scoring items and attributes. Strictly following\cite{lei2020estimation}, we use the training set of $O_I$ to optimize FM offline. We refer the readers to the original paper \cite{lei2020estimation} for more Information.
\textbf{Stage 2:} Pretraining of two samplers. The pretrained FM is used to provide reward scores for two sampling networks. The detailed pretraining process of samplers are shown in Algorithm 1 and 2 respectively.
\textbf{Stage 3:} Online training of interact network. We use a user simulator to interact with the user to train the policy network. The interaction algorithm flow of CRS based on our KGenSam is shown in Algorithm 3.
\begin{algorithm}[ht]
\caption{Pretrain the Active Sampler}
\begin{algorithmic}
\REQUIRE $O_A$, FM
\ENSURE Active Sampler Policy $\pi$
\FOR{$(u,p^+,p^-)\in O_A$}
\STATE $Pool_{p^*} \leftarrow A$
\FOR{$t=1$ to $T$}
\STATE select a batch ranked $p^*_t$ from $Pool_{p^*}$
\IF{$p^*_t$ in $A^+_u$}
\STATE $p^+_u \leftarrow p^*_t$
\STATE update FM based on $p^+_u$
\ELSE
\STATE $p^-_u \leftarrow p^*_t$
\STATE update FM based on $p^-_u$
\ENDIF
\STATE remove $p^*_{t}$ from $Pool_{p^*}$
\ENDFOR
\STATE evaluate FM on the validation set to get the reward signal $\sum_{t=1}^T reward^t$
\STATE update $\pi$ with policy gradient
\ENDFOR
\end{algorithmic}
\end{algorithm}
\begin{algorithm}[ht]
\caption{Pretrain the Negative Sampler}
\begin{algorithmic}
\REQUIRE $O_I$, FM
\ENSURE Negative Sampler Policy $\pi$
\FOR{$(u,i^+,i^-)\in O_I$}
\STATE $ Batch_{i^-} \leftarrow \{\}, Pool_{i^-} \leftarrow \{\}, i_s \leftarrow i^+ $
\FOR{$s=1$ to $steps$}
\STATE get neighbors $Neighbors_{i_s}$ of $i_s$
\STATE traverse $Neighbors_{i_s}$ as $i_s$
\FOR{each $i_{neighbor} \in Neighbors_{i_s}$}
\IF{$i_{neighbor} \notin I^+$}
\STATE add $i_{neighbor}$ to $Pool_{i^-}$
\ENDIF
\ENDFOR
\STATE select a batch ranked $Batch_{i^-}$ from $Pool_{i^-}$
\STATE update FM based on $Batch_{i^-}$ and $i^+$
\STATE evaluate FM on the validation set to get the reward signal $\sum_{t=1}^B reward^t$
\STATE update $\pi$ with policy gradient
\ENDFOR
\ENDFOR
\end{algorithmic}
\end{algorithm}
\begin{algorithm}[ht]
\caption{KGenSam-based CRS Algorithm}
\begin{algorithmic}
\REQUIRE $O_I$, $O_A$, FM, Active Sampler, Negative Sampler
\ENSURE Conversation Recommendation Policy $\pi$
\FOR{$(u,i^+,i^-)\in O_I, (u,p^+,p^-)\in O_A $}
\STATE $P^+_u \leftarrow \{p^+_0\}, P^-_u \leftarrow \{\}, Pool_{p^*} \leftarrow A$
\STATE $Pool_{i^-} \leftarrow Neighbors_{i^+}-I^+ $
\FOR{$t=1$ to $T$}
\STATE select an action $a_t$ from $\{ask,rec\}$
\STATE Negative Sampler selects negative batch $Batch_{i^-}$
\IF{$a_t = ask$}
\STATE remove $p^+_{t-1}$ from $Pool_{p^*}$
\STATE Active Sampler selects a $p^*_t$ from $Pool_{p^*}$
\IF{user $u$ accepts $p^*_t$}
\STATE add $p^*_t$ to $P^+_u$
\ELSE
\STATE add $p^*_t$ to $P^-_u$
\ENDIF
\ELSE
\STATE FM outputs $i^+_t$ to recommend
\IF{user $u$ accept $i^+_t$}
\STATE achieve successful recommendation and quit!
\ELSE
\STATE add $i^+_t$ to $Batch_{i^-}$
\ENDIF
\ENDIF
\STATE update FM based on $P^+_u$, $P^-_u$ and $Batch_{i^-}$
\ENDFOR
\STATE reach the maximum turns and quit!
\ENDFOR
\end{algorithmic}
\end{algorithm}
\subsubsection{Comparison Methods}
We compare our method against the following baseline methods:
\begin{itemize}
\item Abs Greedy\cite{Christakopoulou2016TCRS}.This method serves as a baseline where the model only recommends items at each turn and updates itself until it finally makes successful recommendation.
\item Max Entropy. This method is based on the maximum entropy rule. It always chooses an attribute with the maximum entropy within the current candidate item set\cite{Dhingra2016End} to ask. Details can be found at \cite{lei2020estimation}.
\item CRM\cite{sun2018conversational}. This is a CRS model which is originally designed for single-round recommendation. CRM records user’s preference into a belief tracker, and uses reinforcement learning to learn the policy of interaction. We follow \cite{lei2020estimation} to adapt it to the multi-round scenario.
\item EAR \cite{lei2020estimation}. This is the state-of-the-art method which proposed a three stage solution, Estimation Action Reflection.
\item SCPR \cite{lei2020interactive}. This is the simple implementation model of the state-of-the-art work, CPR, which introduces KG in CRS for the first time.
\item FPAN \cite{xv2021FPAN}. This is the latest CRS research work. It pays attention to the correction effect of user feedback on recommendation and adjusts the node representation of KG through the Gate Mechanism, which significantly improves the CRS recommendation results hence being the most comparable model.
\end{itemize}
\subsubsection{Evaluation Metrics}
The evaluation follows \cite{lei2020estimation}. We use the success rate at the 15th turn of conversation (SR@15) to measure the cumulative ratio of successful recommendation by turn 15 . We also use and average average turns (AT) of conversation length when the interaction process ends to record the average number of turns for a successful recommendation. Therefore, the higher SR@15 and the lower AT indicate a higher performance and overall higher efficiency.
\textbf{Success Rate@15.}
\begin{equation}
SR@15=\frac{1}{\#users \times T }\sum_{u\in U}\sum_{t=1}^T \theta_{hit}
\end{equation}
where $\theta_{hit} = 1$, if the instinctive feedback of the recommended item given by the simulator is acceptance.
\textbf{Average Turns.}
\begin{equation}
AT=\frac{1}{\#users} \sum_{u\in U}{T_{ends}}
\end{equation}
where $T_{ends}$ is the total turns when the current conversation session ends.
\subsection{Results and Discussion}
\subsubsection{Overall Performance Comparison}
\begin{table}
\caption{Success Rate@15 and Average Turns}
\label{table:compareall}
\centering
\begin{tabular}{c|c|c|c|c}
\hline
~ & \multicolumn{2}{c|}{\textbf{LastFM}} &\multicolumn{2}{c}{\textbf{Yelp}}\\
\hline
~ & \textbf{SR@15} & \textbf{AT} & \textbf{SR@15} & \textbf{AT}\\
\hline
Abs Greedy & 0.222 & 13.48 & 0.189 & 13.43\\
Max Entropy & 0.283 & 13.91 & 0.398 & 13.42\\
CRM & 0.325 & 13.75 & 0.177 & 13.69\\
EAR & 0.429 & 12.88 & 0.182 & 13.63\\
SCPR & 0.465 & 12.86 & 0.489 & 12.62\\
FPAN & 0.667 & 10.14 & 0.588 & 12.65\\
\hline
\textbf{KGenSam} & \textbf{0.948} & \textbf{6.43} & \textbf{0.810} & \textbf{7.72}\\
\hline
improve & 42.0\%↑ & 36.6\%↑ & 37.8\%↑ & 38.8\%↑\\
\hline
\end{tabular}
\end{table}
% experiment-total-compare.pdf
\begin{figure*}[htb]%[!t]
\centering
\includegraphics[width=7in]{pictures/compare-all1.pdf}
\includegraphics[width=7in]{pictures/compare-all2.pdf}
% \includesvg[width=7in]{compare-all1}
% \includesvg[width=7in]{compare-all2}
\caption{SR of compared methods at different turns on LastFM and Yelp.}
\label{fig:compareall}
\end{figure*}
The results are summarized in Table \ref{table:compareall}. Overall, it can be seen that the proposed KGenSam architecture outperforms other baselines in both terms of SR@15 and AT, which validates our hypothesis that enhancing the information modeling of interactive environment is an effective strategy to improve the performance of CRS.
By intuitively presenting the performance comparison in Figure \ref{fig:compareall}, we also have following discoveries:
\begin{itemize}
\item For Yelp dataset which has large attribute space, the performance of all baseline methods is not satisfactory, CRM and EAR models can not even achieve the performance of Max Entropy method.
\item From the improvement trend of success rate in the whole conversation, we can clearly find that the first six rounds of interaction of the baseline methods are in a "Chilling Period", during which the success rate of recommendation is very low, and after the sixth round, the success rate of CRS slowly increases. However, KGenSam does not have such a "Chilling Period", which can improve the success rate after each round of interaction, and the improvement of the early interaction is even greater than that of the later interaction, which is consistent with the trend of the accuracy improvement curve of Active Learning in the classification model annotation task. This consistency verifies our hypothesis that the idea of active learning can effectively integrate with CRS task.
\end{itemize}
\subsubsection{Evaluating Key Designs in KGenSam}
\begin{table*}[!ht]
\caption{Performance of removing Samplers from KGenSam}
\label{table:ablation}
\centering
\resizebox{\textwidth}{13mm}{
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c|c|c}
\hline
~ & \multicolumn{6}{c|}{\textbf{LastFM}} &\multicolumn{6}{c}{\textbf{Yelp}}\\
\hline
~ & \textbf{SR@3} & \textbf{SR@6} & \textbf{SR@9} & \textbf{SR@12} & \textbf{SR@15} & \textbf{AT} & \textbf{SR@3} & \textbf{SR@6} & \textbf{SR@9} & \textbf{SR@12} & \textbf{SR@15} & \textbf{AT}\\
\hline
w/o Active Sampler & 0.000 & 0.085 & 0.324 & 0.561 & 0.7225 (23.8\%↓) & 9.18 &
0.000 & 0.036 & 0.143 & 0.247 & 0.309 (61.9\%↓) & 12.94\\
w/o Negative Sampler & 0.157 & 0.355 & 0.491 & 0.602 & 0.673 (29.0\%↓) & 10.55 &
0.135 & 0.288 & 0.417 & 0.514 & 0.572 (29.4\%↓) & 10.23\\
w/o Samplers & 0.000 & 0.051 & 0.204 & 0.353 & 0.455 (52.0\%↓) & 12.65 &
0.000 & 0.000 & 0.023 & 0.090 & 0.195 (75.9\%↓) & 12.59\\
\hline
\textbf{KGenSam} & \textbf{0.217} & \textbf{0.493} & \textbf{0.690} & \textbf{0.830} & \textbf{0.948} & \textbf{6.43} &
\textbf{0.184} & \textbf{0.394} & \textbf{0.580} & \textbf{0.714} & \textbf{0.810} & \textbf{7.72}\\
\hline
\end{tabular}}
\end{table*}
% experiment-ablation-compare.pdf
\begin{figure*}[htb]%[!t]
\centering
\includegraphics[width=7in]{pictures/ablation1.pdf}
\includegraphics[width=7in]{pictures/ablation2.pdf}
% \includesvg[width=7in]{ablation1}
% \includesvg[width=7in]{ablation2}
\caption{Success rate of different ablation setting KGenSam on LastFM and Yelp.}
\label{fig:ablation}
\end{figure*}
The key designs of KGenSam are the two sampler modules. In order to evaluate whether the two samplers can meet our expectations, that is, to accurately select the asked attribute samples and efficiently update the recommender, we designed the following ablation experiments.
\textbf{(1) w/o Active Sampler :} We select attribute samples with the maximum scores (calculated by propagating messages like CPR\cite{lei2020interactive}) to ask.
\textbf{(2) w/o Negative Sampler :} We randomly select item samples without interaction as negative samples to update the recommender.
\textbf{(3) w/o Samplers :} Neither of the two sampling modules works. We only use KG as the environment, and the selection methods of asked attributes and negative items are consistent with (1) and (2).
Table \ref{table:ablation} shows the success rate of every three turns and the average rounds of a session under the above three ablation experimental settings. It can be seen that when two samplers work at the same time, the performance of the complete KGenSam is the best.
In addition, we have the following conclusions based on the success rate trend in Figure \ref{fig:ablation}:
\begin{itemize}
\item The Active Sampler changes the shape of the success rate curve and makes it convex upward, indicating that the Active Sampler changes the magnitude of user preference information acquisition per turn. As indicated in Figure \ref{fig:ablation}, the Active Sampler can greatly improve the performance of CRS in the first six rounds of the dialogue, which means it can help CRS solve the problem of "Chilling Period" (the phenomenon mentioned in the previous section). Figure \ref{fig:ablation} illustrates that the fuzzy attribute samples output by the Active Sampler are indeed the most uncertain user preferences in the current system, which means the attribute knowledge enhanced by the Active Learning idea can help CRS ask the user preference information accurately.
\item As shown in Figure \ref{fig:ablation}, from the blue line to the black line and from the orange line to the red line, we can see that the Negative Sampler makes the success rate curve move upward as a whole, which indicates that the Negative Sampler improves the learning speed of user preference information, and makes the success rate rise to a relatively high level earlier.
\item From Figure \ref{fig:ablation}, we can also notice that KGenSam with Active and Negative Samplers removed separately behaves differently on the two datasets. For the LastFm with fewer attributes, the effect of removing the active sampler is less than that of removing the Negative Sampler; while for Yelp with more attributes, the opposite is true. Through the analysis of this result, we believe that the Active Sampler can play a greater role in the complex data environment because it focuses on the selection of attribute nodes. Thus, the Active Sampler has a greater impact on Yelp which has a large number of attributes.
\item Table \ref{table:ablation} and Figure \ref{fig:ablation} verify that the Active Samplers are responsible for obtaining a large amount of user preference information, and the Negative Samplers are responsible for learning the acquired information. Two key designs achieve a high-performance CRS by jointly enhancing the environmental information in KG.
\end{itemize}
\subsubsection{Performance under Different Reward}
\label{sec:section 5.2.3}
\begin{table}
\caption{Reward Settings of Interact Policy Network in CRS}
\label{table:reward}
\centering
\setlength{\tabcolsep}{1mm}{
\begin{tabular}{c|c|c|c|c}
\hline
& $\boldsymbol{R_{CPR}}$ & $\boldsymbol{R_{ask\_more}}$ & $\boldsymbol{R_{rec\_more}}$ & $\boldsymbol{R_{EAR}}$\\
\hline
$reward_{ask\_suc}$ & 0.01 & 0.1 & 0.01 & 0.01+0.1\\
$reward_{ask\_fail}$ & -0.1 & -0.1 & -0.1 & 0.01+0\\
$reward_{rec\_suc}$ & 1 & 1 & 1 & 0.01+1\\
$reward_{rec\_fail}$ & -0.1 & -1 & -0.01 & 0.01+0\\
$reward_{reach\_max\_turn}$ & -0.3 & -0.3 & -0.3 & -0.3\\
\hline
\end{tabular}}
\end{table}
\begin{figure}[htb]%[!t]
\centering
\includegraphics[width=3in]{pictures/lastfm_Rec_Ratio.pdf}
% \includesvg[width=3in]{lastfm_Rec_Ratio}
\caption{Rec Ratios of KGenSam-based CRS trained with different reward settings in LastFM.}
\label{fig:recratio}
\end{figure}
In the process of experiment, we found a phenomenon worthy of attention, that is, the reward setting of interact policy network has a great impact on the final evaluation result of CRS, which may be due to the current evaluation metrics of CRS is not comprehensive enough. For example, the value settings of reward in the two representative CRS works, EAR and CPR, are totally different. We use the same reward setting of CPR which is also based on KG as the default reward setting in our KGenSam framework. In order to further verify the effect of reward setting in CRS, we design several kinds of reward settings as shown in Table \ref{table:reward}, in which $reward_{ask\_suc}$, $reward_{ask\_fail}$, $reward_{rec\_suc}$, $reward_{rec\_fail}$ and $reward_{reach\_max\_turn}$ represent "ask attribute succeed", "ask attribute fail", "recommend succeed", "recommend fail" and "reach maximum turn" respectively. $R_{CPR}$(default setting) and $R_{EAR}$ are the reward settings of CPR and EAR. In addition, we increase $reward_{ask\_suc}$ and decrease $reward_{rec\_fail}$ to set $R_{ask\_more}$ which can tend to $ask$, in the same way, we set $R_{rec\_more}$ to incline $recommend$.
It can be seen from Figure \ref{fig:recratio} that reward setting can affect the probability of taking $recommend$ action. The reward setting of CPR makes the rec ratio of each round extremely high, and the final performance on both SR@T an AT is also the best, which verifies our previous conjecture. We boldly draw the conclusion that rec ratio is positively correlated with the two evaluation indicators, which needs to be verified by more research in the future.
\section{Conclusion}
In this paper, we provide a new idea for the research of CRS, that is to solve the problem of insufficient environmental information modeling in CRS from the perspective of knowledge enhancement. The experimental results show that this perspective is reliable and effective. We construct an Active Sampler to focus on the fuzzy attribute nodes in the graph to maximize the amount of information obtained by each $ask$ turn. At the same time, the Negative Sampler we constructed focus on the high-quality negative samples in KG, so as to maximize the learning speed of each recommender update. As far as we know, we are the first to define the idea of Active Learning into the CRS problem and achieve the experimental results overcoming "Chilling Period" problem. This idea of Active Learning to screen fuzzy samples in KG can also be applied to the cold start problem of KG based recommendation system. In addition, we also introduce hard negative Samples into CRS to achieve efficient CRS online update and promote the commercial online application of CRS.
In the future, we will try to introduce more evaluation metrics to solve the incomplete evaluation problem in CRS research. In addition, the current CRS research is based on single session (including single-round and multi-rounds). Our KGenSam method has achieved high evaluation results in the single session scenario where the conversation ends after only one successful recommendation, but we believe that single-session CRS is obviously not in line with the real recommendation scenario. Therefore, from the perspective of sequential recommendation modeling, we will study multi-session modeling of CRS
to extend our KGenSam framework, which may improve the possibility of CRS being widely used in business scenarios.
% if have a single appendix:
%\appendix[Proof of the Zonklar Equations]
% or
%\appendix % for no appendix heading
% do not use \section anymore after \appendix, only \section*
% is possibly needed
% use appendices with more than one appendix
% then use \section to start each appendix
% you must declare a \section before using any
% \subsection or using \label (\appendices by itself
% starts a section numbered zero.)
%
% \appendices
% \section{Details of pretraining in KGenSam}
% Appendix one text goes here.
% % you can choose not to have a title for an appendix
% % if you want by leaving the argument blank
% \section{Additional experimental results}
% Appendix two text goes here.
% use section* for acknowledgment
\ifCLASSOPTIONcompsoc
% The Computer Society usually uses the plural form
\section*{Acknowledgments}
\else
% regular IEEE prefers the singular form
\section*{Acknowledgment}
\fi
This work is supported by the National Key R\&D Program of China (2018AAA0100604), the Fundamental Research Funds for the Central Universities (2021RC217), the Beijing Natural Science Foundation (JQ20023), the National Natural Science Foundation of China (61632002, 61832004, 62036012, 61720106006).
% Can use something like this to put references on a page
% by themselves when using endfloat and the captionsoff option.
\ifCLASSOPTIONcaptionsoff
\newpage
\fi
% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}
% references section
% can use a bibliography generated by BibTeX as a .bbl file
% BibTeX documentation can be easily obtained at:
% http://mirror.ctan.org/biblio/bibtex/contrib/doc/
% The IEEEtran BibTeX style support page is at:
% http://www.michaelshell.org/tex/ieeetran/bibtex/
\bibliographystyle{IEEEtran}
% argument is your BibTeX string definitions and bibliography database(s)
\bibliography{references}
%
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
% biography section
%
% If you have an EPS/PDF photo (graphicx package needed) extra braces are
% needed around the contents of the optional argument to biography to prevent
% the LaTeX parser from getting confused when it sees the complicated
% \includegraphics command within an optional argument. (You could create
% your own custom macro containing the \includegraphics command to make things
% simpler here.)
%\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{mshell}}]{Michael Shell}
% or if you just want to reserve a space for a photo:
% \begin{IEEEbiography}{Michael Shell}
% Biography text here.
% \end{IEEEbiography}
% % if you will not have a photo at all:
% \begin{IEEEbiographynophoto}{John Doe}
% Biography text here.
% \end{IEEEbiographynophoto}
% insert where needed to balance the two columns on the last page with
% biographies
%\newpage
% \begin{IEEEbiographynophoto}{Jane Doe}
% Biography text here.
% \end{IEEEbiographynophoto}
% You can push biographies down or up by placing
% a \vfill before or after them. The appropriate
% use of \vfill depends on what kind of text is
% on the last page and whether or not the columns
% are being equalized.
%\vfill
% Can be used to pull up biographies so that the bottom of the last one
% is flush with the other column.
%\enlargethispage{-5in}
% that's all folks
\end{document}
% Generated by IEEEtran.bst, version: 1.14 (2015/08/26)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{2020WangDenoising}
\BIBentryALTinterwordspacing
W.~Wang, F.~Feng, X.~He, L.~Nie, and T.~Chua, ``Denoising implicit feedback for
recommendation,'' in \emph{{WSDM} '21, The Fourteenth {ACM} International
Conference on Web Search and Data Mining, Virtual Event, Israel, March 8-12,
2021}, L.~Lewin{-}Eytan, D.~Carmel, E.~Yom{-}Tov, E.~Agichtein, and
E.~Gabrilovich, Eds.\hskip 1em plus 0.5em minus 0.4em\relax {ACM}, 2021, pp.
373--381. [Online]. Available: \url{https://doi.org/10.1145/3437963.3441800}
\BIBentrySTDinterwordspacing
\bibitem{2020WangClicknotLike}
\BIBentryALTinterwordspacing
W.~Wang, F.~Feng, X.~He, H.~Zhang, and T.~Chua, ``"click" is not equal to
"like": Counterfactual recommendation for mitigating clickbait issue,''
\emph{CoRR}, vol. abs/2009.09945, 2020. [Online]. Available:
\url{https://arxiv.org/abs/2009.09945}
\BIBentrySTDinterwordspacing
\bibitem{2019LeeMeLU}
H.~Lee, J.~Im, S.~Jang, H.~Cho, and S.~Chung, ``Melu: Meta-learned user
preference estimator for cold-start recommendation,'' \emph{ACM}, 2019.
\bibitem{2020CenControllableMultiInterest}
\BIBentryALTinterwordspacing
Y.~Cen, J.~Zhang, X.~Zou, C.~Zhou, H.~Yang, and J.~Tang, ``Controllable
multi-interest framework for recommendation,'' in \emph{{KDD} '20: The 26th
{ACM} {SIGKDD} Conference on Knowledge Discovery and Data Mining, Virtual
Event, CA, USA, August 23-27, 2020}, R.~Gupta, Y.~Liu, J.~Tang, and B.~A.
Prakash, Eds.\hskip 1em plus 0.5em minus 0.4em\relax {ACM}, 2020, pp.
2942--2951. [Online]. Available:
\url{https://doi.org/10.1145/3394486.3403344}
\BIBentrySTDinterwordspacing
\bibitem{2019GaoBLOMA}
C.~Gao, S.~Yuan, Z.~Zhang, H.~Yin, and J.~Shao, \emph{BLOMA: Explain
Collaborative Filtering via Boosted Local Rank-One Matrix
Approximation}.\hskip 1em plus 0.5em minus 0.4em\relax Complexity in Polish
Phonotactics, 2019.
\bibitem{2019JagermanOffPolicyEvaluation}
R.~Jagerman, I.~Markov, and M.~Rijke, ``When people change their mind:
Off-policy evaluation in non-stationary recommendation environments,'' in
\emph{the Twelfth ACM International Conference}, 2019.
\bibitem{2013WangContingencyApproachInteractionModes}
W.~Wang and I.~Benbasat, ``{A Contingency Approach to Investigating the Effects
of User-System Interaction Modes of Online Decision Aids},''
\emph{{INFORMATION SYSTEMS RESEARCH}}, vol.~{24}, no.~{3}, pp. {861--876},
{SEP} {2013}.
\bibitem{lei2020estimation}
W.~Lei, X.~He, Y.~Miao, Q.~Wu, R.~Hong, M.-Y. Kan, and T.-S. Chua,
``Estimation-action-reflection: Towards deep interaction between
conversational and recommender systems,'' in \emph{Proceedings of the 13th
International Conference on Web Search and Data Mining}, 2020, pp. 304--312.
\bibitem{2012RendleBPR}
S.~Rendle, C.~Freudenthaler, Z.~Gantner, and L.~Schmidt-Thieme, ``Bpr: Bayesian
personalized ranking from implicit feedback,'' \emph{AUAI Press}, 2012.
\bibitem{2017HeFastMF}
\BIBentryALTinterwordspacing
X.~He, H.~Zhang, M.~Kan, and T.~Chua, ``Fast matrix factorization for online
recommendation with implicit feedback,'' in \emph{Proceedings of the 39th
International {ACM} {SIGIR} conference on Research and Development in
Information Retrieval, {SIGIR} 2016, Pisa, Italy, July 17-21, 2016},
R.~Perego, F.~Sebastiani, J.~A. Aslam, I.~Ruthven, and J.~Zobel, Eds.\hskip
1em plus 0.5em minus 0.4em\relax {ACM}, 2016, pp. 549--558. [Online].
Available: \url{https://doi.org/10.1145/2911451.2911489}
\BIBentrySTDinterwordspacing
\bibitem{2008PanOneClass}
\BIBentryALTinterwordspacing
R.~Pan, Y.~Zhou, B.~Cao, N.~N. Liu, R.~M. Lukose, M.~Scholz, and Q.~Yang,
``One-class collaborative filtering,'' in \emph{Proceedings of the 8th {IEEE}
International Conference on Data Mining {(ICDM} 2008), December 15-19, 2008,
Pisa, Italy}.\hskip 1em plus 0.5em minus 0.4em\relax {IEEE} Computer Society,
2008, pp. 502--511. [Online]. Available:
\url{https://doi.org/10.1109/ICDM.2008.16}
\BIBentrySTDinterwordspacing
\bibitem{zhang2018towards}
Y.~Zhang, X.~Chen, Q.~Ai, L.~Yang, and W.~B. Croft, ``Towards conversational
search and recommendation: System ask, user respond,'' in \emph{Proceedings
of the 27th acm international conference on information and knowledge
management}, 2018, pp. 177--186.
\bibitem{li2018towards}
R.~Li, S.~Kahou, H.~Schulz, V.~Michalski, L.~Charlin, and C.~Pal, ``Towards
deep conversational recommendations,'' \emph{arXiv preprint
arXiv:1812.07617}, 2018.
\bibitem{christakopoulou2018q}
K.~Christakopoulou, A.~Beutel, R.~Li, S.~Jain, and E.~H. Chi, ``Q\&r: A
two-stage approach toward interactive recommendation,'' in \emph{Proceedings
of the 24th ACM SIGKDD International Conference on Knowledge Discovery \&
Data Mining}, 2018, pp. 139--148.
\bibitem{sun2018conversational}
Y.~Sun and Y.~Zhang, ``Conversational recommender system,'' in \emph{The 41st
international acm sigir conference on research \& development in information
retrieval}, 2018, pp. 235--244.
\bibitem{lei2020interactive}
W.~Lei, G.~Zhang, X.~He, Y.~Miao, X.~Wang, L.~Chen, and T.-S. Chua,
``Interactive path reasoning on graph for conversational recommendation,'' in
\emph{Proceedings of the 26th ACM SIGKDD International Conference on
Knowledge Discovery \& Data Mining}, 2020, pp. 2073--2083.
\bibitem{xv2021FPAN}
\BIBentryALTinterwordspacing
K.~Xu, J.~Yang, J.~Xu, S.~Gao, J.~Guo, and J.~Wen, ``Adapting user preference
to online feedback in multi-round conversational recommendation,'' in
\emph{{WSDM} '21, The Fourteenth {ACM} International Conference on Web Search
and Data Mining, Virtual Event, Israel, March 8-12, 2021}, L.~Lewin{-}Eytan,
D.~Carmel, E.~Yom{-}Tov, E.~Agichtein, and E.~Gabrilovich, Eds.\hskip 1em
plus 0.5em minus 0.4em\relax {ACM}, 2021, pp. 364--372. [Online]. Available:
\url{https://doi.org/10.1145/3437963.3441791}
\BIBentrySTDinterwordspacing
\bibitem{2018ZhaoKB4Rec}
\BIBentryALTinterwordspacing
W.~X. Zhao, G.~He, H.~Dou, J.~Huang, S.~Ouyang, and J.~Wen, ``Kb4rec: {A}
dataset for linking knowledge bases with recommender systems,'' \emph{CoRR},
vol. abs/1807.11141, 2018. [Online]. Available:
\url{http://arxiv.org/abs/1807.11141}
\BIBentrySTDinterwordspacing
\bibitem{2019CaoUnifying}
\BIBentryALTinterwordspacing
------, ``Kb4rec: {A} dataset for linking knowledge bases with recommender
systems,'' \emph{CoRR}, vol. abs/1807.11141, 2018. [Online]. Available:
\url{http://arxiv.org/abs/1807.11141}
\BIBentrySTDinterwordspacing
\bibitem{2019VolodymyrHuman}
M.~Volodymyr, K.~Koray, S.~David, .~Rusu, Andrei~A, V.~Joel, .~Bellemare,
Marc~G, G.~Alex, R.~Martin, .~Fidjeland, Andreas~K, and O.~Georg,
``Human-level control through deep reinforcement learning,'' \emph{Nature},
vol. 518, no. 7540, pp. 529--33, 2019.
\bibitem{2020HuGPA}
\BIBentryALTinterwordspacing
S.~Hu, Z.~Xiong, M.~Qu, X.~Yuan, M.~C{\^{o}}t{\'{e}}, Z.~Liu, and J.~Tang,
``Graph policy network for transferable active learning on graphs,'' in
\emph{Advances in Neural Information Processing Systems 33: Annual Conference
on Neural Information Processing Systems 2020, NeurIPS 2020, December 6-12,
2020, virtual}, H.~Larochelle, M.~Ranzato, R.~Hadsell, M.~Balcan, and H.~Lin,
Eds., 2020. [Online]. Available:
\url{https://proceedings.neurips.cc/paper/2020/hash/73740ea85c4ec25f00f9acbd859f861d-Abstract.html}
\BIBentrySTDinterwordspacing
\bibitem{2020WangKGPolicy}
\BIBentryALTinterwordspacing
X.~Wang, Y.~Xu, X.~He, Y.~Cao, M.~Wang, and T.~Chua, ``Reinforced negative
sampling over knowledge graph for recommendation,'' in \emph{{WWW} '20: The
Web Conference 2020, Taipei, Taiwan, April 20-24, 2020}, Y.~Huang, I.~King,
T.~Liu, and M.~van Steen, Eds.\hskip 1em plus 0.5em minus 0.4em\relax {ACM} /
{IW3C2}, 2020, pp. 99--109. [Online]. Available:
\url{https://doi.org/10.1145/3366423.3380098}
\BIBentrySTDinterwordspacing
\bibitem{2006KissEvaluatingCentrality}
C.~Kiss, A.~Scholz, and M.~Bichler, ``Evaluating centrality measures in large
call graphs,'' in \emph{IEEE International Conference on E-commerce
Technology \& the IEEE International Conference on Enterprise Computing},
2006.
\bibitem{1990HurvichKL}
C.~M. Hurvich and S.~C.~L. Tsai, ``Improved estimators of kullback–leibler
information for autoregressive model selection in small samples,''
\emph{Biometrika}, vol.~77, no.~4, pp. 709--719, 1990.
\bibitem{2005SmithKL}
A.~Smith, P.~A. Naik, and C.~L. Tsai, ``Markov-switching model selection using
kullback–leibler divergence,'' \emph{SSRN Electronic Journal}, vol. 134,
no.~2, pp. 553--577, 2005.
\bibitem{2016gcn}
\BIBentryALTinterwordspacing
T.~N. Kipf and M.~Welling, ``Semi-supervised classification with graph
convolutional networks,'' in \emph{5th International Conference on Learning
Representations, {ICLR} 2017, Toulon, France, April 24-26, 2017, Conference
Track Proceedings}.\hskip 1em plus 0.5em minus 0.4em\relax OpenReview.net,
2017. [Online]. Available: \url{https://openreview.net/forum?id=SJU4ayYgl}
\BIBentrySTDinterwordspacing
\bibitem{1999SuttonPolicyGradient}
R.~S. Sutton, D.~Mcallester, S.~Singh, and Y.~Mansour, ``Policy gradient
methods for reinforcement learning with function approximation,''
\emph{Submitted to Advances in Neural Information Processing Systems},
vol.~12, 1999.
\bibitem{park2018Adversarial}
\BIBentryALTinterwordspacing
D.~H. Park and Y.~Chang, ``Adversarial sampling and training for
semi-supervised information retrieval,'' in \emph{The World Wide Web
Conference, {WWW} 2019, San Francisco, CA, USA, May 13-17, 2019}, L.~Liu,
R.~W. White, A.~Mantrach, F.~Silvestri, J.~J. McAuley, R.~Baeza{-}Yates, and
L.~Zia, Eds.\hskip 1em plus 0.5em minus 0.4em\relax {ACM}, 2019, pp.
1443--1453. [Online]. Available:
\url{https://doi.org/10.1145/3308558.3313416}
\BIBentrySTDinterwordspacing
\bibitem{wang2017IRGAN}
J.~Wang, L.~Yu, W.~Zhang, Y.~Gong, and D.~Zhang, ``Irgan: A minimax game for
unifying generative and discriminative information retrieval models,'' in
\emph{SIGIR 2017 - 40th International ACM SIGIR Conference on Research and
Development in Information Retrieval}, 2017.
\bibitem{Christakopoulou2016TCRS}
\BIBentryALTinterwordspacing
K.~Christakopoulou, F.~Radlinski, and K.~Hofmann, ``Towards conversational
recommender systems,'' in \emph{Proceedings of the 22nd {ACM} {SIGKDD}
International Conference on Knowledge Discovery and Data Mining, San
Francisco, CA, USA, August 13-17, 2016}, B.~Krishnapuram, M.~Shah, A.~J.
Smola, C.~C. Aggarwal, D.~Shen, and R.~Rastogi, Eds.\hskip 1em plus 0.5em
minus 0.4em\relax {ACM}, 2016, pp. 815--824. [Online]. Available:
\url{https://doi.org/10.1145/2939672.2939746}
\BIBentrySTDinterwordspacing
\bibitem{Dhingra2016End}
\BIBentryALTinterwordspacing
B.~Dhingra, L.~Li, X.~Li, J.~Gao, Y.~Chen, F.~Ahmed, and L.~Deng, ``Towards
end-to-end reinforcement learning of dialogue agents for information
access,'' in \emph{Proceedings of the 55th Annual Meeting of the Association
for Computational Linguistics, {ACL} 2017, Vancouver, Canada, July 30 -
August 4, Volume 1: Long Papers}, R.~Barzilay and M.~Kan, Eds.\hskip 1em plus
0.5em minus 0.4em\relax Association for Computational Linguistics, 2017, pp.
484--495. [Online]. Available: \url{https://doi.org/10.18653/v1/P17-1045}
\BIBentrySTDinterwordspacing
\end{thebibliography}