Multi-Kernel Correntropy for Robust Learning
Extracted source text · arXiv 1905.10115
%% bare_jrnl.tex
%% V1.4b
%% 2015/08/26
%% by Michael Shell
%% see http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.8b or later) with an IEEE
%% journal paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/pkg/ieeetran
%% and
%% http://www.ieee.org/
%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE!
%% User assumes all risk.
%% In no event shall the IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including **
%% ** renaming them and changing author support contact information. **
%%*************************************************************************
% *** Authors should verify (and, if needed, correct) their LaTeX system ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. The IEEE's font choices and paper sizes can ***
% *** trigger bugs that do not appear when using other class files. *** ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/
\documentclass[journal]{IEEEtran}
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[journal]{../sty/IEEEtran}
\usepackage[numbers,sort&compress]{natbib}
\usepackage{amsmath,graphicx}
\usepackage{multirow}
\usepackage{booktabs}
\usepackage{bm}
\usepackage{mathrsfs}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{setspace}
\usepackage{subfigure}
\usepackage[justification=centering]{caption}
\usepackage{float}
% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)
% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}eq4
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
% % pdf code
% \else
% % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/pkg/ifpdf
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.
% *** CITATION PACKAGES ***
%
%\usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of the IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off
% such as if a citation ever needs to be enclosed in parenthesis.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 5.0 (2009-03-20) and later if using hyperref.sty.
% The latest version can be obtained at:
% http://www.ctan.org/pkg/cite
% The documentation is contained in the cite.sty file itself.
% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
% \usepackage[pdftex]{graphicx}
% declare the path(s) where your graphic files are
% \graphicspath{{../pdf/}{../jpeg/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
% or other class option (dvipsone, dvipdf, if not using dvips). graphicx
% will default to the driver specified in the system graphics.cfg if no
% driver is specified.
% \usepackage[dvips]{graphicx}
% declare the path(s) where your graphic files are
% \graphicspath{{../eps/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation
% can be obtained at:
% http://www.ctan.org/pkg/graphicx
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found at:
% http://www.ctan.org/pkg/epslatex
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). The IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex
% *** MATH PACKAGES ***
%
%\usepackage{amsmath}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics.
%
% Note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/amsmath
% *** SPECIALIZED LIST PACKAGES ***
%
%\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as the IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/pkg/algorithms
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/pkg/algorithmicx
% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/array
% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.
% *** SUBFIGURE PACKAGES ***
%\ifCLASSOPTIONcompsoc
% \usepackage[caption=false,font=normalsize,labelfont=sf,textfont=sf]{subfig}
%\else
% \usepackage[caption=false,font=footnotesize]{subfig}
%\fi
% subfig.sty, written by Steven Douglas Cochran, is the modern replacement
% for subfigure.sty, the latter of which is no longer maintained and is
% incompatible with some LaTeX packages including fixltx2e. However,
% subfig.sty requires and automatically loads Axel Sommerfeldt's caption.sty
% which will override IEEEtran.cls' handling of captions and this will result
% in non-IEEE style figure/table captions. To prevent this problem, be sure
% and invoke subfig.sty's "caption=false" package option (available since
% subfig.sty version 1.3, 2005/06/28) as this is will preserve IEEEtran.cls
% handling of captions.
% Note that the Computer Society format requires a larger sans serif font
% than the serif footnote size font used in traditional IEEE formatting
% and thus the need to invoke different subfig.sty package options depending
% on whether compsoc mode has been enabled.
%
% The latest version and documentation of subfig.sty can be obtained at:
% http://www.ctan.org/pkg/subfig
% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure.
% Be aware that LaTeX2e kernels dated 2015 and later have fixltx2e.sty's
% corrections already built into the system in which case a warning will
% be issued if an attempt is made to load fixltx2e.sty as it is no longer
% needed.
% The latest version and documentation can be found at:
% http://www.ctan.org/pkg/fixltx2e
%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/stfloats
% Do not use the stfloats baselinefloat ability as the IEEE does not allow
% \baselineskip to stretch. Authors submitting work to the IEEE should note
% that the IEEE rarely uses double column equations and that authors should try
% to avoid such use. Do not be tempted to use the cuted.sty or midfloat.sty
% packages (also by Sigitas Tolusis) as the IEEE does not format its papers in
% such ways.
% Do not attempt to use stfloats with fixltx2e as they are incompatible.
% Instead, use Morten Hogholm'a dblfloatfix which combines the features
% of both fixltx2e and stfloats:
%
% \usepackage{dblfloatfix}
% The latest version can be found at:
% http://www.ctan.org/pkg/dblfloatfix
%\ifCLASSOPTIONcaptionsoff
% \usepackage[nomarkers]{endfloat}
% \let\MYoriglatexcaption\caption
% \renewcommand{\caption}[2][\relax]{\MYoriglatexcaption[#2]{#2}}
%\fi
% endfloat.sty was written by James Darrell McCauley, Jeff Goldberg and
% Axel Sommerfeldt. This package may be useful when used in conjunction with
% IEEEtran.cls' captionsoff option. Some IEEE journals/societies require that
% submissions have lists of figures/tables at the end of the paper and that
% figures/tables without any captions are placed on a page by themselves at
% the end of the document. If needed, the draftcls IEEEtran class option or
% \CLASSINPUTbaselinestretch interface can be used to increase the line
% spacing as well. Be sure and use the nomarkers option of endfloat to
% prevent endfloat from "marking" where the figures would have been placed
% in the text. The two hack lines of code above are a slight modification of
% that suggested by in the endfloat docs (section 8.4.1) to ensure that
% the full captions always appear in the list of figures/tables - even if
% the user used the short optional argument of \caption[]{}.
% IEEE papers do not typically make use of \caption[]'s optional argument,
% so this should not be an issue. A similar trick can be used to disable
% captions of packages such as subfig.sty that lack options to turn off
% the subcaptions:
% For subfig.sty:
% \let\MYorigsubfloat\subfloat
% \renewcommand{\subfloat}[2][\relax]{\MYorigsubfloat[]{#2}}
% However, the above trick will not work if both optional arguments of
% the \subfloat command are used. Furthermore, there needs to be a
% description of each subfigure *somewhere* and endfloat does not add
% subfigure captions to its list of figures. Thus, the best approach is to
% avoid the use of subfigure captions (many IEEE journals avoid them anyway)
% and instead reference/explain all the subfigures within the main caption.
% The latest version of endfloat.sty and its documentation can obtained at:
% http://www.ctan.org/pkg/endfloat
%
% The IEEEtran \ifCLASSOPTIONcaptionsoff conditional can also be used
% later in the document, say, to conditionally put the References on a
% page by themselves.
% *** PDF, URL AND HYPERLINK PACKAGES ***
%
%\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/url
% Basically, \url{my_url_here}.
% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex). ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )
% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
%
% paper title
% Titles are generally capitalized except for words such as a, an, and, as,
% at, but, by, for, in, nor, of, on, or, the, to and up, which are usually
% not capitalized unless they are the first or last word of the title.
% Linebreaks \\ can be used within to get better formatting as desired.
% Do not put math or special symbols in the title.
\title{\huge {Multi-Kernel Correntropy for Robust Learning}}
%
%
% author names and IEEE memberships
% note positions of commas and nonbreaking spaces ( ~ ) LaTeX will not break
% a structure at a ~ so this keeps an author's name from being broken across
% two lines.
% use \thanks{} to gain access to the first footnote area
% a separate \thanks must be used for each paragraph as LaTeX2e's \thanks
% was not built to handle multiple paragraphs
%
\author{Badong~Chen,~\IEEEmembership{Senior Member,~IEEE,}
Xin~Wang,~\IEEEmembership{Student Member,~IEEE,}
Zejian~yuan,~\IEEEmembership{Member,~IEEE,}
Pengju~Ren,~\IEEEmembership{Member,~IEEE,}
and~Jing~Qin,~\IEEEmembership{Member,~IEEE}% <-this % stops a space
\thanks{This work was supported by the National NSF of China (No. 91648208), the National Natural Science Foundation-Shenzhen Joint Research Program (No.U1613219) and the National Key R$\&$D Program of China (2017YFB1002501).}
\thanks{Badong~Chen, Xin~Wang, Zejian~Yuan and Pengju~Ren(chenbd@mail.xjtu\newline.edu.cn, wangxin0420@ stu.xjtu.edu.cn, yuan.ze.jian @mail.xjtu.edu.cn and pengjuren@gmail.com) are with the School of Electronic and Information Engineering,Xi'an Jiaotong University, Xi'an 710049, Shaanxi, China.}% <-this % stops a space
\thanks{Jing~Qin(harry.qin@polyu.edu.hk) is with the Center of Smart Health, School of Nursing, Hong Kong Polytechnic University, Hongkong,China.}
}
% note the % following the last \IEEEmembership and also \thanks -
% these prevent an unwanted space from occurring between the last author name
% and the end of the author line. i.e., if you had this:
%
% \author{....lastname \thanks{...} \thanks{...} }
% ^------------^------------^----Do not want these spaces!
%
% a space would be appended to the last name and could cause every name on that
% line to be shifted left slightly. This is one of those "LaTeX things". For
% instance, "\textbf{A} \textbf{B}" will typeset as "A B" not "AB". To get
% "AB" then you have to do: "\textbf{A}\textbf{B}"
% \thanks is no different in this regard, so shield the last } of each \thanks
% that ends a line with a % and do not let a space in before the next \thanks.
% Spaces after \IEEEmembership other than the last one are OK (and needed) as
% you are supposed to have spaces between the names. For what it is worth,
% this is a minor point as most people would not even notice if the said evil
% space somehow managed to creep in.
% The paper headers
%\markboth{Journal of \LaTeX\ Class Files,~Vol.~14, No.~8, August~2015}%
%{Shell \MakeLowercase{\textit{et al.}}: Bare Demo of IEEEtran.cls for IEEE Journals}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the twoside option.
%
% *** Note that you probably will NOT want to include the author's ***
% *** name in the headers of peer review papers. ***
% You can use \ifCLASSOPTIONpeerreview for conditional compilation here if
% you desire.
% If you want to put a publisher's ID mark on the page you can do it like
% this:
%\IEEEpubid{0000--0000/00\$00.00~\copyright~2015 IEEE}
% Remember, if you use this you must call \IEEEpubidadjcol in the second
% column for its text to clear the IEEEpubid mark.
% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}
% make the title area
\maketitle
% As a general rule, do not put math, special symbols or citations
% in the abstract or keywords.
\begin{abstract}
As a novel similarity measure that is defined as the expectation of a kernel function between two random variables, correntropy has been successfully applied in robust machine learning and signal processing to combat large outliers. The kernel function in correntropy is usually a zero-mean Gaussian kernel. In a recent work, the concept of mixture correntropy (MC) was proposed to improve the learning performance, where the kernel function is a mixture Gaussian kernel, namely a linear combination of several zero-mean Gaussian kernels with different widths. In both correntropy and mixture correntropy, the center of the kernel function is, however, always located at zero. In the present work, to further improve the learning performance, we propose the concept of multi-kernel correntropy (MKC), in which each component of the mixture Gaussian kernel can be centered at a different location. The properties of the MKC are investigated and an efficient approach is proposed to determine the free parameters in MKC. Experimental results show that the learning algorithms under the maximum multi-kernel correntropy criterion (MMKCC) can outperform those under the original maximum correntropy criterion (MCC) and the maximum mixture correntropy criterion (MMCC).
\end{abstract}
% Note that keywords are not normally used for peerreview papers.
\begin{IEEEkeywords}
Correntropy, mixture correntropy, multi-kernel correntropy, robust learning, outliers.
\end{IEEEkeywords}
% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle
\section{Introduction}
% The very first letter is a 2 line initial drop letter followed
% by the rest of the first word in caps.
%
% form to use if the first word consists of a single letter:
% \IEEEPARstart{A}{demo} file is ....
%
% form to use if you need the single drop letter followed by
% normal text (unknown if ever used by the IEEE):
% \IEEEPARstart{A}{}demo file is ....
%
% Some journals put the first two words in caps:
% \IEEEPARstart{T}{his demo} file is ....
%
% Here we have the typical use of a "T" for an initial drop letter
% and "HIS" in caps to complete the first word.
\IEEEPARstart{A} key problem in supervised machine learning is how to define an objective function to measure the similarity between model output and a target variable. The mean square error (MSE) is one of the most popular similarity measures, which is computationally simple and easy to use as a performance index in many signal processing and machine learning applications. The MSE is, however, vulnerable to non-Gaussian noises, such as impulsive noises or outliers, because the solution that minimizes the squared difference (the error in ${L_2}$ norm) can deviate far from the optimal solution in the presence of large outliers. To address this problem, many non-MSE similarity measures were proposed in the literature, such as the mean absolute error (MAE)\cite{lin1990adaptive,coyle1988stack}, mean p-power error (MPE)\cite{2pei1994least}, M-estimation cost \cite{liu2006error} and logarithmic cost \cite{sayin2014novel}. In particular in recent years, the correntropy as a local similarity measure in kernel space has found many successful applications in robust regression \cite{chen2012recursive,13feng2015learning}, classification \cite{xu2018robust,syed2012correntropy,singh2014c,shi2018training}, PCA \cite{he2011robust}, feature extraction \cite{zhou2017maximum}, adaptive filtering \cite{zhao2011kernel,chen2016generalized,ma2015maximum,wu2015robust,wu2015kernel,cinar2012hidden,liu2017maximum} and so on. Correntropy defines a non-homogeneous metric (Correntropy Induced Metric- CIM) that behaves like different norms (from ${L_2}$ to ${L_0}$) depending on the actual distance between samples, which can be used as an outlier-robust error measure in robust signal processing or a sparsity penalty term in sparse signal processing \cite{liu2007correntropy}.
The original correntropy is defined as the expectation of a kernel function between two random variables, where the kernel function is usually a zero-mean Gaussian kernel \cite{liu2007correntropy}. The learning methods under maximum correntropy criterion (MCC) may however perform poorly when the kernel function in correntropy is limited to a single Gaussian kernel. It is likely that the combination of several kernel functions can perform much better. The mixture correntropy (MC) was thus proposed in a recent work to improve the learning performance, in which the kernel function is implemented by a linear combination of several zero-mean Gaussian kernels with different widths \cite{chen2018mixture}. However, there is still a shortcoming in the mixture correntropy that only allows the combination of zero-mean Gaussian kernels. To further improve the learning performance, in the present work, we propose the concept of multi-kernel correntropy (MKC), where each component of the mixture Gaussian kernel can be centered at a different location (not limited to zero-mean). Some important properties of the MKC are also studied. The MKC involves more free parameters than the MC, so a challenging issue is how to determine the free parameters in a practical application. To address this issue, we propose an efficient approach in this paper to optimize the free parameters in MKC by minimizing a distance between the mixture Gaussian function and the error's probability density function (PDF). Experimental results have confirmed the satisfactory performance of the learning methods under maximum multi-kernel correntropy criterion (MMKCC).
The rest of the paper is organized as follows. In section II, we define the MKC and present several properties. In section III, we propose an effective method to optimize the free parameters in MKC. Experimental results are then presented in section IV and finally, conclusion is given in section V.
%\hfill mds
%\hfill August 26, 2015
\section{MULTI-KERNEL CORRENTROPY}
\subsection{Definitions}
Given two random variables $X \in \mathbf{R}$ and $Y \in \mathbf{R}$ with joint PDF ${p_{XY}}(x,y)$, correntropy is defined by \cite{liu2007correntropy}
\begin{equation}
\label{eq1}
\begin{aligned}
V(X,Y) = \rm{\textbf{E}}\left[ {\kappa (\emph{X,Y})} \right] = \iint{{\kappa \emph{(x,y)}}\emph{p}_{\emph{XY}}}\emph{(x,y)dxdy}
\end{aligned}
\end{equation}
where $\kappa (.,.)$ is usually a radial kernel, and $\textbf{E}[.]$ denotes the expectation operator. If the kernel function $\kappa (.,.)$ satisfies Mercer¡¯s condition, correntropy can be expressed as a correlation measure in a functional Hilbert space $\mathcal{F}$:
\begin{equation}
\label{eq2}
\begin{aligned}
V(X,Y) = \rm{\textbf{E}}\left[ {{{\left\langle {\varphi (\emph{X}),\varphi (\emph{Y})} \right\rangle }_\mathcal{F}}} \right]
\end{aligned}
\end{equation}
where $\varphi (.)$ is a nonlinear mapping induced by the kernel to transform the variables from the original space to the functional space $\mathcal{F}$, and ${\left\langle {.,.} \right\rangle _\mathcal{F}}$ stands for the inner product in $\mathcal{F}$. Without explicit mention, the kernel function in correntropy is the well-known Gaussian kernel:
\begin{equation}
\label{eq3}
\begin{aligned}
\kappa (X,Y) = {\kappa _\sigma }(e) = \frac{1}{{\sqrt {2\pi } \sigma }}\exp \left( { - \frac{{{e^2}}}{{2{\sigma ^2}}}} \right)
\end{aligned}
\end{equation}
where $e = X - Y$ is the error between $X$ and $Y$, and $\sigma$ is the kernel bandwidth ($\sigma>0$). It is easy to understand that correntropy measures how similar two random variables are in a local region of the error space controlled by the kernel bandwidth. Correntropy can easily be estimated from finite samples:
\begin{equation}
\label{eq4}
\begin{aligned}
{\hat V_\sigma }(X,Y) = \frac{1}{N}\sum\limits_{i = 1}^N {{\kappa _\sigma }({x_i} - {y_i})}
\end{aligned}
\end{equation}
where $\left\{ {{x_i},{y_i}} \right\}_{i = 1}^N$ are $N$ samples of the random variables $X$ and $Y$. In particular, the function $CIM(\tilde X,\tilde Y) = \sqrt {{\kappa _\sigma }(0) - {{\hat V}_\sigma }(X,Y)}$ defines a metric, namely the correntropy induced metric (CIM) in the sample space, where $\tilde X = {\left[ {{x_1}, \cdots ,{x_N}} \right]^T}$, $\tilde Y = {\left[ {{y_1}, \cdots ,{y_N}} \right]^T}$. The CIM behaves like an $L_2$ norm distance if samples are close and like an $L_1$ norm distance as samples get further apart and eventually will approach the $L_0$ norm as samples far apart. This property elucidates the robustness of correntropy for outlier rejection. Under the maximum correntropy criterion (MCC), the detrimental effect of outliers can effectively be eliminated by maximizing the correntropy between the model output and target response \cite{chen2017maximum}.
The kernel function in correntropy is usually limited to a zero-mean Gaussian kernel and this may seriously restricts its performance when used as a cost function in machine learning. To improve the learning performance, the mixture correntropy (MC) was proposed in a recent paper \cite{chen2018mixture} by using a linear combination of several zero-mean Gaussian kernels (with different bandwidths) as the kernel function. The mixture correntropy with $m$ sub-kernels is
\begin{equation}
\label{eq5}
\begin{aligned}
{V_{\lambda ,\sigma }}(X,Y) &= \sum\limits_{i = 1}^m {{\lambda _i}{V_{{\sigma _i}}}(X,Y)} \\
{\rm{ }} &= \rm{\textbf{E}}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\emph{X - Y})} } \right]\\
{\rm{ }} &= \iint{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(x - y)} } \right)}p_{XY}}(x,y)dxdy
\end{aligned}
\end{equation}
where $\bm{\lambda} = {\left[ {{\lambda _1},{\lambda _2}, \cdots ,{\lambda _m}} \right]^T}$ is the mixture coefficient vector, and $\bm{\sigma} = {\left[ {{\sigma _1},{\sigma _2}, \cdots ,{\sigma _m}} \right]^T}$ is the bandwidth vector. Usually, the mixture coefficient vector satisfies $\sum\limits_{i = 1}^m {{\lambda _i}} = 1$ with ${\lambda _i} \ge 0$($i = 1, \cdots ,m$). In \cite{chen2018mixture}, for simplicity, only the case of $m=2$ is considered. There is still a limitation in the mixture correntropy, that is, all the sub-kernels are centered at zero. To solve this limitation and further enhance the learning performance, in the present paper, we propose a more general definition of correntropy, namely, the multi-kernel correntropy (MKC), in which the sub-kernels can be centered at different locations (not limited to zero-mean). Specifically, the MKC between random variables $X$ and $Y$ is defined by
\begin{equation}
\label{eq6}
\begin{aligned}
&{V_{\bm{\lambda ,c,\sigma} }}(X,Y)= \textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(X - Y - {c_i})} } \right]\\
&=\iint{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(x - y - {c_i})} } \right)}p_{XY}}(x,y)dxdy
\end{aligned}
\end{equation}
where $\bm{c} = {\left[ {{c_1},{c_2}, \cdots ,{c_m}} \right]^T} \in {\mathbf{R}^m}$ is the center vector.
\emph{Remark}: The kernel function in the above MKC is a multi-Gaussian function that usually does not satisfy Mercer's condition. This is not a problem, however, because for a similarity measure the Mercer's condition is not necessary.
The MKC ${V_{\bm{\lambda ,c,\sigma} }}(X,Y)$ will reduce to the MC ${V_{\bm{\lambda ,\sigma} }}(X,Y)$ when $\bm{c} = {\left[ {0, \cdots ,0} \right]^T}$. Fig. 1 shows the kernel functions of the mixture correntropy($m=2, \lambda_1=0.5, \lambda_2=0.5, \sigma_1=0.5, \sigma_2=1.5$) and multi-kernel correntropy($m=2, \lambda_1=0.5, \lambda_2=0.5, \sigma_1=0.5, \sigma_2=1.5, c_1=-1.0, c_2=2.0$). Compared with the MC, the MKC is much more general and flexible and can adapt to more complicated error distribution, such as skewed, multi-peak, discrete-valued distribution, and hence it may achieve much better performance with proper setting of the centers when used as a cost function in machine learning. However, the MKC contains $3m$ free parameters, which have to be determined in practical applications. We will develop an efficient method in section IV to determine these free parameters.
\begin{figure}[]
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
%\makeatletter
%\def\@captype{figure}
%\makeatother
%\begin{minipage}[t]{0.5\linewidth}
\centering
\includegraphics[width=2.8in,height=1.8in]{fig1.eps}
\caption{Kernel functions of the mixture correntropy and multi-kernel correntropy}
\label{fig1}
\end{figure}
\subsection{Properties}
In the following, we present several properties of the MKC. The first and second properties are very straightforward and will not be proved here.
Property 1: The MKC ${V_{\bm{\lambda ,c,\sigma} }}(X,Y)$ is positive and bounded: $0 < {V_{\bm{\lambda ,c,\sigma} }}(X,Y) \le \sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sqrt {2\pi } {\sigma _i}}}} $
Property 2: The MKC ${V_{\bm{\lambda ,c,\sigma} }}(X,Y)$ involves all the even moments of the error $e = X - Y$ about the centers $\left\{ {{c_i}} \right\}$, that is,
\begin{small}
\begin{equation}
\label{eq7}
\begin{aligned}
{V_{\bm{\lambda ,c,\sigma} }}(X,Y) = \sum\limits_{i = 1}^m {\left( {\frac{{{\lambda _i}}}{{\sqrt {2\pi } {\sigma _i}}}\sum\limits_{n = 0}^\infty {\frac{{{{\left( { - 1} \right)}^n}}}{{{2^n}n!}}\textbf{E}\left[ {\frac{{{{\left( {e - {c_i}} \right)}^{2n}}}}{{\sigma _i^{2n}}}} \right]} } \right)}
\end{aligned}
\end{equation}
\end{small}
\emph{Remark}: As $\left\{ {{\sigma _i}} \right\}$ increases, the high-order moments will decay fast, and the second-order moments will tend to dominate the value.
Property 3: As $min\left\{ {{\sigma _i}} \right\}$ is large enough, it holds that
\begin{equation}
\label{eq8}
\begin{aligned}
{V_{\bm{\lambda ,c,\sigma} }}(X,Y) \approx \sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sqrt {2\pi } {\sigma _i}}}} \left( {1 - \frac{1}{{2\sigma _i^2}}\textbf{E}\left[ {{{\left( {e - {c_i}} \right)}^2}} \right]} \right)
\end{aligned}
\end{equation}
Proof: Since $\exp (x) \approx 1 + x$ for $x$ small enough, as $min\left\{ {{\sigma _i}} \right\}$ is large enough, we have
\begin{equation}
\label{eq9}
\begin{aligned}
{V_{\bm{\lambda ,c,\sigma} }}(X,Y) &= \textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(X - Y - {c_i})} } \right]\\
{\rm{ }} &\approx \textbf{E}\left[ {\sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sqrt {2\pi } {\sigma _i}}}\left( {1 - \frac{{{{\left( {e - {c_i}} \right)}^2}}}{{2\sigma _i^2}}} \right)} } \right]\\
{\rm{ }} &= \sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sqrt {2\pi } {\sigma _i}}}} \left( {1 - \frac{1}{{2\sigma _i^2}}\textbf{E}\left[ {{{\left( {e - {c_i}} \right)}^2}} \right]} \right)
\end{aligned}
\end{equation}
which completes the proof.
\emph{Remark}: According to Property 3, when $min\left\{ {{\sigma _i}} \right\}$ is very large, maximizing the MKC will be equivalent to minimizing a weighted sum of the error's second-order moments about the centers $\left\{ {{c_i}} \right\}$.
Property 4: Let ${p_e}(.)$ be the PDF of the error variable $e=X-Y$. It holds that
\begin{equation}
\label{eq10}
\begin{aligned}
\mathop {\lim }\limits_{max\left\{ {{\sigma _i}} \right\} \to 0 + } {V_{\bm{\lambda ,c,\sigma} }}(X,Y) = \sum\limits_{i = 1}^m {{\lambda _i}{p_e}({c_i})}
\end{aligned}
\end{equation}
Proof: When $max\left\{ {{\sigma _i}} \right\}$ shrinks to zero, the Gaussian kernel function ${\kappa _{{\sigma _i}}}(.)$ will approach the Dirac delta function $\delta (.)$. Thus we have
\begin{small}
\begin{equation}
\label{eq11}
\begin{aligned}
\mathop {\lim }\limits_{max\left\{ {{\sigma _i}} \right\} \to 0 + }&{V_{\bm{\lambda ,c,\sigma} }}(X,Y) = \mathop {\lim }\limits_{max\left\{ {{\sigma _i}} \right\} \to 0 + } \textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(e - {c_i})} } \right]\\
&= \mathop {\lim }\limits_{max\left\{ {{\sigma _i}} \right\} \to 0 + } \sum\limits_{i = 1}^m {{\lambda _i}\int {{\kappa _{{\sigma _i}}}(\varepsilon - {c_i}){p_e}(\varepsilon )d\varepsilon } } \\
&= \sum\limits_{i = 1}^m {{\lambda _i}\int {\delta (\varepsilon - {c_i}){p_e}(\varepsilon )d\varepsilon } } \\
&= \sum\limits_{i = 1}^m {{\lambda _i}{p_e}({c_i})}
\end{aligned}
\end{equation}
\end{small}
which completes the proof.
\emph{Remark}: According to Property 4, when $max\left\{ {{\sigma _i}} \right\}$ is very small, the MKC will approach a weighted sum of the values of ${p_e}(\varepsilon )$ evaluated at $\varepsilon = {c_i}$ ($i = 1, \cdots ,m$).
\section{MAXIMUM MULTI-KERNEL CORRENTROPY CRITERION}
The proposed MKC can be used to build new cost functions in many machine learning applications. Consider a supervised learning setting where the goal is to optimize a model $M$ that receives a random variable $X$ and outputs $Y=M(X)$ that should approximate a target variable (or teaching variable) $T$. Here $M(.)$ denotes an unknown mapping from the input to output that needs to be learned. A central problem in this learning task is the definition of a loss function (or a similarity measure) to compare $Y$ with $T$. The well-known minimum mean square error (MMSE) criterion has been the workhorse of supervised learning, which aims to minimize the MSE cost $\textbf{E}\left[ {{e^2}} \right]$ with $e=T-Y$ being the error variable. The combination of the linear feedforward model and MSE yields a set of equations that can be solved analytically. However, MSE is only optimal when the error variable is Gaussian distributed, which is seldom the case in real world applications. The error distributions tend to be skewed and with long tails, which create problems for MSE. Therefore, many ¡°optimal solutions¡± are indeed not practical, simply because of the criterion that is used in the optimization. Many non-MSE optimization criterion were proposed in the literature to address the limitations of the MSE. The maximum correntropy criterion (MCC) is one of the hotspots of current research, which performs very well particularly when the error distribution is heavy-tailed \cite{wang2017maximum}. Under the MCC, the model is optimized (or trained) to maximize the correntropy between the target $T$ and output $Y$:
\begin{equation}
\label{eq12}
\begin{aligned}
{M^*} &= \mathop {\arg \max }\limits_{M \in \textbf{M}} {V_\sigma }\left( {T,Y} \right) \\
&= \mathop {\arg \max }\limits_{M \in \textbf{M}} \textbf{E}\left[ {{\kappa _\sigma }\left( e \right)} \right]
\end{aligned}
\end{equation}
where $M^*$ denotes the optimal model and $\textbf{M}$ stands for the hypothesis space. To improve the learning performance, the maximum mixture correntropy criterion (MMCC) was proposed in \cite{chen2018mixture}. To further improve the flexibility and robustness, in the present paper, we propose the maximum multi-kernel correntropy criterion (MMKCC), where the optimal model is obtained by maximizing the MKC, that is
\begin{equation}
\label{eq13}
\begin{aligned}
{M^*} &= \mathop {\arg \max }\limits_{M \in \textbf{M}} {V_{\bm{\lambda ,c,\sigma} }}(T,Y) \\
&= \mathop {\arg \max }\limits_{M \in \textbf{M}} \textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(e - {c_i})} } \right]
\end{aligned}
\end{equation}
In a practical situation, given finite input-target samples $\left\{ {{x_j},{t_j}} \right\}_{j = 1}^N$, the model can be trained through maximizing a sample estimator of the MKC:
\begin{equation}
\label{eq14}
\begin{aligned}
{M^*} &= \mathop {\arg \max }\limits_{M \in \textbf{M}} {\hat V_{\bm{\lambda ,c,\sigma} }}(T,Y) \\
&= \mathop {\arg \max }\limits_{M \in \textbf{M}} \frac{1}{N}\sum\limits_{j = 1}^N {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}({e_j} - {c_i})} }
\end{aligned}
\end{equation}
where ${e_j} = {t_j} - {y_j} = {t_j} - M({x_j})$ is the $j$-th error sample.
In the following, we present a simple example to show how to solve the optimal solution under MMKCC. Consider a linear-in-parameter (LIP) model in which the $j$-th output sample is
\begin{equation}
\label{eq15}
\begin{aligned}
{y_j} &= {\bm{h}_j}\bm{\beta} \\
&= \left[ {{\varphi _1}({\bm{x}_j}),{\varphi _2}({\bm{x}_j}), \cdots ,{\varphi _L}({\bm{x}_j})} \right]{\left[ {{\beta _1},{\beta _2}, \cdots ,{\beta _L}} \right]^T}
\end{aligned}
\end{equation}
where ${\bm{h}_j} = \left[ {{\varphi _1}({\bm{x}_j}),{\varphi _2}({\bm{x}_j}), \cdots ,{\varphi _L}({\bm{x}_j})} \right] \in {\rm{\textbf{R}}^L}$ is the $j$-th nonlinearly mapped input vector (a row vector), with ${\varphi _l}(.)$ being the $l$-th nonlinear mapping function ($l = 1,2, \cdots ,L$), and $\bm{\beta} = {\left[ {{\beta _1},{\beta _2}, \cdots ,{\beta _L}} \right]^T} \in {\rm{\textbf{R}}^L}$ is the output weight vector to be learned. Based on the MMKCC, the optimal weight vector ${\bm{\beta} ^*}$ can be solved by maximizing the following objective function:
\begin{equation}
\label{eq16}
\begin{aligned}
{\bm{\beta} ^*} &= \mathop {\arg \max }\limits_{\bm{\beta} \in {\rm{\textbf{R}}^L}} J(\bm{\beta} ) \\
&= \frac{1}{N}\sum\limits_{j = 1}^N {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}({e_j} - {c_i})} } - \gamma {\left\| \bm{\beta} \right\|^2}
\end{aligned}
\end{equation}
where ${e_j} = {t_j} - {\bm{h}_j}\bm{\beta} $ , and $\gamma \ge 0$ is a regularization parameter. Setting ${{\partial J(\bm{\beta} )} \mathord{\left/
{\vphantom {{\partial J(\bm{\beta} )} {\partial \bm{\beta} }}} \right.
\kern-\nulldelimiterspace} {\partial \beta }} = \bm{0}$, we have
\begin{footnotesize}
\begin{equation}
\label{eq17}
\begin{aligned}
&\frac{1}{N}\sum\limits_{j = 1}^N {\sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sigma _i^2}}{\kappa _{{\sigma _i}}}({e_j} - {c_i})({e_j} - {c_i})\bm{h}_j^T} } - 2\gamma \bm{\beta} = 0\\
&\Rightarrow \sum\limits_{j = 1}^N {\sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sigma _i^2}}{\kappa _{{\sigma _i}}}({e_j} - {c_i})({t_j} - {\bm{h}_j}\bm{\beta} - {c_i})\bm{h}_j^T} } - \gamma '\bm{\beta} = 0\\
&\Rightarrow \sum\limits_{j = 1}^N {\psi ({e_j})\bm{h}_j^T{\bm{h}_j}\beta } + \gamma '\bm{\beta} = \sum\limits_{j = 1}^N {\psi ({e_j}){t_j}\bm{h}_j^T} - \sum\limits_{j = 1}^N {\zeta ({e_j})\bm{h}_j^T}
\end{aligned}
\end{equation}
\end{footnotesize}
where $\psi ({e_j}) = \sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sigma _i^2}}{\kappa _{{\sigma _i}}}({e_j} - {c_i})}$, $\zeta ({e_j}) = \sum\limits_{i = 1}^m {\frac{{{\lambda _i}{c_i}}}{{\sigma _i^2}}{\kappa _{{\sigma _i}}}({e_j} - {c_i})}$, and $\gamma ' = 2N\gamma$. From (17), one can easily derive
\begin{footnotesize}
\begin{equation}
\label{eq18}
\begin{aligned}
\bm{\beta} &= {\left( {\sum\limits_{j = 1}^N {\psi ({e_j})\bm{h}_j^T{\bm{h}_j}} + \gamma 'I} \right)^{ - 1}}\left( {\sum\limits_{j = 1}^N {\psi ({e_j}){t_j}\bm{h}_j^T} - \sum\limits_{j = 1}^N {\zeta ({e_j})\bm{h}_j^T} } \right)\\
&= {\left( {{\textbf{H}^T}\bm{\Lambda} \textbf{H} + \gamma 'I} \right)^{ - 1}}\left( {{\textbf{H}^T}\bm{\Lambda} \textbf{T} - {\textbf{H}^T}\bm{\theta} } \right)
\end{aligned}
\end{equation}
\end{footnotesize}
where $\textbf{H} = \left[ {{\bm{h}_{jl}}} \right]$ is an $N \times L$ dimensional matrix with ${\bm{h}_{jl}} = {\varphi _l}({\bm{x}_j})$, $\bm{\Lambda}$ is an $N \times N$ diagonal matrix with diagonal elements ${\bm{\Lambda}_{jj}} = \psi ({e_j})$, $\textbf{T} = {\left[ {{t_1}, \cdots ,{t_N}} \right]^T}$ , and $\bm{\theta} = {\left[ {\zeta ({e_1}), \cdots ,\zeta ({e_N})} \right]^T}$.
The equation (18) is not a closed-form solution and it is actually a fixed-point equation because the diagonal matrix $\bm{\Lambda}$ and $\bm{\theta}$ vector on the right-hand side depend on the weight vector $\bm{\beta}$ through ${e_j} = {t_j} - {\bm{h}_j}\bm{\beta} $. However, the optimal solution of $\bm{\beta}$ can be obtained via a fixed-point iterative algorithm based on the equation (18), as described in \textbf{Algorithm 1}.
\begin{algorithm}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
\caption{Fixed-point iterative algorithm under MMKCC}
\label{alg:1}
\begin{algorithmic}[1]
\REQUIRE training samples $\{ {\bm{x}_i},{t_i}\} _{i = 1}^N$, number of nonlinear mappers $L$, mixture coefficient vector $\bm{\lambda}$, bandwidth vector $\bm{\sigma}$, center vector $\bm{c}$,regularization parameter $\gamma '$, maximum iteration number $K$, termination tolerance $\xi$ and the initial weight vector ${\bm{\beta} _0}{\rm{ = }}\textbf{0}$.
\ENSURE weight vector $\bm{\beta}$\\
\FORALL{$k = 1,2,...,K$}
\STATE Compute the errors based on ${\bm{\beta} _{k - 1}}$: ${e_i} = {t_i} - {\bm{h}_i}{\bm{\beta} _{k - 1}}$, $i = 1,2, \cdots ,N$
\STATE Compute the diagonal matrix ${\bf{\Lambda }}$: ${\bm{\Lambda} _{jj}} = \sum\limits_{i = 1}^m {\frac{{{\lambda _i}}}{{\sigma _i^2}}{\kappa _{{\sigma _i}}}({e_j} - {c_i})}$,$j = 1,2, \cdots ,N$
\STATE Compute the vector $\bm{\theta}$: $\bm{\theta} = {\left[ {\zeta ({e_1}), \cdots ,\zeta ({e_N})} \right]^T}$
\STATE Update the weight vector $\bm{\beta}$: ${\bm{\beta} _k} = {\left( {{\bf{H}^T}\bm{\Lambda} \bf{H} + \gamma 'I} \right)^{ - 1}}\left( {{\bf{H}^T}\bm{\Lambda} \bf{T} - {\bf{H}^T}\bm{\theta} } \right)$
\STATE \textbf{Until} $\left| {{J}({\bm{\beta} _k}) - {J}({\bm{\beta} _{k - 1}})} \right| < \xi $
\ENDFOR
\end{algorithmic}
\end{algorithm}
\section{DETERMINATION OF FREE PARAMETERS IN MMKCC}
One of the most challenging problems in MMKCC is how to determine the $3m$ free parameters, namely the vectors $\bm{\lambda} = {\left[ {{\lambda _1},{\lambda _2}, \cdots ,{\lambda _m}} \right]^T}$, $\bm{c} = {\left[ {{c_1},{c_2}, \cdots ,{c_m}} \right]^T}$ and $\bm{\sigma} = {\left[ {{\sigma _1},{\sigma _2}, \cdots ,{\sigma _m}} \right]^T}$. If this problem is not solved, the MMKCC will not be practical. To address this problem, we consider again the supervised learning setting in the previous section. First, we divide the MMKCC into three terms:
\begin{equation}
\label{eq19}
\begin{aligned}
&{V_{\bm{\lambda ,c,\sigma} }}(T,Y) = \textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(e - {c_i})} } \right]\\
&=\frac{1}{2}\int {{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\varepsilon - {c_i})} } \right)}^2}d\varepsilon } {\rm{ + }}\frac{1}{2}\int {{{\left( {{p_e}(\varepsilon )} \right)}^2}d\varepsilon } \\
&- \frac{1}{2}\int {{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\varepsilon - {c_i})} - {p_e}(\varepsilon )} \right)}^2}d\varepsilon }
\end{aligned}
\end{equation}
The first term is independent of the model $M$, so we have
\begin{equation}
\label{eq20}
\begin{aligned}
{M^*} &= \mathop {\arg \max }\limits_{M \in \textbf{M}} {V_{\bm{\lambda ,c,\sigma} }}(T,Y) \\
&= \mathop {\arg \max }\limits_{M \in \textbf{M}} {U_{\bm{\lambda ,c,\sigma} }}(T,Y)
\end{aligned}
\end{equation}
where ${U_{\bm{\lambda ,c,\sigma} }}(T,Y) = \frac{1}{2}\int {{{\left( {{p_e}(\varepsilon )} \right)}^2}d\varepsilon } - \frac{1}{2}\int {{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\varepsilon - {c_i})} - {p_e}(\varepsilon )} \right)}^2}d\varepsilon }$.
To determine the free parameters, in this study we propose the optimization in (21),
\begin{table*}
\begin{equation}
\label{eq21}
\begin{aligned}
&\left( {{M^*},\bm{{\lambda ^*},{c^*},{\sigma ^*}}} \right) = \mathop {\arg \max }\limits_{M \in \textbf{M},\bm{\lambda} \in {\bm{\Omega _\lambda} },\bm{c} \in {\bm{\Omega _c}},\bm{\sigma} \in {\bm{\Omega _\sigma} }} {U_{\bm{\lambda ,c,\sigma} }}(T,Y)\\
&= \mathop {\arg \max }\limits_{M \in \textbf{M},\bm{\lambda} \in {\bm{\Omega _\lambda} },\bm{c} \in {\bm{\Omega _c}},\bm{\sigma} \in {\bm{\Omega _\sigma} }} \frac{1}{2}\int {{{\left( {{p_e}(\varepsilon )} \right)}^2}d\varepsilon } - \frac{1}{2}\int {{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\varepsilon - {c_i})} - {p_e}(\varepsilon )} \right)}^2}d\varepsilon } \\
&= \mathop {\arg \max }\limits_{M \in \textbf{M},\bm{\lambda} \in {\bm{\Omega _\lambda} },\bm{c} \in {\bm{\Omega _c}},\bm{\sigma} \in {\bm{\Omega _\sigma} }} - \frac{1}{2}\int {{{\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\varepsilon - {c_i})} } \right)}^2}d\varepsilon } + \textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(e - {c_i})} } \right]
\end{aligned}
\end{equation}
\end{table*}
where ${\bm{\Omega _\lambda}}$, ${\bm{\Omega _c}}$ and ${\bm{\Omega _\sigma}}$ denote the admissible sets of the parameter vectors $\bm{\lambda}$, $\bm{c}$ and $\bm{\sigma}$.
\emph{Remark}: It is worth noting that the objective function ${U_{\bm{\lambda ,c,\sigma}}}(T,Y)$ can be expressed as
\begin{equation}
\label{eq22}
\begin{aligned}
{U_{\bm{\lambda ,c,\sigma}}}(T,Y) &= \frac{1}{2}QIP\left( e \right) \\
&- \frac{1}{2}{D_{ED}}\left( {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(\varepsilon - {c_i})} \left\| {{p_e}(\varepsilon )} \right.} \right)
\end{aligned}
\end{equation}
where $QIP\left( e \right) = \int {{{\left( {{p_e}(\varepsilon )} \right)}^2}d\varepsilon }$ is the quadratic information potential (QIP) \cite{principe2010information} of the error $e$, and ${D_{ED}}\left( {.\left\| . \right.} \right)$ denotes the Euclidean distance between PDFs \cite{santamaria2002adaptive,heravi2018new}, defined by ${D_{ED}}\left( {p(x)\left\| {q(x)} \right.} \right) = \int {{{(p(x) - q(x))}^2}dx} $. Therefore, maximizing the objective function ${U_{\bm{\lambda ,c,\sigma}}}(T,Y)$ will try to maximize the QIP (or minimize Renyi's quadratic entropy) of the error and at the same time, minimize the Euclidean distance between the multi-Gaussian kernel function and the error's PDF.
If $N$ error samples $\left\{ {{e_j}} \right\}_{j = 1}^N$ are available, we have $\textbf{E}\left[ {\sum\limits_{i = 1}^m {{\lambda _i}{\kappa _{{\sigma _i}}}(e - {c_i})} } \right] \approx {\bm{\lambda} ^T}\bm{\tilde h}$, where $\bm{\tilde h} = \frac{1}{N}\sum\limits_{j = 1}^N {\bm{\tilde g}({e_j})}$, with $\bm{\tilde g}({e_j}) = {\left[ {{\kappa _{{\sigma _1}}}({e_j} - {c_1}), \cdots ,{\kappa _{{\sigma _m}}}({e_j} - {c_m})} \right]^T}$. Thus by (21), we have (23),
\begin{table*}
\begin{equation}
\label{eq23}
\begin{aligned}
\left( {{M^*},{\bm{\lambda ^*},{c^*},{\sigma ^*}}} \right) &= \mathop {\arg \max }\limits_{M \in \textbf{M},\bm{\lambda \in {\Omega _\lambda },c \in {\Omega _c},\sigma \in {\Omega _\sigma }}} - \frac{1}{2}{\bm{\lambda} ^T}\left( {\int {\bm{\tilde g}(\varepsilon )\bm{\tilde g}{{(\varepsilon )}^T}} d\varepsilon } \right)\bm{\lambda}{\rm{ + }}{\bm{\lambda} ^T}\bm{\tilde h}\\
&= \mathop {\arg \max }\limits_{M \in \textbf{M},\bm{\lambda \in {\Omega _\lambda },c \in {\Omega _c},\sigma \in {\Omega _\sigma }}} - \frac{1}{2}{\bm{\lambda} ^T}\mathbf{\tilde K}\bm{\lambda} {\rm{ + }}{\bm{\lambda} ^T} \bm{\tilde h}
\end{aligned}
\end{equation}
\end{table*}
where $\mathbf{\tilde K}$ is expressed in (24).
\begin{table*}
\begin{equation}
\label{eq24}
\begin{aligned}
\bf{\tilde K} = \left[ {\begin{array}{*{20}{c}}
{\frac{1}{{\sqrt {2\pi } \sqrt {{\sigma _1}^2 + {\sigma _1}^2} }}\exp (\frac{{{{({c_1} - {c_1})}^2}}}{{2({\sigma _1}^2 + {\sigma _1}^2)}})}& \ldots &{\frac{1}{{\sqrt {2\pi } \sqrt {{\sigma _1}^2 + {\sigma _m}^2} }}\exp (\frac{{{{({c_1} - {c_m})}^2}}}{{2({\sigma _1}^2 + {\sigma _m}^2)}})}\\
\vdots & \ddots & \vdots \\
{\frac{1}{{\sqrt {2\pi } \sqrt {{\sigma _m}^2 + {\sigma _1}^2} }}\exp (\frac{{{{({c_m} - {c_1})}^2}}}{{2({\sigma _m}^2 + {\sigma _1}^2)}})}& \ldots &{\frac{1}{{\sqrt {2\pi } \sqrt {{\sigma _M}^2 + {\sigma _m}^2} }}\exp (\frac{{{{({c_m} - {c_m})}^2}}}{{2({\sigma _m}^2 + {\sigma _m}^2)}})}
\end{array}} \right]
\end{aligned}
\end{equation}
\end{table*}
According to (23), the model $M$ and $3m$ free parameters are jointly optimized via maximizing the objective function ${\hat U_{\bm{\lambda ,c,\sigma}}}(T,Y) = - \frac{1}{2}{\bm{\lambda} ^T}\mathbf{\tilde K}\bm{\lambda} {\rm{ + }}{\bm{\lambda} ^T}\bm{\tilde h}$. This is a very complicated optimization problem. To simplify the optimization, one can adopt an alternative optimization method: i) given a model (hence the $N$ error samples are given), we solve the free parameters by maximizing ${\hat U_{\bm{\lambda ,c,\sigma}}}(T,Y)$ (with error samples fixed); ii) after the free parameters are determined, we solve a new model by maximizing ${\hat U_{\bm{\lambda ,c,\sigma}}}(T,Y)$ (with free parameters fixed). The above procedure can be repeated until convergence and the flow chart is shown in Fig. 2.
\begin{figure}[t]
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
%\makeatletter
%\def\@captype{figure}
%\makeatother
%\begin{minipage}[t]{0.5\linewidth}
\centering
\includegraphics[width=2.4in,height=2.0in]{flow2.eps}
\caption{Alternative optimization for model and free parameters}
\label{fig1}
\end{figure}
In a practical application, there are usually two approaches to find the free parameters and the optimal model. The first one is an online approach, in which the model is optimized by an iterative method and at each iteration, the $3m$ free parameters are determined based on the error samples at that iteration. The second one is a two-stage approach, which contains two stages: 1) train the model using a simple method (usually with very few free parameters) to obtain the error samples, and determine the $3m$ free parameters based on these errors; 2) train the model again under the MMKCC with the obtained free parameters, and during the training these free parameters are fixed.
Next, we describe how to determine the $3m$ free parameters given a model. First, to simplify the optimization, we just apply some clustering technique such as the K-means on the error samples to obtain the center vector ${\bm{c}^*}$ (whose elements are the clustering centers). Then by (23), one can easily obtain the mixture coefficient vector:
\begin{equation}
\label{eq25}
\begin{aligned}
{\bm{\lambda} ^{\rm{*}}} = {\bf{\tilde K}^{ - {\rm{1}}}}\bm{\tilde h}
\end{aligned}
\end{equation}
In order to avoid numerical problem in the matrix inversion, a regularized solution can be used:
\begin{equation}
\label{eq26}
\begin{aligned}
{\bm{\lambda} ^{\rm{*}}} = {(\bf{\tilde K} + \eta I)^{ - {\rm{1}}}}\bm{\tilde h}
\end{aligned}
\end{equation}
where $\eta$ is a regularization parameter. Substituting (26) into (23), we solve the bandwidth vector as follows:
\begin{equation}
\label{eq27}
\begin{aligned}
{\bm{\sigma} ^*} &= \mathop {\arg \max }\limits_{\bm{\sigma \in {\Omega _\sigma }}} - \frac{1}{2}{\left[ {{{(\mathbf{\tilde K} + \eta I)}^{ - {\rm{1}}}}\bm{\tilde h}} \right]^T}\mathbf{\tilde K}{(\mathbf{\tilde K} + \eta I)^{ - {\rm{1}}}}\bm{\tilde h} \\
&+ {\left[ {{{(\mathbf{\tilde K} + \eta I)}^{ - {\rm{1}}}}\bm{\tilde h}} \right]^T}\bm{\tilde h}
\end{aligned}
\end{equation}
In order to reduce the computational complexity of the optimization problem in (27), one can alternately optimize every dimension of the bandwidth vector over a finite set of values. Specifically, given a finite set of bandwidths ${{\bm{\Omega} _\sigma}}$, we optimize each element of the bandwidth vector $\bm{\sigma}$ one by one and repeat this procedure until convergence. The proposed procedure for free parameters determination is summarized in \textbf{Algorithm 2}.
\begin{algorithm}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
\caption{Determination of the free parameters}
\label{alg:1}
\begin{algorithmic}[1]
\REQUIRE error samples $\left\{ {{e_j}} \right\}_{j = 1}^N$, parameter dimension $m$, regularization parameter $\eta$, a finite set of bandwidths ${{\bm{\Omega} _\sigma}}$ and initialize ${\sigma _1} = \cdots = {\sigma _m} = {\sigma _0}$.
\ENSURE free parameters $\bm{{\lambda ^*},{c^*},{\sigma ^*}}$ \\
\STATE Determine the center vector $\bm{c^*}$ by applying the K-means clustering on the error samples $\left\{ {{e_j}} \right\}_{j = 1}^N$
\STATE Alternately optimize every dimension of the bandwidth vector $\bm{\sigma}$ and repeat $S$ times:
\FORALL{$s = 1,2,...,S$}
\FORALL{$i = 1,2,...,m$}
\STATE ${\sigma_i^*} = \mathop {\arg \max }\limits_{\sigma_i \in {{\bm{\Omega} _\sigma }}} - \frac{1}{2}{\left[ {{{(\mathbf{\tilde K} + \eta I)}^{ - {\rm{1}}}}\bm{\tilde h}} \right]^T}\mathbf{\tilde K}{(\mathbf{\tilde K} + \eta I)^{ - {\rm{1}}}}\bm{\tilde h} + {\left[ {{{(\mathbf{\tilde K} + \eta I)}^{ - {\rm{1}}}}\bm{\tilde h}} \right]^T}\bm{\tilde h}$, with $\bm{c=c^*}$ and other $m-1$ elements of $\bm{\sigma}$ being fixed
\ENDFOR
\ENDFOR
\STATE Compute ${\bm{\lambda} ^{\rm{*}}} = {(\bf{\tilde K} + \eta I)^{ - {\rm{1}}}}\bm{\tilde h}$ with $\bm{\sigma=\sigma^*}$ and $\bm{c=c^*}$
\textbf{Return:} $\bm{\lambda^*,\sigma^*,c^*}$
\end{algorithmic}
\end{algorithm}
\section{EXPERIMENTAL RESULTS}
In this section, we present experimental results to demonstrate the desirable performance of learning methods under the proposed MMKCC criterion. Without explicit mention, the dimension number is $m=2$, the regularization parameter is $\eta = 10^{-4}$ and the iteration number $S$ is $S=3$.
\subsection{Linear Regression}
First, we consider a simple linear regression example where the input-target samples are generated by a two-dimensional linear system: ${t_i} = {\bm{\beta}^*}^T{\bm{x}_i} + {\rho _i}$, where ${\bm{\beta}^*} = {[1,2]^T}$ is the weight vector to be estimated, and ${\rho _i}$ denotes an additive noise. The input samples $\{ {\bm{x}_i}\}$ are uniformly distributed over $[ - 2.0,2.0] \times [ - 2.0,2.0]$. The noise ${\rho _i}$ comprises two mutually independent noises, namely the inner noise ${B _i}$ and the outlier noise ${O_i}$. Specifically, ${\rho _i}$ is given by $\rho {}_i = {\rm{(}}1 - {g_i}{\rm{)}}{B_i} + {g_i}{O_i}$, where ${g _i}$ is a binary variable with probability mass $\Pr {\rm{\{ }}{g_i} = 1{\rm{\} }} = p$, $\Pr {\rm{\{ }}{g_i} = 0{\rm{\} }} = 1 - p$, $(0 \le p \le 1)$, which is assumed to be independent of both $B_i$ and $O_i$. In this example, $p$ is set at $0.1$, and the outlier $O_i$ is drawn from a zero-mean Gaussian distribution with variance $10000$. As for the inner noise $B_i$, we consider three cases: 1) $B_i \sim$ 0.5$\mathcal{N}$(4.0,1.0)+0.5$\mathcal{N}$(-4.0,1.0), where $\mathcal{N}(u,\sigma^2)$ denotes the Gaussian density function with mean $u$ and variance $\sigma^2$ ; 2) $B_i \sim$ 1/3$\mathcal{N}$(5.0,1.0)+2/3$\mathcal{N}$(-2.0,1.0). 3) $B_i \sim$ 0.5$\mathcal{N}$(0,1.0)+0.5$\mathcal{N}$(0,5.0). The root mean squared error (RMSE) is employed to measure the performance, computed by $RMSE = \sqrt {\frac{1}{2}{{\left\| {{\bm{\beta}_k} - {\bm{\beta}^*}} \right\|}^2}}$, where $\bm{\beta}_k$ and $\bm{\beta}^*$ denote the estimated and the target weight vectors respectively.
We compare the performance of four learning criteria, namely MSE, MCC, MMCC and MMKCC. For MSE, there is a closed-form solution, so no iteration is needed. For MCC, MMCC and MMKCC, a fixed-point iteration is used to solve the model (see the \textbf{Algorithm 1} for the fixed-point algorithm under MMKCC). The mean $\pm$ deviation results of the RMSEs and computing times over 100 Monte Carlo runs are presented in Table I. In the simulation, the sample number is $N$ = 400, the fixed-point iteration number is $K$ = 10, and the initial weight vector is set to ${\beta _0} = {[{\rm{0}},{\rm{0}}]^T}$. For each learning criterion, the parameters are experimentally selected to achieve the best results, except that the free parameters of MMKCC are determined by \textbf{Algorithm 2}. The finite set ${\bm{\Omega} _\sigma}$ in \textbf{Algorithm 2} is equally spaced over [0.1, 2.0] with step size 0.2. The simulations are carried out with MATLAB 8.6 running in Core 4 Quad, 3.4-GHZ CPU with 20-GB RAM. From Table I, we observe: i) MCC, MMCC and MMKCC can significantly outperform MSE although they have no closed-form solution; ii) MMKCC can achieve better performance than MCC and MMCC especially for noises with multi-peak or asymmetric distributions; iii) although the MMKCC is computationally more expensive than MCC and MMCC, the computing times of three learning criteria are in the same order of magnitude. For the noise case 2), the error distributions and multi-Gaussian kernel functions (determined by \textbf{Algorithm 2} after each fixed-point iteration) at different fixed-point iterations under MMKCC are shown in Fig.3. As expected, the multi-Gaussian kernel function matches the error distribution very well at every iteration (as discussed earlier, the free parameters in MMKCC have been optimized to minimize the Euclidean distance between the multi-Gaussian kernel function and the error PDF). The average RMSE convergence curves of three learning criteria are illustrated in Fig. 4.
\begin{figure*}[htbp]
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
%\makeatletter
%\def\@captype{figure}
%\makeatother
%\begin{minipage}[t]{0.5\linewidth}
\centering
\subfigure[]{
\includegraphics[width=2.5in,height=1.9in]{itera.eps}}
\subfigure[]{
\includegraphics[width=2.5in,height=1.9in]{iterb.eps}}
\subfigure[]{
\includegraphics[width=2.5in,height=1.9in]{iterc.eps}}
\subfigure[]{
\includegraphics[width=2.5in,height=1.9in]{iterd.eps}}
\caption{Error distributions and multi-Gaussian kernel functions at different fixed-point iterations: (a) first iteration; (b)second iteration, (c)third iteration, (d)fourth iteration}
\label{fig2}
\end{figure*}
\begin{table*}[!htbp]
\centering
\caption{ RMSEs and computing times(sec) of different learning criteria}\label{RMSE}
\begin{tabular}{ccccccc}
\hline
& &MSE&MCC&MMCC&MMKCC \\
\hline
\multirow{2}{*}{case 1)}&RMSE& $0.5427\pm0.3175$ &0.0881$\pm0.0431$ &$0.0831\pm0.0375$ &$\textbf{0.0342}\pm\textbf{0.0259}$\\
&TIME(sec)&N/A&$0.0832\pm0.0020$ &$0.1027\pm0.0026$&$0.3328\pm0.0070$\\
\multirow{2}{*}{case 2)}&RMSE& $0.5031\pm0.2483$ &$0.0754\pm0.0414$ &$0.0674\pm0.0334$ &$\textbf{0.0224}\pm\textbf{0.0115}$\\
&TIME(sec)&N/A&$0.0814\pm0.0018$ &$0.1014\pm0.0024$&$0.3415\pm0.0075$\\
\multirow{2}{*}{case 3)}&RMSE& $0.5494\pm0.3418$ &$0.0391\pm0.0191$ &$0.0353\pm0.0176$ &$\textbf{0.0335}\pm\textbf{0.0168}$\\
&TIME(sec)&N/A&$0.0841\pm0.0022$ &$0.1021\pm0.0027$&$0.3297\pm0.0068$\\
\hline
\end{tabular}
\end{table*}
\begin{figure}[]
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
%\makeatletter
%\def\@captype{figure}
%\makeatother
%\begin{minipage}[t]{0.5\linewidth}
\centering
\includegraphics[width=2.5in,height=2.0in]{converge.eps}
\caption{RMSE convergence curves of different learning criteria}
\label{fig2}
\end{figure}
\subsection{Non-linear regression with benchmark datasets}
In the second example, we show the superior performance of the MMKCC criterion in nonlinear regression with five benchmark data sets from UCI machine learning repository \cite{frank2010uci}. The descriptions of the data sets are given in Table II. In the experiment, the training and testing samples from each data set are randomly chosen and the data values are normalized into [0, 1.0]. The extreme learning machine (ELM) is adopted as the regression model to be trained, which is a linear-in-parameter (LIP) model with randomly generated hidden nodes\cite{huang2006extreme,huang2011extreme,19miche2010op,tang2016extreme,kasun2016dimension,yang2016multilayer,yang2015data,huang2017efficient,hossain2017application}. Under the MMKCC, the model is trained by the fixed-point iterative algorithm in \textbf{Algorithm 1} and we call it the ELM-MMKCC algorithm. In this example, the performance of the ELM-MMKCC is compared with that of several other ELM based algorithms, including RELM\cite{huang2012extreme}, ELM-RCC\cite{xing2013training} and ELM-MMCC\cite{chen2018mixture}, where ELM-RCC and ELM-MMCC are developed based on MCC and MMCC respectively. The parameters of each algorithm are selected through fivefold cross-validation, except that the free parameters of MMKCC are determined by \textbf{Algorithm 2}. The finite set ${\bm{\Omega} _\sigma}$ in \textbf{Algorithm 2} is equally spaced over [0.1, 3.0] with step size 0.1. The training and testing RMSEs over 100 runs are presented in Table III. Clearly, the ELM-MMKCC outperforms the RELM, ELM-RCC and ELM-MMCC for all the data sets.
\begin{table}[]
\renewcommand\arraystretch{1.5}
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
\centering
\caption{Specification of the datasets}
\begin{tabular}{cccc}
\toprule
\multirow{2}*{Datasets} & \multirow{2}*{Features} & \multicolumn{2}{c}{Observations}\\
\cline{3-4}
&{}&Training&Testing\\
\midrule
Servo &5 &83 &83\\
Slump &10 &52 &51\\
Concrete &9 &515 &515\\
Airfoil &5 &751 &751\\
Yacht &6 &154 &154\\
\bottomrule
\end{tabular}
\end{table}
\begin{table*}[]\scriptsize
\renewcommand\arraystretch{1.5}
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{5pt}
\centering
\caption{ Training and testing RMSEs of several ELM algorithms}
\begin{tabular}{p{1cm}p{1.5cm}p{1.5cm}p{1.5cm}p{1.5cm}p{1.5cm}p{1.5cm}p{1.5cm}p{1.5cm}}
\toprule
\multicolumn{1}{c}{\multirow{2}*{Datasets}} & \multicolumn{2}{c}{RELM}& \multicolumn{2}{c}{ELM-RCC}& \multicolumn{2}{c}{ELM-MMCC}& \multicolumn{2}{c}{ELM-MMKCC}\\
\cline{2-9}
&\multicolumn{1}{c}{Training RMSE} &\multicolumn{1}{c}{Testing RMSE}&\multicolumn{1}{c}{Training RMSE} &\multicolumn{1}{c}{Testing RMSE} &\multicolumn{1}{c}{Training RMSE} &\multicolumn{1}{c}{Testing RMSE} &\multicolumn{1}{c}{Training RMSE} &\multicolumn{1}{c}{Testing RMSE}\\
Servo &$0.0556\pm0.0102$&$0.1064\pm0.0176$ &$0.0714\pm0.0119$&$0.1054\pm0.0159$&$0.0781\pm0.0194$&$0.1024\pm0.0182$&$0.0783\pm0.0236$&$\textbf{0.0997}\pm\textbf{0.0188}$\\
Slump &$0.0081\pm0.0013$&$0.0455\pm0.0099$ &$0.0000\pm0.0000$&$0.0435\pm0.0102$&$0.0000\pm0.0000$&$0.0432\pm0.0112$&$0.0003\pm0.0000$&$\textbf{0.0411}\pm\textbf{0.0098}$\\
Concrete&$0.0734\pm0.0023$&$0.0919\pm0.0044$ &$0.0559\pm0.0018$&$0.0879\pm0.0070$&$0.0557\pm0.0019$&$0.0874\pm0.0068$&$0.0567\pm0.0019$&$\textbf{0.0859}\pm\textbf{0.0056}$\\
Airfoil &$0.0924\pm0.0023$&$0.0991\pm0.0029$ &$0.0796\pm0.0023$&$0.0906\pm0.0033$&$0.0796\pm0.0025$&$0.0905\pm0.0032$&$0.0735\pm0.0022$&$\textbf{0.0895}\pm\textbf{0.0051}$\\
Yacht &$0.0300\pm0.0021$&$0.0487\pm0.0068$ &$0.0125\pm0.0008$&$0.0333\pm0.0082$&$0.0125\pm0.0007$&$0.0330\pm0.0074$&$0.0105\pm0.0007$&$\textbf{0.0311}\pm\textbf{0.0092}$\\
\bottomrule
\end{tabular}
\end{table*}
\begin{table*}[]
\centering
\caption{ Testing RMSEs of TDNNs trained under different criteria}\label{RMSE}
\begin{tabular}{cccccc}
\hline
&MSE&MCC&MMCC&MMKCC \\
\hline
RMSE& 0.0427&0.0309&0.0302&\textbf{0.0277}\\
\hline
\end{tabular}
\end{table*}
\subsection{Chaotic time series prediction}
In the third example, we apply different learning criteria (MSE, MCC, MMCC, MMKCC) to train a time delay neural network (TDNN)\cite{waibel1995phoneme} to predict the Mackey-Glass chaotic time series\cite{kuo1993nonlinear}. The TDNN has a single hidden layer and six nonlinear processing elements in the hidden layer, and its inputs consist of six delayed values. A sigmoid nonlinearity was used in each of the hidden processing elements, while the output processing element was linear. The sequence for training has an additive noise, and the training samples are generated by
\begin{equation}
\label{eq26}
\begin{aligned}
x(t) = - bx(t - 1) + \frac{{ax(t - \tau )}}{{1 + x{{(t - \tau )}^{10}}}} + {\rho _t}
\end{aligned}
\end{equation}
with $b=0.1$, $a=0.2$,$\tau {\rm{ = 30}}$ and ${\rho _t}\sim 0.45\mathcal{N}(-0.05,0.05) + 0.45\mathcal{N}(0.05,0.05)+0.1\mathcal{N}(0,0.2)$. The TDNN is trained to predict the next sample of the time series by using six previous samples, with a segment of 200 samples. The trained networks are tested on clean data set (without additive noise) of length 1000. The kernel size of MCC is experimentally set at $\sigma=2.0$, and the kernel sizes of MMCC are ${\sigma _1} = {\rm{1}}.0$, ${\sigma _2} = {\rm{2}}.0$, and the mixture coefficient in MMCC is $\alpha = 0.8$. For MMKCC, the finite set ${\bm{\Omega} _\sigma}$ is equally spaced over [0.1, 3.0] with step size 0.1. The PDFs of the testing error averaged over 10 Monte Carlo runs are illustrated in Fig.5 and the corresponding testing RMSEs are presented in Table IV. Evidently, the TDNN trained under MMKCC achieves the best performance with the most concentrated error distribution and the lowest RMSE.
\begin{figure}[]
\setlength{\abovecaptionskip}{0pt}
\setlength{\belowcaptionskip}{0pt}
%\makeatletter
%\def\@captype{figure}
%\makeatother
%\begin{minipage}[t]{0.5\linewidth}
\centering
\includegraphics[width=2.4in,height=1.8in]{tdnn.eps}
\caption{Testing error PDFs of TDNNs trained under different criteria}
\label{fig2}
\end{figure}
\section{Conclusion}
A new generalized version of correntropy, called multi-kernel correntropy (MKC), was proposed in this study, where the kernel function is a mixture Gaussian kernel with different widths and centers. The original correntropy and the recently proposed mixture correntropy are both special cases of the new definition. Some important properties of the MKC were presented. In addition, a novel approach was proposed to determine the free parameters of MKC when used in supervised learning. The superior performance of the proposed learning method has been confirmed by experimental results of linear regression, nonlinear regression with benchmark datasets and chaotic time series prediction.
% Can use something like this to put references on a page
% by themselves when using endfloat and the captionsoff option.
\ifCLASSOPTIONcaptionsoff
\newpage
\fi
\bibliographystyle{IEEEtran}
\bibliography{MMKCC}
\end{document}
% Generated by IEEEtran.bst, version: 1.13 (2008/09/30)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{lin1990adaptive}
J.-H. Lin, T.~M. Sellke, and E.~J. Coyle, ``Adaptive stack filtering under the
mean absolute error criterion,'' \emph{IEEE transactions on acoustics,
speech, and signal processing}, vol.~38, no.~6, pp. 938--954, 1990.
\bibitem{coyle1988stack}
E.~J. Coyle and J.-H. Lin, ``Stack filters and the mean absolute error
criterion,'' \emph{IEEE Transactions on Acoustics, Speech, and Signal
Processing}, vol.~36, no.~8, pp. 1244--1254, 1988.
\bibitem{2pei1994least}
S.-C. Pei and C.-C. Tseng, ``Least mean p-power error criterion for adaptive
fir filter,'' \emph{IEEE Journal on Selected Areas in Communications},
vol.~12, no.~9, pp. 1540--1547, 1994.
\bibitem{liu2006error}
W.~Liu, P.~Pokharel, and J.~Principe, ``Error entropy, correntropy and
m-estimation,'' in \emph{2006 16th IEEE Signal Processing Society Workshop on
Machine Learning for Signal Processing}.\hskip 1em plus 0.5em minus
0.4em\relax IEEE, 2006, pp. 179--184.
\bibitem{sayin2014novel}
M.~O. Sayin, N.~D. Vanli, and S.~S. Kozat, ``A novel family of adaptive
filtering algorithms based on the logarithmic cost,'' \emph{IEEE Transactions
on signal processing}, vol.~62, no.~17, pp. 4411--4424, 2014.
\bibitem{chen2012recursive}
X.~Chen, J.~Yang, J.~Liang, and Q.~Ye, ``Recursive robust least squares support
vector regression based on maximum correntropy criterion,''
\emph{Neurocomputing}, vol.~97, pp. 63--73, 2012.
\bibitem{13feng2015learning}
Y.~Feng, X.~Huang, L.~Shi, Y.~Yang, and J.~A. Suykens, ``Learning with the
maximum correntropy criterion induced losses for regression.'' \emph{Journal
of Machine Learning Research}, vol.~16, pp. 993--1034, 2015.
\bibitem{xu2018robust}
G.~Xu, B.-G. Hu, and J.~C. Principe, ``Robust c-loss kernel classifiers,''
\emph{IEEE transactions on neural networks and learning systems}, vol.~29,
no.~3, pp. 510--522, 2018.
\bibitem{syed2012correntropy}
M.~N. Syed, J.~C. Principe, and P.~M. Pardalos, ``Correntropy in data
classification,'' in \emph{Dynamics of Information Systems: Mathematical
Foundations}.\hskip 1em plus 0.5em minus 0.4em\relax Springer, 2012, pp.
81--117.
\bibitem{singh2014c}
A.~Singh, R.~Pokharel, and J.~Principe, ``The c-loss function for pattern
classification,'' \emph{Pattern Recognition}, vol.~47, no.~1, pp. 441--453,
2014.
\bibitem{shi2018training}
W.~Shi, Y.~Gong, X.~Tao, and N.~Zheng, ``Training dcnn by combining max-margin,
max-correlation objectives, and correntropy loss for multilabel image
classification,'' \emph{IEEE transactions on neural networks and learning
systems}, vol.~29, no.~7, pp. 2896--2908, 2018.
\bibitem{he2011robust}
R.~He, B.-G. Hu, W.-S. Zheng, and X.-W. Kong, ``Robust principal component
analysis based on maximum correntropy criterion,'' \emph{IEEE Transactions on
Image Processing}, vol.~20, no.~6, pp. 1485--1494, 2011.
\bibitem{zhou2017maximum}
N.~Zhou, Y.~Xu, H.~Cheng, Z.~Yuan, and B.~Chen, ``Maximum correntropy criterion
based sparse subspace learning for unsupervised feature selection,''
\emph{IEEE Transactions on Circuits and Systems for Video Technology}, 2017.
\bibitem{zhao2011kernel}
S.~Zhao, B.~Chen, and J.~C. Principe, ``Kernel adaptive filtering with maximum
correntropy criterion,'' in \emph{The 2011 International Joint Conference on
Neural Networks}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2011, pp.
2012--2017.
\bibitem{chen2016generalized}
B.~Chen, L.~Xing, H.~Zhao, N.~Zheng, and J.~C. Principe, ``Generalized
correntropy adaptive filtering,'' \emph{IEEE Transactions on Signal
Processing}, vol.~64, no.~13, pp. 3376--3387, 2016.
\bibitem{ma2015maximum}
W.~Ma, H.~Qu, G.~Gui, L.~Xu, J.~Zhao, and B.~Chen, ``Maximum correntropy
criterion based sparse adaptive filtering algorithms for robust channel
estimation under non-gaussian environments,'' \emph{Journal of the Franklin
Institute}, vol. 352, no.~7, pp. 2708--2727, 2015.
\bibitem{wu2015robust}
Z.~Wu, S.~Peng, B.~Chen, and H.~Zhao, ``Robust hammerstein adaptive filtering
under maximum correntropy criterion,'' \emph{Entropy}, vol.~17, no.~10, pp.
7149--7166, 2015.
\bibitem{wu2015kernel}
Z.~Wu, J.~Shi, X.~Zhang, W.~Ma, B.~Chen, and I.~Senior~Member, ``Kernel
recursive maximum correntropy,'' \emph{Signal Processing}, vol. 117, pp.
11--16, 2015.
\bibitem{cinar2012hidden}
G.~T. Cinar and J.~C. Pr{\'\i}ncipe, ``Hidden state estimation using the
correntropy filter with fixed point update and adaptive kernel size,'' in
\emph{Neural Networks (IJCNN), The 2012 International Joint Conference
on}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2012, pp. 1--6.
\bibitem{liu2017maximum}
X.~Liu, B.~Chen, B.~Xu, Z.~Wu, and P.~Honeine, ``Maximum correntropy unscented
filter,'' \emph{International Journal of Systems Science}, vol.~48, no.~8,
pp. 1607--1615, 2017.
\bibitem{liu2007correntropy}
W.~Liu, P.~P. Pokharel, and J.~C. Pr{\'\i}ncipe, ``Correntropy: Properties and
applications in non-gaussian signal processing,'' \emph{IEEE Transactions on
Signal Processing}, vol.~55, no.~11, pp. 5286--5298, 2007.
\bibitem{chen2018mixture}
B.~Chen, X.~Wang, N.~Lu, S.~Wang, J.~Cao, and J.~Qin, ``Mixture correntropy for
robust learning,'' \emph{Pattern Recognition}, vol.~79, pp. 318--327, 2018.
\bibitem{chen2017maximum}
B.~Chen, X.~Liu, H.~Zhao, and J.~C. Principe, ``Maximum correntropy kalman
filter,'' \emph{Automatica}, vol.~76, pp. 70--77, 2017.
\bibitem{wang2017maximum}
F.~Wang, Y.~He, S.~Wang, and B.~Chen, ``Maximum total correntropy adaptive
filtering against heavy-tailed noises,'' \emph{Signal Processing}, vol. 141,
pp. 84--95, 2017.
\bibitem{principe2010information}
J.~C. Principe, \emph{Information theoretic learning: Renyi's entropy and
kernel perspectives}.\hskip 1em plus 0.5em minus 0.4em\relax Springer Science
\& Business Media, 2010.
\bibitem{santamaria2002adaptive}
I.~Santamar{\'\i}a, C.~Pantale{\'o}n, L.~Vielva, and J.~C. Principe, ``Adaptive
blind equalization through quadratic pdf matching,'' in \emph{2002 11th
European Signal Processing Conference}.\hskip 1em plus 0.5em minus
0.4em\relax IEEE, 2002, pp. 1--4.
\bibitem{heravi2018new}
A.~R. Heravi and G.~A. Hodtani, ``A new information theoretic relation between
minimum error entropy and maximum correntropy,'' \emph{IEEE Signal Processing
Letters}, vol.~25, no.~7, pp. 921--925, 2018.
\bibitem{frank2010uci}
A.~Frank and A.~Asuncion, ``Uci machine learning repository [http://archive.
ics. uci. edu/ml]. irvine, ca: University of california,'' \emph{School of
information and computer science}.
\bibitem{huang2006extreme}
G.-B. Huang, Q.-Y. Zhu, and C.-K. Siew, ``Extreme learning machine: theory and
applications,'' \emph{Neurocomputing}, vol.~70, no.~1, pp. 489--501, 2006.
\bibitem{huang2011extreme}
G.-B. Huang, D.~H. Wang, and Y.~Lan, ``Extreme learning machines: a survey,''
\emph{International journal of machine learning and cybernetics}, vol.~2,
no.~2, pp. 107--122, 2011.
\bibitem{19miche2010op}
Y.~Miche, A.~Sorjamaa, P.~Bas, O.~Simula, C.~Jutten, and A.~Lendasse, ``Op-elm:
optimally pruned extreme learning machine,'' \emph{IEEE transactions on
neural networks}, vol.~21, no.~1, pp. 158--162, 2010.
\bibitem{tang2016extreme}
J.~Tang, C.~Deng, and G.-B. Huang, ``Extreme learning machine for multilayer
perceptron,'' \emph{IEEE transactions on neural networks and learning
systems}, vol.~27, no.~4, pp. 809--821, 2016.
\bibitem{kasun2016dimension}
L.~L.~C. Kasun, Y.~Yang, G.-B. Huang, and Z.~Zhang, ``Dimension reduction with
extreme learning machine,'' \emph{IEEE Transactions on Image Processing},
vol.~25, no.~8, pp. 3906--3918, 2016.
\bibitem{yang2016multilayer}
Y.~Yang and Q.~J. Wu, ``Multilayer extreme learning machine with subnetwork
nodes for representation learning,'' \emph{IEEE transactions on cybernetics},
vol.~46, no.~11, pp. 2570--2583, 2016.
\bibitem{yang2015data}
Y.~Yang, Q.~J. Wu, Y.~Wang, K.~Zeeshan, X.~Lin, and X.~Yuan, ``Data partition
learning with multiple extreme learning machines,'' \emph{IEEE transactions
on cybernetics}, vol.~45, no.~8, pp. 1463--1475, 2015.
\bibitem{huang2017efficient}
Z.~Huang, Y.~Yu, J.~Gu, and H.~Liu, ``An efficient method for traffic sign
recognition based on extreme learning machine,'' \emph{IEEE transactions on
cybernetics}, vol.~47, no.~4, pp. 920--933, 2017.
\bibitem{hossain2017application}
M.~Hossain, S.~Mekhilef, M.~Danesh, L.~Olatomiwa, and S.~Shamshirband,
``Application of extreme learning machine for short term output power
forecasting of three grid-connected pv systems,'' \emph{Journal of Cleaner
Production}, vol. 167, no.~2, pp. 395--405, 2017.
\bibitem{huang2012extreme}
G.-B. Huang, H.~Zhou, X.~Ding, and R.~Zhang, ``Extreme learning machine for
regression and multiclass classification,'' \emph{IEEE Transactions on
Systems, Man, and Cybernetics, Part B (Cybernetics)}, vol.~42, no.~2, pp.
513--529, 2012.
\bibitem{xing2013training}
H.-J. Xing and X.-M. Wang, ``Training extreme learning machine via regularized
correntropy criterion,'' \emph{Neural Computing and Applications}, vol.~23,
no. 7-8, pp. 1977--1986, 2013.
\bibitem{waibel1995phoneme}
A.~Waibel, T.~Hanazawa, G.~Hinton, K.~Shikano, and K.~J. Lang, ``Phoneme
recognition using time-delay neural networks,'' \emph{Backpropagation:
Theory, Architectures and Applications}, pp. 35--61, 1995.
\bibitem{kuo1993nonlinear}
J.-M. Kuo, ``Nonlinear dynamic modeling with artificial neural networks,''
Ph.D. dissertation, Citeseer, 1993.
\end{thebibliography}