Unsupervised Learning for Cell-level Visual Representation in Histopathology Images with Generative Adversarial Networks
Extracted source text · arXiv 1711.11317
%% bare_jrnl.tex
%% V1.3
%% 2007/01/11
%% by Michael Shell
%% see http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.7 or later) with an IEEE journal paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/tex-archive/macros/latex/contrib/IEEEtran/
%% and
%% http://www.ieee.org/
% *** Authors should verify (and, if needed, correct) their LaTeX system ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. IEEE's font choices can trigger bugs that do ***
% *** not appear when using other class files. ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/
%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE!
%% User assumes all risk.
%% In no event shall IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including **
%% ** renaming them and changing author support contact information. **
%%
%% File list of work: IEEEtran.cls, IEEEtran_HOWTO.pdf, bare_adv.tex,
%% bare_conf.tex, bare_jrnl.tex, bare_jrnl_compsoc.tex
%%*************************************************************************
% Note that the a4paper option is mainly intended so that authors in
% countries using A4 can easily print to A4 and see how their papers will
% look in print - the typesetting of the document will not typically be
% affected with changes in paper size (but the bottom and side margins will).
% Use the testflow package mentioned above to verify correct handling of
% both paper sizes by the user's LaTeX system.
%
% Also note that the "draftcls" or "draftclsnofoot", not "draft", option
% should be used if it is desired that the figures are to be displayed in
% draft mode.
%
\documentclass[journal]{IEEEtran}
\usepackage{blindtext}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{dsfont}
\usepackage{multirow}
\usepackage{amsmath}
\usepackage{float}
\usepackage{subfigure}
%\usepackage{caption}
\usepackage[section]{placeins}
\usepackage{array}
\usepackage{colortbl}
\usepackage{color}
\newcolumntype{L}[1]{>{\raggedright\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcolumntype{C}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcolumntype{R}[1]{>{\raggedleft\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\bibliographystyle{ieeetr}
% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)
% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
% % pdf code
% \else
% % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/tex-archive/macros/latex/contrib/oberdiek/
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.
% *** CITATION PACKAGES ***
%
%\usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 4.0 (2003-05-27) and later if using hyperref.sty. cite.sty does
% not currently provide for hyperlinked citations.
% The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/cite/
% The documentation is contained in the cite.sty file itself.
% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
% \usepackage[pdftex]{graphicx}
% declare the path(s) where your graphic files are
% \graphicspath{{../pdf/}{../jpeg/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
% or other class option (dvipsone, dvipdf, if not using dvips). graphicx
% will default to the driver specified in the system graphics.cfg if no
% driver is specified.
% \usepackage[dvips]{graphicx}
% declare the path(s) where your graphic files are
% \graphicspath{{../eps/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
% \DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/graphics/
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found as epslatex.ps or
% epslatex.pdf at: http://www.ctan.org/tex-archive/info/
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex
% *** MATH PACKAGES ***
%
%\usepackage[cmex10]{amsmath}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics. If using
% it, be sure to load this package with the cmex10 option to ensure that
% only type 1 fonts will utilized at all point sizes. Without this option,
% it is possible that some math symbols, particularly those within
% footnotes, will be rendered in bitmap form which will result in a
% document that can not be IEEE Xplore compliant!
%
% Also, note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/
% *** SPECIALIZED LIST PACKAGES ***
%
%\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithms/
% There is also a support site at:
% http://algorithms.berlios.de/index.html
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/tex-archive/macros/latex/contrib/algorithmicx/
% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/required/tools/
%\usepackage{mdwmath}
%\usepackage{mdwtab}
% Also highly recommended is Mark Wooding's extremely powerful MDW tools,
% especially mdwmath.sty and mdwtab.sty which are used to format equations
% and tables, respectively. The MDWtools set is already installed on most
% LaTeX systems. The lastest version and documentation is available at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/mdwtools/
% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.
%\usepackage{eqparbox}
% Also of notable interest is Scott Pakin's eqparbox package for creating
% (automatically sized) equal width boxes - aka "natural width parboxes".
% Available at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/eqparbox/
%\usepackage[tight,footnotesize]{subfigure}
% *** SUBFIGURE PACKAGES ***
%\usepackage[tight,footnotesize]{subfigure}
% subfigure.sty was written by Steven Douglas Cochran. This package makes it
% easy to put subfigures in your figures. e.g., "Figure~1a and 1b". For IEEE
% work, it is a good idea to load it with the tight package option to reduce
% the amount of white space around the subfigures. subfigure.sty is already
% installed on most LaTeX systems. The latest version and documentation can
% be obtained at:
% http://www.ctan.org/tex-archive/obsolete/macros/latex/contrib/subfigure/
% subfigure.sty has been superceeded by subfig.sty.
%\usepackage[caption=false]{caption}
%\usepackage[font=footnotesize]{subfig}
% subfig.sty, also written by Steven Douglas Cochran, is the modern
% replacement for subfigure.sty. However, subfig.sty requires and
% automatically loads Axel Sommerfeldt's caption.sty which will override
% IEEEtran.cls handling of captions and this will result in nonIEEE style
% figure/table captions. To prevent this problem, be sure and preload
% caption.sty with its "caption=false" package option. This is will preserve
% IEEEtran.cls handing of captions. Version 1.3 (2005/06/28) and later
% (recommended due to many improvements over 1.2) of subfig.sty supports
% the caption=false option directly:
%\usepackage[caption=false,font=footnotesize]{subfig}
%
% The latest version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/subfig/
% The latest version and documentation of caption.sty can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/caption/
% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure. The latest version and documentation can be found at:
% http://www.ctan.org/tex-archive/macros/latex/base/
%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/sttools/
% Documentation is contained in the stfloats.sty comments as well as in the
% presfull.pdf file. Do not use the stfloats baselinefloat ability as IEEE
% does not allow \baselineskip to stretch. Authors submitting work to the
% IEEE should note that IEEE rarely uses double column equations and
% that authors should try to avoid such use. Do not be tempted to use the
% cuted.sty or midfloat.sty packages (also by Sigitas Tolusis) as IEEE does
% not format its papers in such ways.
%\ifCLASSOPTIONcaptionsoff
% \usepackage[nomarkers]{endfloat}
% \let\MYoriglatexcaption\caption
% \renewcommand{\caption}[2][\relax]{\MYoriglatexcaption[#2]{#2}}
%\fi
% endfloat.sty was written by James Darrell McCauley and Jeff Goldberg.
% This package may be useful when used in conjunction with IEEEtran.cls'
% captionsoff option. Some IEEE journals/societies require that submissions
% have lists of figures/tables at the end of the paper and that
% figures/tables without any captions are placed on a page by themselves at
% the end of the document. If needed, the draftcls IEEEtran class option or
% \CLASSINPUTbaselinestretch interface can be used to increase the line
% spacing as well. Be sure and use the nomarkers option of endfloat to
% prevent endfloat from "marking" where the figures would have been placed
% in the text. The two hack lines of code above are a slight modification of
% that suggested by in the endfloat docs (section 8.3.1) to ensure that
% the full captions always appear in the list of figures/tables - even if
% the user used the short optional argument of \caption[]{}.
% IEEE papers do not typically make use of \caption[]'s optional argument,
% so this should not be an issue. A similar trick can be used to disable
% captions of packages such as subfig.sty that lack options to turn off
% the subcaptions:
% For subfig.sty:
% \let\MYorigsubfloat\subfloat
% \renewcommand{\subfloat}[2][\relax]{\MYorigsubfloat[]{#2}}
% For subfigure.sty:
% \let\MYorigsubfigure\subfigure
% \renewcommand{\subfigure}[2][\relax]{\MYorigsubfigure[]{#2}}
% However, the above trick will not work if both optional arguments of
% the \subfloat/subfig command are used. Furthermore, there needs to be a
% description of each subfigure *somewhere* and endfloat does not add
% subfigure captions to its list of figures. Thus, the best approach is to
% avoid the use of subfigure captions (many IEEE journals avoid them anyway)
% and instead reference/explain all the subfigures within the main caption.
% The latest version of endfloat.sty and its documentation can obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/endfloat/
%
% The IEEEtran \ifCLASSOPTIONcaptionsoff conditional can also be used
% later in the document, say, to conditionally put the References on a
% page by themselves.
% *** PDF, URL AND HYPERLINK PACKAGES ***
%
\usepackage{url}
\urlstyle{same}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version can be obtained at:
% http://www.ctan.org/tex-archive/macros/latex/contrib/misc/
% Read the url.sty source comments for usage information. Basically,
% \url{my_url_here}.
% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex). ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )
% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
%
% paper title
% can use linebreaks \\ within to get better formatting as desired
\title{Unsupervised Learning for Cell-level Visual Representation in Histopathology Images with Generative Adversarial Networks}
\author{Bo Hu$^\sharp$, Ye Tang$^\sharp$, Eric I-Chao Chang, Yubo Fan, Maode Lai and Yan Xu*
\thanks{This work is supported by the Technology and Innovation Commission of Shenzhen in China under Grant shenfagai2016-627, Microsoft Research under the eHealth program, the National Natural Science Foundation in China under Grant 81771910, the National Science and Technology Major Project of the Ministry of Science and Technology in China under Grant 2017YFC0110903, the Beijing Natural Science Foundation in China under Grant 4152033, Beijing Young Talent Project in China, the Fundamental Research Funds for the Central Universities of China under Grant SKLSDE-2017ZX-08 from the State Key Laboratory of Software Development Environment in Beihang University in China, the 111 Project in China under Grant B13003. \emph{* indicates corresponding author; $^\sharp$ indicates equal contribution.}}
\thanks{Bo Hu, Ye Tang, Yubo Fan and Yan Xu are with the State Key Laboratory of Software Development Environment and the Key Laboratory of Biomechanics and Mechanobiology of Ministry of Education and Research Institute of Beihang University in Shenzhen and Beijing Advanced Innovation Centre for Biomedical Engineering, Beihang University, Beijing 100191, China (email: bohu1996@gmail.com; yetang1995@gmail.com; yubofan@buaa.edu.cn; xuyan04@gmail.com).}
\thanks{Maode Lai is with the Department of Pathology, School of Medicine, Zhejiang
University (email: lmd@zju.edu.cn).}
\thanks{Eric I-Chao Chang, and Yan Xu are with Microsoft Research, Beijing 100080, China (email: echang@microsoft.com; v-yanx@microsoft.com).}
}
%
%
% author names and IEEE memberships
% note positions of commas and nonbreaking spaces ( ~ ) LaTeX will not break
% a structure at a ~ so this keeps an author's name from being broken across
% two lines.
% use \thanks{} to gain access to the first footnote area
% a separate \thanks must be used for each paragraph as LaTeX2e's \thanks
% was not built to handle multiple paragraphs
%
% note the % following the last \IEEEmembership and also \thanks -
% these prevent an unwanted space from occurring between the last author name
% and the end of the author line. i.e., if you had this:
%
% \author{....lastname \thanks{...} \thanks{...} }
% ^------------^------------^----Do not want these spaces!
%
% a space would be appended to the last name and could cause every name on that
% line to be shifted left slightly. This is one of those "LaTeX things". For
% instance, "\textbf{A} \textbf{B}" will typeset as "A B" not "AB". To get
% "AB" then you have to do: "\textbf{A}\textbf{B}"
% \thanks is no different in this regard, so shield the last } of each \thanks
% that ends a line with a % and do not let a space in before the next \thanks.
% Spaces after \IEEEmembership other than the last one are OK (and needed) as
% you are supposed to have spaces between the names. For what it is worth,
% this is a minor point as most people would not even notice if the said evil
% space somehow managed to creep in.
% The paper headers
\markboth{}%
{Shell \MakeLowercase{\textit{et al.}}: Bare Demo of IEEEtran.cls for Journals}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the twoside option.
%
% *** Note that you probably will NOT want to include the author's ***
% *** name in the headers of peer review papers. ***
% You can use \ifCLASSOPTIONpeerreview for conditional compilation here if
% you desire.
% If you want to put a publisher's ID mark on the page you can do it like
% this:
%\IEEEpubid{0000--0000/00\$00.00~\copyright~2007 IEEE}
% Remember, if you use this you must call \IEEEpubidadjcol in the second
% column for its text to clear the IEEEpubid mark.
% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}
% make the title area
\maketitle
\begin{abstract}
The visual attributes of cells, such as the nuclear morphology and chromatin openness, are critical for histopathology image analysis. By learning cell-level visual representation, we can obtain a rich mix of features that are highly reusable for various tasks, such as cell-level classification, nuclei segmentation, and cell counting. In this paper, we propose a unified generative adversarial networks architecture with a new formulation of loss to perform robust cell-level visual representation learning in an unsupervised setting. Our model is not only label-free and easily trained but also capable of cell-level unsupervised classification with interpretable visualization, which achieves promising results in the unsupervised classification of bone marrow cellular components. Based on the proposed cell-level visual representation learning, we further develop a pipeline that exploits the varieties of cellular elements to perform histopathology image classification, the advantages of which are demonstrated on bone marrow datasets.
\end{abstract}
\begin{IEEEkeywords}
unsupervised learning, representation learning, generative
adversarial networks, classification, cell.
\end{IEEEkeywords}
\section{Introduction}
\begin{figure}[h]
\centering
\includegraphics[width=0.48\textwidth]{{pathology_compare/cell_type}.png}
\caption{Examples of five types of cellular elements in bone marrow: $\displaystyle (a)$ granulocytes precursors such as myeloblasts, $\displaystyle (b)$ cells with dark, dense, and close phased nuclei, the candidates of which are most likely lymphocytes and normoblasts, $\displaystyle (c)$ granulocytes such as neutrophils, $\displaystyle (d)$ monocytes, and $\displaystyle (e)$ megakaryocytes. Five types of cells can be distinguished by the chromatin openness, the density of nuclei, and if nuclei show the appearance of being segmented. Megakaryocytes appear the least often, as well are the most distinguished due to their massive size.}
\label{fig1}
\end{figure}
\begin{figure}[h]
\centering
\subfigure[abnormal]{ \label{fig:a}
\includegraphics[width=0.22\textwidth]{{pathology_compare/pic1}.png}
}
\subfigure[normal]{ \label{fig:b}
\includegraphics[width=0.22\textwidth]{{pathology_compare/pic2}.png}
}
\caption{Examples of bone marrow images sliced from Whole Slide Images (WSI). Too many myeloblasts in (a) indicate the presence of blood disease.}
\label{fig2}
\end{figure}
\PARstart{H}{istopathology} images are considered to be the gold standard in the diagnosis of many diseases \cite{Gurcan2009histopathological}. In many situations, the cellular components are an important determinant. For example, in the biopsy sections of bone marrow, the abnormal cellular constitution indicates the presence of blood disease \cite{Bennett1976Proposals}. Bone marrow is the key component of both the hematopoietic system and the lymphatic system by producing large amounts of blood cells. The cell lines undergoing maturation in the marrow mostly include myeloid cells (granulocytes, monocytes, megakaryocytes, and their precursors), erythroid cells (normoblasts), and lymphoid cells (lymphocytes and their precursors). Figure~\ref{fig1} are examples of five main cellular components in bone marrow. These components are significant to both the systemic circulation and the immune system. Several kinds of cancer are characterized by the cellular constitution in bone marrow \cite{Bennett1976Proposals}. For instance, too many granulocytes precursors such as myeloblasts indicate the presence of chronic myeloid leukemia. Having large, abnormal lymphocytes heralds the presence of lymphoma. Figure~\ref{fig2} shows the difference between normal and abnormal bone marrow histopathology images from the perspective of cells.
As described above, cell-level information is irreplaceable for histopathology image analysis. Cell-level visual attributes such as the morphological features of nuclei and the openness of chromatin are helpful for various tasks such as cell-level classification and nuclei segmentation. We define cell-level images as the output from nuclei segmentation. Each cell-level image contains only one cell. We opt to perform representation learning on these cell-level images, in which the visual attributes such as the nuclei morphology and chromatin openness are distinguished. The learned features are further utilized to assist tasks such as cell counting to highlight the quantification of certain types of cells.
To achieve this, the main obstacle is the labeling of cells. There are massive amounts of cells in each histopathology image, which makes manual labeling ambiguous and laborious. Therefore, an unsupervised cell-level visual representation learning method based on unlabeled data is believed to be more reasonable than fully supervised methods. Unsupervised cell-level visual representation learning is known to be difficult. First, geometrical and morphological appearances of cells from the same category can have a distinct diversity due to factors such as cell cycles. Furthermore, the staining conditions of histopathology images can be pretty diverse, resulting in inconsistent color characteristics of nuclei and cytoplasm.
Recently, deep learning has been proven to be powerful in histopathology image analysis such as classification \cite{Xu2014Deep,xu2015deep}, segmentation \cite{xu2016deep,Chen2016DCAN}, and detection \cite{chen2014deep,cirecsan2013mitosis}. Generative Adversarial Networks (GANs) \cite{goodfellow2014generative} are a class of generative models that use unlabeled data to perform representation learning. GAN is capable of transforming noise variables into visually appealing image samples by learning a model distribution that imitates the real data distribution. Several GAN architectures such as Deep Convolutional Generative Adversarial Nets (DCGAN) \cite{radford2015unsupervised} have proven their advantages in various natural images datasets. Recently, Wasserstein-GAN (WGAN) \cite{arjovsky2017wasserstein} and WGAN with gradient penalty (WGAN-GP) \cite{gulrajani2017improved} have greatly improved the stability of training GAN. More complex network structures such as residual networks \cite{he2016deep} can now be fused into GAN models.
Meanwhile, Information Maximizing Generative Adversarial Networks (InfoGAN) \cite{chen2016infoGAN} makes a modification that encourages GAN to learn interpretable and meaningful representations. InfoGAN maximizes the mutual information between the chosen random variables and the observations to make variables represent interpretable semantic features. The problem is that InfoGAN utilizes a DCGAN architecture, which requires meticulous attention towards hyperparameters. For our problem, it suffers a severe convergence problem.
%Furthermore, our model is capable of cell-level unsupervised classification. Instead of using classifiers on top of the learned features, we instead conduct the process of maximizing mutual information by training a unified GAN architecture to learn the disentangled representation of cells. Inspired by Information Maximizing Generative Adversarial Nets (InfoGAN) \cite{chen2016infoGAN}, we maximize mutual information between the chosen random noise variables and the generated samples. In this way, we allocate the cell-level images into corresponding categories according to their most significant semantic features.
%A categorical random variable is chosen to be combined with random noises. By changing its value in the test process, generated samples are respectively different type cells. In addition, real data set is allocated to corresponding dimensions using trained model.
%Defining a categorical random factor, we precisely capture the semantic variance between each type of cells. In our experiment, a 5-dimensional categorical random variable is chosen to be combined with random noises. By changing its value in the test process, generated samples are respectively myeloblasts, lymphocytes, and segmented cells such as neutrophils and monocytes. Besides, real data set is allocated to corresponding dimensions using trained model, which achieves 0.89 purity viewed as a clustering task.
%Inspired by Information Maximizing Generative Adversarial Nets (InfoGAN) \cite{chen2016infoGAN}, we utilized a unified GAN architecture to learn a disentangled representation of cells. Different type of white blood cells is distinguished in respect of their sizes, roundness, color characteristics, the number of nucleolar lobes and so on. Maximizing the mutual information between the chosen random variables and the generated samples of GAN, we can allocate the data distribution into different dimensions according to their most significant semantic features. Defining a categorical random factor, we precisely capture the semantic variance between each type of cells. In our experiment, we find this method quite effective. We use a 5-dimensional categorical random variable to constrain the output of the generator. First, by changing its value, generated samples are respectively myeloblasts, lymphocytes, and segmented cells such as neutrophils and monocytes. Sec
Inspired by WGAN-GP and InfoGAN, we present an unsupervised representation learning method for cell-level images using a unified GAN architecture with a new formulation of loss, which inherits the superiority from both WGAN-GP and InfoGAN. We observe great improvements followed by the setting of WGAN-GP. Introducing mutual information into our formulation, we are capable of learning interpretable and disentangled cell-level visual representations, as well as allocate cells into different categories according to their most significant semantic features. Our method achieves promising results in the unsupervised classification of bone marrow cellular components.
Based on the cell-level visual representations, the quantification of each cellular component can be obtained by the trained model. Followed by this, cell proportions for each histopathology image can then be calculated to assist image-level classification. We further develop a pipeline combining cell-level unsupervised classification and nuclei segmentation to conduct image-level classification of histopathology images, which shows its advantages via experimentations on bone marrow datasets.
%To summarize, we propose an unsupervised method for cell-level visual representation learning using a unified GAN architecture. We unsupervised classify cell-level images into corresponding categories according to their most significant semantic features by maximizing mutual information. Meanwhile, interpretable representation for each class can be generated for visualization. Followed by this architecture, we develop a pipeline that combines nuclei segmentation and cell-level visual representation learning to conduct image-level unsupervised classification, which demonstrates a convincing result on bone marrow histopathology images.As described before, cell-level information is clinical in the diagnoses of diseases such as leukemia and lymphoma.
The contributions of this work include the following: (1) We present an unsupervised framework to perform cell-level visual representation learning using generative adversarial networks. (2) A unified GAN architecture with a new formulation of loss is proposed to generate representations that are both high-quality and interpretable, which also endows our model the capability of cell-level unsupervised classification. (3) A pipeline is developed that exploits the varieties of cell-level elements to perform image-level classification of histopathology images.
%In this paper, (1)We propose unsupervised learning of cell-level visual representation with InfoGAN combined WGan. In this way we automatically learn the typical representations of white blood cells (2)We develop a classification of white blood cells method using cell-level visual representation.We first use existing architecture to detect and segment cells. Then we use GAN to simultaneously learn the cell-level visual representation and perform clustering. Simply applying a linear model above the quantities of different types of white blood cells, we make a visualization that cell distribution between normal and abnormal pathology images can have a distinct difference. With the help of a pathologist, we labeled all the cells into four main categories: myeloblasts, lymphocytes, neutrophils, and monocytes. Using proposed model as a classifier, we can achieve a purity of 0.89, which outperforms the k-means method with a purity of 0.69. We also used the trained network as a feature extractor to assist supervised classification, which is proved to outperform the existing manually feature extractor such as HOG.
%In this paper, we propose an unsupervised method for cell-level visual representation learning using generative adversarial networks. Here we learn the interpretable disentangled representation of cells using InfoGAN, as well as perform clustering using the learned distribution. Training a classifier on top of cell proportions of each histopathology image, we conduct image-level classification, which achieves a promising score.
%In this paper, we propose an unsupervised method for cell-level visual representation learning using generative adversarial networks. Here we learn an interpretable disentangled representation of cells using InfoGAN, as well as perform clustering of white blood cells using the trained model. Analyzing the proportions of different clusters of white blood cells from each single large-scale histopathology image, we perform a visualization to show that the cell proportions between normal and abnormal histopathology images have a distinct difference. With the help of a pathologist, we labeled part of our dataset into four main categories: myeloblasts, lymphocytes, neutrophils, and monocytes. Using proposed model as an unsupervised classifier, we achieve 0.89 purity in the clustering task, which outperforms the k-means with a feature extractor method of 0.69 purity. We also used the trained networks as a feature extractor to assist supervised classification, which is proved to exceed manual feature extractors【resnet feature呢】. Based on the result of cell classification, finally we perform a 4-fold cross validation on image level diagnosis and achieve 0.925 accuracy which outperform than other histopathology image classification
%The contributions of our work are as follows: (1) We introduce generative adversarial networks into cell-level visual representation learning. (2) We combine nuclei segmentation method with unsupervised algorithms to achieve promising performance in cell-level clustering. (3) According to the proportions of each cluster of white blood cells, we propose a pipeline to determine whether a bone marrow histopathology image indicates the presence of blood diseases.
%In this paper, We proposed a modified GAN architecture, as well combined it with existing cell detection and segmentation architecture. In this way, we simultaneously learn the typical representations of white blood cells.With the help of a pathologist, we labeled all the cells into four main categories: myeloblasts, lymphocytes, neutrophils and monocytes. Using proposed model as a classifier, we can achieve a purity of 0.89, which outperforms the k-means method with a purity of 0.69. We also used the trained network as a feature extractor to assist supervised classification, which is proved to outperform the existing manually feature extractor such as HOG.
%The contribution of our work includes:(1)We combine Wasserstein generative adversarial networks with gradient penalty and mutual information to perform disentangled representation learning. (2) Base on the cell level, we combines cell segmentation method and deep learning algorithms, achieves state-of-the-art performance in the task of clustering. Simply applying a linear model above the quantity of different types of white blood cells, we make a visualization that cell distribution between normal and abnormal pathology images can have a distinct difference.
%要改 Generative Adversarial Networks(GAN), an unsupervised framework which could generate meaningful representations from the whole dataset, is suitable for our case. Compared with other generative methods such as Boltzmann Machines, GAN generates images that are much closer to the original data distribution. GAN has achieved a tremendous work in the area of natural images, which proves the advantage of GAN among generative models. In the latest research, information Maximizing Generative Adversarial Nets (InfoGAN) introduced mutual information into GAN architecture, which makes unsupervised clustering using GAN possible. Traditional clustering methods like k-means requires an extra feature extractor such as HoG before conducting clustering to images. For InfoGAN, the task of feature extractor is substituted by internal neural networks architecture. In our case, we found that GAN does have a better result of clustering.
%要改 In this paper, we develop a pipeline system applying cell-level visual representation in myeloid leukemia diagnosis. We proposed a modified informatics maximum generative adversarial networks, as well combined it with existing cell detection and segmentation architecture. In this way, we simultaneously learn the typical representations of bone marrow cells and performed an unsupervised clustering of them. We then used the trained network as a feature extractor, which outperforms the existing feature extractor method such as HOG and CNN.
%要改 The contribution of our work includes:(1)We combine Wasserstein generative adversarial networks with gradient penalty and mutual information to perform both cell-level visual representation learning and cell clustering. (2) Base on the cell level, we develop a pipeline that combines cell segmentation method and deep learning and achieves state-of-the-art performance in the task of unsupervised clustering. A visualization can be done to show that normal pathology images and abnormal ones have a huge difference in respect of cell distribution.
\section{related works}
\subsection{Directly Related Works}
\subsubsection{Generative Adversarial Networks}
Goodfellow et al. \cite{goodfellow2014generative} propose GANs, a class of unsupervised generative models consisting of a generator neural network and an adversarial discriminator neural network. While the generator is encouraged to produce synthetic samples, the discriminator learns to discriminate between generated and real samples. This process is described as a minimax game. Radford et al. \cite{radford2015unsupervised} propose one of the most frequently used GAN architectures DCGAN.
Arjovsky et al. \cite{arjovsky2017wasserstein} propose WGAN, which modifies the objective function, securing the training process to be more stable. For regular GANs, the training process optimizes a lower bound of the Jensen-Shannon (JS) divergence between the generator distribution and the real data distribution. WGAN modifies this by optimizing an approximation of the Earth-Mover (EM) distance. The only challenge is how to enforce the Lipschitz constraint on the discriminator. While Arjovsky et al. \cite{arjovsky2017wasserstein} use weight-clipping, Gulrajani et al. \cite{gulrajani2017improved} propose WGAN-GP, which adds a gradient penalty on the discriminator. For our bone marrow datasets, even if we have tried multiple hyperparameters, DCGAN still suffers from a severe convergence difficulty. While DCGAN leads to the failure for our datasets, WGAN-GP greatly eases this problem.
Chen et al. \cite{chen2016infoGAN} introduce mutual information into GAN architecture. Mutual information describes the dependencies between two separate variables. Maximizing mutual information between the chosen random variables and the generated samples, InfoGAN produces representations that are meaningful and interpretable. To exploit the varieties of cellular components, the superior ability of InfoGAN in learning disentangled and discrete representations is what a regular GAN lacks.
Therefore, we propose a unified GAN architecture with a new formulation of loss, which inherits the superiority of both WGAN-GP and InfoGAN. The outstanding stability of WGAN-GP eases the difficulty in tuning the complicated hyperparameters of InfoGAN. Introducing mutual information into our model, we are capable of learning interpretable cell-level visual representations, as well as allocate cells into different categories according to their most significant semantic features.
%GAN allows network training with a wealth of unlabeled data. The analogue to the human learning process and the novel learning ability make it much more data efficient. We can learn to recognize or classify objects and structures without the specific labels, meaning that GAN is more suitable to some specific task, such as representation learning \cite{radford2015unsupervised}. Therefore in our work, we introduce a unified GAN architecture learn the representation of cells.
%For nucleus segmentation, Chiao et al. \cite{Chiao1998Statistics} and Reinhard et al. \cite{Reinhard2001Color} proposed Reinhard color normalization that transforms color characteristics of pathology images into the same standard. Later on, \cite{Macenko2009A} proposed a fancy method for unsupervised color deconvolution. For a two-stain image or matrix in SDA space, this method works by computing a best-fit plane with PCA, wherein it selects the stain vectors as percentiles in the “angle distribution” in that plane.Single image is separated by the RGB values of two stains. Based on these two steps above, we can precisely segment nucleus in bone marrow biopsies.
\subsubsection{Classification of Blood Disease}
%In the diagnosis of Acute Lymphocytic Leukemia (acute lymphocytic leukemia), according to the size, shape and other traits, pathologists always perform the Complete Blood Count (CBC) to classify and measure the amounts of different types of White Blood Cells (white blood cells) under the microscope. Abnormal changes in the percentage of white blood cells, such as high proportion of myeloblasts, suggest leukemia.
%Nazlibilek et al. \cite{Nazlibilek2014Automatic} propose a system to help automatically diagnose acute lymphocytic leukemia. This system consists of several stages: segmenting white blood cells in blood smear images based on intensity thresholding; counting the total number and extracting the features of WBCs; and classifying cells into five different subtypes using neural networks. Though disease diagnosis is out of the scope of their paper, in their future work, they claim that the classification result can be used in the diagnosis of acute lymphocytic leukemia.
Nazlibilek et al. \cite{Nazlibilek2014Automatic} propose a system to help automatically diagnose acute lymphocytic leukemia. This system consists of several stages: nuclei segmentation, feature extraction, cell-level classification, and cell counting. In their future work, they claim that the result of cell counting can be used for further diagnosis of acute lymphocytic leukemia.
In our work, we design a similar workflow which consists of nuclei segmentation, cell-level classification, and image-level classification. Our advantages lie in the novelty of an unsupervised setting and the convincing performance of image-level classification based on the calculated cell proportions.
\subsection {Cell-level Representation}
The representation of individual cells can be used for a variety of tasks such as cell classification. Traditional cell-level visual representation for classification tasks can be categorized into four categories \cite{Y2014Methods}: morphological \cite{Muthu2012Hybrid}, texture \cite{Xu2015Dual,Lorenzo2013Cervical}, intensity \cite{Dundar2011Computerized}, and cytology features \cite{Nguyen2011Prostate}. These traditional methods have been employed in the representation of white blood cells \cite{Tai2011Blood,Putzu2014Leucocyte,Su2014A}. However, the features used above need to be manually designed by experienced experts according to the characteristics of different types of cells. While images suffer from a distinct variance, discovering, characterizing and selecting good handcraft features can be extremely difficult.
To remedy the limitations of manual features in cell classification, Convolutional Neural Network (CNN) learns higher-level latent features, whose convolution layer can act as a feature extractor \cite{xu2017large}. Xie et al. \cite{xie2016unsupervised} propose Deep Embedding Clustering (DEC) that simultaneously learns feature representations and cluster assignments using deep neural networks.
Variational Autoencoder (VAE) \cite{kingma2013auto} serves as a convincing unsupervised strategy in cell-level visual representation learning \cite{Xu2016Stacked,Cruzroa2013A, zhang2016fusing}. However, how to use VAE to learn categorical and discrete latent variables is still under investigation. Dilokthanakul et al. \cite{dilokthanakul2016deep} and Jiang et al. \cite{jiang2017variational} design models combining VAE with Gaussian Mixture Model (GMM). But they demonstrate their experiment on one-dimensional datasets such as MNIST. To perform clustering and embedding on a higher-dimensional dataset, their methods still need a feature extractor.
GANs such as Categorical GAN \cite{springenberg2015unsupervised} can merge categorical variables into the model with little effort, which makes learned representations disentangled and interpretable. This ability is critical in medical image analysis where accountability is especially needed.
\subsection {Cell-level Histopathology Image Analysis}
\subsubsection {Classification}
Cell classification has been performed in diverse histopathology related works such as breast cancer \cite{Malon2013Classification}, acute lymphocytes leukemia \cite{Mohapatra2014An,Zhao2016Automatic}, and colon cancer \cite{Sirinukunwattana2016Locality}.
Based on the result of cell classification, some approaches have been proposed to determine the presence or location of cancer \cite{Nguyen2011Prostate}, \cite{hou2016automatic}. In prostate cancer, Nguyen et al. \cite{Nguyen2011Prostate} innovatively employ cell classification for automatic cancer detection and grading. They distinguish the cancer nuclei and normal nuclei, which are combined with textural features to classify the image as normal or cancerous and then detect and grade the cancer regions. In the diagnosis of Glioma, Hou et al. \cite{hou2016automatic} apply CNN to the classification of morphological attributes of nuclei. They also claim that the nuclei classification result provides clinical information for diagnosing and classifying glioma into subtypes and grades. {Zhang et al. \cite{zhang2015weighted, zhang2015towards, zhang2015high} and Shi et al. \cite{shi2017cell} use either supervised or semi-supervised hashing models for cell-level analysis.}
All of these works require a large amount of accurately annotated data. Obtaining such annotated data is time-consuming and labor-intensive while GAN can optimally leverage the wealth of unlabeled data.
\subsubsection{Segmentation}
Nuclei segmentation is of great importance for cell-level classification. Nuclei segmentation methods can be roughly categorized as follows: intensity thresholding \cite{Callau2015Evaluation}, \cite{Wienert2012Detection}, morphology operation \cite{Dorini2013Semiautomatic}, \cite{Schmitt2009Morphological}, deformable models \cite{Dzyubachyk2010Advanced}, watershed transform \cite{Long2009A}, clustering \cite{Hai2014Automatic}, \cite{Bueno2012A}, and graph-based methods \cite{Chang2013Invariant}, \cite{Arslan2013Attributed}. The methods above have been broadly applied to the segmentation of white blood cells.
\subsection {Generative Adversarial Networks in Medical Images}
Recently, several works involving GAN have gathered great attention in medical image analysis.
In medical image synthesizing, Nie et al. \cite{nie2017medical} estimate the CT image from its corresponding MR image with context-aware GAN. In medical image reconstruction, Li et al. \cite{Li2017Reconstruction} use GAN to reconstruct medical images with the thinner sliced thickness from regular thick-slice images. Mahapatra et al. \cite{Mahapatra2017Image} propose a super resolution method that takes a low-resolution input fundus image to generate a high-resolution super-resolved image. Wolterink et al. \cite{Wolterink2017Generative} employ GAN to reduce the noise in low-dose CT images. All these recent works demonstrate the great potential of GAN in solving complicated medical problems.
\section{Methods}
In this section, we first introduce an unsupervised method for cell-level visual representation learning using GAN. Then we present the details of how image-level classification is performed on histopathology images based on cell-level representation.
\begin{figure}[H]
\subfigure[Training process. Random variables are composed of Gaussian variables $z$ and the discrete variable $c$. Besides playing the minimax game between the generator ($G$) and the discriminator ($D$) through the EM distance, we also minimize the negative Log-likelihood between $c$ and the output of the auxiliary network ($Q(c|G(c,z)$) to maximize mutual information.]{
\label{fig:mini:subfig:train}
\begin{minipage}[b]{0.48\textwidth}
\centering
\includegraphics[width=1\textwidth]{{pipeline/10135}.png}
\end{minipage}
}
\subfigure[Test process. Real samples are classified into five categories by the auxiliary network $Q$. At the same time, fake samples are generated by giving noises with the chosen $c$ for each class. In the example of generated samples (fake), one row contains five samples from the same category in $c$, and a column shows the generated images for 5 possible categories in $c$ with $z$ fixed.]{
\label{fig:mini:subfig:b}
\begin{minipage}[b]{0.48\textwidth}
\centering
\includegraphics[width=0.95\textwidth]{{pipeline/10130}.png}
\end{minipage}}
\subfigure[Illustration of residual blocks (resblocks) in the architecture. There are three different types of residual blocks considering whether they include nearest-neighbor upsampling or mean pooling for downsampling. Batch normalization layers are used in our generator to help stabilize training.]{
\label{fig:mini:subfig:c}
\begin{minipage}[b]{0.48\textwidth}
\centering
\includegraphics[width=0.95\textwidth]{{pipeline/10132}.png}
\end{minipage}}
\caption{Network architecture of our cell-level visual representation learning: (a) Training process. (b) Test process. (c) The architecture of residual blocks (written as resblock in (a) and (b)).}
\label{networks architecture}
\end{figure}
\subsection{Cell-level Visual Representation Learning}
Given cell-level images that come from nuclei segmentation as the real data, we define a generator network $G$, a discriminator network $D$, and an auxiliary network $Q$. The architecture of these networks are shown in Figure~\ref{networks architecture}. In the training process, we learn a generator distribution that matches the real data distribution by playing a minimax game between $G$ and $D$ by optimizing an approximation of the Earth-Mover (EM) distance. Meanwhile, we maximize mutual information between the chosen random variables and the generated samples using an auxiliary network $Q$. In the test process, the generator generates the representations for each category of cells according to different values of the chosen random variables. Cell images can be allocated to the corresponding categories by the auxiliary network $Q$.
\begin{figure*}[t]
\includegraphics[width=1\textwidth]{{pipeline/pipeline2}.png} \caption{Overview of our pipeline as follows: (a) Nuclei segmentation is performed on histopathology images. (b) Using the trained GAN architecture, Cell-level clustering is performed using the learned auxiliary network $Q$. Cell proportions are then calculated for each histopathology image. (c) Image-level prediction is given based on cell proportions. (d) For visualization, the generator $G$ can generate the interpretable representation for each category of cells by changing the noises.}
\label{pipeline}
\end{figure*}
\subsubsection {Training Process}
Given cell-level images sampled from the real data distribution $x\sim \mathbb{P}_r$, the first goal is to learn a generator distribution $\mathbb{P}_g$ that matches the real data distribution $\mathbb{P}_r$.
We first define a random noise variable $z$. The input noise $z$ is transformed by the generator into a sample $\tilde x = G(z), z \sim p(z)$. $\tilde x$ can be viewed as following the generator distribution $\mathbb{P}_g$. Inspired by WGAN \cite{arjovsky2017wasserstein}, we optimize networks through the WGAN objective $W(\mathbb{P}_r, \mathbb{P}_g)$:
\begin{equation}
W(\mathbb{P}_r, \mathbb{P}_g) = \sup_{\lVert f \lVert_{L \leq 1}} \ \mathbb{E}_{x \sim \mathbb{P}_r}[f(x)] - \mathbb{E}_{\tilde x \sim \mathbb{P}_g}[f(\tilde x)].
\end{equation}
$W(\mathbb{P}_r, \mathbb{P}_g)$ is an efficient approximation of the EM distance, which is constructed using the Kantorovich-Rubinstein duality \cite{arjovsky2017wasserstein}. The EM distance measures how close the generator distribution and the data distribution are. To distinguish two distributions $\mathbb{P}_g$ and $\mathbb{P}_r$, the adversarial discriminator network $D$ is trained to learn the function $f$ that maximizes $W(\mathbb{P}_r, \mathbb{P}_g)$. To make $\mathbb{P}_g$ approach $\mathbb{P}_r$, the generator instead is trained to minimize $W(\mathbb{P}_r, \mathbb{P}_g)$. The value function $V(D,G)$ is written as follows:
\begin{equation}
V(D,G) = \mathbb{E}_{x \sim \mathbb{P}_r} [D(x)] - \mathbb{E}_{z \sim p(z)} [D(G(z))].
\end{equation} This minimax game between the generator and the discriminator is written as:
\begin{equation}
\min_{G}\max_{D \in \mathcal{D}} V(D,G).
\end{equation}
Followed by the work of WGAN-GP \cite{gulrajani2017improved}, a gradient penalty is added on the discriminator to enforce the Lipschitz constraint to make sure that the discriminator lies within the space of 1-Lipschitz functions $D\in\mathcal{D}$. The loss of the discriminator with a hyperparameter $\lambda_1$ is written as:
\begin{equation}
\resizebox{1\hsize}{!}{$L_D = \mathbb{E}_{z \sim p(z)} [D(G(z))] - \mathbb{E}_{x \sim \mathbb{P}_r} [D(x)]+ \lambda_1 \mathbb{E}_{\hat x \sim \mathbb{P}_{\hat x}} [ || \nabla_{\hat x} D(\hat x) ||_p - 1 ]^2,$}
\end{equation} where $\mathbb{P}_{\hat x}$ is defined sampling uniformly along straight lines between pairs of points sampled from the data distribution $\mathbb{P}_r$ and the generator distribution $\mathbb{P}_g$.
In this way, our model is capable of generating visually appealing cell-level images. But still, it fails to exploit information of categories of cells since the noise variable $z$ doesn't correspond to any interpretable feature. Motivated by this, our second goal is to make the chosen variables represent meaningful and interpretable semantic features of cells. Inspired by InfoGAN \cite{chen2016infoGAN}, we introduce mutual information into our model:
\begin{equation}
I(X;Y) = \mathrm {H} (X)-\mathrm {H} (X|Y) = \mathrm {H} (Y) -\mathrm {H} (Y|X).
\label{mutual}
\end{equation}
$I(X;Y)$ describes the dependencies between two separate variables $X$ and $Y$. It measures the different aspects of the association between two random variables. If the chosen random variables correspond to certain semantic features, it's reasonable to assume that mutual information between generated samples and random variables should be high.
We define a latent variable $c$ sampled from a fixed noise distribution $p(c)$. The concatenation of the random noise variable $z$ and the latent variable $c$ is then transformed by the generator G into a sample $G(z,c)$. Since we encourage the latent variable to correspond with meaningful semantic features, there should be high mutual information between $c$ and $G(z, c)$. Therefore, the next step is to maximize mutual information $I(c;G(z,c))$, which can be written as:
\begin{equation}
I(c;G(z,c)) = H(c) - H(c \vert G(z,c)).
\end{equation} Followed by this, a lower bound $L_I$ is given by:
\begin{equation}
L_I(G, Q) = \mathbb{E}_{z \sim p(z), c \sim p(c)}[\log Q(c \vert G(z,c))] + H(c),
\end{equation} where $H(c)$ is the entropy of the variable sampled from a fixed noise distribution. Maximizing this lower bound, we maximize mutual information $I(c;G(z,c))$. The proof can be found in InfoGAN \cite{chen2016infoGAN}.
Since we introduce the latent variable $c$ into the model, the value function $V(D,G)$ is replaced by:
\begin{equation}
V (D,G) \leftarrow \mathbb{E}_{x \sim \mathbb{P}_r} [D(x)] - \mathbb{E}_{z \sim p(z), c \sim p(c)}[D(G(z,c))].
\label{fig:wgan}
\end{equation}As we combine the adversarial process with the process of maximizing mutual information, this information-regularized minimax game with a hyperparameter $\lambda_2$ can be written as follows:
\begin{equation}
\min_{G,Q}\max_{D \in \mathcal{D}}V (D,G)-\lambda_2 L_I(G, Q).
\end{equation} The loss of $D$ can be replaced by:
\begin{equation}
\resizebox{1\hsize}{!}{$L_D \leftarrow \mathbb{E}_{z \sim p(z), c \sim p(c)}[D(G(z,c))] - \mathbb{E}_{x\sim \mathbb{P}_r}[D(x)] + \lambda_1 \mathbb{E}_{\hat x \sim \mathbb{P}_{\hat x}} [ || \nabla_{\hat x} D(\hat x) ||_p - 1 ]^2,$}
\end{equation} Since $H(c)$ can be viewed as a constant, the loss of the auxiliary network $Q$ can be written as the negative log-likelihood between $Q(c|G(c,z))$ and the discrete variable $c$. The losses of $G$ and $Q$ can be interpreted as below:
\begin{equation}
L_G = -\mathbb{E}_{z \sim p(z), c \sim p(c)}[D(G(z,c))],
\end{equation}
\begin{equation}
L_Q = - \lambda_2 \mathbb{E}_{z \sim p(z), c \sim p(c)}[\log Q(c|G(z,c))].
\label{fig:InfoGAN}
\end{equation} Figure~\ref{trainingpicture} shows how noises are transformed into interpretable samples during the training process.
\begin{figure}[h!]
\centering
\includegraphics[width=0.46\textwidth]{{gan_update/stage_0}.png}
\caption{Example of how a set of noise vectors are transformed into interpretable image samples over generator iterations. We use a 5-dimensional categorical variable $c$ and 32 Gaussian noise variables $z$ as input. Different rows correspond to different values of $z$. Different columns correspond to different values of $c$. The value of $c$ largely corresponds to cell types.}
\label{trainingpicture}
\end{figure}
\subsubsection {Test Process}
In the training process, a generator distribution is learned to imitate the real data distribution. An auxiliary distribution is learned to maximize the lower bound. Especially if $c$ is sampled from a categorical distribution, a softmax function is applied as the final layer of $Q$. Under this circumstance, $Q$ can act as a classifier in the test process, since the posterior $Q(c|x)$ is discrete. Assuming that each category in $c$ corresponds to a type of cells, the auxiliary network $Q$ can divide cell-level images into different categories while the generator $G$ can generate the interpretable representation for each category of cells.
\subsection{Image-level Classification}
Based on the cell-level visual representation learning, we propose a pipeline combining nuclei segmentation and cell-level visual representation to highlight the varieties of cellular elements. Image-level classification is performed using the calculated cell proportions. The illustration of this pipeline is shown in Figure~\ref{pipeline}.
\subsubsection{Nuclei Segmentation}
An unsupervised nuclei segmentation approach is ultilized consisting of four stages: normalization, unsupervised color deconvolution, intensity thresholding and postprocessing to segment nuclei from the background. Figure~\ref{segmentation} is an overview of our segmentation pipeline.
\begin{figure}[t]
\includegraphics[width=0.48\textwidth]{segmentation/seg.png}
\caption{Overview of segementation process: $\displaystyle (a)$ the cropped image, $\displaystyle (b)$ the normalized image, $\displaystyle (c)$ the separated hematoxylin stain image using color deconvolution, $\displaystyle (d)$ the binary image generated by intensity thresholding, $\displaystyle (e)$ the labeled image after postprocessing where different grayscale values stand for different segmented instances, and $\displaystyle (f)$ the final segmentation image.}
\label{segmentation}
\end{figure}
\textbf{Color Normalization:} We employ Reinhard color normalization \cite{Reinhard2001Color} to convert the color characteristics of all images into the desired standard by computing the mean and standard deviations of a target image in LAB space.
\textbf{Color Deconvolution:} Using the PCA-based `Macenko' method \cite{Macenko2009A}, unsupervised color deconvolution is performed to separate the normalized image into two stains. We project pixels onto a best-fit plane, wherein it selects the stain vectors as percentiles in the `angle distribution' of the corresponding plane. With the correct stain matrix for color deconvolution, the normalized image can be separated into hematoxylin stain and eosin stain.
\textbf{Intensity Thresholding:} To sufficiently segment cells, we apply intensity thresholding in the hematoxylin stain image where the intensity distribution of cells is consistently distinct from the background. By converting the hematoxylin stain image into a binary image with a constant global threshold, the cells are roughly segmented.
\textbf{Postprocessing:} In image postprocessing, objects with fewer pixels than the minimum area threshold will be removed from the binary image. Then we employ the method in \cite{Wienert2012Detection} to remove thin protrusions from cells. Furthermore, we use opening operation to separate a few touched cells.
\subsubsection{Classification}
We utilize the model distribution trained in our unsupervised representation learning as the cell-level classifier. Assuming that we use a $k$-dimensional categorical variable as the chosen variable in the training process, the real data (cell-level images) distribution is allocated into $k$ dimensions. In the test process, cell-level images are unsupervised classified into $k$ corresponding categories.
For each histopathology image, we count the numbers of cell-level instances in each category as the representation of its cellular constitution, denoted as $\{X_1, X_2, X_3, \ldots, X_k\}$. For cellular element $i$, the ratio of the number of this cellular element to the total number of the cellular constitution in this image is calculated by $P_i=\frac{X_i}{\sum_{i=1}^{k}{X_i}}$. We define $P_i$ as the cell proportion of cellular element $i$.
Given cell proportions $\{P_1, P_2, P_3, \ldots, P_k\}$ as the feature vector of histopathology images, we utilize either k-means or SVM to give image-level predictions.
\section{Experiments and Results}
\subsection{Dataset} \label{subsection:dataset}
All our experiments are conducted on bone marrow histopathology images stained with hematoxylin and eosin. As described before, the cellular constitution in bone marrow is a determinant in diagnoses of blood disease.
\textbf{Dataset A:} Publicly available dataset \cite{kainz2015you} which consists of eleven images of healthy bone marrow with a resolution of $1200 \times 1200$ pixels. Each image contains around 200 cells. The whole dataset includes 1995 cell-level images in total. We label all cell-level images into four categories: 34 neutrophils, 751 myeloblasts, 495 monocytes, and 715 lymphocytes. Images are carefully labeled by two pathologists. When the two pathologists disagree on a particular image, a senior pathologist makes a decision over the discord.
\textbf{Dataset B:} Dataset provided by the First Affiliated Hospital of Zhejiang University which contains whole slides of bone marrow from 24 patients with blood diseases. Each patient matchs with one whole slide. We randomly crop 29 images with a resolution of $1500 \times 800$ pixels from all whole slides. Dataset B contains around 12000 cells in total. For this dataset, we label 600 cell-level images into three categories for evaluation: 200 myeloblasts, 200 monocytes, and 200 lymphocytes. The labeling process is conducted in the same manner as Dataset A.
\textbf{Dataset C:} Combination of Datasets A and B, which results in 29 abnormal and 11 normal histopathology images.
\textbf{Dataset D:} Dataset includes whole slides from 28 patients with bone marrow hematopoietic tissue hyperplasia (negative) and 56 patients with leukemia (positive). Each patient matchs with one whole slide. We randomly crop images with a resolution of $1500 \times 800$ pixels from all whole slides. This results in 72 negative and 132 positive images. After segmentation, Dataset D contains around 80000 cells in total.
\subsection{Implementation}
\textbf{Network Parameters:} Our generator $G$, discriminator $D$ and auxiliary network $Q$ all have the structures of residual networks. In the training process, all three networks are updated by Adam optimizer ($\alpha=0.0001$, $\beta_1=0.5$, $\beta_2=0.9$, $lr=2 \times 10^{-4}$) \cite{kingma2014adam} with a batch size of 64. All our experiments use hyperparameters $\lambda_1 = 10$ and $\lambda _2 = 1$. For each training iteration, we update $D$, $G$ and $Q$ in turn. One training iteration consists of five discriminator iterations, one generator iteration, and one auxiliary network iteration. For each training process, we augment the training set by rotating images with angles $90^\circ$, $180^\circ$, $270^\circ$. We train ten epochs for our model in each experiment.
\textbf{Noise Sources:} The noise fed into the network is the combination of a 5-dimensional categorical variable and 32 Gaussian noise variables for the training of Dataset A or Dataset B. We use the combination of a 5-dimensional categorical variable and 64 Gaussian noise variables for Dataset C.
\textbf{Segmentation Parameters:} The mean value of the standard image in three channels is $[8.98\pm 0.64, 0.08\pm 0.11, 0.02\pm 0.03]$ for color normalization. Vectors for color deconvolution are picked from 1\% to 99\% angle distribution while the magnitude below 16 is excluded from the computation. We use the threshold value of 120 for intensity thresholding. In the post-process, objects with pixels smaller than 200 will be removed. An opening operation with $7 \times 7$ kernel size is performed to separate touched cells. When the edge of the bounding box of a cell-level image is larger than 32 pixels, we rescale the image to make the larger edge match to 32. Each cell is centered in a $32 \times 32$ pixel image where blank is filled with $[255, 255, 255]$.
\textbf{Bounding Box:} To prevent the color and texture contrast from troubling the feature extraction process, we use instances without segmentation for baseline methods. If we depose the nuclei in the center with the loose bounding box in the same manner as our previous experiments, cells will suffer from severe overlapping. Thus, we crop the minimum bounding box region along each segmented instance, and then resize it into $32 \times 32$ pixels as our dataset.
\textbf{Software:} We implement our experiments on framework Pytorch for deep learning models and framework HistomicsTK for nuclei segmentation. Our model is compared with multiple sources of baselines. Three main types of baselines are claimed to be relevant as follows: (1) feature extractors including manual features, HOG and DNN extractor; (2) supervised classifiers including SVM and DNN; (3) clustering algorithms including DEC and K-means. The rich mix of different sources of baselines, including deep learning algorithms, provides a stronger demonstration to our experiments. We utilize k-means++ \cite{arthur2007k} to choose the initial values when using k-means to perform clustering. The feature code\footnote{Implementation details can be found at \url{https://github.com/bohu615/nu_gan}} is Python implementation in all these algorithms.
\textbf{Hardware:} For hardware, we use one pair of Tesla K80 GPU for parallel training and testing of neural network models. Other baseline experiments are conducted on Intel(R) Xeon(R) CPU E5-2690 v3 @ 2.60GHz. For our model, with a batch size of 64, using one pair of K80 GPU for parallel computation, each generator iteration costs 3.2 seconds in the training process when each batch costs 0.18 seconds in the test process.
\subsection{Cell-level Classification Using Various Features}
To demonstrate the quality of our representation learning, we apply the trained model as a feature extractor. The experiment is conducted on Dataset A. In this experiment, 1596 cell-level images are used for training; 399 cell-level images are used for testing.
\textbf{Comparison:} (1) MF: 188-dimensional manual feature combined of SIFT \cite{lowe2004distinctive}, LBP \cite{ojala2002multiresolution}, and $L \times a \times b$ color histogram. (2) DNN: DNN+k-means: DNN features extracted by ResNet-50 trained on Imagenet-1K, on top of which k-means is performed. (3) Our Method: We downsample the features after each residual block of the discriminator into a $4 \times 4$ spatial grid using max pooling. These features are flattened and concatenated to form an 8192-dimensional vector. On top of the feature vectors, an L2-SVM is trained to perform classification.
Different processing strategies are used as follows: (1) w/ Seg: using the output generated by nuclei segmentation; (2) w/o Seg: using the minimum bounding box along each cell-level instance.
\textbf{Evaluation:} For each class, we denote the number of true positives $TP$, the number of false positives $FP$ and the number of false negatives $FN$. The precision, recall and F-score ($F_1$) for each class are defined as follows: \begin{equation}
\begin{aligned}
& precision=\frac{TP}{TP+FP}, \\
& recall=\frac{TP}{TP+FN}, \\
& F_1 = \frac{2 \cdot precision \cdot recall}{precision + recall}. \\
\end{aligned}
\label{eq:fscore}
\end{equation}The average precision, recall and F-socre are calculated weighted by support (the number of true instances of each class).
\textbf{Results:} We randomly choose correctly classified and misclassified samples displayed in Figure~\ref{cell_right_wrong}. The comparison of results is shown as Table~\ref{cell-level classification}, which proves the advantages of our representation learning method. The manual feature extractor can generate a better result based on the bounding box regions, but its performance is still lower than ours. The color of the background can provide useful information for the color histogram channel in manual features but is viewed as noise for the DNN based extractor. Though the dimensions of the feature vectors of our method are higher, the clustering ability of our model ensures further unsupervised applications. Furthermore, we apply mean pooling on top of feature maps to prove that using less dimensional features can also generate a comparable result. In this manner, we achieve 0.850 F-score using 2048 dimensional features and 0.840 F-score using 512 dimensional features.
\begin{figure}[H]
\centering
\includegraphics[width=0.38\textwidth]{{classification_compare/cell_right_wrong}.png}
\caption{Visualization of cell-level classification performed on Dataset A: $(up)$ correctly classified samples and $(down)$ misclassified samples. misclassified samples can be illegible for pathologists either.}
\label{cell_right_wrong}
\end{figure}
\begin{table}[H]
\centering
\caption{{\textnormal{Performance of cell-level classification using various features.}}}
\label{cell-level classification}
\begin{tabular}{|c|c|c|c|c|c|c|}
\hline
\multirow{2}{*}{Methods} & \multicolumn{2}{c|}{Precision} & \multicolumn{2}{c|}{Recall} & \multicolumn{2}{c|}{F-score} \\ \cline{2-7}
& \tiny{w/ Seg} & \tiny{{w/o Seg}} & \tiny{{w/ Seg}} & \tiny{{w/o Seg}} & \tiny{{w/ Seg}} & \tiny{{w/o Seg}} \\ \hline
MF & 0.821 & {0.837} & 0.803 & {0.847} & 0.811 & {0.842} \\
DNN & 0.838 & {0.760} & 0.817 & {0.769} & 0.827 & {0.764} \\
Our Method & \textbf{0.865} & {/} & \textbf{0.848} & {/} & \textbf{0.857} & {/} \\ \hline
\end{tabular}
\end{table}
\subsection{Cell-level Clustering}
As the priority of image-level classification of histopathology images, cell-level clustering is performed using the trained auxiliary network $Q$. We conduct experiments on the three datasets described in Section~\ref{subsection:dataset}.
\textbf{Comparison:} (1) MF+k-means: Manual features with k-means. (2) DNN+k-means: DNN features extracted by ResNet-50 trained on Imagenet-1K, on top of which k-means is performed. (3) HOG+DEC: Deep Embedded Clustering (DEC) \cite{xie2016unsupervised} on 2048-dimensional HOG features. (4) Our Method: Cell images are unsupervised allocated to five clusters by the auxiliary network $Q$. We also test models such as Categorical GAN (CatGAN) \cite{springenberg2015unsupervised}, InfoGAN (under DCGAN architecture), and Gaussian Mixture VAE (GMVAE) \cite{dilokthanakul2016deep} on our datasets under different hyperparameters, but find them fail to converge.
{The following processing strategies are also used: (1) w/ Seg: using the output generated by nuclei segmentation; (2) w/o Seg: using the minimum bounding box along each cell-level instance.}
\textbf{Evaluation:} We evaluate the performance of clustering using the average F-score, purity, and entropy. For the set of clusters $\{ \omega_1, \omega_2, \ldots, \omega_K \}$ and the set of classes $\{ c_1,c_2,\ldots,c_J \}$, we assume that each cluster $\omega_k$ is assigned to only one class ${\mathop{\mathrm{argmax}}\nolimits}_j (\vert\omega_k \cap c_j\vert)$. The F-score for class $c_j$ is then given by Equation \ref{eq:fscore}. The average F-score is given calculated by the number of true instances in each class.
Purity and Entropy are also used as evaluation metrics, which are written as follows:
\begin{equation}
\begin{aligned}
& purity = \frac{1}{N} \sum_k \max_j \vert\omega_k \cap c_j\vert, \\
& entropy = -\frac{1}{N}\sum_k\vert \omega_k\vert \log \frac{\vert \omega_k\vert}{N}. \\
\end{aligned}
\label{eq:purity}
\end{equation}Larger purity and smaller entropy indicate better clustering results.
For nuclei segmentation, we use Intersection over Union (IoU) and the F-score as evaluation metrics. A segmented instance (I) is matched with the ground truth (G) only if they intersect at least 50\% (i.e., $\vert I \cap G \vert>0.5G$). For each matched instance and its ground truth, the overlapping pixels are counted as true positive ($TP$). The pixels of instance remain unmatched are counted as false positive ($FP$) while the pixels of ground truth remaining unmatched are counted as false negative ($FN$). The F-score is then calculated using Equation \ref{eq:fscore}.
For k-means based methods, the average F-score is approximately the same ($\pm 0.02$) using either four, five, or six clusters.
\begin{figure*}[t]
\centering
\includegraphics[width=\textwidth]{{clustering/display}.png}
\caption{Visualization of clustering. We randomly select 60 samples from each one of five clusters, displayed as (a) to (e). Instances in the same cluster have a distinct consistency. In (b), cells in marrow with dark, dense, and close phased nuclei tend to be lymphocytes or erythroid precursors. In (c) and (e), cells with dispersed chromatin are most likely granulocytes precursors such as myeloblasts.}
\label{cluster_display}
\end{figure*}
\textbf{Annotations:} To evaluate the capability of nuclei segmentation, We randomly choose 20 patches from Dataset C with a resolution of $200 \times 200$ pixels. The ground truth is carefully labeled by two pathologists. When the two pathologists disagree on a particular image, a senior pathologist makes a decision over the discord.
\textbf{Results:} For nuclei segmentation, our method achieves 0.56 mean IoU and 0.70 F-score.
For cell-level clustering, the comparison shown as Table~\ref{table:cluster} shows the superiority of our method. To explicitly reveal the semantic features our model has captured, we randomly choose 60 samples from each of the five clusters displayed in Figure~\ref{cluster_display}, which shows a distinct consistency within each cluster. Reasonable interpretations can be given. Cells are clustered according to the semantic features such as the chromatin openness, the darkness and density of nuclei, and if nuclei show the appearance of being segmented.
{When it comes to unsupervised classification, none of the baseline methods can benefit from the bounding box. We observe that the color context of the background can be disturbing when the classification is under the fully unsupervised manner.}
% \usepackage{multirow}
\begin{table}[H]
\centering
\caption{\textnormal{Performance of cell-level clustering.}}
\label{table:cluster}%
\resizebox{1\hsize}{!}{
\begin{tabular}{|c|c|l|c|c|c|c|c|c|}
\hline
\multirow{2}{*}{Dataset} & \multicolumn{2}{c|}{\multirow{2}{*}{Methods}} & \multicolumn{2}{c|}{Purity} & \multicolumn{2}{c|}{Entropy} & \multicolumn{2}{c|}{F-score} \\ \cline{4-9}
& \multicolumn{2}{c|}{} & \tiny{{w/ Seg}} & \tiny{{w/o Seg}} & \tiny{{w/ Seg}} & \tiny{{w/o Seg}} & \tiny{{w/ Seg}} & \tiny{{w/o Seg}} \\ \hline
\multirow{4}{*}{A} & \multicolumn{2}{c|}{MF+k-means} & 0.579 &{0.442} & 1.376 &{1.598} & 0.603 &{0.510} \\
& \multicolumn{2}{c|}{DNN+k-means} & 0.667 &{0.470} & 1.256 &{1.552} & 0.677 &{0.501} \\
& \multicolumn{2}{c|}{HOG+DEC} & 0.729 &{0.637} & 1.086 &{1.167} & 0.737 &{0.664} \\
& \multicolumn{2}{c|}{Our Method} & \textbf{0.855} & {/} & \textbf{0.750} &{ /} & \textbf{0.863} & {/} \\ \hline
\multirow{4}{*}{B} & \multicolumn{2}{c|}{MF+k-means} & 0.392 &{0.421} & 1.561 &{1.545} & 0.409 &{0.454} \\
& \multicolumn{2}{c|}{DNN+k-means} & 0.719 &{0.406} & 0.844 &{1.557} & 0.760 &{0.435} \\
& \multicolumn{2}{c|}{HOG+DEC} & 0.771 & {0.681} & 0.697 &{1.161} & 0.812 &{0.693} \\
& \multicolumn{2}{c|}{Our Method} & \textbf{0.874} & {/} & \textbf{0.431} & {/} & \textbf{0.841} & {/} \\ \hline
\multirow{4}{*}{C} & \multicolumn{2}{c|}{MF+k-means} & 0.459 &{0.446} & 1.533 &{1.597} & 0.484 &{0.514} \\
& \multicolumn{2}{c|}{DNN+k-means} & 0.578 &{0.458} & 1.377 &{1.575} & 0.601 &{0.485} \\
& \multicolumn{2}{c|}{HOG+DEC} & 0.667 & {0.602} & 1.217 &{1.334} & 0.682 &{0.621} \\
& \multicolumn{2}{c|}{Our Method} & \textbf{0.769} & {/ } & \textbf{0.977} & {/ } & \textbf{0.777} &{ / } \\ \hline
\end{tabular}}
\end{table}
\iffalse
\begin{table}[H]
\centering
\caption{\textnormal{Performance of cell-level clustering.}}
\label{table:cluster}
\begin{tabular}{|c|c|c|c|c|}
\hline
Dataset & Methods & Purity & Entropy & F-score \\ \hline
\multirow{5}{*}{A} & MF+k-means & 0.579 & 1.376 & 0.603 \\
& DNN+k-means & 0.667 & 1.256 & 0.677 \\
& HOG+DEC & 0.729 & 1.086 & 0.737 \\
& Our Method & \textbf{0.855} & \textbf{0.750} & \textbf{0.863} \\ \hline
\multirow{5}{*}{B} & MF+k-means & 0.392 & 1.561 & 0.409 \\
& DNN+k-means & 0.719 & 0.844 & 0.760 \\
& HOG+DEC & 0.771 & 0.697 & 0.812 \\
& Our Method & \textbf{0.874} & \textbf{0.431} & \textbf{0.841} \\ \hline
\multirow{5}{*}{C} & MF+k-means & 0.459 & 1.533 & 0.484 \\
& DNN+k-means & 0.578 & 1.377 & 0.601 \\
& HOG+DEC & 0.667 & 1.217 & 0.682 \\
& Our Method & \textbf{0.769} & \textbf{0.977} & \textbf{0.777} \\ \hline
\end{tabular}
\end{table}
\fi
Especially for Dataset A, Figure~\ref{loss} shows the convergence of $V(D,G)$ (see Equation (\ref{fig:wgan})) and $L_Q$ (see Equation (\ref{fig:InfoGAN})). $V(D,G)$ is used to evaluate how well the generator distribution matches the real data distribution \cite{gulrajani2017improved}. $L_Q$ approaching zero indicates that mutual information is maximized \cite{chen2016infoGAN}. Figure~\ref{purity} shows how the purity of clustering increases in the training process.
\begin{figure}[H]
\subfigure[]{
\label{loss}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{loss_figure/loss1}.png}
\end{minipage}}
\subfigure[]{
\label{purity}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{loss_figure/loss2}.png}
\end{minipage}}
\label{fig:mini:subfig}
\caption{Visualization of cell-level clustering performed on Dataset A: (a) Training losses converge as the network trains. (b) The purity increases gradually over generator iterations.}
\end{figure}
\textbf{Impacts of the Number of Clusters:}
For our method, it is easy to change the number of clusters by sampling the categorical noise from a different dimension. We compare the results of choosing different numbers of clusters shown in Table~\ref{different_k}, which shows there is no distinct difference between choosing four and five clusters. We choose five clusters (a 5-dimensional categorical random variable) in change for a slightly better performance.
\begin{table}[H]
\centering
\caption{\textnormal{Performance when choosing different numbers of clusters.}}
\label{different_k}
\begin{tabular}{|c|c|c|c|}
\hline
Clusters & 4 & 5 & 6 \\ \hline
F-score & 0.831 & 0.863 & 0.789 \\ \hline
\end{tabular}
\end{table}
{{
\textbf{Impacts of Uninformative Representations}: The uninformative representations such as the staining color and rotations can be interference factors in the process of classification. Besides using color normalization and data augmentation to ease this problem, we also demonstrate that these features are more likely to be latent encoded in Gaussian random variables which do not influence the classification task. As is shown in Figure~\ref{fig:Uninformative}, we fix the value of the chosen categorical variable $c$ while walking through the random space of the Gaussian noise variable $z$. The result shows that uninformative representations tend to be encoded in noise variables through the process of maximizing the mutual information.
\begin{figure}[H]
\centering
\subfigure[]{
\label{staining}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{uninformative/staining}.png}
\end{minipage}}
\subfigure[]{
\label{rotation}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{uninformative/rotation}.png}
\end{minipage}}
\caption{{{Examples of how uninformative representations are encoded in Gaussian noise variables $z$. Different columns share the same value of the chosen categorical variable $c$. A random walk is performed between two points in the space of $z$. It can be seen that (a) the staining color and (b) the rotation are both latent encoded in the Gaussian noise variables.}}}
\label{fig:Uninformative}
\end{figure}
}}
\subsection{Image-level Classification}
{{We perform image-level classification experiments on Dataset C and Dataset D respectively. Dataset C includes 29 positive and 11 negative images. Dataset D includes 132 positive and 72 negative images. Each dataset is randomly split into four folds for the 4-fold cross-validation. Each score is reported averagely. Each experiment is repeated for four times with different random split for cross-validation. The scores are reported four times to show confidence intervals.}}
{\textbf{Comparison:} (1) DNN (cell-level based): We use ResNet-50 features extracted from cell-level instances to perform cell-level clustering. Then we train an L2-SVM on top of the cell proportions to perform image-level classification. (2) DNN (image-level based): We use ResNet-50 pre-trained on Imagenet-1K as an image-level feature extractor. Images with a resolution of $1500 \times 800$ are normalized and center cropped to $800 \times 800$ pixels, then resized into $224 \times 224$ pixels. An L2-SVM is trained on the feature vectors. We observe this produces a better result than fine-tuning or directly training a ResNet-50 without pre-train.} (3) Our method (w/ k-means): We first train our GAN architecture on the training set, then conduct the cell-level clustering on both the training set and test set using the trained model. Cluster centers are calculated given cell proportions of each sample in the training set. The predict label is given by the closest cluster that each sample in the test set belongs to. (4) Our method (w/ SVM): An L2-SVM instead of k-means is used as the final classifier.
\textbf{Evaluation:} We use the precision, recall and F-score for evaluation, the details of which have been described in Equation \ref{eq:fscore}. The difference is that the labels are binary in this experiment.
%\textbf{Using Patches Instead of Entire Images:}
{{\textbf{Results:} Following the proposed pipeline, the GAN architecture is trained on the segmentation output of the split training set. For cell-level clustering task, we achieve 0.791 F-score trained on 12000 training instances of Dataset C and 0.771 F-score trained on 60000 training instances of Dataset D, both evaluated by labeled cells of Dataset A.
Given the cell proportions, when using k-means to perform image-level unsupervised classification, we achieve 0.931 F-score on Dataset C and 0.875 F-score on Dataset D, which is comparative to the DNN method with 0.933 and 0.888 F-score. The advantage is that our model is interpretable. The proportion of which category of cells is irregular is recognizable.
Since there are a large number of cell-level images on both Dataset C and D, it is difficult to test our method under full-supervision with a similar pipeline. We instead train an L2-SVM on cell proportions, taking image-level labels of histopathology images as targets. As the comparison shown in Table~\ref{imagelevel}, our method achieves 0.950 F-score on Dataset C and 0.902 F-score on Dataset D.}}
\begin{table*}[tbh!]
\centering
\caption{{\textnormal{Performance of image-level classification. Each experiment is repeated for four times with different random split for cross-validation. The scores are reported four times to show confidence intervals.}}}
\label{imagelevel}
\begin{tabular}{|c|c|c|l|l|l|c|l|l|l|c|l|l|l|}
\hline
Datasets & Methods & \multicolumn{4}{c|}{Precision} & \multicolumn{4}{c|}{Recall} & \multicolumn{4}{c|}{F-score} \\ \hline
\multirow{4}{*}{C} & {DNN (cell-level based)} &{0.539}&{0.598}&{0.688}&{0.524}
&{0.711}& {0.723}&{0.734}&{0.678}
&{0.636}&{0.678}&{0.701}&{0.621} \\
& DNN (image-level based)&0.906 & {0.913} & {0.901} &{0.921}
& \textbf{0.969} &{0.958} &{0.943} &{0.965}
& 0.933 & {0.929} & {0.924} & {0.937} \\
& Our Method (w/ k-means) & 0.936 & {0.945} & {0.939} &{0.937} &0.933 &{0.944} &{0.946} &{0.938} &0.931 &{0.941} &{0.948} & {0.939} \\
& Our Method (w/ SVM) & \textbf{0.950} & {\textbf{0.948}} &{\textbf{0.940}} & {\textbf{0.946}} &\textbf{0.969} &{\textbf{0.968}} & {\textbf{0.950}} &{\textbf{0.966}} & \textbf{0.950} &{\textbf{0.949}} &{\textbf{0.940}} &{\textbf{0.949}} \\ \hline
\multirow{4}{*}{{D}} & {DNN (cell-level based)} &{0.469}&{0.579}&{0.498}&{0.581} &{0.697}&{0.654}&{0.643}&{0.665} &{0.558}&{0.612}&{0.583}&{0.621} \\
& {DNN (image-level based)} &{0.863}&{\textbf{0.900}}&{0.887}&{0.869} &{\textbf{0.863}}&{0.886}&{0.871}&{0.865} &{\textbf{0.863}}&{0.888}&{0.879}&{0.866} \\
& {Our Method (w/ k-means)} &{0.858} &{0.879} &{0.881} &{0.868} &{0.857} &{0.868} &{0.873} &{0.865} &{0.862} &{0.870} &{0.875} &{0.867} \\
& {Our Method (w/ SVM)} &{\textbf{0.864}} &{0.897} &{\textbf{0.901}} &{\textbf{0.882}} &{0.858} &{\textbf{0.892}} &{\textbf{0.898}} &{\textbf{0.878}} &{\textbf{0.863}} &{\textbf{0.891}} &{\textbf{0.902}} &{\textbf{0.880}} \\ \hline
\end{tabular}
\end{table*}
On Dataset C, we use Principal Components Analysis (PCA) to perform a dimensionality reduction, cell proportions of each histopathology image are projected onto a two-dimension plane to show that there is a distinct difference between normal and abnormal images, shown in Figure~\ref{visualization}.
\begin{figure}[tbh]
\includegraphics[width=0.46\textwidth]{{loss_figure/fig}.png}
\caption{Visualization of unsupervised classification using cell proportions. It can be observed that the points representing normal and abnormal samples are distinctly distributed in two different clusters.}
\label{visualization}
\end{figure}
{{\textbf{Impacts of the Segmentation Parameters:} To validate the impacts of the segmentation performance on the image-level classification result, we change the value of intensity threshold in the segmentation process of experiments on Dataset C. We randomly choose 20 patches with a resolution of $200 \times 200$ pixels in Dataset C for evaluation, which includes 335 nuclei as counted. We use missing instances (nuclei that are missing in outputs), false alarms (mis-segmented background instances), and the F-score for evaluation.
As is shown in Table~\ref{intensity}, both results of segmentation and classification are the highest when the intensity threshold remains 120. Followed by the decreasing of segmentation performance, the classification performance will stay within an acceptable range. Too bad segmentation performance will worsen the classification result since the quality and quantity of the segmentation outputs are not enough to reveal the distinct representation of each image-level instance.}}
\begin{table}[H]
\centering
\caption{\textnormal{{Performance when changing the segmentation parameters.}}}
\label{intensity}
\resizebox{1\hsize}{!}{
\begin{tabular}{|c|c|c|c|c|c|c|c|}
\hline
{\tiny{Intensity threshold}} & {60} & {80} & {100} & {120} & {140} & {160} & {180} \\ \hline
{\tiny{Missing Instances}} & {127} & {48} & {21} & {7} & {14} & {64} & {184} \\ \hline
{\tiny{False Alarms}} & {3} & {4} & {15} & {5} & {20} & {30} & {35} \\ \hline
{\tiny{Segmentation F-score}} & {0.315} & {0.413} & {0.602} & {0.701} & {0.656} & {0.534} & {0.218} \\ \hline
{\tiny{Classification F-score}} & {0.579} & {0.814} & {0.932} & {0.950} & {0.941} & {0.901} & {0.576} \\ \hline
\end{tabular}}
\end{table}
{{\textbf{Impacts of the Number of Clusters:} For image-level classification of Dataset C, we conduct experiments choosing different number of clusters. Table~\ref{image-level-cluster} shows that there is no distinct difference of performance between choosing five and six clusters. We still choose five clusters for a better performance.}}
\begin{table}[H]
\centering
\caption{\textnormal{{Performance when choosing different numbers of clusters.}} }
\label{image-level-cluster}
\begin{tabular}{|c|c|c|c|c|}
\hline
{\tiny{Clusters}}& {4} & {5} & {6} & {7} \\ \hline
{\tiny{Cell-level Classification F-score}} & {0.711} & {0.791} & {0.762} & {0.710} \\ \hline
{\tiny{Image-level Classification F-score}} & {0.897} & {0.950} & {0.944} & {0.899} \\ \hline
\end{tabular}
\end{table}
{\textbf{Patch-level Classification}: We perform classification based on patches. Using a sliding window with a window size of 224 and a stride of 224, we separately transfer the normalized images from the training set and test set from Dataset C into labeled image patches. This results in 588 positive and 288 negative patches for training, 224 positive and 108 negative patches for testing. If 50\% of the patches of an image-level instance are positive, we will consider this instance as positive. In this manner, we achieve 0.851 F-score using DNN feature extractor with SVM and 0.831 F-score using our method, which is not comparative to our image-level classification results.}
\textbf{Discussion:}
Analyzing the results, we find that the cell proportions \{$P_1, P_2, \cdots, P_5$\} can indicate the presence of blood diseases.
For our experiment, cell-level clustering shows that \{$P_1$, $P_4$\} correspond to myeloblasts, \{$P_5$\} corresponds to lymphocytes and erythroid precursors, and \{$P_2$, $P_3$\} correspond to monocytes and glanulocytes. For all normal images, $P_1$ and $P_4$ are relatively lower. This matches the constitution in normal bone marrow where the lymphocytes, glanulocytes and erythroid precursors are in the majority when the percentage of cells with open phased nuclei (such as myeloblasts, under some circumstances plasma cells) is relatively lower (less than 10\%). In Figure~\ref{visualization}, abnormal images that are confidently discriminated are reflected in the numerous presence of the supposed minority myeloblasts or plasma cells, which in turn is reflected in the sharp increase of $P_1$ and $P_4$.
However, there are three abnormal images that are exceptional. To analyze what causes the failure, we display the example image in Figure~\ref{wrong}.
\begin{figure}[H]
\centering
\includegraphics[width=0.3\textwidth]{{staining_compare/wrong}.png}
\caption{Example of the failed samples. Too many erythroid precursors indicate the presence of blood disease. The overlap of nuclei and the lousy staining condition add to the difficulties of cell-level classification.}
\label{wrong}
\end{figure}
In these images, the irregular proportion of erythroid precursors indicates the presence of blood disease. We find that our model does not correctly classify these cells. The reason could be that the staining condition of these cells is not as good as expected. A typical erythroid precursor should have a close phased, dark-staining nucleus that appears almost black. As Figure~\ref{difference} shows, the color of nuclei segmented from these images differ from the rest of the dataset. Particularly in these images, our model is still not robust enough to capture the most significant semantic variance in an unsupervised setting. Therefore, acquiring high-quality histopathology images is still a priority.
\begin{figure}[H]
\centering
\subfigure[]{
\label{dataset2_lymph}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{staining_compare/difference_0002}.png}
\end{minipage}}
\subfigure[]{
\label{dataset2_yuanshi}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{staining_compare/difference_0001}.png}
\end{minipage}}
\subfigure[]{
\label{dataset1_lymph}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{staining_compare/difference_0000}.png}
\end{minipage}}
\subfigure[]{
\label{dataset1_yuanshi}
\begin{minipage}{0.23\textwidth}
\includegraphics[width=\textwidth]{{staining_compare/difference_0003}.png}
\end{minipage}}
\caption{Variance of staining conditions. $(a)$ and $(b)$ are erythroid precursors and myeloblasts randomly chosen from failed images. $(c)$ and $(d)$ are samples selected from correctly predicted images. Our model mistakes erythroid precursors for myeloblasts particularly in failed images.}
\label{difference}
\end{figure}
\section{Conclusion}
In this paper, we introduce a unified GAN architecture with a new formulation of the loss function into cell-level visual representation learning of histopathology images. Cell-level unsupervised classification with interpretable visualization is performed by maximizing mutual information. Based on this model, we exploit cell-level information by calculating the cell proportions of histopathology images. Followed by this, we propose a novel pipeline combining cell-level visual representation learning and nuclei segmentation to highlight the varieties of cellular elements, which achieves promising results when tested on bone marrow datasets.
In future work, some improvements can be made to our method. First, the segmentation method and the computational time can be further improved. The gradient penalty added on the network architecture requires the computation of the second order derivative, which is time-consuming in the training process. Secondly, in addition to cell proportions, other information about the patients should be carefully considered, such as clinical trials and gene expression data. By allocating and annotating the relevant genetic variants, the risk can be re-evaluated. In clinical practice, doctors need to consolidate more critical information to make a confident diagnosis. For example, bone marrow cells of children might not be as varied as those of adults'. To classify cells in a more fine-grained manner, the peculiar distribution information such as erythroid cells more likely form clusters (erythroid islands) can be considered.
\section{Acknowledgment}
The authors would like to thank the First Affiliated Hospital of Zhejiang University and Dr. Xiaodong Teng from Department of Pathology, the First Affiliated Hospital of Zhejiang University for providing data and help.
\medskip
\bibliographystyle{plain}
\begin{thebibliography}{10}
\bibitem{Gurcan2009histopathological}
M.~N. Gurcan, L.~E. Boucheron, A.~Can, A.~Madabhushi, N.~M. Rajpoot, and
B.~Yener, ``Histopathological image analysis: A review,'' {\em IEEE Reviews
in Biomedical Engineering}, vol.~2, no.~1, pp.~147--171, 2009.
\bibitem{Bennett1976Proposals}
J.~M. Bennett, D.~Catovsky, M.~T. Daniel, G.~Flandrin, D.~A. Galton, H.~R.
Gralnick, and C.~Sultan, ``Proposals for the classification of the acute
leukaemias. french-american-british (fab) co-operative group,'' {\em British
Journal of Haematology}, vol.~33, no.~4, p.~451–458, 1976.
\bibitem{Xu2014Deep}
Y.~Xu, T.~Mo, Q.~Feng, P.~Zhong, M.~Lai, and I.~C. Chang, ``Deep learning of
feature representation with multiple instance learning for medical image
analysis,'' in {\em IEEE International Conference on Acoustics, Speech and
Signal Processing}, pp.~1626--1630, 2014.
\bibitem{xu2015deep}
Y.~Xu, Z.~Jia, Y.~Ai, F.~Zhang, M.~Lai, I.~Eric, and C.~Chang, ``Deep
convolutional activation features for large scale brain tumor histopathology
image classification and segmentation,'' in {\em IEEE International
Conference on Acoustics, Speech and Signal Processing}, pp.~947--951, 2015.
\bibitem{xu2016deep}
J.~Xu, X.~Luo, G.~Wang, H.~Gilmore, and A.~Madabhushi, ``A deep convolutional
neural network for segmenting and classifying epithelial and stromal regions
in histopathological images,'' {\em Neurocomputing}, vol.~191, no.~1,
pp.~214--223, 2016.
\bibitem{Chen2016DCAN}
H.~Chen, X.~Qi, L.~Yu, and P.-A. Heng, ``Dcan: Deep contour-aware networks for
accurate gland segmentation,'' in {\em Proceedings of the IEEE conference on
Computer Vision and Pattern Recognition}, pp.~2487--2496, 2016.
\bibitem{chen2014deep}
T.~Chen and C.~Chefd’hotel, ``Deep learning based automatic immune cell
detection for immunohistochemistry images,'' in {\em International Workshop
on Machine Learning in Medical Imaging}, pp.~17--24, 2014.
\bibitem{cirecsan2013mitosis}
D.~C. Cire{\c{s}}an, A.~Giusti, L.~M. Gambardella, and J.~Schmidhuber,
``Mitosis detection in breast cancer histology images with deep neural
networks,'' in {\em International Conference on Medical Image Computing and
Computer-assisted Intervention}, pp.~411--418, 2013.
\bibitem{goodfellow2014generative}
I.~Goodfellow, J.~Pouget-Abadie, M.~Mirza, B.~Xu, D.~Warde-Farley, S.~Ozair,
A.~Courville, and Y.~Bengio, ``Generative adversarial nets,'' in {\em
Advances in neural information processing systems}, pp.~2672--2680, 2014.
\bibitem{radford2015unsupervised}
A.~Radford, L.~Metz, and S.~Chintala, ``Unsupervised representation learning
with deep convolutional generative adversarial networks,'' {\em arXiv
preprint arXiv:1511.06434}, 2015.
\bibitem{arjovsky2017wasserstein}
M.~Arjovsky, S.~Chintala, and L.~Bottou, ``Wasserstein gan,'' {\em arXiv
preprint arXiv:1701.07875}, 2017.
\bibitem{gulrajani2017improved}
I.~Gulrajani, F.~Ahmed, M.~Arjovsky, V.~Dumoulin, and A.~Courville, ``Improved
training of wasserstein gans,'' {\em arXiv preprint arXiv:1704.00028}, 2017.
\bibitem{he2016deep}
K.~He, X.~Zhang, S.~Ren, and J.~Sun, ``Deep residual learning for image
recognition,'' in {\em Proceedings of the IEEE conference on Computer Vision
and Pattern Recognition}, pp.~770--778, 2016.
\bibitem{chen2016infoGAN}
X.~Chen, Y.~Duan, R.~Houthooft, J.~Schulman, I.~Sutskever, and P.~Abbeel,
``Infogan: Interpretable representation learning by information maximizing
generative adversarial nets,'' in {\em Advances in neural information
processing systems}, pp.~2172--2180, 2016.
\bibitem{Nazlibilek2014Automatic}
S.~Nazlibilek, D.~Karacor, T.~Ercan, M.~H. Sazli, O.~Kalender, and Y.~Ege,
``Automatic segmentation, counting, size determination and classification of
white blood cells,'' {\em Measurement}, vol.~55, no.~3, pp.~58--65, 2014.
\bibitem{Y2014Methods}
Y.~Sun and P.~A. Sermon, ``Methods for nuclei detection, segmentation, and
classification in digital histopathology: A review--current status and
future potential,'' {\em IEEE Reviews in Biomedical Engineering}, vol.~7,
no.~1-5, p.~97, 2014.
\bibitem{Muthu2012Hybrid}
M.~Muthu, Rama~Krishnan, C.~Chakraborty, R.~R. Paul, and A.~K. Ray, ``Hybrid
segmentation, characterization and classification of basal cell nuclei from
histopathological images of normal oral mucosa and oral submucous fibrosis,''
{\em Expert Systems with Applications}, vol.~39, no.~1, pp.~1062--1077, 2012.
\bibitem{Xu2015Dual}
X.~Xu, F.~Lin, C.~Ng, and K.~P. Leong, ``Dual spatial pyramid on rotation
invariant texture feature for hep-2 cell classification,'' in {\em
International Joint Conference on Neural Networks}, pp.~1--8, 2015.
\bibitem{Lorenzo2013Cervical}
J.~V. Lorenzo-Ginori, W.~Curbelo-Jardines, J.~D. López-Cabrera, and S.~B.
Huergo-Suárez, {\em Cervical Cell Classification Using Features Related to
Morphometry and Texture of Nuclei}.
\newblock Springer Berlin Heidelberg, 2013.
\bibitem{Dundar2011Computerized}
M.~M. Dundar, S.~Badve, G.~Bilgin, V.~Raykar, R.~Jain, O.~Sertel, and M.~N.
Gurcan, ``Computerized classification of intraductal breast lesions using
histopathological images.,'' {\em IEEE Transactions on Biomedical
Engineering}, vol.~58, no.~7, pp.~1977--1984, 2011.
\bibitem{Nguyen2011Prostate}
K.~Nguyen, A.~K. Jain, and B.~Sabata, ``Prostate cancer detection: Fusion of
cytological and textural features,'' {\em Journal of Pathology Informatics},
vol.~2, no.~1, p.~1, 2011.
\bibitem{Tai2011Blood}
W.~L. Tai, R.~M. Hu, C.~W.~H. Han, R.~M. Chen, and J.~J.~P. Tsai, ``Blood cell
image classification based on hierarchical svm,'' in {\em IEEE International
Symposium on Multimedia}, pp.~129--136, 2011.
\bibitem{Putzu2014Leucocyte}
L.~Putzu, G.~Caocci, and C.~D. Ruberto, ``Leucocyte classification for
leukaemia detection using image processing techniques,'' {\em Artificial
Intelligence in Medicine}, vol.~62, no.~3, pp.~179--191, 2014.
\bibitem{Su2014A}
M.~C. Su, C.~Y. Cheng, and P.~C. Wang, ``A neural-network-based approach to
white blood cell classification,'' {\em The scientific world journal},
vol.~2014, no.~4, p.~796371, 2014.
\bibitem{xu2017large}
Y.~Xu, Z.~Jia, L.-B. Wang, Y.~Ai, F.~Zhang, M.~Lai, I.~Eric, and C.~Chang,
``Large scale tissue histopathology image classification, segmentation, and
visualization via deep convolutional activation features,'' {\em BMC
bioinformatics}, vol.~18, no.~1, p.~281, 2017.
\bibitem{xie2016unsupervised}
J.~Xie, R.~Girshick, and A.~Farhadi, ``Unsupervised deep embedding for
clustering analysis,'' in {\em International Conference on Machine Learning},
pp.~478--487, 2016.
\bibitem{kingma2013auto}
D.~P. Kingma and M.~Welling, ``Auto-encoding variational bayes,'' {\em arXiv
preprint arXiv:1312.6114}, 2013.
\bibitem{Xu2016Stacked}
J.~Xu, L.~Xiang, Q.~Liu, H.~Gilmore, J.~Wu, J.~Tang, and A.~Madabhushi,
``Stacked sparse autoencoder (ssae) for nuclei detection on breast cancer
histopathology images.,'' {\em IEEE Transactions on Medical Imaging},
vol.~35, no.~1, pp.~119--130, 2016.
\bibitem{Cruzroa2013A}
A.~A. Cruzroa, J.~E. Arevalo~Ovalle, A.~Madabhushi, and F.~A. González~Osorio,
``A deep learning architecture for image representation, visual
interpretability and automated basal-cell carcinoma cancer detection.,'' in
{\em International Conference on Medical Image Computing and
Computer-Assisted Intervention}, pp.~403--410, 2013.
\bibitem{zhang2016fusing}
X.~Zhang, W.~Liu, H.~Dou, T.~Ju, J.~Xu, and S.~Zhang, ``Fusing heterogeneous features from stacked sparse autoencoder for histopathological image analysis,''{\em
IEEE Journal of Biomedical and Health Informatics}, vol.~20, no.~5, pp.~1377--1383, 2016.
\bibitem{dilokthanakul2016deep}
N.~Dilokthanakul, P.~A. Mediano, M.~Garnelo, M.~C. Lee, H.~Salimbeni,
K.~Arulkumaran, and M.~Shanahan, ``Deep unsupervised clustering with gaussian
mixture variational autoencoders,'' {\em arXiv preprint arXiv:1611.02648},
2016.
\bibitem{jiang2017variational}
Z.~Jiang, Y.~Zheng, H.~Tan, B.~Tang, and H.~Zhou, ``Variational deep embedding:
An unsupervised and generative approach to clustering,'' in {\em
International Joint Conference on Artificial Intelligence}, 2017.
\bibitem{springenberg2015unsupervised}
J.~T. Springenberg, ``Unsupervised and semi-supervised learning with
categorical generative adversarial networks,'' {\em arXiv preprint
arXiv:1511.06390}, 2015.
\bibitem{Malon2013Classification}
C.~D. Malon and C.~Eric, ``Classification of mitotic figures with convolutional
neural networks and seeded blob features,'' {\em Journal of Pathology
Informatics}, vol.~4, no.~1, p.~9, 2013.
\bibitem{Mohapatra2014An}
S.~Mohapatra, D.~Patra, and S.~Satpathy, {\em An ensemble classifier system for
early diagnosis of acute lymphoblastic leukemia in blood microscopic images}.
\newblock Springer-Verlag, 2014.
\bibitem{Zhao2016Automatic}
J.~Zhao, M.~Zhang, Z.~Zhou, J.~Chu, and F.~Cao, ``Automatic detection and
classification of leukocytes using convolutional neural networks,'' {\em
Medical \& biological engineering \& computing}, vol.~55, no.~8,
pp.~1287--1301, 2017.
\bibitem{Sirinukunwattana2016Locality}
K.~Sirinukunwattana, S.~E. Ahmed~Raza, Y.~W. Tsang, D.~R. Snead, I.~A. Cree,
and N.~M. Rajpoot, ``Locality sensitive deep learning for detection and
classification of nuclei in routine colon cancer histology images,'' {\em
IEEE Transactions on Medical Imaging}, vol.~35, no.~5, p.~1196, 2016.
\bibitem{hou2016automatic}
L.~Hou, K.~Singh, D.~Samaras, T.~M. Kurc, Y.~Gao, R.~J. Seidman, and J.~H.
Saltz, ``Automatic histopathology image analysis with cnns,'' in {\em
Scientific Data Summit (NYSDS)}, pp.~1--6, 2016.
\bibitem{zhang2015weighted}
X.~Zhang, H.~Su, L.~Yang, and S.~Zhang, ``Weighted hashing with multiple cues
for cell-level analysis of histopathological images,'' in {\em International
Conference on Information Processing in Medical Imaging}, pp.~303--314,
Springer, 2015.
\bibitem{zhang2015towards}
X.~Zhang, W.~Liu, M.~Dundar, S.~Badve, and S.~Zhang, ``Towards large-scale histopathological image analysis: Hashing-based image retrieval,''{\em
IEEE Transactions on Medical Imaging}, vol.~34, no.~2, pp.~496--506, 2015.
\bibitem{zhang2015high}
X.~Zhang, F.~Xing, H.~Su, L.~Yang, and S.~Zhang, ``High-throughput histopathological image analysis via robust cell segmentation and hashing,''{\em
Medical image analysis}, vol.~26, no.~1, pp.~306--315, 2015.
\bibitem{shi2017cell}
X.~Shi, F.~Xing, Y.~Xie, H.~Su, and L.~Yang, ``Cell encoding for histopathology
image classification,'' in {\em International Conference on Medical Image
Computing and Computer-Assisted Intervention}, pp.~30--38, Springer, 2017.
\bibitem{Callau2015Evaluation}
C.~Callau, M.~Lejeune, A.~Korzynska, M.~García, G.~Bueno, R.~Bosch, J.~Jaén,
G.~Orero, T.~Salvadó, and C.~López, ``Evaluation of cytokeratin-19 in
breast cancer tissue samples: a comparison of automatic and manual
evaluations of scanned tissue microarray cylinders.,'' {\em Biomedical
Engineering Online}, vol.~14, no.~S2, p.~S2, 2015.
\bibitem{Wienert2012Detection}
S.~Wienert, D.~Heim, S.~Kai, A.~Stenzinger, M.~Beil, P.~Hufnagl, M.~Dietel,
C.~Denkert, and F.~Klauschen, ``Detection and segmentation of cell nuclei in
virtual microscopy images: A minimum-model approach,'' {\em Scientific
Reports}, vol.~2, no.~7, p.~503, 2012.
\bibitem{Dorini2013Semiautomatic}
L.~B. Dorini, R.~Minetto, and N.~J. Leite, ``Semiautomatic white blood cell
segmentation based on multiscale analysis.,'' {\em IEEE Journal of Biomedical
and Health Informatics}, vol.~17, no.~1, p.~250, 2013.
\bibitem{Schmitt2009Morphological}
O.~Schmitt and M.~Hasse, ``Morphological multiscale decomposition of connected
regions with emphasis on cell clusters,'' {\em Computer Vision \& Image
Understanding}, vol.~113, no.~2, pp.~188--201, 2009.
\bibitem{Dzyubachyk2010Advanced}
O.~Dzyubachyk, W.~A. van Cappellen, J.~Essers, W.~J. Niessen, and E.~Meijering,
``Advanced level-set-based cell tracking in time-lapse fluorescence
microscopy.,'' {\em IEEE Transactions on Medical Imaging}, vol.~29, no.~3,
pp.~852--867, 2010.
\bibitem{Long2009A}
F.~Long, H.~Peng, X.~Liu, S.~K. Kim, and E.~Myers, ``A 3d digital atlas of c.
elegans and its application to single-cell analyses.,'' {\em Nature Methods},
vol.~6, no.~9, p.~667, 2009.
\bibitem{Hai2014Automatic}
S.~Hai, F.~Xing, J.~D. Lee, C.~A. Peterson, and Y.~Lin, ``Automatic myonuclear
detection in isolated single muscle fibers using robust ellipse fitting and
sparse representation,'' {\em IEEE/ACM Transactions on Computational Biology
\& Bioinformatics}, vol.~11, no.~4, pp.~714--726, 2014.
\bibitem{Bueno2012A}
G.~Bueno, R.~González, O.~Déniz, M.~Garcíarojo, J.~Gonzálezgarcía, M.~M.
Fernándezcarrobles, N.~Vállez, and J.~Salido, ``A parallel solution for
high resolution histological image analysis.,'' {\em Computer Methods \&
Programs in Biomedicine}, vol.~108, no.~1, pp.~388--401, 2012.
\bibitem{Chang2013Invariant}
H.~Chang, J.~Han, A.~Borowsky, L.~Loss, J.~W. Gray, P.~T. Spellman, and
B.~Parvin, ``Invariant delineation of nuclear architecture in glioblastoma
multiforme for clinical and molecular association,'' {\em IEEE Transactions
on Medical Imaging}, vol.~32, no.~4, pp.~670--682, 2013.
\bibitem{Arslan2013Attributed}
S.~Arslan, T.~Ersahin, R.~Cetin-Atalay, and C.~Gunduz-Demir, ``Attributed
relational graphs for cell nucleus segmentation in fluorescence microscopy
images,'' {\em IEEE Transactions on Medical Imaging}, vol.~32, no.~6,
pp.~1121--1131, 2013.
\bibitem{nie2017medical}
D.~Nie, R.~Trullo, J.~Lian, C.~Petitjean, S.~Ruan, Q.~Wang, and D.~Shen,
``Medical image synthesis with context-aware generative adversarial
networks,'' in {\em International Conference on Medical Image Computing and
Computer-Assisted Intervention}, pp.~417--425, 2017.
\bibitem{Li2017Reconstruction}
Z.~Li, Y.~Wang, and J.~Yu, ``Reconstruction of thin-slice medical images using
generative adversarial network,'' in {\em International Workshop on Machine
Learning in Medical Imaging}, pp.~325--333, Springer, 2017.
\bibitem{Mahapatra2017Image}
D.~Mahapatra, B.~Bozorgtabar, S.~Hewavitharanage, and R.~Garnavi, ``Image super
resolution using generative adversarial networks and local saliency maps for
retinal image analysis,'' in {\em International Conference on Medical Image
Computing and Computer-Assisted Intervention}, pp.~382--390, 2017.
\bibitem{Wolterink2017Generative}
J.~M. Wolterink, T.~Leiner, M.~A. Viergever, and I.~Isgum, ``Generative
adversarial networks for noise reduction in low-dose ct.,'' {\em IEEE
Transactions on Medical Imaging}, vol.~PP, no.~99, p.~1, 2017.
\bibitem{Reinhard2001Color}
E.~Reinhard, M.~Ashikhmin, B.~Gooch, and P.~Shirley, ``Color transfer between
images,'' {\em IEEE Computer Graphics \& Applications}, vol.~21, no.~5,
pp.~34--41, 2001.
\bibitem{Macenko2009A}
M.~Macenko, M.~Niethammer, J.~S. Marron, D.~Borland, J.~T. Woosley, X.~Guan,
C.~Schmitt, and N.~E. Thomas, ``A method for normalizing histology slides for
quantitative analysis,'' in {\em IEEE International Conference on Symposium
on Biomedical Imaging: From Nano To Macro}, pp.~1107--1110, 2009.
\bibitem{kainz2015you}
P.~Kainz, M.~Urschler, S.~Schulter, P.~Wohlhart, and V.~Lepetit, ``You should
use regression to detect cells,'' in {\em International Conference on Medical
Image Computing and Computer Assisted Intervention}, pp.~276--283, 2015.
\bibitem{arthur2007k}
D.~Arthur and S.~Vassilvitskii, ``k-means++: The advantages of careful
seeding,'' in {\em Proceedings of the eighteenth annual ACM-SIAM symposium on
Discrete algorithms}, pp.~1027--1035, Society for Industrial and Applied
Mathematics, 2007.
\bibitem{kingma2014adam}
D.~Kingma and J.~Ba, ``Adam: A method for stochastic optimization,'' {\em arXiv
preprint arXiv:1412.6980}, 2014.
\bibitem{lowe2004distinctive}
D.~G. Lowe, ``Distinctive image features from scale-invariant keypoints,'' {\em
International journal of computer vision}, vol.~60, no.~2, pp.~91--110, 2004.
\bibitem{ojala2002multiresolution}
T.~Ojala, M.~Pietikainen, and T.~Maenpaa, ``Multiresolution gray-scale and
rotation invariant texture classification with local binary patterns,'' {\em
IEEE Transactions on Pattern Analysis and Machine Intelligence}, vol.~24,
no.~7, pp.~971--987, 2002.
\end{thebibliography}
\end{document}