Pyramid Embedded Generative Adversarial Network for Automated Font Generation
Extracted source text · arXiv 1811.08106
%% bare_conf.tex
%% V1.4b
%% 2015/08/26
%% by Michael Shell
%% See:
%% http://www.michaelshell.org/
%% for current contact information.
%%
%% This is a skeleton file demonstrating the use of IEEEtran.cls
%% (requires IEEEtran.cls version 1.8b or later) with an IEEE
%% conference paper.
%%
%% Support sites:
%% http://www.michaelshell.org/tex/ieeetran/
%% http://www.ctan.org/pkg/ieeetran
%% and
%% http://www.ieee.org/
%%*************************************************************************
%% Legal Notice:
%% This code is offered as-is without any warranty either expressed or
%% implied; without even the implied warranty of MERCHANTABILITY or
%% FITNESS FOR A PARTICULAR PURPOSE!
%% User assumes all risk.
%% In no event shall the IEEE or any contributor to this code be liable for
%% any damages or losses, including, but not limited to, incidental,
%% consequential, or any other damages, resulting from the use or misuse
%% of any information contained here.
%%
%% All comments are the opinions of their respective authors and are not
%% necessarily endorsed by the IEEE.
%%
%% This work is distributed under the LaTeX Project Public License (LPPL)
%% ( http://www.latex-project.org/ ) version 1.3, and may be freely used,
%% distributed and modified. A copy of the LPPL, version 1.3, is included
%% in the base LaTeX documentation of all distributions of LaTeX released
%% 2003/12/01 or later.
%% Retain all contribution notices and credits.
%% ** Modified files should be clearly indicated as such, including **
%% ** renaming them and changing author support contact information. **
%%*************************************************************************
% *** Authors should verify (and, if needed, correct) their LaTeX system ***
% *** with the testflow diagnostic prior to trusting their LaTeX platform ***
% *** with production work. The IEEE's font choices and paper sizes can ***
% *** trigger bugs that do not appear when using other class files. *** ***
% The testflow support page is at:
% http://www.michaelshell.org/tex/testflow/
\documentclass[a4paper,conference]{IEEEtran}
% Some Computer Society conferences also require the compsoc mode option,
% but others use the standard conference format.
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[conference]{../sty/IEEEtran}
\usepackage{amsfonts}
\usepackage{url}
% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)
% *** MISC UTILITY PACKAGES ***
%
%\usepackage{ifpdf}
% Heiko Oberdiek's ifpdf.sty is very useful if you need conditional
% compilation based on whether the output is pdf or dvi.
% usage:
% \ifpdf
% % pdf code
% \else
% % dvi code
% \fi
% The latest version of ifpdf.sty can be obtained from:
% http://www.ctan.org/pkg/ifpdf
% Also, note that IEEEtran.cls V1.7 and later provides a builtin
% \ifCLASSINFOpdf conditional that works the same way.
% When switching from latex to pdflatex and vice-versa, the compiler may
% have to be run twice to clear warning/error messages.
% *** CITATION PACKAGES ***
%
%\usepackage{cite}
% cite.sty was written by Donald Arseneau
% V1.6 and later of IEEEtran pre-defines the format of the cite.sty package
% \cite{} output to follow that of the IEEE. Loading the cite package will
% result in citation numbers being automatically sorted and properly
% "compressed/ranged". e.g., [1], [9], [2], [7], [5], [6] without using
% cite.sty will become [1], [2], [5]--[7], [9] using cite.sty. cite.sty's
% \cite will automatically add leading space, if needed. Use cite.sty's
% noadjust option (cite.sty V3.8 and later) if you want to turn this off
% such as if a citation ever needs to be enclosed in parenthesis.
% cite.sty is already installed on most LaTeX systems. Be sure and use
% version 5.0 (2009-03-20) and later if using hyperref.sty.
% The latest version can be obtained at:
% http://www.ctan.org/pkg/cite
% The documentation is contained in the cite.sty file itself.
% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
\usepackage[pdftex]{graphicx}
% declare the path(s) where your graphic files are
\graphicspath{{../pdf/}{../jpeg/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
\DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
% or other class option (dvipsone, dvipdf, if not using dvips). graphicx
% will default to the driver specified in the system graphics.cfg if no
% driver is specified.
\usepackage[dvips]{graphicx}
% declare the path(s) where your graphic files are
\graphicspath{{../eps/}}
% and their extensions so you won't have to specify these with
% every instance of \includegraphics
\DeclareGraphicsExtensions{.eps}
\fi
% graphicx was written by David Carlisle and Sebastian Rahtz. It is
% required if you want graphics, photos, etc. graphicx.sty is already
% installed on most LaTeX systems. The latest version and documentation
% can be obtained at:
% http://www.ctan.org/pkg/graphicx
% Another good source of documentation is "Using Imported Graphics in
% LaTeX2e" by Keith Reckdahl which can be found at:
% http://www.ctan.org/pkg/epslatex
%
% latex, and pdflatex in dvi mode, support graphics in encapsulated
% postscript (.eps) format. pdflatex in pdf mode supports graphics
% in .pdf, .jpeg, .png and .mps (metapost) formats. Users should ensure
% that all non-photo figures use a vector format (.eps, .pdf, .mps) and
% not a bitmapped formats (.jpeg, .png). The IEEE frowns on bitmapped formats
% which can result in "jaggedy"/blurry rendering of lines and letters as
% well as large increases in file sizes.
%
% You can find documentation about the pdfTeX application at:
% http://www.tug.org/applications/pdftex
% *** MATH PACKAGES ***
%
%\usepackage{amsmath}
% A popular package from the American Mathematical Society that provides
% many useful and powerful commands for dealing with mathematics.
%
% Note that the amsmath package sets \interdisplaylinepenalty to 10000
% thus preventing page breaks from occurring within multiline equations. Use:
%\interdisplaylinepenalty=2500
% after loading amsmath to restore such page breaks as IEEEtran.cls normally
% does. amsmath.sty is already installed on most LaTeX systems. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/amsmath
% *** SPECIALIZED LIST PACKAGES ***
%
%\usepackage{algorithmic}
% algorithmic.sty was written by Peter Williams and Rogerio Brito.
% This package provides an algorithmic environment fo describing algorithms.
% You can use the algorithmic environment in-text or within a figure
% environment to provide for a floating algorithm. Do NOT use the algorithm
% floating environment provided by algorithm.sty (by the same authors) or
% algorithm2e.sty (by Christophe Fiorio) as the IEEE does not use dedicated
% algorithm float types and packages that provide these will not provide
% correct IEEE style captions. The latest version and documentation of
% algorithmic.sty can be obtained at:
% http://www.ctan.org/pkg/algorithms
% Also of interest may be the (relatively newer and more customizable)
% algorithmicx.sty package by Szasz Janos:
% http://www.ctan.org/pkg/algorithmicx
% *** ALIGNMENT PACKAGES ***
%
%\usepackage{array}
% Frank Mittelbach's and David Carlisle's array.sty patches and improves
% the standard LaTeX2e array and tabular environments to provide better
% appearance and additional user controls. As the default LaTeX2e table
% generation code is lacking to the point of almost being broken with
% respect to the quality of the end results, all users are strongly
% advised to use an enhanced (at the very least that provided by array.sty)
% set of table tools. array.sty is already installed on most systems. The
% latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/array
% IEEEtran contains the IEEEeqnarray family of commands that can be used to
% generate multiline equations as well as matrices, tables, etc., of high
% quality.
% *** SUBFIGURE PACKAGES ***
%\ifCLASSOPTIONcompsoc
% \usepackage[caption=false,font=normalsize,labelfont=sf,textfont=sf]{subfig}
%\else
% \usepackage[caption=false,font=footnotesize]{subfig}
%\fi
% subfig.sty, written by Steven Douglas Cochran, is the modern replacement
% for subfigure.sty, the latter of which is no longer maintained and is
% incompatible with some LaTeX packages including fixltx2e. However,
% subfig.sty requires and automatically loads Axel Sommerfeldt's caption.sty
% which will override IEEEtran.cls' handling of captions and this will result
% in non-IEEE style figure/table captions. To prevent this problem, be sure
% and invoke subfig.sty's "caption=false" package option (available since
% subfig.sty version 1.3, 2005/06/28) as this is will preserve IEEEtran.cls
% handling of captions.
% Note that the Computer Society format requires a larger sans serif font
% than the serif footnote size font used in traditional IEEE formatting
% and thus the need to invoke different subfig.sty package options depending
% on whether compsoc mode has been enabled.
%
% The latest version and documentation of subfig.sty can be obtained at:
% http://www.ctan.org/pkg/subfig
% *** FLOAT PACKAGES ***
%
%\usepackage{fixltx2e}
% fixltx2e, the successor to the earlier fix2col.sty, was written by
% Frank Mittelbach and David Carlisle. This package corrects a few problems
% in the LaTeX2e kernel, the most notable of which is that in current
% LaTeX2e releases, the ordering of single and double column floats is not
% guaranteed to be preserved. Thus, an unpatched LaTeX2e can allow a
% single column figure to be placed prior to an earlier double column
% figure.
% Be aware that LaTeX2e kernels dated 2015 and later have fixltx2e.sty's
% corrections already built into the system in which case a warning will
% be issued if an attempt is made to load fixltx2e.sty as it is no longer
% needed.
% The latest version and documentation can be found at:
% http://www.ctan.org/pkg/fixltx2e
%\usepackage{stfloats}
% stfloats.sty was written by Sigitas Tolusis. This package gives LaTeX2e
% the ability to do double column floats at the bottom of the page as well
% as the top. (e.g., "\begin{figure*}[!b]" is not normally possible in
% LaTeX2e). It also provides a command:
%\fnbelowfloat
% to enable the placement of footnotes below bottom floats (the standard
% LaTeX2e kernel puts them above bottom floats). This is an invasive package
% which rewrites many portions of the LaTeX2e float routines. It may not work
% with other packages that modify the LaTeX2e float routines. The latest
% version and documentation can be obtained at:
% http://www.ctan.org/pkg/stfloats
% Do not use the stfloats baselinefloat ability as the IEEE does not allow
% \baselineskip to stretch. Authors submitting work to the IEEE should note
% that the IEEE rarely uses double column equations and that authors should try
% to avoid such use. Do not be tempted to use the cuted.sty or midfloat.sty
% packages (also by Sigitas Tolusis) as the IEEE does not format its papers in
% such ways.
% Do not attempt to use stfloats with fixltx2e as they are incompatible.
% Instead, use Morten Hogholm'a dblfloatfix which combines the features
% of both fixltx2e and stfloats:
%
% \usepackage{dblfloatfix}
% The latest version can be found at:
% http://www.ctan.org/pkg/dblfloatfix
% *** PDF, URL AND HYPERLINK PACKAGES ***
%
%\usepackage{url}
% url.sty was written by Donald Arseneau. It provides better support for
% handling and breaking URLs. url.sty is already installed on most LaTeX
% systems. The latest version and documentation can be obtained at:
% http://www.ctan.org/pkg/url
% Basically, \url{my_url_here}.
% *** Do not adjust lengths that control margins, column widths, etc. ***
% *** Do not use packages that alter fonts (such as pslatex). ***
% There should be no need to do such things with IEEEtran.cls V1.6 and later.
% (Unless specifically asked to do so by the journal or conference you plan
% to submit to, of course. )
% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
%
% paper title
% Titles are generally capitalized except for words such as a, an, and, as,
% at, but, by, for, in, nor, of, on, or, the, to and up, which are usually
% not capitalized unless they are the first or last word of the title.
% Linebreaks \\ can be used within to get better formatting as desired.
% Do not put math or special symbols in the title.
\title{Pyramid Embedded Generative Adversarial Network \\for Automated Font Generation}
% author names and affiliations
% use a multiple column layout for up to three different
% affiliations
\author{\IEEEauthorblockN{Donghui Sun}
\IEEEauthorblockA{Alibaba\\
ruya.sdh@alibaba-inc.com}
\and
\IEEEauthorblockN{Qing Zhang}
\IEEEauthorblockA{Alibaba\\
sensi.zq@alibaba-inc.com}
\and
\IEEEauthorblockN{Jun Yang}
\IEEEauthorblockA{Alibaba\\
muzhuo.yj@alibaba-inc.com}}
% conference papers do not typically use \thanks and this command
% is locked out in conference mode. If really needed, such as for
% the acknowledgment of grants, issue a \IEEEoverridecommandlockouts
% after \documentclass
% for over three affiliations, or if they all won't fit within the width
% of the page, use this alternative format:
%
%\author{\IEEEauthorblockN{Michael Shell\IEEEauthorrefmark{1},
%Homer Simpson\IEEEauthorrefmark{2},
%James Kirk\IEEEauthorrefmark{3},
%Montgomery Scott\IEEEauthorrefmark{3} and
%Eldon Tyrell\IEEEauthorrefmark{4}}
%\IEEEauthorblockA{\IEEEauthorrefmark{1}School of Electrical and Computer Engineering\\
%Georgia Institute of Technology,
%Atlanta, Georgia 30332--0250\\ Email: see http://www.michaelshell.org/contact.html}
%\IEEEauthorblockA{\IEEEauthorrefmark{2}Twentieth Century Fox, Springfield, USA\\
%Email: homer@thesimpsons.com}
%\IEEEauthorblockA{\IEEEauthorrefmark{3}Starfleet Academy, San Francisco, California 96678-2391\\
%Telephone: (800) 555--1212, Fax: (888) 555--1212}
%\IEEEauthorblockA{\IEEEauthorrefmark{4}Tyrell Inc., 123 Replicant Street, Los Angeles, California 90210--4321}}
% use for special paper notices
%\IEEEspecialpapernotice{(Invited Paper)}
% make the title area
\maketitle
% As a general rule, do not put math, special symbols or citations
% in the abstract
\begin{abstract}
In this paper, we investigate the Chinese font synthesis problem and propose a Pyramid Embedded Generative Adversarial Network (PEGAN) to automatically generate Chinese character images. The PEGAN
consists of one generator and one discriminator. The generator is built using one encoder-decoder structure with cascaded refinement connections and mirror skip connections. The cascaded refinement connections embed a multi-scale pyramid of down-sampled original input into the encoder feature maps of different layers, and multi-scale feature maps from the encoder are connected to the corresponding feature maps in the decoder to make the mirror skip connections. Through combining the generative adversarial loss, pixel-wise loss, category loss and perceptual loss, the generator and discriminator can be trained alternately to synthesize character images. In order to verify the effectiveness of our proposed PEGAN, we first build one evaluation set, in which the characters are selected according to their stroke number and frequency of use, and then use both qualitative and quantitative metrics to measure the performance of our model comparing with the baseline method. The experimental results demonstrate the effectiveness of our proposed model, it shows the potential to automatically extend small font banks into complete ones.
\end{abstract}
% no keywords
% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle
\section{Introduction}
Font, as one basic element, has been widely used in various aspects of the art and design. However, the design of font is a very time-consuming task. During the first stage of developing a font bank, calligrapher needs to write a large amount of characters as templates . And for the second stage, font designers will spend quite a long time to digitize these character templates and manually create all other characters that do not exist in the templates. Particularly, it takes even more time to design one Chinese font which has a much larger dictionary compared with English or Latin that includes
only tens of letters. Therefore, a more efficient way to automatically generate a font bank is desperately needed.
There are some attempts to synthesize Chinese characters automatically. The most typical method is based on stroke extraction \cite{cao2000model}. In this kind of methods, font generation procedure is divided into stroke extraction and recombination of isolated strokes. Effective stroke extraction from character image plays a crucial role to determine the performance of the font generation. However, the current stroke extraction algorithms can not always work well due to the complexity and diversity of Chinese characters.
Generative adversarial network (GAN) \cite{Goodfellow2014Generative} is increasingly being used for image generation. The original GAN has one generator and one discriminator. The generator can generate realistic images and try to fool the discriminator. The discriminator is used to classify real images and generated images. The generator and discriminator can be trained alternately by one minmax policy \cite{Goodfellow2014Generative}. In \cite{mirza2014conditional}, conditional GAN is proposed by feeding an auxiliary information such as image class labels together with original input of GAN to generator and discriminator. In \cite{Isola2016Image}, $pix2pix$ is proposed to transform images from one style to another based on conditional GAN. Similar to $pix2pix$, $zi2zi$ \cite{zi2zi} is also built on conditional GAN and it is the first attempt to automatically generate font images.
There is another type of method which is not based on adversarial learning and also can be used for image generation. In \cite{Chen2017Photographic}, one convolutional network which is denoted as cascaded refinement network (CRN) is proposed to synthesize photographic images conditioned on pixel-wise semantic layouts. The backbone of CRN is one feedforward convolutional network that consists of several cascaded refinement modules. Each module operates at a given resolution, which is doubled between consecutive modules.
%The semantic layout $L$ is down-sampled to $w\times h$ and fed into the first module. The second module receives a concatenation of down-sampled layout $L$ ($2w\times2h$) and outputs of the first module, takes it as the input and computes its output feature maps. All other modules are of the same structure.
This design feeds low level information of input image to different layers, preserves much more detailed information during training and results in high-resolution images generation. However, the input and output of CRN are supposed to have identical layout. Thus, it cannot handle the image transformation.
%In this paper, we also treat the task of font generation as image to image transformation.
%The most relevant work of our model is $zi2zi$ \cite{zi2zi} and cascaded refinement network (CRN) \cite{Chen2017Photographic}. The generator of $zi2zi$ uses U-Net structure. The input images go through multiple layers. And the low level information will be reduced during feature extraction if no extra supplement. CRN \cite{Chen2017Photographic} is not built on adversarial learning. It deploys a convolutional network to synthesize photographic images conditioned on pixel-wise semantic layouts. The backbone of CRN is one feedforward convolutional network that consisting of several cascaded refinement modules. Each module operates at a given resolution. Resolution is doubled between consecutive modules. The semantic layout $L$ is downsampled into $w\times h$ and then fed into the first module. The second module receives a concatenation of the layout $L$ (downsampled to $2w\times2h$) and the featuremaps of first module as input and produce new featuremaps. All other modules are identical structure. This design feeds low level information of input image to different layers, preserves much more detailed information during training and can be used to generated high-resolution images. However, the input and output of CRN have identical layout. Thus, it cannot handle image transformation.
In this work, we treat the task of font generation as image to image transformation and propose a new method for automated font generation. The contributions of this paper are as follows:
\begin{enumerate}
\item
%Inspired by $zi2zi$ and CRN,
We propose PEGAN by embedding multi-scale pyramid of refinement information into U-Net \cite{ronneberger2015u}. This design can enable generator preserve much more detailed information and is beneficial for model performance.
\item We deploy four loss functions in training procedure: adversarial loss, pixel-wise loss, category loss and perceptual loss. The adversarial loss is beneficial to generate images with fine details. The pixel loss indicates the pixel space distance between real and generated images, while perceptual loss measures the discrepancy between them from perceptual aspect. The category loss, which is important for pre-training, enables the model to learn from multiple styles simultaneously.
\item We build a character set for evaluation based on stroke number and frequency of use. We conduct both qualitative and quantitative measurements to evaluate the performance of proposed font generation model.
\end{enumerate}
The organization of the rest of paper is as follows. In the next Section, we review recent methods for font generation. In Section \ref{pegan}, the framework of PEGAN is introduced, as well as loss functions and implementation details. Section \ref{exp} shows experimental results, where we report our results using Microsoft HeiTi to generate HuaKang font. Also, we deploy PEGAN performing small font banks extension. Finally, we conclude the paper in Section \ref{conclusion}.
\section{Related Work}
\label{related_work}
In this section we review the relevant work for font generation. There are mainly two categories: stroke extraction based methods and image generation based methods.
\subsection{Font generation based on stroke extraction}
Stroke is the basic unit of Chinese characters. The stroke extraction based methods are the dominant way to generate font in early stage. In \cite{cao2000model}, the strokes of both source font and target font are first extracted. Then autoencoder and self-organizing maps are adopted to cluster the extracted strokes into 100 different groups. Finally, new characters of target font are generated by stroke replacement. In \cite{miyazaki2017automatic}, one automatic extrapolation method for small font is proposed. The strokes of characters from one small subset are first extracted to form a stroke pool. In addition, one transformation matrix between source font and target font is learnt by the corresponding relationship between their skeletons. The missing target characters are generated based on strokes and transformed skeletons. This kind of methods mainly rely on stroke extraction. Moreover, the stroke extraction of calligraphic font is still one challenging problem.
\subsection{Font generation based on GAN}
Recently, GAN is becoming a very hot topic in machine learning community \cite{Goodfellow2014Generative, Chen2016InfoGAN}. As GAN is capable of mapping one distribution into another distribution, these kinds methods can be used to generate realistic fonts.
Inspired by \cite{Isola2016Image}, $zi2zi$ \cite{zi2zi} is the first attempt to generate fonts using GAN. The generator of $zi2zi$ is based on the U-Net structure, as shown in Fig. \ref{fig_unet}. It employs different loss functions, including adversarial loss, L1 loss, and const loss. The $zi2zi$ also introduces category loss such that it can perform one-to-many learning mode. Besides $zi2zi$, there are several GAN based methods for font generation. In \cite{Chang2017Chinese}, the category loss is dropped because it argues that embedded category information can degrade the generation quality for calligraphic fonts. In \cite{Lyu2017Auto}, apart from generator which captures the layout of input font image, it also applys another encoder-decoder network acting as supervisor which reconstructs source images to guide the generator to learn the detailed stroke information.
Compared with stroke based methods, GAN based methods consider font generation as image-to-image transformation. It does not rely stroke extraction and can be trained by end to end. %As GAN is capable of mapping one distribution into another distribution. Thus, these kinds methods can generate realistic fonts.
\section{Pyramid Embedded GAN}
\label{pegan}
In this section, we introduce the proposed PEGAN for automated font generation. Firstly, we explain the whole framework of the proposed PEGAN. Then, we explain four different loss functions deployed in PEGAN: the adversarial, pixel-wise, category and perceptual losses, respectively. Finally, we illustrate the details of the training procedure.
\subsection{Framework overview}
We develop PEGAN framework, which consists of an image generator $G$, and a discriminator $D$, as shown in Fig. \ref{fig_pegan}.
\subsubsection{Generator}
Encoder-decoder network has been used in many previous solutions for image-to-image transformation \cite{hinton2006reducing}. In such a network, the input is progressively down-sampled and passed through multiple layers until a bottleneck layer, as shown in Fig. \ref{fig_unet}. For image-to-image transformation task the input image of high resolution are mapped to another image. The input and output differ in appearance, but have similar underlying structures. Thus, it should be considered that input and output should share a great deal of low level information.
In \cite{Isola2016Image}, the structure of U-Net with skip connections is introduced for shuttling this information directly from encoder to decoder. Specifically, skip connections are added between each module $i$ and module $n-i$, where $n$ is the total number of modules. Each skip connection simply concatenates all channels at module $i$ in encoder with those at module $n-i$ in decoder.
Based on this design, we propose a refinement U-Net by embedding a multi-scale pyramid of down-sampled original input into encoder feature maps of different layers. We show our design in Fig. \ref{fig_pegan}. To be more specific, the first module $e_{1}$ of encoder receives the original source font images ($w\times h$) as input and produces a feature map $F_{1}$ at resolution $\frac{w}{2} \times \frac{h}{2}$ as output. All other modules $e_{i}$ (for i $\neq$ 8) are structurally identical: $e_{i}$ receives a concatenation of the source font images (downsampled to $\frac{w}{2^{i}}\times \frac{h}{2^{i}}$) and the feature layer $F_{i-1}$ as input, and produces feature layer $F_{i}$ as output. The feature maps, $F_{i}$, are connected to corresponding feature maps in decoder to make mirror skip connections.
\begin{figure*}[!t]
\centering
\includegraphics[width=0.9\linewidth]{eps/unet}
\caption{Three choices for the architecture of the generator. The common choice is encoder-decoder. The U-Net is an encoder-decoder with skip connections between mirrored layers in the encoder and decoder stacks. We design the refinement U-Net by adding multiple scale inputs for encoder. Besides the original input, downsampled inputs are fed into each module of encoder. The skips from encoder to decoder are also reserved.}
\label{fig_unet}
\end{figure*}
\begin{figure*}[!t]
\centering
\includegraphics[width=0.9\linewidth,height=8cm]{eps/pegan}
% where an .eps filename suffix will be assumed under latex,
% and a .pdf suffix will be assumed for pdflatex; or what has been declared
% via \DeclareGraphicsExtensions.
\caption{The PEGAN framework. The backbone of Generator is U-Net with cascaded refinement connections. $G$ mainly consists of one $Encoder$ and $Decoder$. Both $Encoder$ and $Decoder$ are formed by a series of convolutional modules, $e_{i}$, $d_{i}, i=1,2,...8$, respectively. $e_{i}$ is composed of LReLU-Conv-BN. The $e_{1}$ takes source images as input. The rest of $e_{i}$ takes two different inputs: the first are feature maps outputted by preceding modules, the second are downsampled source images. $d_{i}$ is composed by ReLU-DeConv-BN-Dropout. The $d_{1}$ takes $e_{8}$'s feature map as input. Besides the preceding module's feature map, the rest of $d_{i}$ also takes the feature maps generated in $Encoder$. For example, the feature map of $d_{1}$ and $e_{7}$ are concatenated along $depth$ channel and then fed to $d_{2}$. New images generated by $d_{8}$ are fed to discriminator $D$. The $D$ is composed by Conv-LReLU-\{Conv-BN-LReLU\}x3-FC-Sigmoid. The discriminator $D$ outputs two confidence values, one is for category to which input font images belong, the other one is used to determine whether the input of $D$ are real images or generated images. The pre-trained VGG-19 is used to compute perceptual loss.}
\label{fig_pegan}
\end{figure*}
\subsubsection{Discriminator}
The discriminator $D$ takes real font images and generated images as inputs, and makes classification between them. $D$ is composed of Conv and Leaky ReLU (LReLU), followed by three Conv-BN-LReLU module blocks. The last two layers are full connected layer and Sigmoid layer. $D$ outputs two confidence values. One is the predicted probability of which category the input images belongs to and the other is the probability that input images are ``real" images.
\subsection{Loss functions}
The generator $G$ is trained to produce ``fake'' images that cannot be distinguished from ``real'' images by an adversarially trained discriminator, $D$. In contrary, $D$ is trained to classify ``fake" images from ``real'' images. Thus, it matters how to measure the discrepancy of ``real'' images and ``fake'' images in training process. Four different loss functions are adopted in PEGAN.
\subsubsection{Adversarial loss}
We begin with the generative adversarial loss from original GAN. The generator $G$ is trained to map samples from noise $p_{z}$ to real distribution $p_{data}$ by one minmax game with discriminator $D$. In training stage, $G$ tries to generate fake images with realistic appearance to confuse $D$. Meanwhile, $D$ aims to distinguish the real samples and the generated samples.
%The minmax game can be formulated as:
%
%$$
%\min_{G} \max_{D} \mathbb {E}_{x \in p_{data}} \left[\log D(x) \right ] + \mathbb{E}_{z \in p_{z}} \left[\log (1- D(G(z)) \right ] \eqno{(1)}
%$$
%
As for the task of image to image transformation, we focus on transforming real images in one style to another rather than from random noises to real images, thus the adversarial loss can be written as:
$$
\pounds _{adv} = \mathbb {E}_{x \in p_{data}} \left[\log D(x) \right ] + \\
\mathbb{E}_{z \in p_{input}} \left[\log (1- D(G(z)) \right ] \eqno{(1)}
$$
%\begin{equation}\label{eq:adv_loss}
% \pounds _{adv} = \mathbb {E}_{x \in p_{data}} \left[\log D(x) \right ] + \\
% \mathbb{E}_{z \in p_{input}} \left[\log (1- D(G(z)) \right ] %\eqno{(1)}
% %L=\sum_iL_i^{det}+\lambda \sum_jL_j^{ctc},
%\end{equation}
where $D$ tries to maximize $\pounds _{adv}$ while $G$ minimizes it.
\subsubsection{Pixel-wise loss}
The most widely used pixel-wise loss for generated images and real images are L2 distance \cite{dong2016image, shi2016real-time} and L1 distance \cite{Isola2016Image}. It has been found it is beneficial to mix the GAN adversarial loss with traditional pixel wise loss \cite{Isola2016Image}. We use L1 distance rather than L2 as L1 makes less blurring:
$$
\pounds _{L1} = \mathbb{E}_{x \in p_{data}, z \in p_{input}} \left \| x - G(z)\right \|_{1} \eqno{(2)}
$$
\subsubsection{Category loss}
In order to generate high quality font images, it is important to make the model not only aware of its own style, but other font styles as well. Thus it is essential to enable model to learn multiple font styles at the same time \cite{Johnson2016Google}, thereby a category embedding is used by concatenating a non-trainable gaussian noise as style embedding to the character embedding, right before it enters decoder. To prevent the model from mixing the styles together and generating characters that do not look like any of the provided targets, the multi-class category loss is added to supervise the discriminator to predict the style of the generated characters \cite{zi2zi}. In our implementation, the category loss $\pounds _{cate}$ is calculated using sigmoid cross entropy.
\subsubsection{Perceptual loss}
Both the adversarial loss and pixel wise loss are common metrics in image generation task. %The pixel-wise loss penalizes the discrepancy occurring in the pixel space, but often leads to blurry results.
%Based on the idea of \cite{johnson2016perceptual},
Besides, we use another loss function that is closer to perceptual similarity. The perceptual loss explores the discrepancy between high-dimensional representations of images extracted from a well-trained CNN \cite{johnson2016perceptual}. We define the perceptual loss based on ReLU activation layers of the pre-trained VGG-19 network described in \cite{simonyan2015very}. Layers of different depths can represent image features at different abstraction levels: from edges, colors to object patterns. Matching both lower-level and higher-level activations in the perception network guides the synthesis network to learn fine-grained details as well as global arrangements \cite{Chen2017Photographic}.
Let $\phi$ denotes the layers of VGG-19, for one training pair $\left \langle x,z \right \rangle$ ( $x \in p_{data}, z \in p_{input}$ ), the perceptual loss is defined as
$$
\pounds _{perp} = \sum_{l} \lambda_{l} \left \| \phi_{l}(x) - \phi_{l}(G(z))\right \|_{1} \eqno{(3)}
$$
We use the configuration in \cite{Chen2017Photographic}. For layers $\phi_{l} $ $(l \geq 1)$, the `$conv1\_2$', `$conv2\_2$', `$conv3\_2$', `$conv4\_2$', and `$conv5\_2$' in VGG-19 are chosen to calculate perceptual loss. The $\lambda$ is weight of different layers. In our implementation, we set $\lambda_{1}=\lambda_{2}=\lambda_{3}=\lambda_{4}=1, \lambda_{5}=10$.
In total, the loss of PEGAN is weighted sum of these four losses.
$$
\pounds _{PEGAN}= w_{adv} \pounds _{adv} + w_{L1}\pounds _{L1} + w_{perp}\pounds _{perp} + w_{cate}\pounds _{cate} \eqno{(4)}
$$
Here we set $w_{adv}$ = $w_{perp}$ = $w_{cate}$ = 1, $w_{L1}$ = 100.
\subsection{Implementation details}
In this paper, all images are scaled into 256$\times$256. We take data augmentation by firstly enlarge images and then random cropping. Apart from VGG-19, all convolutional and deconvolutional layers in the above mentioned parts have 5$\times$5 kernel size and 2$\times$2 stride. We choose initial learning rate of 1e-3 and train the proposed model with Adam \cite{kingma2015adam} optimization method.
The entire training process consists of \textit{pre-training} stage and \textit{tuning} stage. In the \textit{pre-training} stage, one source font is used to generate $N$ different fonts. We denote this as one-to-many \textit{pre-training} mode. In the \textit{tuning} stage, we choose one font from these $N$ fonts as the final target. In addition, we freeze the encoder of generator $G$ and update the rest parameters during tuning. In our implementation, we set $N$=20.
%------Experiment-------%
\section{Experiments}
\label{exp}
In this section, we first explain the experimental configuration and the measurements used for image quality evaluation. Then, we evaluate the proposed PEGAN and compare our result with that of baseline.
\subsection{Evaluation setting up and Metrics}
It is an open problem that image quality evaluation in image generation task. The quality evaluation of generated character images are different from other image generation tasks. The basic element of Chinese character is stroke. In general, characters with more strokes are much more difficult to generate with high quality, thus different characters have different difficulty to create. Considering this, we build one evaluation character set according to the number of strokes and frequency of use. As shown in Table \ref{table_easy_mid_hard}, we define three levels of difficulty based on stroke numbers.
Each level contains 100 most commonly used characters.
\begin{table}[htp]
\caption{The evaluation character set}
\begin{center}
\begin{tabular}{c|c|c|c}
\hline
Level & Easy & Mid & Hard \\
\hline
Stroke number& $\leq$ 5 & $\geq$ 6 $\&$ $\leq$ 9 & $\geq$ 10 \\
\hline
Character number & 100 & 100 & 100 \\
\hline
\end{tabular}
\end{center}
\label{table_easy_mid_hard}
\end{table}%
In this work, we used qualitative and quantitative metrics to evaluate the performance of proposed font generated model. For qualitative evaluation, we deployed the `Visual Turing Test' introduced in \cite{shrivastava2017learning}, and designed a simple user study where subjects were asked to classify images as real or generated.
We had 14 subjects in this visual test. Each subject was shown a random selection of 20 real character images and 20 generated character images in a random order, and was asked to label the character images as either real or generated. We use average classification accuracy as qualitative metrics. Meanwhile, we chose several commonly used image quality methods for quantitative evaluation, such as Peak Signal to Noise Ratio (PSNR), Structural Similarity Index (SSIM) \cite{wang2004image}, Universal Quality Index (UQI) \cite{Wang2002A}. Given one pair images, PSNR estimates absolute errors, while SSIM and UQI are perception-based models that consider image quality degradation as perceived change in structural information, and they also incorporates important perceptual phenomena, including both luminance masking and contrast masking terms \cite{wang2003multiscale}.
Besides above two kinds of measurements, we also applied one evaluation method based on character recognition model. We assumed one moderate character recognition model trained using a lot of real images could recognize the generated character images correctly if generated characters have basic shape and layout. On the other hand, if the generated character images suffer from serious quality problem such as lacking of strokes and stroke incompleteness, they cannot be recognized by the model. Thus, the recognition accuracy can be used to indicate the quality of the generated character images.
\subsection{Microsoft HeiTi generates HuaKang POP3 Std W12}
This font, HuaKang POP3 Std W12, is very popular for calligraphic designers. We tried to use Micsosoft HeiTi to generate this font. In \textit{pretraining} stage, we sampled 2000 characters from Microsoft HeiTi and another 20 target fonts as training data, and compared our method with baseline $zi2zi$ method which is built on conditional GAN. The generated result is shown in Fig. \ref{fig_huakang}, from which we come up with a conclusion that some incomplete strokes that emerges in $zi2zi$ are improved a lot by the proposed PEGAN.
The comparison of quantitative evaluation result are shown in Table \ref{table_eval}. Our proposed PEGAN outperforms $zi2zi$ at first three metrics, means that PEGAN can generate images with less absolute errors of pixel content and perceptual structure information. In term of character recognition accuracy, the generated images by PEGAN is easier to be recognized correctly than $zi2zi$. The result of Visual Turing Test was shown in Table \ref{table_turing}. The average human classification accuracy of PEGAN output is 65\%, while that of $zi2zi$ result is 73\%. The lower human classification accuracy means that character images are generated with more realistic appearance so that they can confuse human. It can be concluded that the proposed PEGAN improve the quality of generated character images.
\begin{table}[htp]
\caption{The comparison for quantitative evaluation results of PEGAN and baseline}
\begin{center}
\begin{tabular}{c|c|c|c|c}
\hline
& PSNR & SSIM & UQI & Character recognition model \\
\hline
$zi2zi$ \cite{zi2zi} & 17.45 & 0.83 & 0.26 & 95$\%$ \\
\hline
PEGAN & 17.52 & 0.84 & 0.27 & 96$\%$ \\
\hline
\end{tabular}
\end{center}
\label{table_eval}
\end{table}%
\begin{table}[htp]
\caption{The comparison of Visual Turing Test Result}
\begin{center}
\begin{tabular}{c|c}
\hline
& Classification Accuracy \\
\hline
$zi2zi$ & 73\% \\
\hline
PEGAN & 65\% \\
\hline
\end{tabular}
\end{center}
\label{table_turing}
\end{table}%
\begin{figure}[!t]
\centering
\includegraphics[width=0.9\linewidth]{eps/huakang-res}
\caption{The comparison between generated KuaHang POP3 Std W12 using the proposed PEGAN and $zi2zi$ \cite{zi2zi}, ground truth (GT). In general, $zi2zi$ also has a good result. However, incomplete strokes exist in some characters. We use red circles label these bad cases in $zi2zi$`s result. These bad cases get improved by the proposed PEGAN.}
\label{fig_huakang}
\end{figure}
\subsection{Small font banks expansion}
Besides the generation of HuaKang POP3 Std W12, we apply PEGAN expanding three small font banks: Baizhou KaiTi, ZhaoHeTi and ZCool KuHei. The number of Chinese character in this three fonts are relatively small but all of them are often used in the design of package and banner. We show some examples of these three font styles in Fig. \ref{fig_smallfonts}. Baizhou KaiTi is one Japanese style font containing about 670 Chinese characters. ZhaoHeTi is a Japanese style art font with about 2000 Chinese characters. ZCool KuHei is a crowdfunding font, and it contains 3563 Chinese characters. The three generated characters are shown in Fig. \ref{fig_baizhou}, Fig. \ref{fig_zhaohe} and Fig. \ref{fig_zcool}. All the characters shown in the figures are not included in the training data. We can find that even these characters are not seen by the model during training process, it can use the knowledge obtained by training to handle new characters and achieve style transforming successfully. As a result, the new proposed PEGAN is capable of generating supplementary characters for a font style and automatically extend small font banks into complete ones (6763 characters).
\begin{figure}[!t]
\centering
\includegraphics[width=0.85\linewidth]{eps/smallfonts}
\caption{Style demonstration of three small fonts.}
\label{fig_smallfonts}
\end{figure}
\begin{figure}[!t]
\centering
\includegraphics[width=0.85\linewidth]{eps/baizhoukaiti}
\caption{The generated Baizhou KaiTi images (testing data) using PEGAN. From left to right: Source character, Generated character, Target character ground truth. The number of training sample for Baizhou KaiTi is very limited. It only contains 670 Chinese characters. We also achieve satisfactory results by using the proposed PEGAN.}
\label{fig_baizhou}
\end{figure}
\begin{figure}[!t]
\centering
\includegraphics[width=0.9\linewidth]{eps/zhaoheti}
\caption{The generated ZhaoHeTi testing data using PEGAN.}
\label{fig_zhaohe}
\end{figure}
\begin{figure}[!t]
\centering
\includegraphics[width=0.88\linewidth]{eps/zcool}
%\fbox{\rule{0pt}{2in} \rule{0.8\linewidth}{0pt}}
\caption{The generated ZCool KuHei testing data using PEGAN,}
\label{fig_zcool}
\end{figure}
\section{Conclusion}
\label{conclusion}
In this paper, we propose Pyramid Embedded GAN to synthesize Chinese characters automatically. In PEGAN, the generator mainly consists of U-Net structure with cascaded refinement connections and skip connections. We embed a multi-scale pyramid of down-sampled original input into encoder feature maps of different layers on the one hand, on the other hand, we also shuttle the multi-scale feature maps from encoder to corresponding feature maps in decoder by using skip connections.
In order to measure the discrepancy of real images and generated images in training stage, four different loss functions are deployed: adversarial loss, category loss, pixel-wise loss and perceptual loss. We split the training process into $pre-training$ stage and $tuning$ stage. For a proper evaluation of generated character image quality, we build one evaluation character set based on stroke number and frequency of use. Moreover, we take both qualitative and quantitative metrics to measure the generated character image quality, and build one evaluation method based on character recognition model as well. To verify the performance of our proposed PEGAN, we perform experiment using Microsoft HeiTi to generate HuaKang POP3 Std W12, both qualitative and quantitative evaluation result indicates the effectiveness of PEGAN. We also use PEGAN to extend three small font banks. Although the minimum number of Chinese character of this small fonts is only 670, we still obtain a good result to make more characters for this font. In the future, we will investigate new methods to further improve the fine detailed information of generated character image and try to develop font bank for business use.
% An example of a floating figure using the graphicx package.
% Note that \label must occur AFTER (or within) \caption.
% For figures, \caption should occur after the \includegraphics.
% Note that IEEEtran v1.7 and later has special internal code that
% is designed to preserve the operation of \label within \caption
% even when the captionsoff option is in effect. However, because
% of issues like this, it may be the safest practice to put all your
% \label just after \caption rather than within \caption{}.
%
% Reminder: the "draftcls" or "draftclsnofoot", not "draft", class
% option should be used if it is desired that the figures are to be
% displayed while in draft mode.
%
%\begin{figure}[!t]
%\centering
%\includegraphics[width=2.5in]{myfigure}
% where an .eps filename suffix will be assumed under latex,
% and a .pdf suffix will be assumed for pdflatex; or what has been declared
% via \DeclareGraphicsExtensions.
%\caption{Simulation results for the network.}
%\label{fig_sim}
%\end{figure}
% Note that the IEEE typically puts floats only at the top, even when this
% results in a large percentage of a column being occupied by floats.
% An example of a double column floating figure using two subfigures.
% (The subfig.sty package must be loaded for this to work.)
% The subfigure \label commands are set within each subfloat command,
% and the \label for the overall figure must come after \caption.
% \hfil is used as a separator to get equal spacing.
% Watch out that the combined width of all the subfigures on a
% line do not exceed the text width or a line break will occur.
%
%\begin{figure*}[!t]
%\centering
%\subfloat[Case I]{\includegraphics[width=2.5in]{box}%
%\label{fig_first_case}}
%\hfil
%\subfloat[Case II]{\includegraphics[width=2.5in]{box}%
%\label{fig_second_case}}
%\caption{Simulation results for the network.}
%\label{fig_sim}
%\end{figure*}
%
% Note that often IEEE papers with subfigures do not employ subfigure
% captions (using the optional argument to \subfloat[]), but instead will
% reference/describe all of them (a), (b), etc., within the main caption.
% Be aware that for subfig.sty to generate the (a), (b), etc., subfigure
% labels, the optional argument to \subfloat must be present. If a
% subcaption is not desired, just leave its contents blank,
% e.g., \subfloat[].
% An example of a floating table. Note that, for IEEE style tables, the
% \caption command should come BEFORE the table and, given that table
% captions serve much like titles, are usually capitalized except for words
% such as a, an, and, as, at, but, by, for, in, nor, of, on, or, the, to
% and up, which are usually not capitalized unless they are the first or
% last word of the caption. Table text will default to \footnotesize as
% the IEEE normally uses this smaller font for tables.
% The \label must come after \caption as always.
%
%\begin{table}[!t]
%% increase table row spacing, adjust to taste
%\renewcommand{\arraystretch}{1.3}
% if using array.sty, it might be a good idea to tweak the value of
% \extrarowheight as needed to properly center the text within the cells
%\caption{An Example of a Table}
%\label{table_example}
%\centering
%% Some packages, such as MDW tools, offer better commands for making tables
%% than the plain LaTeX2e tabular which is used here.
%\begin{tabular}{|c||c|}
%\hline
%One & Two\\
%\hline
%Three & Four\\
%\hline
%\end{tabular}
%\end{table}
% Note that the IEEE does not put floats in the very first column
% - or typically anywhere on the first page for that matter. Also,
% in-text middle ("here") positioning is typically not used, but it
% is allowed and encouraged for Computer Society conferences (but
% not Computer Society journals). Most IEEE journals/conferences use
% top floats exclusively.
% Note that, LaTeX2e, unlike IEEE journals/conferences, places
% footnotes above bottom floats. This can be corrected via the
% \fnbelowfloat command of the stfloats package.
% conference papers do not normally have an appendix
% use section* for acknowledgment
%\section*{Acknowledgment}
%The authors would like to thank...
% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}
% references section
% can use a bibliography generated by BibTeX as a .bbl file
% BibTeX documentation can be easily obtained at:
% http://mirror.ctan.org/biblio/bibtex/contrib/doc/
% The IEEEtran BibTeX style support page is at:
% http://www.michaelshell.org/tex/ieeetran/bibtex/
%\bibliographystyle{IEEEtran}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{IEEEabrv,../bib/paper}
%
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
%\begin{thebibliography}{1}
%
%%\bibitem{IEEEhowto:kopka}
%
%\bibitem{guocreate}
%Haoli Guo, Tao Jia, and Yujie Zheng.
%\newblock Create your own chinese calligraphy artwork.
%
%\bibitem{cao2000model}
%Ruini Cao and Chew~Lim Tan.
%\newblock A model of stroke extraction from Chinese character images.
%\newblock In {\em Pattern Recognition, 2000. Proceedings. 15th International
% Conference on}, volume~4, pages 368--371. IEEE, 2000.
%
%\bibitem{miyazaki2017automatic}
%Tomo Miyazaki, Tatsunori Tsuchiya, Yoshihiro Sugaya, Shinichiro Omachi,
% Masakazu Iwamura, Seiichi Uchida, and Koichi Kise.
%\newblock Automatic generation of typographic font from a small font subset.
%\newblock {\em arXiv preprint arXiv:1701.05703}, 2017.
%
%\bibitem{Goodfellow2014Generative}
%Ian~J. Goodfellow, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David
% Warde-Farley, Sherjil Ozair, Aaron Courville, and Yoshua Bengio.
%\newblock Generative adversarial nets.
%\newblock In {\em International Conference on Neural Information Processing
% Systems}, pages 2672--2680, 2014.
%
% \bibitem{mirza2014conditional}
%Mehdi Mirza and Simon Osindero.
%\newblock Conditional generative adversarial nets.
%\newblock {\em arXiv preprint arXiv:1411.1784}, 2014.
%
% \bibitem{Chen2016InfoGAN}
%Xi~Chen, Yan Duan, Rein Houthooft, John Schulman, Ilya Sutskever, and Pieter
% Abbeel.
%\newblock Infogan: Interpretable representation learning by information
% maximizing generative adversarial nets.
%\newblock 2016.
%
%\bibitem{Isola2016Image}
%Phillip Isola, Jun~Yan Zhu, Tinghui Zhou, and Alexei~A Efros.
%\newblock Image-to-image translation with conditional adversarial networks.
%\newblock 2016.
%
%\bibitem{zi2zi}
%kaonashi tyc.
%\newblock zi2zi: Master chinese calligraphy with conditional adversarial
% networks.
%\newblock \url{https://kaonashi-tyc.github.io/2017/04/06/zi2zi.html}.
%\newblock Accessed: 2017-05-30.
%
%\bibitem{Chang2017Chinese}
%Jie Chang and Yujun Gu.
%\newblock Chinese typography transfer.
%\newblock 2017.
%
%\bibitem{Johnson2016Google}
%Melvin Johnson, Mike Schuster, Quoc~V Le, Maxim Krikun, Yonghui Wu, Zhifeng
% Chen, Nikhil Thorat, Fernanda Viégas, Martin Wattenberg, and Greg Corrado.
%\newblock Google's multilingual neural machine translation system: Enabling
% zero-shot translation.
%\newblock 2016.
%
%\bibitem{Lyu2017Auto}
%Pengyuan Lyu, Xiang Bai, Cong Yao, Zhen Zhu, Tengteng Huang, and Wenyu Liu.
%\newblock Auto-encoder guided gan for chinese calligraphy synthesis.
%\newblock 2017.
%
%
%\bibitem{Radford2015Unsupervised}
%Alec Radford, Luke Metz, and Soumith Chintala.
%\newblock Unsupervised representation learning with deep convolutional
% generative adversarial networks.
%\newblock {\em Computer Science}, 2015.
%
%
%
%\bibitem{hinton2006reducing}
%Geoffrey~E Hinton and Ruslan Salakhutdinov.
%\newblock Reducing the dimensionality of data with neural networks.
%\newblock {\em Science}, 313(5786):504--507, 2006.
%
%\bibitem{dong2016image}
%Chao Dong, Chen~Change Loy, Kaiming He, and Xiaoou Tang.
%\newblock Image super-resolution using deep convolutional networks.
%\newblock {\em IEEE Transactions on Pattern Analysis and Machine Intelligence},
% 38(2):295--307, 2016.
%
%\bibitem{shi2016real-time}
%Wenzhe Shi, Jose Caballero, Ferenc Huszar, Johannes Totz, Andrew~P Aitken, Rob
% Bishop, Daniel Rueckert, and Zehan Wang.
%\newblock Real-time single image and video super-resolution using an efficient
% sub-pixel convolutional neural network.
%\newblock {\em computer vision and pattern recognition}, pages 1874--1883,
% 2016.
%
%\bibitem{ssim}
%Wikipedia.
%\newblock Structural similarity.
%\newblock \url{https://en.wikipedia.org/wiki/Structural_similarity}.
%\newblock Accessed: 2017-10-19.
%
%\bibitem{johnson2016perceptual}
%Justin Johnson, Alexandre Alahi, and Li~Feifei.
%\newblock Perceptual losses for real-time style transfer and super-resolution.
%\newblock {\em european conference on computer vision}, pages 694--711, 2016.
%
%\bibitem{simonyan2015very}
%Karen Simonyan and Andrew Zisserman.
%\newblock Very deep convolutional networks for large-scale image recognition.
%\newblock {\em international conference on learning representations}, 2015.
%
%
%\bibitem{Chen2017Photographic}
%Qifeng Chen and Vladlen Koltun.
%\newblock Photographic image synthesis with cascaded refinement networks.
%\newblock 2017.
%
%\bibitem{kingma2014adam}
%Diederik Kingma and Jimmy Ba.
%\newblock Adam: A method for stochastic optimization.
%\newblock {\em arXiv preprint arXiv:1412.6980}, 2014.
%
%\bibitem{shrivastava2016learning}
%Ashish Shrivastava, Tomas Pfister, Oncel Tuzel, Josh Susskind, Wenda Wang, and
% Russ Webb.
%\newblock Learning from simulated and unsupervised images through adversarial
% training.
%\newblock {\em arXiv preprint arXiv:1612.07828}, 2016.
%
%
%\bibitem{wang2004image}
%Zhou Wang, Alan~C Bovik, Hamid~R Sheikh, and Eero~P Simoncelli.
%\newblock Image quality assessment: from error visibility to structural
% similarity.
%\newblock {\em IEEE transactions on image processing}, 13(4):600--612, 2004.
%
%\bibitem{Wang2002A}
%Zhou Wang and Alan~C Bovik.
%\newblock A universal image quality index.
%\newblock {\em IEEE Signal Processing Letters}, 9(3):81--84, 2002.
%
%\bibitem{zcool}
%ZCool.
%\newblock Zcool kuheiti.
%\newblock \url{http://www.hellorf.com/font/show/10}.
%\newblock Accessed: 2017-07-30.
%
%\bibitem{ronneberger2015u}
%Olaf Ronneberger, Philipp Fischer, and Thomas Brox.
%\newblock U-net: Convolutional networks for biomedical image segmentation.
%\newblock In {\em International Conference on Medical Image Computing and
% Computer-Assisted Intervention}, pages 234--241. Springer, 2015.
%
%
%H.~Kopka and P.~W. Daly, \emph{A Guide to \LaTeX}, 3rd~ed.\hskip 1em plus
% 0.5em minus 0.4em\relax Harlow, England: Addison-Wesley, 1999.
%
%\end{thebibliography}
\bibliographystyle{IEEEtran}
\bibliography{refs}
% that's all folks
\end{document}
% Generated by IEEEtran.bst, version: 1.12 (2007/01/11)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{cao2000model}
R.~Cao and C.~L. Tan, ``A model of stroke extraction from chinese character
images,'' in \emph{Proceedings 15th International Conference on Pattern
Recognition. ICPR-2000}, vol.~4, 2000, pp. 368--371.
\bibitem{Goodfellow2014Generative}
I.~J. Goodfellow, J.~Pouget-Abadie, M.~Mirza, B.~Xu, D.~Warde-Farley, S.~Ozair,
A.~Courville, and Y.~Bengio, ``Generative adversarial nets,'' in
\emph{International Conference on Neural Information Processing Systems},
2014, pp. 2672--2680.
\bibitem{mirza2014conditional}
M.~Mirza and S.~Osindero, ``Conditional generative adversarial nets,''
\emph{arXiv preprint arXiv:1411.1784}, 2014.
\bibitem{Isola2016Image}
P.~Isola, J.~Y. Zhu, T.~Zhou, and A.~A. Efros, ``Image-to-image translation
with conditional adversarial networks,'' in \emph{The IEEE Conference on
Computer Vision and Pattern Recognition}, 2016, pp. 5967--5976.
\bibitem{zi2zi}
kaonashi tyc, ``zi2zi: Master chinese calligraphy with conditional adversarial
networks,'' \url{https://kaonashi-tyc.github.io/2017/04/06/zi2zi.html},
accessed: 2017-05-30.
\bibitem{Chen2017Photographic}
Q.~Chen and V.~Koltun, ``Photographic image synthesis with cascaded refinement
networks,'' in \emph{The IEEE International Conference on Computer Vision
(ICCV)}, vol.~1, 2017.
\bibitem{ronneberger2015u}
O.~Ronneberger, P.~Fischer, and T.~Brox, ``U-net: Convolutional networks for
biomedical image segmentation,'' in \emph{International Conference on Medical
Image Computing and Computer-Assisted Intervention}.\hskip 1em plus 0.5em
minus 0.4em\relax Springer, 2015, pp. 234--241.
\bibitem{miyazaki2017automatic}
T.~Miyazaki, T.~Tsuchiya, Y.~Sugaya, S.~Omachi, M.~Iwamura, S.~Uchida, and
K.~Kise, ``Automatic generation of typographic font from a small font
subset,'' \emph{arXiv preprint arXiv:1701.05703}, 2017.
\bibitem{Chen2016InfoGAN}
X.~Chen, Y.~Duan, R.~Houthooft, J.~Schulman, I.~Sutskever, and P.~Abbeel,
``Infogan: Interpretable representation learning by information maximizing
generative adversarial nets,'' in \emph{Advances in Neural Information
Processing Systems 29: Annual Conference on Neural Information Processing
Systems 2016, December 5-10, 2016, Barcelona, Spain}, 2016, pp. 2172--2180.
\bibitem{Chang2017Chinese}
J.~Chang and Y.~Gu, ``Chinese typography transfer,'' \emph{arXiv preprint
arXiv:1707.04904}, 2017.
\bibitem{Lyu2017Auto}
P.~Lyu, X.~Bai, C.~Yao, Z.~Zhu, T.~Huang, and W.~Liu, ``Auto-encoder guided gan
for chinese calligraphy synthesis,'' in \emph{2017 14th IAPR International
Conference on Document Analysis and Recognition (ICDAR)}, vol.~01, Nov 2017,
pp. 1095--1100.
\bibitem{hinton2006reducing}
G.~E. Hinton and R.~Salakhutdinov, ``Reducing the dimensionality of data with
neural networks.'' \emph{Science}, vol. 313, no. 5786, pp. 504--507, 2006.
\bibitem{dong2016image}
C.~Dong, C.~C. Loy, K.~He, and X.~Tang, ``Image super-resolution using deep
convolutional networks,'' \emph{IEEE Transactions on Pattern Analysis and
Machine Intelligence}, vol.~38, no.~2, pp. 295--307, 2016.
\bibitem{shi2016real-time}
W.~Shi, J.~Caballero, F.~Huszar, J.~Totz, A.~P. Aitken, R.~Bishop, D.~Rueckert,
and Z.~Wang, ``Real-time single image and video super-resolution using an
efficient sub-pixel convolutional neural network,'' in \emph{The IEEE
Conference on Computer Vision and Pattern Recognition (CVPR)}, 2016, pp.
1874--1883.
\bibitem{Johnson2016Google}
M.~Johnson, M.~Schuster, Q.~V. Le, M.~Krikun, Y.~Wu, Z.~Chen, N.~Thorat, F.~B.
Vi{\'{e}}gas, M.~Wattenberg, G.~Corrado, M.~Hughes, and J.~Dean, ``Google's
multilingual neural machine translation system: Enabling zero-shot
translation,'' \emph{{TACL}}, vol.~5, pp. 339--351, 2017.
\bibitem{johnson2016perceptual}
J.~Johnson, A.~Alahi, and L.~Feifei, ``Perceptual losses for real-time style
transfer and super-resolution,'' in \emph{European Conference on Computer
Vision}, 2016, pp. 694--711.
\bibitem{simonyan2015very}
K.~Simonyan and A.~Zisserman, ``Very deep convolutional networks for
large-scale image recognition,'' in \emph{International Conference on
Learning Representations}, 2015.
\bibitem{kingma2015adam}
D.~P. Kingma and J.~Ba, ``Adam: {A} method for stochastic optimization,''
\emph{CoRR}, vol. abs/1412.6980, 2014.
\bibitem{shrivastava2017learning}
A.~Shrivastava, T.~Pfister, O.~Tuzel, J.~Susskind, W.~Wang, and R.~Webb,
``Learning from simulated and unsupervised images through adversarial
training,'' in \emph{2017 IEEE Conference on Computer Vision and Pattern
Recognition}, July 2017, pp. 2242--2251.
\bibitem{wang2004image}
Z.~Wang, A.~C. Bovik, H.~R. Sheikh, and E.~P. Simoncelli, ``Image quality
assessment: from error visibility to structural similarity,'' \emph{IEEE
transactions on image processing}, vol.~13, no.~4, pp. 600--612, 2004.
\bibitem{Wang2002A}
Z.~Wang and A.~C. Bovik, ``A universal image quality index,'' \emph{IEEE Signal
Processing Letters}, vol.~9, no.~3, pp. 81--84, 2002.
\bibitem{wang2003multiscale}
Z.~Wang, E.~P. Simoncelli, and A.~C. Bovik, ``Multiscale structural similarity
for image quality assessment,'' in \emph{The Thrity-Seventh Asilomar
Conference on Signals, Systems Computers, 2003}, vol.~2, Nov 2003, pp.
1398--1402.
\end{thebibliography}