Vector Grimoire: Codebook-based Shape Generation under Raster Image Supervision
Extracted source text · arXiv 2410.05991
\documentclass{article}
% ready for submission
\usepackage[preprint]{latex_style}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{hyperref} % hyperlinks
\usepackage{url} % simple URL typesetting
\usepackage{booktabs} % professional-quality tables
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{xcolor} % colors
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%% NOT FROM THE TEMPLATE %%%%%%
% Include other packages here, before hyperref.
\usepackage{graphicx}
\graphicspath{{./figures/}}
\usepackage{booktabs}
\usepackage{subcaption}
% math formulas
\usepackage{amsmath}
% For xmark anc cmark
\usepackage{pifont}
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
% added by marco
\usepackage{makecell} % multi line in tab header
\usepackage{multirow}
\usepackage{mathabx}
\usepackage{caption}
\usepackage{array}
\usepackage{tikz}
\usepackage{numprint}
\setcitestyle{round}
% Support for ORCID icon
\usepackage{orcidlink}
% Extra settings
\renewcommand{\sectionautorefname}{Section}
\renewcommand{\subsectionautorefname}{Section}
\renewcommand{\subsubsectionautorefname}{Section}
\newcommand{\marco}[1]{{\color{teal} Marco: [{#1}]}}
\title{\textsc{Vector Grimoire}: Codebook-based Shape Generation under Raster Image Supervision}
\newcommand{\ours}{\textsc{Grimoire}}
\newcommand{\websitelink}{\url{https://potpov.github.io/grimoire-web/}}
% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to LaTeX to determine where to break the
% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
% authors names on the first line, and the last on the second line, try using
% \AND instead of \And before the third author name.
\author{%
Moritz Feuerpfeil$^{*}$ \quad Marco Cipriano$^{*}$ \quad Gerard de Melo \\
Hasso Plattner Institute (HPI) \\
\texttt{marco.cipriano@hpi.de}\\
\websitelink \\
}
\begin{document}
\maketitle
\makeatletter\def\Hy@Warning#1{}\makeatother
\def\thefootnote{*}\footnotetext{Equal contribution.}
\def\thefootnote{\arabic{footnote}}
\input{macro}
\input{sec/0-abstract}
\input{sec/1-intro}
\input{sec/2-related}
\input{sec/3-method}
\input{sec/4-data}
\input{sec/5-results}
\input{sec/6-conclusion}
% % ---- Bibliography ----
% %
\bibliographystyle{abbrvnat}
\bibliography{main}
% \section*{References}
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \appendix
% \section{Appendix / supplemental material}
\clearpage
\input{sec/supplementary}
\end{document}
% font macros
\newcommand{\funfixed}[1]{\includegraphics[width=0.14\columnwidth]{figures/generations/fonts/unfixed_#1.pdf}}
\newcommand{\fpc}[1]{\includegraphics[width=0.14\columnwidth]{figures/generations/fonts/pc_fixed_#1.pdf}}
\newcommand{\fpi}[1]{\includegraphics[width=0.14\columnwidth]{figures/generations/fonts/pi_fixed_#1.pdf}}
% generative with temp macros
\newcommand{\gentone}[1]{\includegraphics[width=0.12\columnwidth]{figures/generations/icons/showcase/#1.pdf}}
% generative with token conditioning macros
\newcommand{\notoken}[1]{\includegraphics[width=0.12\columnwidth]{figures/generations/icons/token_conditioning/vq_context_0_#1.pdf}}
\newcommand{\tokenone}[1]{\includegraphics[width=0.12\columnwidth]{figures/generations/icons/token_conditioning/vq_context_3_#1.pdf}}
\newcommand{\tokentwo}[1]{\includegraphics[width=0.12\columnwidth]{figures/generations/icons/token_conditioning/vq_context_9_#1.pdf}}
\newcommand{\tokenthree}[1]{\includegraphics[width=0.12\columnwidth]{figures/generations/icons/token_conditioning/vq_context_30_#1.pdf}}
\newcommand{\tokenfour}[1]{\includegraphics[width=0.12\columnwidth]{figures/generations/icons/token_conditioning/vq_context_45_#1.pdf}}
% qualitative comparison with im2vec
\newcommand{\imvecfont}[1]{\includegraphics[width=0.11\columnwidth]{figures/generations/comparison/imvec_a_#1.pdf}}
\newcommand{\imvecstar}[1]{\includegraphics[width=0.11\columnwidth]{figures/generations/comparison/imvec_star_#1.pdf}}
\newcommand{\imvecmnist}[1]{\includegraphics[width=0.11\columnwidth]{figures/generations/comparison/im2vec_mnist_0#1.pdf}}
\newcommand{\grimfont}[1]{\includegraphics[width=0.11\columnwidth]{figures/generations/comparison/ours_a_#1.pdf}}
\newcommand{\grimstar}[1]{\includegraphics[width=0.11\columnwidth]{figures/generations/comparison/ours_star_#1.pdf}}
\newcommand{\grimmnist}[1]{\includegraphics[width=0.11\columnwidth]{figures/generations/comparison/ours_mnist_0#1.pdf}}
\newcommand{\na}{\textit{n/a}}
\newcommand{\figrdata}{FIGR-8}
\newcommand{\imtovec}{Im2Vec}
\newcommand{\fonts}{Fonts}
\newcommand{\mnist}{MNIST}
\newcommand{\CLIP}{CLIP~$\upuparrows$}
\newcommand{\CLIPp}{CLIP$_p$~$\upuparrows$}
\newcommand{\MSE}{MSE~$\downdownarrows$}
\newcommand{\FID}{FID~$\downdownarrows$}
% appendix
\newcommand{\appgrimGenFonts}[1]{\includegraphics[width=0.15\columnwidth]{figures/appendix/generations/grim/fonts/fonts_pi_fixed_#1.pdf}}
\newcommand{\appgrimGenMnist}[1]{\includegraphics[width=0.15\columnwidth]{figures/appendix/generations/grim/mnist_showcase/#1.pdf}}
\newcommand{\appgrimGenIcons}[1]{\includegraphics[width=0.15\columnwidth]{figures/appendix/generations/grim/icons/pi_fixed_#1.pdf}}
\newcommand{\appImtovecGenIcons}[1]{\includegraphics[width=0.15\columnwidth]{figures/appendix/generations/imvec/imvecc_sample_#1.pdf}}
\newcommand{\appgrimRecFonts}[2]{\includegraphics[width=0.15\columnwidth]{figures/appendix/recon/fonts/#1/#2.pdf}}
\newcommand{\appgrimRecIcons}[2]{\includegraphics[width=0.15\columnwidth]{figures/appendix/recon/icons/#1/#2.pdf}}
\newcommand{\appCodebookTopStroke}[1]{\includegraphics[width=0.1\columnwidth]{figures/appendix/codebook/top_strokes/#1.pdf}}
\newcommand{\appReductionImages}[1]{\includegraphics[width=0.33\columnwidth]{figures/appendix/codebook/reductions/#1.png}}
\newcommand{\appgrimRecPngFonts}[2]{\includegraphics[width=0.15\columnwidth]{figures/appendix/recon/fonts/#1/#2.png}}
\newcommand{\appgrimRecPngIcons}[2]{\includegraphics[width=0.15\columnwidth]{figures/appendix/recon/icons/#1/#2.png}}
% color mnist macros
\newcommand{\mniorig}[1]{\includegraphics[width=0.14\columnwidth]{figures/appendix/generations/grim/mnist/orig_#1.png}}
\newcommand{\mnirecon}[1]{\includegraphics[width=0.14\columnwidth]{figures/appendix/generations/grim/mnist/rec_#1.png}}
\newcommand{\mnimerg}[1]{\includegraphics[width=0.14\columnwidth]{figures/appendix/generations/grim/mnist/mer_#1.png}}
\begin{abstract}
Scalable Vector Graphics (SVG) is a popular format on the web and in the design industry. However, despite the great strides made in generative modeling, SVG has remained underexplored due to the discrete and complex nature of such data.
We introduce \ours{}, a text-guided SVG generative model that is comprised of two modules:
A Visual Shape Quantizer (VSQ) learns to map raster images onto a discrete codebook by reconstructing them as vector shapes, and an Auto-Regressive Transformer (ART) models the joint probability distribution over shape tokens, positions, and textual descriptions, allowing us to generate vector graphics from natural language.
Unlike existing models that require direct supervision from SVG data, \ours{} learns shape image patches using only raster image supervision which opens up vector generative modeling to significantly more data.
We demonstrate the effectiveness of our method by fitting \ours{} for closed filled shapes on MNIST and for outline strokes on icon and font data, surpassing previous image-supervised methods in generative quality and the vector-supervised approach in flexibility.
\end{abstract}
\section{Introduction}
\label{sec:intro}
In the domain of computer graphics, Scalable Vector Graphics (SVG) has emerged as a versatile format, enabling the representation of 2D graphics with precision and scalability. SVG is an XML-based vector graphics format that describes a series of parametrized shape primitives rather than a limited-resolution raster of pixel values.
While modern generative models have made significant advancements in producing high-quality raster images~\citep{ho2020denoising,isola2017image,saharia2022photorealistic,nichol2021glide}, SVG generation remains a less explored task. Existing works that have aimed to train a deep neural network for this goal primarily adopted language models to address the problem~\citep{wu2023iconshop,tang2024strokenuwa}.
In general, existing approaches share two key limitations: they necessitate SVG data for direct supervision which inherently limits the available data and increases the burden of data pre-processing, and they are not easily extendable when it comes to visual attributes such as color or stroke properties. The extensive pre-processing is required due to the diverse nature of an SVG file that can express shapes as a series of different basic primitives such as circles, lines, and squares -- each having different properties -- that can overlap and occlude each other.
An ideal generative model for SVG should however benefit from \emph{visual guidance for supervision}, which is not possible when merely training to reproduce tokenized SVG primitives, as there is no differentiable mapping to the generated raster imagery.
In this paper, we present \ours{} (Shape Generation with raster image supervision), a novel pipeline explicitly designed to generate SVG files with only raster image supervision. Our approach incorporates a differentiable rasterizer, DiffVG~\citep{li2020differentiable}, to bridge the vector graphics primitives and the raster image domain. We adopt a VQ-VAE recipe~\citep{van2017neural}, which pairs a codebook-based discrete auto-encoder with an auto-regressive Transformer that models the image space implicitly by learning the distribution of codes that resemble them.
We find this approach particularly promising for vector graphics generation, as it breaks the complexity of this task into two stages.
In the first stage of our method, we decompose images into primitive shapes represented as patches. A vector-quantized auto-encoder learns to encode and map each patch into a discrete codebook, and decode these codes to an SVG approximation of the input patch, which is trained under raster supervision. In the second stage, the series of raster patches containing primitives are encoded and the prior distribution of codes is learned by an auto-regressive Transformer model conditioned on a textual description. At inference, a full series of codes can be generated from textual input, or other existing shape codes. Therefore, \ours{} supports text-to-SVG generation and SVG auto-completion as possible downstream tasks out-of-the-box.
The key contributions of this work are:
\begin{enumerate}
\item We frame the problem of image-supervised SVG generation as the prediction of a series of individual shapes and their positions on a shared canvas.
\item We train the first text-conditioned generative model that learns to draw vector graphics with only raster image supervision.
\item We compare our model with alternative frameworks showing superior performance in generative capabilities on diverse datasets.
\item Upon acceptance, release the code of this work to the research community.
\end{enumerate}
\begin{figure*}[t]
\centering
\setlength{\tabcolsep}{2pt}
\begin{tabular}{p{0.02\textwidth}cccccccc}
\toprule
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{Ours}} & \grimstar{1} & \grimstar{2} & \grimstar{3} & \grimstar{4} & \grimfont{1} & \grimfont{2} & \grimfont{3} & \grimfont{4} \\\hline
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{\imtovec}} & \imvecstar{1} & \imvecstar{3} & \imvecstar{4} & \imvecstar{5} & \imvecfont{1} & \imvecfont{2} & \imvecfont{3} & \imvecfont{4} \\\hline
\noalign{\vskip 1mm}
\end{tabular}
\caption{Generative results for fonts and icons from \ours{} and \imtovec. Since \imtovec{} does not accept any conditioning, we sample after training \imtovec{} only on icons of stars or the letter A, respectively. For \ours{} we use the models trained on the full dataset conditioned on the respective class.}
\label{fig:results-icon-generation}
\end{figure*}
\begin{figure*}[ht]
\centering
\includegraphics[width=\linewidth]{model_2}
\caption{Overview of \ours{}. On the left, the training process of our VSQ module is depicted, where raster input patches are encoded into discrete codes and reconstructed as SVG shapes using visual supervision. In the top right, each image is encoded into a series of discrete codes using the trained VSQ encoder and its textual description. The bottom right illustrates how the ART module learns the joint distribution of these codes and the corresponding text.}
\label{fig:model-overview}
\end{figure*}
\section{Related Work}
\label{sec:related}
\subsection{SVG Generative Models}
The field of vector graphics generation has witnessed increasing interest.
Following the extraordinary success of Large Language Models (LLM), the most recent approaches~\citep{lopes2019learned,aoki2022svg,wu2023iconshop,tang2024strokenuwa} have recast the problem as an NLP task, learning a distribution over tokenized SVG commands. Iconshop \citep{wu2023iconshop} introduced a method of tokenizing SVG paths that makes them suitable input for causal language modeling. To add conditioning, they employed a pre-trained language model to tokenize and embed textual descriptions, which are concatenated with the SVG tokens to form sequences that the auto-regressive Transformer can learn a joint probability on.
StrokeNUWA~\citep{tang2024strokenuwa} introduced Vector Quantized Strokes to compress SVG strokes into a codebook with SVG supervision and fine-tune a pre-trained Encoder--Decoder LLM to predict these tokens given textual input. However, both of these approaches suffer from a number of limitations. First, they require a corpus of SVG data for training, which hinges upon large pre-processing pipelines to remove redundancies, convert non-representable primitives, and standardize the representations.
Secondly, there is no supervision of the visual rendering, which makes the models prone to data quality errors, e.g., excessive occlusion of shapes.
Finally, these models lack any straightforward extensibility towards the inclusion of new visual features such as colours, stroke widths, or fillings and alpha values.
Hence, another line of work has sought to incorporate visual supervision. These approaches generally rely on recent advances in differentiable rasterization, which
enables backpropagation of raster-based losses through different types of vectorial primitives such as B\'ezier curves, circles, and squares. The most important development in this area is DiffVG~\citep{li2020differentiable}, which removed the need for approximations and introduced techniques to handle antialiasing.
They further pioneered image-supervised SVG generative models by training a Variational Autoencoder (VAE) and a Generative Adversarial Network (GAN)~\citep{goodfellow2014generative} on MNIST~\citep{lecun1998gradient} and QuickDraw~\citep{ha2017neural}. These generative capabilities have subsequently been extended in Im2Vec~\citep{reddy2021im2vec}, which adopts a VAE including a recurrent neural network to generate vector graphics as sets of deformed and filled circular paths, which are differentiably composited and rasterized, allowing for back-propagation of a multi-resolution MSE-based pyramid loss. However, all of these models lack versatile conditioning (such as text) and focus on either image vectorization, i.e., the task of creating the closest vector representation of a raster prior, or vector graphics interpolation. We show in \autoref{sec:results} that these approaches fail to capture the diversity and complexity of datasets such as \figrdata{}, and generate repetitive samples.
A different type of SVG generation enabled by DiffVG is painterly rendering~\citep{ganin2018synthesizing,nakano2019neural}, where an algorithm iteratively fits a given set of vector primitives to match an image, guided by a deep perceptual loss function. To achieve this goal, CLIPDraw~\citep{frans2022clipdraw} rasterized a set of randomly initialized SVG paths and encoded these with a pre-trained CLIP~\citep{radford2021learning} image encoder, iteratively minimizing the cosine distance between such embeddings and the text description. A similar approach was adopted by CLIPasso~\citep{vinker2022clipasso} to translate images into strokes. Vector Fusion~\citep{jain2023vectorfusion} leveraged Score Distillation Sampling (SDS)~\citep{poole2022dreamfusion} to induce abstract semantic knowledge from an off-the-shelf Stable Diffusion model~\citep{rombach2022high}. A very similar approach, based on SDS, has also been applied to fonts~\citep{iluz2023word}. However, all painterly rendering methods come as iterative algorithms making them very computationally expensive and impractical in real world use cases. They frequently create many unnecessary and redundant shapes to minimize the perceptual loss.
\subsection{Vector Quantization}
VQ-VAE~\citep{van2017neural} is a well-known improved architecture for training Variational Autoencoders~\citep{kingma2013auto,rezende2014stochastic}. Instead of focusing on representations with continuous features as in most prior work~\citep{vincent2010stacked,denton2016semi,hinton2006reducing,chen2016infogan}, the encoder in a VQ-VAE emits discrete rather than continuous codes. Each code maps to the closest embedding in a codebook of limited size. The decoder learns to reconstruct the original input image from the chosen codebook embedding. Both the encoder--decoder architecture and the codebook are trained jointly. After training, the autoregressive distribution over the latent codes is learnt by a second model, which then allows for generating new images via ancestral sampling. Latent discrete representations were already pioneered in previous work~\citep{mnih2014neural,courville2011spike}, but none of the above methods close the performance gap of VAEs with continuous latent variables, where one can use the Gaussian reparametrization trick, which benefits from much lower variance in the gradients. \citet{mentzer2023finite} simplified the design of the vector quantization in VQ-VAE with a scheme called finite scalar quantization (FSQ), where the encoded representation of an image is projected to the nearest position on a low-dimensional hypercube. In this case, no additional codebook must be learned, but rather it is given implicitly, which simplifies the loss formulation.
Our work builds in part on the VQ-VAE framework and includes the FSQ mechanism.
\section{Method}
\label{sec:method}
\begin{figure*}[b]
\centering
\includegraphics[width=0.8\linewidth]{preprocessing_2}
\caption{Overview of the data generation process for \ours{}. For the \mnist{} digits, we simply create patches from a $6\times6$ Grid. For \figrdata{}, we extract the outlines of each icon and create small centered raster segments. We save the original anchor position of each segment for the second stage of our training pipeline. More information about the outline extraction is provided in \autoref{subsec:pre-processing}. \fonts{} comes in vector format and can be easily manipulated to extract strokes, similarly to \figrdata.
}
\label{fig:stage1data}
\end{figure*}
\subsection{Stage 1 -- Visual Shape Quantizer}
\label{subset:vsq-model}
The first stage of our model employs a \textbf{V}isual \textbf{S}hape \textbf{Q}uantizer (VSQ), a vector-quantized auto-encoder, whose encoder $E_{\mathrm{VSQ}}$ maps an input image $I$ onto a discrete codebook $\mathbb{V}$ through vector-quantization and decodes that quantized vector into shape parameters of cubic B\'ezier curves through the decoder $D_{\mathrm{VSQ}}$. Instead of learning the codebook~\citep{van2017neural}, we adopt the more efficient approach of defining our codebook $V$ as a set of equidistant points in a hypercube with $q$ dimensions. Each dimension has $l$ unique values: $L = [l_1, l_2, \ldots, l_q]$. The size of the codebook $|\mathbb{V}|$ is hence defined by the product of values of all $q$ dimensions.
We define $q=5$ and $L = [7, 5, 5, 5, 5]$ for a target codebook size of 4,375 unique codes, following the recommendations of the original authors \citep{mentzer2023finite}.
Before being fed to the encoder $E_{\mathrm{VSQ}}$, each image $I\in\mathbb{R}^{C\times H\times W}$ is divided into patches
$\mathbf{S} = (s_1, s_2, \ldots, s_n)$, with $s_i\in\mathbb{R}^{C\times 128\times128}$, where $C=3$ is the number of channels. A set of discrete anchor coordinates $\mathbf{\Theta} = (\theta_1, \theta_2, \ldots, \theta_n)$ with $\theta_i\in\mathbb{N}^{2}$ being the center coordinate of $s_i$ in the original image $I$ is also saved. The original image $I$ can then be reconstructed using $S$ and $\Theta$.
We experiment on three datasets (see Section \ref{sec:data}). For \mnist{}, the patches are obtained by tiling each image into a $6\times6$ grid. For \fonts{} and \figrdata{}, each patch depicts part of the target outline as shown in~\autoref{fig:stage1data}.
We utilize a contour-finding algorithm \citep{find_contour} to extract outlines from raster images, which are then divided into several shorter segments. Additional details regarding this extraction process can be found in \autoref{subsec:pre-processing}. In contrast, the \fonts{} dataset is natively available in vector format, making it easier to manipulate, similar to icons, before undergoing rasterization.
The VSQ encoder $E_{\mathrm{VSQ}}$ maps each patch $s_i\in\mathbb{R}^{C\times 128\times128}$ to $\xi$ codes on the hypercube $E_{\mathrm{VSQ}} : \mathbb{R}^{C\times128\times128} \mapsto V$ as follows.
Each centered raster patch $s_i$ is encoded with a ResNet-18~\citep{he2016deep} into a latent variable $z_i \in \mathcal{Z} \subset \mathbb{R}^{d\times \xi}$ with $d=512$. Successively, each of the $\xi$ codes is projected to $q$ dimensions through a linear mapping layer and finally quantized, resulting in $\hat z_i\in\mathbb{N}^{q}$. The final code value $v_i\in\mathbb{V}$ is then computed as the weighted sum of all $q$ dimensions of $\hat{z}_i$:
\[
v_i = \sum_{j=1}^{q} \hat{z}_{ij} \cdot b_j,
\]
where the basis $b_j$ is derived as $b_j = \prod_{k=1}^{j-1} l_k$, with $b_1 = 1$.
This transformation ensures that each unique combination of quantized values $\hat{z}_i$ is mapped to a unique code $v_i$ in the codebook $\mathbb{V}$.
This approach avoids auxiliary losses on the codebook while maintaining competitive expressiveness.
The decoder $D_{\mathrm{VSQ}}$ consists of a projection layer, which transforms all the $\xi$ predicted codes back into the latent space $\mathcal{Z}$, and a lightweight neural network $\Phi_\textrm{points}$, which predicts the control points of $\nu$ cubic Bézier curves that form a single connected path. We propose two variants of $\Phi_\textrm{points}$, a fully-connected neural network $\Phi^\textrm{stroke}_\textrm{points} : \mathcal{Z} \mapsto \mathbb{R}^{(2\times (\nu \times 3 + 1))}$, which predicts connected strokes, and a 1-D CNN $\Phi^\textrm{shape}_\textrm{points} : \mathcal{Z} \mapsto \mathbb{R}^{(2\times (\nu \times 3))}$, which outputs a closed shape.
Finally, the predicted path of $\nu$ B\'ezier curves from $\Phi_\textrm{points}$ passes through the differentiable rasterizer to obtain a raster output $\hat{s}_i = \textrm{DiffVG}(D_{\mathrm{VSQ}}(E_{\mathrm{VSQ}}(s_i)))$.
In order to learn to reconstruct strokes and shapes, we train the VSQ module using the mean squared error:
\[\mathcal{L_\text{recons}}=(s - \hat{s})^2.\]
$D_{\mathrm{VSQ}}$ can be extended to predict continuous values for any visual attribute supported by the differentiable rasterizer. Hence, we also propose series of other fully-connected prediction heads that can optionally be enabled: $\Phi_\textrm{width} : \mathcal{Z} \mapsto \mathbb{R}$ predicts the stroke width of the overall shape, and $\Phi_\textrm{color} : \mathcal{Z} \mapsto \mathbb{R^C}$ outputs the stroke color or the filling color for the output of $\Phi^\textrm{stroke}_\textrm{points}$ and $\Phi^\textrm{shape}_\textrm{points}$, respectively. All the modules are followed by a sigmoid activation function.
While $\mathcal{L_\text{recons}}$ would suffice for training the VSQ, operating only on the visual domain could lead to degenerate strokes and undesirable local minima. To mitigate this, we propose a novel geometric constraint $\mathcal{L_{\text{geom}}}$, which punishes control point placement of irregular distances measured between all combinations of points predicted by $\Phi^\textrm{stroke}_\textrm{points}$.
Let $P=(p_{1},p_{2},...,p_{\nu+1})$ be the set of all start and end points of a stroke with $p_{i} = (p^{x}_{i}, p^{y}_{i})$ and $p^{x}_{i}, p^{y}_{i} \in [0, 1]$. Then $\rho_{i,j}$ is defined as the Euclidean distance between two points $p_{i}$ and $p_{j}$, $\overline{\rho}_j$ is defined as the mean scaled inner distance for point $p_j$ to all other points in $P$, and $\delta_{j}$ as the average squared deviation from that mean for point $p_j$:
\[
\overline{\rho}_j = \frac{1}{\nu} \sum_{\substack{i=1 \\ i \neq j}}^{\nu+1} \frac{\rho_{i,j}}{|i-j|}
\qquad
\delta_{j} = \frac{1}{\nu} \sum_{\substack{i=1 \\ i \neq j}}^{\nu+1} \left(\frac{\rho_{i,j}}{|i-j|} - \overline{\rho}_j \right)^2
\]
$\mathcal{L_{\text{geom}}}$ is finally defined as the average of the deviations for all start and end points in $P$. $\mathcal{L_{\text{geom}}}$ is then weighted with $\alpha$ and added to the reconstruction loss.
\[
\mathcal{L_{\text{geom}}} = \frac{1}{\nu + 1} \sum_{j=1}^{\nu+1} \delta_{j}
\quad \quad
\mathcal{L_{\text{geom}}} = \mathcal{L_{\text{geom}}} + \alpha \times \mathcal{L_{\text{geom}}}
\]
We use this component only for the experiments with $\Phi^\textrm{stroke}_\textrm{points}$ and set $\alpha=0.4$.
We opt to train the ResNet encoder from scratch during this stage, since the target images belong to a very specific domain. The amount of trainable parameters is $15.36M$ for the encoder and $0.8M$ for the decoder. We stress the importance of the skewed balance between the two parameter counts, as the encoding of images is only required for training the model and encoding the training data for the auto-regressive Transformer in the next step. The final inference pipeline discards the encoder and only requires the trained decoder $D_{\mathrm{VSQ}}$, hence resulting in more lightweight inference. The overall scheme of \ours{} including the first stage of training is depicted in~\autoref{fig:model-overview}.
\subsection{Stage 2 -- Auto-Regressive Transformer}
After the VSQ is trained, each patch $s_i$ can be mapped onto an index code $v_i$ of the codebook $V$ using the encoder $E_{\mathrm{VSQ}}$ and the quantization method. However, the predicted patch $\hat{s_i}$ captured by the VSQ does not describe a complete SVG, as the centering leads to a loss of information about their global position $\theta_i$ on the original canvas. Also, the sequence of tokens is still missing the text conditioning. This is addressed in the second stage of \ours{}. The second stage consists of an \textbf{A}uto-\textbf{R}egressive \textbf{T}ransformer (ART) that learns for each image $I$ the joint distribution over the text, positions, and stroke tokens.
A textual description $T$ of $I$ is tokenized into $\mathcal{T} = (\tau_1, \tau_2, \ldots, \tau_t)$ using a pre-trained BERT encoder~\citep{devlin2018bert} and embedded. $I$ is visually encoded by transforming its patches $s_i$ onto $v_i\in V$ via the encoder $E_{\mathrm{VSQ}}$, whereas each original patch position $\theta_i \in \Theta$ is mapped into the closest position in a $256 \times 256$ grid resulting in $256^2$ possible position tokens. Special tokens \texttt{<SOS>}, \texttt{<BOS>}, and \texttt{<EOS>} indicate the start of a full sequence, beginning of the patch token sequence, and end of sequence, respectively. Each patch token is alternated with its position token. The final input sequence for a given image to the ART module becomes:
\[ x = (\texttt{<SOS>}, \tau_1, \ldots, \tau_t, \texttt{<BOS>}, \theta_1, v_1, \dots \theta_n, v_n, \texttt{<EOS>})\]
The total amount of representable token values then has a dimensionality of $|V|+256^2+3 = 69,914$ for $|V| = \text{4,375}$. A learnable weight matrix $W\in \mathbb{R}^{d\times \text{69,914}}$ embeds the position and visual tokens into a vector of size $d$. The BERT text embeddings are projected into the same $d$-dimensional space using a trainable linear mapping layer. The ART module consists of $12$ and $16$ standard Transformer decoder blocks with causal multi-head attention with $8$ attention heads for fonts and icons, respectively. The final loss for the ART module is defined as:
\[
\mathcal{L}_{\text{Causal}} = - \sum_{i=1}^{N} \log p(x_i \mid x_{<i}; \theta)
\]
During inference, the input to the ART module is represented as $ x = (\texttt{<SOS>}, \tau_1, \ldots, \tau_t, \texttt{<BOS>})$, where new tokens are predicted auto-regressively until the $\texttt{<EOS>}$ token is generated. Additionally, visual strokes can be incorporated into the input sequence to condition the generation process.
\section{Data}
\label{sec:data}
\noindent \textbf{\mnist.} We conduct our initial experiments on the \mnist{} dataset~\citep{lecun1998gradient}. We upscale each digit to $128\times128$ pixels and generate the texual description using the prompt ``\textit{x} in black color", where \textit{x} is the class of each digit. We adopt the original train and test split.
\noindent \textbf{\fonts.} For our experiments on fonts, we use a subset of the SVG-Fonts dataset \citep{lopes2019learned}. We remove fonts where capital and lowercase glyphs are identical, and consider only 0--9, a--z, and A--Z glyphs, which leads to 32,961 unique fonts for a corpus of $\sim$2M samples. The font features -- such as type of character or style -- are extracted from the \textit{.TTF} file metadata.
% However, upon manual inspection, we find those information noisy.
The final textual description for a sample glyph $g$ in font style $s$ is built using the prompt: ``[capital] $g$ in $s$ font'', where ``capital '' is included only for the glyphs A-Z. We use 80\%, 10\%, and 10\% for training, testing, and validation respectively.
\noindent \textbf{\figrdata.} We validate our method on more complex data and further use a subset of \figrdata{} \citep{clouatre2019figr}, where we select the 75 majority classes (excluding ``arrow'') and any class that contains those, e.g., the selection of ``house'' further entails the inclusion of ``dog house''. This procedure yields 427K samples, of which we select 90\% for training, 5\% for validation, and 5\% for testing. We use the class names as textual descriptions without further processing besides minor spelling correction. Since the black strokes of \figrdata{} mark the background rather than the actual icon, we invert the full dataset before applying our additional pre-processing described in \autoref{subsec:pre-processing}.
\section{Results}
\label{sec:results}
This section presents our findings in two primary categories. First, we examine the \textbf{quality} of the reconstructions and generations produced by \ours{} in comparison to existing methods. Second, we highlight the \textbf{flexibility} of our approach, demonstrating how \ours{} can be easily extended to incorporate additional SVG features.
\subsection{Reconstructions}
\label{subsec:recon-result-vsq}
\noindent \textbf{Closed Paths}. We begin by presenting the reconstruction results of our VSQ module on the \mnist{} dataset. In our experiments, we model each patch shape using a total of $15$ segments. Increasing the number of segments beyond this point did not yield any significant improvement in reconstruction quality. Given the simplicity of the target shapes, we adopted a single code per shape.
We also conducted a comparative analysis of the reconstruction capabilities of our VSQ module against \imtovec. To assess the generative quality of our samples, we employed the Fréchet Inception Distance (FID) \citep{heusel2017gans} and CLIPScore \citep{radford2021learning}, both of which are computed using the image features of a pre-trained CLIP encoder. Additionally, to validate our VSQ module, we considered the reconstruction loss $\mathcal{L_\text{recons}}$, as it directly reflects the maximum achievable performance of the network and provides a more reliable metric.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Result: Table1: Reconstruction (us vs im2vec full)
\begin{table*}[b]
\centering
\caption{Results for reconstructions of \ours{} and \imtovec{} on the full datasets. The last row includes post-processing.}
\label{tab:recons-ours-imvec-full}
\resizebox{\linewidth}{!}{%
\begin{tabular}{l ccc ccc ccc}
& \multicolumn{3}{c}{MNIST} & \multicolumn{3}{c}{Fonts} & \multicolumn{3}{c}{\figrdata} \\
\cmidrule(lr){2-4} \cmidrule(lr){5-7} \cmidrule(lr){8-10}
Model & \MSE & \FID & \CLIP & \MSE & \FID & \CLIP & \MSE & \FID & \CLIP \\
\midrule
\imtovec\ (filled) & 0.140 & \textbf{1.33} & 25.02 & 0.140& 2.04& 26.82& 0.330 & 16.10& 26.17 \\
\imtovec & \na & \na & \na & 0.050& 5.64& 26.72 & 0.050 & 13.90& 26.17 \\
\midrule
VSQ & \textbf{0.090} & 7.09 & \textbf{25.24} & 0.014& 4.45& 28.61 & 0.004 & 1.42 & 31.09\\
VSQ + PI & \na & \na & \na & \textbf{0.011}& \textbf{0.29}& \textbf{28.96}& \textbf{0.002} & \textbf{0.05} & \textbf{32.03} \\
\bottomrule
\end{tabular}
}
\end{table*}
\npnoround
As shown in \autoref{tab:recons-ours-imvec-full}, our VSQ module consistently achieves a lower reconstruction error compared to \imtovec{} across all MNIST digits. In \autoref{tab:recons-ours-imvec-subset}, we also report the reconstruction error for a subset of the dataset, selecting the digit zero due to its particularly challenging topology. Again, our method exhibits superior performance with lower reconstruction errors. For \mnist{}, we fill the predicted shapes from \imtovec{}, since the raster ground truth images are only in a filled format. However, we present both filled and unfilled versions for all other scenarios.
The CLIPScore of our reconstructions is higher in both cases. Notably, FID is the only metric where \imtovec{} occasionally shows superior results. We attribute this to the lower resolution of the ground truth images, which introduces instability in the FID metric. The CLIPScore, however, mitigates this issue by comparing the similarity with the textual description.
\textbf{Strokes}. For \fonts{} and \figrdata{}, we conduct a deeper investigation to validate the reconstruction errors of VSQ under different configurations, varying the amount of segments and codes per shape, and the maximum length of the input strokes. Our findings show that for \fonts, more than one segment per shape consistently degrades the reconstruction quality, possibly because the complexity of the strokes in our datasets does not require many Beziér curves to reconstruct an input patch. We also find that shorter thresholds on the stroke length help the reconstruction quality, as the MSE decreases when moving from $11\%$ to $7\%$ and eventually to $4\%$ of the maximum stroke length with respect to the image size. Intuitively, shorter strokes are easier to model, but could also lead to very scattered predictions for overly short settings.
The best reconstructions are achieved by using multiple codes per centered stroke. The two-codes configuration has an average decrease in MSE of 18.28\%, 41.46\%, and 26.09\% for the respective stroke lengths. However, the best-performing configuration with two codes per shape is just 11.36\% better than the best single code representative, which we believe does not justify twice the number of required visual tokens for the second stage training.
Throughout our experiments, the configurations with multiple segments do consistently benefit from our geometric constraint.
Ultimately, for our final experiments we choose $(\nu = 2$, $\xi = 1)$ for \fonts, and $(\nu = 4$, $\xi = 2)$ for \figrdata{}.
Regarding the comparison with \imtovec{}, \autoref{tab:recons-ours-imvec-subset} shows that the text-conditioned \ours{} on a single glyph or icon has superior reconstruction performance even if \imtovec{} is specifically trained on that subset of data. In \autoref{tab:recons-ours-imvec-full}, we also report the values after training on the full datasets. In this case, \ours{} substantially outperforms \imtovec{}, which is unable to cope with the complexity of the data.
Finally, as \ours{} quickly learns to map basic strokes or shapes onto its finite codebook and due to the similarities between those primitive traits among various samples in the dataset, we find \ours{} to converge even before completing a full epoch on any dataset. Despite the reconstruction error being considerably higher, we also notice reasonable domain transfer capabilities between \figrdata{} images and \fonts{} when training the VSQ module only on one dataset and keeping the maximum stroke length consistent. Qualitative examples of the re-usability of the VSQ module are reported in the Appendix.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Result: Table1: Reconstruction (us vs im2vec subset)
\begin{table*}[h]
\centering
\caption{Results for reconstructions of \ours{} and \imtovec{} on subsets. The last row includes post-processing.}
\label{tab:recons-ours-imvec-subset}
\resizebox{\linewidth}{!}{%
\begin{tabular}{l ccc ccc ccc}
& \multicolumn{3}{c}{MNIST (0)} & \multicolumn{3}{c}{Fonts (A)} & \multicolumn{3}{c}{Icons (Star)} \\
\cmidrule(lr){2-4} \cmidrule(lr){5-7} \cmidrule(lr){8-10}
Model & \MSE & \FID & \CLIP & \MSE & \FID & \CLIP & \MSE & \FID & \CLIP \\
\midrule
\imtovec\ (filled) & 0.218 & \textbf{2.20} & 24.61 & 0.087 & 1.64& 26.27& 0.120& 2.40 & 30.90 \\
\imtovec & \na & \na & \na & 0.060 & 6.33 & 25.78& 0.110& 11.17 & 30.40 \\
\midrule
VSQ & \textbf{0.130} & 11.2 & \textbf{26.68} & 0.020 & 4.50 & 29.13 & 0.002 & 1.26 & 31.64 \\
VSQ + PI & \na & \na & \na & \textbf{0.012} & \textbf{0.61} & \textbf{29.46} & \textbf{0.001} & \textbf{0.07} & \textbf{32.94} \\
\bottomrule
\end{tabular}
}
\end{table*}
\npnoround
\subsection{Generations}
\label{subsubsec:generative-result}
\textbf{Text Conditioning.} We compare \ours{} with \imtovec{} by generating glyphs and icons and handwritten digits, and report the results in \autoref{tab:results-gen-competitor}. Despite \imtovec{} being tailored for single classes only, our general model shows superior performance in CLIPScore for all datasets. \imtovec{} shows a generally lower FID score in the experiments with filled shapes, which we attribute again to the lower resolution of the ground truth images (\mnist{}) and a bias in the metric itself as CLIP struggles to produces meaningful visual embeddings for sparse images~\citep{chowdhury2022fs} as for \fonts, \figrdata. In contrast, in the generative results on unfilled shapes, \ours{} almost consistently outperforms \imtovec{} by a large margin for glyphs and icons.
Note that we establish new baseline results for the complete datasets, as \imtovec{} does not support text or class conditioning.
\begin{table*}[b]
\centering
\caption{Results for generations of \ours{} and \imtovec. \ours{} is trained on the full dataset and conditioned to the respective classes using the text description.}
\label{tab:results-gen-competitor}
\resizebox{\linewidth}{!}{
\begin{tabular}{l cc cc cc cc cc cc}
\toprule
& \multicolumn{2}{c}{MNIST (0)} & \multicolumn{2}{c}{MNIST (Full)} & \multicolumn{2}{c}{Fonts (A)} & \multicolumn{2}{c}{Fonts (Full)} & \multicolumn{2}{c}{\figrdata (Star)} & \multicolumn{2}{c}{\figrdata (Full)}\\
\cmidrule(lr){2-3}
\cmidrule(lr){4-5}
\cmidrule(lr){6-7}
\cmidrule(lr){8-9}
\cmidrule(lr){10-11}
\cmidrule(lr){12-13}
Model & \FID & \CLIP & \FID & \CLIP & \FID & \CLIP & \FID & \CLIP & \FID & \CLIP & \FID & \CLIP \\
\midrule
\imtovec\ (filled) & \textbf{2.22} & 24.69 & \na&\na& \textbf{1.20}& 25.81& \na & \na & \textbf{2.97}& 31.72& \na& \na \\
\imtovec & \na & 25.21 & \na&\na& 5.36& 25.39& \na & \na & 11.59& 31.88& \na& \na \\
\ours{} (ours) & 12.25 & \textbf{26.60}& \textbf{9.25} & \textbf{25.25} & 5.61& \textbf{30.60} & \textbf{1.67}& \textbf{28.64} & 6.25& \textbf{32.24}& \textbf{3.58}& \textbf{27.45}\\
\bottomrule
\end{tabular}
}
\end{table*}
Looking at qualitative samples in \autoref{fig:results-icon-generation-overview} and \autoref{fig:results-icon-generation}, one can see that contrary to the claim that surplus shapes collapse to a point \citep{reddy2021im2vec}, there are multiple redundant shapes present in the generations of \imtovec. A single star might then be represented by ten overlapping almost identical paths. The qualitative results in \autoref{fig:results-mnist-generation} confirm this behaviour on the \mnist{} dataset. We also show that setting \imtovec{} to predict only one single SVG path leads the model to compress the shape area and use its filling as a stroke width.
Overall, \ours{} produces much cleaner samples with less redundancy, which makes them easier to edit and visually more pleasing. The text conditioning also allows for more flexibility.
The generations are also diverse, as can be seen in \autoref{fig:results-icon-generation-overview} where we showcase multiple generations for the same classes from \figrdata{}. Additional generations on all datasets are provided in the Appendix.
\begin{figure*}[h]
\centering
\setlength{\tabcolsep}{3pt}
\begin{tabular}{ccccccc}
\hline
\noalign{\vskip 1mm}
\gentone{1} & \gentone{2} & \gentone{3}& \gentone{4} & \gentone{5} & \gentone{6} & \gentone{7}\\\hline
\noalign{\vskip 1mm}
\gentone{8} & \gentone{9} & \gentone{10} & \gentone{11} & \gentone{12} & \gentone{13} & \gentone{14} \\\hline
\noalign{\vskip 1mm}
\gentone{15} & \gentone{16} & \gentone{17} & \gentone{18} & \gentone{19} & \gentone{20} & \gentone{21} \\\hline
Phone & Heart & Light bulb & Arrow & User & Home & Settings \\
\end{tabular}
\caption{Examples of text-conditioned icon generation from \ours{}.}
\label{fig:results-icon-generation-overview}
\end{figure*}
\begin{figure*}[h]
\centering
\setlength{\tabcolsep}{0.7pt}
\begin{tabular}{p{0.02\textwidth} ccc| ccc | ccc }
\toprule
% Multicolumn header
& \multicolumn{3}{c}{\textbf{\ours}} & \multicolumn{3}{c}{\textbf{\imtovec{} (One path)}} & \multicolumn{3}{c}{\textbf{\imtovec{} (Ten Paths)}}\\
% Column sub-header
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{Generations}} &
\grimmnist{1} & \grimmnist{2} & \grimmnist{3} &
\imvecmnist{1} & \imvecmnist{2} & \imvecmnist{3} &
\imvecmnist{7} & \imvecmnist{8} & \imvecmnist{9} \\
\hline
\noalign{\vskip 1mm}
\end{tabular}
\caption{Generative results for the MNIST dataset from \ours{} and \imtovec with the number of predicted paths fixed to one and ten respectively. Since \imtovec{} does not accept any conditioning, we sample after training \imtovec{} only on the digit Zero. For \ours{}, we use the models trained on the full dataset conditioned on the respective class.}
\label{fig:results-mnist-generation}
\end{figure*}
\textbf{Vector Conditioning.}
We also evaluate \ours{} on another task previously unavailable for image-supervised vector graphic generative models, which is text-guided icon completion. \autoref{fig:results-icon-generation-context} shows the capability of our model to complete an unseen icon, based on a set of given context strokes that start at random positions. \ours{} can meaningfully complete various amounts of contexts, even when the strokes of the context stem from disconnected parts of the icon. We provide a quantitative analysis in \autoref{subsec:clip-on-token-generation}. The results in this section are all obtained with the default pipeline that post-processes the generation of our model. A detailed analysis of our post-processing is provided in \autoref{subsec:post-processing} and \autoref{subsec:postproc-generative-metrics}.
\begin{figure*}[t]
\centering
\setlength{\tabcolsep}{3pt}
\begin{tabular}{p{0.04\textwidth}ccccc}
& $\nu_\text{context}=0$ & $\nu_\text{context}=1$ & $\nu_\text{context}=3$ & $\nu_\text{context}=10$ & \makecell{$\nu_\text{context}=15$} \\
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{Map Pin}} & \notoken{1} & \tokenone{1} & \tokentwo{1} & \tokenthree{1} & \tokenfour{1} \\\hline
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{Cloud}} & \notoken{2} & \tokenone{2} & \tokentwo{2} & \tokenthree{2} & \tokenfour{2} \\\hline
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{Document}} & \notoken{3} & \tokenone{3} & \tokentwo{3} & \tokenthree{3} & \tokenfour{3} \\\hline
\end{tabular}
\caption{Different completions with varying number of context segments $\nu_\text{context}$ (marked in red). \ours{} can meaningfully complete irregular starting positions of the context strokes.}
\label{fig:results-icon-generation-context}
\end{figure*}
\subsection{Flexibility}
Finally, we demonstrate the flexibility of \ours{} through additional qualitative results on new SVG attributes.
One of the advantages of splitting the generative pipeline into two parts is that the ART module can be fully decoupled from the visual attributes of the SVG primitives. Instead, the vector prediction head of the VSQ can be extended to include any visual attribute supported by the differentiable rasterizer.
Specifically, we activate the prediction heads $\Phi_\textrm{width}$ and $\Phi_\textrm{color}$ —outlined in \autoref{subset:vsq-model}— to enable learning of stroke width and color, respectively. We train the VSQ module on input patches while varying the values of those attributes and present the qualitative outcomes in \autoref{fig:vis-attr-recon}, where each stroke is randomly colored using an eight-color palette and a variable stroke width. The VSQ module accurately learns these features without requiring altering the size of the codebook or modifying any other network configurations.
A similar analysis is conducted with closed shapes, and the results are reported in \autoref{fig:recon-shape-colors}, showing that the VSQ module jointly maps both shape and color to a single code. This highlights the minimal requirements of \ours{} in supporting additional SVG features. In contrast, other state-of-the-art vector-based generative models often rely on complex tokenization pipelines, making the extension to new SVG attributes more cumbersome and less flexible.
\begin{figure}[h]
\centering
\includegraphics[width=0.9\linewidth]{figures/visual_attributes/visual_attributes_GT.png}
\includegraphics[width=0.9\linewidth]{figures/visual_attributes/visual_attributes_RECONS.png}
\caption{Inputs (top) and corresponding reconstructions (bottom) generated by a VSQ model trained to predict not only the shape but also the visual attributes of the input strokes, such as color and stroke width.}
\label{fig:vis-attr-recon}
\end{figure}
\begin{figure*}[ht]
\centering
\setlength{\tabcolsep}{4pt}
\begin{tabular}{cc | cc | cc}
Orig & VSQ & {Orig} & VSQ & Orig & VSQ \\
\noalign{\vskip 1mm}
\mniorig{3} & \mnirecon{3} & \mniorig{4} & \mnirecon{4} & \mniorig{5} & \mnirecon{5} \\
\multicolumn{2}{c}{Two in royal blue} & \multicolumn{2}{c}{Four in purple} & \multicolumn{2}{c}{Six in in teal} \\
\hline
\noalign{\vskip 1mm}
\end{tabular}
\caption{Reconstruction of \mnist{} digits when the VSQ module also predicts the filling color. The left side shows the tiling of the original raster images, the right side reports the reconstructions from the VSQ module. No post-processing is applied.
}
\label{fig:recon-shape-colors}
\end{figure*}
\section{Conclusion}
This work presents \ours{}, a novel framework for generating and completing complex SVGs, trained solely on raster images. \ours{} improves existing raster-supervised SVG generative networks in output quality, while offering significantly greater flexibility through text-conditioned generation. We validate \ours{} on filled shapes using a simple tile-patching strategy to create the input data, and on strokes using fonts and icons datasets. Our results demonstrate the superior performance of \ours{} compared to existing models, even when adapted to specific image classes. Additionally, we show that \ours{} can be seamlessly extended to support new SVG attributes when included in the training data.
Future work could explore incorporating additional vector primitives, expanding visual features, or employing a hierarchical approach to patch extraction.
\section{Appendix}
\label{sec:appendix}
\subsection{Glossary of Notation}
Due to the number of notation used in this work, in \autoref{subsec:glossary} we have reported a recap of the most important with a brief description of their meaning.
\begin{table*}[ht]
\centering
\setlength{\tabcolsep}{4pt}
\caption{Glossary of relevant notations in this work.}
\label{subsec:glossary}
\begin{tabular}{ll}
\hline
Notation & Meaning \\
\midrule
$E$ & Network encoder \\
$D$ & Network decoder \\
$I$ & Image from the dataset \\
$V$ & Codebook \\
$v$ & Codes from the codebook \\
$L$ & Set of values per dimension of our codebook \\
$l$ & Single dimensional value \\
$q$ & Number of dimensions of the codebook \\
$\mathbf{S}$ & Series of patches \\
$s$ & Single patch \\
$C$ & Color channels \\
$n$ & Number of patches \\
$\mathbf{\Theta}$ & Set of discrete coordinates \\
$\theta$ & Single coordinate pair \\
$\mathcal{Z}$ & Latent space \\
$\hat{z}$ & Projected embedding \\
$d$ & Dimension of latent space \\
$z$ & Latent embedding \\
$\hat{s}$ & Reconstructed patch \\
$\nu$ & Number of segments \\
$P$ & Set of points \\
$p$ & Point pair \\
$\rho$ & Euclidian distance between two points \\
$\Phi$ & Generic Neural network \\
$\xi$ & Number of codebook codes \\
$T$ & Text description \\
$\mathcal{T}$ & Tokenized description \\
$\tau$ & Text token \\
$t$ & Number of text tokens \\
\hline
\end{tabular}
\end{table*}
\subsection{Pre-Processing}
\label{subsec:pre-processing}
This section provides additional information regarding the pre-processing and extraction techniques on the employed datasets.
\textbf{Shapes.} No pre-processing is conducted for the \mnist{} dataset. Images are simply tiled using a $6 \times 6$ grid and the central position of each tile in the original image is saved.
\noindent \textbf{Strokes.} For the \figrdata{} dataset, the pixels outlining the icons are isolated using a contour finding algorithm~\citep{find_contour} and the coordinates are then used to convert them into vector paths. This simple procedure available in our code repository allows us to efficiently apply a standard pre-processing pipeline defined in~\citet{carlier2020deepsvg} and already adopted by other studies~\citep{wu2023iconshop, tang2024strokenuwa}. The process involves normalizing all strokes and breaking them into shorter units if their length exceeds a certain maximum percentage of the image size. Finally, each resulting path fragment is scaled, translated to the center of a new canvas $s$ by placing the center of its bounding box onto the center of $s$, and rasterized to become part of the training data. Since strokes in $S$ are all translated around the image center, the original center position $\theta$ of the bounding box in $I$ is recorded for each $s$ and saved. These coordinates are discretized in a range of $256 \times 256$ values.
This approach is also used for \fonts{}, but since the data comes in vector format, there is no need for contour finding.
\subsection{Post-Processing}\label{subsec:post-processing}
% Since \ours{} yields various strokes and positions, we further introduce a technique that we invoke to generate a final smooth vector graphics.
Our approach introduces small discrepancies with the ground truth data during tokenization. The VSQ introduces small inaccuracies in the reconstruction of the stroke, and the discretization of the global center positions may sligthly displace said strokes. The latter serve as the training data for the auto-regressive Transformer and therefore represent an upper limit to the final generation quality.
Similarly for \mnist{}, the use of white padding on each patch to facilitate faster convergence results in small background gaps when rendering all shapes together, as shown in \autoref{fig:results-mnist-generation}.
These small errors compound for the full final image and may become fairly visible in the reconstructions.
\begin{figure*}[ht]
\centering
\includegraphics[width=\linewidth]{figures/svg_fixing.png}
\caption{Different SVG post-processing methods visualized. From left to right: raw generation, results of applying PC and PI, results of applying PC and PI by only considering nearest neighbors of consecutive strokes.}
\label{fig:svg_fixing}
\end{figure*}
While we opted not to modify the global reconstructions of \mnist{} generation, for \figrdata{} and \fonts{}, we make use of SVG post-processing similar to prior work \citep{tang2024strokenuwa}, which introduced Path Clipping (PC) and Path Interpolations (PI). In PC, the beginning of a stroke is set to the position of the end of the previous stroke. In PI, a new stroke is added that connects them instead. As we operate on visual supervision, the ordering of the start and end point of a stroke is not consistent. Hence, we adapt these two methods to not consider the start and end point, but rather consider the nearest neighbors of consecutive strokes. We also add a maximum distance parameter to the post-processing in order to avoid intentionally disconnected strokes to get connected. See \autoref{fig:svg_fixing}, \autoref{fig:results-fonts-generation} for a qualitative depiction of this process and \autoref{subsec:postproc-generative-metrics} for a quantitative comparison.
\begin{figure*}[bh]
\centering
\setlength{\tabcolsep}{1pt}
\begin{tabular}{p{0.04\textwidth}p{0.04\textwidth}cccccc}
& & \makecell{capital i in \\ regular font} & \makecell{c in \\ regular font} & \makecell{p in \\ italic font} & \makecell{capital l in \\ regular font} & \makecell{2 in \\ italic font} & \makecell{4 in \\normal font} \\
\noalign{\vskip 1mm}\rotatebox{90}{\ \textbf{Unfixed}} & \rotatebox{90}{\ \textbf{Pred.}} & \funfixed{1} & \funfixed{2} & \funfixed{3} & \funfixed{4} & \funfixed{5} & \funfixed{6}\\\hline
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{PI}} & \rotatebox{90}{\ \textbf{Fixing}} & \fpi{1} & \fpi{2} & \fpi{3} & \fpi{4} & \fpi{5} & \fpi{6} \\\hline
\noalign{\vskip 1mm}
\rotatebox{90}{\ \textbf{PC}} & \rotatebox{90}{\ \textbf{Fixing}} & \fpc{1} & \fpc{2} & \fpc{3} & \fpc{4} & \fpc{5} & \fpc{6}\\\hline
\end{tabular}
\caption{Some examples of text-conditioned glyph generation from \ours{}. The first row shows the unfixed model predictions, the second and third rows depict the final outputs with two different post-processing techniques.
}
\label{fig:results-fonts-generation}
\end{figure*}
\subsection{Results with different Post-processing}
\label{subsec:postproc-generative-metrics}
In \ours{}, the resulting full vector graphic generation is characterized by fragmented segments. This is because the output strokes of the VSQ decoder are each locally centered onto a separate canvas, and the auto-regressive Transformer, which is responsible for the absolute position of each shape, returns only the center coordinates of the predicted shape without controlling the state of connection between different strokes. To cope with this, in \autoref{subsec:post-processing}, we introduced several post-processing algorithms. In this section, we report additional information about the performance of each of them for the VSQ module (reconstruction) and the overall \ours{} (generation). \autoref{tab:post-processing-differences} shows that the PC technique consistently outperforms the alternatives across both datasets in terms of both FID and CLIPScore.
% Since our proposed method generates locally centered and disconnected strokes that are placed on a global canvas, the resulting full vector graphic is characterized by fragmented segments. To make this generation more visually appealing, we introduced post-processing algorithms in \autoref{sec:post-processing}. In our main experiments, we consistently reported numbers for \ours{} with PC post-processing, which is motivated by our findings in \autoref{tab:post-processing-differences} which show that PC consistently performs better for both FID and CLIP score on both datasets.
\begin{table*}[ht]
\centering
\caption{Reconstruction capabilities of our VSQ module and generative performance of \ours{} with different post-processing techniques after training on \fonts{} and \figrdata{}.}
\label{tab:post-processing-differences}
\begin{tabular}{c|ccc|ccc}
Model & \multicolumn{3}{c|}{\fonts} & \multicolumn{3}{c}{\figrdata} \\
&MSE & FID & CLIP & MSE & FID & CLIP \\
\hline
VSQ& 0.0144& 4.45& 28.61& 0.0045&1.29&31.17\\
VSQ (+PC)& 0.0135& \textbf{0.23}& \textbf{29.24}& \textbf{0.0023}&0.10&31.97\\
VSQ (+PI)& \textbf{0.0106}& 0.29& 28.96& 0.0028&\textbf{0.07}&\textbf{32.0}\\
\midrule
\ours{} &\na& 4.44& 28.45& \na&4.20&26.96\\
\ours{} (+PC) &\na& \textbf{1.67}& \textbf{28.64}& \na&\textbf{3.58}&\textbf{27.45}\\
\ours{} (+PI) &\na& 1.86& 28.43& \na&4.57&26.73\\
\bottomrule
\end{tabular}
\end{table*}
\subsection{\imtovec{} on Other Classes}
\label{subsec:im2vec_otherclasses}
We conducted a more in-depth analysis of the generative capabilities in \imtovec{} after training on single subsets of \figrdata{}, and compare the results with \ours{}. We trained \imtovec{} on the top-10 classes of \figrdata{}: Camera (8,818 samples), Home (7,837), User (7,480), Book (7,163), Clock (6,823), Flower (6,698), Star (6,681), Calendar (misspelt as \emph{caledar} in the dataset, 6,230), and Document (6,221). \autoref{tab:single-class-results-app} compares the FID and CLIPScore with \ours{}. Note that we train our model only once on the full \figrdata{} dataset and validate the generative performance using text-conditioning on the target class, whereas
\imtovec{} is unable to handle training on such diverse data.
Despite \imtovec{} appearing to obtain higher scores on several classes such as User or Document, a qualitative inspection reveals how the majority of the generated samples come in the form of meaningless filled blobs or rectangles. The traditional metrics employed in this particular generative field, based on the pre-trained CLIP model, react very strongly to such shapes in contrast to more defined stroke images. We refer reviewers to the qualitative samples in \autoref{fig:im2vec_gen_all_app}. We further observe a low variance in the generations when \imtovec{} learns the representations of certain classes, such as star icons.
\begin{table*}[ht]
\centering
\caption{Generative results for \ours{} and \imtovec{} for the top-10 classes in \figrdata.}
\label{tab:single-class-results-app}
\resizebox{\linewidth}{!}{
\begin{tabular}{c|cc|cc|cc|cc|cc|cc|cc|cc|cc}
\multirow{2}{*}{Model} & \multicolumn{2}{c|}{camera} & \multicolumn{2}{c|}{home} & \multicolumn{2}{c|}{user} & \multicolumn{2}{c|}{book} & \multicolumn{2}{c|}{clock} & \multicolumn{2}{c|}{cloud} & \multicolumn{2}{c|}{flower} & \multicolumn{2}{c|}{calendar} & \multicolumn{2}{c}{document} \\
& FID & CLIP & FID & CLIP & FID & CLIP & FID & CLIP & FID & CLIP & FID & CLIP & FID & CLIP & FID & CLIP & FID & CLIP \\
\hline
\imtovec{} (filled) & 9.21& 27.86& \textbf{3.48} & 26.85& \textbf{2.12} & \textbf{28.92}& 7.18& \textbf{27.26}& 6.12& 26.38& 17.43& 24.38& \textbf{6.61}& \textbf{25.42} & \textbf{4.5}& \textbf{27.26}& 12.19& \textbf{28.65} \\
\imtovec{} & 9.05& 27.18& 9.19& 25.95& 6.33& 27.01& 8.63& 25.84& \textbf{5.09}& 25.69& 25.58& 24.38& 6.8& 23.34& 6.61& 26.22& 16.62& 26.71\\ \hline
\ours{} & 6.74&29.81 &7.16 &27.16 &5.45 &26.81 &6.65 &27.1 & 7.22& 26.32& 6.78& 24.96& 10.27& 22.00& 5.57& 26.23& 4.08& 27.96\\
\ours{} (+PC) &\textbf{5.77} &\textbf{30.22} &7.6 &\textbf{27.41} &4.38 & 27.18& \textbf{5.8} & \textbf{27.24} & 6.79& \textbf{26.45} & \textbf{6.05}& \textbf{25.51}& 9.37& 22.46& 5.09& 26.41& \textbf{3.81}& 28.21\\
\ours{} (+PI) &7.5 &29.46 & 7.44&27.01 &5.95 &26.85 &6.79 &27.08 & 7.63& 26.12& 7.09& 24.73& 9.97& 22.04& 5.87& 25.98& 4.21& 27.89\\
\end{tabular}
\vspace*{2mm}
}
\end{table*}
\subsection{Qualitative Results of the Geometric Loss}
\label{subsec:geometric_qualitative}
The adoption of our geometric constraint improves the overall reconstruction error, which we attribute to the network being encouraged to elongate the stroke as much as possible.
The results in \autoref{tab:geometric-loss-comparison} show the effects on the control points of the reconstructed strokes from the VSQ. With the geometric constraint, the incentive to stretch the stroke works against the MSE objective, which results in an overall longer stroke and therefore in greater connectedness in a full reconstruction and an overall lower reconstruction error. We also present an example with an excessively high geometric constraint weight ($\alpha=5$) demonstrating that beyond a certain threshold, the positive effect diminishes, resulting in degenerated strokes.
\begin{figure}
\centering
\begin{minipage}[]{0.2\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/geometric_loss/gt}
\caption*{Ground Truth}
\label{subfig:gt}
\end{minipage}
\hfill
\begin{minipage}[]{0.2\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/geometric_loss/recons_0}
\caption*{$\alpha = 0$}
\label{subfig:no-geometric}
\end{minipage}
\hfill
\begin{minipage}[]{0.2\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/geometric_loss/recons_0.1}
\caption*{$\alpha = 0.1$}
\label{subfig:geometric}
\end{minipage}
\hfill
\begin{minipage}[]{0.2\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/geometric_loss/recons_5}
\caption*{$\alpha = 5$}
\label{subfig:geometric-toomuch}
\end{minipage}
\caption{Samples from the test set when training the VSQ module with and without our geometric constraint. Each stroke consists of two cubic Bézier segments. Embedded within each stroke, the red dots mark the start and end points, while the green and blue dot pairs are the control points of each segment.}
\label{tab:geometric-loss-comparison}
\end{figure}
\subsection{Implementation Details}
\label{subsec:hyperparams}
%Our models are built using PyTorch Lightning~\citep{Falcon_PyTorch_Lightning_2019}.
We use AdamW optimization and train the VSQ module for 1 epoch for \fonts{} and \figrdata{} and five epochs for \mnist{}. We use a learning rate of $\lambda=2 \times 10^{-5}$, while the auto-regressive Transformer is trained for $\sim$30 epochs with $\lambda=6 \times 10^{-4}$. The Transformer has a context length of 512.
Before proceeding to the second stage, we filter out icons represented by fewer than ten or more than 512 VSQ tokens, which affects 12.16\% of samples. We use p-sampling for our generations with \ours{}. Training the VSQ module on six NVIDIA H100 takes approximately 48, 15, and 12 hours for \mnist{}, \figrdata{}, and \fonts{}, respectively; the ART module takes considerably fewer resources, requiring around 8 hours depending on the configuration.
Regarding \imtovec{}, we replace the Ranger scheduler with AdamW~\citep{loshchilov2017decoupled} and enable the weighting factor for the Kullback–Leibler (KL) divergence in the loss function to \textit{0.1}, as it was disabled by default in the code repository, preventing any sampling. We train \imtovec{} with six paths for 105 epochs with a learning rate of $\lambda=2 \times 10^{-4}$ with early stopping if the validation loss does not decrease after seven epochs.
Regarding the generative metrics, we utilized CLIP with a ViT-16 backend for FID and CLIPScore.
\subsection{Generative Scores with Completion}
\label{subsec:clip-on-token-generation}
To evaluate if \ours{} generalizes and learns to meaningfully complete previously unseen objects, we compare the CLIPScore and FID of completions with varying lengths of context. The context and text prompts are extracted from 1,000 samples of the test set of the \figrdata{} dataset. The results are shown in \autoref{tab:completions-generations}.
While \ours{} can meaningfully complete unseen objects, the quality of these completions is generally lower than the generations under text-only conditioning. This is expected, as prompts in the test set are also encountered during training (the class names). The CLIPScore generally drops to its lowest point with the least amount of context and then recovers when more context is given to the model, which coincides with our qualitative observations that with only a few context strokes, \ours{} occasionally ignores them completely or completes them in an illogical way, reducing the visual appearance.
\begin{table*}
\centering
\caption{Generation quality of \ours{} with different lengths of provided context on \fonts{} and \figrdata{}. Post-processing is conducted for all setups. \ours{} uses textual input for all generations.}
\label{tab:completions-generations}
\begin{tabular}{c|cc|cc}
Model & \multicolumn{2}{c|}{\fonts} & \multicolumn{2}{c}{\figrdata} \\
& FID & CLIP & FID & CLIP \\
\hline
\ours{} (w/o context) & \textbf{1.67} & \textbf{28.64} & \textbf{3.58} & \textbf{27.45} \\
\ours{} (+ 3 stroke context) & 2.78 & 27.25 & 4.65 & 25.31 \\
\ours{} (+ 6 stroke context) & 3.16 & 27.25 & 5.46 & 25.54 \\
\ours{} (+ 12 stroke context) & 2.95 & 27.57 & 6.04 & 25.85 \\
\ours{} (+ 24 stroke context) & 2.25 & 28.12 & 6.05 & 26.39 \\
\end{tabular}\vspace*{1mm}
\end{table*}
\subsection{Domain Transfer Capabilities for Reconstruction}
\label{subsec:domain-transfer}
To validate how the strokes learned during the first training stage adapt to different domains, we use our VSQ module to reconstruct \fonts{} after training on \figrdata{}, and vice versa. \autoref{fig:zeroshot-performance} provides a qualitative example for each setting. Despite the loss value for each image being around one order of magnitude higher than the in-domain test-set (MSE$\approx 0.05$), the VSQ module uses reasonable codes to reconstruct the shapes and picks curves in the correct directions. Straight lines end up being the easiest to decode in both cases.
\begin{figure}
\centering
\rule[1ex]{\linewidth}{1pt}
\begin{minipage}[b]{0.45\textwidth}
\centering
\includegraphics[width=0.9\textwidth]{figures/appendix/zeroshot/icons_on_fonts.png}
\caption*{Icons on \fonts.}
\label{subfig:icons-on-fonts}
\end{minipage}
\hfill
\begin{minipage}[b]{0.45\textwidth}
\centering
\includegraphics[width=0.9\textwidth]{figures/appendix/zeroshot/fonts_on_icons.png}
\caption*{\fonts{} on icons.}
\label{subfig:fonts-on-icons}
\end{minipage}
\rule[2ex]{\linewidth}{1pt}
\caption{Qualitative zero-shot reconstructions from the test-set of \figrdata{} and \fonts{} after training the VSQ module solely on the respective other dataset.}
\label{fig:zeroshot-performance}
\end{figure}
\subsection{Codebook Usage for Strokes}
\label{subsec:codebook-usage}
As described in \autoref{subset:vsq-model}, for FSQ, we fixed the number of dimensions of the hypercube to 5 and set the individual number of values for each dimension as $L = [7, 5, 5, 5, 5]$ for a total codebook size of $|B| = $ 4,375. In this section, we want to share some interesting findings about the learnt codebook. For this, we shall use the VSQ trained on \figrdata{} with $n_{\rm code}=1$, $n_{\rm seg}=2$, a maximum stroke length of $3.0$, and the geometric constraint with $\alpha = 0.2$.
After training the VSQ on \figrdata{}, we tokenize the full dataset. The resulting VQ tokens stem from 60.09\% of the codebook, while 39.91\% of the available codes remained unused. The ten most used strokes make up 41.24\% of the dataset, while the top 24 and 102 strokes make up roughly 50\% and 75\%, respectively. These findings indicate that for these particular VSQ settings, one could experiment with smaller codebook sizes.
To balance out the stroke distribution, one could use a different subset of \figrdata{}. Currently, the classes ``menu'', ``credit card'', ``laptop'', and ``monitor'' are contributing the most to the stroke imbalance, with 26\%, 24.3\%, 24.05\%, and 23.8\% of their respective strokes being the most frequent horizontal one in \autoref{tab:codebook-top-strokes}.
\begin{table*}
\centering
\setlength{\tabcolsep}{4pt}
\caption{Top ten most used strokes of the VSQ module trained on icons and their relative occurrences in our subset of \figrdata{}.}
\label{tab:codebook-top-strokes}
\resizebox{\linewidth}{!}{
\begin{tabular}{p{0.1\textwidth}|c|c|c|c|c|c|c|c|cc}
\appCodebookTopStroke{0_3648}& \appCodebookTopStroke{1_718} & \appCodebookTopStroke{2_543} & \appCodebookTopStroke{3_738} & \appCodebookTopStroke{4_1720} & \appCodebookTopStroke{5_563} & \appCodebookTopStroke{6_2595}&\appCodebookTopStroke{7_213}&\appCodebookTopStroke{8_725}&\appCodebookTopStroke{9_38} \\
\midrule
18.76\% & 12.26\% & 2.56\% &1.73\% &1.16\% &1.12\% &0.99\% &0.94\% &0.92\% &0.80\% \\
\end{tabular}
\vspace*{1mm}
}
\end{table*}
\subsection{Average Strokes in Codebook}
In \autoref{subsec:codebook-usage}, we show the ten most used strokes of our trained VSQ, but after inspecting the full codebook we notice how neighbouring codes often express very similar strokes. Therefore, to visualize the codebook more effectively, we plot mean and minimum reductions of the full codebook in \autoref{fig:codebook-reduction}. Additionally, we tokenize the full \figrdata{} dataset and plot the same reductions in \autoref{fig:figr8-codebook-reduction} to show the composition of the dataset. %The heatmap in XXX shows the distribution of positions in \figrdata{}. % Also plot unused strokes
\begin{figure}
\centering
\rule[1ex]{\linewidth}{1pt}
\begin{minipage}[b]{0.45\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/codebook/reductions/codebook_mean_strokes.png}
\caption*{codebook mean strokes}
\label{subfig:codebook-mean-strokes}
\end{minipage}
\hfill
\begin{minipage}[b]{0.45\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/codebook/reductions/codebook_min_reduce.png}
\caption*{all codebook strokes}
\label{subfig:codebook-all-strokes}
\end{minipage}
\rule[1ex]{\linewidth}{1pt}
\caption{Different reductions of all 4,375 strokes from the VSQ codebook. The model seems to have learned an expressive codebook-decoder mapping as the figure on the left shows a smooth and evenly distributed stroke profile and the figure on the right displays strokes in almost every direction.}
\label{fig:codebook-reduction}
\end{figure}
\begin{figure}
\centering
\rule[1ex]{\linewidth}{1pt}
\begin{minipage}[b]{0.3\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/codebook/reductions/figr8_mean_strokes.png}
\caption*{\figrdata{} mean strokes}
\label{subfig:figr8_mean_strokes}
\end{minipage}
\hfill
\begin{minipage}[b]{0.3\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/codebook/reductions/figr8_mean_strokes_without_top_ten.png}
\caption*{\figrdata{} mean strokes excluding top ten strokes}
\label{subfig:figr8_mean_strokes_without_top_ten}
\end{minipage}
\hfill
\begin{minipage}[b]{0.3\textwidth}
\centering
\includegraphics[width=\textwidth]{figures/appendix/codebook/reductions/figr8_min_reduce.png}
\caption*{all \figrdata{} strokes}
\label{subfig:figr8-min}
\end{minipage}
\rule[1ex]{\linewidth}{1pt}
\caption{Different reductions of all strokes from the tokenized \figrdata{} dataset. The visualization on left shows the dominance of the two most occurring strokes, the middle shows that the distribution of strokes is skewed. The missing 39.91\% of strokes are also visible in the right figure, where certain diagonal strokes that are available in the codebook are never used.}
\label{fig:figr8-codebook-reduction}
\end{figure}
\subsection{Qualitative Results -- Reconstruction }
\label{subsec:im2vec_reconstruction}
In \autoref{fig:grim_rec_fonts_app} and \autoref{fig:grim_rec_icons_app}, we provide several qualitative examples of vector reconstructions using \imtovec{} and our VSQ module on the \fonts{} and \figrdata{} datasets, respectively. We fill the shapes of the images when using \imtovec{}, since the model creates SVGs as series of filled circles and would not be able to learn from strokes with a small width. \imtovec{} does not converge when trained on the full datasets, whereas it returns some approximate reconstruction of the input when only a single class is adopted. In contrast, the VSQ module generalizes over the full dataset.
\subsection{Qualitative Results -- Generation}
In this section, we provide qualitative examples of our reconstruction and generative pipeline, and compared those with \imtovec{}.
\autoref{fig:grim_gen_icons_app} reports a few examples of icons generated with \ours{} using only text-conditioning on classes. In \autoref{fig:grim_gen_mnist_app} we report some generations for \mnist. In \autoref{fig:grim_gen_fonts_app}, we report generative results for \fonts{}. Thanks to the conditioning, we can generate upper-case and lower-case glyphs in bold, italic, light styles, and more. As can be seen in the table, \ours{} also learns to properly mix those styles only based on text.
Finally, in \autoref{fig:im2vec_gen_all_app}, we report some generative results on icons and \fonts{} for \imtovec{} on a single class dataset. The results show how the pipeline typically fails to produce meaningful or sufficiently diverse samples.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%% RECON
\begin{figure}[ht]
\centering
\setlength{\tabcolsep}{4pt}
\begin{tabular}{p{0.1\textwidth}ccccc}
\toprule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecPngFonts{A}{gt_filled_0} & \appgrimRecPngFonts{A}{gt_filled_14} & \appgrimRecPngFonts{A}{gt_filled_29} & \appgrimRecPngFonts{A}{gt_filled_30} & \appgrimRecPngFonts{A}{gt_filled_50} \\
\rotatebox{90}{\ \textbf{Im2Vec}} & \appgrimRecFonts{A}{im2vec_reconstruction_0} & \appgrimRecFonts{A}{im2vec_reconstruction_14} & \appgrimRecFonts{A}{im2vec_reconstruction_29} & \appgrimRecFonts{A}{im2vec_reconstruction_30} & \appgrimRecFonts{A}{im2vec_reconstruction_50} \\
\midrule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecFonts{A}{gt_drawing_13} & \appgrimRecFonts{A}{gt_drawing_51} & \appgrimRecFonts{A}{gt_drawing_93} & \appgrimRecFonts{A}{gt_drawing_206} & \appgrimRecFonts{A}{gt_drawing_318} \\
\rotatebox{90}{\ \textbf{VSQ}} & \appgrimRecFonts{A}{pi_drawing_13} & \appgrimRecFonts{A}{pi_drawing_51} & \appgrimRecFonts{A}{pi_drawing_93} & \appgrimRecFonts{A}{pi_drawing_206} & \appgrimRecFonts{A}{pi_drawing_318} \\
\midrule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecPngFonts{Full}{gt_filled_74} & \appgrimRecPngFonts{Full}{gt_filled_1504} & \appgrimRecPngFonts{Full}{gt_filled_2757} & \appgrimRecPngFonts{Full}{gt_filled_5276} & \appgrimRecPngFonts{Full}{gt_filled_5778} \\
\rotatebox{90}{\ \textbf{Im2Vec}} & \appgrimRecFonts{Full}{im2vec_reconstruction_74} & \appgrimRecFonts{Full}{im2vec_reconstruction_1504} & \appgrimRecFonts{Full}{im2vec_reconstruction_2757} & \appgrimRecFonts{Full}{im2vec_reconstruction_5276} & \appgrimRecFonts{Full}{im2vec_reconstruction_5778} \\
\midrule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecFonts{Full}{gt_drawing_36421} & \appgrimRecFonts{Full}{gt_drawing_36463} & \appgrimRecFonts{Full}{gt_drawing_79131} & \appgrimRecFonts{Full}{gt_drawing_91988} & \appgrimRecFonts{Full}{gt_drawing_93850} \\
\rotatebox{90}{\ \textbf{VSQ}} & \appgrimRecFonts{Full}{pi_drawing_36421} & \appgrimRecFonts{Full}{pi_drawing_36463} & \appgrimRecFonts{Full}{pi_drawing_79131} & \appgrimRecFonts{Full}{pi_drawing_91988} & \appgrimRecFonts{Full}{pi_drawing_93850} \\
\bottomrule
\end{tabular}
\caption{Examples of various reconstructions of our VSQ module after training on \fonts{} compared to reconstructions of \imtovec{} trained on the letter "A" (first row) and \imtovec{} trained on the full \fonts{} dataset (third row).}
\label{fig:grim_rec_fonts_app}
\end{figure}
\begin{figure}[ht]
\centering
\setlength{\tabcolsep}{4pt}
\begin{tabular}{p{0.1\textwidth}ccccc}
\toprule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecPngIcons{star}{gt_filled_1} & \appgrimRecPngIcons{star}{gt_filled_2} & \appgrimRecPngIcons{star}{gt_filled_3} & \appgrimRecPngIcons{star}{gt_filled_4} & \appgrimRecPngIcons{star}{gt_filled_5} \\
\rotatebox{90}{\ \textbf{Im2Vec}} & \appgrimRecIcons{star}{im2vec_reconstruction_1} & \appgrimRecIcons{star}{im2vec_reconstruction_2} & \appgrimRecIcons{star}{im2vec_reconstruction_3} & \appgrimRecIcons{star}{im2vec_reconstruction_4} & \appgrimRecIcons{star}{im2vec_reconstruction_5} \\
\midrule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecIcons{star}{gt_drawing_1} & \appgrimRecIcons{star}{gt_drawing_2} & \appgrimRecIcons{star}{gt_drawing_3} & \appgrimRecIcons{star}{gt_drawing_4} & \appgrimRecIcons{star}{gt_drawing_5} \\
\rotatebox{90}{\ \textbf{VSQ}} & \appgrimRecIcons{star}{pi_drawing_1} & \appgrimRecIcons{star}{pi_drawing_2} & \appgrimRecIcons{star}{pi_drawing_3} & \appgrimRecIcons{star}{pi_drawing_4} & \appgrimRecIcons{star}{pi_drawing_5} \\
\midrule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecPngIcons{Full}{f_gt_filled_1} & \appgrimRecPngIcons{Full}{f_gt_filled_2} & \appgrimRecPngIcons{Full}{f_gt_filled_3} & \appgrimRecPngIcons{Full}{f_gt_filled_4} & \appgrimRecPngIcons{Full}{f_gt_filled_5} \\
\rotatebox{90}{\ \textbf{Im2Vec}} & \appgrimRecIcons{Full}{f_im2vec_reconstruction_1} & \appgrimRecIcons{Full}{f_im2vec_reconstruction_2} & \appgrimRecIcons{Full}{f_im2vec_reconstruction_3} & \appgrimRecIcons{Full}{f_im2vec_reconstruction_4} & \appgrimRecIcons{Full}{f_im2vec_reconstruction_5} \\
\midrule
\rotatebox{90}{\ \textbf{GT}} & \appgrimRecIcons{Full}{f_gt_drawing_1} & \appgrimRecIcons{Full}{f_gt_drawing_2} & \appgrimRecIcons{Full}{f_gt_drawing_3} & \appgrimRecIcons{Full}{f_gt_drawing_4} & \appgrimRecIcons{Full}{f_gt_drawing_5} \\
\rotatebox{90}{\ \textbf{VSQ} } & \appgrimRecIcons{Full}{f_pi_drawing_1} & \appgrimRecIcons{Full}{f_pi_drawing_2} & \appgrimRecIcons{Full}{f_pi_drawing_3} & \appgrimRecIcons{Full}{f_pi_drawing_4} & \appgrimRecIcons{Full}{f_pi_drawing_5} \\
\bottomrule
\end{tabular}
\caption{Examples of various reconstructions of our VSQ module after training on icons compared to reconstructions of \imtovec{} trained on one class (first row) and \imtovec{} trained on the full dataset (third row).}
\label{fig:grim_rec_icons_app}
\end{figure}
%%%%%%%%%%% END RECON
%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%% GENERATIONS grim
\begin{figure*}[ht]
\centering
\setlength{\tabcolsep}{5pt}
\begin{tabular}{ccccc}
\toprule
\appgrimGenIcons{1} & \appgrimGenIcons{2} & \appgrimGenIcons{3} & \appgrimGenIcons{4} & \appgrimGenIcons{5} \\
Clock & Luggage & Shopping Bag & Camera & Like \\ \midrule
\appgrimGenIcons{6} & \appgrimGenIcons{7} & \appgrimGenIcons{8} & \appgrimGenIcons{9} & \appgrimGenIcons{10} \\
Home & Mail & Share & Target & Arrow \\ \midrule
\appgrimGenIcons{11} & \appgrimGenIcons{12} & \appgrimGenIcons{13} & \appgrimGenIcons{14} & \appgrimGenIcons{15} \\
Microphone & Gift & Clock & Eye & Cube \\ \midrule
\appgrimGenIcons{16} & \appgrimGenIcons{17} & \appgrimGenIcons{18} & \appgrimGenIcons{19} & \appgrimGenIcons{20} \\
Sun & Bell & Bell & Smile & Clip \\ \midrule
\appgrimGenIcons{21} & \appgrimGenIcons{22} & \appgrimGenIcons{23} & \appgrimGenIcons{24} & \appgrimGenIcons{25} \\
Smile & Share & Bin & Book & Home \\
\bottomrule
\end{tabular}
\caption{Examples of various samples generated with \ours{} after training on icons, using only text conditioning.}
\label{fig:grim_gen_icons_app}
\end{figure*}
\begin{figure*}[ht]
\centering
\setlength{\tabcolsep}{2pt}
\begin{tabular}{ccccc}
\toprule
\appgrimGenMnist{0} & \appgrimGenMnist{1} & \appgrimGenMnist{2} & \appgrimGenMnist{3} & \appgrimGenMnist{4} \\
\appgrimGenMnist{5} & \appgrimGenMnist{6} & \appgrimGenMnist{7} & \appgrimGenMnist{8} & \appgrimGenMnist{9} \\
\bottomrule
\end{tabular}
\caption{Examples of a samples generated with \ours{} for each digit of the \mnist{} dataset.}
\label{fig:grim_gen_mnist_app}
\end{figure*}
\begin{figure*}[ht]
\centering
\setlength{\tabcolsep}{4pt}
\begin{tabular}{p{0.03\textwidth}ccccc}
\toprule
\rotatebox{90}{\ \textbf{A}} & \appImtovecGenIcons{00} & \appImtovecGenIcons{01} & \appImtovecGenIcons{02} & \appImtovecGenIcons{03} & \appImtovecGenIcons{04} \\
\rotatebox{90}{\ \textbf{Star}} & \appImtovecGenIcons{10} & \appImtovecGenIcons{11} & \appImtovecGenIcons{12} & \appImtovecGenIcons{13} & \appImtovecGenIcons{14} \\
\rotatebox{90}{\ \textbf{User}} & \appImtovecGenIcons{20} & \appImtovecGenIcons{21} & \appImtovecGenIcons{22} & \appImtovecGenIcons{23} & \appImtovecGenIcons{24} \\
\rotatebox{90}{\ \textbf{Document}} & \appImtovecGenIcons{30} & \appImtovecGenIcons{31} & \appImtovecGenIcons{32} & \appImtovecGenIcons{33} & \appImtovecGenIcons{34} \\
\rotatebox{90}{\ \textbf{Camera}} & \appImtovecGenIcons{40} & \appImtovecGenIcons{41} & \appImtovecGenIcons{42} & \appImtovecGenIcons{43} & \appImtovecGenIcons{44} \\
\rotatebox{90}{\ \textbf{Book}} & \appImtovecGenIcons{55} & \appImtovecGenIcons{51} & \appImtovecGenIcons{52} & \appImtovecGenIcons{53} & \appImtovecGenIcons{54} \\
\bottomrule
\end{tabular}
\caption{Examples of filled samples generated with \imtovec{} after training the model on specific classes of the dataset. For most classes, \imtovec{} could not capture the diversity of the data and failed to meaningfully converge.}
\label{fig:grim_gen_fonts_app}
\end{figure*}
\begin{figure*}[ht]
\centering
\setlength{\tabcolsep}{2pt}
\begin{tabular}{p{0.03\textwidth}cccccc}
\toprule
\rotatebox{90}{\ \textbf{Capital}} & \appgrimGenFonts{00} & \appgrimGenFonts{01} & \appgrimGenFonts{02} & \appgrimGenFonts{03} & \appgrimGenFonts{04} & \appgrimGenFonts{05} \\
\rotatebox{90}{\ \textbf{Capital}} & \appgrimGenFonts{10} & \appgrimGenFonts{11} & \appgrimGenFonts{12} & \appgrimGenFonts{13} & \appgrimGenFonts{14} & \appgrimGenFonts{15} \\
\rotatebox{90}{\ \textbf{Minuscule}} & \appgrimGenFonts{20} & \appgrimGenFonts{21} & \appgrimGenFonts{22} & \appgrimGenFonts{23} & \appgrimGenFonts{24} & \appgrimGenFonts{25} \\
\rotatebox{90}{\ \textbf{Minuscule}} & \appgrimGenFonts{30} & \appgrimGenFonts{31} & \appgrimGenFonts{32} & \appgrimGenFonts{33} & \appgrimGenFonts{34} & \appgrimGenFonts{35} \\
& \textbf{Regular} & \textbf{Regular} & \textbf{Italic} & \textbf{Bold} & \textbf{Bold-italic} & \textbf{Light}\\
\bottomrule
\end{tabular}
\caption{Examples of various samples generated with \ours{} after training on \fonts, using only text conditioning.}
\label{fig:im2vec_gen_all_app}
\end{figure*}
%%%%%%%% ENDS GENERATIONS GRIM
%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{thebibliography}{40}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Aoki and Aizawa(2022)]{aoki2022svg}
H.~Aoki and K.~Aizawa.
\newblock Svg vector font generation for chinese characters with transformer.
\newblock In \emph{2022 IEEE International Conference on Image Processing (ICIP)}, pages 646--650. IEEE, 2022.
\bibitem[Carlier et~al.(2020)Carlier, Danelljan, Alahi, and Timofte]{carlier2020deepsvg}
A.~Carlier, M.~Danelljan, A.~Alahi, and R.~Timofte.
\newblock Deepsvg: A hierarchical generative network for vector graphics animation.
\newblock \emph{Advances in Neural Information Processing Systems}, 33:\penalty0 16351--16361, 2020.
\bibitem[Chen et~al.(2016)Chen, Duan, Houthooft, Schulman, Sutskever, and Abbeel]{chen2016infogan}
X.~Chen, Y.~Duan, R.~Houthooft, J.~Schulman, I.~Sutskever, and P.~Abbeel.
\newblock Infogan: Interpretable representation learning by information maximizing generative adversarial nets.
\newblock \emph{Advances in neural information processing systems}, 29, 2016.
\bibitem[Chowdhury et~al.(2022)Chowdhury, Sain, Bhunia, Xiang, Gryaditskaya, and Song]{chowdhury2022fs}
P.~N. Chowdhury, A.~Sain, A.~K. Bhunia, T.~Xiang, Y.~Gryaditskaya, and Y.-Z. Song.
\newblock Fs-coco: Towards understanding of freehand sketches of common objects in context.
\newblock In \emph{European Conference on Computer Vision}, pages 253--270. Springer, 2022.
\bibitem[Clou{\^a}tre and Demers(2019)]{clouatre2019figr}
L.~Clou{\^a}tre and M.~Demers.
\newblock Figr: Few-shot image generation with reptile.
\newblock \emph{arXiv preprint arXiv:1901.02199}, 2019.
\bibitem[Courville et~al.(2011)Courville, Bergstra, and Bengio]{courville2011spike}
A.~Courville, J.~Bergstra, and Y.~Bengio.
\newblock A spike and slab restricted boltzmann machine.
\newblock In \emph{Proceedings of the fourteenth international conference on artificial intelligence and statistics}, pages 233--241. JMLR Workshop and Conference Proceedings, 2011.
\bibitem[Denton et~al.(2016)Denton, Gross, and Fergus]{denton2016semi}
E.~Denton, S.~Gross, and R.~Fergus.
\newblock Semi-supervised learning with context-conditional generative adversarial networks.
\newblock \emph{arXiv preprint arXiv:1611.06430}, 2016.
\bibitem[Devlin et~al.(2018)Devlin, Chang, Lee, and Toutanova]{devlin2018bert}
J.~Devlin, M.-W. Chang, K.~Lee, and K.~Toutanova.
\newblock Bert: Pre-training of deep bidirectional transformers for language understanding.
\newblock \emph{arXiv preprint arXiv:1810.04805}, 2018.
\bibitem[Frans et~al.(2022)Frans, Soros, and Witkowski]{frans2022clipdraw}
K.~Frans, L.~Soros, and O.~Witkowski.
\newblock Clipdraw: Exploring text-to-drawing synthesis through language-image encoders.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 5207--5218, 2022.
\bibitem[Ganin et~al.(2018)Ganin, Kulkarni, Babuschkin, Eslami, and Vinyals]{ganin2018synthesizing}
Y.~Ganin, T.~Kulkarni, I.~Babuschkin, S.~A. Eslami, and O.~Vinyals.
\newblock Synthesizing programs for images using reinforced adversarial learning.
\newblock In \emph{International Conference on Machine Learning}, pages 1666--1675. PMLR, 2018.
\bibitem[Goodfellow et~al.(2014)Goodfellow, Pouget-Abadie, Mirza, Xu, Warde-Farley, Ozair, Courville, and Bengio]{goodfellow2014generative}
I.~Goodfellow, J.~Pouget-Abadie, M.~Mirza, B.~Xu, D.~Warde-Farley, S.~Ozair, A.~Courville, and Y.~Bengio.
\newblock Generative adversarial nets.
\newblock \emph{Advances in neural information processing systems}, 27, 2014.
\bibitem[Ha and Eck(2017)]{ha2017neural}
D.~Ha and D.~Eck.
\newblock A neural representation of sketch drawings.
\newblock \emph{arXiv preprint arXiv:1704.03477}, 2017.
\bibitem[He et~al.(2016)He, Zhang, Ren, and Sun]{he2016deep}
K.~He, X.~Zhang, S.~Ren, and J.~Sun.
\newblock Deep residual learning for image recognition.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 770--778, 2016.
\bibitem[Heusel et~al.(2017)Heusel, Ramsauer, Unterthiner, Nessler, and Hochreiter]{heusel2017gans}
M.~Heusel, H.~Ramsauer, T.~Unterthiner, B.~Nessler, and S.~Hochreiter.
\newblock Gans trained by a two time-scale update rule converge to a local nash equilibrium.
\newblock \emph{Advances in neural information processing systems}, 30, 2017.
\bibitem[Hinton and Salakhutdinov(2006)]{hinton2006reducing}
G.~E. Hinton and R.~R. Salakhutdinov.
\newblock Reducing the dimensionality of data with neural networks.
\newblock \emph{science}, 313\penalty0 (5786):\penalty0 504--507, 2006.
\bibitem[Ho et~al.(2020)Ho, Jain, and Abbeel]{ho2020denoising}
J.~Ho, A.~Jain, and P.~Abbeel.
\newblock Denoising diffusion probabilistic models.
\newblock \emph{Advances in neural information processing systems}, 33:\penalty0 6840--6851, 2020.
\bibitem[Iluz et~al.(2023)Iluz, Vinker, Hertz, Berio, Cohen-Or, and Shamir]{iluz2023word}
S.~Iluz, Y.~Vinker, A.~Hertz, D.~Berio, D.~Cohen-Or, and A.~Shamir.
\newblock Word-as-image for semantic typography.
\newblock \emph{arXiv preprint arXiv:2303.01818}, 2023.
\bibitem[Isola et~al.(2017)Isola, Zhu, Zhou, and Efros]{isola2017image}
P.~Isola, J.-Y. Zhu, T.~Zhou, and A.~A. Efros.
\newblock Image-to-image translation with conditional adversarial networks.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 1125--1134, 2017.
\bibitem[Jain et~al.(2023)Jain, Xie, and Abbeel]{jain2023vectorfusion}
A.~Jain, A.~Xie, and P.~Abbeel.
\newblock Vectorfusion: Text-to-svg by abstracting pixel-based diffusion models.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 1911--1920, 2023.
\bibitem[Kingma and Welling(2013)]{kingma2013auto}
D.~P. Kingma and M.~Welling.
\newblock Auto-encoding variational bayes.
\newblock \emph{arXiv preprint arXiv:1312.6114}, 2013.
\bibitem[LeCun et~al.(1998)LeCun, Bottou, Bengio, and Haffner]{lecun1998gradient}
Y.~LeCun, L.~Bottou, Y.~Bengio, and P.~Haffner.
\newblock Gradient-based learning applied to document recognition.
\newblock \emph{Proceedings of the IEEE}, 86\penalty0 (11):\penalty0 2278--2324, 1998.
\bibitem[Li et~al.(2020)Li, Luk{\'a}{\v{c}}, Gharbi, and Ragan-Kelley]{li2020differentiable}
T.-M. Li, M.~Luk{\'a}{\v{c}}, M.~Gharbi, and J.~Ragan-Kelley.
\newblock Differentiable vector graphics rasterization for editing and learning.
\newblock \emph{ACM Transactions on Graphics (TOG)}, 39\penalty0 (6):\penalty0 1--15, 2020.
\bibitem[Lopes et~al.(2019)Lopes, Ha, Eck, and Shlens]{lopes2019learned}
R.~G. Lopes, D.~Ha, D.~Eck, and J.~Shlens.
\newblock A learned representation for scalable vector graphics.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 7930--7939, 2019.
\bibitem[Lorensen and Cline(1987)]{find_contour}
W.~E. Lorensen and H.~E. Cline.
\newblock Marching cubes: A high resolution 3d surface construction algorithm.
\newblock In \emph{Proceedings of the 14th Annual Conference on Computer Graphics and Interactive Techniques}, SIGGRAPH '87, page 163–169, New York, NY, USA, 1987. Association for Computing Machinery.
\newblock ISBN 0897912276.
\newblock \doi{10.1145/37401.37422}.
\newblock URL \url{https://doi.org/10.1145/37401.37422}.
\bibitem[Loshchilov and Hutter(2017)]{loshchilov2017decoupled}
I.~Loshchilov and F.~Hutter.
\newblock Decoupled weight decay regularization.
\newblock \emph{arXiv preprint arXiv:1711.05101}, 2017.
\bibitem[Mentzer et~al.(2023)Mentzer, Minnen, Agustsson, and Tschannen]{mentzer2023finite}
F.~Mentzer, D.~Minnen, E.~Agustsson, and M.~Tschannen.
\newblock Finite scalar quantization: Vq-vae made simple.
\newblock \emph{arXiv preprint arXiv:2309.15505}, 2023.
\bibitem[Mnih and Gregor(2014)]{mnih2014neural}
A.~Mnih and K.~Gregor.
\newblock Neural variational inference and learning in belief networks.
\newblock In \emph{International Conference on Machine Learning}, pages 1791--1799. PMLR, 2014.
\bibitem[Nakano(2019)]{nakano2019neural}
R.~Nakano.
\newblock Neural painters: A learned differentiable constraint for generating brushstroke paintings.
\newblock \emph{arXiv preprint arXiv:1904.08410}, 2019.
\bibitem[Nichol et~al.(2021)Nichol, Dhariwal, Ramesh, Shyam, Mishkin, McGrew, Sutskever, and Chen]{nichol2021glide}
A.~Nichol, P.~Dhariwal, A.~Ramesh, P.~Shyam, P.~Mishkin, B.~McGrew, I.~Sutskever, and M.~Chen.
\newblock Glide: Towards photorealistic image generation and editing with text-guided diffusion models.
\newblock \emph{arXiv preprint arXiv:2112.10741}, 2021.
\bibitem[Poole et~al.(2022)Poole, Jain, Barron, and Mildenhall]{poole2022dreamfusion}
B.~Poole, A.~Jain, J.~T. Barron, and B.~Mildenhall.
\newblock Dreamfusion: Text-to-3d using 2d diffusion.
\newblock \emph{arXiv preprint arXiv:2209.14988}, 2022.
\bibitem[Radford et~al.(2021)Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, et~al.]{radford2021learning}
A.~Radford, J.~W. Kim, C.~Hallacy, A.~Ramesh, G.~Goh, S.~Agarwal, G.~Sastry, A.~Askell, P.~Mishkin, J.~Clark, et~al.
\newblock Learning transferable visual models from natural language supervision.
\newblock In \emph{International conference on machine learning}, pages 8748--8763. PMLR, 2021.
\bibitem[Reddy et~al.(2021)Reddy, Gharbi, Lukac, and Mitra]{reddy2021im2vec}
P.~Reddy, M.~Gharbi, M.~Lukac, and N.~J. Mitra.
\newblock Im2vec: Synthesizing vector graphics without vector supervision.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 7342--7351, 2021.
\bibitem[Rezende et~al.(2014)Rezende, Mohamed, and Wierstra]{rezende2014stochastic}
D.~J. Rezende, S.~Mohamed, and D.~Wierstra.
\newblock Stochastic backpropagation and approximate inference in deep generative models.
\newblock In \emph{International conference on machine learning}, pages 1278--1286. PMLR, 2014.
\bibitem[Rombach et~al.(2022)Rombach, Blattmann, Lorenz, Esser, and Ommer]{rombach2022high}
R.~Rombach, A.~Blattmann, D.~Lorenz, P.~Esser, and B.~Ommer.
\newblock High-resolution image synthesis with latent diffusion models.
\newblock In \emph{Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 10684--10695, 2022.
\bibitem[Saharia et~al.(2022)Saharia, Chan, Saxena, Li, Whang, Denton, Ghasemipour, Gontijo~Lopes, Karagol~Ayan, Salimans, et~al.]{saharia2022photorealistic}
C.~Saharia, W.~Chan, S.~Saxena, L.~Li, J.~Whang, E.~L. Denton, K.~Ghasemipour, R.~Gontijo~Lopes, B.~Karagol~Ayan, T.~Salimans, et~al.
\newblock Photorealistic text-to-image diffusion models with deep language understanding.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 36479--36494, 2022.
\bibitem[Tang et~al.(2024)Tang, Wu, Zhang, Ni, Yin, Liu, Yang, Wang, Liu, Li, et~al.]{tang2024strokenuwa}
Z.~Tang, C.~Wu, Z.~Zhang, M.~Ni, S.~Yin, Y.~Liu, Z.~Yang, L.~Wang, Z.~Liu, J.~Li, et~al.
\newblock Strokenuwa: Tokenizing strokes for vector graphic synthesis.
\newblock \emph{arXiv preprint arXiv:2401.17093}, 2024.
\bibitem[Van Den~Oord et~al.(2017)Van Den~Oord, Vinyals, et~al.]{van2017neural}
A.~Van Den~Oord, O.~Vinyals, et~al.
\newblock Neural discrete representation learning.
\newblock \emph{Advances in neural information processing systems}, 30, 2017.
\bibitem[Vincent et~al.(2010)Vincent, Larochelle, Lajoie, Bengio, Manzagol, and Bottou]{vincent2010stacked}
P.~Vincent, H.~Larochelle, I.~Lajoie, Y.~Bengio, P.-A. Manzagol, and L.~Bottou.
\newblock Stacked denoising autoencoders: Learning useful representations in a deep network with a local denoising criterion.
\newblock \emph{Journal of machine learning research}, 11\penalty0 (12), 2010.
\bibitem[Vinker et~al.(2022)Vinker, Pajouheshgar, Bo, Bachmann, Bermano, Cohen-Or, Zamir, and Shamir]{vinker2022clipasso}
Y.~Vinker, E.~Pajouheshgar, J.~Y. Bo, R.~C. Bachmann, A.~H. Bermano, D.~Cohen-Or, A.~Zamir, and A.~Shamir.
\newblock Clipasso: Semantically-aware object sketching.
\newblock \emph{ACM Transactions on Graphics (TOG)}, 41\penalty0 (4):\penalty0 1--11, 2022.
\bibitem[Wu et~al.(2023)Wu, Su, Ma, and Liao]{wu2023iconshop}
R.~Wu, W.~Su, K.~Ma, and J.~Liao.
\newblock Iconshop: Text-guided vector icon synthesis with autoregressive transformers.
\newblock \emph{ACM Transactions on Graphics (TOG)}, 42\penalty0 (6):\penalty0 1--14, 2023.
\end{thebibliography}