Prompt Augmentation for Self-supervised Text-guided Image Manipulation
Extracted source text · arXiv 2412.13081
% CVPR 2024 Paper Template; see https://github.com/cvpr-org/author-kit
\documentclass[10pt,twocolumn,letterpaper]{article}
%%%%%%%%% PAPER TYPE - PLEASE UPDATE FOR FINAL VERSION
\usepackage{cvpr} % To produce the CAMERA-READY version
% \usepackage[review]{cvpr} % To produce the REVIEW version
% \usepackage[pagenumbers]{cvpr} % To force page numbers, e.g. for an arXiv version
\usepackage{tikz}
\usepackage{cancel}
\usepackage{pifont}% http://ctan.org/pkg/pifont
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
% Import additional packages in the preamble file, before hyperref
\input{preamble}
% It is strongly recommended to use hyperref, especially for the review version.
% hyperref with option pagebackref eases the reviewers' job.
% Please disable hyperref *only* if you encounter grave issues,
% e.g. with the file validation for the camera-ready version.
%
% If you comment hyperref and then uncomment it, you should delete *.aux before re-running LaTeX.
% (Or just hit 'q' on the first LaTeX run, let it finish, and you should be clear).
\definecolor{cvprblue}{rgb}{0.21,0.49,0.74}
\usepackage[pagebackref,breaklinks,colorlinks,citecolor=cvprblue]{hyperref}
%%%%%%%%% PAPER ID - PLEASE UPDATE
\def\paperID{11915} % *** Enter the Paper ID here
\def\confName{CVPR}
\def\confYear{2024}
%%%%%%%%% TITLE - PLEASE UPDATE
% \title{Prompt Augmentation for Text-guided Image Editing}
\title{Prompt Augmentation for Self-supervised Text-guided Image Manipulation}
%%%%%%%%% AUTHORS - PLEASE UPDATE
% \author{Rumeysa Bodur\\
% Imperial College London, UK\\
% {\tt\small r.bodur18@imperial.ac.uk}
% % For a paper whose authors are all at the same institution,
% % omit the following lines up until the closing ``}''.
% % Additional authors and addresses can be added with ``\and'',
% % just like the second author.
% % To save space, use either the email address or home page, not both
% \and
% Binod Bhattarai\\
% University of Aberdeen, UK\\
% {\tt\small binod.bhattarai@abdn.ac.uk}
% \and
% Tae-Kyun Kim\\
% KAIST, South Korea\\
% {\tt\small kimtaekyun@kaist.ac.kr}
% }
\author{Rumeysa Bodur$^1$, Binod Bhattarai$^2$, Tae-Kyun Kim$^{1,3}$ \\%\thanks{Thanks to XYZ agency for funding.}}\\
$^1$Imperial College London, UK, $^2$University of Aberdeen, UK, $^3$KAIST, South Korea \\
\tt\small{r.bodur18@imperial.ac.uk, binod.bhattarai@abdn.ac.uk, kimtaekyun@kaist.ac.kr}}
% \address{\tt\small{\{r.bodur18, b.bhattarai, tk.kim\}@imperial.ac.uk}}
\begin{document}
\maketitle
\input{sec/0_abstract}
\input{sec/1_intro}
\input{sec/2_related_work}
\input{sec/3_method}
\input{sec/4_evaluation}
\input{sec/5_conclusion}
\newpage
{
\small
\bibliographystyle{ieeenat_fullname}
\bibliography{main}
}
% WARNING: do not forget to delete the supplementary pages from your submission
% \input{sec/X_suppl}
\end{document}
%
% --- inline annotations
%
% \usepackage[dvipsnames]{xcolor}
\newcommand{\red}[1]{{\color{red}#1}}
% \newcommand{\todo}[1]{{\color{red}#1}}
\newcommand{\TODO}[1]{\textbf{\color{red}[TODO: #1]}}
% --- disable by uncommenting
% \renewcommand{\TODO}[1]{}
% \renewcommand{\todo}[1]{#1}
\documentclass[10pt,twocolumn,letterpaper]{article}
\usepackage[rebuttal]{cvpr}
% Include other packages here, before hyperref.
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
% Import additional packages in the preamble file, before hyperref
\input{preamble}
% If you comment hyperref and then uncomment it, you should delete
% egpaper.aux before re-running latex. (Or just hit 'q' on the first latex
% run, let it finish, and you should be clear).
\definecolor{cvprblue}{rgb}{0.21,0.49,0.74}
\usepackage[pagebackref,breaklinks,colorlinks,citecolor=cvprblue]{hyperref}
% Support for easy cross-referencing
\usepackage[capitalize]{cleveref}
\crefname{section}{Sec.}{Secs.}
\Crefname{section}{Section}{Sections}
\Crefname{table}{Table}{Tables}
\crefname{table}{Tab.}{Tabs.}
% If you wish to avoid re-using figure, table, and equation numbers from
% the main paper, please uncomment the following and change the numbers
% appropriately.
%\setcounter{figure}{2}
%\setcounter{table}{1}
%\setcounter{equation}{2}
% If you wish to avoid re-using reference numbers from the main paper,
% please uncomment the following and change the counter for `enumiv' to
% the number of references you have in the main paper (here, 6).
%\let\oldthebibliography=\thebibliography
%\let\oldendthebibliography=\endthebibliography
%\renewenvironment{thebibliography}[1]{%
% \oldthebibliography{#1}%
% \setcounter{enumiv}{6}%
%}{\oldendthebibliography}
%%%%%%%%% PAPER ID - PLEASE UPDATE
\def\paperID{11915} % *** Enter the Paper ID here
\def\confName{CVPR}
\def\confYear{2024}
\begin{document}
%%%%%%%%% TITLE - PLEASE UPDATE
\title{Prompt Augmentation for Self-supervised Text-guided Image Manipulation} % **** Enter the paper title here
\maketitle
\thispagestyle{empty}
\appendix
%%%%%%%%% BODY TEXT - ENTER YOUR RESPONSE BELOW
\noindent We thank reviewers for taking the time to review and provide constructive feedback.
% , significantly contributing to refining our paper.
As they note, our work presents \textit{a novel and sensible idea}, is \textit{well-motivated}, \textit{easy to follow}, involves \textit{comprehensive experiments}.
\begin{table}[ht]
\vspace{-0.2cm}
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{lccccc}
\toprule
\textbf{Method} & \textbf{SDEdit} & \textbf{DiffEdit} & \textbf{DALL-E 2} & \textbf{IP2P} & \textbf{Ours} \\
\midrule
% \textbf{Preference} & \textcolor{blue}{76.3}\%/\textcolor{red}{23.7}\% & \textcolor{blue}{73.84}\%/\textcolor{red}{26.16}\% & \textcolor{blue}{68.3}\%/\textcolor{red}{31.7}\% & \textcolor{blue}{62.15}\%/\textcolor{red}{37.85}\% & - \\
\textbf{Preference} & \textcolor{blue}{76.3}\% & \textcolor{blue}{73.84}\% & \textcolor{blue}{68.3}\% & \textcolor{blue}{62.15}\% & - \\
& \textcolor{red}{23.7}\% & \textcolor{red}{26.16}\% & \textcolor{red}{31.7}\% & \textcolor{red}{37.85}\% & - \\
\textbf{CLIP-R-Prec } & 75.91 $\pm$ 1.5 & 67.60 $\pm$ 1.7 & 81.83 $\pm$ 1.4 & 77.45 $\pm$ 1.6 & 80.69 $\pm$ 1.45\\
\bottomrule
\end{tabular}
}
\vspace{-0.2cm}
\caption{User Study. \textcolor{red}{Red} shows the baseline and \textcolor{blue}{blue} our method.}
\vspace{-0.1cm}
\label{tab:user_study}
\end{table}
\vspace{-0.1cm}
\noindent\textbf{Human Study [Zr6o, bsLE, 9Hzf].}
% In response to reviewer suggestions, we conducted a human study on Microworkers to validate our findings.
Following established protocols [2,12], on Microworkers, we presented users with $540$ pairs, consisting of input images, target prompts, and manipulation results from both our method and a baseline. For each pair, $10$ different participants selected the more successful manipulation based on faithfulness to the prompt and input image, resulting in $5.4$K responses.
% Tab.~\ref{tab:user_study} presents the percentages of users \emph{preferring our results over the respective baseline}, with \textcolor{blue}{blue} indicating a preference for our method and \textcolor{red}{red} for the counterpart baseline.
In Tab.~\ref{tab:user_study}, \textcolor{blue}{blue} indicates preference for our method and \textcolor{red}{red} for the counterpart baseline. The results align more closely with our qualitative observations than the automatic metrics. DALL-E, despite achieving the highest ClipScore (CS), is preferred less than our method and InstructPix2Pix (IP2P). This is attributed to its low fidelity to masked content, emphasising nuances that automatic metrics might overlook. Similarly, IP2P's weaknesses, not directly measurable automatically, contribute to its lag behind our method. We will provide details of this study in the revised manuscript.
\noindent\textbf{CLIP-R-Prec(CP)[Zr6o].} As suggested, we evaluated our method using CP (R=1)(Tab.~\ref{tab:user_study}), known for better alignment with human preferences compared to CS. While CP aligns more closely with our human study results, it still falls short in capturing the identified shortcomings of DALL-E, highlighted in our user study and qualitative findings. We will include these results and further analysis in the final version.
\noindent\textbf{Prompt Augmentation Strategy [Zr6o].}
We admit that the focus of our method on augmenting prompts is primarily nouns and adjectives, chosen for their role in shaping context and visual attributes, amongst endless manipulation possibilities. While we acknowledge the need for advanced augmentation techniques, our work lays the foundation for future exploration in this direction. Notably, our use of the soft-CL is designed to capture nuanced similarities, as evident in the ablation study (Fig.4), which shows the adaptability of our approach to handle more subtle variations.
\noindent \textbf{Comparison with Imagen Editor [Zr6o].}
% The purppose of the Imagen Editor is similar to that of DALL-E 2 which is text-guided image inpainting. }
% %We appreciate the reviewers for highlighting the missing references. We will add the missing works in the relevant section in the final version.
Imagen Editor~[31] is designed to fine-tune Imagen by Google Research on text-guided image inpainting, providing it with the capability of editing in a way similar to DALL-E. Hence, as another inpainting-based method, it shares similar limitations, requiring masks during inference and lacking fidelity to the masked content. Unfortunately, the unavailability of both code and a demo hinders a direct comparison. Also, EditBench, tailored for inpainting tasks, does not align with the specific requirements of evaluating our local editing task.
\noindent\textbf{Experiments and Evaluation Set [bsLE].} We acknowledge concerns about the evaluation set size. Fig.~\ref{fig:subset_quant} shows its impact on the metrics. Notably, with smaller sizes, there is fluctuation, but as the set surpasses 80 images, CS shows stability within the range of 77.83 and 78.19. This plateau indicates a level of stability in performance, a trend that becomes more pronounced after reaching 110. Similar stability is observed in SSIM-$\overline{M}$, achieved even more promptly (at 60).
% While we anticipate extending our test set for the final manuscript, these trends lead us to expect a similar outcome in both metrics.
Moreover, creating an evaluation set for localised image editing is a manual process taking $\sim20$ minutes per sample for generating/retrieving an image, writing meaningful target prompts, manually constructing masks (for calculating SSIM) and inference for all baselines. Also, we politely disagree that our experiments are relatively weak.
We extensively evaluated our method qualitatively and quantitatively using 3 metrics (CS, SSIM, FID) and added another one (CP) as well as a user study (Tab.~\ref{tab:user_study}).
% We acknowledge concerns about the evaluation set size. Creating an evaluation set for localised image editing is a manual process taking $\sim20$ minutes per sample for generating/retrieving an image, writing meaningful target prompts, manually constructing masks (for calculating SSIM) and inference for all baselines. Fig.~\ref{fig:subset_quant} shows the impact of the evaluation set size. Notably, with smaller sizes, there is fluctuation, but as the set surpasses 80 images, CS demonstrates stability within the range of 77.83 and 78.19. This plateau indicates a level of stability in performance, a trend that becomes more pronounced after reaching 110. Similar stability is observed in SSIM-$\overline{M}$, achieved even more promptly (at 60). While we anticipate extending our test set for the final manuscript, these trends lead us to expect a similar outcome in both metrics.
\begin{figure}
\centering
\includegraphics[trim=0.8cm 0.1cm 1cm 0.1cm, clip, width=\linewidth]{figs/rebuttal/subset_quant.pdf}
\vspace{-0.8cm}
\caption{Quantitative results on subsets of the evaluation set.}
\label{fig:subset_quant}
\vspace{-0.7cm}
\end{figure}
\noindent\textbf{Motivation for Self-Supervision [bsLE].}
% We appreciate the reviewer's feedback regarding the clarity of our motivation.
Traditional generic image manipulation methods often rely on training with unpaired data~[5,10,12,20] and use pre-trained text-to-image models [5,13,25](L41-44) due to the absence of large-scale datasets explicitly designed for the task, despite domain-specific datasets being available (L44-46). While IP2P is an exception, employing another model to generate synthetic target images, our innovative approach uses prompt augmentation to identify regions requiring manipulation, enhancing the model's contextual understanding of image manipulation. In the revised manuscript, we will elaborate on the advantages of our methodology, including its non-reliance on masks during inference and the challenges associated with acquiring large-scale paired datasets for this task.
While our method is inherently scalable for more extensive data usage, practical constraints such as limited computational resources have posed challenges in training on a scale comparable to baselines. Our method, due to resource constraints, is trained on 20K images and 60K augmented prompts, in contrast to IP2P, which leverages 450K images, and shows a competitive performance outperforming it on 3 (out of 5) metrics. Hence, we anticipate that using our method as a pre-training step will further boost it.
%Despite these limitations, we believe our approach demonstrates promise, and due to time limitation a more precise size comparison within our resource constraints can be included in the final version.
\noindent \textbf{Missing References [9Hzf].} Thank you for referring to missing references. We will add them in the final version.\\
%%%%%%%%% REFERENCES
% {
% \small
% \bibliographystyle{ieeenat_fullname}
% \bibliography{main}
% }
\end{document}
\begin{abstract}
Text-guided image editing finds applications in various creative and practical fields. While recent studies in image generation have advanced the field, they often struggle with the dual challenges of coherent image transformation and context preservation. In response, our work introduces prompt augmentation, a method amplifying a single input prompt into several target prompts, strengthening textual context and enabling localised image editing. Specifically, we use the augmented prompts to delineate the intended manipulation area. We propose a Contrastive Loss tailored to driving effective image editing by displacing edited areas and drawing preserved regions closer. Acknowledging the continuous nature of image manipulations, we further refine our approach by incorporating the similarity concept, creating a Soft Contrastive Loss. The new losses are incorporated to the diffusion model, demonstrating improved or competitive image editing results on public datasets and generated images over state-of-the-art approaches.
\end{abstract}
\vspace{-0.6cm}
\section{Introduction}
\label{sec:intro}
% taks decription and general challenges:
The field of text-guided image generation has made significant progress, particularly with the advent of diffusion models~\cite{rombach2022sd, singh2022highfidelity, ramesh2022dalle, kim2024arbitrary}, ushering in a new era of content creation. This progress extends to text-guided image manipulation, which offers a wide range of applications, spanning from artistic expression to the enhancement of image interpretability. Text-guided image manipulation involves altering an input image based on a user-provided textual prompt, such as changing the appearance or shape of objects, modifying the background, and adding, removing or replacing features. Some sample manipulations performed by our method can be seen in Figure~\ref{fig:motivation}. In this context, two intertwined challenges are present: the task of transforming image content in accordance with a provided textual description, and the need to preserve the salient aspects of the original visual information that remain contextually relevant. While recent methodologies \cite{meng2022sdedit, brooks2022instructpix2pix, hertz2022prompt} have excelled in the domain of image manipulation, they often encounter limitations in addressing both of these challenges concurrently.
\begin{figure}[t!]
\centering
\resizebox{0.82\linewidth}{!}{%
\begin{tabular}{ccc}
\includegraphics[width=3cm]{figs/motivation/00003.png} & \includegraphics[width=3cm]{figs/motivation/0008.png} &
\includegraphics[width=3cm]{figs/motivation/00053.png} \\
``a \textcolor{brown}{leopard}'' & ``a \textcolor{brown}{wedding} & ``a penguin \\
& \textcolor{brown}{ cake}'' & \textcolor{brown}{walking on the beach''}\\
\includegraphics[width=3cm]{figs/motivation/00344.png}
&\includegraphics[width=3cm]{figs/motivation/0008_gen.png}
&\includegraphics[width=3cm]{figs/motivation/00799_ours.png}\\
\end{tabular}%
}
\vspace{-0.2cm}
\caption{\textbf{Text-guided image manipulation.} Illustrative examples generated by our method (bottom row) with localised manipulations based on given text prompts and input images (top row).}
\vspace{-0.5cm}
\label{fig:motivation}
\end{figure}
% \begin{figure}
% \centering
% \begin{tikzpicture}
% \draw[black, fill=lightgray] (0, 0) rectangle (6, 4); % Adjust the coordinates and size as needed
% \end{tikzpicture}
% \caption{\textbf{Text-guided image manipulation.} Illustrative examples demonstrating the effectiveness of our method in generating images with precise, localised manipulations based on given text prompts.}
% \label{fig:placeholder}
% \end{figure}
%challenges with sota
Many pioneering text-guided image editing methods rely on text-guided image generation models~\cite{meng2022sdedit, hertz2022prompt, kawar2023imagic, diffedit, parmar2023zeroshoti2i, wu2023zeroshotediting}, which tend to be ill-equipped to address the nuanced challenges of image manipulation. Additionally, methods tailored specifically for image editing often require domain-specific training~\cite{bar2022text2live, kim2022diffusionclip}. These methods require training for each instance or translation domain, limiting their scalability across diverse tasks. Despite excelling within their trained domains, they often struggle with the broader generalisation needed for accommodating various editing scenarios. Although there exist generic methods, such as InstructPix2Pix~\cite{brooks2022instructpix2pix}, they often struggle to achieve both the faithful transformation of image content and the preservation of contextual details.
Some existing methods incorporate masks at inference time~\cite{diffedit, avrahami2021blendeddiff, ramesh2022dalle} or during training~\cite{wang2023imageneditor}. Although these methods show a good performance at content preservation, Dall-E 2~\cite{ramesh2022dalle} and Imagen Editor~\cite{wang2023imageneditor} fall short as they treat it as an inpainting task without taking into consideration the masked content and DiffEdit~\cite{diffedit} shows weaknesses as it relies on a pre-trained image generation model that is not specifically trained for manipulation purposes and is sensitive to the mask detected at inference time.
% prompt augmentaion and mask
Augmentation techniques have proven useful for enhancing image translation~\cite{cao2021remix, bodur2021densegeo, bodur2020hierarchical}, but their application in label space remains underexplored~\cite{bodur2020auglabel, lu2023specialistdiffusion, liu2023equitable, bhattarai2020inducing}.
Hence, to overcome the aforementioned challenges, this paper introduces the concept of prompt augmentation, which extends a single input prompt into a multitude of target prompts. This augmentation provides the model with a broader and more nuanced understanding of the image editing task at hand. Taking inspiration from DiffEdit~\cite{diffedit}, which uses user given source and target prompts to generate masks at inference time, we leverage augmented prompts to automatically derive an attention mask that serves as a guide for image editing at training time. This mask plays a pivotal role in distinguishing which regions of the input image should undergo alterations and which should remain unaltered, contributing to the task's overall precision in localising the desired edit.
% cl and soft-cl
In addition to facilitating attention map computation, augmented prompts empower us to establish relations between their respective manipulations. Specifically, areas falling into the obtained masked for each pair should exhibit dissimilarity, reflecting the diverse edits dictated by different prompts, while the unmasked regions should maintain similarity, preserving aspects unaffected by the prompt. With this aim we propose to incorporate a Contrastive Loss (CL) that serves this dual purpose, compelling the model to simultaneously displace edited areas while drawing closer the preserved regions.
%By doing so, it reinforces the idea that the edited content aligns with the target prompt, thus facilitating the desired visual transformation while preserving the areas unaffected by the prompt.
As manipulation is not a discreet task and the required amount of modification is influenced by the target prompt's relation to the source image, we propose to soften our contrastive loss by considering this relation. Building upon this foundation, we introduce a novel loss function, the Soft Contrastive Loss (Soft CL) to incorporate the concept of similarity between textual prompts. This approach results in a more dynamic and nuanced interaction between textual prompts and the image, contributing to a higher level of performance.
\begin{itemize}
\item We introduce prompt augmentation, expanding input prompts into multiple targets, in order to enhance contextual understanding for image editing, and compute a dynamic attention mask to guide editing localisation.
\item We introduce CL to encourage effective editing, pulling preserved regions closer while pushing edited areas to align with the target prompt. Our novel Soft CL incorporates similarity for dynamic prompt interaction, improving adaptability and performance.
\item We assess the performance of our method through a comparative analysis with state-of-the-art approaches, achieving competitive results without relying on masks during inference or utilising a paired dataset.
\end{itemize}
\vspace{-0.2cm}
\section{Related Work}
\label{sec:formatting}
\vspace{-0.2cm}
\noindent\textbf{Text-guided Image Manipulation.}
Text-guided image manipulation using diffusion models is an evolving field that harnesses diffusion models' progressive and attention-based architecture to facilitate fine-grained image adjustments. This research trajectory aims to enhance controllability and flexibility while maintaining the fidelity of the input image.
% CLIP:
DiffusionCLIP~\cite{kim2022diffusionclip} leverages CLIP-based losses to guide the diffusion process where they fine-tune a pretrained diffusion model for a specific domain.
% Specific domain or instance
Imagic~\cite{kawar2023imagic}, Text2Live~\cite{bar2022text2live} and Dreambooth~\cite{ruiz2022dreambooth} involve fine-tuning the entire model for each image, primarily generating variations for objects.
% cross-attention maps
Plug-and-Play~\cite{tumanyan2022plugandplay} explores the injection of spatial features and self-attention maps to maintain the overall structural integrity of the image.
Prompt-to-Prompt (P2P)~\cite{hertz2022prompt} dispenses with fine-tuning, instead retaining image structure by assigning cross-attention maps from the original image to the edited one based on corresponding text tokens.
Another study~\cite{epstein2023selfguidance} uses self-guidance by constraining attention maps or intermediate activations to control the sampling process. Similarly, MasaCtrl~\cite{cao2023masactrl} proposes to exploit innate features to use mutual attention during inference.
% target images
InstructPix2Pix~\cite{brooks2022instructpix2pix} utilises P2P to generate target images by accommodating human-like instructions for image editing.
% denoise-noise + masks
SDEdit~\cite{meng2022sdedit} adopts a two-step approach, where they first introduce noise into the input image and then employ the SDE prior for denoising, ultimately enhancing realism while aligning with user guidance.
BlendedDiffusion~\cite{avrahami2021blendeddiff} follows a similar approach but introduces manually created masks at inference time. Similarly, DiffEdit~\cite{diffedit} also uses masks at inference but automatically predicts them by using difference of latent noises obtained by input and target prompts provided by the user.
Imagen Editor~\cite{wang2023imageneditor} incorporates masks during training by using an object detector for randomly masking out objects to propose inpainting masks during training.
In contrast, our method integrates masks during training, specifically tailored to the target manipulation. We determine these masks through the generation of multiple augmentations of the source prompt enabling us to employ a strategy akin to~\cite{diffedit} to obtain an attention mask.\\
\noindent\textbf{Prompt generation and augmentation for image generation.}
InstructPix-to-Pix~\cite{brooks2022instructpix2pix} proposes to generate instructions for image editing by using a GPT model that they train on manually constructed pairs of captions and instruction.
Specialist Diffusion~\cite{lu2023specialistdiffusion} augments the prompts to define the same image with multiple captions that convey the \emph{same} meaning in order to improve the generalisation of the image generation network. They retrieve similar captions, replace words with synonyms or reflect the image augmentation, e.g. horizontal flip, in the text prompt.
Another work~\cite{lin2023regeneration} proposes to generate random sentences including source and target domain in order to calculate a mean difference that will serve as a direction while editing. iEdit~\cite{bodur2023iedit} generates target prompts by changing words in the input caption in order to retrieve pseudo-target images and guide the model.
In~\cite{liu2023equitable}, the authors propose to augment the prompts with cultural descriptions in order to reduce the culture bias in generative models.
Our approach uniquely integrates prompt augmentation during training, automatically generating multiple augmentations of the source prompt that define possible target prompts. We leverage these augmented prompts to determine masks tailored specifically to the target manipulation, and to enable self-supervised learning, which enhance the model's ability to learn editing tasks.\\
\noindent\textbf{Contrastive Loss for image generation.}
Being a powerful self-supersived learning method, contrastive learning ensures the consistency of image representations across various augmentations by comparing positive pairs against negative ones. While it has delivered remarkable outcomes in numerous fields, its application in image generation, and more specifically, image manipulation, remains relatively uncharted. In Cntr-GAN\cite{zhao2020cntrgan} the discriminator is trained with a loss to push different image representations apart, while drawing augmentations of the same image closer.
The first study to explore contrastive learning in the domain of image manipulation~\cite{park2020contrastive} proposes PatchNCE loss to minimise the distance between the feature representations of patches from a source image and corresponding patches from generated images while treating randomly sampled patches from other locations as negative samples. XMC-GAN~\cite{zhang2021cross} proposes to enforce text–image resemblance with a contrastive discriminator for text-to-image generation.\\
\noindent\textbf{Soft Contrastive Loss.} Most studies utilising contrastive learning rely on hard assignment of samples~\cite{duan2019deepeL, park2020contrastive,zhao2020cntrgan,zhang2021cross} while only a few explore soft assignment. In a study~\cite{kim2019deep} tested on image retrieving, the authors propose a log ratio loss to enforce the label and feature representation distances to be proportional.
Another study~\cite{thoma2020softcl} follows a similar approach for visual localisation, in which they aim to have the ordering of euclidean distances between features respect the ordering of geometric proximity measure between the corresponding images.
Given the continuous nature of textual prompts and image manipulation, in our work, we leverage the similarity between augmented prompts to guide the soft assignment of image representations.
% \vspace{-0.2cm}
\section{Methodology}
% \vspace{-0.2cm}
% \begin{figure*}
% \centering
% \begin{tikzpicture}
% \draw[black, fill=lightgray] (0, 0) rectangle (12, 4); % Adjust the coordinates and size as needed
% \end{tikzpicture}
% \caption{Overview of our method.}
% \label{fig:method_overview}
% \end{figure*}
\begin{figure*}
\centering
\includegraphics[width=0.87\linewidth]{figs/fig2_pipeline.pdf}
\caption{\textbf{Overview of our method.} (a) Prompt Augmentation: In order to augment the prompts to facilitate localised image editing we start by refining textual descriptions for source images using the BLIP captioning model~\cite{li2022blip}, resulting in cleaner captions suitable for further processing. Subsequently, we augment this input prompt by generating a range of target prompts using masked language modeling and exploiting word relations. (b) Soft Contrastive Loss (Soft-CL): The augmented prompts are instrumental in computing an attention mask based on the differences between the generated images. This attention mask is used to bring the inverse masked areas of the generated images closer while pushing away masked areas considering their similarity to the prompts. }
\label{fig:method_overview}
\end{figure*}
In this section, we present a comprehensive description of our proposed approach. We start by explaining the prompt augmentation process, followed by how these augmented prompts contribute to obtaining an attention mask and training the model using a contrastive loss. We then delve into the finer refinements of our approach, aiming to accommodate the continuous nature of textual prompts through the integration of Soft-CL and soft prompt augmentation. For a visual overview of our methodology, refer to Figure~\ref{fig:method_overview}.
\subsection{Prompt Augmentation}
\label{sec:prompt_augmentation}
We implement prompt augmentation as a critical component of our approach, with the primary goal of generating multiple target prompts to facilitate \emph{self-supervision} for localised image editing. We employ a publicly available large-scale dataset, i.e LAION-5B~\cite{schuhmann2022laionb}, that contains image-caption pairs. However, given the inherent noise in captions from web-derived datasets, manipulating these captions is challenging. To address this, we employ an image captioning model, i.e. BLIP~\cite{li2022blip}, to generate cleaner descriptions associated with the source images. This step is imperative as it enhances the clarity of captions, providing a structured foundation essential for subsequent modifications. For instance, the original caption for the LAION image shown in Figure~\ref{fig:method_overview} is \textit{`2004 Jaguar XJ8 Daphne AL'}, which is not descriptive of the image and cannot be easily manipulated to obtain target prompts. The caption generated with BLIP, \textit{`the front-end view of a black car'}, is cleaner, more descriptive and enables easier manipulation.
With the cleaner captions in hand, we proceed to augment the single input prompt by creating a range of target prompts. This process begins by masking a random noun or adjective within the input prompt. Leveraging the capabilities of a masked language model, specifically BERT~\cite{devlin2018bert}, we generate potential replacement words for the masked term. Additionally, we enrich our candidate word pool by incorporating semantically related words such as synonyms, antonyms and co-hyponyms, gathered through the NLTK\footnote{https://www.nltk.org/} library. From this pool, we randomly select a set of words to generate multiple variations of the input prompt. For instance, for the aforementioned sample in Figure~\ref{fig:method_overview}, some generated target prompts are: \textit{`the front-end view of a blue car'}, \textit{`the front-end view of a red car'}, \textit{`the front-end view of a white car'}. Please refer to the supplementary material for more samples. These augmented prompts not only diversify the translation capabilities of our network but also lay the groundwork for localised image manipulation by facilitating our self-supervised framework.
% \vspace{-0.1cm}
\subsection{On-The-Fly Mask Generation}
Diffusion models conditioned on text generate different noise estimates for each prompt. Drawing inspiration from the approach used in DiffEdit during inference, we exploit this characteristic to reveal the intended areas of manipulation within images during training. In our work, we use Latent Diffusion Model (LDM)~\cite{rombach2022sd}, which operates in the latent space to mitigate computational complexities associated with diffusion models. The loss for conditional LDM is given in Eqn.~\ref{eqn:ldm_simple_loss}, where $\epsilon_{\theta}$ represents the noise estimation network, which is a UNet~\cite{ronneberger2015unet}. $\tau_{\theta}$ is a domain-specific encoder projecting the conditioning target prompt to an intermediate representation, and the step $t$ is uniformly sampled from $\{1,2,...,T\}$. The network parameters $\theta$ are optimised to predict the noise ${\epsilon_1}{\sim}\mathcal{N}(0,1)$ that is used for corrupting the encoded version of the input image.
\vspace{-0.3cm}
\begin{equation}
{\mathcal{L}_{LDM}} = \mathbb{E}_{\mathcal{E}(x),y, \epsilon\sim\mathcal{N}(0,1),t}[||\epsilon - \epsilon_{\theta}(z_{t},t,\tau_{\theta}(y))||^2]
\label{eqn:ldm_simple_loss}
\end{equation}
By passing the augmented prompts through the LDM, the model outputs a set of estimated noises. We use the differences between these estimations to delineate the areas of image manipulation. Given the corrupted version of an encoded input image $z_t$ and $N$ augmented target prompts $y_1, y_2, \ldots, y_N$, the noise estimation network, $\epsilon_\theta$, outputs $N$ estimated noises during each iteration. We calculate the average of the differences of these estimation as shown in Eqn.~\ref{eqn:noise_difference} where \(N_p\) represents the number of unique pairs, and it is given by \(N_p = \frac{N \cdot (N-1)}{2}\).
\vspace{-0.3cm}
\begin{equation}
\Delta\epsilon = \frac{1}{N_p} \sum_{i,j} |\epsilon_{\theta}(z_{t},t,\tau_{\theta}(y_i)) - \epsilon_{\theta}(z_{t},t,\tau_{\theta}(y_j))|
\label{eqn:noise_difference}
\end{equation}
The outcome, $\Delta \epsilon$, provides valuable insights into the varying degrees of noise introduced by different prompts, signifying areas in the image that are most susceptible to modifications. Subsequently, these dissimilarity values are thresholded to establish a binary mask that distinctly highlights regions of alteration within the image. An example can be seen in Figure~\ref{fig:method_overview}, where the mask obtained for the car image outlines only areas that would be manipulated in case the colour of the car would change. In our particular implementation, we employ a threshold empirically set at $0.4$ of the range of absolute differences in noise estimates. The resulting binary mask, $M$, serves as a crucial component for defining the areas targeted for image manipulation within the generated images.
% \begin{equation}
% \Delta\epsilon = | \epsilon_{\theta}(z_{t},t,\tau_{\theta}(y_1)) - \epsilon_{\theta}(z_{t},t,\tau_{\theta}(y_2))|.
% \end{equation}
\subsection{Contrastive Loss (CL)}
% \vspace{-0.1cm}
Using the mask $M$ derived from augmented prompts, our goal is to guide the model to encourage the masked areas of the latent images, $z_1, z_2, \ldots, z_N$, generated with augmented prompts, $y_1, y_2, \ldots, y_N$, to be in alignment with its respective prompt, while preserving other relevant regions. To achieve this, we introduce a CL that promotes dissimilarity in the masked areas, while drawing the inverse masked areas $\overline{M}$ of the images closer to each other and the source image. The \emph{dissimilarity} within the masked regions is expressed in the first component of the CL as follows:
% By using the mask, $M$, generated through the augmented prompts, our aim is to ensure that the model concentrates on transforming the masked area of the image in accordance with the selected prompt while preserving other pertinent regions. In pursuit of this objective, we introduce a CL, which leverages this mask to encourage manipulations in the designated area by the masks while drawing the inverse masked areas, $\overline{M}$, of the images $I_1, I_2, \ldots, I_N$ generated using $y_1, y_2, \ldots, y_N$, closer to each other and the source image. Within the CL, the first component encourages \emph{dissimilarity} within the masked regions of the generated images. This part of the loss functions can be expressed as follows:
\vspace{-0.1cm}
\begin{equation}
L_{\text{d}} = \frac{1}{N_p} \sum_{i,j} \left(1 -\left| (M \odot z_i) - (M \odot z_j) \right|_2\right)
\label{eqn:cl_dissimilarity}
\end{equation}
where $\odot$ represents element-wise multiplication. This term promotes diversity within the areas of the images that should undergo edits, ensuring that the edited content aligns closely with the target prompts. The second component of CL is equally significant. It operates in parallel, aiming to minimise differences within unmasked regions of the images. These regions are critical for \emph{preserving} the parts of the image that should remain unaltered during the editing process. This part of the loss can be defined as:
\vspace{-0.1cm}
\begin{equation}
L_{\text{p}} = \frac{1}{N_p} \sum_{i,j} \left| (\overline{M} \odot z_i) - (\overline{M} \odot z_j) \right|_2
\end{equation}
The combined CL, unifying both the dissimilarity and preservation components, can be represented as follows:
\vspace{-0.15cm}
\begin{equation}
L_{\text{CL}} = L_{\text{p}} + \beta \cdot L_{\text{d}}
\label{eqn:objective_cl}
\end{equation}
The overall objective to fine-tune the LDM is:
\vspace{-0.15cm}
\begin{equation}
L_{\text{obj}} = L_{LDM} + \alpha \cdot L_{\text{CL}}
\label{eqn:objective}
\end{equation}
\subsection{Soft Contrastive Loss (Soft-CL)}
In image editing, textual prompts present a spectrum of demands that may vary from subtle modifications to profound transformations. For instance, let's consider the difference between transforming `a girl playing in a park' into `a girl playing in a garden' or `a girl playing in a sandbox'. The change from a park to a garden, for example, requires fewer modifications compared to the more extensive transformation into a sandbox.
To cater to this diversity in editing requirements, we introduce a refinement to our method in the form of Soft Contrastive Loss (Soft-CL). Soft-CL represents an enhancement over the conventional Contrastive Loss, infusing the loss function with the concept of similarity. To accommodate the varying editing demands, we adapt our loss function by revising Eqn.~\ref{eqn:cl_dissimilarity} as follows:
\vspace{-0.15cm}
\begin{equation}
L_{{d}}^{soft} = \frac{1}{N_p} \sum_{i,j} \left(1 -\left| (M_i \odot z_i) - (M_j \odot z_j)\right|_2^2 \cdot {\gamma}(y_i, y_j) \right)
\end{equation}
where $\gamma()$ refers to a similarity measure between the prompts, for which we have used the cosine distance between the CLIP embeddings of the prompts.
In accordance with the Soft-CL we have also enhanced our prompt augmentation strategy to mine prompts in a softer manner rather than selecting them randomly from the pool we construct. For soft prompt augmentation (Soft-PA), we choose the prompts based on differing similarity levels. As we have done in Soft-CL we specifically use the distance between their CLIP embeddings. This process results in a diverse set of prompts, further enhancing the Soft-CL, and consequently the model's ability to handle a wide range of textual inputs to produce contextually relevant image edits.
% \begin{equation}
% \begin{split}
% L = \sum_{i=1}^{N} \sum_{j=1}^{N, j \neq i}& \left(-\alpha \cdot \left| (M_i \odot I_i) - (M_j \odot I_j) \right|_2^2 \cdot \text{Sim}(i, j) \right.\ \cr
% & \left. + \left|(\mathbf{1} - M_i) \odot I_i - (\mathbf{1} - M_j) \odot I_j\right|_2^2\right)
% \end{split}
% \end{equation}
\vspace{-0.1cm}
\section{Experiments}
\begin{figure*}
\centering
\resizebox{0.87\textwidth}{!}{%
\begin{tabular}{ccccccc}
input & & SDEdit & DALL-E 2 & DiffEdit & InstructPix2Pix & Ours \\
\includegraphics[width=3cm]{figs/qual_res/input/0000.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{red}{strawberry} \\cake}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0000.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0000.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0000.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0000.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0000.png} \\
& & \includegraphics[width=3cm]{figs/qual_res/sdedit/0000_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0000_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0000_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0000_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0000_heatmap.png} \\
\includegraphics[width=3cm]{figs/qual_res/input/0059.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a yellow \cancel{flower} \textcolor{orange}{rose}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0059.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0059.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0059.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0059.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0059.png}\\
& & \includegraphics[width=3cm]{figs/qual_res/sdedit/0059_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0059_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0059_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0059_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0059_heatmap.png} \\
\includegraphics[width=3cm]{figs/qual_res/input/0065.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{brown}{victorian} lamp}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0065.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0065.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0065.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0065.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0065.png}\\ & & \includegraphics[width=3cm]{figs/qual_res/sdedit/0065_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0065_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0065_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0065_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0065_heatmap.png} \\
\includegraphics[width=3cm]{figs/qual_res/input/0066.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{green}{green and white check} pillow}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0066.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0066.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0066.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0066.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0066.png}\\ & & \includegraphics[width=3cm]{figs/qual_res/sdedit/0066_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0066_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0066_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0066_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0066_heatmap.png} \\
\includegraphics[width=3cm]{figs/qual_res/input/real/0002.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \cancel{dog} \textcolor{brown}{bear}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0002.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0002.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0002.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0002.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0002.png}\\ & & \includegraphics[width=3cm]{figs/qual_res/sdedit/0002_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0002_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0002_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0002_heatmap.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0002_heatmap.png} \\
% \includegraphics[width=3cm]{figs/qual_res/input/real/0013.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a bowl of \cancel{fruits} \textcolor{red}{apples}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0013.png}\\
\end{tabular}%
}
\caption{Qualitative comparison of our method against SDEdit~\cite{meng2022sdedit}, DALL-E 2~\cite{ramesh2022dalle}, DiffEdit~\cite{diffedit} and InstructPixtoPix~\cite{brooks2022instructpix2pix} using both generated and real images.}
\label{fig:qual_sota_comparison}
\end{figure*}
In this section, we analyse our method's performance through experiments, demonstrating its performance with qualitative and quantitative results, comparing to state-of-the-art methods. We also conduct an ablation study, a hyper-parameter study, and discuss limitations.
\vspace{-0.1cm}
\subsection{Experimental Setup}
\noindent\textbf{Datasets.} For training, we used Laion-5B images ~\cite{schuhmann2022laionb} with aesthetics scores greater than 7. For evaluation, we combined images generated using Stable Diffusion v1.4~\cite{rombach2022sd} with real images from COCO~\cite{lin2014coco} and ImageNet~\cite{deng2009imagenet}, totaling 135 images.
\noindent\textbf{Baselines.} Our method is compared against several state-of-the-art baselines, namely SDEdit~\cite{meng2022sdedit}, DiffEdit~\cite{diffedit}, DALL-E 2~\cite{ramesh2022dalle} and InstructPix2Pix~\cite{brooks2022instructpix2pix}. It should be noted that DALL-E 2 and DiffEdit use manually created or automatically generated masks at inference time, and InstructPix2Pix is trained on a paired dataset. Hence, these methods are not directly comparable to ours.
\noindent\textbf{Evaluation Metrics.} To assess the fidelity of our image translation, we utilise CLIPScore~\cite{hessel2021clipscore}, which measures the faithfulness of the image to the target prompt. Additionally, SSIM (Structural Similarity Index) is employed to gauge the faithfulness of the translated image to the input image. We also compute the FID~\cite{seitzer2020fid} in order to evaluate the quality of the generated samples. %In addition to these quantitative metrics, we present qualitative results.% to provide a comprehensive evaluation of our method.
\subsection{Qualitative Results}
\vspace{-0.1cm}
In Figure~\ref{fig:qual_sota_comparison}, we present a comparison of the results generated by our method against SDEdit~\cite{meng2022sdedit}, DALL-E 2~\cite{ramesh2022dalle}, DiffEdit~\cite{diffedit} and InstructPixtoPix~\cite{brooks2022instructpix2pix} using both generated and real images along wtih heatmaps showing the pixel differences between input an generated images.
We observe that SDEdit~\cite{meng2022sdedit} often encounters challenges in preserving background details, e.g.~\textit{``a victorian lamp''}. Furthermore, the editing performance of SDEdit may fall short, e.g.~\textit{``a strawberry cake''}. In some scenarios, it struggles with both aspects, as seen in \textit{``a green and white check pillow''}. These limitations can be attributed to the inherent trade-off within SDEdit, balancing between editing and preservation. Additionally, SDEdit is not specifically trained for editing purposes, which contributes to its challenges in maintaining both background integrity and effective editing.
\begin{table*}
\centering
\resizebox{0.76\linewidth}{!}{
\begin{tabular}{lccccc}
\toprule
\textbf{Method} & \textbf{CLIPScore $(\%) \uparrow$} & \textbf{FID $\downarrow$} & \textbf{SSIM-$\overline{M}(\%) \uparrow$} & \textbf{CLIP-R-Precision$(\%) \uparrow$} & \textbf{Human Study}\\
\midrule
SDEdit~\cite{meng2022sdedit} & 76.76 & 174 & 63.07 & 75.91 $\pm$ 1.5 & \textcolor{blue}{76.3}\% - \textcolor{red}{23.7}\% \\
DiffEdit~\cite{diffedit} & 72.77 & 85 & 85.14 & 67.60 $\pm$ 1.7 &\textcolor{blue}{73.84}\% - \textcolor{red}{26.16}\% \\
DALL-E 2~\cite{ramesh2022dalle} & 78.47 & 151 & 96.74 & 81.83 $\pm$ 1.4 & \textcolor{blue}{68.3}\% - \textcolor{red}{31.7}\%\\
InstructPix2Pix~\cite{brooks2022instructpix2pix} & 77.66 & 123 & 81.71 & 77.45 $\pm$ 1.6 & \textcolor{blue}{62.15}\% - \textcolor{red}{37.85}\% \\
Our Method & 78.19 & 133 & 70.39 & 80.69 $\pm$ 1.45 & - \\
\bottomrule
\end{tabular}}
\caption{Quantitative comparison of baselines and our method.
(↑ higher is better, ↓ lower is better).}
\label{tab:quantitative-comparison}
\vspace{-0.5cm}
\end{table*}
As an inpainting method requiring manual user-provided masks for the intended area of manipulation, DALL-E 2~\cite{ramesh2022dalle} falls short in faithfully reproducing the characteristics of the input image, despite its effective preservation of the background. Notably, in instances such as \textit{``a strawberry cake''}, the shape of the cake undergoes significant alterations, compromising the fidelity to the original. Similarly, the orientation of the flower in \textit{``a yellow rose''} is not adequately preserved. These observed shortcomings underscore the challenges inherent in maintaining fidelity to input details in the inpainting-based DALL-E 2 framework. DiffEdit~\cite{diffedit} employs masks derived from input and output prompts during inference. However, DiffEdit's performance proves to be sensitive to the precision of the obtained masks and, consequently, the prompts given by the user. While it yields commendable results when the mask is accurately detected, e.g. \textit{``a yellow rose''}, it faces challenges in other scenarios, such as \textit{``a strawberry cake''}, where the change predominantly occurs in the plates due to inferred mask inaccuracies. It should be noted that DiffEdit, designed primarily for inference time, can be seamlessly integrated into our method. InstructPix2Pix~\cite{brooks2022instructpix2pix} undergoes training with fairly aligned target images generated by P2P~\cite{hertz2022prompt}. Despite its ability to preserve the background and while achieving successful translations in many cases, it carries a notable caveat associated with the use of generated images. The outcomes may exhibit non-realistic qualities, e.g. \textit{``a bear''}, or manipulations can extend beyond the intended region, impacting the entire image, as evident in the check patterned background of \textit{``a green and white check pillow''}. This limitation stems from weaknesses inherited from P2P.%, emphasising the need for careful consideration of its impact on the overall realism of the generated results.
On the other hand, our method demonstrates proficient editing capabilities, preserving the background with minimal undesired changes.
\vspace{-0.1cm}
\subsection{Quantitative Results}
\vspace{-0.1cm}
In Table~\ref{tab:quantitative-comparison}, we present a quantitative comparison of our method with state-of-the-art approaches. The CLIPScore metric indicates the translation coherency regarding the target prompt, while SSIM-$\overline{M}$ scores offer a perspective on background preservation, providing a comprehensive assessment of diverse methodological approaches. SDEdit, serving as our baseline, achieves a CLIPScore of 76.76 and an SSIM-$\overline{M}$ score of 63.07, aligning with qualitative observations of challenges in preserving background details. DiffEdit, relying on automatically generated masks, obtains a CLIPScore of 72.77 and an SSIM-$\overline{M}$ score of 85.14, showing that despite proficiency in background preservation, it falls short in prompt-based translation. DALL-E 2, as an inpainting method with manual masks, demonstrated a CLIPScore of 78.47 and an outstanding SSIM-$\overline{M}$ score of 96.74, showcasing remarkable background preservation due to its inpainting nature. InstructPix2Pix, trained with fairly aligned target images, achieved a CLIPScore of 77.66 and an SSIM-$\overline{M}$ score of 81.71. While competitive, the SSIM-$\overline{M}$ score indicates some compromise in background preservation, aligning with qualitative analysis. Our method, with a CLIPScore of 78.19 and an SSIM-$\overline{M}$ score of 70.39, demonstrates effective editing with a moderate level of background preservation despite not being trained on a paired dataset or using masks at inference. \\
Examining the FID values in Table~\ref{tab:quantitative-comparison} provides valuable insights into the perceptual quality and realism of generated images across different methods. SDEdit reveals a higher FID, indicating a noticeable gap in quality when compared to source images. DiffEdit, relying on automatically generated masks, presents a lower FID, suggesting a closer resemblance to source images. This lower score may be attributed to its effective use of masks for coherent image generation. DALL-E 2 exhibits a moderate FID, which could be attributed to the challenges of seamlessly blending manipulated regions with the rest of the image, impacting the overall perceptual quality. InstructPix2Pix showcases a competitive FID, maintaining a balance between translation and background preservation. Our method maintains a moderate FID, indicating a balance between effective editing and maintaining a satisfactory level of image quality. \\
We further evaluated our method using CLIP-R-Precision score~\cite{park2021clipprecision} (R=1)(Tab.~\ref{tab:quantitative-comparison}), known for better alignment with human preferences compared to CLIPScore. We observe that it aligns more closely with our qualitative results as our method is outperformed only by DALL-E 2. We argue that, CLIP-Precision-Score still falls short in capturing the identified shortcomings of DALL-E, highlighted in our qualitative findings.\\
Following established protocols~\cite{bar2022text2live, kawar2023imagic}, we conduct a user study on Microworkers, where participants were presented pairs if images and selected the more successful manipulation based on faithfulness to the prompt and input image.
In Tab.~\ref{tab:quantitative-comparison}, \textcolor{blue}{blue} indicates preference for our method and \textcolor{red}{red} for the counterpart baseline. The results align more closely with our qualitative observations than the automatic metrics. DALL-E, despite achieving the highest ClipScore, is preferred less than our method and InstructPix2Pix. This is attributed to its low fidelity to masked content, emphasising nuances that automatic metrics might overlook. Similarly, IP2P's weaknesses, not directly measurable automatically, contribute to its lag behind our method. Further details of this study can be found in the supplementary.
\subsection{Ablation Study}
\vspace{-0.1cm}
\begin{figure}[!t]
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{cccccc}
{ Input} && { Baseline} & { $+$ CL} & { $+$ Soft-CL} & { $+$ Soft-PA} \\
\includegraphics[width=3cm]{figs/qual_res/input/0000.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{orange}\\ cake}}} & \includegraphics[width=3cm]{figs/ablation/b_0.png} & \includegraphics[width=3cm]{figs/ablation/d_0.png} & \includegraphics[width=3cm]{figs/ablation/f_0.png}& \includegraphics[width=3cm]{figs/ablation/g_0.png}
\\
\includegraphics[width=3cm]{figs/qual_res/input/0000.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{mandarin} \\cake}}} & \includegraphics[width=3cm]{figs/ablation/b_1.png} & \includegraphics[width=3cm]{figs/ablation/d_1.png} & \includegraphics[width=3cm]{figs/ablation/f_1.png} & \includegraphics[width=3cm]{figs/ablation/g_1.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/0000.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{brown}{chocolate} \\cake}}} & \includegraphics[width=3cm]{figs/ablation/b_2.png} & \includegraphics[width=3cm]{figs/ablation/d_2.png} & \includegraphics[width=3cm]{figs/ablation/f_2.png} & \includegraphics[width=3cm]{figs/ablation/g_2.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/0000.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{brown}{marble} \\cake}}} & \includegraphics[width=3cm]{figs/ablation/b_3.png} & \includegraphics[width=3cm]{figs/ablation/d_3.png} & \includegraphics[width=3cm]{figs/ablation/f_3.png} & \includegraphics[width=3cm]{figs/ablation/g_3.png}\\
\end{tabular}%
}
\caption{Qualitative comparison of ablation study results.}
\label{fig:qualitative_ablation}
\end{figure}
Table~\ref{tab:quantitative_ablation} provides insights from our ablation study, investigating the influence of key components in our proposed method. The absence of CL, Soft-CL, Soft-PA is indicated by \xmark{} in the respective columns. Remarkably, the introduction of contrastive loss alone leads to a noticeable enhancement in all metrics, indicating improved translation coherency and content preservation compared to the baseline, which is equivalent to stable diffusion-based SDEdit.While the introduction of soft contrastive loss also shows a rise in all metrics, it becomes superior to CL only with the introduction of soft prompt augmentation, which refines the method's ability to balance translation coherence and background preservation. In summary, our ablation study underscores the pivotal role of contrastive loss in enhancing translation quality. The incorporation of soft contrastive loss, coupled with soft prompt augmentation, contributes to substantial improvements in overall performance.\\
\begin{table}
\centering
\resizebox{0.85\linewidth}{!}{
\begin{tabular}{lcccccc}
\toprule
\textbf{CL} & \textbf{Soft-CL} & \textbf{Soft-PA} & \textbf{CLIPScore $(\%) \uparrow$} & \textbf{FID $\downarrow$} & \textbf{SSIM-$\overline{M}(\%) \uparrow$}\\
\midrule
\xmark & \xmark & \xmark & 76.76 & 174 & 63.07 \\
\cmark & \xmark & \xmark & 77.94 & 159 & 68.37 \\
\xmark & \cmark & \xmark & 77.53 & 140 & 67.21 \\
\xmark & \cmark & \cmark &78.19 & 133 & 70.39 \\
\bottomrule
\end{tabular}}
\caption{Ablation study results for the proposed method components. CL denotes contrastive loss, Soft-CL is soft contrastive loss, and Soft-PA is soft prompt augmentation.}
\vspace{-0.3 cm}
\label{tab:quantitative_ablation}
\vspace{-0.2cm}
\end{table}
In Figure~\ref{fig:qualitative_ablation}, the qualitative ablation study unveils the nuanced impact of each component in refining the translation. The samples, generating different variations of a cake: ``an orange cake'', ``a mandarin cake'', ``a chocolate cake'' and ``a marble cake'' illustrate the model's behavior in the absence of each component. The baseline model produces results with some difference between "orange" and "mandarin," despite their similar semantics. Additionally, it struggles to synthesise a ``chocolate cake'' and ``a marble cake''. The introduction of hard-assigned CL improves generation outputs but does not establish a clear relation between the prompts. With the incorporation of Soft-CL the model becomes adept at discerning fine-grained differences. In this context, ``an orange cake'' and ``a mandarin cake'' demonstrate the model's ability to generate semantically related descriptions, reflecting heightened sensitivity to shared characteristics between these citrus fruits. Soft-PA amplifies these effects by introducing diversity in the input prompts. In the case of ``a chocolate cake'' Soft-PA encourages the model to explore a broader spectrum of possibilities, resulting in varied and contextually rich outputs while showing a clear similarity between `orange' and `mandarin' results. The qualitative examples underscore the role of Soft-CL in enhancing the model's capacity to capture semantic similarities, fostering more coherent and contextually relevant image manipulations.
The interplay between Soft-CL and Soft-PA contributes to the generation of diverse and nuanced outputs, showcasing the effectiveness of these components in refining the editing capabilities of our method.
\subsection{Hyper-parameter Study}
\begin{table}
\centering
\resizebox{0.8\linewidth}{!}{
\begin{tabular}{cccccccc}
\toprule
\textbf{Weight ($\alpha$)} & \textbf{CLIPScore $(\%) \uparrow$} & \textbf{FID $\downarrow$} & \textbf{SSIM-$\overline{M}(\%) \uparrow$}\\
\midrule
0.25 & 76.11 & 148 & 70.75 \\
0.5 & 76.54 & 146 & 70.3 \\
1.0 & 78.20 & 133 & 70.39\\
2.0 & 72.04 & 121 & 73.22\\
\bottomrule
\end{tabular}}
\caption{Impact of different weight values for $\alpha$ in Eqn.~\ref{eqn:objective} on model performance metrics.}
\label{tab:hyperparam-results}
\vspace{-0.6cm}
\end{table}
In Table~\ref{tab:hyperparam-results}, the impact of the weight hyperparameter ($\alpha$ in Eqn.~\ref{eqn:objective}) on our model's performance metrics is shown. Increasing $\alpha$ correlates positively with CLIPScore, indicating improved translation coherency. However, a sharp decrease in CLIPScore is observed with a high value of $\alpha=2.0$. Concurrently, the FID metric decreases with higher $\alpha$ values, reflecting enhanced image quality. The SSIM-$\overline{M}$ score, emphasising fidelity to input images, peaks at $\alpha=2.0$. Our analysis suggests that $\alpha=1.0$ strikes a favorable equilibrium, yielding strong performance across CLIPScore, FID, and SSIM-$\overline{M}$, making it a judicious choice for our method.
\section{Conclusion}
\vspace{-0.1cm}
We introduced a novel method using prompt augmentation for dynamic mask generation and self-supervised learning in image manipulation. Our soft-contrastive loss achieves effective translations in delineated areas while preserving the rest with minimal undesired changes. Addressing challenges in existing methods, our approach provides a promising solution for localised image manipulations. Experimental evaluations, including comparisons with state-of-the-art techniques and ablation studies, were conducted both qualitatively and quantitatively. Our proposed image editing method stands as a significant contribution, paving the way for prompt augmentation and striking a nuanced balance between successful translations and background preservation.\\
\textbf{Acknowledgements.} R. Bodur is funded by the Turkish Ministry of National Education. This work was in part sponsored by NST grant (CRC 21011, MSIT), KOCCA grant (R2022020028, MCST), IITP grant (RS-2023-00228996, MSIT). BB is supported by NCS University of Aberdeen Startup grant.
% \section{Possible Improvements}
% \subsection{Prompt Augmentation}
% \begin{itemize}
% \item \textbf{Semantic Similarity}: Currently, prompts are generated by masking a random noun or adjective in the input and replacing it with a related word or a top token from BERT's predictions. To improve this, we can consider incorporating semantic similarity measures between prompts. We can incorporate semantic similarity measures between prompts to specifically serve for the soft CL proposed.
% \item \textbf{Learning the prompt generation process}: We can explore reinforcement learning or other algorithms to fine-tune prompts for generating more relevant textual descriptions for image editing for more coherent and task-specific prompt generation.
% \end{itemize}
% \subsection{Soft Contrastive Loss (Soft CL)}
% \begin{itemize}
% \item \textbf{Loss Function Variations}: Experiment with different loss functions.
% \subitem \textbf{Multimodal Embeddings}: Utilise multimodal embeddings, like CLIP, to capture richer relationships between images and prompts. A CLIP-based CL.
% \item \textbf{Semantic Segmentation-Guided Loss}: Incorporate semantic segmentation maps to guide the loss functions. We can ensure that the model's attention aligns with the intended areas of manipulation.
% \subitem \textbf{Cross-attention maps:} Using the cross attention maps from the UNet
% \end{itemize}
\clearpage
\setcounter{page}{1}
\setcounter{figure}{0}
\renewcommand{\thefigure}{S\arabic{figure}}%
\setcounter{table}{0}
\renewcommand{\thetable}{S\arabic{table}}%
\setcounter{section}{0}
\renewcommand{\thesection}{\Alph{section}}
\maketitlesupplementary
In this supplementary material, we delve deeper into the capabilities of our proposed method for text-guided image manipulation. Included are implementation details, limitations, illustrative samples of prompt augmentations, insights into the mask generation process, and supplementary qualitative comparisons that further enrich the findings presented in the main paper.
\section{Implementation Details} We employed a pretrained LDM, specifically Stable Diffusion v1.4, as the foundation of our experiments. Fine-tuning the UNet was conducted for 20,000 steps, utilising a learning rate of 1.0e-05. The experiments were executed on an NVIDIA Titan RTX with 24GB of GPU memory, employing a batch size of 1. The image size used was 384, and we augmented each image with 3 target prompts as decsribed in Sec.~\ref{sec:prompt_augmentation}.
\section{Limitations}
While our work demonstrates notable achievements in image manipulation, it is crucial to acknowledge its limitations.
Prompt augmentation, while beneficial for introducing diversity during training, presents challenges. Overreliance on augmented prompts may carry the risk of bias, potentially hindering the model's generalisation to new prompts. Our current strategy, focusing on the manipulation of adjectives and nouns, represents a subset of the infinite possibilities for image edits. Hence, the augmentation process remains suboptimal, emphasising the need for more sophisticated techniques, potentially involving the training of a dedicated large language model for this purpose. Additionally, the resource limitations with a batch size of 1 limits the potential of our approach.
\section{Prompt Augmentation and Mask Generation}
\begin{table}[t]
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{ccc}
\toprule
\textbf{Input} & \textbf{Augmented Prompts} & \textbf{Generated Mask} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/6_org.jpg} \\
\emph{Two wolves in the snow}
\end{tabular}
&
\begin{tabular}{c}
Two \textit{werewolves} in the snow\\
Two \textit{cats} in the snow\\
Two \textit{trees} in the snow\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/6_mask.png}
\\
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/212_org.png} \\
\emph{A woman wearing a pearl necklace}
\end{tabular}
&
\begin{tabular}{c}
A woman wearing a \textit{silver} necklace\\
A woman wearing a \textit{sapphire} necklace\\
A woman wearing a \textit{charm} necklace\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/212_mask.png}
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/134_org.jpg} \\
\emph{A woman wearing a silver headpiece}
\end{tabular}
&
\begin{tabular}{c}
A woman wearing a \textit{silvery} headpiece\\
A woman wearing a \textit{sterling} headpiece\\
A woman wearing a \textit{jade} headpiece\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/134_mask.png}
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/1_org.png} \\
\emph{A blue dress}
\end{tabular}
&
\begin{tabular}{c}
A \textit{black} dress\\
A \textit{white} dress\\
A \textit{red} dress\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/1_mask.png}
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/865_org.jpg} \\
\emph{A woman with blue hair}
\end{tabular}
&
\begin{tabular}{c}
A woman with \textit{black} hair\\
A woman with \textit{green} hair\\
A woman with \textit{red} hair\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/865_mask.png}
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/17_org.jpg} \\
\emph{An owl in the snow}
\end{tabular}
&
\begin{tabular}{c}
An \textit{eagle} in the snow\\
A \textit{goose} in the snow\\
An \textit{elf} in the snow\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/17_mask.png}
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/85_org.jpg} \\
\emph{A group of men standing in front of an airplane}
\end{tabular}
&
\begin{tabular}{c}
A group of \textit{gentlemen} standing in front of an airplane\\
A group of \textit{monks} standing in front of an airplane\\
A group of \textit{pirates} standing in front of an airplane\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/85_mask.png}
\end{tabular} \\
\midrule
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/154_org.png} \\
\emph{A woman wearing a hat}
\end{tabular}
&
\begin{tabular}{c}
A woman wearing a \textit{cap}\\
A woman wearing a \textit{hood}\\
A woman wearing a \textit{cloak}\\
\end{tabular}
&
\begin{tabular}{c}
\includegraphics[width=3cm]{figs/supp_augmentation/154_mask.png}
\end{tabular} \\
\bottomrule
\end{tabular}}
\caption{Samples for prompt augmentation and the masks generated by using these prompts.}
\label{tab:yourlabel}
\end{table}
In Figure \ref{tab:yourlabel}, we present a set of samples illustrating the augmentation process and mask generation. The "Input" column features images dynamically captioned by using BLIP~\cite{li2022blip}, offering diverse scenes.
The ``Augmented Prompts" column displays variations for each input, providing insights into augmentation diversity. For example, for an image with wolves, augmented prompts include ``Two werewolves in the snow," ``Two cats in the snow," and ``Two trees in the snow." These variations serve the purpose of the proposed soft contrastive loss.
In the ``Generated Mask" column, masks are featured, generated by leveraging the augmented prompts. The alignment of images and masks in the table provides a clear view of the augmentation and mask generation results.
\section{Additional Qualitative Comparisons}
\begin{figure*}
\centering
\resizebox{0.9\textwidth}{!}{%
\begin{tabular}{ccccccc}
input & & SDEdit & DALL-E 2 & DiffEdit & InstructPix2Pix & Ours \\
\includegraphics[width=3cm]{figs/qual_res/input/0071.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a penguin walking on the \cancel{ice} \textcolor{red}{beach}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0071.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0071.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0071.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0071.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0071.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/0072.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \cancel{cat}\textcolor{red}{leopard}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0072.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0072.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0072.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0072.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0072.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/0023.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{red}{boat} in the sea}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0023.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0023.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0023.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0023.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0023.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/real/0023.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{red}{panda}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0023.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0023.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0023.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0023.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0023_real.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/0006.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{red}{pink} car}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0006.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0006.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0006.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0006.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0006.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/0070.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{red}{parrot}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/0070.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/0070.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/0070.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/0070.png} & \includegraphics[width=3cm]{figs/qual_res/ours/0070.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/real/0013.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a bowl of \cancel{fruits} \textcolor{red}{apples}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0013.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0013.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/real/0027.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering an \textcolor{red}{autumn} tree}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0027.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0027.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0027.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0027.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0027_real.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/real/0043.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{red}{dog}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0043.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0043.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0043.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0043.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0043_real.png}\\
\includegraphics[width=3cm]{figs/qual_res/input/real/0046.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering \scriptsize{a doughnut \textcolor{red}{with\\ white chocolate and raspberry sauce}}}}} & \includegraphics[width=3cm]{figs/qual_res/sdedit/real/0046.png} & \includegraphics[width=3cm]{figs/qual_res/dalle/real/0046.png} & \includegraphics[width=3cm]{figs/qual_res/diffedit/real/0046.png} & \includegraphics[width=3cm]{figs/qual_res/ip2p/real/0046.png} & \includegraphics[width=3cm]{figs/qual_res/ours/real/0046_real.png}\\
\end{tabular}%
}
\caption{Additional qualitative results comparing our method against SDEdit~\cite{meng2022sdedit}, DALL-E 2~\cite{ramesh2022dalle}, DiffEdit~\cite{diffedit} and InstructPixtoPix~\cite{brooks2022instructpix2pix} using both generated and real images.}
\label{fig:supp_qual_sota_comparison}
\end{figure*}
Figure~\ref{fig:supp_qual_sota_comparison} showcases additional qualitative comparisons between our method and prominent approaches, including SDEdit~\cite{meng2022sdedit}, DALL-E 2~\cite{ramesh2022dalle}, DiffEdit~\cite{diffedit}, and InstructPix2Pix~\cite{brooks2022instructpix2pix}. Our observations echo those discussed in the main paper. Notably, SDEdit~\cite{meng2022sdedit} grapples with the challenge of maintaining a balance between fidelity to the input image and adherence to the target prompt, as evident in examples like `a boat in the sea', `a bowl of apples', and `a pink car'. DALL-E 2~\cite{ramesh2022dalle}, functioning as an inpainting method reliant on masks, tends to discard content within the masked area while preserving the inverse mask area. This often leads to mismatches between input and edited content, as seen in the positions of `a leopard', `a panda', `an autumn tree', and the quantity of `apples'. Additionally, it encounters difficulty in seamlessly blending with the rest of the image, exemplified by the case of pink car'. DiffEdit's sensitivity to detected masks, dependent on parameters and the employed image generation model, results in occasional failures in accurate mask detection (a leopard', a panda', a bowl of apples') or the selection of completely incorrect areas (`doughnut', `autumn tree'). InstructPix2Pix may generate non-realistic images (`a panda', `a leopard'), face translation issues (e.g., `teddy bear translated into a dog'), impact the entire image with the target prompt (`a parrot', `autumn tree', `a pink car'), or struggle with multiple simultaneous changes (e.g., `doughnut with raspberry and white chocolate sauce'). Despite not using masks at inference like DALL-E 2 and DiffEdit and not being trained on a dataset with target images like InstructPix2Pix, our method achieves more successful translations with high fidelity to the target prompt and input image, accompanied by minimal undesired changes.
\section{User Study}
As described in the main paper, we executed a user study adhering to established procedures on Microworkers. A total of 540 pairs were presented to users, each consisting of input images, target prompts, and manipulation outcomes generated by both our proposed method and a baseline.
Each pair underwent evaluation by 10 distinct participants, tasked with identifying the more effective manipulation in terms of fidelity to the prompt and input image. This process yielded a comprehensive dataset of 5.4K responses. The outcomes of this survey are detailed in Table~\ref{tab:quantitative-comparison} of the main paper. The results consistently demonstrate a preference for our proposed method over the baseline.
\section{Integrating DiffEdit into Our Framework}
In Figure~\ref{fig:diffours}, we provide additional qualitative comparisons between our method and DiffEdit~\cite{diffedit}. The first column shows the input image, and the second and third columns display DiffEdit and our results, respectively. As DiffEdit is exclusively utilised at inference time, we seamlessly incorporate it into our method, presenting the combined outcomes in the last column. Identical parameters are employed for computations in both DiffEdit and DiffEdit$+$Ours. DiffEdit utilises Stable Diffusion's checkpoint v1.4, the same pretrained model used for fine-tuning with our approach.
\begin{figure}[!t]
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{ccccc}
{ Input} && { DiffEdit} & {Ours} & {DiffEdit $+$ Ours} \\
\includegraphics[width=3cm]{figs/supp_diffedit_ours/0020.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a girl \textcolor{orange}{wearing\\ a hat}}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0020_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0020_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0020_diffours.png} \\
\includegraphics[width=3cm]{figs/supp_diffedit_ours/0002.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{three-tier}\\ cake}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0002_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0002_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0002_diffours.png}
\\
\includegraphics[width=3cm]{figs/supp_diffedit_ours/0002.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{strawberry}\\ cake}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0000_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0000_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0000_diffours.png}
\\
% \includegraphics[width=3cm]{figs/supp_diffedit_ours/0021.png}& \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{kite in the} \\sky}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0021_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0021_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0021_diffours.png} \\
\includegraphics[width=3cm]{figs/supp_diffedit_ours/0024.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{moon in the} sky}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0024_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0024_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0024_diffours.png} \\
\includegraphics[width=3cm]{figs/supp_diffedit_ours/0019.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering \textcolor{orange}{lavenders} \cancel{tulips} in \\ a \textcolor{orange}{glass} vase}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0019_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0019_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0019_diffours.png} \\
\includegraphics[width=3cm]{figs/supp_diffedit_ours/0001.png} & \raisebox{0.5cm}{\rotatebox{90}{\parbox{2cm}{\centering a \textcolor{orange}{yorkshire terrier}}}} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0001_diffedit.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0001_ours.png} & \includegraphics[width=3cm]{figs/supp_diffedit_ours/0001_diffours.png} \\
\end{tabular}%
}
\caption{Qualitative comparison of DiffEdit, our method and their combination.}
\label{fig:diffours}
\end{figure}
When using DiffEdit in conjunction with our method, we observe improved mask detection. For example, in `a girl wearing a hat', DiffEdit alters the girl's face while adding a hat. Upon integration into our method at inference, the face remains unaltered while incorporating the hat. Our method alone achieves a successful translation with minimal undesired modifications in the face area, and this is further mitigated with the integration of DiffEdit. A similar scenario is evident in "a moon in the sky," where DiffEdit manipulates an unrelated area on the shores. Integrated into our method, it produces a successful translation while preserving unrelated areas effectively. Once again, our method alone, while placing the moon in the desired location, introduces some undesired alterations, which are minimized through the integration of DiffEdit. In `a strawberry cake', DiffEdit inaccurately manipulates the plates rather than the cake. While our method achieves a successful translation with minimal undesired changes, integration with DiffEdit further prevents these changes, preserving all parts of the image perfectly. These results highlight that our fine-tuning strategy enhances the model's ability to detect editing masks compared to the baseline.
Consistent with our earlier observations, while our method enhances the accuracy of mask detection in DiffEdit, it remains sensitive to the detected masks and corresponding parameters. An example is seen in `lavenders', where integrating DiffEdit into our framework leads to undesired manipulations in the background and fails to successfully translate the vase into a glass one although it preserves some details better. Despite DiffEdit detecting a relatively accurate mask in this case, it struggles to translate tulips into lavenders successfully. In summary, while integrating DiffEdit often improves our method, particularly in preserving the background, its performance is still influenced by parameter sensitivity in mask detection. Further refinement of these parameters could enhance the overall results.
\begin{thebibliography}{43}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Avrahami et~al.(2022)Avrahami, Lischinski, and Fried]{avrahami2021blendeddiff}
Omri Avrahami, Dani Lischinski, and Ohad Fried.
\newblock Blended diffusion for text-driven editing of natural images.
\newblock \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 18208--18218, 2022.
\bibitem[Bar-Tal et~al.(2022)Bar-Tal, Ofri-Amar, Fridman, Kasten, and Dekel]{bar2022text2live}
Omer Bar-Tal, Dolev Ofri-Amar, Rafail Fridman, Yoni Kasten, and Tali Dekel.
\newblock Text2live: Text-driven layered image and video editing.
\newblock In \emph{Proceedings of the European Conference on Computer Vision (ECCV)}, pages 707--723. Springer, 2022.
\bibitem[Bhattarai and Kim(2020)]{bhattarai2020inducing}
Binod Bhattarai and Tae-Kyun Kim.
\newblock Inducing optimal attribute representations for conditional gans.
\newblock In \emph{Proceedings of the European Conference on Computer Vision (ECCV)}, 2020.
\bibitem[Bhattarai et~al.(2020)Bhattarai, Bodur, and Kim]{bodur2020auglabel}
Binod Bhattarai, Rumeysa Bodur, and Tae-Kyun Kim.
\newblock Auglabel: Exploiting word representations to augment labels for face attribute classification.
\newblock \emph{International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, pages 2308--2312, 2020.
\bibitem[Bodur et~al.(2021)Bodur, Bhattarai, and Kim]{bodur2021densegeo}
Rumeysa Bodur, Binod Bhattarai, and Tae-Kyun Kim.
\newblock 3d dense geometry-guided facial expression synthesis by adversarial learning.
\newblock In \emph{Proceedings of the Winter Conference on Applicationn of Computer Vision (WACV)}, 2021.
\bibitem[Bodur et~al.(2023{\natexlab{a}})Bodur, Bhattarai, and Kim]{bodur2020hierarchical}
Rumeysa Bodur, Binod Bhattarai, and Tae-Kyun Kim.
\newblock Joint training of hierarchical gans and semantic segmentation for expression translation.
\newblock \emph{International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, pages 2308--2312, 2023{\natexlab{a}}.
\bibitem[Bodur et~al.(2023{\natexlab{b}})Bodur, Gundogdu, Bhattarai, Kim, Donoser, and Bazzani]{bodur2023iedit}
Rumeysa Bodur, Erhan Gundogdu, Binod Bhattarai, Tae-Kyun Kim, Michael Donoser, and Loris Bazzani.
\newblock iedit: Localised text-guided image editing with weak supervision.
\newblock \emph{arXiv preprint arXiv:2305.05947}, 2023{\natexlab{b}}.
\bibitem[Brooks et~al.(2023)Brooks, Holynski, and Efros]{brooks2022instructpix2pix}
Tim Brooks, Aleksander Holynski, and Alexei~A. Efros.
\newblock Instructpix2pix: Learning to follow image editing instructions.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Cao et~al.(2021)Cao, Hou, Yang, He, and Sun]{cao2021remix}
Jie Cao, Luanxuan Hou, Ming-Hsuan Yang, Ran He, and Zhenan Sun.
\newblock Remix: Towards image-to-image translation with limited data.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2021.
\bibitem[Cao et~al.(2023)Cao, Wang, Qi, Shan, Qie, and Zheng]{cao2023masactrl}
Mingdeng Cao, Xintao Wang, Zhongang Qi, Ying Shan, Xiaohu Qie, and Yinqiang Zheng.
\newblock Masactrl: Tuning-free mutual self-attention control for consistent image synthesis and editing.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, pages 22560--22570, 2023.
\bibitem[Couairon et~al.(2023)Couairon, Verbeek, Schwenk, and Cord]{diffedit}
Guillaume Couairon, Jakob Verbeek, Holger Schwenk, and Matthieu Cord.
\newblock {DiffEdit: Diffusion-based Semantic Image Editing with Mask Guidance}.
\newblock In \emph{International Conference on Learning Representations (ICLR)}, 2023.
\bibitem[Deng et~al.(2009)Deng, Dong, Socher, Li, Li, and Fei-Fei]{deng2009imagenet}
Jia Deng, Wei Dong, Richard Socher, Li-Jia Li, Kai Li, and Li Fei-Fei.
\newblock Imagenet: A large-scale hierarchical image database.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2009.
\bibitem[Devlin et~al.(2019)Devlin, Chang, Lee, and Toutanova]{devlin2018bert}
Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova.
\newblock Bert: Pre-training of deep bidirectional transformers for language understanding.
\newblock In \emph{Proceedings of the 2019 Conference of the North}. Association for Computational Linguistics, 2019.
\bibitem[Duan et~al.(2019)Duan, Chen, Lu, and Zhou]{duan2019deepeL}
Yueqi Duan, Lei Chen, Jiwen Lu, and Jie Zhou.
\newblock Deep embedding learning with discriminative sampling policy.
\newblock \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 4959--4968, 2019.
\bibitem[Epstein et~al.(2023)Epstein, Jabri, Poole, Efros, and Holynski]{epstein2023selfguidance}
Dave Epstein, Allan Jabri, Ben Poole, Alexei Efros, and Aleksander Holynski.
\newblock Diffusion self-guidance for controllable image generation.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, pages 16222--16239. Curran Associates, Inc., 2023.
\bibitem[Hertz et~al.(2022)Hertz, Mokady, Tenenbaum, Aberman, Pritch, and Cohen-Or]{hertz2022prompt}
Amir Hertz, Ron Mokady, Jay Tenenbaum, Kfir Aberman, Yael Pritch, and Daniel Cohen-Or.
\newblock Prompt-to-prompt image editing with cross attention control.
\newblock In \emph{International Conference on Learning Representations (ICLR)}, 2022.
\bibitem[Hessel et~al.(2021)Hessel, Holtzman, Forbes, Bras, and Choi]{hessel2021clipscore}
Jack Hessel, Ari Holtzman, Maxwell Forbes, Ronan~Le Bras, and Yejin Choi.
\newblock Clipscore: A reference-free evaluation metric for image captioning.
\newblock In \emph{EMNLP}, 2021.
\bibitem[Kawar et~al.(2023)Kawar, Zada, Lang, Tov, Chang, Dekel, Mosseri, and Irani]{kawar2023imagic}
Bahjat Kawar, Shiran Zada, Oran Lang, Omer Tov, Huiwen Chang, Tali Dekel, Inbar Mosseri, and Michal Irani.
\newblock Imagic: Text-based real image editing with diffusion models.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Kim et~al.(2022)Kim, Kwon, and Ye]{kim2022diffusionclip}
Gwanghyun Kim, Taesung Kwon, and Jong~Chul Ye.
\newblock Diffusionclip: Text-guided diffusion models for robust image manipulation.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 2426--2435, 2022.
\bibitem[Kim and Kim(2024)]{kim2024arbitrary}
Jinseok Kim and Tae-Kyun Kim.
\newblock Arbitrary-scale image generation and upsampling using latent diffusion model and implicit neural decoder.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2024.
\bibitem[Kim et~al.(2019)Kim, Seo, Laptev, Cho, and Kwak]{kim2019deep}
Sungyeon Kim, Minkyo Seo, Ivan Laptev, Minsu Cho, and Suha Kwak.
\newblock Deep metric learning beyond binary supervision.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 2288--2297, 2019.
\bibitem[Li et~al.(2022)Li, Li, Xiong, and Hoi]{li2022blip}
Junnan Li, Dongxu Li, Caiming Xiong, and Steven Hoi.
\newblock Blip: Bootstrapping language-image pre-training for unified vision-language understanding and generation.
\newblock In \emph{International Conference on Machine Learning (ICML)}, 2022.
\bibitem[Lin et~al.(2014)Lin, Maire, Belongie, Hays, Perona, Ramanan, Doll{\'a}r, and Zitnick]{lin2014coco}
Tsung-Yi Lin, Michael Maire, Serge Belongie, James Hays, Pietro Perona, Deva Ramanan, Piotr Doll{\'a}r, and C~Lawrence Zitnick.
\newblock Microsoft coco: Common objects in context.
\newblock In \emph{Proceedings of the European Conference on Computer Vision (ECCV)}. Springer, 2014.
\bibitem[Lin et~al.()Lin, Zhang, Yang, Wang, and Shi]{lin2023regeneration}
Yupei Lin, Sen Zhang, Xiaojun Yang, Xiao Wang, and Yukai Shi.
\newblock Regeneration learning of diffusion models with rich prompts for zero-shot image translation.
\newblock arXiv preprint arXiv:2305.04651, 2023.
\bibitem[Liu et~al.()Liu, Shin, Okogwu, Yun, Coleman, Schaldenbrand, Kim, and Oh]{liu2023equitable}
Zhixuan Liu, Youeun Shin, Beverley-Claire Okogwu, Youngsik Yun, Lia Coleman, Peter Schaldenbrand, Jihie Kim, and Jean Oh.
\newblock Towards equitable representation in text-to-image synthesis models with the cross-cultural understanding benchmark (ccub) dataset.
\newblock arXiv preprint arXiv:2301.12073, 2023.
\bibitem[Lu et~al.(2023)Lu, Tunanyan, Wang, Navasardyan, Wang, and Shi]{lu2023specialistdiffusion}
Haoming Lu, Hazarapet Tunanyan, Kai Wang, Shant Navasardyan, Zhangyang Wang, and Humphrey Shi.
\newblock Specialist diffusion: Plug-and-play sample-efficient fine-tuning of text-to-image diffusion models to learn any unseen style.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 14267--14276, 2023.
\bibitem[Meng et~al.(2022)Meng, He, Song, Song, Wu, Zhu, and Ermon]{meng2022sdedit}
Chenlin Meng, Yutong He, Yang Song, Jiaming Song, Jiajun Wu, Jun-Yan Zhu, and Stefano Ermon.
\newblock Sdedit: Guided image synthesis and editing with stochastic differential equations.
\newblock In \emph{International Conference on Learning Representations (ICLR)}, 2022.
\bibitem[Park et~al.(2021)Park, Azadi, Liu, Darrell, and Rohrbach]{park2021clipprecision}
Dong~Huk Park, Samaneh Azadi, Xihui Liu, Trevor Darrell, and Anna Rohrbach.
\newblock Benchmark for compositional text-to-image synthesis.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS) Track on Datasets and Benchmarks}. Curran, 2021.
\bibitem[Park et~al.(2020)Park, Efros, Zhang, and Zhu]{park2020contrastive}
Taesung Park, Alexei~A. Efros, Richard Zhang, and Jun-Yan Zhu.
\newblock Contrastive learning for conditional image synthesis.
\newblock In \emph{Proceedings of the European Conference on Computer Vision (ECCV)}, 2020.
\bibitem[Parmar et~al.(2023)Parmar, Kumar~Singh, Zhang, Li, Lu, and Zhu]{parmar2023zeroshoti2i}
Gaurav Parmar, Krishna Kumar~Singh, Richard Zhang, Yijun Li, Jingwan Lu, and Jun-Yan Zhu.
\newblock Zero-shot image-to-image translation.
\newblock In \emph{Proceedings of the Special Interest Group on Computer Graphics and Interactive Techniques Conference (SIGGRAPH)}. ACM, 2023.
\bibitem[Ramesh et~al.(2022)Ramesh, Dhariwal, Nichol, Chu, and Chen]{ramesh2022dalle}
Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, and Mark Chen.
\newblock Hierarchical text-conditional image generation with clip latents.
\newblock \emph{arXiv preprint arXiv:2204.06125}, 2022.
\bibitem[Rombah et~al.(2022)Rombah, Blattmann, Lorenz, Esser, and Ommer]{rombach2022sd}
Robin Rombah, Andreas Blattmann, Dominik Lorenz, Patrick Esser, and Björn Ommer.
\newblock High-resolution image synthesis with latent diffusion models.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2022.
\bibitem[Ronneberger et~al.(2015)Ronneberger, Fischer, and Brox]{ronneberger2015unet}
Olaf Ronneberger, Philipp Fischer, and Thomas Brox.
\newblock U-net: Convolutional networks for biomedical image segmentation.
\newblock In \emph{Medical Image Computing and Computer-Assisted Intervention (MICCAI)}, pages 234--241, Cham, 2015.
\bibitem[Ruiz et~al.(2022)Ruiz, Li, Jampani, Pritch, Rubinstein, and Aberman]{ruiz2022dreambooth}
Nataniel Ruiz, Yuanzhen Li, Varun Jampani, Yael Pritch, Michael Rubinstein, and Kfir Aberman.
\newblock Dreambooth: Fine tuning text-to-image diffusion models for subject-driven generation.
\newblock \emph{arXiv preprint arxiv:2208.12242}, 2022.
\bibitem[Schuhmann et~al.(2022)Schuhmann, Beaumont, Vencu, Gordon, Wightman, Cherti, Coombes, Katta, Mullis, Wortsman, Schramowski, Kundurthy, Crowson, Schmidt, Kaczmarczyk, and Jitsev]{schuhmann2022laionb}
Christoph Schuhmann, Romain Beaumont, Richard Vencu, Cade~W. Gordon, Ross Wightman, Mehdi Cherti, Theo Coombes, Aarush Katta, Clayton Mullis, Mitchell Wortsman, Patrick Schramowski, Srivatsa~R. Kundurthy, Katherine Crowson, Ludwig Schmidt, Robert Kaczmarczyk, and Jenia Jitsev.
\newblock {LAION}-5b: An open large-scale dataset for training next generation image-text models.
\newblock In \emph{Conference on Neural Information Processing Systems (NeurIPS)}, 2022.
\bibitem[Seitzer(2020)]{seitzer2020fid}
Maximilian Seitzer.
\newblock {pytorch-fid: FID Score for PyTorch}.
\newblock \url{https://github.com/mseitzer/pytorch-fid}, 2020.
\newblock Version 0.3.0.
\bibitem[Singh et~al.(2023)Singh, Gould, and Zheng]{singh2022highfidelity}
Jaskirat Singh, Stephen Gould, and Liang Zheng.
\newblock High-fidelity guided image synthesis with latent diffusion models.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Thoma et~al.(2020)Thoma, Paudel, and Gool]{thoma2020softcl}
Janine Thoma, Danda~Pani Paudel, and Luc~V Gool.
\newblock Soft contrastive learning for visual localization.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, pages 11119--11130, 2020.
\bibitem[Tumanyan et~al.(2022)Tumanyan, Geyer, Bagon, and Dekel]{tumanyan2022plugandplay}
Narek Tumanyan, Michal Geyer, Shai Bagon, and Tali Dekel.
\newblock Plug-and-play diffusion features for text-driven image-to-image translation.
\newblock \emph{arXiv preprint arXiv:2211.12572}, 2022.
\bibitem[Wang et~al.(2023)Wang, Saharia, Montgomery, Pont-Tuset, Noy, Pellegrini, Onoe, Laszlo, Fleet, Soricut, Baldridge, Norouzi, Anderson, and Chan]{wang2023imageneditor}
Su Wang, Chitwan Saharia, Ceslee Montgomery, Jordi Pont-Tuset, Shai Noy, Stefano Pellegrini, Yasumasa Onoe, Sarah Laszlo, David~J. Fleet, Radu Soricut, Jason Baldridge, Mohammad Norouzi, Peter Anderson, and William Chan.
\newblock Imagen editor and editbench: Advancing and evaluating text-guided image inpainting.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 18359--18369, 2023.
\bibitem[Wu and De~la Torre(2023)]{wu2023zeroshotediting}
Chen~Henry Wu and Fernando De~la Torre.
\newblock A latent space of stochastic diffusion models for zero-shot image editing and guidance.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, pages 7378--7387, 2023.
\bibitem[Zhang et~al.(2021)Zhang, Koh, Baldridge, Lee, and Yang]{zhang2021cross}
Han Zhang, Jing~Yu Koh, Jason Baldridge, Honglak Lee, and Yinfei Yang.
\newblock Cross-modal contrastive learning for text-to-image generation.
\newblock \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 2021.
\bibitem[Zhao et~al.(2020)Zhao, Zhang, Chen, Singh, and Zhang]{zhao2020cntrgan}
Zhengli Zhao, Zizhao Zhang, Ting Chen, Sameer Singh, and Han Zhang.
\newblock Image augmentations for gan training.
\newblock \emph{arXiv preprint arXiv:2006.02595}, 2020.
\end{thebibliography}