Deblur4DGS: 4D Gaussian Splatting from Blurry Monocular Video
Extracted source text ยท arXiv 2412.06424
% CVPR 2025 Paper Template; see https://github.com/cvpr-org/author-kit
\documentclass[10pt,twocolumn,letterpaper]{article}
%%%%%%%%% PAPER TYPE - PLEASE UPDATE FOR FINAL VERSION
% \usepackage{cvpr} % To produce the CAMERA-READY version
% \usepackage[review]{cvpr} % To produce the REVIEW version
\usepackage[pagenumbers]{cvpr} % To force page numbers, e.g. for an arXiv version
% Import additional packages in the preamble file, before hyperref
\input{preamble}
% It is strongly recommended to use hyperref, especially for the review version.
% hyperref with option pagebackref eases the reviewers' job.
% Please disable hyperref *only* if you encounter grave issues,
% e.g. with the file validation for the camera-ready version.
%
% If you comment hyperref and then uncomment it, you should delete *.aux before re-running LaTeX.
% (Or just hit 'q' on the first LaTeX run, let it finish, and you should be clear).
\definecolor{cvprblue}{rgb}{0.21,0.49,0.74}
\usepackage[pagebackref,breaklinks,colorlinks,citecolor=cvprblue]{hyperref}
\usepackage[accsupp]{axessibility} % Improves PDF readability for those with visual impairments.
% \usepackage{cite}
% \usepackage{mathrsfs}
% \usepackage{amsfonts}
% \usepackage{amsmath}
% \usepackage{amssymb}
% \usepackage{bbding}
% \usepackage{booktabs}
% \usepackage{tabularx}
\usepackage{multirow}
\usepackage{xcolor}
% \usepackage[abs]{overpic}
% \usepackage{MnSymbol}
\usepackage[abs]{overpic}
\newcommand{\tabincell}[2]{\begin{tabular}{@{}#1@{}}#2\end{tabular}}
%%%%%%%%% PAPER ID - PLEASE UPDATE
\def\paperID{6516} % *** Enter the Paper ID here
\def\confName{CVPR}
\def\confYear{2025}
%%%%%%%%% TITLE - PLEASE UPDATE
\title{Deblur4DGS: 4D Gaussian Splatting from Blurry Monocular Video}
% with Dynamic Gaussian Splatting Reconstructing 4D Reconstruction
% DybluGS: Dynamic Gaussian Splatting from Blurry Monocular Video
%%%%%%%%% AUTHORS - PLEASE UPDATE
% TODO FINAL: Replace with your author list.
% Include the authors' OCRID for the camera-ready version, if at all possible.
% \author{Renlong Wu \and Zhilu Zhang \and Mingyang Chen \and Xiaopeng Fan \and \\ Zifei Yan \and Wangmeng Zuo
% }
\author{
Renlong Wu, Zhilu Zhang, Mingyang Chen, Xiaopeng Fan, Zifei Yan, Wangmeng Zuo \\
Harbin Institute of Technology\\
{\tt\small hirenlongwu@gmail.com,cszlzhang@outlook.com, youngmchan269@gmail.com,} \\
{\tt\small fxp@hit.edu.cn,yanzifei@hit.edu.cn,wmzuo@hit.edu.cn}\\
\url{https://deblur4dgs.github.io/}
}
% For a paper whose authors are all at the same institution,
% omit the following lines up until the closing ``}''.
% Additional authors and addresses can be added with ``\and'',
% just like the second author.
% To save space, use either the email address or home page, not both
% \and
% Second Author\\
% Institution2\\
% First line of institution2 address\\
% {\tt\small secondauthor@i2.org}
% }
% % TODO FINAL: Replace with your institution list.
% \institute{
% Harbin Institute of Technology, China
% \\
% \email{hirenlongwu@gmail.com,cszlzhang@outlook.com,
% \\youngmchan269@gmail.com,fxp@hit.edu.cn,
% \\yanzifei@hit.edu.cn,wmzuo@hit.edu.cn}
% }
% \author{
% Renlong Wu \\
% Institution1\\
% Institution1 address\\
% {\tt\small firstauthor@i1.org}
% % For a paper whose authors are all at the same institution,
% % omit the following lines up until the closing ``}''.
% % Additional authors and addresses can be added with ``\and'',
% % just like the second author.
% % To save space, use either the email address or home page, not both
% \and
% Second Author\\
% Institution2\\
% First line of institution2 address\\
% {\tt\small secondauthor@i2.org}
% }
\begin{document}
\maketitle
\input{sec/0_abstract}
\input{sec/1_intro}
\clearpage
\input{sec/X_suppl}
% \input{sec/2_formatting}
% \input{sec/3_finalcopy}
% % WARNING: do not forget to delete the supplementary pages from your submission
% \input{sec/X_suppl}
\clearpage
{
\small
\bibliographystyle{ieeenat_fullname}
\bibliography{main}
}
\end{document}
%
% --- inline annotations
%
\newcommand{\red}[1]{{\color{red}#1}}
\newcommand{\todo}[1]{{\color{red}#1}}
\newcommand{\TODO}[1]{\textbf{\color{red}[TODO: #1]}}
% --- disable by uncommenting
% \renewcommand{\TODO}[1]{}
% \renewcommand{\todo}[1]{#1}
\documentclass[10pt,twocolumn,letterpaper]{article}
\usepackage[rebuttal]{cvpr}
% Include other packages here, before hyperref.
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
% Import additional packages in the preamble file, before hyperref
\input{preamble}
% If you comment hyperref and then uncomment it, you should delete
% egpaper.aux before re-running latex. (Or just hit 'q' on the first latex
% run, let it finish, and you should be clear).
\definecolor{cvprblue}{rgb}{0.21,0.49,0.74}
\usepackage[pagebackref,breaklinks,colorlinks,allcolors=cvprblue]{hyperref}
% If you wish to avoid re-using figure, table, and equation numbers from
% the main paper, please uncomment the following and change the numbers
% appropriately.
%\setcounter{figure}{2}
%\setcounter{table}{1}
%\setcounter{equation}{2}
% If you wish to avoid re-using reference numbers from the main paper,
% please uncomment the following and change the counter value to the
% number of references you have in the main paper (here, 100).
%\makeatletter
%\apptocmd{\thebibliography}{\global\c@NAT@ctr 100\relax}{}{}
%\makeatother
%%%%%%%%% PAPER ID - PLEASE UPDATE
\def\paperID{*****} % *** Enter the Paper ID here
\def\confName{CVPR}
\def\confYear{2023}
\begin{document}
%%%%%%%%% TITLE - PLEASE UPDATE
\title{\LaTeX\ Guidelines for Author Response} % **** Enter the paper title here
\maketitle
\thispagestyle{empty}
\appendix
%%%%%%%%% BODY TEXT - ENTER YOUR RESPONSE BELOW
\section{Introduction}
After receiving paper reviews, authors may optionally submit a rebuttal to address the reviewers' comments, which will be limited to a {\bf one page} PDF file.
Please follow the steps and style guidelines outlined below for submitting your author response.
The author rebuttal is optional and, following similar guidelines to previous conferences, is meant to provide you with an opportunity to rebut factual errors or to supply additional information requested by the reviewers.
It is NOT intended to add new contributions (theorems, algorithms, experiments) that were absent in the original submission and NOT specifically requested by the reviewers.
You may optionally add a figure, graph, or proof to your rebuttal to better illustrate your answer to the reviewers' comments.
Per a passed 2018 PAMI-TC motion, reviewers should refrain from requesting significant additional experiments for the rebuttal or penalize for lack of additional experiments.
Authors should refrain from including new experimental results in the rebuttal, especially when not specifically requested to do so by the reviewers.
Authors may include figures with illustrations or comparison tables of results reported in the submission/supplemental material or in other papers.
Just like the original submission, the rebuttal must maintain anonymity and cannot include external links that reveal the author identity or circumvent the length restriction.
The rebuttal must comply with this template (the use of sections is not required, though it is recommended to structure the rebuttal for ease of reading).
%-------------------------------------------------------------------------
\subsection{Response length}
Author responses must be no longer than 1 page in length including any references and figures.
Overlength responses will simply not be reviewed.
This includes responses where the margins and formatting are deemed to have been significantly altered from those laid down by this style guide.
Note that this \LaTeX\ guide already sets figure captions and references in a smaller font.
%------------------------------------------------------------------------
\section{Formatting your Response}
{\bf Make sure to update the paper title and paper ID in the appropriate place in the tex file.}
All text must be in a two-column format.
The total allowable size of the text area is $6\frac78$ inches (17.46 cm) wide by $8\frac78$ inches (22.54 cm) high.
Columns are to be $3\frac14$ inches (8.25 cm) wide, with a $\frac{5}{16}$ inch (0.8 cm) space between them.
The top margin should begin 1 inch (2.54 cm) from the top edge of the page.
The bottom margin should be $1\frac{1}{8}$ inches (2.86 cm) from the bottom edge of the page for $8.5 \times 11$-inch paper;
for A4 paper, approximately $1\frac{5}{8}$ inches (4.13 cm) from the bottom edge of the page.
Please number any displayed equations.
It is important for readers to be able to refer to any particular equation.
Wherever Times is specified, Times Roman may also be used.
Main text should be in 10-point Times, single-spaced.
Section headings should be in 10 or 12 point Times.
All paragraphs should be indented 1 pica (approx.~$\frac{1}{6}$ inch or 0.422 cm).
Figure and table captions should be 9-point Roman type as in \cref{fig:onecol}.
List and number all bibliographical references in 9-point Times, single-spaced,
at the end of your response.
When referenced in the text, enclose the citation number in square brackets, for example~\cite{Alpher05}.
Where appropriate, include the name(s) of editors of referenced books.
\begin{figure}[t]
\centering
\fbox{\rule{0pt}{0.5in} \rule{0.9\linewidth}{0pt}}
%\includegraphics[width=0.8\linewidth]{egfigure.eps}
\caption{Example of caption. It is set in Roman so that mathematics
(always set in Roman: $B \sin A = A \sin B$) may be included without an
ugly clash.}
\label{fig:onecol}
\end{figure}
To avoid ambiguities, it is best if the numbering for equations, figures, tables, and references in the author response does not overlap with that in the main paper (the reviewer may wonder if you talk about \cref{fig:onecol} in the author response or in the paper).
See \LaTeX\ template for a workaround.
%-------------------------------------------------------------------------
\subsection{Illustrations, graphs, and photographs}
All graphics should be centered.
Please ensure that any point you wish to make is resolvable in a printed copy of the response.
Resize fonts in figures to match the font in the body text, and choose line widths which render effectively in print.
Readers (and reviewers), even of an electronic copy, may choose to print your response in order to read it.
You cannot insist that they do otherwise, and therefore must not assume that they can zoom in to see tiny details on a graphic.
When placing figures in \LaTeX, it is almost always best to use \verb+\includegraphics+, and to specify the figure width as a multiple of the line width as in the example below
{\small\begin{verbatim}
\usepackage{graphicx} ...
\includegraphics[width=0.8\linewidth]
{myfile.pdf}
\end{verbatim}
}
%%%%%%%%% REFERENCES
{
\small
\bibliographystyle{ieeenat_fullname}
\bibliography{main}
}
\end{document}
\begin{abstract}
Recent 4D reconstruction methods have yielded impressive results but rely on sharp videos as supervision. However, motion blur often occurs in videos due to camera shake and object movement, while existing methods render blurry results when using such videos for reconstructing 4D models. Although a few NeRF-based approaches attempted to address the problem, they struggled to produce high-quality results, due to the inaccuracy in estimating continuous dynamic representations within the exposure time. Encouraged by recent works in 3D motion trajectory modeling using 3D Gaussian Splatting (3DGS), we suggest taking 3DGS as the scene representation manner, and propose the first 4D Gaussian Splatting framework to reconstruct a high-quality 4D model from blurry monocular video, named Deblur4DGS. Specifically, we transform continuous dynamic representations estimation within an exposure time into the exposure time estimation. Moreover, we introduce exposure regularization to avoid trivial solutions, as well as multi-frame and multi-resolution consistency ones to alleviate artifacts. Furthermore, to better represent objects with large motion, we suggest blur-aware variable canonical Gaussians. Beyond novel-view synthesis, Deblur4DGS can be applied to improve blurry video from multiple perspectives, including deblurring, frame interpolation, and video stabilization. Extensive experiments on the above four tasks show that Deblur4DGS outperforms state-of-the-art 4D reconstruction methods. The codes are available at \url{https://github.com/ZcsrenlongZ/Deblur4DGS}.
\end{abstract}
\section{Introduction}
\label{sec:intro}
\begin{figure}[t!]
\centering
\includegraphics[width=0.99\linewidth]{figures/fig1.png}
\vspace{-3mm}
\caption{
Performance and rendering speed comparisons for novel-view synthesis on the 720 $\times$ 1080 images.
%
Deblur4DGS produces better results with real-time rendering speed.
}
\label{fig:intro}
\vspace{-6mm}
\end{figure}
Substantial efforts have been made for 4D reconstruction, which has extensive applications in augmented reality and virtual reality.
%
To model static scenes, Neural Radiance Field (NeRF)~\cite{mildenhall2021nerf} and 3D Gaussian Splatting (3DGS)~\cite{kerbl20233d} propose implicit neural representation manner and explicit Gaussian ellipsoids one, respectively.
%
To model dynamic objects, implicit neural fields~\cite{lu20243d,zhu2024motiongs,yang2024deformable,4dgaussians,cao2023hexplane,pumarola2021d,yan2023nerf} and explicit deformation~\cite{4d-rotor20244d,chu2024dreamscene4d,luiten2023dynamic,yang2023real,katsumata2024compact,kratimenos2023dynmf,lin2024gaussian,li2024spacetime,wang2024shape} are suggested for motion representation.
%
While achieving great progress, most methods rely on synchronized multi-view videos.
%
They yield unsatisfactory results when applied to monocular video, where dynamic objects are only observed once at each timestamp.
%
To alleviate the under-constrained nature of the problem, recent studies have introduced data-driven priors, such as depth maps~\cite{lee2023fast,yang48953364d}, optical flows~\cite{gao2021dynamic,tian2023mononerf,gao2024gaussianflow,guo2024motion,liu2024modgs,zhu2024motiongs,lee2023fast,wang2024gflow}, tracks~\cite{wang2024shape,seidenschwarz2024dynomo,stearns2024dynamic,lei2024mosca}, and generative models~\cite{wu2025sc4d, chu2024dreamscene4d,yu20244real,zeng2025stag4d} to improve 4D reconstruction.
%
Unfortunately, motion blur often arises in the videos due to camera shake and object movement, while the above methods heavily rely on sharp videos for supervision.
%
When reconstructing the 4D scene from the blurry video, these methods can render blurry results.
%
The first step to solving this problem is to deal with camera motion blur, which is relatively simple.
%
Some NeRF-based~\cite{lee2023dp, ma2022deblur,wang2023bad,lee2023exblurf,li2023usb} and 3DGS-based~\cite{zhao2024bad,peng2024bags,chen2024deblur,oh2024deblurgs} methods have suggested jointly optimizing 3D static scene representation and camera poses within the exposure time by calculating the reconstruction loss between the synthetic blurry images and the input blurry frames.
%
In contrast, the object motion blur is more challenging to address, as the solution has to estimate continuous and sharp dynamic representations within the exposure time to simulate blurry frames.
%
As far as we know, only DybluRF~\cite{sun2024dyblurf, bui2023dyblurf} attempted this with NeRF but struggled to produce high-quality and real-time rendering results, as shown in \cref{fig:intro}.
%
In this work, we propose taking 3DGS~\cite{kerbl20233d} as the scene representation manner to explore the problem, driven by two main motivations.
%
First, the real-time rendering capability of 3DGS and its successful application in 4D reconstruction make this method highly promising.
%
Second, the explicit representation property of 3DGS makes it easier to model 3D object motion than NeRF. In particular, recent advances~\cite{kratimenos2023dynmf,wang2024shape,seidenschwarz2024dynomo,stearns2024dynamic} in reconstructing 3D motion trajectories present an opportunity to simplify this continuous dynamic representations estimation problem.
%
Based on this, the challenging problem can be transformed into simple exposure time estimation, without the extra motion trajectory modeling in DybluRF~\cite{sun2024dyblurf, bui2023dyblurf}.
%
With the estimated exposure time, continuous dynamic representations can be obtained by directly interpolating between representations at the nearest integer timestamps.
%
Specifically, we name it Deblur4DGS, which is the first Gaussian Splatting framework designed for 4D reconstruction from blurry monocular video.
%
For the static scene, we present a camera motion predictor to learn camera poses at exposure start and end, which is jointly optimized with static Gaussians.
%
For the dynamic objects, we optimize learnable exposure time parameters and dynamic Gaussians of the integer timestamps, simultaneously.
%
Then continuous camera poses and dynamic Gaussians within exposure time can be obtained by interpolation, and they are used to render continuous sharp frames to calculate the reconstruction loss.
%
Moreover, the exposure regularization term is introduced to avoid trivial solutions, while multi-frame and multi-resolution consistency regularization terms are used to alleviate artifacts.
%
Furthermore, existing 4D reconstruction methods generally select Gaussians at a single timestamp as canonical Gaussians (\eg, the ones at the first timestamp).
%
However, it may produce results with missing details in scenes with large motion, especially when processing blurry videos with a low frame rate.
%
To alleviate this issue, we suggest variable canonical Gaussians as time progresses based on the image blur level.
%
Gaussians corresponding to the sharper frame are selected as the canonical ones for better blur removal, and each canonical Gaussian is only used for some nearby timestamps to reduce the difficulty of modeling large motion.
%
%
Blurry videos suffer from not only motion blur, but also low frame rates and scene shake generally.
%
Beyond novel-view synthesis, the optimized Deblur4DGS can also be applied to address these problems, achieving deblurring, frame interpolation, and video stabilization.
%
We evaluate the 4D reconstruction quality of Deblur4DGS from all four perspectives.
%
%
Extensive experiments demonstrate that Deblur4DGS outperforms state-of-the-art 4D reconstruction methods quantitatively and qualitatively while maintaining real-time rendering speed.
%
Furthermore, Deblur4DGS has competitive capabilities in comparison with task-specific video processing models trained in a supervised manner.
The main contributions can be summarized as follows:
%
\begin{itemize}
\item We propose Deblur4DGS, the first 4D Gaussian Splatting framework specially designed to reconstruct a high-quality 4D model from blurry monocular video.
\item We propose transforming dynamic representation estimation into exposure time estimation, where a series of regularizations are suggested to tackle the under-constrained optimization and blur-aware variable canonical Gaussians is present to better represent objects with large motion.
%
\item Extensive experiments on novel-view synthesis, deblurring, frame interpolation, and video stabilization tasks show that Deblur4DGS significantly outperforms state-of-the-art 4D reconstruction methods.
\end{itemize}
\section{Related Work}
%
\subsection{Image and Video Deblurring}
%
Deep learning-based image~\cite{sun2015learning,ren2023multiscale,li2023self,kong2023efficient,wang2022uformer,zamir2022restormer,li2022learning, zhang2022self,lin2024improving,zhang2024bracketing} and video~\cite{hyun2017online, nah2019recurrent, son2021recurrent, wieschollek2017learning, zhong2023real,liang2024vrt, liang2022recurrent, Pan_2023_CVPR, zhang2024blur, shang2023joint,zhong2023blur,jiang2022erdn,zhong2020efficient,chan2022basicvsr++,zhang2018adversarial} deblurring methods have been widely explored.
%
Compared to image deblurring methods, video ones can leverage temporal clues between consecutive frames for more effective restoration.
%
For example, DBLRNet~\cite{zhang2018adversarial} employs spatial-temporal 3D convolution for spatial and temporal features aggregation.
%
DSTNet~\cite{Pan_2023_CVPR} develops a deep discriminative spatial and
temporal network.
%
%
BasicVSR++~\cite{chan2022basicvsr++} improves feature fusion with second-order feature propagation and flow-guided deformable alignment.
%
BSSTNet~\cite{zhang2024blur} introduces a blur map to sufficiently utilize the entire video, achieving recent state-of-the-art.
%
%
When reconstructing from a blurry video, pre-processing it with the 2D deblurring method is a straightforward manner.
%
However, 2D deblurring methods cannot perceive 3D structures and maintain scene geometric consistency, leading to unsatisfactory scene reconstruction.
%
\begin{figure*}[t!]
\centering
\includegraphics[width=0.99\linewidth]{figures/deblur4dgs.pdf}
\vspace{-3mm}
\caption{
(a) Training of Deblur4DGS.
%
When processing $t$-th frame, we first discretize its exposure time into $N$ timestamps.
%
Then, we estimate continuous camera poses $\{\mathbf{P}_{t,i}\}_{i=1}^{N}$ (see \cref{section:estimate_camera_poses}) and dynamic Gaussians $\{\mathbf{D}_{t,i}\}_{i=1}^{N}$ (see \cref{section:continuous_dynamic_objects}) within exposure time.
%
Next, we render each latent sharp image $\hat{\mathbf{I}}_{t,i}$ with the camera pose $\mathbf{P}_{t,i}$, dynamic Gaussians $\mathbf{D}_{t,i}$ and static Gaussians $\mathbf{S}$.
%
Finally, $\{\hat{\mathbf{I}}_{t,i}\}_{i=1}^{N}$ are averaged to obtain the synthetic blurry image $\hat{\mathbf{B}}_{t}$, which is used to calculate the reconstruction loss $\mathcal{L}_{rec}$ with the given blurry frame $\mathbf{B}_{t}$.
%
To regularize the under-constrained optimization, we introduce exposure regularization $\mathcal{L}_{e}$, multi-frame consistency regularization $\mathcal{L}_{mfc}$ and multi-resolution consistency regularization $\mathcal{L}_{mrc}$ (see \cref{section:regularization_terms}).
%
(b) Rendering of Deblur4DGS. Deblur4DGS produces the sharp image with user-provided timestamp $t$ and camera pose $\mathbf{P}_{t}$.
}
\label{fig:pipeline}
\vspace{-5mm}
\end{figure*}
\subsection{3D and 4D Reconstruction}
%
To reconstruct 3D models, NeRF~\cite{mildenhall2021nerf} and 3DGS~\cite{kerbl20233d} introduce implicit neural representation manner and explicit Gaussian ellipsoids one respectively, where the latter generally achieves better results in both image quality and rendering speed.
%
To reconstruct 4D models, most works~\cite{pumarola2021d, yan2023nerf,cao2023hexplane,fridovich2023k,fang2022fast,liu2022devrf,guo2023forward, zhao2023nerf2, feng20233d, somraj2024factorized,duan20244d,lu20243d,luiten2023dynamic,yang2023real,kratimenos2023dynmf,huang2024sc,lin2024gaussian,li2024spacetime,shawswings,sun20243dgstream,4dgaussians,yang2024deformable,bae2024per,mihajlovic2024splatfields,4d-rotor20244d,chu2024dreamscene4d,katsumata2024compact} incorporate implicit neural fields and explicit deformation for motion representation.
%
For example, DeformableGS~\cite{yang2024deformable} achieves it with multi-layer perceptrons (MLPs).
%
4DGasusians~\cite{bae2024per} improves it with a multi-resolution HexPlane~\cite{cao2023hexplane}.
%
E-D3DGS~\cite{4dgaussians} introduces per-Gaussian embeddings and temporal embeddings.
%
Moreover, to better reconstruct from monocular video, some studies enhance 4D reconstruction with data-driven priors, such as depth maps~\cite{lee2023fast,yang48953364d}, optical flows~\cite{gao2021dynamic,tian2023mononerf,gao2024gaussianflow,guo2024motion,liu2024modgs,zhu2024motiongs,lee2023fast,wang2024gflow}, tracks~\cite{wang2024shape,seidenschwarz2024dynomo,stearns2024dynamic,lei2024mosca}, and generative models~\cite{wu2025sc4d, chu2024dreamscene4d,yu20244real,zeng2025stag4d,zhang2023controlvideo}.
%
Note that these methods heavily rely on high-quality sharp videos for supervision and perform poorly when facing blurry inputs.
%
To process camera motion in static areas, recent works~\cite{lee2023dp, ma2022deblur,wang2023bad,lee2023exblurf,li2023usb, lee2024deblurring,zhao2024bad,peng2024bags,lee2024crim,chen2024deblur,oh2024deblurgs} suggest jointly optimizing the scene representation and recovering the camera poses within the exposure time.
%
To process object motion blur in dynamic scenes, DybluRF~\cite{sun2024dyblurf, bui2023dyblurf} incorporates object motion blur formation into dynamic model optimization but faces challenges in producing high-quality images and achieving real-time rendering.
%
In this work, with 3DGS~\cite{kerbl20233d} as the scene representation manner, we develop Deblur4DGS to reconstruct a high-quality 4D model from a blurry video.
%
\section{Preliminary}
\subsection{4D Gaussian Splatting}
\label{section:Preliminary}
%
A 3D Gaussian~\cite{kerbl20233d} is parameterized by $\{\mathbf{x}, \mathbf{r}, \mathbf{s}, \mathbf{o}, \mathbf{c}\}$, where $\mathbf{x}$ characterizes the center position in the world space, rotation matrix $\mathbf{r}$ and scale matrix $\mathbf{s}$ define the shape, $\alpha$ is opacity, and spherical harmonics (SH) coefficients $\mathbf{c}$ represent the view-dependent color.
In 4D Gaussian Splatting (4DGS), the static and dynamic regions are usually processed, respectively.
%
Static regions can be represented by a set of 3D Gaussians, named $\mathbf{S}$.
%
For the dynamic areas, 4DGS generally selects a timestamp (\eg, the first timestamp) and represents the objects by canonical dynamic Gaussians, \ie, $\mathbf{C}$.
%
Then, $\mathbf{C}$ is deformed to other timestamps by an implicit neural field~\cite{lu20243d,zhu2024motiongs,yang2024deformable,4dgaussians,cao2023hexplane,pumarola2021d,yan2023nerf} or explicit manner~\cite{4d-rotor20244d,chu2024dreamscene4d,luiten2023dynamic,yang2023real,katsumata2024compact,kratimenos2023dynmf,lin2024gaussian,li2024spacetime,wang2024shape} for motion representation.
%
Denote by $\mathbf{D}_{t}$ the dynamic Gaussians at $t$-th timestamp, it can be written as,
%
\vspace{-2mm}
\begin{equation}
\mathbf{D}_{t} = \mathcal{F}(\mathbf{C}, t; \Theta_{\mathcal{F}}).
\vspace{-2mm}
\end{equation}
%
$\mathcal{F}$ is the deformation operation with parameters $\Theta_{\mathcal{F}}$.
%
The Gaussians for $t$-th timestamp is the union of $\mathbf{S}$ and $\mathbf{D}_{t}$.
%
%
%
Collectively, 4DGS models a scene with static Gaussians $\mathbf{S}$, canonical dynamic Gaussians $\mathbf{C}$, and a deformation operation $\mathcal{F}$.
%
With the provided camera pose, the Gaussians at $t$-th timestamp $\mathbf{D}_{t}$ can be projected into 2D spaces and rasterized to obtain the corresponding image.
%
\subsection{Motion Blur Formation}
%
Motion blur occurs due to camera shake and object movement, which can be simply regarded as the integration of a latent sharp image sequence~\cite{nah2017deep}, \ie,
%
\vspace{-2mm}
\begin{equation}
\mathbf{B}(u,v)=\phi \int_0^\tau \mathbf{I}_t(u,v) d t.
\vspace{-2mm}
\end{equation}
%
$\mathbf{B}(u,v) \in \mathbb{R}^{H\times W \times 3}$ is the blurry image and $\mathbf{I}_t(u,v)$ is the latent sharp one at $t$-th timestamp.
%
$(u,v)$ is pixel location, $\tau$ is the camera exposure time, and $\phi$ is a normalization factor.
%
To approximate the integral operation, recent works~\cite{zhao2024bad,sun2024dyblurf,wang2023bad} divide the exposure time into $N$ timestamps and regard the blurry image as the average of $N$ sharp images, \ie,
\vspace{-2mm}
\begin{equation}
\label{eq:formation}
\mathbf{B}(u,v) \approx \frac{1}{N} \sum_{i=0}^{N-1} \mathbf{I}_i(u,v).
\vspace{-2mm}
\end{equation}
In this work, we reconstruct a high-quality 4D model from a blurry video by incorporating the physical motion blur formation into model optimization.
%
\section{Proposed Method}
\label{section:overview}
Let $\{\mathbf{B}_{t}\}_{t=1}^{T}$ and $\{\mathbf{M}_{t}\}_{t=1}^{T}$ denote a blurry video with $T$ timestamps and the corresponding masks indicating dynamic areas, respectively.
%
As shown in \cref{fig:pipeline}(a), when processing $t$-th frame, we first evenly divide its camera exposure time into $N$ timestamps.
%
Then, we estimate continuous camera poses $\{\mathbf{P}_{t,i}\}_{i=1}^{N}$ and dynamic Gaussians $\{\mathbf{D}_{t,i}\}_{i=1}^{N}$ to simulate camera shake and object movement.
%
Next, we render each sharp image $\hat{\mathbf{I}}_{t, i}$ with the corresponding camera pose $\mathbf{P}_{t,i}$, dynamic Gaussians $\mathbf{D}_{t,i}$ and static Gaussians $\mathbf{S}$.
%
After that, we average $\{\hat{\mathbf{I}}_{t,i}\}_{i=1}^{N}$ to obtain the synthetic blurry image $\hat{\mathbf{B}}_{t}$,
which is used to calculate the reconstruction loss $\mathcal{L}_{rec}$ with the given blurry frame $\mathbf{B}_{t}$, \ie,
\vspace{-1mm}
\begin{equation}
\label{eq:loss_rec}
\mathcal{L}_{rec} = (1-\beta)\mathcal{L}_{1}(\hat{\mathbf{B}}_{t}, \mathbf{B}_{t}) + \beta\mathcal{L}_{ssim}(\hat{\mathbf{B}}_{t}, \mathbf{B}_{t}).
\vspace{-1mm}
\end{equation}
%
$\mathcal{L}_{1}$ and $\mathcal{L}_{ssim}$ are $\ell_1 $ loss and SSIM~\cite{wang2004image} loss, respectively.
%
$\beta$ is set to $0.2$.
%
The settings all follow 3DGS~\cite{kerbl20233d}.
%
%
%
\subsection{Continuous Camera Poses Estimation}
\label{section:estimate_camera_poses}
%
To estimate continuous camera poses, recent methods~\cite{zhao2024bad,peng2024bags,chen2024deblur,oh2024deblurgs} directly
optimize exposure start and end poses (\ie, $\mathbf{P}_{t,1}$ and $\mathbf{P}_{t,N}$).
%
Then, the linear interpolation is performed between $\mathbf{P}_{t,1}$ and $\mathbf{P}_{t,N}$ in the Lie-algebra of $\mathbf{SE}(3)$ to obtain the camera pose at $i$-th intermediate timestamp (\ie, $\mathbf{P}_{t,i}$).
%
It can be written as,
%
\vspace{-2mm}
\begin{equation}
\mathbf{P}_{t,i} = \mathbf{P}_{t,1} \odot \texttt{exp}(\frac{i-1}{N-1} \odot \texttt{log}( \frac{\mathbf{P}_{t, N}}{\mathbf{P}_{t,1}})).
\vspace{-2mm}
\end{equation}
%
%
$\texttt{exp}$ and $\texttt{log}$ are exponential and logarithmic functions, respectively.
%
$\odot$ is a pixel-wise multiply operation.
%
We follow the manner but deploy a tiny MLP as the camera motion predictor (see details in \textit{Suppl}.) for more stable optimization.
%
We pre-train the camera motion predictor and static Gaussians $\mathbf{S}$ with static reconstruction loss $\mathcal{L}_{rec}^{s}$, \ie,
\vspace{-1mm}
\begin{equation}
\label{eq:loss_rec_static}
\mathcal{L}_{rec}^{s} = (1-\beta)\mathcal{L}_{1}(\hat{\mathbf{B}}_{t}^{s}, \mathbf{B}_{t}^{s}) + \beta\mathcal{L}_{ssim}(\hat{\mathbf{B}}_{t}^{s}, \mathbf{B}_{t}^{s}).
\vspace{-1mm}
\end{equation}
$\hat{\mathbf{B}}_{t}^{s} = (1 - \mathbf{M}_{t}) \odot \hat{\mathbf{B}}_{t}$ and $\mathbf{B}_{t}^{s} = (1 - \mathbf{M}_{t}) \odot \mathbf{B}_{t}$ are the static areas of $\hat{\mathbf{B}}_{t}$ and $\mathbf{B}_{t}$, respectively.
%
%
\subsection{Continuous Dynamic Gaussians Estimation}
\label{section:continuous_dynamic_objects}
We first introduce blur-aware variable canonical Gaussians for better dynamic representation at integer timestamps.
%
Then, we describe Gaussian deformation manner.
%
Finally, we detail how to take learnable exposure time parameters to obtain continuous dynamic Gaussians within exposure time.
%
%
\noindent\textbf{Blur-Aware Variable Canonical Gaussians.}
%
Existing 4D reconstruction methods generally select a single canonical Gaussians $\mathbf{C}$ across the entire video, which may produce results with missing details in scenes with large motion.
%
To alleviate the issue, we suggest variable the canonical Gaussians as time progresses.
%
In such case, the $k$-th canonical Gaussians $\mathbf{C}_{k}$ is only used for some nearby timestamps thus reducing the difficulty of modeling large motion.
%
One way to achieve this is to uniformly divide the video into $L$ segments and select $\mathbf{C}_{k}$ for $k$-th segment.
%
Although it improves performance, selecting the one corresponding to the sharper frame is better for blur removal.
%
%
In particular, we first uniformly divide the video into $L$ segments and calculate the blur level $b_{t}$ of dynamic areas for $t$-th frame following ~\cite{bansal2016blur,ren2020video}.
%
It can be written as,
\vspace{-2mm}
\begin{equation}
b_{t} = \sum_{(u,v)\in \mathbf{M}_{t}}(\Delta\mathbf{B}_{t}(u,v) -\overline{\Delta\mathbf{B}_{t}})^2,
\vspace{-2mm}
\end{equation}
%
where $\mathbf{M}_{t}$ indicates dynamic areas.
%
$\Delta\mathbf{B}_{t}$ is the image Laplacian and $\overline{\Delta\mathbf{B}_{t}}$ is its mean value.
%
The larger $b_{t}$ is, the sharper the frame is.
%
To make the start and end frame of the segment as sharp as possible, we look for the sharp frame among their surrounding $H$ frames and redefine them as the start and end of the current segment.
%
After that, we select the Gaussians corresponding to the sharpest frame in each segment as its canonical ones.
\noindent\textbf{Gaussian Deformation.}
%
We follow Shape-of-Motion~\cite{wang2024shape} to deform dynamic Gaussians, where a set of rigid transformation matrices are deployed.
%
Let $\{\mathbf{x}_{c}, \mathbf{r}_{c}, \mathbf{s}, \mathbf{o}, \mathbf{c}\}$, $\{\mathbf{x}_{t}, \mathbf{r}_{t}, \mathbf{s}, \mathbf{o}, \mathbf{c}\}$, and $\{\mathbf{A}_{t}, \mathbf{E}_{t}\}$ denote a Gaussian in $\mathbf{C}_{k}$, the ones in $\mathbf{D}_{t}$, and the corresponding transformation matrix, respectively.
%
It can be written as,
\vspace{-2mm}
\begin{equation}
\label{eqn:deform}
\mathbf{x}_{t} = \mathbf{A}_{t} \mathbf{x}_{c} + \mathbf{E}_{t}, \qquad
\mathbf{r}_{t} = \mathbf{A}_{t} \mathbf{r}_{c}.
\vspace{-2mm}
\end{equation}
%
Kindly refer to Shape-of-Motion~\cite{wang2024shape} for details.
%
%
\noindent\textbf{Interpolation with Exposure Time Parameters.}
%
%
To get continuous dynamic Gaussians $\{\mathbf{D}_{t,i}\}_{i=1}^{N}$, one straightforward way is to deploy a series of learnable Gaussian or deformation parameters, but it is unstable to optimize.
%
With the explicit object motion representation in \cref{eqn:deform}, $\mathbf{D}_{t, i}$ can be calculated by interpolating between the ones at the nearest integer timestamps, \ie,
%
\vspace{-1mm}
\begin{equation}
\small
\label{eq:estimate_continuous_gaussians}
\begin{split}
\mathbf{D}_{t, i} = \mathbf{w}_{t, i} \odot \mathbf{D}_{t-1} + (1 - \mathbf{w}_{t, i}) \odot \mathbf{D}_{t}, \text{ }{i} \in [1, N/2],
\\
\mathbf{D}_{t, i} = (1 - \mathbf{w}_{t, i}) \odot \mathbf{D}_{t} + \mathbf{w}_{t, i} \odot \mathbf{D}_{t+1}, \text{ }{i} \in [N/2, N].
\vspace{-1mm}
\end{split}
\end{equation}
%
$\mathbf{w}_{t, i}$ is the normalized time interval between $\mathbf{D}_{t, i}$ and $\mathbf{D}_{t, N/2}$.
%
Thus, the problem is transformed to estimate $\mathbf{w}_{t, i}$.
%
%
In the implementation, we can estimate the one at exposure start and end (\ie, $\mathbf{w}_{t, 1}$ and $\mathbf{w}_{t, N}$) and then interpolate between them to get the $i$-th intermediate one $\mathbf{w}_{t, i}$, \ie,
\vspace{-2mm}
\begin{equation}
\label{eq:w_t_interpolate}
\mathbf{w}_{t,i} = (1 - \frac{i-1}{N-1}) \odot \mathbf{w}_{t,1} + \frac{i-1}{N-1} \odot \mathbf{w}_{t,N}.
\vspace{-2mm}
\end{equation}
%
As the object motion within the exposure can be regarded as uniform, the absolute value of $\mathbf{w}_{t, 1}$ and $\mathbf{w}_{t, N}$ are equal, which is half the exposure time $\mathbf{w}_{t}$.
%
Thus, \cref{eq:w_t_interpolate} can be re-written as,
\vspace{-1mm}
\begin{equation}
\label{eq:w_t_interpolate_final}
\mathbf{w}_{t,i} = (1 - \frac{i-1}{N-1}) \odot \frac{\mathbf{w}_{t}}{2} + \frac{i-1}{N-1} \odot (-\frac{\mathbf{w}_{t}}{2}).
\vspace{-1mm}
\end{equation}
%
Thus, we set learnable parameters $\mathbf{w}_{t}$ in Deblur4DGS for continuous dynamic Gaussians estimation within the exposure time.
%
The canonical Gaussians, Gaussian deformation modules, and $\mathbf{w}_{t}$ are jointly optimized.
%
The reconstruction loss for dynamic areas is similar to \cref{eq:loss_rec_static}.
\begin{figure}[t!]
\centering
\includegraphics[width=0.99\linewidth]{figures/fig3.png}
\vspace{-3mm}
\caption{
%
Illustration of artifacts in dynamic areas.
%
After optimization with~\cref{eq:loss_rec}, static areas of $\hat{\mathbf{I}}_{t, i}$ are sharp while dynamic areas exist notable artifacts, as shown in the yellow and red boxes in (b), respectively.
%
The proposed regularization terms $\mathcal{L}_{reg}$ (see \cref{eq:regularization}) improve the visual effect, as shown in (c).
%
}
\label{fig:borken_solution}
\vspace{-5mm}
\end{figure}
\subsection{Regularization Terms}
%
\label{section:regularization_terms}
After optimization with \cref{eq:loss_rec}, static areas of $\hat{\mathbf{I}}_{t, i}$ are sharp while dynamic areas can with notable artifacts, as shown in the yellow and red boxes of \cref{fig:borken_solution}(b) respectively.
%
The reasons are below.
%
(1) Note that multiple solutions exist for the model to fulfill \cref{eq:loss_rec}.
%
The most ideal one is that every $\hat{\mathbf{I}}_{t,i}$ is sharp, and the most trivial one is that every $\hat{\mathbf{I}}_{t,i}$ is as blurry as $\mathbf{B}_{t}$.
%
(2) As static areas are consistent across the entire video, the model tends to learn the underlying sharp representation for inter-frame consistency.
%
In other words, the inter-frame consistency implicitly regularizes model optimization.
%
To further validate this, we conduct an experiment that removes the inter-frame consistency by reducing the number of frames to one.
%
In such a case, the static areas are blurry after optimization with \cref{eq:loss_rec}, which supports our confirmation.
%
(3) Compared to static areas, the inter-frame consistency in dynamic ones is weaker due to object motion.
%
It may provide insufficient regularization to guide sharp representation learning, thus leading to results with artifacts.
%
To avoid this, we introduce some regularization terms, including exposure regularization $\mathcal{L}_{e}$, multi-frame consistency regularization $\mathcal{L}_{mfc}$, and multi-resolution consistency regularization $\mathcal{L}_{mrc}$.
%
%
First, the continuous dynamic Gaussians $\{\mathbf{D}_{t, i}\}_{i=1}^{N}$ should not be the same.
%
In other words, the value of exposure time parameters $\mathbf{w}_{t}$ should not be too small.
%
If $\mathbf{w}_{t}$ is too small, $\mathbf{D}_{t,i}$ is nearly the same as $\mathbf{D}_{t}$, leading to trivial solutions.
%
We constrain $\mathbf{w}_{t}$ by $\mathcal{L}_{e}$, as,
\vspace{-2mm}
\begin{equation}
\label{eq:L_w}
\mathcal{L}_{e} = \texttt{max}(0, \mathbf{\epsilon}-\mathbf{w}_{t}).
\vspace{-1mm}
\end{equation}
%
$\texttt{max}$ is the maximum function and $\mathbf{\epsilon}$ is a threshold.
%
%
Second, despite different motions, the content of multiple frames within exposure time should be similar.
%
We utilize $\mathcal{L}_{fc}$ to constrain consistency between neighbor frames, and that between each frame and the first frame, \ie,
%
\vspace{-2mm}
\begin{equation}
\label{eq:L_cc}
\begin{split}
\mathcal{L}_{mfc} = \frac{1}{N-1} & \sum_{i=2}^{N} (\left\|\mathbf{M}_{t,i} \odot ( \hat{\mathbf{I}}_{t,{i-1 \rightarrow i}} - \hat{\mathbf{I}}_{t,i})\right\|_1
\\ & \; \; \: + \left\|\mathbf{M}_{t,1} \odot( \hat{\mathbf{I}}_{t,{i \rightarrow 1}} - \hat{\mathbf{I}}_{t,1})\right\|_1).
\vspace{-2mm}
\end{split}
\end{equation}
%
$\hat{\mathbf{I}}_{t,{i-1 \rightarrow i}}$ and $\hat{\mathbf{I}}_{t,{i \rightarrow 1}}$ are obtained by aligning $\hat{\mathbf{I}}_{t,{i-1}}$ to $\hat{\mathbf{I}}_{t,i}$ and aligning $\hat{\mathbf{I}}_{t,i}$ to $\hat{\mathbf{I}}_{t,1}$ with a pre-trained optical flow network~\cite{sun2018pwc}, respectively.
%
$\mathbf{M}_{t,i}$ and $\mathbf{M}_{t,1}$ are masks that indicate dynamic areas in $\hat{\mathbf{I}}_{t,i}$ and $\hat{\mathbf{I}}_{t,1}$, respectively.
%
%
Third, the blur in the lower resolution is lower level and is easier to remove~\cite{kim2022mssnet,tao2018scale}, thus the artifacts are less in models trained with down-sampled blurry video.
%
Taking advantage of this, we impose $\mathcal{L}_{mrc}$ to assist the optimization of high-resolution models with results from low-resolution models.
%
It can be written as,
%
\vspace{-2mm}
\begin{equation}
\label{eq:L_sc}
\mathcal{L}_{mrc} = ||(\mathbf{M}_{t,i})_{\downarrow}\odot((\hat{\mathbf{I}}_{t,i})_{\downarrow}- \texttt{sg}(\hat{\mathbf{I}}_{t, i}^{l}))||_1.
\vspace{-2mm}
\end{equation}
%
$\hat{\mathbf{I}}_{t, i}^{l}$ is the rendered sharp image from the low-resolution model, which is pre-trained by taking the down-sampled video as supervision.
%
$(\cdot)_{\downarrow}$ is an image down-sampling operation.
%
$\texttt{sg}$ is the stop-gradient operation.
%
%
\begin{figure*}[t!]
\centering
\begin{subfigure}{0.99\textwidth}
\ContinuedFloat
\begin{overpic}[width=0.99\textwidth,grid=False]
{figures/vis_high_novel.png}
\put(50,-9){{\fontsize{7}{7}\selectfont Blury Frame}}
\put(130,-9){{\fontsize{7}{7}\selectfont DeformableGS~\cite{yang2024deformable}}}
\put(190,-9){{\fontsize{7}{7}\selectfont 4DGaussians~\cite{4dgaussians}}}
\put(243,-9){{\fontsize{7}{7}\selectfont E-D3DGS~\cite{bae2024per}}}
\put(287,-9){{\fontsize{7}{7}\selectfont DybluRF~\cite{sun2024dyblurf}}}
\put(330,-9){{\fontsize{7}{7}\selectfont Shape-of-Motion~\cite{wang2024shape}}}
\put(407,-9){{\fontsize{7}{7}\selectfont Ours}}
\put(450,-9){{\fontsize{7}{7}\selectfont Sharp GT}}
\end{overpic}
\vspace{2mm}
\end{subfigure}
\caption{Visual comparisons of novel-view synthesis on the $720 \times 1080 $ images.
%
Our method produces more photo-realistic details in both static and dynamic areas, as marked with yellow and red boxes respectively.
}
\label{fig:vis_high_novel}
\vspace{-3mm}
\end{figure*}
Overall, the proposed regularization terms $\mathcal{L}_{reg}$ can be denoted as,
\vspace{-1mm}
\begin{equation}
\label{eq:regularization}
\mathcal{L}_{reg} = \lambda_{e}\mathcal{L}_{e} + \lambda_{mfc}\mathcal{L}_{mfc} + \lambda_{mrc}\mathcal{L}_{mrc}.
\vspace{-1mm}
\end{equation}
%
$\lambda_{e}$, $\lambda_{mfc}$, and $\lambda_{mrc}$ are set to 0.1, 2, 1, respectively.
%
Besides, following Shape-of-Motion~\cite{wang2024shape}, we also use some other regularization terms $\mathcal{L}_{oth}$ to help reconstruct 3D motion better, and the details can be seen in the \textit{Suppl}.
\subsection{Application to Multiple Tasks}
%
%
The blurry videos suffer from not only motion blur, but also low frame rates and scene shake generally.
%
Beyond novel-view synthesis, Deblur4DGS can adjust the camera poses and timestamps to address these problems, achieving video deblurring, frame interpolation, and video stabilization.
%
First, when inputting camera poses of the blurry video, Deblur4DGS can render corresponding deblurring results.
%
Second, when feeding the interpolated camera poses and timestamps, Deblur4DGS can produce frame-interpolated results.
%
Third, Deblur4DGS can render a more stable video with the smoothed camera poses as inputs.
%
\section{Experiments}
\subsection{Experimental Settings}
%
\noindent\textbf{Datasets.}
%
We evaluate methods on 6 scenes with significant motion blur from Stereo Blur Dataset~\cite{zhou2019davanet}, where each one contains blurry stereo videos and the corresponding sharp ones.
%
For each scene, we extract 24 frames from the original video and use COLMAP~\cite{schonberger2016structure} to calibrate the camera extrinsic and intrinsic parameters, following DybluRF~\cite{sun2024dyblurf}.
%
%
Note that DybluRF~\cite{sun2024dyblurf} conducts experiments on $\times 2.5$ down-sampled data.
%
We employ both $\times 2.5$ down-sampled ones (\ie, $288 \times 512$) and the original ones (\ie, $720 \times 1080$).
\noindent\textbf{Training Configurations.}
%
We use the blurry image sequences from the left camera for training.
%
For stable optimization, we pre-train the camera motion predictor and static Gaussians $\mathbf{S}$ for $200$ epochs.
%
After that, we jointly optimize the camera motion predictor, $\mathbf{S}$, canonical dynamic Gaussians $\{\mathbf{C}_{k}\}_{k=1}^{K}$, deformable operation $\mathcal{F}$ and exposure time parameters $\{\mathbf{w}_{t}\}_{t=1}^{T}$ for $200$ epochs.
%
The learning rate for the camera motion predictor is set to $5\times10^{-4}$ and decayed to $1\times10^{-5}$.
%
The learning rate for $\{\mathbf{w}_{t}\}_{t=1}^{T}$ is set to $1\times10^{-1}$ and decayed to $1\times10^{-5}$.
%
The learning rate for $\mathbf{S}$, $\{\mathbf{C}_{k}\}_{k=1}^{K}$ and $\mathcal{F}$ are the same as Shape-of-Motion~\cite{wang2024shape}.
%
$N$ is set to 11.
%
$L$ and $H$ are set to 5 and 3 respectively.
%
$\mathbf{\epsilon}$ is set to $1.0$.
%
All experiments are conducted with PyTorch~\cite{paszke2019pytorch} on one Nvidia GeForce RTX A6000 GPU.
\begin{table}[t!]
\small
\renewcommand\arraystretch{1}
\begin{center}
\caption{Quantitative comparisons for \textbf{novel view synthesis}.
%
$\uparrow$ denotes the higher metric the better, and $\downarrow$ denotes the lower one the better.
%
The best is \textbf{boldfaced}, and the second is \underline{underlined}.
}
\label{tab:quant_novel}
\vspace{-2mm}
\scalebox{0.8}{
\begin{tabular}{lcccc}
\toprule
Methods & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$ } & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$} &\\
\midrule
DeformableGS~\cite{yang2024deformable} & 17.82 / 0.722 / 0.292 & 17.12 / 0.751 / 0.363\\
4DGaussians~\cite{4dgaussians} &21.94 / 0.833 / 0.180 & 21.69 / 0.852 / 0.238\\
E-D3DGS~\cite{bae2024per} & 22.85 / 0.843 / 0.175 & 22.38 / 0.854 / 0.240\\
DyBluRF~\cite{sun2024dyblurf} & 27.09 / 0.919 / \underline{0.090} & 26.46 / 0.913 / \underline{0.142}\\
Shape-of-Motion~\cite{wang2024shape} & \underline{27.89} / \underline{0.928} / 0.116 & \underline{27.49} / \underline{0.922} / 0.192\\
\midrule
Deblur4DGS (Ours) & \textbf{28.92} / \textbf{0.949} / \textbf{0.060} & \textbf{28.88} / \textbf{0.947} / \textbf{0.098}
\\
\bottomrule
\end{tabular}
}
\end{center}
\vspace{-6mm}
\end{table}
\begin{table}
\centering
\caption{Quantitative results that pre-process the blurry video with an image or video deblurring method before 4D reconstruction.
%
`None' denotes no image or video deblurring method being used.
}
\vspace{-2mm}
\scalebox{0.6}{\begin{tabular}{c c c c}
\toprule
\tabincell{c}{Reconstruction \\ Methods}
& \tabincell{c}{Deblurring \\ Methods} & \tabincell{c}{PSNR$\uparrow$ / SSIM$\uparrow$ / LPIPS$\downarrow$ \\ $288 \times 512$} & \tabincell{c}{PSNR$\uparrow$ / SSIM$\uparrow$ / LPIPS$\downarrow$ \\ $720 \times 1080$} \\
\midrule
\multirow{5}{*}{\begin{tabular}[c]{@{}c@{}} E-D3DGS~\cite{bae2024per} \end{tabular}} & None & 22.85 / 0.843 / 0.175 &22.38 / 0.854 / 0.240 \\
\multicolumn{1}{c}{} & Restormer~\cite{zamir2022restormer} & \textbf{23.55} / 0.854 / 0.142 & 22.77 / \underline{0.867} / 0.186\\
\multicolumn{1}{c}{} & DSTNet~\cite{Pan_2023_CVPR}& 23.21 / \textbf{0.863} / \underline{0.132} & \underline{22.80} / 0.865 / \underline{0.177}\\
\multicolumn{1}{c}{} & BSSTNet~\cite{zhang2024blur}& \underline{23.28} / \underline{0.857} / \textbf{0.128} & \textbf{23.08} / \textbf{0.876} / \textbf{0.163}\\
\midrule
\multirow{5}{*}{\begin{tabular}[c]{@{}c@{}} Shape-of-Motion~\cite{wang2024shape} \end{tabular}} & None & 27.89 / 0.928 / 0.116 & 27.49 / 0.922 / 0.192\\
\multicolumn{1}{c}{} & Restormer~\cite{zamir2022restormer} & \textbf{28.39} / \textbf{0.941} / 0.080 & \textbf{28.25} / 0.940 / 0.117\\
\multicolumn{1}{c}{} & DSTNet~\cite{Pan_2023_CVPR} & 28.07 / 0.938 / \textbf{0.072} & 28.22 / \underline{0.941} / \underline{0.104}\\
\multicolumn{1}{c}{} & BSSTNet~\cite{zhang2024blur}& \underline{28.34} / \underline{0.940} / \underline{0.079} & \underline{28.24} / \textbf{0.943} / \textbf{0.100}\\
\midrule
Deblur4DGS (Ours) & None & \textbf{28.92} / \textbf{0.949} / \textbf{0.060} & \textbf{28.88} / \textbf{0.947} / \textbf{0.098}\\
\bottomrule
\end{tabular}}
\label{tab:deblur_4d}
\vspace{-5mm}
\end{table}
%
\noindent\textbf{Evaluation Configurations.}
%
We evaluate methods on novel-view synthesis, deblurring, frame interpolation, and video stabilization tasks.
%
To evaluate novel-view synthesis and deblurring performance, we use the sharp image sequences of the right and left cameras as ground truth respectively.
%
Due to calibrated camera parameters are not absolutely accurate, the rendering results may be slightly spatially misaligned with the ground truth.
%
We adopt the aligned PSNR~\cite{huynh2008scope}, SSIM~\cite{wang2004image} and LPIPS~\cite{zhang2018unreasonable} as evaluation metrics, following similar works~\cite{bhat2021deep,dudhane2022burst,wu2024self}.
%
Specifically, we align rendering results to the ground truth by a pre-trained optical flow network (\ie, PWC-Net~\cite{sun2018pwc}) and then calculate metrics between the aligned results and the ground truth.
%
Simultaneously, we report results that calculate metrics between the original rendering images and the ground truth in the \textit{Suppl}.
%
As there is no ground truth for frame interpolation and video stabilization tasks, we evaluate method performance with recent no-reference metrics, \ie, CLIPIQA~\cite{wang2023exploring} and MUSIQ~\cite{ke2021musiq}.
\begin{table}[t!]
\small
\renewcommand\arraystretch{1}
\begin{center}
\caption{Quantitative comparisons for \textbf{deblurring}.
%
The first category is supervised image or video deblurring methods, and the second is 4D reconstruction-based ones.
%
The best one in each category is \textbf{boldfaced}, and the second one is \underline{underlined}.
}
\label{tab:quant_deblur}
\vspace{-2mm}
\scalebox{0.8}{
\begin{tabular}{lcccc}
\toprule
Methods & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$} & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$} &\\
\midrule
Restormer~\cite{zamir2022restormer} & \underline{33.33} / \underline{0.974} / 0.043 & \underline{32.43} / \underline{0.962} / 0.064 \\
DSTNet~\cite{ Pan_2023_CVPR} & 33.13 / 0.971 / \underline{0.039} & 32.42 / \underline{0.962} / \underline{0.059} \\
BSSTNet~\cite{zhang2024blur} & \textbf{34.13} / \textbf{0.980} / \textbf{0.027} & \textbf{33.63} / \textbf{0.973} / \textbf{0.039} \\
\midrule
DeformableGS~\cite{yang2024deformable} & 30.92 / \underline{0.949} / 0.103 & 29.11 / 0.926 / 0.191\\
4DGaussians~\cite{4dgaussians} & 31.25 / \underline{0.949} / 0.097 & 29.95 / 0.932 / 0.169 \\
E-D3DGS~\cite{bae2024per} & \underline{31.28} / \underline{0.949} / 0.098 & \underline{29.98} / 0.931 / 0.170 \\
DyBluRF~\cite{sun2024dyblurf} & 29.87 / 0.947 / 0.054 & 29.23 / \underline{0.943} / \underline{0.105} \\
Shape-of-Motion~\cite{wang2024shape} & 29.83 / 0.946 / 0.107 & 28.93 / 0.932 / 0.190 \\
Deblur4DGS (Ours) & \textbf{32.55} / \textbf{0.975} / \textbf{0.041} & \textbf{31.60} / \textbf{0.965} / \textbf{0.084} \\
\bottomrule
\end{tabular}
}
\end{center}
\vspace{-7mm}
\end{table}
\subsection{Comparison with State-of-the-Art Methods}
%
We compare Deblur4DGS with 5 state-of-the-art 4D reconstruction methods (\ie,
DeformableGS~\cite{yang2024deformable},
4DGaussians~\cite{4dgaussians},
E-D3DGS~\cite{bae2024per},
DyBluRF~\cite{sun2024dyblurf}, and Shape-of-Motion~\cite{wang2024shape}) on two resolution (\ie, $228 \times 512$ and $720 \times 1080$) datasets.
%
We input poses of right and left cameras to produce novel-view synthesis and deblurring results, respectively.
%
We linearly interpolate the left camera poses and timestamps to generate $\times16$ frame interpolation results.
%
We employ a Gaussian filter to smooth left camera poses for video stabilization, following ~\cite{peng20243d}.
%
The results for novel-view synthesis, deblurring, frame interpolation, and video stabilization are summarized in \cref{tab:quant_novel}, \cref{tab:quant_deblur}, \cref{tab:quant_fi}, and \cref{tab:quant_stab}, respectively.
%
Besides, rendering speed comparisons on $720\times1080$ images are reported in \cref{fig:intro}.
%
%
%
%
\noindent\textbf{Novel-view synthesis Comparisons.}
%
As shown in \cref{tab:quant_novel}, among all competing methods, DyBluRF~\cite{sun2024dyblurf} gets the best LPIPS score, while Shape-of-Motion~\cite{wang2024shape} achieves the best PSNR and SSIM scores.
%
Benefiting the explicit 3D representation manner and elaborate design for processing motion blur, our Deblur4DGS obtains all the best scores on both datasets.
%
In addition, Deblur4DGS removes blur more clearly and produces more photo-realistic details in both static and dynamic areas, as shown in \cref{fig:vis_high_novel}.
%
More visual comparisons can be seen in the \textit{Suppl}.
%
%
In addition, to further demonstrate the effectiveness of Deblur4DGS, we first pre-process the blurry videos with state-of-the-art image~\cite{zamir2022restormer} or video~\cite{Pan_2023_CVPR,zhang2024blur} deblurring methods and then perform 4D reconstruction.
%
The results are summarized in \cref{tab:deblur_4d}.
%
Compared with reconstruction from blurry videos, the incorporation of deblurring models improves performance.
%
This is because the deblurring models remove some blur, facilitating sharp scene reconstruction.
%
However, as the image and video deblurring methods cannot perceive 3D structure and maintain scene geometric consistency, the scene reconstruction results are still unsatisfactory.
%
In contrast, Deblur4DGS jointly reconstructs scene geometry and processes motion blur in 3D space, achieving better scene reconstruction results.
%
\begin{table}[t!]
\small
\renewcommand\arraystretch{1}
\begin{center}
\caption{Quantitative comparisons for \textbf{frame interpolation}.
%
The first category is supervised video frame interpolation methods, and the second is 4D reconstruction-based ones.
}
\label{tab:quant_fi}
\vspace{-2mm}
\scalebox{0.8}{
\begin{tabular}{lcccc}
\toprule
Methods & \tabincell{c}{CLIPIQA$\uparrow$/MUSIQ$\uparrow$ \\ $288 \times 512$} & \tabincell{c}{CLIPIQA$\uparrow$/MUSIQ$\uparrow$ \\ $720 \times 1080$} &\\
\midrule
RIFE~\cite{huang2022real} & 0.179 / \underline{44.090}& 0.147 / 32.637\\
EMAVFI~\cite{zhang2023extracting} & \underline{0.181} / 42.773 & \underline{0.170} / 32.564\\
VIDUE~\cite{shang2023joint} & \textbf{0.267} / \textbf{60.854} & \textbf{0.185} / \textbf{49.560}\\
\midrule
DeformableGS~\cite{yang2024deformable} & 0.172 / 41.603 & 0.167 / 31.205\\
4DGaussians~\cite{4dgaussians} & 0.177 / 44.132 & 0.177 / 33.836\\
E-D3DGS~\cite{bae2024per} & \underline{0.178} / 43.250 & 0.181 / 32.945\\
DyBluRF~\cite{sun2024dyblurf} & 0.166 / \underline{51.513} & 0.132 / \underline{36.768}\\
Shape-of-Motion~\cite{wang2024shape} & 0.176 / 43.020 & \underline{0.202} / 33.817 \\
Deblur4DGS (Ours) & \textbf{0.200} / \textbf{54.043} & \textbf{0.213} / \textbf{42.139}\\
\bottomrule
\end{tabular}
}
\end{center}
\vspace{-6mm}
\end{table}
\begin{table}[t!]
\small
\renewcommand\arraystretch{1}
\begin{center}
\caption{Quantitative comparisons for \textbf{video stabilization}.
%
The first category is 2D video stabilization methods, and the second is 4D reconstruction-based ones.
}
\label{tab:quant_stab}
\vspace{-2mm}
\scalebox{0.8}{
\begin{tabular}{lcccc}
\toprule
Methods &\tabincell{c}{CLIPIQA$\uparrow$/MUSIQ$\uparrow$ \\ $288 \times 512$} & \tabincell{c}{CLIPIQA$\uparrow$/MUSIQ$\uparrow$ \\ $720 \times 1080$} &\\
\midrule
MeshFlow~\cite{liu2016meshflow}& \underline{0.140} / \underline{38.289} & \underline{0.116} / \textbf{34.140}\\
NNDVS~\cite{zhang2023minimum} & \textbf{0.186} / \textbf{45.613}& \textbf{0.137} / \underline{33.269}\\
\midrule
DeformableGS~\cite{yang2024deformable} & 0.170 / 40.846 & 0.168 / 30.971\\
4DGaussians~\cite{4dgaussians} & 0.145 / 44.673 & 0.172 / 34.416 \\
E-D3DGS~\cite{bae2024per} & \underline{0.181} / 44.364 & \underline{0.182} / 33.828\\
DyBluRF~\cite{sun2024dyblurf} & 0.164 / \underline{53.115} &0.132 / \underline{37.556} \\
Shape-of-Motion~\cite{wang2024shape} & 0.174 / 43.293 & 0.202 / 33.750\\
Deblur4DGS (Ours)& \textbf{0.199} / \textbf{54.299} & \textbf{0.211} / \textbf{42.241}\\
\bottomrule
\end{tabular}
}
\end{center}
\vspace{-7mm}
\end{table}
\noindent\textbf{Deblurring Comparisons.}
%
Apart from 4D reconstruction-based methods, we additionally compare with some state-of-the-art image~\cite{zamir2022restormer} and video~\cite{Pan_2023_CVPR,zhang2024blur} deblurring ones.
%
As shown in \cref{tab:quant_deblur}, Deblur4DGS obtains better results than 4D reconstruction-based methods and comparable ones to the deblurring-specific (\ie, Restormer~\cite{zamir2022restormer}, DSTNet~\cite{Pan_2023_CVPR}, and BSSTNet~\cite{ zhang2024blur}) methods.
%
Compared with the former, Deblur4DGS better reconstructs the scene, thus performing better.
%
Note that the latter ones are trained on large paired data in a fully supervised manner while Deblur4DGS is optimized only with the given blurry video in a self-supervised manner.
%
Although the large data prior makes these methods perform better, Deblur4DGS is more convenient to use without paired data collection.
\noindent\textbf{Frame Interpolation Comparisons.}
%
We compare Deblur4DGS with 4D reconstruction-based methods and some state-of-the-art supervised video frame interpolation ones (\ie, RIFE~\cite{huang2022real}, EMAVFI~\cite{zhang2023extracting}, and VIDUE~\cite{ shang2023joint}), as shown in \cref{tab:quant_fi}.
%
Deform4DGS generally performs satisfactorily.
%
Note that VIDUE~\cite{ shang2023joint} is specially designed and trained with large paired data in a supervised manner for joint deblurring and
fame Interpolation, thus achieving better results.
%
%
%
%
\noindent\textbf{Video Stabilization Comparisons.}
%
The results are summarized in \cref{tab:quant_stab}.
%
Benefiting from the better geometry reconstruction, Deblur4DGS achieves pleasant scores compared with both 2D video stabilization methods (\ie, MeshFlow~\cite{liu2016meshflow} and NNDVS~\cite{zhang2023minimum}) and 4D reconstruction-based ones, which further demonstrates its effectiveness.
\begin{figure}[t!]
\centering
\includegraphics[width=0.99\linewidth]{figures/fig4.png}
\vspace{-2mm}
\caption{
%
Effect about the estimation of continuous camera poses (ECP) and dynamic Gaussians (EDG).
%
Static areas are marked with yellow boxes while dynamic ones are marked with red boxes.
%
Without ECP and EDG, the results are almost as blurry as the input frame, as shown in (b).
%
Only with ECP, static areas are sharp but may lead to visual artifacts in dynamic areas, as shown in (c).
%
Further introducing EDG improves this, as shown in (d).
%
}
\label{fig:ecp_edg}
\vspace{-4mm}
\end{figure}
\section{Ablation Study}
%
We first discuss the estimation of continuous camera poses (ECP) and dynamic Gaussians (EDG).
%
Then, we validate the effect of regularization terms, as well as the blur-aware variable (BAV) canonical Gaussians.
%
% .
%
\subsection{Effect of ECP and EDG}
%
ECP and EDG are introduced to process camera motion blur and object motion blur, respectively.
%
Visual comparisons are shown in \cref{fig:ecp_edg}, and quantitative results are in the \textit{Suppl}.
%
First, without ECP and EDG, the results are almost as blurry as the input frame, as shown in \cref{fig:ecp_edg}(b).
%
%
Second, only with ECP, the static areas are sharp but may lead to visual artifacts in dynamic areas, as shown in \cref{fig:ecp_edg}(c).
%
It is because ECP cannot simulate the object movement.
%
Third, we further introduce EDG to simulate that produces visually pleasant results in both areas, as shown in \cref{fig:ecp_edg}(d).
%
\subsection{Effect of Regularization Terms}
%
The effect of exposure regularization $\mathcal{L}_{e}$ (see \cref{eq:L_w}, multi-frame consistency regularization $\mathcal{L}_{mfc}$ (see \cref{eq:L_cc}) and multi-resolution consistency regularization $\mathcal{L}_{mrc}$ (see \cref{eq:L_sc}) are shown in \cref{tab:regularization_terms}.
%
Without these regularization terms, the performance drops significantly and the visual results in \cref{fig:borken_solution} show that dynamic areas are with notable artifacts.
%
By regularizing the object motion within the exposure time distinguished, $\mathcal{L}_{e}$ alleviates the trivial solutions and improves performance.
%
Besides, $\mathcal{L}_{mfc}$ and $\mathcal{L}_{mrc}$ additionally regularize multi-frame and multi-resolution consistency respectively, helping to alleviate artifacts.
%
Their combinations perform best.
%
\subsection{Effect of BAV Canonical Gaussians.}
%
The effect of BAV canonical Gaussians is summarized in \cref{tab:blur_awrae_canonical}.
%
First, selecting a single canonical Gaussians across the entire video (\ie, None) leads to poor performance, due to the challenge of modeling large object motion.
%
Second, selecting variable canonical Gaussians uniformly (\ie, w/o Blur-Aware) alleviates the challenge, thus leading to performance improvement.
%
We also experiment with an optical flow-based strategy~\cite{shawswings} to select the canonical Gaussians, and it achieves a similar performance to the uniform selection.
%
This may be due to the inaccurate estimation of optical flow from blurry images.
%
Third, our blur-aware selection is better, as the canonical Gaussians corresponding to the sharper frame help blur removal.
%
\begin{table}
\centering
\caption{Ablation studies on regularization terms (see \cref{eq:regularization}).}
\vspace{-2mm}
\small
\scalebox{1.0}{\begin{tabular}{c c c c}
\toprule
$\mathcal{L}_{e}$ & $\mathcal{L}_{mfc}$ & $\mathcal{L}_{mrc}$ & PSNR$\uparrow$ / SSIM$\uparrow$ / LPIPS$\downarrow$\\
\midrule
$\times$&$\times$ & $\times$ & 28.69 / 0.947 / 0.063\\
$\times$& $\checkmark$ & $\checkmark$ & 28.86 / 0.949 / 0.060 \\
\midrule
$\checkmark$ & $\times$& $\times$ & 28.73 / 0.947 / 0.060\\
$\checkmark$ & $\checkmark$ & $\times$ & 28.87 / 0.948 / 0.060\\
$\checkmark$ & $\times$& $\checkmark$ & 28.86 / 0.948 / 0.060\\
\midrule
$\checkmark$ & $\checkmark$ & $\checkmark$& 28.92 / 0.949 / 0.060 \\
\bottomrule
\end{tabular}}
\label{tab:regularization_terms}
\vspace{-1mm}
\end{table}
\begin{table}
\centering
\caption{Effect of blur-aware variable (BAV) canonical Gaussians.
%
`None' denotes selecting a single one across the entire video.
}
\small
\vspace{-2mm}
\scalebox{1.0}{\begin{tabular}{c c c}
\toprule
Methods & PSNR$\uparrow$ / SSIM$\uparrow$ / LPIPS$\downarrow$\\
\midrule
None & 28.67 / 0.946 / 0.065\\
w/o Blur-Aware & 28.85 / 0.949 / 0.062\\
\midrule
Ours & 28.92 / 0.949 / 0.060\\
\bottomrule
\end{tabular}}
\label{tab:blur_awrae_canonical}
\vspace{-4mm}
\end{table}
\section{Conclusions}
%
In this work, we propose Deblur4DGS, the first 4D Gaussian Splatting framework to reconstruct a high-quality 4D model from blurry monocular video.
%
In particular, with the explicit motion trajectory modeling based on 3D Gaussian Splatting, we propose to transform the challenging continuous dynamic representation estimation within an exposure time into the exposure time estimation, where a series of regularizations are suggested to tackle the under-constrained optimization.
%
Besides, a blur-aware variable canonical Gaussians is present to represent objects with large motion better.
%
Beyond novel-view synthesis, Deblur4DGS can improve blurry video quality from multiple perspectives, including deblurring, frame interpolation, and video stabilization.
% %
%
Extensive results show Deblur4DGS outperforms state-of-the-art 4D reconstruction methods.
% %
\clearpage
\setcounter{page}{9}
\setcounter{section}{0}
\setcounter{figure}{0}
\setcounter{table}{0}
\setcounter{equation}{0}
\maketitlesupplementary
\renewcommand{\thesection}{\Alph{section}}
\renewcommand{\thetable}{\Alph{table}}
\renewcommand{\thefigure}{\Alph{figure}}
\renewcommand{\theequation}{\Alph{equation}}
\noindent{The content of the supplementary material involves:}
\vspace{2mm}
\begin{itemize}
\item Structure of camera motion predictor in Sec.~\ref{sec:camera_motion_predictor}.
\vspace{2mm}
\item Other regularization terms in Sec.~\ref{sec:more_regularization_terms}.
\vspace{2mm}
\item More result comparisons in Sec.~\ref{sec:more_result_comparisons}.
\vspace{2mm}
\item Effect of ECP and EDG in Sec.~\ref{sec:ECP_EDG}.
\vspace{2mm}
\item Effect of regularization terms in Sec.~\ref{sec:regularization_terms}.
\vspace{2mm}
\item Effect of BAV canonical Gaussians in Sec.~\ref{sec:BAV}.
\vspace{2mm}
\end{itemize}
\section{Structure of Camera Motion Predictor}
\label{sec:camera_motion_predictor}
%
The structure of camera motion predictor is provided in~\cref{fig:camera_motion_predictor}.
%
It first embeds the camera pose $\mathbf{P}_{t}$ to a higher dimensional space, following~\cite{mildenhall2021nerf}.
%
Then, three FC blocks are stacked, each consisting of an FC layer followed by a ReLU operation.
%
Finally, we deploy two heads to predict the camera pose at exposure start and end (\ie, $\mathbf{P}_{t,1}$ and $\mathbf{P}_{t,N}$), respectively.
\section{Other Regularization Terms}
\label{sec:more_regularization_terms}
%
Following Shape-of-Motion~\cite{wang2024shape}, we use some other regularization terms $\mathcal{L}_{oth}$ to help reconstruct 3D motion better, including mask regularization $\mathcal{L}_{mask}$, 2D tracks regularization $\mathcal{L}_{track}$ and distance-preserving regularization $\mathcal{L}_{rigid}$.
%
%
Specifically, we render the masks within the exposure time $\{\hat{\mathbf{M}}_{t, i}\}_{i=1}^{N}$ to indicate dynamic areas.
%
To supervise the training, we synthesize the mask $\hat{\mathbf{M}}_{t}^{B}$ for the synthetic blurry image $\hat{\mathbf{B}}_{t}$ as,
\begin{equation}
\label{eq:syn_mask}
\hat{\mathbf{M}}_{t}^{B}(u,v) = \texttt{max}\{\hat{\mathbf{M}}_{t, 1}(u,v), \hat{\mathbf{M}}_{t, 2}(u,v),..., \hat{\mathbf{M}}_{t, N}(u,v)\}.
\end{equation}
%
$(u,v)$ is the pixel location.
%
The mask regularization $\mathcal{L}_{mask}$ can be written as,
\begin{equation}
\label{eq:mask_reg}
\mathcal{L}_{mask} = \mathcal{L}_{1}(\hat{\mathbf{M}}_{t}^{B}, \mathbf{M}_{t}),
\end{equation}
%
where $\mathbf{M}_{t}$ is the mask that obtained by applying SAM2~\cite{ravi2024sam2} to the ground truth blurry frame.
%
Besides, we render the 2D tracks $\hat{\mathbf{U}}_{t \rightarrow t^\prime}$ from a pair of randomly sampled query time $t$ and target time $t^\prime$.
%
We supervise it by the lifted long-range 2D tracks $\mathbf{U}_{t \rightarrow t^\prime}$ that are extracted from TAPIR~\cite{doersch2023tapir}, \ie,
\begin{equation}
\label{eq:track_reg}
\mathcal{L}_{track} = \mathcal{L}_{1}(\hat{\mathbf{U}}_{t \rightarrow t^\prime}, \mathbf{U}_{t \rightarrow t^\prime}).
\end{equation}
%
Finally, we enforce a distance-preserving loss $\mathcal{L}_{rigid}$ between randomly sampled dynamic Gaussians and their $J$-nearest neighbors.
%
Let $\mathbf{x}_{t}$ and $\mathbf{x}_{t^\prime}$ denote the position of a Gaussian at time $t$ and $t^\prime$, and $\mathcal{C}_{J}(\mathbf{x}_{t})$ denote the set of $J$-nearest neighbors of $\mathbf{x}_{t}$.
%
$\mathcal{L}_{rigid}$ can be written as,
\begin{equation}
\label{eq:rigid_reg}
\mathcal{L}_{rigid} = ||\texttt{dist} (\hat{\mathbf{x}}_{t}, \mathcal{C}_{J}(\mathbf{x}_{t})) - \texttt{dist} (\hat{\mathbf{x}}_{t^\prime}, \mathcal{C}_{J}(\mathbf{x}_{t^\prime})) ||_{2}^{2}.
\end{equation}
%
$\texttt{dist}$ measures the Euclidean distance.
%
%
\begin{figure}[t!]
\centering
\includegraphics[width=0.99\linewidth]{figures/camera_mlp.pdf}
\caption{
Structure of camera motion predictor.
}
\label{fig:camera_motion_predictor}
\vspace{-2mm}
\end{figure}
%
Overall, $\mathcal{L}_{oth}$ can be written as,
\begin{equation}
\label{eq:rigid_reg}
\mathcal{L}_{oth} = \lambda_{mask}\mathcal{L}_{mask} + \lambda_{track}\mathcal{L}_{track} + \lambda_{rigid}\mathcal{L}_{rigid}.
\end{equation}
%
$\lambda_{mask}$, $\lambda_{track}$ and $\lambda_{rigid}$ are set to 1, 2, and 2, respectively.
\section{More Result Comparisons}
\label{sec:more_result_comparisons}
Due to calibrated camera parameters are not absolutely accurate, the rendering results may be slightly spatially misaligned with the ground truth.
%
In the main text, we calculate the metrics (\ie, PSNR~\cite{huynh2008scope}, SSIM~\cite{wang2004image} and LIPIS~\cite{zhang2018unreasonable}) between the aligned rendering results and the ground truth.
%
Here, we provide the novel-view synthesis and deblurring comparisons that directly calculate metrics between the rendering results and the ground truth, as shown in \cref{tab:quant_novel} and \cref{tab:quant_deblur} respectively.
%
It shows that Deblur4DGS still outperforms state-of-the-art 4D reconstruction-based methods.
%
More visual results for novel view synthesis in \cref{fig:vis_supp} show that our method produces more photo-realistic details.
%
%
Moreover, visual results for deblurring in \cref{fig:vis_deblurring} show that compared with 4D reconstruction methods, Deblur4DGS produces sharper contents and fewer visual artifacts.
%
Furthermore, the frame interpolation and video stabilization results, as well as some novel view synthesis videos are shown in the \href{https://deblur4dgs.github.io/}{site}.
%
\section{Effect of ECP and EDG}
\label{sec:ECP_EDG}
%
The visual results are provided in the main text.
%
Here we provide the quantitative results in \cref{tab:ect_oct}.
%
Without ECP and EDG, the reconstruction results are unsatisfactory due to motion blur.
%
ECP and EDG enable processing of the camera motion blur and object motion blur respectively, thus leading to performance improvement.
% %
\section{Effect of regularization terms}
\label{sec:regularization_terms}
%
We validate the effect of exposure regularization $\mathcal{L}_{e}$, multi-frame consistency regularization $\mathcal{L}_{mfc}$ and multi-resolution consistency regularization $\mathcal{L}_{mrc}$.
%
In the main text, we provide the results on $288 \times 512$ data.
%
Here, we provide that on $720 \times 1080$ data in \cref{tab:regularization_terms}.
%
The results are consistent for both data.
%
$\mathcal{L}_{e}$ regularizes the object motion within the exposure time distinguished, alleviating the trivial solutions and improving performance.
%
$\mathcal{L}_{mfc}$ and $\mathcal{L}_{mrc}$ regularize multi-frame and multi-resolution consistency respectively, helping to alleviate artifacts.
%
Their combinations perform best.
\section{Effect of BAV canonical Gaussians}
\label{sec:BAV}
%
\cref{tab:blur_awrae_canonical} additionally provides the quantitative results on $720 \times 1080$ data, which are consistent with the ones on $288 \times 512$ data.
%
Compared with selecting a single one across the entire video (\ie, None), selecting variable canonical Gaussians uniformly (\ie, w/o Blur-Aware) alleviates the challenge of modeling large object motion, thus leading to performance improvement.
%
Our method selects the canonical Gaussians corresponding to the sharper frame, which is beneficial to blur removal and performs better.
\vspace{60mm}
\begin{table}[t!]
\small
\renewcommand\arraystretch{1}
\begin{center}
\caption{Quantitative comparisons for \textbf{novel view synthesis}.
%
The metrics are directly calculated between rendering results and ground truth.
%
The best is \textbf{boldfaced}, and the second is \underline{underlined}.
}
\label{tab:quant_novel}
\vspace{-2mm}
\scalebox{0.8}{
\begin{tabular}{lcccc}
\toprule
Methods & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$ } & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$} &\\
\midrule
DeformableGS~\cite{yang2024deformable} & 16.32 / 0.644 / 0.356 & 16.06 / 0.711 / 0.424\\
4DGaussians~\cite{4dgaussians} & 20.58 / 0.795 / 0.186 & 20.31 / 0.821 / 0.258 \\
E-D3DGS~\cite{bae2024per} & 20.99 / 0.804 / 0.179&20.95 / 0.829 / 0.257\\
DyBluRF~\cite{sun2024dyblurf} & 23.39 / 0.823 / \underline{0.088} & 22.59 / 0.815 / \underline{0.168}\\
Shape-of-Motion~\cite{wang2024shape} &\underline{26.47} / \underline{0.908} / 0.104 &\underline{25.76} / \underline{0.896} / 0.195\\
\midrule
Deblur4DGS (Ours) & \textbf{26.69} / \textbf{0.918} / \textbf{0.052}& \textbf{25.93} / \textbf{0.904} / \textbf{0.102}\\
\bottomrule
\end{tabular}
}
\end{center}
\end{table}
\begin{table}[t!]
\small
\renewcommand\arraystretch{1}
\begin{center}
\caption{Quantitative comparisons for \textbf{deblurring}.
%
The metrics are directly calculated between rendering results and ground truth.
%
The first category is supervised image or video deblurring methods, and the second is 4D reconstruction-based ones.
%
}
\label{tab:quant_deblur}
\vspace{-2mm}
\scalebox{0.8}{
\begin{tabular}{lcccc}
\toprule
Methods & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$} & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$} &\\
\midrule
Restormer~\cite{zamir2022restormer} & \underline{36.74} / \underline{0.987} / 0.019 &\textbf{35.97} / \underline{0.981} / 0.028\\
DSTNet~\cite{ Pan_2023_CVPR} & 36.49 / 0.986 / \underline{0.015} & 35.24 / 0.977 / \underline{0.027}\\
BSSTNet~\cite{zhang2024blur} & \textbf{36.88} / \textbf{0.988} / \textbf{0.013}& 35.87 / \textbf{0.983} / \textbf{0.021}\\
\midrule
DeformableGS~\cite{yang2024deformable} & 30.97 / 0.950 / 0.096 & 28.95 / 0.924 / 0.189\\
4DGaussians~\cite{4dgaussians} & 31.30 / \underline{0.951} / 0.090 & \underline{29.76} / 0.929 / 0.167\\
E-D3DGS~\cite{bae2024per} & \underline{31.34} / \underline{0.951} / 0.091 & 29.74 / 0.929 / 0.170\\
DyBluRF~\cite{sun2024dyblurf} & 29.32 / 0.942 / \underline{0.049} & 28.67 / \underline{0.936} / \underline{0.101}\\
Shape-of-Motion~\cite{wang2024shape} & 29.51 / 0.945 / 0.095 & 28.35 / 0.927 / 0.187 \\
Deblur4DGS (Ours) & \textbf{31.77} / \textbf{0.972} / \textbf{0.033}& \textbf{30.48} / \textbf{0.956} / \textbf{0.081}\\
\bottomrule
\end{tabular}
}
\end{center}
\end{table}
\begin{table}
\centering
\caption{Effect about the estimation of continuous camera poses (ECP) and dynamic Gaussians (EDG).}
\vspace{-2mm}
\scalebox{0.8}{\begin{tabular}{c c c c}
\toprule
ECP & EDG & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$} & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$}\\
\midrule
$\times$ & $\times$ & 28.00 / 0.929 / 0.119 & 27.70 / 0.923 / 0.191\\
$\checkmark$ & $\times$ & 28.86 / 0.948 / 0.062 & 28.68 / 0.945 / 0.100\\
\midrule
$\checkmark$ & $\checkmark$ & 28.92 / 0.949 / 0.060 & 28.88 / 0.947 / 0.098\\
\bottomrule
\end{tabular}}
\label{tab:ect_oct}
\end{table}
\begin{table}
\centering
\caption{Ablation studies on regularization terms.}
\vspace{-2mm}
\small
\scalebox{0.8}{\begin{tabular}{c c c c c}
\toprule
$\mathcal{L}_{e}$ & $\mathcal{L}_{mfc}$ & $\mathcal{L}_{mrc}$ & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$} & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$}\\
\midrule
$\times$&$\times$ & $\times$ & 28.69 / 0.947 / 0.063 & 28.70 / 0.946 / 0.099\\
$\times$& $\checkmark$ & $\checkmark$ & 28.86 / 0.949 / 0.060 & 28.86 / 0.947 / 0.098\\
\midrule
$\checkmark$ & $\times$& $\times$ & 28.73 / 0.947 / 0.060 & 28.74 / 0.945 / 0.099\\
$\checkmark$ & $\checkmark$ & $\times$ & 28.87 / 0.948 / 0.060 & 28.78 / 0.946 / 0.098\\
$\checkmark$ & $\times$& $\checkmark$ & 28.86 / 0.948 / 0.060 & 28.79 / 0.946 / 0.099\\
\midrule
$\checkmark$ & $\checkmark$ & $\checkmark$& 28.92 / 0.949 / 0.060 & 28.88 / 0.947 / 0.098 \\
\bottomrule
\end{tabular}}
\label{tab:regularization_terms}
\vspace{-1mm}
\end{table}
\begin{table}
\centering
\caption{Effect of blur-aware variable (BAV) canonical Gaussians.
%
`None' denotes selecting a single one across the entire video.
}
\small
\vspace{-2mm}
\scalebox{0.8}{\begin{tabular}{c c c c}
\toprule
Methods & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $288 \times 512$} & \tabincell{c}{PSNR$\uparrow$/SSIM$\uparrow$/LPIPS$\downarrow$ \\ $720 \times 1080$}\\
\midrule
None & 28.67 / 0.946 / 0.065 & 28.66 / 0.945 / 0.104\\
w/o Blur-Aware & 28.85 / 0.949 / 0.062 & 28.83 / 0.947 / 0.098\\
\midrule
Ours & 28.92 / 0.949 / 0.060 & 28.88 / 0.947 / 0.098\\
\bottomrule
\end{tabular}}
\label{tab:blur_awrae_canonical}
\vspace{-4mm}
\end{table}
\begin{figure*}[t!]
\centering
\begin{subfigure}{0.99\textwidth}
\ContinuedFloat
\begin{overpic}[width=0.99\textwidth,grid=False]
{figures/novel_vis.png}
\put(50,-9){{\fontsize{7}{7}\selectfont Blury Frame}}
\put(130,-9){{\fontsize{7}{7}\selectfont DeformableGS~\cite{yang2024deformable}}}
\put(190,-9){{\fontsize{7}{7}\selectfont 4DGaussians~\cite{4dgaussians}}}
\put(243,-9){{\fontsize{7}{7}\selectfont E-D3DGS~\cite{bae2024per}}}
\put(287,-9){{\fontsize{7}{7}\selectfont DybluRF~\cite{sun2024dyblurf}}}
\put(330,-9){{\fontsize{7}{7}\selectfont Shape-of-Motion~\cite{wang2024shape}}}
\put(407,-9){{\fontsize{7}{7}\selectfont Ours}}
\put(450,-9){{\fontsize{7}{7}\selectfont Sharp GT}}
\end{overpic}
\vspace{2mm}
\end{subfigure}
\caption{Visual comparisons of novel-view synthesis on the $720 \times 1080 $ images.
%
Our method produces more photo-realistic details in both static and dynamic areas, as marked with yellow and red boxes respectively.
}
\label{fig:vis_supp}
\vspace{-3mm}
\end{figure*}
\begin{figure*}[t!]
\centering
\begin{subfigure}{0.99\textwidth}
\ContinuedFloat
\begin{overpic}[width=0.99\textwidth,grid=False]
{figures/deblur_vis.png}
\put(50,-9){{\fontsize{7}{7}\selectfont Blury Frame}}
\put(130,-9){{\fontsize{7}{7}\selectfont DeformableGS~\cite{yang2024deformable}}}
\put(190,-9){{\fontsize{7}{7}\selectfont 4DGaussians~\cite{4dgaussians}}}
\put(243,-9){{\fontsize{7}{7}\selectfont E-D3DGS~\cite{bae2024per}}}
\put(287,-9){{\fontsize{7}{7}\selectfont DybluRF~\cite{sun2024dyblurf}}}
\put(330,-9){{\fontsize{7}{7}\selectfont Shape-of-Motion~\cite{wang2024shape}}}
\put(407,-9){{\fontsize{7}{7}\selectfont Ours}}
\put(450,-9){{\fontsize{7}{7}\selectfont Sharp GT}}
\end{overpic}
\vspace{2mm}
\end{subfigure}
\caption{Visual comparisons of deblurring on the $720 \times 1080 $ images.
%
Compared with 4D reconstruction-based methods, Deblur4DGS produces sharper contents and fewer artifacts in both static and dynamic areas, as marked with yellow and red boxes respectively.
}
\label{fig:vis_deblurring}
\vspace{-3mm}
\end{figure*}
\clearpage
\begin{thebibliography}{108}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Bae et~al.(2024)Bae, Kim, Yun, Lee, Bang, and Uh]{bae2024per}
Jeongmin Bae, Seoha Kim, Youngsik Yun, Hahyun Lee, Gun Bang, and Youngjung Uh.
\newblock Per-gaussian embedding-based deformation for deformable 3d gaussian splatting.
\newblock \emph{arXiv preprint arXiv:2404.03613}, 2024.
\bibitem[Bansal et~al.(2016)Bansal, Raj, and Choudhury]{bansal2016blur}
Raghav Bansal, Gaurav Raj, and Tanupriya Choudhury.
\newblock Blur image detection using laplacian operator and open-cv.
\newblock In \emph{2016 International Conference System Modeling \& Advancement in Research Trends (SMART)}, pages 63--67. IEEE, 2016.
\bibitem[Bhat et~al.(2021)Bhat, Danelljan, Van~Gool, and Timofte]{bhat2021deep}
Goutam Bhat, Martin Danelljan, Luc Van~Gool, and Radu Timofte.
\newblock Deep burst super-resolution.
\newblock In \emph{CVPR}, 2021.
\bibitem[Bui et~al.(2023)Bui, Park, Oh, and Kim]{bui2023dyblurf}
Minh-Quan~Viet Bui, Jongmin Park, Jihyong Oh, and Munchurl Kim.
\newblock Dyblurf: Dynamic deblurring neural radiance fields for blurry monocular video.
\newblock \emph{arXiv preprint arXiv:2312.13528}, 2023.
\bibitem[Cao and Johnson(2023)]{cao2023hexplane}
Ang Cao and Justin Johnson.
\newblock Hexplane: A fast representation for dynamic scenes.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 130--141, 2023.
\bibitem[Chan et~al.(2022)Chan, Zhou, Xu, and Loy]{chan2022basicvsr++}
Kelvin~CK Chan, Shangchen Zhou, Xiangyu Xu, and Chen~Change Loy.
\newblock Basicvsr++: Improving video super-resolution with enhanced propagation and alignment.
\newblock In \emph{Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 5972--5981, 2022.
\bibitem[Chen and Liu(2024)]{chen2024deblur}
Wenbo Chen and Ligang Liu.
\newblock Deblur-gs: 3d gaussian splatting from camera motion blurred images.
\newblock \emph{Proceedings of the ACM on Computer Graphics and Interactive Techniques}, 7\penalty0 (1):\penalty0 1--15, 2024.
\bibitem[Chu et~al.(2024)Chu, Ke, and Fragkiadaki]{chu2024dreamscene4d}
Wen-Hsuan Chu, Lei Ke, and Katerina Fragkiadaki.
\newblock Dreamscene4d: Dynamic multi-object scene generation from monocular videos.
\newblock \emph{arXiv preprint arXiv:2405.02280}, 2024.
\bibitem[Doersch et~al.(2023)Doersch, Yang, Vecerik, Gokay, Gupta, Aytar, Carreira, and Zisserman]{doersch2023tapir}
Carl Doersch, Yi Yang, Mel Vecerik, Dilara Gokay, Ankush Gupta, Yusuf Aytar, Joao Carreira, and Andrew Zisserman.
\newblock Tapir: Tracking any point with per-frame initialization and temporal refinement.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 10061--10072, 2023.
\bibitem[Duan et~al.(2024{\natexlab{a}})Duan, Wei, Dai, He, Chen, and Chen]{4d-rotor20244d}
Yuanxing Duan, Fangyin Wei, Qiyu Dai, Yuhang He, Wenzheng Chen, and Baoquan Chen.
\newblock 4d-rotor gaussian splatting: towards efficient novel view synthesis for dynamic scenes.
\newblock In \emph{ACM SIGGRAPH 2024 Conference Papers}, pages 1--11, 2024{\natexlab{a}}.
\bibitem[Duan et~al.(2024{\natexlab{b}})Duan, Wei, Dai, He, Chen, and Chen]{duan20244d}
Yuanxing Duan, Fangyin Wei, Qiyu Dai, Yuhang He, Wenzheng Chen, and Baoquan Chen.
\newblock 4d gaussian splatting: Towards efficient novel view synthesis for dynamic scenes.
\newblock \emph{arXiv preprint arXiv:2402.03307}, 2024{\natexlab{b}}.
\bibitem[Dudhane et~al.(2022)Dudhane, Zamir, Khan, Khan, and Yang]{dudhane2022burst}
Akshay Dudhane, Syed~Waqas Zamir, Salman Khan, Fahad~Shahbaz Khan, and Ming-Hsuan Yang.
\newblock Burst image restoration and enhancement.
\newblock In \emph{CVPR}, 2022.
\bibitem[Fang et~al.(2022)Fang, Yi, Wang, Xie, Zhang, Liu, Nie{\ss}ner, and Tian]{fang2022fast}
Jiemin Fang, Taoran Yi, Xinggang Wang, Lingxi Xie, Xiaopeng Zhang, Wenyu Liu, Matthias Nie{\ss}ner, and Qi Tian.
\newblock Fast dynamic radiance fields with time-aware neural voxels.
\newblock In \emph{SIGGRAPH Asia 2022 Conference Papers}, pages 1--9, 2022.
\bibitem[Feng et~al.(2023)Feng, Alzayer, Rubinstein, Freeman, and Huang]{feng20233d}
Brandon~Y Feng, Hadi Alzayer, Michael Rubinstein, William~T Freeman, and Jia-Bin Huang.
\newblock 3d motion magnification: Visualizing subtle motions from time-varying radiance fields.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 9837--9846, 2023.
\bibitem[Fridovich-Keil et~al.(2023)Fridovich-Keil, Meanti, Warburg, Recht, and Kanazawa]{fridovich2023k}
Sara Fridovich-Keil, Giacomo Meanti, Frederik~Rahb{\ae}k Warburg, Benjamin Recht, and Angjoo Kanazawa.
\newblock K-planes: Explicit radiance fields in space, time, and appearance.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 12479--12488, 2023.
\bibitem[Gao et~al.(2021)Gao, Saraf, Kopf, and Huang]{gao2021dynamic}
Chen Gao, Ayush Saraf, Johannes Kopf, and Jia-Bin Huang.
\newblock Dynamic view synthesis from dynamic monocular video.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 5712--5721, 2021.
\bibitem[Gao et~al.(2024)Gao, Xu, Cao, Mildenhall, Ma, Chen, Tang, and Neumann]{gao2024gaussianflow}
Quankai Gao, Qiangeng Xu, Zhe Cao, Ben Mildenhall, Wenchao Ma, Le Chen, Danhang Tang, and Ulrich Neumann.
\newblock Gaussianflow: Splatting gaussian dynamics for 4d content creation.
\newblock \emph{arXiv preprint arXiv:2403.12365}, 2024.
\bibitem[Guo et~al.(2023)Guo, Sun, Dai, Chen, Ye, Tan, Ding, Zhang, and Wang]{guo2023forward}
Xiang Guo, Jiadai Sun, Yuchao Dai, Guanying Chen, Xiaoqing Ye, Xiao Tan, Errui Ding, Yumeng Zhang, and Jingdong Wang.
\newblock Forward flow for novel view synthesis of dynamic scenes.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 16022--16033, 2023.
\bibitem[Guo et~al.(2024)Guo, Zhou, Li, Wang, and Li]{guo2024motion}
Zhiyang Guo, Wengang Zhou, Li Li, Min Wang, and Houqiang Li.
\newblock Motion-aware 3d gaussian splatting for efficient dynamic scene reconstruction.
\newblock \emph{arXiv preprint arXiv:2403.11447}, 2024.
\bibitem[Huang et~al.(2024)Huang, Sun, Yang, Lyu, Cao, and Qi]{huang2024sc}
Yi-Hua Huang, Yang-Tian Sun, Ziyi Yang, Xiaoyang Lyu, Yan-Pei Cao, and Xiaojuan Qi.
\newblock Sc-gs: Sparse-controlled gaussian splatting for editable dynamic scenes.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 4220--4230, 2024.
\bibitem[Huang et~al.(2022)Huang, Zhang, Heng, Shi, and Zhou]{huang2022real}
Zhewei Huang, Tianyuan Zhang, Wen Heng, Boxin Shi, and Shuchang Zhou.
\newblock Real-time intermediate flow estimation for video frame interpolation.
\newblock In \emph{European Conference on Computer Vision}, pages 624--642. Springer, 2022.
\bibitem[Huynh-Thu and Ghanbari(2008)]{huynh2008scope}
Quan Huynh-Thu and Mohammed Ghanbari.
\newblock Scope of validity of psnr in image/video quality assessment.
\newblock \emph{Electronics letters}, 44\penalty0 (13):\penalty0 800--801, 2008.
\bibitem[Hyun~Kim et~al.(2017)Hyun~Kim, Mu~Lee, Scholkopf, and Hirsch]{hyun2017online}
Tae Hyun~Kim, Kyoung Mu~Lee, Bernhard Scholkopf, and Michael Hirsch.
\newblock Online video deblurring via dynamic temporal blending network.
\newblock In \emph{Proceedings of the IEEE international conference on computer vision}, pages 4038--4047, 2017.
\bibitem[Jiang et~al.(2022)Jiang, Xie, Xia, Li, and Liu]{jiang2022erdn}
Bangrui Jiang, Zhihuai Xie, Zhen Xia, Songnan Li, and Shan Liu.
\newblock Erdn: Equivalent receptive field deformable network for video deblurring.
\newblock In \emph{European Conference on Computer Vision}, pages 663--678. Springer, 2022.
\bibitem[Katsumata et~al.(2024)Katsumata, Vo, and Nakayama]{katsumata2024compact}
Kai Katsumata, Duc~Minh Vo, and Hideki Nakayama.
\newblock A compact dynamic 3d gaussian representation for real-time dynamic view synthesis.
\newblock In \emph{ECCV}, 2024.
\bibitem[Ke et~al.(2021)Ke, Wang, Wang, Milanfar, and Yang]{ke2021musiq}
Junjie Ke, Qifei Wang, Yilin Wang, Peyman Milanfar, and Feng Yang.
\newblock Musiq: Multi-scale image quality transformer.
\newblock In \emph{Proceedings of the IEEE/CVF international conference on computer vision}, pages 5148--5157, 2021.
\bibitem[Kerbl et~al.(2023)Kerbl, Kopanas, Leimk{\"u}hler, and Drettakis]{kerbl20233d}
Bernhard Kerbl, Georgios Kopanas, Thomas Leimk{\"u}hler, and George Drettakis.
\newblock 3d gaussian splatting for real-time radiance field rendering.
\newblock \emph{ACM Trans. Graph.}, 42\penalty0 (4):\penalty0 139--1, 2023.
\bibitem[Kim et~al.(2022)Kim, Lee, and Cho]{kim2022mssnet}
Kiyeon Kim, Seungyong Lee, and Sunghyun Cho.
\newblock Mssnet: Multi-scale-stage network for single image deblurring.
\newblock In \emph{European conference on computer vision}, pages 524--539. Springer, 2022.
\bibitem[Kong et~al.(2023)Kong, Dong, Ge, Li, and Pan]{kong2023efficient}
Lingshun Kong, Jiangxin Dong, Jianjun Ge, Mingqiang Li, and Jinshan Pan.
\newblock Efficient frequency domain-based transformers for high-quality image deblurring.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 5886--5895, 2023.
\bibitem[Kratimenos et~al.(2023)Kratimenos, Lei, and Daniilidis]{kratimenos2023dynmf}
Agelos Kratimenos, Jiahui Lei, and Kostas Daniilidis.
\newblock Dynmf: Neural motion factorization for real-time dynamic view synthesis with 3d gaussian splatting.
\newblock \emph{arXiv preprint arXiv:2312.00112}, 2023.
\bibitem[Lee et~al.(2024{\natexlab{a}})Lee, Lee, Sun, Ali, and Park]{lee2024deblurring}
Byeonghyeon Lee, Howoong Lee, Xiangyu Sun, Usman Ali, and Eunbyung Park.
\newblock Deblurring 3d gaussian splatting.
\newblock \emph{arXiv preprint arXiv:2401.00834}, 2024{\natexlab{a}}.
\bibitem[Lee et~al.(2023{\natexlab{a}})Lee, Lee, Shin, and Lee]{lee2023dp}
Dogyoon Lee, Minhyeok Lee, Chajin Shin, and Sangyoun Lee.
\newblock Dp-nerf: Deblurred neural radiance field with physical scene priors.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 12386--12396, 2023{\natexlab{a}}.
\bibitem[Lee et~al.(2023{\natexlab{b}})Lee, Oh, Rim, Cho, and Lee]{lee2023exblurf}
Dongwoo Lee, Jeongtaek Oh, Jaesung Rim, Sunghyun Cho, and Kyoung~Mu Lee.
\newblock Exblurf: Efficient radiance fields for extreme motion blurred images.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 17639--17648, 2023{\natexlab{b}}.
\bibitem[Lee et~al.(2024{\natexlab{b}})Lee, Kim, Lee, Cho, and Lee]{lee2024crim}
Junghe Lee, Donghyeong Kim, Dogyoon Lee, Suhwan Cho, and Sangyoun Lee.
\newblock Crim-gs: Continuous rigid motion-aware gaussian splatting from motion blur images.
\newblock \emph{arXiv preprint arXiv:2407.03923}, 2024{\natexlab{b}}.
\bibitem[Lee et~al.(2023{\natexlab{c}})Lee, Zhang, Blackburn-Matzen, Niklaus, Zhang, Huang, and Liu]{lee2023fast}
Yao-Chih Lee, Zhoutong Zhang, Kevin Blackburn-Matzen, Simon Niklaus, Jianming Zhang, Jia-Bin Huang, and Feng Liu.
\newblock Fast view synthesis of casual videos.
\newblock \emph{arXiv preprint arXiv:2312.02135}, 2023{\natexlab{c}}.
\bibitem[Lei et~al.(2024)Lei, Weng, Harley, Guibas, and Daniilidis]{lei2024mosca}
Jiahui Lei, Yijia Weng, Adam Harley, Leonidas Guibas, and Kostas Daniilidis.
\newblock Mosca: Dynamic gaussian fusion from casual videos via 4d motion scaffolds.
\newblock \emph{arXiv preprint arXiv:2405.17421}, 2024.
\bibitem[Li et~al.(2022)Li, Zhang, Cheung, Wang, Qin, and Li]{li2022learning}
Dasong Li, Yi Zhang, Ka~Chun Cheung, Xiaogang Wang, Hongwei Qin, and Hongsheng Li.
\newblock Learning degradation representations for image deblurring.
\newblock In \emph{European conference on computer vision}, pages 736--753. Springer, 2022.
\bibitem[Li et~al.(2023{\natexlab{a}})Li, Wang, Nan, and Ji]{li2023self}
Ji Li, Weixi Wang, Yuesong Nan, and Hui Ji.
\newblock Self-supervised blind motion deblurring with deep expectation maximization.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 13986--13996, 2023{\natexlab{a}}.
\bibitem[Li et~al.(2023{\natexlab{b}})Li, Wang, Zhao, Liao, and Liu]{li2023usb}
Moyang Li, Peng Wang, Lingzhe Zhao, Bangyan Liao, and Peidong Liu.
\newblock Usb-nerf: Unrolling shutter bundle adjusted neural radiance fields.
\newblock \emph{arXiv preprint arXiv:2310.02687}, 2023{\natexlab{b}}.
\bibitem[Li et~al.(2024)Li, Chen, Li, and Xu]{li2024spacetime}
Zhan Li, Zhang Chen, Zhong Li, and Yi Xu.
\newblock Spacetime gaussian feature splatting for real-time dynamic view synthesis.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 8508--8520, 2024.
\bibitem[Liang et~al.(2022)Liang, Fan, Xiang, Ranjan, Ilg, Green, Cao, Zhang, Timofte, and Gool]{liang2022recurrent}
Jingyun Liang, Yuchen Fan, Xiaoyu Xiang, Rakesh Ranjan, Eddy Ilg, Simon Green, Jiezhang Cao, Kai Zhang, Radu Timofte, and Luc~V Gool.
\newblock Recurrent video restoration transformer with guided deformable attention.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 378--393, 2022.
\bibitem[Liang et~al.(2024)Liang, Cao, Fan, Zhang, Ranjan, Li, Timofte, and Van~Gool]{liang2024vrt}
Jingyun Liang, Jiezhang Cao, Yuchen Fan, Kai Zhang, Rakesh Ranjan, Yawei Li, Radu Timofte, and Luc Van~Gool.
\newblock Vrt: A video restoration transformer.
\newblock \emph{IEEE Transactions on Image Processing}, 2024.
\bibitem[Lin et~al.(2024{\natexlab{a}})Lin, Zhang, Wei, Ren, Jiang, Tian, and Zuo]{lin2024improving}
Jingbo Lin, Zhilu Zhang, Yuxiang Wei, Dongwei Ren, Dongsheng Jiang, Qi Tian, and Wangmeng Zuo.
\newblock Improving image restoration through removing degradations in textual representations.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 2866--2878, 2024{\natexlab{a}}.
\bibitem[Lin et~al.(2024{\natexlab{b}})Lin, Dai, Zhu, and Yao]{lin2024gaussian}
Youtian Lin, Zuozhuo Dai, Siyu Zhu, and Yao Yao.
\newblock Gaussian-flow: 4d reconstruction with dynamic 3d gaussian particle.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 21136--21145, 2024{\natexlab{b}}.
\bibitem[Liu et~al.(2022)Liu, Cao, Mao, Zhang, Zhang, Keppo, Shan, Qie, and Shou]{liu2022devrf}
Jia-Wei Liu, Yan-Pei Cao, Weijia Mao, Wenqiao Zhang, David~Junhao Zhang, Jussi Keppo, Ying Shan, Xiaohu Qie, and Mike~Zheng Shou.
\newblock Devrf: Fast deformable voxel radiance fields for dynamic scenes.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 36762--36775, 2022.
\bibitem[Liu et~al.(2024)Liu, Liu, Wang, Lv, Wang, Wang, and Hou]{liu2024modgs}
Qingming Liu, Yuan Liu, Jiepeng Wang, Xianqiang Lv, Peng Wang, Wenping Wang, and Junhui Hou.
\newblock Modgs: Dynamic gaussian splatting from causually-captured monocular videos.
\newblock \emph{arXiv preprint arXiv:2406.00434}, 2024.
\bibitem[Liu et~al.(2016)Liu, Tan, Yuan, Sun, and Zeng]{liu2016meshflow}
Shuaicheng Liu, Ping Tan, Lu Yuan, Jian Sun, and Bing Zeng.
\newblock Meshflow: Minimum latency online video stabilization.
\newblock In \emph{Computer Vision--ECCV 2016: 14th European Conference, Amsterdam, The Netherlands, October 11-14, 2016, Proceedings, Part VI 14}, pages 800--815. Springer, 2016.
\bibitem[Lu et~al.(2024)Lu, Guo, Hui, Chen, Yang, Tang, Zhu, and Dai]{lu20243d}
Zhicheng Lu, Xiang Guo, Le Hui, Tianrui Chen, Min Yang, Xiao Tang, Feng Zhu, and Yuchao Dai.
\newblock 3d geometry-aware deformable gaussian splatting for dynamic view synthesis.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 8900--8910, 2024.
\bibitem[Luiten et~al.(2023)Luiten, Kopanas, Leibe, and Ramanan]{luiten2023dynamic}
Jonathon Luiten, Georgios Kopanas, Bastian Leibe, and Deva Ramanan.
\newblock Dynamic 3d gaussians: Tracking by persistent dynamic view synthesis.
\newblock \emph{arXiv preprint arXiv:2308.09713}, 2023.
\bibitem[Ma et~al.(2022)Ma, Li, Liao, Zhang, Wang, Wang, and Sander]{ma2022deblur}
Li Ma, Xiaoyu Li, Jing Liao, Qi Zhang, Xuan Wang, Jue Wang, and Pedro~V Sander.
\newblock Deblur-nerf: Neural radiance fields from blurry images.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 12861--12870, 2022.
\bibitem[Mihajlovic et~al.(2024)Mihajlovic, Prokudin, Tang, Maier, Bogo, Tung, and Boyer]{mihajlovic2024splatfields}
Marko Mihajlovic, Sergey Prokudin, Siyu Tang, Robert Maier, Federica Bogo, Tony Tung, and Edmond Boyer.
\newblock Splatfields: Neural gaussian splats for sparse 3d and 4d reconstruction.
\newblock \emph{arXiv preprint arXiv:2409.11211}, 2024.
\bibitem[Mildenhall et~al.(2021)Mildenhall, Srinivasan, Tancik, Barron, Ramamoorthi, and Ng]{mildenhall2021nerf}
Ben Mildenhall, Pratul~P Srinivasan, Matthew Tancik, Jonathan~T Barron, Ravi Ramamoorthi, and Ren Ng.
\newblock Nerf: Representing scenes as neural radiance fields for view synthesis.
\newblock \emph{Communications of the ACM}, 65\penalty0 (1):\penalty0 99--106, 2021.
\bibitem[Nah et~al.(2017)Nah, Hyun~Kim, and Mu~Lee]{nah2017deep}
Seungjun Nah, Tae Hyun~Kim, and Kyoung Mu~Lee.
\newblock Deep multi-scale convolutional neural network for dynamic scene deblurring.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 3883--3891, 2017.
\bibitem[Nah et~al.(2019)Nah, Son, and Lee]{nah2019recurrent}
Seungjun Nah, Sanghyun Son, and Kyoung~Mu Lee.
\newblock Recurrent neural networks with intra-frame iterations for video deblurring.
\newblock In \emph{Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 8102--8111, 2019.
\bibitem[Oh et~al.(2024)Oh, Chung, Lee, and Lee]{oh2024deblurgs}
Jeongtaek Oh, Jaeyoung Chung, Dongwoo Lee, and Kyoung~Mu Lee.
\newblock Deblurgs: Gaussian splatting for camera motion blur.
\newblock \emph{arXiv preprint arXiv:2404.11358}, 2024.
\bibitem[Pan et~al.(2023)Pan, Xu, Dong, Ge, and Tang]{Pan_2023_CVPR}
Jinshan Pan, Boming Xu, Jiangxin Dong, Jianjun Ge, and Jinhui Tang.
\newblock Deep discriminative spatial and temporal network for efficient video deblurring.
\newblock In \emph{The IEEE Conference on Computer Vision and Pattern Recognition(CVPR)}, 2023.
\bibitem[Paszke et~al.(2019)Paszke, Gross, Massa, Lerer, Bradbury, Chanan, Killeen, Lin, Gimelshein, Antiga, et~al.]{paszke2019pytorch}
Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, et~al.
\newblock Pytorch: An imperative style, high-performance deep learning library.
\newblock \emph{Advances in neural information processing systems}, 32, 2019.
\bibitem[Peng et~al.(2024{\natexlab{a}})Peng, Tang, Zhou, Wang, Liu, Li, and Chellappa]{peng2024bags}
Cheng Peng, Yutao Tang, Yifan Zhou, Nengyu Wang, Xijun Liu, Deming Li, and Rama Chellappa.
\newblock Bags: Blur agnostic gaussian splatting through multi-scale kernel modeling.
\newblock \emph{arXiv preprint arXiv:2403.04926}, 2024{\natexlab{a}}.
\bibitem[Peng et~al.(2024{\natexlab{b}})Peng, Ye, Zhao, Liu, Sun, Li, and Cao]{peng20243d}
Zhan Peng, Xinyi Ye, Weiyue Zhao, Tianqi Liu, Huiqiang Sun, Baopu Li, and Zhiguo Cao.
\newblock 3d multi-frame fusion for video stabilization.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 7507--7516, 2024{\natexlab{b}}.
\bibitem[Pumarola et~al.(2021)Pumarola, Corona, Pons-Moll, and Moreno-Noguer]{pumarola2021d}
Albert Pumarola, Enric Corona, Gerard Pons-Moll, and Francesc Moreno-Noguer.
\newblock D-nerf: Neural radiance fields for dynamic scenes.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 10318--10327, 2021.
\bibitem[Ravi et~al.(2024)Ravi, Gabeur, Hu, Hu, Ryali, Ma, Khedr, R{\"a}dle, Rolland, Gustafson, Mintun, Pan, Alwala, Carion, Wu, Girshick, Doll{\'a}r, and Feichtenhofer]{ravi2024sam2}
Nikhila Ravi, Valentin Gabeur, Yuan-Ting Hu, Ronghang Hu, Chaitanya Ryali, Tengyu Ma, Haitham Khedr, Roman R{\"a}dle, Chloe Rolland, Laura Gustafson, Eric Mintun, Junting Pan, Kalyan~Vasudev Alwala, Nicolas Carion, Chao-Yuan Wu, Ross Girshick, Piotr Doll{\'a}r, and Christoph Feichtenhofer.
\newblock Sam 2: Segment anything in images and videos.
\newblock \emph{arXiv preprint arXiv:2408.00714}, 2024.
\bibitem[Ren et~al.(2023)Ren, Delbracio, Talebi, Gerig, and Milanfar]{ren2023multiscale}
Mengwei Ren, Mauricio Delbracio, Hossein Talebi, Guido Gerig, and Peyman Milanfar.
\newblock Multiscale structure guided diffusion for image deblurring.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 10721--10733, 2023.
\bibitem[Ren et~al.(2020)Ren, Qian, and Chen]{ren2020video}
Xuanchi Ren, Zian Qian, and Qifeng Chen.
\newblock Video deblurring by fitting to test data.
\newblock \emph{arXiv preprint arXiv:2012.05228}, 2020.
\bibitem[Schonberger and Frahm(2016)]{schonberger2016structure}
Johannes~L Schonberger and Jan-Michael Frahm.
\newblock Structure-from-motion revisited.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 4104--4113, 2016.
\bibitem[Seidenschwarz et~al.(2024)Seidenschwarz, Zhou, Duisterhof, Ramanan, and Leal-Taix{\'e}]{seidenschwarz2024dynomo}
Jenny Seidenschwarz, Qunjie Zhou, Bardienus Duisterhof, Deva Ramanan, and Laura Leal-Taix{\'e}.
\newblock Dynomo: Online point tracking by dynamic online monocular gaussian reconstruction.
\newblock \emph{arXiv preprint arXiv:2409.02104}, 2024.
\bibitem[Shang et~al.(2023)Shang, Ren, Yang, Zhang, Ma, and Zuo]{shang2023joint}
Wei Shang, Dongwei Ren, Yi Yang, Hongzhi Zhang, Kede Ma, and Wangmeng Zuo.
\newblock Joint video multi-frame interpolation and deblurring under unknown exposure time.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 13935--13944, 2023.
\bibitem[Shaw et~al.()Shaw, Nazarczuk, Song, Moreau, Catley-Chandar, Dhamo, and P{\'e}rez-Pellitero]{shawswings}
Richard Shaw, Michal Nazarczuk, Jifei Song, Arthur Moreau, Sibi Catley-Chandar, Helisa Dhamo, and Eduardo P{\'e}rez-Pellitero.
\newblock Swings: Sliding windows for dynamic 3d gaussian splatting.
\bibitem[Somraj et~al.(2024)Somraj, Choudhary, Mupparaju, and Soundararajan]{somraj2024factorized}
Nagabhushan Somraj, Kapil Choudhary, Sai~Harsha Mupparaju, and Rajiv Soundararajan.
\newblock Factorized motion fields for fast sparse input dynamic view synthesis.
\newblock In \emph{ACM SIGGRAPH 2024 Conference Papers}, pages 1--12, 2024.
\bibitem[Son et~al.(2021)Son, Lee, Lee, Cho, and Lee]{son2021recurrent}
Hyeongseok Son, Junyong Lee, Jonghyeop Lee, Sunghyun Cho, and Seungyong Lee.
\newblock Recurrent video deblurring with blur-invariant motion estimation and pixel volumes.
\newblock \emph{ACM Transactions on Graphics (TOG)}, 40\penalty0 (5):\penalty0 1--18, 2021.
\bibitem[Stearns et~al.(2024)Stearns, Harley, Uy, Dubost, Tombari, Wetzstein, and Guibas]{stearns2024dynamic}
Colton Stearns, Adam Harley, Mikaela Uy, Florian Dubost, Federico Tombari, Gordon Wetzstein, and Leonidas Guibas.
\newblock Dynamic gaussian marbles for novel view synthesis of casual monocular videos.
\newblock \emph{arXiv preprint arXiv:2406.18717}, 2024.
\bibitem[Sun et~al.(2018)Sun, Yang, Liu, and Kautz]{sun2018pwc}
Deqing Sun, Xiaodong Yang, Ming-Yu Liu, and Jan Kautz.
\newblock Pwc-net: Cnns for optical flow using pyramid, warping, and cost volume.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 8934--8943, 2018.
\bibitem[Sun et~al.(2024{\natexlab{a}})Sun, Li, Shen, Ye, Xian, and Cao]{sun2024dyblurf}
Huiqiang Sun, Xingyi Li, Liao Shen, Xinyi Ye, Ke Xian, and Zhiguo Cao.
\newblock Dyblurf: Dynamic neural radiance fields from blurry monocular video.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 7517--7527, 2024{\natexlab{a}}.
\bibitem[Sun et~al.(2015)Sun, Cao, Xu, and Ponce]{sun2015learning}
Jian Sun, Wenfei Cao, Zongben Xu, and Jean Ponce.
\newblock Learning a convolutional neural network for non-uniform motion blur removal.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 769--777, 2015.
\bibitem[Sun et~al.(2024{\natexlab{b}})Sun, Jiao, Li, Zhang, Zhao, and Xing]{sun20243dgstream}
Jiakai Sun, Han Jiao, Guangyuan Li, Zhanjie Zhang, Lei Zhao, and Wei Xing.
\newblock 3dgstream: On-the-fly training of 3d gaussians for efficient streaming of photo-realistic free-viewpoint videos.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 20675--20685, 2024{\natexlab{b}}.
\bibitem[Tao et~al.(2018)Tao, Gao, Shen, Wang, and Jia]{tao2018scale}
Xin Tao, Hongyun Gao, Xiaoyong Shen, Jue Wang, and Jiaya Jia.
\newblock Scale-recurrent network for deep image deblurring.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 8174--8182, 2018.
\bibitem[Tian et~al.(2023)Tian, Du, and Duan]{tian2023mononerf}
Fengrui Tian, Shaoyi Du, and Yueqi Duan.
\newblock Mononerf: Learning a generalizable dynamic radiance field from monocular videos.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 17903--17913, 2023.
\bibitem[Wang et~al.(2023{\natexlab{a}})Wang, Chan, and Loy]{wang2023exploring}
Jianyi Wang, Kelvin~CK Chan, and Chen~Change Loy.
\newblock Exploring clip for assessing the look and feel of images.
\newblock In \emph{Proceedings of the AAAI Conference on Artificial Intelligence}, pages 2555--2563, 2023{\natexlab{a}}.
\bibitem[Wang et~al.(2023{\natexlab{b}})Wang, Zhao, Ma, and Liu]{wang2023bad}
Peng Wang, Lingzhe Zhao, Ruijie Ma, and Peidong Liu.
\newblock Bad-nerf: Bundle adjusted deblur neural radiance fields.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 4170--4179, 2023{\natexlab{b}}.
\bibitem[Wang et~al.(2024{\natexlab{a}})Wang, Ye, Gao, Austin, Li, and Kanazawa]{wang2024shape}
Qianqian Wang, Vickie Ye, Hang Gao, Jake Austin, Zhengqi Li, and Angjoo Kanazawa.
\newblock Shape of motion: 4d reconstruction from a single video.
\newblock \emph{arXiv preprint arXiv:2407.13764}, 2024{\natexlab{a}}.
\bibitem[Wang et~al.(2024{\natexlab{b}})Wang, Yang, Shen, Jiang, and Wang]{wang2024gflow}
Shizun Wang, Xingyi Yang, Qiuhong Shen, Zhenxiang Jiang, and Xinchao Wang.
\newblock Gflow: Recovering 4d world from monocular video.
\newblock \emph{arXiv preprint arXiv:2405.18426}, 2024{\natexlab{b}}.
\bibitem[Wang et~al.(2004)Wang, Bovik, Sheikh, and Simoncelli]{wang2004image}
Zhou Wang, Alan~C Bovik, Hamid~R Sheikh, and Eero~P Simoncelli.
\newblock Image quality assessment: from error visibility to structural similarity.
\newblock \emph{IEEE transactions on image processing}, 13\penalty0 (4):\penalty0 600--612, 2004.
\bibitem[Wang et~al.(2022)Wang, Cun, Bao, Zhou, Liu, and Li]{wang2022uformer}
Zhendong Wang, Xiaodong Cun, Jianmin Bao, Wengang Zhou, Jianzhuang Liu, and Houqiang Li.
\newblock Uformer: A general u-shaped transformer for image restoration.
\newblock In \emph{Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 17683--17693, 2022.
\bibitem[Wieschollek et~al.(2017)Wieschollek, Hirsch, Scholkopf, and Lensch]{wieschollek2017learning}
Patrick Wieschollek, Michael Hirsch, Bernhard Scholkopf, and Hendrik Lensch.
\newblock Learning blind motion deblurring.
\newblock In \emph{Proceedings of the IEEE international conference on computer vision}, pages 231--240, 2017.
\bibitem[Wu et~al.(2024{\natexlab{a}})Wu, Yi, Fang, Xie, Zhang, Wei, Liu, Tian, and Wang]{4dgaussians}
Guanjun Wu, Taoran Yi, Jiemin Fang, Lingxi Xie, Xiaopeng Zhang, Wei Wei, Wenyu Liu, Qi Tian, and Xinggang Wang.
\newblock 4d gaussian splatting for real-time dynamic scene rendering.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 20310--20320, 2024{\natexlab{a}}.
\bibitem[Wu et~al.(2024{\natexlab{b}})Wu, Zhang, Zhang, Gou, Chen, Zhang, Chen, and Zuo]{wu2024self}
Renlong Wu, Zhilu Zhang, Shuohao Zhang, Longfei Gou, Haobin Chen, Lei Zhang, Hao Chen, and Wangmeng Zuo.
\newblock Self-supervised video desmoking for laparoscopic surgery.
\newblock \emph{arXiv preprint arXiv:2403.11192}, 2024{\natexlab{b}}.
\bibitem[Wu et~al.(2025)Wu, Yu, Jiang, Cao, Wang, and Bai]{wu2025sc4d}
Zijie Wu, Chaohui Yu, Yanqin Jiang, Chenjie Cao, Fan Wang, and Xiang Bai.
\newblock Sc4d: Sparse-controlled video-to-4d generation and motion transfer.
\newblock In \emph{European Conference on Computer Vision}, pages 361--379. Springer, 2025.
\bibitem[Yan et~al.(2023)Yan, Li, and Lee]{yan2023nerf}
Zhiwen Yan, Chen Li, and Gim~Hee Lee.
\newblock Nerf-ds: Neural radiance fields for dynamic specular objects.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 8285--8295, 2023.
\bibitem[Yang et~al.()Yang, Xie, Fu, Fan, and Dong]{yang48953364d}
Xiaodong Yang, Weixing Xie, Yihang Fu, Wentao Fan, and Xiao Dong.
\newblock 4d gaussian splatting for high-fidelity dynamic reconstruction of single-view scenes.
\newblock \emph{Available at SSRN 4895336}.
\bibitem[Yang et~al.(2023)Yang, Yang, Pan, and Zhang]{yang2023real}
Zeyu Yang, Hongye Yang, Zijie Pan, and Li Zhang.
\newblock Real-time photorealistic dynamic scene representation and rendering with 4d gaussian splatting.
\newblock \emph{arXiv preprint arXiv:2310.10642}, 2023.
\bibitem[Yang et~al.(2024)Yang, Gao, Zhou, Jiao, Zhang, and Jin]{yang2024deformable}
Ziyi Yang, Xinyu Gao, Wen Zhou, Shaohui Jiao, Yuqing Zhang, and Xiaogang Jin.
\newblock Deformable 3d gaussians for high-fidelity monocular dynamic scene reconstruction.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 20331--20341, 2024.
\bibitem[Yu et~al.(2024)Yu, Wang, Zhuang, Menapace, Siarohin, Cao, Jeni, Tulyakov, and Lee]{yu20244real}
Heng Yu, Chaoyang Wang, Peiye Zhuang, Willi Menapace, Aliaksandr Siarohin, Junli Cao, Laszlo~A Jeni, Sergey Tulyakov, and Hsin-Ying Lee.
\newblock 4real: Towards photorealistic 4d scene generation via video diffusion models.
\newblock \emph{arXiv preprint arXiv:2406.07472}, 2024.
\bibitem[Zamir et~al.(2022)Zamir, Arora, Khan, Hayat, Khan, and Yang]{zamir2022restormer}
Syed~Waqas Zamir, Aditya Arora, Salman Khan, Munawar Hayat, Fahad~Shahbaz Khan, and Ming-Hsuan Yang.
\newblock Restormer: Efficient transformer for high-resolution image restoration.
\newblock In \emph{Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 5728--5739, 2022.
\bibitem[Zeng et~al.(2025)Zeng, Jiang, Zhu, Lu, Lin, Zhu, Hu, Cao, and Yao]{zeng2025stag4d}
Yifei Zeng, Yanqin Jiang, Siyu Zhu, Yuanxun Lu, Youtian Lin, Hao Zhu, Weiming Hu, Xun Cao, and Yao Yao.
\newblock Stag4d: Spatial-temporal anchored generative 4d gaussians.
\newblock In \emph{European Conference on Computer Vision}, pages 163--179. Springer, 2025.
\bibitem[Zhang et~al.(2023{\natexlab{a}})Zhang, Zhu, Wang, Chen, Wu, and Wang]{zhang2023extracting}
Guozhen Zhang, Yuhan Zhu, Haonan Wang, Youxin Chen, Gangshan Wu, and Limin Wang.
\newblock Extracting motion and appearance via inter-frame attention for efficient video frame interpolation.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 5682--5692, 2023{\natexlab{a}}.
\bibitem[Zhang et~al.(2024{\natexlab{a}})Zhang, Xie, and Yao]{zhang2024blur}
Huicong Zhang, Haozhe Xie, and Hongxun Yao.
\newblock Blur-aware spatio-temporal sparse transformer for video deblurring.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 2673--2681, 2024{\natexlab{a}}.
\bibitem[Zhang et~al.(2018{\natexlab{a}})Zhang, Luo, Zhong, Ma, Liu, and Li]{zhang2018adversarial}
Kaihao Zhang, Wenhan Luo, Yiran Zhong, Lin Ma, Wei Liu, and Hongdong Li.
\newblock Adversarial spatio-temporal learning for video deblurring.
\newblock \emph{IEEE Transactions on Image Processing}, 28\penalty0 (1):\penalty0 291--301, 2018{\natexlab{a}}.
\bibitem[Zhang et~al.(2018{\natexlab{b}})Zhang, Isola, Efros, Shechtman, and Wang]{zhang2018unreasonable}
Richard Zhang, Phillip Isola, Alexei~A Efros, Eli Shechtman, and Oliver Wang.
\newblock The unreasonable effectiveness of deep features as a perceptual metric.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and pattern recognition}, pages 586--595, 2018{\natexlab{b}}.
\bibitem[Zhang et~al.(2023{\natexlab{b}})Zhang, Wei, Jiang, Zhang, Zuo, and Tian]{zhang2023controlvideo}
Yabo Zhang, Yuxiang Wei, Dongsheng Jiang, Xiaopeng Zhang, Wangmeng Zuo, and Qi Tian.
\newblock Controlvideo: Training-free controllable text-to-video generation.
\newblock \emph{arXiv preprint arXiv:2305.13077}, 2023{\natexlab{b}}.
\bibitem[Zhang et~al.(2022)Zhang, Xu, Liu, Yan, and Zuo]{zhang2022self}
Zhilu Zhang, RongJian Xu, Ming Liu, Zifei Yan, and Wangmeng Zuo.
\newblock Self-supervised image restoration with blurry and noisy pairs.
\newblock \emph{Advances in Neural Information Processing Systems}, 35:\penalty0 29179--29191, 2022.
\bibitem[Zhang et~al.(2023{\natexlab{c}})Zhang, Liu, Tan, Zeng, and Liu]{zhang2023minimum}
Zhuofan Zhang, Zhen Liu, Ping Tan, Bing Zeng, and Shuaicheng Liu.
\newblock Minimum latency deep online video stabilization.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 23030--23039, 2023{\natexlab{c}}.
\bibitem[Zhang et~al.(2024{\natexlab{b}})Zhang, Zhang, Wu, Yan, and Zuo]{zhang2024bracketing}
Zhilu Zhang, Shuohao Zhang, Renlong Wu, Zifei Yan, and Wangmeng Zuo.
\newblock Bracketing is all you need: Unifying image restoration and enhancement tasks with multi-exposure images.
\newblock \emph{arXiv preprint arXiv:2401.00766}, 2024{\natexlab{b}}.
\bibitem[Zhao et~al.(2024)Zhao, Wang, and Liu]{zhao2024bad}
Lingzhe Zhao, Peng Wang, and Peidong Liu.
\newblock Bad-gaussians: Bundle adjusted deblur gaussian splatting.
\newblock \emph{arXiv preprint arXiv:2403.11831}, 2024.
\bibitem[Zhao et~al.(2023)Zhao, An, Pan, and Yang]{zhao2023nerf2}
Xiaopeng Zhao, Zhenlin An, Qingrui Pan, and Lei Yang.
\newblock Nerf2: Neural radio-frequency radiance fields.
\newblock In \emph{Proceedings of the 29th Annual International Conference on Mobile Computing and Networking}, pages 1--15, 2023.
\bibitem[Zhong et~al.(2020)Zhong, Gao, Zheng, and Zheng]{zhong2020efficient}
Zhihang Zhong, Ye Gao, Yinqiang Zheng, and Bo Zheng.
\newblock Efficient spatio-temporal recurrent neural network for video deblurring.
\newblock In \emph{Computer Vision--ECCV 2020: 16th European Conference, Glasgow, UK, August 23--28, 2020, Proceedings, Part VI 16}, pages 191--207. Springer, 2020.
\bibitem[Zhong et~al.(2023{\natexlab{a}})Zhong, Cao, Ji, Zheng, and Sato]{zhong2023blur}
Zhihang Zhong, Mingdeng Cao, Xiang Ji, Yinqiang Zheng, and Imari Sato.
\newblock Blur interpolation transformer for real-world motion from blur.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 5713--5723, 2023{\natexlab{a}}.
\bibitem[Zhong et~al.(2023{\natexlab{b}})Zhong, Gao, Zheng, Zheng, and Sato]{zhong2023real}
Zhihang Zhong, Ye Gao, Yinqiang Zheng, Bo Zheng, and Imari Sato.
\newblock Real-world video deblurring: A benchmark dataset and an efficient recurrent neural network.
\newblock \emph{International Journal of Computer Vision}, 131\penalty0 (1):\penalty0 284--301, 2023{\natexlab{b}}.
\bibitem[Zhou et~al.(2019)Zhou, Zhang, Zuo, Xie, Pan, and Ren]{zhou2019davanet}
Shangchen Zhou, Jiawei Zhang, Wangmeng Zuo, Haozhe Xie, Jinshan Pan, and Jimmy~S Ren.
\newblock Davanet: Stereo deblurring with view aggregation.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 10996--11005, 2019.
\bibitem[Zhu et~al.(2024)Zhu, Liang, Chang, Deng, Lu, Yang, Zhang, and Zhang]{zhu2024motiongs}
Ruijie Zhu, Yanzhe Liang, Hanzhi Chang, Jiacheng Deng, Jiahao Lu, Wenfei Yang, Tianzhu Zhang, and Yongdong Zhang.
\newblock Motiongs: Exploring explicit motion guidance for deformable 3d gaussian splatting.
\newblock \emph{arXiv preprint arXiv:2410.07707}, 2024.
\end{thebibliography}