Free-Moving Object Reconstruction and Pose Estimation with Virtual Camera
Extracted source text · arXiv 2405.05858
% CVPR 2024 Paper Template; see https://github.com/cvpr-org/author-kit
\documentclass[10pt,twocolumn,letterpaper]{article}
%%%%%%%%% PAPER TYPE - PLEASE UPDATE FOR FINAL VERSION
% \usepackage{cvpr} % To produce the CAMERA-READY version
% \usepackage[review]{cvpr} % To produce the REVIEW version
\usepackage[pagenumbers]{cvpr} % To force page numbers, e.g. for an arXiv version
\usepackage{comment}
\usepackage[symbol]{footmisc}
\usepackage{gensymb}
\usepackage{subcaption}
\usepackage{colortbl}
\usepackage{arydshln}
\usepackage{amssymb}% http://ctan.org/pkg/amssymb
\usepackage{pifont}% http://ctan.org/pkg/pifont
\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%
% Import additional packages in the preamble file, before hyperref
\input{preamble}
% It is strongly recommended to use hyperref, especially for the review version.
% hyperref with option pagebackref eases the reviewers' job.
% Please disable hyperref *only* if you encounter grave issues,
% e.g. with the file validation for the camera-ready version.
%
% If you comment hyperref and then uncomment it, you should delete *.aux before re-running LaTeX.
% (Or just hit 'q' on the first LaTeX run, let it finish, and you should be clear).
\definecolor{cvprblue}{rgb}{0.21,0.49,0.74}
\usepackage[pagebackref,breaklinks,colorlinks,citecolor=cvprblue]{hyperref}
%%%%%%%%% PAPER ID - PLEASE UPDATE
\def\paperID{*****} % *** Enter the Paper ID here
\def\confName{CVPR}
\def\confYear{2024}
%%%%%%%%% TITLE - PLEASE UPDATE
\title{Free-Moving Object Reconstruction and Pose Estimation with Virtual Camera}
% \title{Joint Object Reconstruction and Pose Estimation with Guided Virtual Camera}
% \title{Free-Moving Object Reconstruction and Pose Estimation without Segments}
%%%%%%%%% AUTHORS - PLEASE UPDATE
\author{%
{Haixin Shi$^1$\footnotemark[1], ~Yinlin Hu$^2$, ~Daniel Koguciuk$^2$, ~Juan-Ting Lin$^2$, ~Mathieu Salzmann$^{1}$, ~David Ferstl$^2$} \\
{$^1$EPFL, \quad $^2$MagicLeap} \\
Project page: \href{https://haixinshi.github.io/fmov}{https://haixinshi.github.io/fmov} \\
}
\begin{document}
% from https://tex.stackexchange.com/a/55766
\twocolumn[{%
\renewcommand\twocolumn[1][]{#1}%
\maketitle
\begin{center}
\centering
\captionsetup{type=figure}
\includegraphics[width=\textwidth]{Introduction_Figure/Main_Comparison_Row_v3.png}
\begin{tabular}{m{0.1\textwidth}m{0.45\textwidth}m{0.45\textwidth}}
\small{(a) Settings} & ~~(b) \small{Hampali~\etal.~\cite{meta-obj} versus ours with fixed camera} & \hspace{-1em}\small{(c) BARF~\cite{barf} versus ours on egocentric sequences} \\
\end{tabular}
\vspace{-1.5em}
\captionof{figure}{
{\bf Joint object reconstruction and pose estimation from a monocular RGB video.}
{\bf (a)} {\bf Top}: the standard reconstruction setting with static object captured by a moving camera, which relies on the geometry cue from the whole scene and is inapplicable to dynamic objects. {\bf Middle}: the setting with rotating a hand-held object in front of a fixed camera. {\bf Bottom}: the setting does not assume any prior, which allows the objects to be moved freely with any grasping style. {\bf (b, c)} Our method outperforms state of the art on the HO3D dataset with fixed camera, and produces accurate results on free-moving objects with egocentric views.
}
\label{fig:first}
\end{center}%
}]
% {\bf (a)} The standard reconstruction setting with static object captured by a moving camera, which relies on the geometry cue from the entire scene including both foreground and background. {\bf (b)} A more usable setting with rotating a hand-held object in front a fixed camera, which, however, usually assume that the hand contact points with the object should be fixed during the whole capturing. {\bf (c)} Our setting does not assume any hand prior or camera poses, which allows the objects to be moved in any manner, or freely manipulated with any grasping style if moved by hands, with either a static or moving camera.
\maketitle
\footnotetext[1]{Work done as part of Haixin’s Master thesis.}
\input{sec/00_abstract}
\input{sec/01_introduction}
\input{sec/02_related_work}
\input{sec/03_approach}
\input{sec/04_experiment}
\input{sec/05_conclusion}
{
\small
\bibliographystyle{ieeenat_fullname}
\bibliography{main}
}
% WARNING: do not forget to delete the supplementary pages from your submission
%\input{sec/X_suppl}
\end{document}
%
% --- inline annotations
%
\usepackage[dvipsnames]{xcolor}
\newcommand{\red}[1]{{\color{red}#1}}
\newcommand{\todo}[1]{{\color{red}#1}}
\newcommand{\TODO}[1]{\textbf{\color{red}[TODO: #1]}}
% --- disable by uncommenting
% \renewcommand{\TODO}[1]{}
% \renewcommand{\todo}[1]{#1}
% !TEX root = top.tex
% !TEX spellcheck = en-US
\newif\ifdraft
\draftfalse
\drafttrue
\usepackage{xcolor}
\usepackage{multirow} % added by haixin
\definecolor{orange}{rgb}{1,0.5,0}
\definecolor{violet}{RGB}{70,0,170}
% enable comments
% \newcommand{\HS}[1]{{\color{red}{\bf HS: #1}}}
% \newcommand{\hs}[1]{{\color{red} #1}}
% \newcommand{\YH}[1]{{\color{violet}{\bf YH: #1}}}
% \newcommand{\yh}[1]{{\color{violet}{#1}}}
% \newcommand{\DK}[1]{{\color{orange}{\bf DK: #1}}}
% \newcommand{\dk}[1]{{\color{orange}{#1}}}
% \newcommand{\JT}[1]{{\color{orange}{\bf JT: #1}}}
% \newcommand{\jt}[1]{{\color{orange}{#1}}}
% \newcommand{\MS}[1]{{\color{green}{\bf MS: #1}}}
% \newcommand{\ms}[1]{{\color{green} #1}}
% \newcommand{\DF}[1]{{\color{blue}{\bf DF: #1}}}
% \newcommand{\df}[1]{{\color{blue}{#1}}}
% disable all comments
\newcommand{\HS}[1]{}
\newcommand{\YH}[1]{}
\newcommand{\DK}[1]{}
\newcommand{\JT}[1]{}
\newcommand{\MS}[1]{}
\newcommand{\DF}[1]{}
% \newcommand{\comment}[1]{}
\newcommand{\parag}[1]{\vspace{-3mm}\paragraph{#1}}
% Matrices
\newcommand{\bA}{\mathbf{A}}
\newcommand{\bI}{\mathbf{I}}
\newcommand{\bF}{\mathbf{F}}
\newcommand{\bK}{\mathbf{K}}
\newcommand{\bM}{\mathbf{M}}
\newcommand{\bR}{\mathbf{R}}
\newcommand{\bS}{\mathbf{S}}
\newcommand{\btA}{\tilde{\mathbf{A}}}
\newcommand{\bbA}{\bar{\mathbf{A}}}
\newcommand{\bU}{\mathbf{U}}
\newcommand{\bL}{\mathbf{L}}
\newcommand{\bC}{\mathbf{C}}
\newcommand{\bD}{\mathbf{D}}
\newcommand{\bT}{\mathbf{T}}
\newcommand{\bW}{\mathbf{W}}
\newcommand{\bX}{\mathbf{X}}
\newcommand{\bXe}{\tilde{\mathbf{X}}}
\newcommand{\bbX}{\bar{\mathbf{X}}}
% Vectors
\newcommand{\bu}{\mathbf{u}}
\newcommand{\bv}{\mathbf{v}}
\newcommand{\bw}{\mathbf{w}}
\newcommand{\bq}{\mathbf{q}}
\newcommand{\be}{\mathbf{e}}
\newcommand{\f}{\mathbf{f}}
\newcommand{\bte}{\tilde{\mathbf{e}}}
\newcommand{\bx}{\mathbf{x}}
\newcommand{\bp}{\mathbf{p}}
\newcommand{\bd}{\mathbf{d}}
\newcommand{\bc}{\mathbf{c}}
\newcommand{\bt}{\mathbf{t}}
\newcommand{\br}{\mathbf{r}}
\newcommand{\bm}{\mathbf{m}}
\newcommand{\bn}{\mathbf{n}}
\newcommand{\bg}{\mathbf{g}}
\def\bSigma{\boldsymbol\Sigma}
\newcommand{\minimize}{\operatornamewithlimits{minimize}}
\begin{abstract}
We propose an approach for reconstructing free-moving object from a monocular RGB video. Most existing methods either assume scene prior, hand pose prior, object category pose prior, or rely on local optimization with multiple sequence segments.
We propose a method that allows free interaction with the object in front of a moving camera without relying on any prior, and optimizes the sequence globally without any segments. We progressively optimize the object shape and pose simultaneously based on an implicit neural representation.
A key aspect of our method is a virtual camera system that reduces the search space of the optimization significantly.
We evaluate our method on the standard HO3D dataset and a collection of egocentric RGB sequences captured with a head-mounted device. We demonstrate that our approach outperforms most methods significantly, and is on par with recent techniques that assume prior information.
% We propose an approach for reconstructing free-moving objects from a monocular video. Unlike most existing methods that assume hand pose information or rely on a carefully selection of multiple segments which tend to be local optimal, we propose a method that allows for free rotation of the object in front of a moving camera and globally optimizes the network without segments. We do not rely on depth, camera poses, or any prior information of the object.
% \HS{In addition to virtual camera, should we emphasize some effective tricks in Section \ref{sec:method_progressive}, including relative pose initialization based on rendered depth and also reset Neus(or equivalently segment the sequences) based on relative rotations?} \YH{I think they contain excessive detail for an abstract.}
% The ABSTRACT is to be in fully justified italicized text, at the top of the left-hand column, below the author and affiliation information.
% Use the word ``Abstract'' as the title, in 12-point Times, boldface type, centered relative to the column, initially capitalized.
% The abstract is to be in 10-point, single-spaced type.
% Leave two blank lines after the Abstract, then begin the main text.
% Look at previous \confName abstracts to get a feel for style and length.
\end{abstract}
\section{Introduction}
\label{sec:intro}
%Combined
Understanding 3D objects around us is a fundamental problem in computer vision~\cite{han2021trends}, and also a critical component in many applications, such as augmented reality~(AR)~\cite{Billinghurst2021archallenges} and robot manipulation~\cite{Thalhammer2023ChallengesFM}. This requires an accurate reconstruction and pose estimation of such objects. With a monocular RGB camera, most current work tackle this problem with major simplifications, either by moving the camera around a static object~\cite{unisurf, frodo, yariv2020multiview} or by rotating the object with hands in front of a stationary camera~\cite{rusinkiewicz2002realtime, tzionas2015handobject, weise2008inhandmodelin, weise2009loopclosure}.
% Although the latter setting is more usable in daily scenarios, it relies on the user holding the object firmly throughout the whole capture procedure \cite{hhor-handold, jiang2023hand, showme} to bootstrap the reconstruction with hand pose prior.
In this paper, we investigate a more general setting with the example of an AR device where the object is free-moving in front of a head-mounted camera. We neither assume any object category prior nor any hand prior in this new setting, which allows the objects to be moved in any manner, or freely manipulated with any grasping style if moved by hands, as shown in Fig.~\ref{fig:first}.
% We illustrate different reconstruction settings in Fig.~\ref{fig:setting}.
% \begin{figure}[tb]
% \centering
% \begin{tabular}{ccc}
% \includegraphics[width=0.3\linewidth]{Introduction_Figure/fixed_object_moving_camera.png} &
% \includegraphics[width=0.31\linewidth]{Introduction_Figure/moving_object_fixed_camera_with_hand.png} &
% \includegraphics[width=0.3\linewidth]{Introduction_Figure/ours_single_camera.png} \\
% (a) & (b) & (c)\\
% \end{tabular}
% \caption{\textbf{Different settings of reconstruction.}
% {\bf (a)} The standard reconstruction setting with static object captured by a moving camera, which relies on the geometry cue
% % \DK{should this be geometry cue} \YH{Yes!}
% from the entire scene including both foreground and background. {\bf (b)} A more usable setting with rotating a hand-held object in front a fixed camera, which, however, usually assume that the hand contact points with the object should be fixed during the whole capturing. {\bf (c)} Our setting does not assume any hand prior or camera poses, which allows the objects to be moved in any manner, or freely manipulated with any grasping style if moved by hands, with either a static or moving camera. \YH{Better to embed this figure into the large figure in the first page.}
% }
% \label{fig:setting}
% \end{figure}
\begin{figure}[tb]
\centering
\includegraphics[width=\linewidth]{Introduction_Figure/seg_w_traj_v4.png}
\caption{
{\bf Paradigms of pose-free object reconstruction.} {\bf Top}: The input sequence. {\bf Middle}: Existing method~\cite{meta-obj} relies on segment-wise joint optimization based on multiple easy segments of the sequence, as shown with different colors in the pose trajectories, which tends to be local optimal. {\bf Bottom}: Our method optimizes object shape and pose progressively without any segments, producing globally-consistent shape and pose results.
}
\label{fig:compare_paradigm}
\end{figure}
%yielding two advantages via shared geometry optimization from distinct view angles:(a) incoming new frames can help refine previous frame poses; (b) previous masks can help regularize new frame poses.
% combined (a bit long, moved to related work)
% The new setting exhibits considerable challenges for the reconstruction. The conventional approach to static object reconstruction largely depends on extracting matches across the entire scene to determine camera poses using structure-from-motion (SfM) methods~\cite{snavely2006photo, agarwal2011building}, with COLMAP~\cite{colmap} being a typical example~\cite{colmap}. This process, however, can not handle free-moving objects which exhibit movement patterns distinct from the background, making the geometry cues from the entire scene inapplicable for dynamic scene components. Furthermore, in practice, many real-life objects are texture-less presenting a significant challenge to SfM methods, which rely on distinct scene textures~\cite{cheng2021multi, yi2018learning}. On the other hand, without the hand-object pose initialization from the restriction of firm grasping of the object by hands, reconstruction is inherently challenging.
Without pose initialization, some reconstruction methods~\cite{barf, SCNeRF2021, rosinol2022nerf} propose to optimize the shape representation and poses simultaneously. However, most of them either rely on geometry clues of the background which is inapplicable for free-moving objects, or can only handle a restricted range of viewpoints. On the other hand, some recent methods~\cite{meta-obj} use progressive training to solve this problem and rely on segment-wise optimization based on multiple overlapping segments of the sequence. This strategy, however, suffers mainly in two ways. First, the frame selection of each segment is error-prone, since it typically relies on the changes of mask area of the target between consecutive frames, which hardly can be generalized to different object shapes. Second, the segment-wise optimization is inherently local and sub-optimal, as shown in Fig.~\ref{fig:compare_paradigm}.
% (a bit long, moved to related work)
% There exists some NeRF-based methods~\cite{barf, SCNeRF2021, rosinol2022nerf} that can optimize both the radiance representation and the camera poses jointly without relying on pose initialization. Also, there is concurrent work~\cite{nope-neus} that is NeuS-based and also optimize pose and SDF simultaneously, however, this work only show their performance on DTU dataset~\cite{dtu}, which has much less viewpoint range compared with our target HO3D dataset~\cite{ho3d}. To conclude, most of current methods that simultaneously optimize reconstruction and pose can only handle a restricted range of viewpoints, limiting their usage in comprehensive object reconstruction that necessitates full 360-degree coverage. Some recent work~\cite{3dgs} try to use progressive training to solve this problem, but we found that most of them can only work with static scenes. The RGB-based state-of-art~\cite{meta-obj} can work on 360-degree view scenario also without any strong prior, which is a strong baseline in our paper, but it requires carefully splitting sequence into segments.
We propose a method for joint reconstruction and pose estimation of free-moving objects without any segments, which can be globally optimized with a single network.
% We use an implicit neural representation to optimize the geometry, appearance, and object pose simultaneously.
Our observation is that the unknown pose trajectory of the object can be simplified with a new virtual camera system that always points to the object center with the guidance of 2D object masks, which reduces the search space of the optimization significantly. We first use off-the-shelf 2D segmentation methods to get object masks in each frame, and then optimize the network w.r.t. the virtual camera. To handle the approximation error between virtual camera and real camera, we finally convert the results to real camera coordinate system and refine all the results w.r.t. the real camera. We evaluate our method on both the HO3D dataset~\cite{ho3d} with fixed camera and a collection of data captured using a head-mounted AR device with egocentric views. The experiments show that our method outperforms traditional methods and most baselines significantly.
We summarize our contributions as follows. First, we investigate the problem of existing methods in reconstructing free-moving object from a RGB monocular video. Second, we propose a simple-but-effective strategy with a virtual camera system that simplifies the object trajectory and reduce the search space of the optimization significantly. Finally, we demonstrate the effectiveness of our method on datasets with either a fixed or egocentric camera.
\begin{figure*}[tb]
\centering
\includegraphics[width=0.9\linewidth]{Approach_Figure/Approach_Arch_v8.png}
\caption{{\bf Overview of our method.}
We first use off-the-shelf 2D segmentation methods to get object mask in each frame, and then optimize the MLP networks w.r.t. a virtual camera system, with which the camera always points to areas near the object center, as illustrated as colored 3D axis in the figure. We optimize three MLPs with progressively added images. For each frame with time index $t$, we use the Pose MLP to predict the object pose~$(\bR, s)$, which corresponds to the rotation and the distance from the camera center to the object center, summing up to only 4 degrees of freedom. For each 3D point $\bx$ along the view direction $\bv$, we use the SDF MLP and Color MLP to predict its corresponding SDF value and color opacity, respectively. We compare the rendered image with the input and update the MLP networks based on volume rendering. We finally conduct the virtual-to-real conversion and refine all the results w.r.t. the real camera.
% Guided by mask, we progressively add frames under virtual camera. Within available frames, we sample frame id $t_{i}$ and get 4 degree of freedom(4 DoF) pose from a Pose MLP, namely 1 for the distance $s_{i}$ between camera and object and 3 for rotation $R_{i}$, to define a ray on which we sample points $P_{i}$, together with view direction $V_{i}$ feed into NeuS system\cite{neus} which contains SDF and Color MLP to do volume rendering. We supervise the whole system on rendered results with color from images and also matches across them. Periodically, we will reset NeuS MLPs to solve new surface appearing problem\cite{meta-obj} but without segments.
}
\label{fig:over_view}
\end{figure*}
%Guided by masks, we find and transform the object center to image plane center, building up virtual camera. During this phase we first map frame ID to Gaussian Fourier feature and feed it into Pose MLP to get 4 Degree Of Freedom(4DoF) poses for emitting rays in NeuS system, where we supervise whole system with differential rendering. Under virtual camera, we progressively feed frames and reconstruct the mesh with resetting strategy without segments. Then we go back to original camera via PnP and RANSAC and refine the initialized poses, finally generating high-quality mesh. \YH{Better to highlight parts differentiate our method from others.}
\section{Related Work}
\noindent {\bf 3D Reconstruction} is a fundamental problem in computer vision. Traditional methods~\cite{agarwal2011building, mohr1995relative, pollefeys2004visual, snavely2006photo}, typically COLMAP~\cite{colmap}, first estimate camera parameters with 2D matching and then reconstruct the scene with multi-view-stereo~(MVS) techniques~\cite{cheng2021multi}. Most recent methods solve the reconstruction by optimizing a neural implicit representation with rendering techniques~\cite{nerf, d-nerf, neus, neus2}. Although they achieve high-quality reconstruction results, most of them rely on SfM to obtain accurate camera poses for each frame. However, most SfM-based methods assume static rigid scenes and only work when there are enough textures in the scene~\cite{colmap, theia-manual, Bundler}, which is inapplicable in our setting where the object moves independently of the background and the object is often texture-less. Some recent works try to remove the SfM pose initialization by optimizing the neural representation and camera poses simultaneously~\cite{barf, SCNeRF2021, rosinol2022nerf, nope-nerf, nope-neus}. However, they can only work with forward-facing scenarios with a restricted range of view. The recent method~\cite{meta-obj} tries to divide the whole sequence into multiple easy short segments to facilitate the optimization. However, the frame selection of the segmenting procedure is shape dependent and the segment-wise based optimization is local optimal. By contrast, our method is segment-free and can produce globally-consistent results.
% However, the process of carefully selecting segments is error-prone and can not be generalized to different object types. By contrast, we handle the reconstruction problem of hand-hold objects by reformulating it to a reconstruction problem with respect to a virtual camera with the guidance of 2D object masks, which reduces the search space of the optimization significantly and does not need careful selection of segments.
% The new setting exhibits considerable challenges for the reconstruction. The conventional approach to static object reconstruction largely depends on extracting matches across the entire scene to determine camera poses using structure-from-motion (SfM) methods~\cite{snavely2006photo, agarwal2011building}, with COLMAP~\cite{colmap} being a typical example~\cite{colmap}. This process, however, can not handle free-moving objects which exhibit movement patterns distinct from the background, making the geometry cues from the entire scene inapplicable for dynamic scene components. Furthermore, in practice, many real-life objects are texture-less presenting a significant challenge to SfM methods, which rely on distinct scene textures~\cite{cheng2021multi, yi2018learning}. On the other hand, without the hand-object pose initialization from the restriction of firm grasping of the object by hands, reconstruction is inherently challenging.
% There exists some NeRF-based methods~\cite{barf, SCNeRF2021, rosinol2022nerf} that can optimize both the radiance representation and the camera poses jointly without relying on pose initialization. Also, there is concurrent work~\cite{nope-neus} that is NeuS-based and also optimize pose and SDF simultaneously, however, this work only show their performance on DTU dataset~\cite{dtu}, which has much less viewpoint range compared with our target HO3D dataset~\cite{ho3d}. To conclude, most of current methods that simultaneously optimize reconstruction and pose can only handle a restricted range of viewpoints, limiting their usage in comprehensive object reconstruction that necessitates full 360-degree coverage. Some recent work~\cite{3dgs} try to use progressive training to solve this problem, but we found that most of them can only work with static scenes. The RGB-based state-of-art~\cite{meta-obj} can work on 360-degree view scenario also without any strong prior, which is a strong baseline in our paper, but it requires carefully splitting sequence into segments.
\noindent {\bf Object Pose Estimation} aims to produce accurate 3D rotation and 3D translation of the object w.r.t. the camera, which usually serves as pose initialization for dynamic object reconstruction. Most recent object pose methods~\cite{BOP,PFA,WDR,Zebrapose,GDRNet}
first establish 3D-to-2D correspondences via networks and then use a Perspective-n-Points (PnP) solver~\cite{epnp, moreno2007accurate}
to get the pose results. However, most of them rely on the object's 3D mesh, which is one of the goals of reconstruction and is inapplicable in this work. Some recent category-level methods~\cite{category_pose_1, category_pose_2, category_pose_3} do not rely on the target's mesh explicitly by training the network with mixed data from different instances of the same category. However, they are limited to known categories in the training set and can not be generalized. By contrast, we do not rely on any category prior of the object but optimize the shape, color, and pose of the target simultaneously. On the other hand, most hand-held reconstruction methods~\cite{hhor-handold,hold} rely on the hand-object interaction, and most of them assume a firmly-grasping of the object during the whole capture to leverage hand pose estimators for pose initialization. Differently, our method does not make any assumptions about specific grasping styles and is effective for free-moving objects.
\section{Approach}
The goal of our method is to reconstruct a rigid dynamic object from a sequence of RGB images captured with a calibrated camera. We first explain our capturing and data preprocessing setup in Sec.~\ref{sec:method_setup}. We then present the object representation strategy in Sec.~\ref{sec:method_neus}, and the virtual camera system in Sec.~\ref{sec:virtual_camera}, which reduces the search space of optimization significantly. After that, we discuss segment-free progressive training in Sec.~\ref{sec:method_progressive}. We finally discuss the refinement w.r.t. the real camera and the implementation details in Sec.~\ref{sec:method_global} and Sec.~\ref{sec:impmentation_details}, respectively. We show the overview of our method in Fig.~\ref{fig:over_view}.
% A bit too long
% To solve the problem of local optimum without proper pose initialization in learning the object representation with 360 captures, we propose to progressively optimize the representation with a virtual camera, with the guidance of 2D object masks, which reduces the search space of optimization significantly (Sec.~\ref{sec:method_progressive}). To handle the approximation error between the virtual camera and the real camera, we convert the virtual-camera-based poses to the raw camera coordinate system and finally refine the results with a global optimization (Sec.~\ref{sec:method_global}). Fig.~\ref{fig:over_view} depicts the overview of our framework. Specially, given predicted masks, we find the object center, and transform the image so the object is at the frame center, which is actually our virtual camera setting.Then, based on the robust semi-dense matches from LoFTR~\cite{loftr}, we progressively estimate poses and reconstruct the mesh under virtual camera setting. Then we go back to original image frame, with raw input, refine the poses from virtual camera phase, finally generating high-quality mesh.
\subsection{Capture Setup and Data Pre-Processing}
\label{sec:method_setup}
We capture a sequence of RGB images of dynamic objects with either a fixed or egocentric camera, where the relative pose between the camera and the object is unknown.
We do not assume any object prior or any hand pose prerequisites, which allows the users to rotate the object in any grasping style or even switch between different hands during capture. Users only need to ensure the object is within the field of view of the camera, and all sides of the object are covered during capture to have a full reconstruction.
Our method relies on 2D object masks to separate the object from the background. We obtain object mask of the first frame with some simple clicks based on an interactive segmentation method~\cite{sam}, and then obtain all the object masks in the following frames with a segmentation tracking method~\cite{xmem}.
% On the other hand, we use LoFTR~\cite{loftr} to estimate semi-dense 2D matches for images within a few frame intervals, while ignoring the matching process for images that are too distant.
%This mask extraction process does not rely on any object prior and can be generalized to different types of objects, as shown in Fig.~\ref{fig:noisy_mask}.
% \begin{figure}[tb]
% \centering
% \begingroup
% \setlength{\tabcolsep}{1pt}
% \begin{tabular}{cccc}%ccc
% \includegraphics[width=0.24\linewidth]
% {Approach_Figure/progressive_pose/progressive_traj_1_v4.png} &
% \includegraphics[width=0.24\linewidth]{Approach_Figure/progressive_pose/progressive_traj_2_v4.png} &
% \includegraphics[width=0.24\linewidth]{Approach_Figure/progressive_pose/progressive_traj_3_v4.png} &
% \includegraphics[width=0.24\linewidth]{Approach_Figure/progressive_pose/progressive_traj_4_v4.png}\\
% \end{tabular}
% \endgroup
% \caption{\textbf{Segment-free Optimization On Pose and Shape }
% }
% \label{fig:progressive_pose}
% \end{figure}
\begin{comment}
Could put into Appendix
\begin{figure}[!ht]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cccc}
\includegraphics[width=0.24\linewidth]{Experiment_Figure/noisy_mask/maneki_ref1.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/noisy_mask/maneki_ref2.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/noisy_mask/maneki_ref3.png} &
% \includegraphics[width=0.15\linewidth]{Experiment_Figure/noisy_mask/maneki_ref4.png} &
\includegraphics[width=0.21\linewidth]{Experiment_Figure/noisy_mask/maneki_mesh1.png}
% \includegraphics[width=0.15\linewidth]{Experiment_Figure/noisy_mask/maneki_mesh2.png}
\\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/noisy_mask/spaceinvader_ref1.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/noisy_mask/spaceinvader_ref2.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/noisy_mask/spaceinvader_ref3.png} &
% \includegraphics[width=0.15\linewidth]{Experiment_Figure/noisy_mask/spaceinvader_ref4.png} &
\includegraphics[width=0.20\linewidth, trim={100, 0, 100, 0},clip]{Experiment_Figure/noisy_mask/spaceinvader_mesh1.png}
% \includegraphics[width=0.15\linewidth,trim={140, 40, 140, 0},clip]{Experiment_Figure/noisy_mask/spaceinvader_mesh2.png}
\\
\multicolumn{3}{c}{\small{Reference image and mask}} & \small{Reconstruction}\\
\end{tabular}
\endgroup
\caption{\textbf{Visualization of input masks.}
Despite the presence of severe occlusion and noise in the predicted masks, our method can generate accurate and complete reconstructions.
}
\label{fig:noisy_mask}
\end{figure}
\end{comment}
\subsection{Learning Object Representation}
\label{sec:method_neus}
We use the Signed Distance Function~(SDF)~\cite{neus, volsdf, unisurf} as an implicit representation for the object surface, where the surface of the object is given by the zero-level set of its SDF. We learn the SDF from an image sequence, and compare the input images with volume-rendered images~\cite{neus} after converting the SDF to a radiance field. The final surface mesh is extracted by Marching Cubes~\cite{marching-cubes}.
We follow NeuS~\cite{neus} and use MLP networks to learn the object surface and appearance:
\begin{equation}
(d(\bx),\bc(\bx,\bv)) = F_\theta(\bx,\bv),
\end{equation}
where $F_\theta$ is the MLP networks. For the 3D location $\bx$ and viewing direction $\bv$, $F_\theta$ predicts their SDF value $d(\bx)$ and RGB colors $\bc(\bx,\bv)$. In practice, we use two different MLPs to predict the surface and color field~\cite{neus}, and apply positional encoding~\cite{nerf, InstantNGP} to $\bx$ and $\bv$ to capture high-frequency signals.
We use volume rendering~\cite{volume_rendering} to optimize the implicit representation. The rendered color of each pixel is an integration of colors along the camera ray $\br$ passing through the pixel, which is usually numerically approximated in practice using quadrature~\cite{nerf}:
\begin{align}
\hat{\bC}(\br) = \sum_{k=1}^{N}w_k\bc_k
\label{eq:blending}
\end{align}
with the alpha-blending coefficient $w_k = \text{exp}(-\sum_{i=1}^{k-1}\delta_i\sigma_i)(1-\text{exp}(-\delta_k\sigma_k))
$, where $N$ is the number of sampled points along the ray $\br$, $\delta_k$ is the 3D distance between adjacent sampled points $\bx_{k}$ and $\bx_{k+1}$, and $\sigma_k$ is the volume density of $\bx_k$ after a transformation of its signed distance $d_k$~\cite{neus}. During training, the network parameters are learned using multi-view images with photometric loss:
\begin{align}
\mathcal{L}_{color} = \sum_{\br \in \mathcal{R}}\|\hat{\bC}(\br)-{\bC}(\br)\|
\label{eq:color_loss}
\end{align}
to measure the difference between the rendered color $\hat{\bC}(\br)$ and the observed color ${\bC}(\br)$ of a pixel intersected by the ray $\br$, where $\mathcal{R}$ is the set of camera rays going through sampled pixels. We use the same Eikonal loss and mask loss as in NeuS for network regularization.
% Via volume rendering, we follow the schema from NeuS\cite{neus} to learn neural implicit surface of the object in a disentangled manner\cite{idr} for geometry and appearance.
% \noindent {\bf SDF Network.} Specially, given a 3D point $\bx$ sampled on a ray $\br$, we use a shallow MLP $\theta$ predict its Signed Distance Function (SDF)~\cite{neus, volsdf, unisurf} value $d$ and geometry feature $\bg$: $(d, g) = f_{\theta}(\bx)$. Also we have the normal vector at point $\bx$ as $\bn = \nabla_{\bx}d$.
% \noindent {\bf Rendering.} To predict the color $\bc$ of a 3D point $\bx$ on the ray $\br$, we use another shallow MLP $\pi$: $\bc = f_{\pi}(\bx, \bn, \bg, \bv)$ where $\bv$ is the view direction for the ray while $\bn$ and $\bg$ are from SDF network$\theta$. Having the the color $\bc$, we follow the un-biased and occlusion-aware weight design from NeuS\cite{neus} to get $w$ for every 3D point to do composition and get rendered color for the pixel $k$ passed by ray $\br$ as $\hat{C_{k}}$.
% \noindent {\bf NeuS Supervisions.} Assuming the batch size is $m$ and we sample $n$ points on each ray, we reuse the standard losses from NeuS\cite{neus}:
% \begin{align}
% \mathcal{L}_{color} = \frac{1}{m} \underset{k}\sum (\hat{C_{k}} - C_{k})
% \label{eq:color_loss}
% \end{align}
% \begin{align}
% \mathcal{L}_{eikonal} = \frac{1}{mn} \underset{k, i}\sum(\lVert \bn_{k, i}\rVert - 1)^2
% \label{eq:eikonal_loss}
% \end{align}
% \begin{align}
% \mathcal{L}_{mask} = BCE(M_{k}, \underset{i}\sum w_{k, i})
% \label{eq:mask_loss}
% \end{align}
% \begin{equation}
% \begin{aligned}
% \mathcal{L}_{color} = \frac{1}{m} \underset{k}\sum (\hat{C_{k}} - C_{k})\\
% \mathcal{L}_{eikonal} = \frac{1}{mn} \underset{k, i}\sum(\lVert \bn_{k, i}\rVert - 1)^2 \\
% \mathcal{L}_{mask} = BCE(M_{k}, \underset{i}\sum w_{k, i})
% \label{eq:neus_losses}
% \end{aligned}
% \end{equation}
% where $k$ means $k$th ray in a batch, $i$ means $i$th point on a ray, $ C_{k}$ is GT color while $M_{k}$ is predicted mask label and $BCE$ is the binary cross
% entropy loss.
\begin{figure}[tb]
\centering
\begingroup
% \setlength{\tabcolsep}{1pt}
\begin{tabular}{ccc}
% \includegraphics[width=0.45\linewidth]{example-image}
% \includegraphics[width=0.45\linewidth]{example-image}
% \includegraphics[width=\linewidth]{Approach_Figure/front-facing.png}
% \includegraphics[width=\linewidth]{Approach_Figure/Large_Rel_Pose_row.png}
% \includegraphics[width=\linewidth]{Approach_Figure/current_problems.png}
% \multicolumn{3}{c}{\includegraphics[width=\linewidth]{Approach_Figure/cur_problems/cur_problem_seq.png}} \\
% \includegraphics[width=0.33\linewidth]{Approach_Figure/cur_problems/cur_problem_barf.png} & \includegraphics[width=0.33\linewidth]{Approach_Figure/cur_problems/cur_problem_meta.png} & \includegraphics[width=0.33\linewidth]{Approach_Figure/cur_problems/cur_problem_ours.png} \\
\includegraphics[width=0.30\linewidth]{Approach_Figure/cur_problems/cur_problem_barf_traj_v3.png} & \includegraphics[width=0.30\linewidth]{Approach_Figure/cur_problems/cur_problem_meta_traj_v3.png} & \includegraphics[width=0.30\linewidth]{Approach_Figure/cur_problems/cur_problem_ours_traj_v3.png} \\
\includegraphics[width=0.25\linewidth]{Approach_Figure/cur_problems/cur_problem_barf_mesh_v2.png} & \includegraphics[width=0.25\linewidth]{Approach_Figure/cur_problems/cur_problem_meta_mesh_v2.png} & \includegraphics[width=0.25\linewidth]{Approach_Figure/cur_problems/cur_problem_ours_mesh_v2.png} \\
\small{(a) BARF} & \small{(b) Hampali~\etal.} & \small{(c) Ours}
\end{tabular}
\endgroup
\caption{
{\bf Different methods for joint pose and shape optimization.} {\bf (a)} BARF~\cite{barf} struggles in handling 360-degree sequences. {\bf (b)} The segment-wise optimization of Hampali~\etal.~\cite{meta-obj} is local optimal and suffers in this scenario with large pose changes. {\bf (c)} Our method produces globally-consistent results. We visualize the ground truth pose and the predicted pose in \textcolor[RGB]{3,168,158}{cyan} and \textcolor[RGB]{128,0,128}{purple}, respectively.
}
\label{fig:compare_barf_and_ours}
\end{figure}
\begin{figure}[tb]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cc}%ccc
% \includegraphics[width=0.45\linewidth]{example-image}
% \includegraphics[width=0.45\linewidth]{example-image}
% \includegraphics[width=0.24\linewidth]{example-image} &
% \includegraphics[width=0.48\linewidth]{Approach_Figure/Camera_Difference.png}
% \includegraphics[width=\linewidth]{Approach_Figure/origin_virtual_row.png}\\
% \includegraphics[width=\linewidth]{Approach_Figure/original_virtual_heat.png}
% \includegraphics[width=\linewidth]{Approach_Figure/original_virtual_heat_gaussian_v2.png}
\includegraphics[width=0.45\linewidth]{Approach_Figure/vc_effects/mc1_ref.png} &
\includegraphics[width=0.45\linewidth]{Approach_Figure/vc_effects/mc1_virtual.png} \\
\includegraphics[width=0.45\linewidth]{Approach_Figure/vc_effects/before.png} &
\includegraphics[width=0.45\linewidth]{Approach_Figure/vc_effects/after.png} \\
\small{Real camera} & \small{Virtual camera} \\
% & \small{RC statistics} & \small{VC statistics}
\end{tabular}
\endgroup
\caption{\textbf{Effect of the virtual camera.}
The top row shows the trajectory of the object w.r.t. the real camera and the virtual camera, respectively. The bottom row shows the heatmap of 2D reprojections of the 3D object center across the whole HO3D dataset w.r.t different camera systems. The poses w.r.t the virtual camera do not have significant magnitude in both horizontal and vertical directions, which allows the poses to be approximately captured by only 4 degrees of freedom~(3 for rotation and 1 for distance).
% \HS{I think we can improve this figure like bird-view of pose trajectories of virtual camera and original camera: under virtual camera, we are always looking at the object center while in original camera, there is a large angle.}}(\textit{right twos are heat-maps of object center w/o or w virtual camera.}) \YH{Show the different difficulty levels of the two types of trajectories.
%\YH{Please add image}%pose trajectories between the real and virtual camera.
% \includegraphics[height=5cm]{Approach_Figure/Deformation-Rigid-Body.png}
% \caption{\textbf{Dynamic Scene Deformation And Rigid Object Stands Out.} We are modeling the whole dynamic scene with fine-grained rigid-body transformation. If the object is rigid, poses for all of its pixels should follow a global rigid body transformation. \emph{We may replace it with mesh instead of camera point cloud :)}
}
\label{fig:vc_effect_short_demo}
\end{figure}
\subsection{Optimization with Guided Virtual Camera}
\label{sec:virtual_camera}
To optimize the object representation discussed in the above section, it is essential to specify both the origin and direction of camera rays or camera/object poses.
We do not assume the existence of such poses in our setting. Similar to~\cite{barf, meta-obj, nope-nerf, nope-neus}, we optimize the camera poses and the object representation simultaneously. Since camera rays are functions of camera parameters, we condition the camera rays in Eq.~\ref{eq:color_loss} on learnable camera poses, as illustrated as the Pose MLP in Fig.~\ref{fig:over_view}. We assume the camera intrinsics and lens distortions of the camera are known and only optimize the camera pose in this work.
Typically, joint optimization of camera pose and radiance field can only handle forward-facing scenarios and fails to converge if the images cover a larger range of viewpoints or there are some large pose changes between consecutive images, as shown in Fig.~\ref{fig:compare_barf_and_ours}.
% \YH{Add a small figure to show this and highlight our advantages.}
To handle this problem, we propose to solve the optimization problem w.r.t. a new virtual camera system with the guidance of 2D object masks.
% Specially, we propose to create a virtual camera and then optimize the radiance field and the object pose w.r.t. the virtual camera.
Specially, given a 3D point $\bx$ and the camera intrinsic matrix $\bK$, we have:
\begin{equation}
\begin{aligned}
\bu = \bK(\bR{\bf x}+{\bf t}),
\end{aligned}
\label{eq:perspective}
\end{equation}
where $\bu$ is the reprojected pixel location on the image, and $\bR$ and $\bt$ are the 3D rotation and 3D translation respectively. On the other hand, given the object mask obtained in Sec.~\ref{sec:method_setup}, we crop the object with a transformation matrix $\bM$, and with a virtual camera whose intrinsic matrix is $\bK_v$, we have:
\begin{equation}
\begin{aligned}
\bM\bu & = \bM\bK(\bR{\bf x}+{\bf t}) \\
& = \bK_v\bK_v^{-1}\bM\bK(\bR{\bf x}+{\bf t}) \\
& = \bK_v(\bK_v^{-1}\bM\bK\bR{\bf x}+\bK_v^{-1}\bM\bK{\bf t}), \\
\end{aligned}
\label{eq:virtual_camera_geo}
\end{equation}
where $(\bK_v^{-1}\bM\bK\bR, \bK_v^{-1}\bM\bK{\bf t}) \rightarrow (\bR_v, \bt_v)$ are the 3D rotation and 3D translation w.r.t. the new virtual camera. While note that $(\bR_v, \bt_v)$ is not physically-compliant and can only be approximately estimated. Instead of optimizing $(\bR, \bt)$ directly, we optimize $(\bR_v, \bt_v)$ and the object representation simultaneously w.r.t. the virtual camera.
Since the translation of the target w.r.t. the new virtual camera does not have significant magnitude in both horizontal and vertical directions, as illustrated in Fig.~\ref{fig:vc_effect_short_demo}, we only predict the rotation and the distance from the camera center to the object center for object poses, which has only 4 degrees of freedom. This simplification of pose formulation reduces the search space of network optimization, and our experiments will show it increases the performance significantly.
% Fig.~\ref{fig:vc_effect_short_demo} illustrates this effects, where the right two sub-figures are heat-maps of object center in the whole dataset before and after using virtual camera.
% a bit long
% Empirically, we found that after centralizing the object, in world coordinate there is a very small angle between camera view direction and connected vector from camera position to world origin. Therefore, we typically would have minor translations that are orthogonal to the view direction of camera, which can be ignored during virtual camera pose estimation phase. Therefore, we only need to estimate the translation along the camera view direction. Specially, given $(0, 0, 1)$ unit vector in camera coordinate, the view direction of camera in world coordinate can be retrieved from the last column of $\bR_v$ as $\bR_v[:, 2]$, then we also need to estimate one scale change $t$ along view direction to determine the position of camera: $(\bR_v, \bt_v) \rightarrow (\bR_v, s*\bR_v[:, 2])$, which finally gives us 4 degree of freedom to estimate, namely 3 for $\bR_v$ and 1 for $s$ and reduce the search space a lot.
\begin{comment}
\subsubsection{Relative Pose Initialization}
\HS{The effectiveness of this part for RGB-based method is still under experiments; but this part is very effective for RGB-D method where we replace render depth with GT depth.}
We observe that with large relative pose between two consecutive frames, the system will need more time to converge, during which a lot of noisy rays are emitted to the NeuS system, leading to degradation of shape. To solve this, we propose to learn relative pose based on PnP algorithm\cite{epnp} with rendered depth and Loftr matches. Specially, similar to equation\ref{eq:blending}, we can get the rendered depth with:
\begin{align}
\hat{\bD}(\br) = \sum_{k=1}^{N}w_k\bd_k
\label{eq:blending_depth}
\end{align}
Assume we want to initialize camera to world $Pose_{2}$ from image $I_{2}$ based on relative pose with converged camera to world $Pose_{1}$ for image $I_{1}$ . With render depth from $I_{1}$, we can get corresponding camera coordinate points in $I_{1}$ and based on Loftr matches we can find 2D correspondence in $I_{2}$, finally using RANSAC EPnP~\cite{epnp} to get relative transform $T_{cam_{1} \rightarrow cam_{2}}$ from camera spaces of $I_{1}$ to camera spaces of $I_{2}$. Then we can get the initialized $Pose_{2}$ based on the inverse transform of $T_{cam_{1} \rightarrow cam_{2}}$ :
\begin{align}
Pose_{2} = Pose_{1} T_{cam_{2} \rightarrow cam_{1}}
\label{eq:relative pose}
\end{align}
The reason we select this expression is to avoid the ambiguity of PnP without proper pose initialization: if the 2D correspondence are from the same plane, then there will be possible "flip" pose returned that has much larger error than without initialization.
\end{comment}
% To handle 360-degree captures, we first optimize the object representation and $(\bR_v, \bt_v)$ with the first $B$ images, and then simply add $B$ followed images in the sequence progressively when the previous batches are converged. Thanks to our simplified trajectory with virtual camera, our progressive strategy does not rely on any specific segment selection procedure~\cite{meta-obj}, and we found that the simple progressively-adding already be accurate and generalizes well to different sequences.
\subsection{Segment-Free Progressive Training}
\label{sec:method_progressive}
Although the virtual camera system reduces the search space of optimization, it still can not handle 360-degree sequences. We use progressive training to process the images in the sequence to leverage the temporal information between consecutive images~\cite{meta-obj, 3dgs}.
% \begin{figure}[tb]
% \centering
% % \includegraphics[width=0.45\linewidth]{example-image}
% % \includegraphics[width=0.45\linewidth]{example-image}
% % \includegraphics[width=\linewidth]{Approach_Figure/loftr_matches.png}
% \includegraphics[width=\linewidth]{Approach_Figure/loftr_matches_update.png}
% \caption{
% {\bf x.}
% \YH{Show some visualization of 2D matches across multiple frames}
% }
% \label{fig:matches_across_frames}
% \end{figure}
\begin{figure}[tb]
% \centering
\begingroup
\setlength{\tabcolsep}{0.3pt}
\begin{tabular}{ccc}
\includegraphics[width=0.33\linewidth]
{Approach_Figure/progressive_pose/progressive_traj_1_v3.png} &
\includegraphics[width=0.33\linewidth]{Approach_Figure/progressive_pose/progressive_traj_2_v3.png} &
\includegraphics[width=0.33\linewidth]{Approach_Figure/progressive_pose/progressive_traj_4_v3.png}\\
\small{$t_{1}\rightarrow t_{1}+$10} & \small{$t_{2}\rightarrow t_{2}+$10} & \small{with refinement} \\
\end{tabular}
\endgroup
\caption{
\textbf{Progressive training and global refinement.}
The first two figures show the pose and shape results of two examples during progressive training. The result improves with more images involved. The last figure shows the result with global refinement, which improves the performance further.
}
\label{fig:progressive_pose}
\end{figure}
We use 2D matches between different images to facilitate the optimization:
\begin{equation}
\begin{aligned}
\mathcal{L}_{match}(\br_a, \br_b) = \sum_{k=1}^N (w_{ak}\cdot\|g_b(\bx_{ak}) - \bu_b\|_1) \\ + \sum_{k=1}^N (w_{bk}\cdot\|g_a(\bx_{bk}) - \bu_a\|_1),
\end{aligned}
\label{eq:match_loss}
\end{equation}
where $\br_a$ and $\br_b$ are two camera rays passing through the matched pixel coordinates $\bu_a$ and $\bu_b$ between two different images $a$ and $b$, $\{\bx_{ak}\}$ and $\{\bx_{bk}\}$ are the sampled points along the ray $\br_a$ and $\br_b$, $g_a(\cdot)$ and $g_b(\cdot)$ are the 2D reprojection functions according to the current predicted poses of images $a$ and $b$, $w_{ak}$ and $w_{bk}$ are the predicted weights for the points on the ray.
We use LoFTR~\cite{loftr} to generate sparse 2D matches for images within a few frame intervals~(typically less than 10). We randomly sample available 2D matches between different frames during training.
% To provide a rough relative pose between the consecutive frames which helps the optimization in scenarios with large movements, we use the rendered depth and the 2D matches between the consecutive frames to establish a set of 3D-to-2D correspondences, and use a PnP solver~\cite{1} to estimate the pose of the second frame, which servers as its pose initialization during the optimization.
Another challenge of pose-free reconstruction is the large pose changes where we observe that the newly appeared surface degrades the previously reconstructed shape. To help the networks learn the pose of newly added images, we reset the weights of the shape network when the relative pose of the current frame exceeds a rotation threshold $\tau$ from the last reset, and only keep the weights of the Pose MLP to preserve the learned pose information of previous images. We typically set $\tau$ as 60$\degree$ in our experiments.
% For progressive optimization, we have supervisions:
% \begin{equation}
% \begin{aligned}
% \mathcal{L}_{pro} = \mathcal{L}_{color} + \lambda\mathcal{L}_{eikonal} + \beta\mathcal{L}_{mask} + \gamma\mathcal{L}_{match}
% \end{aligned}
% \end{equation}
% We observe that when the new surface of the object appears or even sharp movement to a new surface, the pose accuracy will degrade sharply since the original NeuS system is over fitting to old surfaces and the new surface is not reconstructed well. This new surface appearing problem was also explored in Hampali \etal.\cite{meta-obj} where they use the maximum and minimum of mask area to segment the sequences. We think this type of segmentation is not able to solve the symmetric objects or heavy occlusions. Instead, we propose to use relative rotation predicted by the system itself to be an indicator to reset the NeuS system. Specially, at the beginning, we have $Pose_{0}$ from the first frame as the reference frame, in the future when we finish predicted $Pose_{i}$ for the frame $i$, if the relative rotation angle between $Pose_{0}$ and $Pose_{i}$ is larger than a threshold, and then we will reset the NeuS related networks but maintain the Pose MLP and restart the training. After this, frame $i$ along with $Pose_{i}$ will be our new reference frame. The resetting process will repeat whenever we reach the angle threshold.
\subsection{Refinement with Real Camera}
\label{sec:method_global}
The optimization target $\{(\bR_v, \bt_v)\}$ of previous sections is based on the virtual camera and not physically-compliant.
It is usually not accurately aligned with the original optimization target $\{(\bR, \bt)\}$.
To address this problem, we use a PnP solver to transform the predicted pose from the virtual camera system back to the real camera system and refine the results w.r.t. the real camera.
Specifically, we sample a set of 3D points from the reconstructed mesh in previous sections, and re-project them to the virtual image plane with the predicted $(\hat{\bR}_v, \hat{\bt}_v)$.
After transforming the reprojected 2D position with $\bM^{-1}$ in Eq.~\ref{eq:virtual_camera_geo}, we establish a set of 3D-to-2D correspondences between 3D points and 2D image locations on the raw image, we use RANSAC EPnP~\cite{epnp} to compute $(\hat{\bR}, \hat{\bt})$ which is the estimated object pose w.r.t. the real camera. We then conduct a global optimization with all available images starting from $\{(\hat{\bR}, \hat{\bt})\}$. We do not use the match loss in this procedure, and Fig.~\ref{fig:progressive_pose} shows some results.
% since it can contain noises and for small adjustment, color loss is enough; and we adopt the ideas from Color-NeuS\cite{colorneus} that set background pixels to zero while removing mask supervision:
% \begin{equation}
% \begin{aligned}
% \mathcal{L}_{global} = \mathcal{L}_{color} + \lambda\mathcal{L}_{eikonal}
% \end{aligned}
% \end{equation}
\subsection{Implementation Details}
\label{sec:impmentation_details}
\noindent {\bf Networks.} Besides the standard SDF MLP and Color MLP used in standard NeuS~\cite{neus}, we use another small Pose MLP to estimate the object pose of the target in every frame.
% and define camera rays in NeuS system.
Similar to~\cite{NeRFtrinsic_Four}, we map each frame ID to a 256-dim feature vector based on Gaussian Fourier features and then use 3 layers of MLP with GELU activation functions to output the poses.
% the Pose MLP is modeled with 3 layers with a hidden size of 64 and GELU activation functions, and after output linear layer, we apply $SO(3)$ as rotation prediction, and 3 dimension for translation; the input frequency parameter m is based on Gaussian Fourier feature mapping and as default set to 128, which results in an embedding size of 256.
% Such high dimension space gives more freedom to learn and we found that a pure vector in $SE(3)$ like BARF\cite{barf} will typically fail in this task due to its low expression ability.
We use a single ADAM optimizer~\cite{adam} for SDP MLP and Color MLP. During training, the learning rate warms up linearly from 0 to 5e-4 during the first 5k iteration and then follows a cosine decay schedule with alpha=0.05. For Pose MLP, we use another ADAM optimizer with a cosine decay schedule of alpha=0.5.
% For loss weight parameters, we pick $\lambda$ = 0.1, $\beta$=5, $\gamma$=0.1.
% for each $B\text{=1}$ consecutive images respectively
\noindent {\bf Sampling.} We use an unit sphere for the initialization of the SDF network, and define the sampling range~(i.e., near and far) within the unit sphere.
For each training step, we randomly sample 512 rays from the input image batch. During the optimization with guided virtual camera, we only sample 32 points along each ray for efficiency.
We progressively train our model with $B$ consecutive images as a group. For every group, we train the networks with a fixed number of training steps (typically 1K). We sample 20\% of the rays from images within previously-converged groups and 80\% from the images within the newly added group.
% after 1k iterations for each with the virtual camera and the typical training iteration is the number of frames times 1k.
% since we found that complex sampling strategy can not only do harm to pose estimation but also make training time-consuming;
In the phase of refinement with the real camera, we use importance-based hierarchy sampling strategies~\cite{neus} to uniformly sample 64 points and then sample another 64 points based on the current predicted SDF values.
% Also, to avoid the sharp change from newly adding frame, we would freeze previous Pose MLP at the first half of 1k iteration and then at the second half optimize all existing Pose MLP together, during which we sample 20\% of the rays from the previously-converged images inside match interval and 80\% for the newly added images.
% Since we have resetting schema, simultaneously, we would independently sample another batch uniformly from seen images to maintain shape.
We train the networks for 150K training steps for refinement. On a typical NVIDIA V100 GPU, the training of a 100-frame sequence takes about 10 hours including both the initialization and refinement.
\section{Experiments}
% BB12 threshold is 9.00
% pose MLP GPMF12: 1.414759
% pose MLP BB12: 2.62
\begin{table*}
\centering
\begin{tabular}{lcccccccccc}
\toprule
% \multirow{2}{*}{Methods} & cracker & sugar & mustard & bleach & potted & power & pitcher & \multirow{2}{*}{mug} & \multirow{2}{*}{banana} & \multirow{2}{*}{Average} \\
Methods & cracker & sugar & mustard & bleach & meatcan & driller & pitcher & mug & banana & Average \\
\midrule
COLMAP~\cite{colmap} & 4.08 & 6.66 & 4.43 & 14.11 & 10.21 & 11.06 & 43.38 & - & - & 13.41\\
Ye~\etal.~\cite{ihoi}& 10.21 & 6.19 & \textbf{2.61} & \textbf{4.18} & 3.43 & 15.15 & 8.87 & - & \textbf{3.47} & 6.76\\
Hampali~\etal.~\cite{meta-obj} & 2.91 & 3.01 & 4.44 & 5.63 & 1.95 & 5.48 & 9.21 & 4.53 & 4.60 & 4.64\\
% the following is by manual align initialization
% \textbf{Ours} & \textbf{1.87}& \textbf{2.06} & 2.70 & \textbf{3.93} & \textbf{1.78} & \textbf{3.37} & \textbf{2.77} & \textbf{2.36} & 4.18 & \textbf{2.78}\\%-RGB
% the following is by automatic algorithm initialization
\textbf{Ours} & \textbf{1.71}& \textbf{1.84} & 3.49 & 5.38 & \textbf{1.80} & \textbf{3.82} & \textbf{2.84} & \textbf{2.78} & 4.54 & \textbf{3.14}\\
\midrule
UNISURF~\cite{unisurf} & 3.40 & 3.49 & 4.34 & 3.41 & 1.54 & 5.33 & 4.63 & - & 3.98 & 3.76 \\
NeuS~\cite{neus} & \textbf{1.75} & \textbf{1.69} & 2.34 & 3.35 & \textbf{1.17} & \textbf{3.13} & 3.48 & \textbf{2.19} &\textbf{2.08} & \textbf{2.35}\\
% NeuS(refine) & 2.13 & 1.84 & \textbf{2.24} & 3.04 & 4.00 & \textbf{2.08} & \textbf{2.55}\\
Patten~\etal.~\cite{rgb-d-baseline} & 3.54 & 3.34 & 3.28 & \textbf{2.43} & 3.26 &3.77 & 4.73 & 4.22 & 2.44 & 3.45\\
% \textbf{Ours-RGBD} & x & x & x & x & x & x & x\\
\bottomrule
\end{tabular}
\caption{\textbf{Mesh evaluation in $HD_{RMSE}\downarrow$.}
Our method outperforms most pose-free methods~(top group), and is on par with methods trained based on ground truth poses~(bottom group).
}
%MC1: cracker box; ShSu12: suger box; SM2: mustard, ABF14: bleach; GPMF12: potted meat; MDF14: power drill; AP13: pitcher base, SMu1: mug, BB12: banana.
\label{tab:recon_comp_ho3d}
\end{table*}
% TODO
\begin{table*}
\centering
\begin{tabular}{lcccccccccc}
\toprule
% \multirow{2}{*}{Methods} & cracker & sugar & mustard & bleach & potted & power & pitcher & \multirow{2}{*}{mug} & \multirow{2}{*}{banana} & \multirow{2}{*}{Average} \\
Methods & cracker & sugar & mustard & bleach & meatcan & driller & pitcher & mug & banana & Average \\
\midrule
COLMAP~\cite{colmap} & 7.4 & 7.4 & 3.5 & 1.5 & 0.1 & 2.8 & 4.1 & 2.4 & 0.0 & 2.9\\
Hampali~\etal.~\cite{meta-obj} & \textbf{7.6} & 6.8 & \textbf{5.2} & 4.7 & \textbf{6.8} & 6.4 & 4.6 & 2.2 & \textbf{0.6} & 4.5\\
\textbf{Ours} & \textbf{7.6} & \textbf{7.6} & 4.2 & \textbf{5.3} & 3.1 & \textbf{8.5} & \textbf{8.7} & \textbf{8.1} & 0.3 & \textbf{5.9}\\%-RGB
\bottomrule
\end{tabular}
\caption{\textbf{Pose evaluation in $AUC_{ATE}\uparrow$.} Our method produces more accurate pose result on most objects.
}
\label{tab:pose_comp_ho3d}
\end{table*}
We evaluate our method systematically in this section. We first introduce the datasets and metrics used in our experiments and then demonstrate the effectiveness of our method by comparing with state-of-the-art methods. We then conduct extensive ablation study to validate different components of our method.
\noindent {\bf Datasets.} We first evaluate our method on the standard HO3D dataset ~\cite{ho3d}, which includes video captures of daily objects with a fixed camera. Since most objects in HO3D are manipulated by one hand, and there exists a fixed relative pose between the hand and the object in most sequences, we collect a more general dataset of free-moving objects with a head-mounted device with egocentric views, where the objects are manipulated by both hands with a free manipulation style.
% Let us now evaluate our method. To compare with the state-of-the-art methods, we first evaluate on the HO3D dataset~\cite{ho3d}, which includes video captures of daily objects with a fixed camera. However, HO3D primarily focuses on evaluating the interactions between hands and objects, and most objects is manipulated by one hand, and often there exists a fixed relative pose between the hand and pose in most sequences. To verify the effectiveness of our methods for free-moving objects, we also captured a dataset of free-moving objects, where the objects are manipulated by both hands with freely grasping style. We capture the data with a head-mounted device where the camera is also moving during the capture.
%Our method predicts both object pose and shape reconstruction.
\noindent {\bf Metrics.} For object reconstruction, we first align the predicted mesh to the GT mesh based on scale-aware ICP~\cite{meta-obj}, then calculate the root mean square of the Hausdorff distance~(mm)~\cite{meta-obj, rgb-d-baseline} between the predicted mesh and the GT mesh ($HD_{RMSE}$). For object pose estimation, we first align predicted poses to GT poses based on similarity transforms~\cite{meta-obj, nope-nerf, 3dgs}, and then compute the area under curve~(AUC) with a threshold of 10cm in Absolute Trajectory Error (ATE)~($AUC_{ATE}$)~\cite{meta-obj}.
% from a plot where y-axis is the percentage of frames less than an ATE threshold while x-axis is thresholds up to 10cm
We also report the pose accuracy in Relative Pose Error~(RPE)~\cite{nope-nerf, 3dgs}, including $RPE_r$~(degree) and $RPE_t$~(cm), corresponding to the rotation error and the translation error, respectively.
% We evaluate both the object reconstruction and pose estimation. For object reconstruction, similar to Hampali \etal.\cite{meta-obj} and Ye \etal.\cite{ihoi}, we use RMSE Hausdorff distance(mm)\HS{In \cite{rgb-d-baseline}, it mentiones "Vertices
% in the reconstruction are sampled and the Euclidean distance
% of each vertex to the nearest vertex in the precise model is
% computed. The root mean square over all distances is reported."} as the metric for quantitative evaluation. We get ground truth YCB-V object\cite{ycbv} meshes from BOP challenges\cite{BOP}, which are adjusted to mm unit. Following \cite{rgb-d-baseline, meta-obj}, we use MeshLab to align estimated mesh to ground truth mesh based on scale-aware ICP algorithm and then we calculate RMSE Hausdorff distance(mm) from internal functions in MeshLab. For object pose estimation, we report the camera rotation and translation error, including the Absolute Trajectory Error (ATE) and Relative Pose Error (RPE) for translations and rotations separately as in \cite{nope-nerf, 3dgs}. For comparison with the state-of-the-arts, we mainly focus quantitative and qualitative analysis of the reconstructed mesh quality. For comprehensive ablation study, we mainly focus on evaluating the effectiveness of different components of our system based on pose accuracy.
\subsection{Evaluation on HO3D}
We report results on the 9 sequences of HO3D as in~\cite{meta-obj, ihoi}, and compare our method with pose-free methods including COLMAP~\cite{colmap}, Ye~\etal.~\cite{ihoi}, and Hampali~\etal.~\cite{meta-obj}, where Ye~\etal. relies on prior hand pose information.
To further validate our method, we compare our reconstructed mesh with the results of UNISURF~\cite{unisurf} and NeuS~\cite{neus} which are trained with ground truth poses. We also compare with another method Patten~\etal.~\cite{rgb-d-baseline} that rely on additional depth images.
% HO3D contains many multi-view sequences with YCB objects~\cite{ycbv}. Some of our selected sequences are only annotated with partial ground truth pose(\textit{some frames miss annotation}). We pick the 6 sequences reported in~\cite{meta-obj} for evaluation. Selected sequences contain objects in different level of textures: for rich texture objects, we have cracker box(MC1) and sugar box(ShSu12); for semi-rich textured objects, we have mustard(SM2) and bleach(ABF14); for textured-less objects, we have power drill(MDF14) and pitcher base(AP13). Also for grasping gesture of hands, we have both fixing gestures including box(MC1), mustard(SM2), pitcher base(AP13), and also dynamic hand gestures including sugar box(ShSu12), bleach(ABF14) and power drill(MDF14).
% Following Hampali \etal.\cite{meta-obj}, we have two types of baselines. The first one is RGB-based pose-free methods, including Hampali \etal.\cite{meta-obj} itself, also one of the classic object pose estimation based on hand poses by Ye \etal.\cite{ihoi}, and finally the traditional algorithm: COLMAP \cite{colmap}.
% Another parts are strong baselines, including UNISURF~\cite{unisurf} and NeuS~\cite{neus} with ground truth poses while another one by Patten \etal.\cite{rgb-d-baseline} that assumes ground truth depth is available.
% Notice that NeuS~\cite{neus} and UNISURF~\cite{unisurf} are concurrent works that improve the standard NeRF\cite{nerf} representation by combining implicit surface representation and volume rendering. And for sure, NeuS~\cite{neus} and UNISURF~\cite{unisurf} are designed for static-scene reconstruction and rely on provided camera poses.
% We borrow the results from Hampali \etal.\cite{meta-obj} which is a strong RGB-based pose-free method that utilizes the UNISURF~\cite{unisurf} as backbone. We ensure our results are comparable with the borrowed results, since we follow the same sequences selected by Hampali \etal.\cite{meta-obj}, and also we follow its frame intervals for each sequences stated in its appendix. Finally, we follow the same evaluation metrics as RMSE Hausdorff distance(mm). Specially, we download the point cloud from project page from Hampali \etal.\cite{meta-obj}. And then we align estimated meshes to ground truth meshes based scale-aware ICP algorithm in Meshlab\cite{meshlab}, and then we calculate the RMSE Hausdorff distance(mm) by adjusting the maximum distance threshold so we can recover the numbers of Hampali \etal.\cite{meta-obj}'s method, and then finally we apply the same threshold on our methods and also NeuS with GT poses.
% For all additional settings, namely Ours and NeuS with GT poses, we apply same predicted masks.
% Also, since Hampali \etal.\cite{meta-obj} didn't offer the frame ids that they select for each sequences and their solution is close-source, we are not able to compare the pose accuracy with them.
We summarize the reconstruction results in Table~\ref{tab:recon_comp_ho3d}.
COLMAP can not produce accurate results on most objects, mainly caused by the lacking of enough textures, especially for objects like bleach and pitcher in the table.
With prior hand pose information, Ye’s method produces better results than COLMAP. With carefully selected multiple easy segments, Hampali’s method outperforms Ye's method. Nevertheless, our method produces better results than most of them, and is even on par with methods relying on ground truth poses (UNISURF and NeuS) or depth images (Patten's method). Fig.~\ref{fig:recon_comp} shows some visualization results of the reconstruction meshes.
% As shown in Table~\ref{tab:recon_comp_ho3d}, we show the quantitative results comparing with other methods in terms of RMSE Hausdorff distance (mm), the lower the better. As for traditional algorithm COLMAP \cite{colmap}, we outperform it with an extreme large margin from 13.95mm to 3.14mm in average. And we can also observe that with the extent of lacking texture increases, the traditional algorithm perform worse, especially for texture-less pitcher base(AP13), which reaches a high error distance as 43.38mm. This shows that traditional algorithms heavily rely on key point matches to infer the poses and further reconstruct mesh. However, our method can stably work well for objects in different texture levels. Also our method outperforms existing state-of-art no-prior Hampali \etal.\cite{meta-obj} method for all objects by large margin from 5.11mm to 3.14mm in average. Also especially for texture-less pitcher base(AP13), for which we can unexpectedly get a perfect shape(as shown in the last row of Figure~\ref{fig:recon_comp}) compared with Hampali \etal, from the numbers reported, our error distance drops from 9.21mm to 3.33mm for this texture-less pitcher base(AP13).
% Though Ye \etal. \cite{ihoi} relies on the robust hand pose prior, we are better than their methods for most of the objects in average from 7.87mm to 3.14mm. This method typically relies on the firm gesture of hand on the object to get reasonable results and it is pre-trained on sequences that contain the specific object. Notice that all objects in HO3D are unknown to our method. Ye \etal. \cite{ihoi} can generate the object shape based on only one image input but it will suffer from the problem that the pre-train datasets contain a lot of frames that the gesture between hand and object are not consistent. Therefore if in all frames the gestures between hand and object are consistent then the Ye \etal. \cite{ihoi} will perform well, if not then it typically outputs bad results. Our method performs better than this method for most of objects since we are not affected by the hand fixing gesture prior assumption. However, our method performs worse for object mustard(SM2), and we observe that for this object the hand gestures among sequences are firmed.
% As for the strong baseline as UNISURF~\cite{unisurf} and NeuS~\cite{neus} with ground truth poses, we found that our performance is comparable with them. We also notice that NeuS~\cite{neus} performs better than UNISURF~\cite{unisurf} given ground truth poses under HO3D dataset, which somehow improves performance upper limit of our model compared with Hampali \etal.\cite{meta-obj} that uses UNISURF~\cite{unisurf} as backbone. We also found that our method even outperforms NeuS~\cite{neus} with ground truth poses for some objects and we think this is because of the inaccurate pose annotation of HO3D dataset. In the future, we will try another stronger baseline: NeuS~\cite{neus} with ground truth poses but also with BARF\cite{barf} to do minor adjustments. Also there is another strong baseline by Patten \etal.\cite{rgb-d-baseline} that assumes ground truth depth is available. We found that for texture-rich objects including cracker box(MC1) and sugar box(ShSu12), depth may not be extremely needed, most of the RGB-based methods including ours and other baselines can get more reasonable results; then in this case, our method can effectively reconstruct the objects that have better quality than Patten \etal. However, for less-textured objects like bleach(ABF14), our method performs worse than this method since this method can rely ground truth depth to reduce geometry ambiguity. From a comprenhensive perspective, in average, our method have higher performance from 3.34mm to 3.14mm compared with Patten \etal. that assumes ground truth depth.
Table~\ref{tab:pose_comp_ho3d} shows the evaluation of pose results. Our method outperforms COLMAP and Hampali’s method significantly.
% We first show visualization of meshes reconstructed by different methods in Figure~\ref{fig:recon_comp}. For rich textured objects including cracker box(MC1) and sugar box(ShSu12), both our methods and Hampali \etal. can generate good results compared to NeuS with GT poses. Notice that all of the three methods are RGB-based so typically the surface would not be very smooth as reality due to the lack of depth information. However, we can find that cracker box and sugar box from Hampali \etal have more unordered shallow holes in the surface and the sugar box suffered more. For semi-textured object mustard(SM2), both ours and Hampali \etal have good shapes for visible parts compared to NeuS with GT poses but for the occluded parts, Hampali \etal tends to generate a lot of noises. The reason that we can omit the occlusion is by borrowing the simple trick from Color-NeuS\cite{colorneus}, where we put colors of all of pixels outside of the masked area to zero, and remove the standard mask loss supervision in standard NeuS\cite{neus}. The key reason is that we should not force the blending weight of parts that are occluded by hands to be zero like the standard NeuS since under the occlusion there actually exist object. Same trick is also applied to our baseline as NeuS with GT pose. For semi-textured object bleach(ABF14), both Hampali \etal and ours generate low quality of mesh compared to NeuS with GT poses. We observe that there are two reasons: first there is a sharp movement in between(see limitation parts later); and also we lack the texture for the side surfaces. Still we have more smooth shape for bleach(ABF14) compared with Hampali \etal. Finally, we start looking at the challenging texture-less objects including power drill(MDF14) and pitcher base(AP13). We observe that NeuS with GT poses didn't offer us high quality meshes though it has pose annotations, for which we doubt that the ground truth annotation is not completely accurate and texture-less object reconstruction is more sensitive to the accuracy of poses. Our method can still generate much smoother meshes compared with Hampali \etal and even NeuS with GT poses. Especially, our method is pose-free, we can reconstruct the very challenging texture-less pitcher base(AP13) very well, which exceeds the existing RGB-based methods a lot, also see our quantitative results in Table~\ref{tab:recon_comp_ho3d}.
\begin{figure}[!ht]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cccc}
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/MC1.png} &
\includegraphics[width=0.24\linewidth,trim={50, 0, 50, 0}]{Experiment_Figure/Final_Recon_Com/MC1_GT02.png} &
\includegraphics[width=0.24\linewidth,trim={50, 0, 50, 0}]{Experiment_Figure/Final_Recon_Com/MC1_Meta01.png} &
\includegraphics[width=0.24\linewidth,trim={50, 0, 50, 0}]{Experiment_Figure/Final_Recon_Com/MC1_Ours00.png}
% \includegraphics[width=0.15\linewidth,trim={50, 0, 50, 0}]{Experiment_Figure/Final_Recon_Com/MC1_Pose03.png}
\\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/ShSu12.png} &
\includegraphics[width=0.24\linewidth,trim={50, 0, 50, 0},clip]{Experiment_Figure/Final_Recon_Com/ShSu12_gt02.png} &
\includegraphics[width=0.24\linewidth,trim={50, 0, 50, 0},clip]{Experiment_Figure/Final_Recon_Com/ShSu12_meta01.png} &
\includegraphics[width=0.24\linewidth,trim={50, 0, 50, 0},clip]{Experiment_Figure/Final_Recon_Com/ShSu12_ours00.png}
% \includegraphics[width=0.15\linewidth,trim={50, 0, 50, 0}]{Experiment_Figure/Final_Recon_Com/ShSu12_pose03.png}
\\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/SM2.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/SM2_gt02.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/SM2_meta01.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/SM2_ours00.png}
% \includegraphics[width=0.15\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/SM2_pose03.png}
\\
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/ABF14.png} &
% \includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/ABF14_gt02.png} &
% \includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/ABF14_meta01.png} &
% \includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/ABF14_ours00.png}
% \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/SMu1.jpg} &
\includegraphics[width=0.24\linewidth,trim={30, 0, 30, 0}]{Experiment_Figure/Final_Recon_Com/SMu1_gt01.png} &
\includegraphics[width=0.24\linewidth,trim={30, 0, 30, 0}]{Experiment_Figure/Final_Recon_Com/SMu1_meta00.png} &
\includegraphics[width=0.24\linewidth,trim={30, 0, 30, 0}]{Experiment_Figure/Final_Recon_Com/SMu1_ours02.png}
% % \includegraphics[width=0.15\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/ABF14_pose03.png}
\\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/MDF14.png} &
% \includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/MDF14_ours00.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/MDF14_gt02.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/MDF14_meta01.png} &
\includegraphics[width=0.24\linewidth,trim={100, 30, 100, 0},clip]{Experiment_Figure/Final_Recon_Com/MDF14_ours_comp04.png}
% \includegraphics[width=0.15\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/MDF14_pose03.png}
\\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/AP13.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/AP13_gt02.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/AP13_meta01.png} &
\includegraphics[width=0.24\linewidth,trim={60, 0, 60, 0},clip]{Experiment_Figure/Final_Recon_Com/AP13_ours00.png}
% \includegraphics[width=0.15\linewidth,trim={60, 0, 60, 0}]{Experiment_Figure/Final_Recon_Com/AP13_pose03.png}
\\
% Reference Image & Ours & Hampali \etal. & NeuS(GT) & NeuS(Refine GT)\\
\small{Reference} & \small{NeuS} & \small{Hampali \etal.} & \small{Ours}\\
\end{tabular}
\endgroup
\caption{\textbf{Reconstruction result on HO3D.}
Our method produces significantly better mesh results compared to Hampali~\etal.'s method~\cite{meta-obj}, and is on par with NeuS~\cite{neus} which is trained with ground truth poses.
}
\label{fig:recon_comp}
\end{figure}
\begin{table}%
\centering
\begin{tabular}{lcccc}
\toprule
\multirow{2}{*}{Strategy}& \multicolumn{3}{c}{Pose} & \multicolumn{1}{c}{Mesh} \\
& $ \!\!\!\!\!\!\!AUC_{ATE}\uparrow\!\!\!$ & $RPE_{t}\downarrow\!\!\!$ & $RPE_{r}\downarrow\!\!\!$ & $HD_{RMSE}\downarrow\!\!\!$\\
% \midrule
% Baseline & 4.327& 4.135& 7.188& 4.152 \\
% VC & 5.424& 1.941& 2.647& 3.900 \\
% VC + refine & \textbf{5.931}& \textbf{1.571}& \textbf{2.199}& \textbf{3.135}\\
\midrule
RC & 4.33& 4.14& 7.19& 4.15 \\
VC & 5.42& 1.94& 2.65& 3.90 \\
VC + RC & \textbf{5.93}& \textbf{1.57}& \textbf{2.20}& \textbf{3.14}\\
\bottomrule
\end{tabular}
\caption{\textbf{Effect of virtual cameras.}
Progressively training w.r.t. the real camera~(RC) typically fails. The proposed virtual camera system~(VC) reduces the search space and improves the results significantly. The global refinement w.r.t. the real camera~(``+RC'') improves the performance further.
}
\label{table:virtual_abalation}
\end{table}
\begin{comment}
\begin{table}%
\centering
\begin{tabular}{ccccccc}
\toprule
\multicolumn{3}{c}{Strategy} & \multicolumn{3}{c}{Pose} & \multicolumn{1}{c}{Mesh} \\
Match & Reset & 4 DoF & $ATE$ & $RPE_{r}$ & $RPE_{t}$ & $HD_{RMSE}$\\
\midrule
\xmark & \cmark & \cmark & 0.370 & 1.881 & 0.117 & 5.060 \\
\cmark & \xmark & \cmark & 0.332 & 1.804 & 0.108 & 5.100 \\
\cmark & \cmark & \xmark & 0.303 & 1.829 & 0.108 & 3.577\\
\cmark & \cmark & \cmark & 0.283 & 1.704 & 0.101 & 2.903\\
\bottomrule
\end{tabular}
\caption{\textbf{Ablation study.}
}
\label{table:ablation_study_1}
\end{table}
\end{comment}
\begin{table}%
\centering
\begin{tabular}{lcccccc}
\toprule
\multirow{2}{*}{Settings} & \multicolumn{3}{c}{Pose} & \multicolumn{1}{c}{Mesh} \\
& $ \!\!\!\!\!\!\!AUC_{ATE}\uparrow\!\!\!$ & $RPE_{t}\downarrow\!\!\!$ & $RPE_{r}\downarrow\!\!\!$ & $HD_{RMSE}\downarrow\!\!\!$\\
\midrule
% Full version &\textbf{ 5.931}& \textbf{1.571}& \textbf{2.199}& \textbf{3.135}\\
% w/o 4D & 5.684& 2.259& 3.486& 3.272\\
% w/o reset & 5.575& 1.637& 2.262& 3.283 \\
% w/o match & 5.208& 1.872& 2.375& 4.096 \\
w/o match & 5.21& 1.87& 2.38& 4.10 \\
w/o reset & 5.58& 1.64& 2.26& 3.28 \\
w/o 4D & 5.68& 2.26& 3.49& 3.27\\
Full &\textbf{ 5.93}& \textbf{1.57}& \textbf{2.20}& \textbf{3.14}\\
\bottomrule
\end{tabular}
\caption{\textbf{Ablation study.}
We evaluate the effectiveness of the proposed components, including reducing the 6 degrees of freedom of poses to only 4~(``w/o 4D''), the periodical reset of shape networks~(``w/o reset''), and the match loss~(``w/o match'').
% We quantitatively evaluate the strategies during progressive optimization: "w/o 4D" uses standard 6 DoF pose and suffer from high relative rotation error due to its high freedom, "w/o reset" disable resetting geometry related networks and lead to poor mesh quality, and "w/o" disable the match supervision shown in Eq.~\ref{eq:match_loss}, leading to worst results.
}
\label{table:ablation_study_1}
\end{table}
% \begin{table}
% \caption{\textbf{HO3D Dataset Pose Trajectory Evaluation}(\textit{$ATE$ and $RPE_{t}$ use in the ratio of reconstructed object radius while $RPE_{r}$ is in degree.})\HS{I would also add textured-less Mug(SMu1) but Hampali \etal.\cite{meta-obj} didn't report the accuracy of this object but they have the mesh on their project webpage.} \YH{Better to use object names to replace the magic sequence IDs, and we only need to mention the index information at one place.}
% }
% \label{table:ho3d_pose_numbers}
% \centering
% \begin{tabular}{lcccccc}
% \toprule
% & \multicolumn{6}{c}{$ATE$/$RPE_{t}$/$RPE_{r}$}\\
% Object & \multicolumn{3}{c}{Ours-RGB} & \multicolumn{3}{c}{Ours-RGB-D}\\
% \midrule
% MC1 & 0.218 & 0.098 & 1.645 & 0.189 & 0.096 & 1.676\\ % exp14
% ShSu12 & 0.257 & 0.141 & 2.442 & 0.196 & 0.140 & 2.430 \\ %exp14
% SM2 & 0.661 & 0.105 & 1.723 & 0.562 & 0.084 & 1.546\\ % exp 12
% ABF14 & 0.462 & 0.142 & 2.551 & 0.366 & 0.121 & 2.293\\ % exp 12
% MDF14 & 0.148 & 0.069 & 1.228 & 0.143 & 0.061 & 1.140\\%exp 14
% AP13 & 0.101 & 0.047 & 0.883 & 0.070 & 0.045 & 0.884\\ %exp 14
% SMu1 & 0.343 & 0.143 & 1.946 & 0.181 & 0.114 & 1.630\\ %exp 12
% \bottomrule
% \end{tabular}
% \end{table}
\begin{comment}
\begin{figure}[!ht]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cccc}
% \includegraphics[width=0.24\linewidth]{Final_Recons/MC1.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/bold_trajs/MC1.png} &
% \includegraphics[width=0.24\linewidth]{Final_Recons/ShSu12.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/bold_trajs/ShSu12.png} \\
\includegraphics[width=0.24\linewidth]{Final_Recons/SM2.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/bold_trajs/SM2.png} &
\includegraphics[width=0.24\linewidth]{Final_Recons/SMu1.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/bold_trajs/SMu1.png} \\
\includegraphics[width=0.24\linewidth]{Final_Recons/MDF14.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/bold_trajs/MDF14.png} &
\includegraphics[width=0.24\linewidth]{Final_Recons/AP13.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/bold_trajs/AP13.png} \\
% Reference Image A & Pose Trajectory A & Reference Image B & Pose Trajectory B\\
\end{tabular}
\endgroup
\caption{\textbf{Pose results on HO3D.}}
\label{fig:ho3d_trajs}
\end{figure}
\end{comment}
\begin{figure}[tb]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cccc}
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/AP13.png} &
\includegraphics[width=0.24\linewidth,trim={80, 0, 80, 0},clip]{Experiment_Figure/virtual_cams/AP13_original00.png} &
\includegraphics[width=0.24\linewidth,trim={80, 0, 80, 0},clip]{Experiment_Figure/virtual_cams/AP13_virtual00.png} &
\includegraphics[width=0.24\linewidth,trim={80, 0, 80, 0},clip]{Experiment_Figure/virtual_cams/AP13_ours00.png} \\
% \includegraphics[width=0.24\linewidth]{Final_Recons/ShSu12.png} &
% \includegraphics[width=0.24\linewidth,trim={80, 0, 80, 0},clip]{Experiment_Figure/virtual_cams/ShSu12_original00.png} &
% \includegraphics[width=0.24\linewidth,trim={80, 0, 80, 0},clip]{Experiment_Figure/virtual_cams/ShSu12_virtual00.png} &
% \includegraphics[width=0.24\linewidth,trim={80, 0, 80, 0},clip]{Experiment_Figure/virtual_cams/ShSu12_ours00.png} \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/ho3d_ref/MDf14} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/virtual_cams/MDF14_baseline02.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/virtual_cams/MDF14_virtual01.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/virtual_cams/MDF14_global00.png} \\
% \includegraphics[width=0.24\linewidth]{Final_Recons/SMu1.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/virtual_cams/SMu1_original01.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/virtual_cams/SMu1_virtual00.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/virtual_cams/SMu1_global02.png} \\
\small{Reference} & \small{RC} & \small{VC} & \small{VC + RC} \\
\end{tabular}
\endgroup
\caption{
{\bf Effect of virtual cameras.}
Progressively training w.r.t. the real camera~(RC) is challenging and typically fails. The proposed virtual camera system~(VC) reduces the search space of optimization and improves the results significantly after refinement w.r.t. the real camera~(``+RC'').
}
\label{fig:effect_of_vc}
\end{figure}
% Since Hampali \etal\cite{meta-obj} didn't offer specific frame ids for the sequences used, although we share the same sequences and same frame intervals, different frame can have different difficulty for pose estimation so we are not able to directly compare with their numbers. We report our estimate pose trajectories in Figure~\ref{fig:ho3d_trajs} and quantitative results on Table~\ref{table:ho3d_pose_numbers}.
\subsection{Ablation Study}
\begin{figure}%[tb]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\centering
\begin{tabular}{cccc}
% \includegraphics[width=0.19\linewidth]{Final_Recons/AP13.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/AP1300.png} &
% \includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/AP13_another00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/AP13_back00_for_match.png} &
% \includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/AP13_another_for_match00.png}
\includegraphics[width=0.24\linewidth,trim={120, 20, 120, 0},clip]{Experiment_Figure/ablation_wo_reset/maneki-neko00.png} &
% \includegraphics[width=0.24\linewidth,trim={100, 30, 100, 0},clip]{Experiment_Figure/ablation_wo_reset/maneki-neko-back-wo-reset00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 20, 120, 0},clip]{Experiment_Figure/abalation_standard/maneki-neko_for_reset00.png}
% \includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/maneki-neko-back-for-reset00.png}
\\
% \includegraphics[width=0.19\linewidth]{Final_Recons/labelprinter.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/labelprinter00.png} &
% \includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/labelprinter_back00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/labelprinter00_for_match.png} &
% \includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/labelprinter_back_for_match00.png}
% \includegraphics[width=0.19\linewidth]{Final_Recons/spaceinvader.png} &
\includegraphics[width=0.24\linewidth,trim={150, 80, 150, 0},clip]{Experiment_Figure/ablation_wo_reset/space_invader_wo_reset00.png} &
% \includegraphics[width=0.24\linewidth,trim={150, 80, 150, 0},clip]{Experiment_Figure/ablation_wo_reset/space_invader_front_reset00.png} &
\includegraphics[width=0.24\linewidth,trim={150, 80, 150, 0},clip]{Experiment_Figure/abalation_standard/spaceinvader_for_reset00.png}
% \includegraphics[width=0.24\linewidth,trim={150, 60, 150, 0},clip]{Experiment_Figure/ablation_ori_6d_4d/space_invader_4d00_front.png}
\\
% \small{Reference} &
\multicolumn{1}{c}{{\cellcolor{red!10} \small{w/o match}}} & \multicolumn{1}{c}{{\cellcolor{green!10} \small{w/ match}}} & \multicolumn{1}{c}{{\cellcolor{red!10} \small{w/o reset}}} & \multicolumn{1}{c}{{\cellcolor{green!10} \small{w/ reset}}} \\
\end{tabular}
\endgroup
\caption{
{\bf Visualization result of ablation study.}
The network benefits from the match loss~(``w/ match'') for recovering detailed shapes, while the periodic reset of shape networks~(``w/ reset'') aids optimization in handling local minima.
% We qualitively evaluate the strategies during progressive optimization: "w/o match" means without the match loss shown in Eq.~\ref{eq:match_loss} and leads to geometry ambiguity, "w/o reset" means without resetting geometry related networks, leading to some fail cases like space invader(\textit{bottom}).
}
\label{fig:mini_ablations_visual}
\end{figure}
\begin{comment}
\begin{figure}%[tb]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{subfigure}{\linewidth}
\centering
\begin{tabular}{cccc}
% \includegraphics[width=0.19\linewidth]{Final_Recons/AP13.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/AP1300.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/AP13_another00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/AP13_back00_for_match.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/AP13_another_for_match00.png} \\
% \includegraphics[width=0.19\linewidth]{Final_Recons/labelprinter.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/labelprinter00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/ablation_wo_match/labelprinter_back00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/labelprinter00_for_match.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/labelprinter_back_for_match00.png}\\
% \small{Reference} &
\multicolumn{2}{c}{{\cellcolor{red!10} \small{w/o match}}} & \multicolumn{2}{c}{{\cellcolor{green!10} \small{w/ match}}} \\
\end{tabular}
\caption{Effect of match loss}
\end{subfigure}
\begin{subfigure}{\linewidth}
\centering
\begin{tabular}{ccccc}
% \includegraphics[width=0.19\linewidth]{Final_Recons/maneki-neko.png} &
\includegraphics[width=0.24\linewidth,trim={120, 20, 120, 0},clip]{Experiment_Figure/ablation_wo_reset/maneki-neko00.png} &
\includegraphics[width=0.24\linewidth,trim={100, 30, 100, 0},clip]{Experiment_Figure/ablation_wo_reset/maneki-neko-back-wo-reset00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 20, 120, 0},clip]{Experiment_Figure/abalation_standard/maneki-neko_for_reset00.png} &
\includegraphics[width=0.24\linewidth,trim={120, 30, 120, 0},clip]{Experiment_Figure/abalation_standard/maneki-neko-back-for-reset00.png} \\
% \includegraphics[width=0.19\linewidth]{Final_Recons/spaceinvader.png} &
\includegraphics[width=0.24\linewidth,trim={150, 80, 150, 0},clip]{Experiment_Figure/ablation_wo_reset/space_invader_wo_reset00.png} &
\includegraphics[width=0.24\linewidth,trim={150, 80, 150, 0},clip]{Experiment_Figure/ablation_wo_reset/space_invader_front_reset00.png} &
\includegraphics[width=0.24\linewidth,trim={150, 80, 150, 0},clip]{Experiment_Figure/abalation_standard/spaceinvader_for_reset00.png} &
\includegraphics[width=0.24\linewidth,trim={150, 60, 150, 0},clip]{Experiment_Figure/ablation_ori_6d_4d/space_invader_4d00_front.png}\\
% \small{Reference} &
\multicolumn{2}{c}{{\cellcolor{red!10} \small{w/o reset}}} & \multicolumn{2}{c}{{\cellcolor{green!10} \small{w/ reset}}} \\
\end{tabular}
\caption{Effect of periodically reset}
\end{subfigure}
\endgroup
\caption{
{\bf Effectiveness of strategies.} We qualitively evaluate the strategies during progressive optimization: "w/o match" means without the match loss shown in Eq.~\ref{eq:match_loss} and leads to geometry ambiguity, "w/o reset" means without resetting geometry related networks, leading to some fail cases like space invader(\textit{bottom}).
}
\label{fig:mini_ablations}
\end{figure}
\end{comment}
% In this section, we will first report comprehensive experiments to demonstrate the advantage of our virtual camera systems. Then we analyze the effectiveness of our strategies regarding match loss, and sampling mechanisms.
We evaluate the design of our method systematically in this section. We report the results averaging across all the 9 sequences in HO3D, if not explicitly mentioned.
\noindent {\bf Effectiveness of guided virtual camera.}
We study the effect of the proposed virtual camera in Table~\ref{table:virtual_abalation}.
In principle, one could leverage the temporal consistency between consecutive frames simply by introducing the progressive training, similar to~\cite{nope-nerf,3dgs}. We compare this strategy with our method. For a fair comparison, we use the same network and the same segmentation mask and 2D matches as those used in our method. The major problem of this strategy is that the pose and shape are optimized w.r.t. the raw camera, which is challenging due to the target’s free movement and the wide range it covers in front of the camera. We denote this method as ``RC'' in the table. By contrast, the proposed virtual camera reduces the search space and improves the results significantly~(``VC''). The result improves further with the refinement w.r.t the real camera~(``+RC''). Fig.~\ref{fig:effect_of_vc} shows some visualization results.
% In this section, we have multiple settings including "Original", "6 DoF Virtual", "4 DoF Virtual" and "Global Refine". "Original" means that we use the raw image as input without modification but we equip it with all tricks we used like progressive optimization and semi-dense match supervision; "6 DoF Virtual" means that we utilize virtual camera system by first estimating the center of the object and then transform the object to frame center in image plane, after which we need to record the transformation matrix $M$ as mentioned in our Section~\ref{sec:method_progressive}. However, here we don't scale the image like cropping a small part of image and scaling it up, for the sake of not modifying the camera intrinsic, which is a sub-example of our virtual camera design. For "6 DoF Virtual", we would estimate 6 degree of freedom, namely 3 for rotation and 3 for translation. As we discussed in Section\ref{sec:method_progressive}, we also propose another setting as 4 degree of freedom(we named it "4 DoF Virtual"), where we estimate the rotation matrix and the change of distance between camera and object center while camera always look at the object center. As for "Global Refine", as mentioned in Section~\ref{sec:method_global}, based the initialization during virtual camera phase, we use PnP to adjust the poses and retrain globally. Here, the Global Refine" was trained based on pose initialization from "4 DoF Virtual".
% We first show the qualitative results in Figure~\ref{fig:virtual_abalation}. Compared with "Original" camera setting, we can find that for nearly all objects except the last row namely pitch base(AP13), "6 DoF Virtual" and "4 DoF Virtual" give us more aligned pose trajectories especially for textured object cracker box(MC1). However, we found that "Original" shows good result in pitch base(AP13) which is somehow reasonable since the object is mainly rotating. With the same trends, for other sequences with worse performances, it is because the objects have a lot of translations that the model need predict. For example, in Figure~\ref{fig:delta_moving}, we can find that the sequence MC1 would have much larger relative translation for every frame compared with AP13, which are the relative translations orthogonal to camera view directions that in "Original" camera setting will put a lot of burdens on model. However, in our virtual camera setting, we first use mask as guidance and move the object to frame center, which compensates a lot such large translations orthogonal to camera view directions, so it would be much easier for the model to learn the poses. Also pose trajectory shapes of "6 DoF Virtual" and "4 DoF Virtual" are somehow similar, but they are not that good as the "Global Refine". For the above conclusion, we can further check it in Table~\ref{table:virtual_abalation}.
% Quantitative results of different camera settings can be found in Table~\ref{table:virtual_abalation}. Notice that all the numbers related to virtual camera settings in this table are converted back to original camera setting during evaluation based on PnP algorithm\cite{epnp} as mentioned in Section~\ref{sec:method_progressive}, so they are totally comparable. We first analyze "6 DoF Virtual" and "4 DoF Virtual", they perform similarly much better than the "Original" by a large margin, $ATE$ from 0.666 to 0.383 and 0.666 to 0.373 respectively. In average($AVG$), "4 DoF Virtual" perform slightly better than "6 DoF Virtual": -0.01 for $ATE$, -0.001 for $RPE_{t}$ and -0.002 for $RPE_{r}$. However, we can find that standard deviation($SD$) of them, "4 DoF Virtual" performs much more stably than "6 DoF Virtual", which can be reflected by $SD$ of $ATE$. "Global Refine" is our delivered setting which is very promising since it reduces all errors of all object during "4 DoF Virtual" phase, which effectively show that our mechanism works perfectly: initialize poses during virtual camera setting and refine initialized poses globally in original camera setting. For sure, "Global Refine" outperforms "Original" by a large margin, which effectively demonstrate the huge advantage of our virtual camera strategy.
% \begin{figure}
% \centering
% \includegraphics[width=0.80\linewidth]{Experiment_Figure/delta_moving_distance.png}
% \caption{\textbf{Image Plane Relative Translation}(\textit{For every frame, we calculate the movement of object center compared with previous frame.})}
% \label{fig:delta_moving}
% \end{figure}
% \begin{figure*}
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.80\linewidth]{Experiment_Figure/diff_cams.png}
% \end{tabular}
% \caption{\textbf{Reconstruction Results With Different Camera Systems}}
% \label{fig:cam_sys}
% \end{figure*}
% \begin{figure*}[tb!]
% \centering
% \begin{tabular}{ccccc}
% \includegraphics[width=0.15\linewidth]{Final_Recons/MC1.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mc1_original.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mc1_virtual_6.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mc1_virtual_4.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mc1_global.png} \\
% \includegraphics[width=0.15\linewidth]{Final_Recons/ShSu12.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/shsu12_original.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/shsu12_virtual_6.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/shsu12_virtual_4.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/shsu12_global.png} \\
% \includegraphics[width=0.15\linewidth]{Final_Recons/SM2.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/sm2_original.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/sm2_virtual_6.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/sm2_virtual_4.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/sm2_global.png}\\
% \includegraphics[width=0.15\linewidth]{Final_Recons/ABF14.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/abf14_original.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/abf14_virtual_6.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/abf14_virtual_4.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/abf14_global.png} \\
% \includegraphics[width=0.15\linewidth]{Final_Recons/MDF14.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mdf14_original.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mdf14_virtual_6.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mdf14_virtua_4.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/mdf14_global.png} \\
% \includegraphics[width=0.15\linewidth]{Final_Recons/AP13.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/ap13_original.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/ap13_virtual_6.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/ap13_virtual_4.png} &
% \includegraphics[width=0.15\linewidth]{virtual_balation/ap13_global.png}\\
% Refrence Image & Original & 6 DoF Virtual & 4 DoF Virtual & Global Refine\\
% \end{tabular}
% \caption{\textbf{Camera System Ablation Study.}\YH{Better to remove 4DoF evaluation if no noticeable improvement.}}
% \label{fig:virtual_abalation}
% \end{figure*}
\noindent {\bf Ablation study of different components.} We study the effect of different components of our method in Table~\ref{table:ablation_study_1}.
Without the match loss or the periodic reset of the shape network, the performance has a significant drop~(denoted as ``w/o match'' and ``w/o reset'' respectively). On the other hand, using the standard 6D pose representation~(denoted as ``w/o 4D'') also suffers in performance.
% We reformulate the pose w.r.t. the virtual camera to have only 4 degree of freedom in our method, which reduces the search space, and achieves the best performance across all configurations.
We use the same refinement procedure for all the results in this table. Fig.~\ref{fig:mini_ablations_visual} shows some visualization results.
\noindent {\bf Ablation study of different hyper-parameters.}
We show ablation results of our method with different hyper-parameters in Table~\ref{fig:ablation_study_2}, including the number of frames $B$ that is processed as a group during progressing training, the degree threshold $\tau$ for periodic reset of the shape network, and the the maximum frame internal $n$ for 2D matching across consecutive images.
As we can see, $B$=1 gives the best results, and as the number of images in each group increases, the performance deteriorates, which we believe is caused by the difficulties introduced with the increased data varieties. On the other hand, $\tau$=60$\degree$ and $n$=10 produce the best results in most of our experiments.
% for pose while $n$=2 for mesh, which indicates the match may contain noises. Notice that we didn't explore further since we think there is a trade-off between match interval and efficiency. Without loss of generality,
% As we discuss in Section~\ref{sec:method_setup}, \ref{sec:method_progressive}, we use LoFTR~\cite{loftr} to generate semi-dense to supervise the model during progressive optimization by match loss. We first show qualitative results in Figure~\ref{fig:ablation_study_match_loss}, where we can see that, for most of the objects, match loss is very effective to regularize the poses. However, we can find that for the last row, the pitch base(AP13), we didn't see very sharp positive effect, and we think that for such texture-less object, LoFTR~\cite{loftr} may likely generate mismatched correspondence. Also for difficult sequence, namely the 4th row bleach(ABF14), matches from LoFTR~\cite{loftr} would not solve the case too much. \YH{Better with ablations based on different matches (RAFT etc.)}
% \begin{figure}[tb!]
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.30\linewidth]{Final_Recons/MC1.png} &
% \includegraphics[width=0.30\linewidth]{wo_match_loss/mc1.png} &
% \includegraphics[width=0.30\linewidth]{virtual_balation/mc1_virtual_4.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/ShSu12.png} &
% \includegraphics[width=0.30\linewidth]{wo_match_loss/shsu12.png} &
% \includegraphics[width=0.30\linewidth]{virtual_balation/shsu12_virtual_4.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/SM2.png} &
% \includegraphics[width=0.30\linewidth]{wo_match_loss/sm2.png} &
% \includegraphics[width=0.30\linewidth]{virtual_balation/sm2_virtual_4.png}\\
% \includegraphics[width=0.30\linewidth]{Final_Recons/ABF14.png} &
% \includegraphics[width=0.30\linewidth]{wo_match_loss/abf14.png} &
% \includegraphics[width=0.30\linewidth]{virtual_balation/abf14_virtual_4.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/MDF14.png} &
% \includegraphics[width=0.30\linewidth]{wo_match_loss/mdf14.png} &
% \includegraphics[width=0.30\linewidth]{virtual_balation/mdf14_virtua_4.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/AP13.png} &
% \includegraphics[width=0.30\linewidth]{wo_match_loss/ap13.png} &
% \includegraphics[width=0.30\linewidth]{virtual_balation/ap13_virtual_4.png}\\
% Refrence Image & W/O Match Loss & W Match Loss\\
% \end{tabular}
% \caption{\textbf{Match Loss Ablation Study.}}
% \label{fig:ablation_study_match_loss}
% \end{figure}
% Then we can view the quantitative results in Table~\ref{table:ablation_study_match_loss}, where we can see the effectiveness of match loss for most of objects. However, for bleach(ABF14), although we can improve $ATE$ from 0.530 to 0.482, we found the negative effects on $RPE_{t}$ and $RPE_{r}$. Also pitch base(AP13), without match loss is better in terms of $ATE$. From a comprehensive perspective, after calculating the average performance of 6 objects in different level of textures, we found the match loss is effective for our system to work.
\begin{comment}
\YH{Better to add the detailed numbers for each sequence to the supplementary (use the avg numbers in the main paper)}
\begin{table}[tb!]
\caption{\textbf{Effectiveness Of Match Loss}(\textit{$ATE$ and $RPE_{t}$ use in the ratio of reconstructed object radius while $RPE_{r}$ is in degree})
}
\centering
\begin{tabular}{lccc}
\toprule
& \multicolumn{3}{c}{$ATE$/$RPE_{t}$/$RPE_{r}$} \\
Object & w/o match loss & & w match loss \\
\midrule
MC1 & 0.938/0.210/3.341 && 0.343/0.124/2.177\\
ShSu12 & 1.730/0.408/5.897 & & 0.321/0.146/2.521\\
SM2 & 0.867/0.139/2.248 & & 0.682/0.119/1.948\\
ABF14 & 0.530/0.199/3.054 & & 0.482/0.204/3.438\\
MDF14 & 0.519/0.113/1.993 & & 0.216/0.073/1.260\\
AP13 & 0.157/0.069/1.271 & & 0.192/0.069/1.260\\
\midrule
Average & 0.957/0.190/3.134 & & \textbf{0.373/0.123/2.107}\\
\bottomrule
\end{tabular}
\label{table:ablation_study_match_loss}
\end{table}
\end{comment}
% \begin{figure*}
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.80\linewidth]{Experiment_Figure/abalation_reset_match.png}
% \end{tabular}
% \caption{\textbf{Abaltion Study On Segment-free Progressive Training}}
% \label{fig:cam_sys}
% \end{figure*}
% \begin{figure}
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.8\linewidth]{Experiment_Figure/abalation_reset.png}
% \end{tabular}
% \caption{\textbf{Abaltion Study On Reset}}
% \label{fig:ablation_reset}
% \end{figure}
% \begin{figure}
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.8\linewidth]{Experiment_Figure/abalation_match.png}
% \end{tabular}
% \caption{\textbf{Abaltion Study On Match Loss}}
% \label{fig:ablation_match}
% \end{figure}
% \subsubsection{Effectiveness Of Relative Pose Initialization}
\begin{comment}
\YH{Better to add the detailed numbers for each sequence to the supplementary}
\begin{table}[tb!]
\caption{\textbf{Effectiveness Of Relative Pose Initialization}(\textit{$ATE$ and $RPE_{t}$ use in the ratio of reconstructed object radius while $RPE_{r}$ is in degree})
}
\centering
\begin{tabular}{lccc}
\toprule
& \multicolumn{3}{c}{$ATE$/$RPE_{t}$/$RPE_{r}$} \\
Object & w/o init & & w init \\
\midrule
MC1 & 0.196/0.090/1.523 && 0.199/0.092/1.561\\
ShSu12 & 0.317/0.165/2.698 & & 0.281/0.162/2.680\\
SM2 & 0.666/0.102/1.680 & & 0.634/0.106/1.753\\
ABF14 & 0.556/0.168/2.692 & & 0.471/0.157/2.593\\
MDF14 & 0.200/0.079/1.369 & & 0.231/0.076/1.299\\
AP13 & 0.092/0.044/0.838 & & 0.082/0.043/0.813\\
SMu1 & 0.249/0.126/1.736 & & 0.258/0.130/1.775 \\
\midrule
AVG & 0.325/0.111/1.791 & & \textbf{0.308/0.109/1.782}\\
SD & 0.193/0.042/0.633 && \textbf{0.171/0.040/0.619} \\
\bottomrule
\end{tabular}
\label{table:ablation_study_relative_pose}
\end{table}
\end{comment}
\begin{figure}
\centering
\begingroup
\setlength{\tabcolsep}{0pt}
% \small{$AUC_{ATE}\uparrow$} \\
{\includegraphics[width=\linewidth]{Experiment_Figure/ablation_statistics/ate_bar_row_v3.png}} \\
% \small{$HD_{RMSE}\downarrow$} \\
% \includegraphics[width=\linewidth] {Experiment_Figure/ablation_statistics/hd_rmse_bar_row_v3.png}\\
\setlength{\tabcolsep}{0pt}
\begin{tabular}{>{\centering\arraybackslash}p{0.06\linewidth}>{\centering\arraybackslash}p{0.30\linewidth}>
{\centering\arraybackslash}p{0.02\linewidth}>
{\centering\arraybackslash}p{0.30\linewidth}>
{\centering\arraybackslash}p{0.02\linewidth}>{\centering\arraybackslash}p{0.30\linewidth}}
&\small{Group size} && \small{Reset degree} & &\small{Match interval}
\end{tabular}
\endgroup
\caption{\textbf{Ablation study of hyper-parameters in $AUC_{ATE}\uparrow$.}
We report the average results of all HO3D sequences. It shows that processing one frame per group in progressive training, resetting shape networks after every 60 degrees, and relying on 2D matches across 10 consecutive frames is the best setting in general.
}
\label{fig:ablation_study_2}
\end{figure}
\begin{figure}[tb!]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cccc}
\includegraphics[width=0.24\linewidth]{Experiment_Figure/generalization/dummy_ref.png} &
\includegraphics[width=0.24\linewidth,trim={0, 0, 80, 0},clip]{Experiment_Figure/generalization/dummy_ml_comp01.png} &
\includegraphics[width=0.24\linewidth,trim={30, 0, 30, 0},clip] {Experiment_Figure/generalization/dummy_ml_comp02.png}&
\includegraphics[width=0.20\linewidth]{Experiment_Figure/generalization/ml_pose_dummy.png} \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/generalization/label_ref.png} &
\includegraphics[width=0.24\linewidth]{Experiment_Figure/generalization/labelprinter_ml_comp01.png} &
\includegraphics[width=0.24\linewidth,trim={100, 40, 100, 0},clip]{Experiment_Figure/generalization/labelprinter_ml_comp02.png}&
\includegraphics[width=0.18\linewidth]{Experiment_Figure/generalization/ml_pose_label.png} \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/generalization/maneki_ref.png} &
\includegraphics[width=0.24\linewidth,trim={0, 0, 0, 30},clip]{Experiment_Figure/generalization/maneki-neko_ml_comp01.png} &
\includegraphics[width=0.24\linewidth,trim={100, 30, 100, 0},clip]{Experiment_Figure/generalization/maneki-neko_ml_comp02.png} &
\includegraphics[width=0.20\linewidth]{Experiment_Figure/generalization/ml_pose_maneki.png} \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/generalization/space-ref.png} &
\includegraphics[width=0.24\linewidth,trim={0, 0, 0, 20},clip]{Experiment_Figure/generalization/spaceinvader_ml_comp01.png} &
\includegraphics[width=0.24\linewidth,trim={100, 30, 100, 0},clip]{Experiment_Figure/generalization/spaceinvader_ml_comp02.png}&
\includegraphics[width=0.20\linewidth]{Experiment_Figure/generalization/ml_pose_space.png} \\
\small{Reference} & \small{BARF} & \small{Ours} & \small{Poses} \\
\end{tabular}
\endgroup
\caption{\textbf{Results on egocentric sequences.}
We visualize the shape results of BARF~\cite{barf} and ours in the middle, and compare their pose results in the last column~(\textcolor[RGB]{128,0,128}{purple}: BARF, \textcolor[RGB]{3,168,158}{cyan}: ours).
Without proper pose initialization for free-moving objects, BARF struggles to produce reasonable results. Our method generates accurate pose and shape results in most cases.
}
% Our method generalizes well to this real setting, and produces accurate meshes for texture-less free-moving objects.
\label{fig:evaluation_on_ml}
\end{figure}
\begin{comment}
% Move sequence-level numbers to supplementary
\begin{table*}
\caption{\textbf{Ablation Study Of Different Components Of The System}(\textit{$ATE$ and $RPE_{t}$ is in the ratio of reconstructed object radius while $RPE_{r}$ is in degree; "Std" means standard setting, while "-match" means without match loss, "-reset" means without resetting NeuS based predicted rotations, and "-global" means without global refinements.}) \HS{I think Table~\ref{table:ablation_study_seg} have similar effects as this table but without detailed performance on different objects. Should we put this table into supplimentary materials?}
}
\label{table:virtual_abalation}
\centering
\begin{tabular}{l|cccc|cccc|cccc}
\toprule
& \multicolumn{4}{c}{$ATE$} & \multicolumn{4}{c}{$RPE_{t}$} & \multicolumn{4}{c}{$RPE_{r}$}\\
Object & Std & -match & -reset & -global & Std & -match & -reset & -global & Std & -match & -reset & -global\\
\midrule
% only w/o reset
% MC1 & 0.196&0.193&0.197&0.236&0.090&0.090&0.093&0.129&1.523&1.526&1.554&2.082\\
% ShSu12 & 0.264&0.232&0.225&0.344&0.149&0.148&0.142&0.169&2.490&2.499&2.412&2.811 \\
% SM2 & 0.596&0.515&0.595&0.708&0.089&0.098&0.107&0.152&1.530&1.729&1.799&2.367\\
% ABF14 & 0.442&0.647&0.460&0.456&0.137&0.214&0.137&0.159&2.534&3.154&2.542&2.796\\
% MDF14 & 0.144&0.478&0.167&0.160&0.073&0.081&0.067&0.085&1.278&1.413&1.207&1.506\\
% AP13 & 0.092&0.187&0.145&0.179&0.044&0.048&0.046&0.063&0.838&0.898&0.866&1.133\\
% SMu1 & 0.249&0.339&0.250&0.299&0.126&0.142&0.124&0.169&1.736&1.951&1.722&2.302\\
% \midrule
% Average & \textbf{0.283}&0.370&0.291&0.340&\textbf{0.101}&0.117&0.102&0.132&\textbf{1.704}&1.881&1.729&2.142 \\
% w/o reset + w/o maintainshape
MC1&0.196&0.193&0.244&0.236&0.090&0.090&0.099&0.129&1.523&1.526&1.651&2.082\\
ShSu12&0.264&0.232&0.272&0.344&0.149&0.148&0.147&0.169&2.490&2.499&2.466&2.811\\
SM2&0.596&0.515&0.653&0.708&0.089&0.098&0.108&0.152&1.530&1.729&1.800&2.367\\
ABF14&0.442&0.647&0.506&0.456&0.137&0.214&0.136&0.159&2.534&3.154&2.620&2.796\\
MDF14&0.144&0.478&0.207&0.160&0.073&0.081&0.075&0.085&1.278&1.413&1.287&1.506\\
AP13&0.092&0.187&0.098&0.179&0.044&0.048&0.045&0.063&0.838&0.898&0.859&1.133\\
SMu1&0.249&0.339&0.342&0.299&0.126&0.142&0.143&0.169&1.736&1.951&1.944&2.302\\
\midrule
Average&\textbf{0.283}&0.370&0.332&0.340&\textbf{0.101}&0.117&0.108&0.132&\textbf{1.704}&1.881&1.804&2.142\\
\bottomrule
\end{tabular}
\end{table*}
\end{comment}
\begin{comment}
\subsubsection{Explore Sampling Strategies}
\begin{figure}[tb!]
\centering
\begin{tabular}{ccc}
\includegraphics[width=0.30\linewidth]{Final_Recons/AP13.png} &
\includegraphics[width=0.30\linewidth]{sample_ablation/hierarchy.png} &
\includegraphics[width=0.30\linewidth]{sample_ablation/16.png} \\
\includegraphics[width=0.30\linewidth]{sample_ablation/32.png} &
\includegraphics[width=0.30\linewidth]{sample_ablation/64.png} &
\includegraphics[width=0.30\linewidth]{sample_ablation/128.png} \\
\end{tabular}
\caption{\textbf{Pose Trajectories For Different Sampling Strategies}(\textit{top row: reference image, hierarchy sampling, uniform sampling with 16 points; bottom row: uniform sampling with 32 points, 64 points, 128 points respectively.})
}
\label{fig:ablation_study_sample}
\end{figure}
The key motivation for exploring sampling strategies can be listed as: first, during progressive optimization under virtual camera, our key goal is to get good initial poses but not very high quality mesh so we can use simple strategies that can not only save training time but also help the model focus on estimating poses instead of refining the shapes. We select pitch base(AP13) to do ablation studies regarding sampling strategies. From the qualitative results in Figure~\ref{fig:ablation_study_sample}, we can find that uniformly sampling with 16 points is too simple to work and it gives us low quality poses. However, for other settings including the hierarchy sampling in standard NeuS\cite{neus} and also uniform sampling with more than 16 points have similar effects.
\begin{table}[!th]
\caption{Different Sampling Strategies During Virtual Camera Pose Estimation(\textit{$ATE$ and $RPE_{t}$ use in the ratio of reconstructed object radius while $RPE_{r}$ is in degree.})
}
\label{table:sampling strategies}
\centering
\begin{tabular}{lcccc}
\toprule
Strategies & $ATE$ & $RPE_{t}$ & $RPE_{r}$ & min/10k-iter\\
\midrule
Hierarchy & 0.202 & 0.065 & 1.138 & 25\\
Uniform 16 & 0.422 & 0.235 & 3.956 & \textbf{7}\\
Uniform 32 & 0.192 & 0.069 & 1.260 & 9\\
Uniform 64 & 0.182 & 0.067 & 1.213 & 14\\
Uniform 128 & \textbf{0.168} & \textbf{0.062} & \textbf{1.106} & 24\\
\bottomrule
\end{tabular}
\end{table}
Now, let us look at the quantitative results in Table~\ref{table:sampling strategies}, we can find similar conclusion about the pose quality in Figure~\ref{fig:ablation_study_sample}. However, we can find that with the increase of more points in uniform samplings, we tend to have better poses, which can be reasoned as: if we have more points during sampling, then the mesh quality will be better, and better mesh can help Pose MLP to estimate the poses better since the supervision signals would be more accurate. There is no free lunch, with the increase of sampled points, the training time increases sharply. Also, we can find that with same sampling points, 128 for hierarchy sampling in standard NeuS\cite{neus} and also 128 for uniform sampling, the latter sharply outperforms hierarchy sampling during pose estimation under virtual camera. To reason this, we think that without good poses at beginning, it is impossible to have good mesh, and then the hierarchy sampling based on existing mesh would mislead the whole system. To balance the pose accuracy and training time, we would select "Uniform 32" as our default setting. Notice that we would add and train every frame for 1k iteration, so for 150 frame of sequence, we would need 2.25 hour to finish the progressive optimization under virtual camera and then we can directly start global refinement, leading to a very efficient system.
\end{comment}
\subsection{Results on Egocentric Sequences}
% \begin{figure}
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.30\linewidth]{Final_Recons/labelprinter.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/label_printer.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/labelprinter00.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/milkbox.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/milkbox.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/milkbox00.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/dummy.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/dummy.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/dummy00.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/maneki-neko.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/maneki-neko.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/maneki-neko00.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/spaceinvader.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/spaceinvader.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/spaceinvader00.png} \\
% Reference Image & Pose Trajectory & Mesh\\
% \end{tabular}
% \caption{\textbf{Egocentric Dataset(part-1): Estimated Poses and Reconstructed Meshes}(\textit{top to down: label-printer, milk-box, dummy, maneki-neko, and space-invader.}) \YH{Show how the shape and trajectory improves with different ingredients (ablations).}}
% \label{fig:ml_reconstruction-1}
% \end{figure}
To verify the generalization ability of our method, we capture some sequences with a head-mounted device with egocentric views~(Magic Leap 2~\cite{magcileap2}), where the camera is naturally moving with the user’s head and the target is freely manipulated by the user.
We illustrate the performance of our approach in Fig.~\ref{fig:evaluation_on_ml}.
Note that our method only relies on RGB images and does not use head pose or depth images provided by the device.
The result shows that our method generalizes well to this real setting, and produces accurate results for free-moving daily objects.
% We capture a dataset with objects manipulated by hands with freely grasping style using an industry head-mounted camera. It contains 7 objects, including both textured and texture-less examples, and with various object types in typical scenarios for AR applications. This dataset is more challenging than HO3D since the camera is also moving and the manipulating manner of the objects is much more complex with little prior relative pose information that we can use between the hands and objects. Since this dataset is not annotated, we typically show qualitative results in Figure~\ref{fig:ml_reconstruction-1},\ref{fig:ml_reconstruction-2},\ref{fig:ml_reconstruction-3} % In Figure~\ref{fig:ml_reconstruction-1},\ref{fig:ml_reconstruction-2},\ref{fig:ml_reconstruction-3}, we explore the most free-moving objects under AR environments where two hands hold the object in different gestures, which make it much hard for methods that rely on hand pose prior.
% \begin{figure}
% \centering
% \begin{tabular}{cccc}
% \includegraphics[width=0.30\linewidth]{Final_Recons/scissor.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/scissor.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/scissors00.png} \\
% \includegraphics[width=0.30\linewidth]{Final_Recons/banana.png} &
% \includegraphics[width=0.30\linewidth]{ml_trajectories/banana.png} &
% \includegraphics[width=0.30\linewidth]{Final_Recons/banana00.png} \\
% Reference Image & Pose Trajectory & Mesh\\
% \end{tabular}
% \caption{\textbf{Egocentric Dataset(part-3): Estimated Poses and Reconstructed Meshes}(\textit{top to down: scissor, banana})}
% \label{fig:ml_reconstruction-3}
% \end{figure}
% Also, from the pose trajectory column of Figure~\ref{fig:ml_reconstruction-1},\ref{fig:ml_reconstruction-2},\ref{fig:ml_reconstruction-3}, we can find that only label printer and milk box, dummy would have complex pose trajectories, leading to 360-degree scanning for a comprehensive mesh reconstruction. However, we found that for maneki-neko, space invader, scissor and banana, our system typically fail when there is very sharp movements or the user turn the object to new surface so we have smaller view coverage, which should be enhanced in our future research. We report the number of frames we can work and the maximum rotation angles of each sequences base on our estimated poses in Table~\ref{table:egoentric_angles}.
%(\textit{we exhaustively calculate the the rotation angle from relative rotation between all possible camera pose pairs and find the maximum one. Notice that ">180" means somehow larger than 180 degree while })
% \begin{table}
% \caption{\textbf{Egocentric Dataset Statistics}(\textit{frame numbers and maximum global rotation degree in global-wise.})
% }
% \label{table:egoentric_angles}
% \centering
% \begin{tabular}{lccc}
% \toprule
% Object & Frame\# & Max. Rot.\\
% \midrule
% label printer & 127 & 360.0 \\
% milkbox & 94 & 360.0 \\
% dummy & 109 & 360.0 \\
% meneki-neko & 30 & 113.3 \\
% spaceinvader & 30 & 137.0 \\
% scissor & 50 & 136.7 \\
% banana & 30 & 180.0 \\
% \bottomrule
% \end{tabular}
% \end{table}
% Notice that among different tables the settings may not be exactly the same since they can happen at different times of master thesis, but we guarantee that for each separate table/figure, all results are consistent and comparable.
\subsection{Limitation}
Although our method produces accurate pose and mesh results in most cases, it can not handle scenarios where some parts of the object is occluded for a long time during capture, as shown in Fig.~\ref{fig:failure_cases}. On the other hand, our method still can not produce accurate results for tiny texture-less object due to the lacking of enough features. Addressing this will be one of our future work.
% Our method is not able to solve tiny or symmetric texture-less object due to the lacking of enough features as shown in Fig.~\ref{fig:failure_cases}.
%Addressing this will be one of our future work.
% The proposed method tends to fail with very large rotation changes among small intervals, as shown in Fig.~\ref{fig:failure_cases}. We can find that in bleach(ABF14), the direct effect of wrong poses is the distortion of the mesh front-facing texture; for mustard(SM2), the wrong poses can distort our contour(see the right bottom part where we are not that symmetric as NeuS with GT). Extracting some rotation initialization from some pre-trained model based on prior category information of the target could relieve this problem, which is a potential future direction.
\begin{figure}[htbp]
\centering
\begingroup
\setlength{\tabcolsep}{1pt}
\begin{tabular}{cccc}
% \includegraphics[width=0.24\linewidth]{Final_Recons/MDF14.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/MDF14_pose.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/MDF_14_occluded00.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/MDF_14_occluded01.png} \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/ABF14_ref.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/ABF14_pose.png} &
\includegraphics[width=0.24\linewidth,trim={100, 0, 100, 0},clip]{Experiment_Figure/fail_cases/ABF14_fail_case00.png} &
% \includegraphics[width=0.24\linewidth,trim={120, 20, 100, 0},clip]{Experiment_Figure/fail_cases/ABF14_fail_case01.png}
% \includegraphics[width=0.20\linewidth]{Experiment_Figure/fail_cases/BB12_ref.png} &
% \includegraphics[width=0.20\linewidth]{Experiment_Figure/fail_cases/BB12_pose.png} &
% \includegraphics[width=0.20\linewidth]{Experiment_Figure/fail_cases/BB1200.png} &
% \includegraphics[width=0.20\linewidth]{Experiment_Figure/fail_cases/BB1201.png} \\
\includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/scissor_ref.png} &
% \includegraphics[width=0.24\linewidth]{Experiment_Figure/fail_cases/scissor_pose.png} &
% \includegraphics[width=0.24\linewidth,trim={100, 0, 100, 0},clip]{Experiment_Figure/fail_cases/fail_scissor_201.png} &
\includegraphics[width=0.24\linewidth,trim={100, 0, 100, 0},clip]{Experiment_Figure/fail_cases/fail_scissor_100.png}
\\
% the following examples are not that typical
% \includegraphics[width=0.20\linewidth]{fail_cases/fail_case1.png} &
% \includegraphics[width=0.20\linewidth]{fail_cases/fail_case2.png} &
% \includegraphics[width=0.20\linewidth]{fail_cases/Ours_MDF1400.png} &
% \includegraphics[width=0.20\linewidth]{fail_cases/abf14_gt00.png} \\
% \includegraphics[width=0.20\linewidth]{fail_cases/fail_case4.png} &
% \includegraphics[width=0.20\linewidth]{fail_cases/fail_case3.png} &
% \includegraphics[width=0.20\linewidth]{fail_cases/Ours_sm200.png} &
% \includegraphics[width=0.20\linewidth]{fail_cases/sm2_gt00.png} \\
% \small{Reference} & \small{Mesh} & \small{Reference} & \small{Mesh}\\
\end{tabular}
\endgroup
\caption{\textbf{Failure cases.} The first two figures show an example where the side of the bleach cleaner is occluded for a long time caused by the firmly holding during the whole capture, resulting in deteriorated meshes. The last two figures show an example of a tiny texture-less object, where our method can generate reasonable results but still has a gap from perfection.
}
\label{fig:failure_cases}
\end{figure}
\section{Conclusion}
We have showed that it is possible to jointly optimize the reconstruction and pose estimation for free-moving objects without relying on any prior information, or any segmenting procedure from a monocular RGB video. It relies on the intuition that, using estimated 2D object masks, one can reformulate the joint optimization problem w.r.t. a virtual camera pointing to the object center, which simplifies the trajectory and reduces the search space of optimization significantly. Although the pose and shape are not physically-compliant in the virtual camera system, our experiments have demonstrated that optimizing w.r.t. the virtual camera yields robust initialization results and produces accurate final results after a refinement w.r.t. the real camera. Future work will focus on making the method more robust and more generic so that it can be used in more scenarios.
\begin{thebibliography}{54}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[mag()]{magcileap2}
Magic Leap 2.
\newblock \url{https://www.magicleap.com/magic-leap-2}.
\newblock Accessed: 2024-04-22.
\bibitem[Agarwal et~al.(2011)Agarwal, Furukawa, Snavely, Simon, Curless, Seitz, and Szeliski]{agarwal2011building}
Sameer Agarwal, Yasutaka Furukawa, Noah Snavely, Ian Simon, Brian Curless, Steven~M. Seitz, and Richard Szeliski.
\newblock Building Rome in a day.
\newblock \emph{Commun. {ACM}}, 2011.
\bibitem[Bian et~al.(2023)Bian, Wang, Li, and Bian]{nope-nerf}
Wenjing Bian, Zirui Wang, Kejie Li, and Jia{-}Wang Bian.
\newblock NoPe-NeRF: Optimising Neural Radiance Field with No Pose Prior.
\newblock \emph{CVPR}, 2023.
\bibitem[Billinghurst(2021)]{Billinghurst2021archallenges}
Mark Billinghurst.
\newblock Grand Challenges for Augmented Reality.
\newblock \emph{Frontiers in Virtual Reality}, 2021.
\bibitem[Chen et~al.(2020)Chen, Li, Wang, and Xu]{category_pose_1}
Dengsheng Chen, Jun Li, Zheng Wang, and Kai Xu.
\newblock Learning Canonical Shape Space for Category-Level 6D Object Pose and Size Estimation.
\newblock \emph{CVPR}, 2020.
\bibitem[Cheng and Schwing(2022)]{xmem}
Ho~Kei Cheng and Alexander~G. Schwing.
\newblock {XMem}: Long-Term Video Object Segmentation with an Atkinson-Shiffrin Memory Model.
\newblock \emph{ECCV}, 2022.
\bibitem[Cheng et~al.(2021)Cheng, Li, Asano, Zheng, and Sato]{cheng2021multi}
Ziang Cheng, Hongdong Li, Yuta Asano, Yinqiang Zheng, and Imari Sato.
\newblock Multi-view 3D Reconstruction of a Texture-less Smooth Surface of Unknown Generic Reflectance.
\newblock \emph{CVPR}, 2021.
\bibitem[Fan et~al.(2024)Fan, Parelli, Kadoglou, Kocabas, Chen, Black, and Hilliges]{hold}
Zicong Fan, Maria Parelli, Maria~Eleni Kadoglou, Muhammed Kocabas, Xu Chen, Michael~J. Black, and Otmar Hilliges.
\newblock {HOLD:} Category-agnostic 3D Reconstruction of Interacting Hands and Objects from Video.
\newblock \emph{CVPR}, 2024.
\bibitem[Fu et~al.(2024)Fu, Liu, Kulkarni, Kautz, Efros, and Wang]{3dgs}
Yang Fu, Sifei Liu, Amey Kulkarni, Jan Kautz, Alexei~A. Efros, and Xiaolong Wang.
\newblock COLMAP-Free 3D Gaussian Splatting.
\newblock \emph{CVPR}, 2024.
\bibitem[Hampali et~al.(2020)Hampali, Rad, Oberweger, and Lepetit]{ho3d}
Shreyas Hampali, Mahdi Rad, Markus Oberweger, and Vincent Lepetit.
\newblock HOnnotate: {A} Method for 3D Annotation of Hand and Object Poses.
\newblock \emph{CVPR}, 2020.
\bibitem[Hampali et~al.(2023)Hampali, Hodan, Tran, Ma, Keskin, and Lepetit]{meta-obj}
Shreyas Hampali, Tomas Hodan, Luan Tran, Lingni Ma, Cem Keskin, and Vincent Lepetit.
\newblock {In-Hand 3D Object Scanning from an {RGB} Sequence}.
\newblock \emph{CVPR}, 2023.
\bibitem[Han et~al.(2021)Han, Laga, and Bennamoun]{han2021trends}
X. Han, H. Laga, and M. Bennamoun.
\newblock Image-Based 3D Object Reconstruction: State-of-the-Art and Trends in the Deep Learning Era.
\newblock \emph{IEEE TPAMI}, 2021.
\bibitem[Hodan et~al.(2018)Hodan, Michel, Brachmann, Kehl, GlentBuch, Kraft, Drost, Vidal, Ihrke, Zabulis, Sahin, Manhardt, Tombari, Kim, Matas, and Rother]{BOP}
Tomas Hodan, Frank Michel, Eric Brachmann, Wadim Kehl, Anders GlentBuch, Dirk Kraft, Bertram Drost, Joel Vidal, Stephan Ihrke, Xenophon Zabulis, Caner Sahin, Fabian Manhardt, Federico Tombari, Tae-Kyun Kim, Jiri Matas, and Carsten Rother.
\newblock BOP: Benchmark for 6D Object Pose Estimation.
\newblock \emph{ECCV}, 2018.
\bibitem[Hu et~al.(2021)Hu, Speierer, Jakob, Fua, and Salzmann]{WDR}
Yinlin Hu, Sebastien Speierer, Wenzel Jakob, Pascal Fua, and Mathieu Salzmann.
\newblock Wide-Depth-Range 6D Object Pose Estimation in Space.
\newblock \emph{CVPR}, 2021.
\bibitem[Hu et~al.(2022)Hu, Fua, and Salzmann]{PFA}
Yinlin Hu, Pascal Fua, and Mathieu Salzmann.
\newblock {Perspective Flow Aggregation for Data-Limited 6D Object Pose Estimation}.
\newblock \emph{ECCV}, 2022.
\bibitem[Huang et~al.(2022)Huang, Ji, He, Sun, He, Shuai, Ouyang, and Zhou]{hhor-handold}
Di Huang, Xiaopeng Ji, Xingyi He, Jiaming Sun, Tong He, Qing Shuai, Wanli Ouyang, and Xiaowei Zhou.
\newblock Reconstructing Hand-Held Objects from Monocular Video.
\newblock \emph{{SIGGRAPH} Asia}, 2022.
\bibitem[Jeong et~al.(2021)Jeong, Ahn, Choy, Anandkumar, Cho, and Park]{SCNeRF2021}
Yoonwoo Jeong, Seokjun Ahn, Christopher Choy, Anima Anandkumar, Minsu Cho, and Jaesik Park.
\newblock Self-Calibrating Neural Radiance Fields.
\newblock \emph{ICCV}, 2021.
\bibitem[Kajiya and Von~Herzen(1984)]{volume_rendering}
James~T Kajiya and Brian~P Von~Herzen.
\newblock RAY TRACING VOLUME DENSITIES.
\newblock \emph{ACM SIGGRAPH}, 1984.
\bibitem[Kingma and Ba(2015)]{adam}
Diederik~P. Kingma and Jimmy Ba.
\newblock Adam: {A} Method for Stochastic Optimization.
\newblock \emph{ICLR}, 2015.
\bibitem[Kirillov et~al.(2023)Kirillov, Mintun, Ravi, Mao, Rolland, Gustafson, Xiao, Whitehead, Berg, Lo, Doll{\'a}r, and Girshick]{sam}
Alexander Kirillov, Eric Mintun, Nikhila Ravi, Hanzi Mao, Chloe Rolland, Laura Gustafson, Tete Xiao, Spencer Whitehead, Alexander~C. Berg, Wan-Yen Lo, Piotr Doll{\'a}r, and Ross~B. Girshick.
\newblock Segment Anything.
\newblock \emph{ICCV}, 2023.
\bibitem[Lepetit et~al.(2009)Lepetit, Moreno-Noguer, and Fua]{epnp}
Vincent Lepetit, Francesc Moreno-Noguer, and Pascal Fua.
\newblock EPnP: An Accurate O(n) Solution to the PnP Problem.
\newblock \emph{IJCV}, 2009.
\bibitem[Lin et~al.(2021)Lin, Ma, and andSimon Lucey]{barf}
Chen{-}Hsuan Lin, Wei{-}Chiu Ma, and Antonio~Torralba andSimon Lucey.
\newblock {BARF:} Bundle-Adjusting Neural Radiance Fields.
\newblock \emph{TCCV}, 2021.
\bibitem[Lorensen and Cline(1987)]{marching-cubes}
William~E. Lorensen and Harvey~E. Cline.
\newblock Marching Cubes: A High Resolution 3D Surface Construction Algorithm.
\newblock \emph{SIGGRAPH}, 1987.
\bibitem[Mildenhall et~al.(2020)Mildenhall, andMatthew Tancik, Barron, Ramamoorthi, and Ng]{nerf}
Ben Mildenhall, Pratul P.~Srinivasan andMatthew Tancik, Jonathan~T. Barron, Ravi Ramamoorthi, and Ren Ng.
\newblock {NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis}.
\newblock \emph{ECCV}, 2020.
\bibitem[Mohr et~al.(1995)Mohr, Quan, and Veillon]{mohr1995relative}
Roger Mohr, Long Quan, and Fran{\c{c}}oise Veillon.
\newblock Relative 3D Reconstruction Using Multiple Uncalibrated Images.
\newblock \emph{The International Journal of Robotics Research}, 1995.
\bibitem[Moreno-Noguer et~al.(2007)Moreno-Noguer, Lepetit, and Fua]{moreno2007accurate}
Francesc Moreno-Noguer, Vincent Lepetit, and Pascal Fua.
\newblock Accurate Non-Iterative O (n) Solution to the P n P Problem.
\newblock \emph{ICCV}, 2007.
\bibitem[M{\"u}ller et~al.(2022)M{\"u}ller, Evans, Schied, and Keller]{InstantNGP}
Thomas M{\"u}ller, Alex Evans, Christoph Schied, and Alexander Keller.
\newblock Instant Neural Graphics Primitives with a Multiresolution Hash Encoding.
\newblock \emph{ACM TOG}, 2022.
\bibitem[Oechsle et~al.(2021)Oechsle, Peng, and Geiger]{unisurf}
Michael Oechsle, Songyou Peng, and Andreas Geiger.
\newblock UNISURF: Unifying Neural Implicit Surfaces and Radiance Fields for Multi-View Reconstruction.
\newblock \emph{ICCV}, 2021.
\bibitem[Patten et~al.(2021)Patten, Park, Leitner, Wolfram, and Vincze]{rgb-d-baseline}
Timothy Patten, Kiru Park, Markus Leitner, Kevin Wolfram, and Markus Vincze.
\newblock Object Learning for 6D Pose Estimation and Grasping from {RGB-D} Videos of In-hand Manipulation.
\newblock \emph{{IROS}}, 2021.
\bibitem[Pollefeys et~al.(2004)Pollefeys, Van~Gool, Vergauwen, Verbiest, Cornelis, Tops, and Koch]{pollefeys2004visual}
Marc Pollefeys, Luc Van~Gool, Maarten Vergauwen, Frank Verbiest, Kurt Cornelis, Jan Tops, and Reinhard Koch.
\newblock Visual Modeling with a Hand-Held Camera.
\newblock \emph{IJCV}, 2004.
\bibitem[Pumarola et~al.(2021)Pumarola, Corona, Pons{-}Moll, and Moreno{-}Noguer]{d-nerf}
Albert Pumarola, Enric Corona, Gerard Pons{-}Moll, and Francesc Moreno{-}Noguer.
\newblock {D-NeRF: Neural Radiance Fields for Dynamic Scenes}.
\newblock \emph{CVPR}, 2021.
\bibitem[Rosinol et~al.(2022)Rosinol, Leonard, and Carlone]{rosinol2022nerf}
Antoni Rosinol, John~J Leonard, and Luca Carlone.
\newblock NeRF-SLAM: Real-Time Dense Monocular SLAM with Neural Radiance Fields.
\newblock \emph{arXiv}, 2022.
\bibitem[Rusinkiewicz et~al.(2002)Rusinkiewicz, Hall-Holt, and Levoy]{rusinkiewicz2002realtime}
Szymon Rusinkiewicz, Olaf Hall-Holt, and Marc Levoy.
\newblock Real-Time 3D Model Acquisition.
\newblock \emph{ACM TOG}, 2002.
\bibitem[Rünz et~al.(2020)Rünz, Li, Tang, Ma, Kong, Schmidt, Reid, Agapito, Straub, Lovegrove, and Newcombe]{frodo}
Martin Rünz, Kejie Li, Meng Tang, Lingni Ma, Chen Kong, Tanner Schmidt, Ian Reid, Lourdes Agapito, Julian Straub, Steven Lovegrove, and Richard Newcombe.
\newblock FroDO: From Detections to 3D Objects.
\newblock \emph{CVPR}, 2020.
\bibitem[Sabae et~al.(2023)Sabae, Baraka, and Hadhoud]{nope-neus}
Mohamed~Shawky Sabae, Hoda~Anis Baraka, and Mayada~Mansour Hadhoud.
\newblock NoPose-NeuS: Jointly Optimizing Camera Poses with Neural Implicit Surfaces for Multi-view Reconstruction.
\newblock \emph{arXiv}, 2023.
\bibitem[Schieber et~al.(2023)Schieber, Deuser, Egger, Oswald, and Roth]{NeRFtrinsic_Four}
Hannah Schieber, Fabian Deuser, Bernhard Egger, Norbert Oswald, and Daniel Roth.
\newblock {NeRFtrinsic Four: An End-To-End Trainable NeRF Jointly Optimizing Diverse Intrinsic and Extrinsic Camera Parameters}.
\newblock \emph{arXiv}, 2023.
\bibitem[Sch{\"{o}}nberger and Frahm(2016)]{colmap}
Johannes~L. Sch{\"{o}}nberger and Jan{-}Michael Frahm.
\newblock Structure-from-Motion Revisited.
\newblock \emph{CVPR}, 2016.
\bibitem[Snavely(2011)]{Bundler}
Noah Snavely.
\newblock Scene Reconstruction and Visualization from Internet Photo Collections: {A} Survey.
\newblock \emph{{IPSJ} Trans. Comput. Vis. Appl.}, 2011.
\bibitem[Snavely et~al.(2006)Snavely, Seitz, and Szeliski]{snavely2006photo}
Noah Snavely, Steven~M. Seitz, and Richard Szeliski.
\newblock Photo Tourism: Exploring Photo Collections in 3D.
\newblock \emph{ACM TOG}, 2006.
\bibitem[Su et~al.(2022)Su, Saleh, Fetzer, Rambach, Navab, Busam, Stricker, and Tombari]{Zebrapose}
Yongzhi Su, Mahdi Saleh, Torben Fetzer, Jason Rambach, Nassir Navab, Benjamin Busam, Didier Stricker, and Federico Tombari.
\newblock ZebraPose: Coarse to Fine Surface Encoding for 6DoF Object Pose Estimation.
\newblock \emph{CVPR}, 2022.
\bibitem[Sun et~al.(2021)Sun, Shen, Wang, Bao, and Zhou]{loftr}
Jiaming Sun, Zehong Shen, Yuang Wang, Hujun Bao, and Xiaowei Zhou.
\newblock {LoFTR}: Detector-Free Local Feature Matching with Transformers.
\newblock \emph{CVPR}, 2021.
\bibitem[Sweeney()]{theia-manual}
Chris Sweeney.
\newblock Theia Multiview Geometry Library: Tutorial \& Reference.
\newblock \url{http://theia-sfm.org}.
\bibitem[Thalhammer et~al.(2023)Thalhammer, Bauer, H{\"o}nig, Weibel, Garc{\'\i}a-Rodr{\'\i}guez, and Vincze]{Thalhammer2023ChallengesFM}
Stefan Thalhammer, Dominik Bauer, Peter H{\"o}nig, Jean-Baptiste Weibel, Jos{\'e} Garc{\'\i}a-Rodr{\'\i}guez, and Markus Vincze.
\newblock Challenges for Monocular 6D Object Pose Estimation in Robotics.
\newblock \emph{arXiv}, 2023.
\bibitem[Tian et~al.(2020)Tian, Ang, and Lee]{category_pose_2}
Meng Tian, Marcelo~H. Ang, and Gim~Hee Lee.
\newblock Shape Prior Deformation for Categorical 6D Object Pose and Size Estimation.
\newblock \emph{ECCV}, 2020.
\bibitem[Tzionas and Gall(2015)]{tzionas2015handobject}
Dimitrios Tzionas and Juergen Gall.
\newblock 3D Object Reconstruction from Hand-Object Interactions.
\newblock \emph{ICCV}, 2015.
\bibitem[Wang et~al.(2021{\natexlab{a}})Wang, Manhardt, Tombari, and Ji]{GDRNet}
Gu Wang, Fabian Manhardt, Federico Tombari, and Xiangyang Ji.
\newblock GDR-Net: Geometry-Guided Direct Regression Network for Monocular 6D Object Pose Estimation.
\newblock \emph{CVPR}, 2021{\natexlab{a}}.
\bibitem[Wang et~al.(2021{\natexlab{b}})Wang, Liu, Liu, Theobalt, Komura, and Wang]{neus}
Peng Wang, Lingjie Liu, Yuan Liu, Christian Theobalt, Taku Komura, and Wenping Wang.
\newblock {NeuS: Learning Neural Implicit Surfaces by Volume Rendering for Multi-view Reconstruction}.
\newblock \emph{NeurIPS}, 2021{\natexlab{b}}.
\bibitem[Wang et~al.(2023)Wang, Han, Habermann, Daniilidis, Theobalt, and Liu]{neus2}
Yiming Wang, Qin Han, Marc Habermann, Kostas Daniilidis, Christian Theobalt, and Lingjie Liu.
\newblock {NeuS2: Fast Learning of Neural Implicit Surfaces for Multi-view Reconstruction}.
\newblock \emph{ICCV}, 2023.
\bibitem[Weise et~al.(2008)Weise, Leibe, and Van~Gool]{weise2008inhandmodelin}
Thibaut Weise, Bastian Leibe, and Luc Van~Gool.
\newblock Accurate and Robust Registration for In-hand Modeling.
\newblock \emph{CVPR}, 2008.
\bibitem[Weise et~al.(2009)Weise, Wismer, Leibe, and Van~Gool]{weise2009loopclosure}
Thibaut Weise, Thomas Wismer, Bastian Leibe, and Luc Van~Gool.
\newblock In-hand Scanning with Online Loop Closure.
\newblock \emph{ICCVW}, 2009.
\bibitem[Weng et~al.(2021)Weng, Wang, Zhou, Qin, Duan, Fan, Chen, Su, and Guibas]{category_pose_3}
Yijia Weng, He Wang, Qiang Zhou, Yuzhe Qin, Yueqi Duan, Qingnan Fan, Baoquan Chen, Hao Su, and Leonidas~J. Guibas.
\newblock {CAPTRA:} CAtegory-level Pose Tracking for Rigid and Articulated Objects from Point Clouds.
\newblock \emph{ICCV}, 2021.
\bibitem[Yariv et~al.(2020)Yariv, Kasten, Moran, Galun, Atzmon, Ronen, and Lipman]{yariv2020multiview}
Lior Yariv, Yoni Kasten, Dror Moran, Meirav Galun, Matan Atzmon, Basri Ronen, and Yaron Lipman.
\newblock Multiview Neural Surface Reconstruction by Disentangling Geometry and Appearance.
\newblock \emph{NeurIPS}, 2020.
\bibitem[Yariv et~al.(2021)Yariv, Gu, Kasten, and Lipman]{volsdf}
Lior Yariv, Jiatao Gu, Yoni Kasten, and Yaron Lipman.
\newblock Volume Rendering of Neural Implicit Surfaces.
\newblock \emph{NeurIPS}, 2021.
\bibitem[Ye et~al.(2022)Ye, Gupta, and Tulsiani]{ihoi}
Yufei Ye, Abhinav Gupta, and Shubham Tulsiani.
\newblock What's in your hands? 3D Reconstruction of Generic Objects in Hands.
\newblock \emph{{CVPR}}, 2022.
\end{thebibliography}