Controllable Human-Object Interaction Synthesis
Extracted source text ยท arXiv 2312.03913
% CVPR 2024 Paper Template; see https://github.com/cvpr-org/author-kit
\documentclass[10pt,twocolumn,letterpaper]{article}
%%%%%%%%% PAPER TYPE - PLEASE UPDATE FOR FINAL VERSION
% \usepackage{cvpr} % To produce the CAMERA-READY version
% \usepackage[review]{cvpr} % To produce the REVIEW version
\usepackage[pagenumbers]{cvpr} % To force page numbers, e.g. for an arXiv version
% Import additional packages in the preamble file, before hyperref
\input{preamble}
\input{marcros}
% It is strongly recommended to use hyperref, especially for the review version.
% hyperref with option pagebackref eases the reviewers' job.
% Please disable hyperref *only* if you encounter grave issues,
% e.g. with the file validation for the camera-ready version.
%
% If you comment hyperref and then uncomment it, you should delete *.aux before re-running LaTeX.
% (Or just hit 'q' on the first LaTeX run, let it finish, and you should be clear).
\definecolor{cvprblue}{rgb}{0.21,0.49,0.74}
\usepackage[pagebackref,breaklinks,colorlinks,citecolor=cvprblue]{hyperref}
%%%%%%%%% PAPER ID - PLEASE UPDATE
\def\paperID{9246} % *** Enter the Paper ID here
\def\confName{CVPR}
\def\confYear{2024}
%%%%%%%%% TITLE - PLEASE UPDATE
\title{Controllable Human-Object Interaction Synthesis}
%%%%%%%%% AUTHORS - PLEASE UPDATE
% \author{Jiaman Li, Alexander Clegg, Roozbeh Mottaghi, Jiajun Wu, Xavier Puig, Karen Liu \\
% Stanford University\\
% % {\tt\small jiamanli@stanford.edu}
% % For a paper whose authors are all at the same institution,
% % omit the following lines up until the closing ``}''.
% % Additional authors and addresses can be added with ``\and'',
% % just like the second author.
% % To save space, use either the email address or home page, not both
% % \and
% % Meta\\
% % {\tt\small secondauthor@i2.org}
% }
\author{Jiaman Li$^1$, Alexander Clegg$^2$, Roozbeh Mottaghi$^2$, Jiajun Wu$^1$, Xavier Puig$^2$\footnotemark[2], C. Karen Liu$^1$\footnotemark[2] \\
$^1$Stanford University, $^2$FAIR, Meta\\
}
\begin{document}
%\maketitle
\twocolumn[{%
\renewcommand\twocolumn[1][]{#1}%
% \vspace{-3em}
\maketitle
\vspace{-10mm}
\begin{center}
\centering
\includegraphics[width=\textwidth]{figures/Figure_teaser.pdf}
\vspace{-6mm}
\captionof{figure}{Given an initial object and human state, a language description, and sparse object waypoints in a 3D scene, CHOIS generates synchronized object motion and human motion at the same time. }
% \karen{I'd move the green text a bit to the left. Maybe start from right above the head of the second blue guy.}
\label{figure1}
%\vspace{-5pt}
\end{center}%
}]
\setcounter{figure}{1}
\input{sec/0_abstract}
\input{sec/1_intro}
\input{sec/2_related}
\input{sec/3_method}
\input{sec/4_experiment}
\input{sec/5_conclusion}
\small
\bibliographystyle{ieeenat_fullname}
\bibliography{main}
% WARNING: do not forget to delete the supplementary pages from your submission
% \input{sec/X_suppl}
\end{document}
%
% --- inline annotations
%
\usepackage[dvipsnames]{xcolor}
\newcommand{\red}[1]{{\color{red}#1}}
\newcommand{\todo}[1]{{\color{red}#1}}
\newcommand{\TODO}[1]{\textbf{\color{red}[TODO: #1]}}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{bm}
\usepackage{multirow}
\usepackage{verbatim}
\usepackage{comment}
\usepackage{adjustbox}
\usepackage{wrapfig}
\usepackage{url} % simple URL typesetting
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{xcolor} % colors
\usepackage{array}
\usepackage{xspace}
% --- disable by uncommenting
% \renewcommand{\TODO}[1]{}
% \renewcommand{\todo}[1]{#1}
% require xspace, array
\newcommand{\stderr}[1]{\scriptsize $\pm #1$}
%% layout
\newcolumntype{L}[1]{>{\raggedright\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcolumntype{C}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcolumntype{R}[1]{>{\raggedleft\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcommand{\xpar}[1]{\noindent\textbf{#1}\ \ }
\newcommand{\vpar}[1]{\vspace{3mm}\noindent\textbf{#1}\ \ }
%% notations
\newcommand{\sect}[1]{Section~\ref{#1}}
\newcommand{\sectapp}[1]{Appendix~\ref{#1}}
\newcommand{\ssect}[1]{\S~\ref{#1}}
\newcommand{\eqn}[1]{Equation~\ref{#1}}
\newcommand{\fig}[1]{Figure~\ref{#1}}
\newcommand{\tbl}[1]{Table~\ref{#1}}
\newcommand{\alg}[1]{Algorithm.~\ref{#1}}
\newcommand{\degree}{\ensuremath{^\circ}\xspace}
\newcommand{\ignore}[1]{}
\newcommand{\norm}[1]{\lVert#1\rVert}
\newcommand{\fcseven}{$\mbox{fc}_7$}
% \DeclarePairedDelimiter\ceil{\lceil}{\rceil}
% \DeclarePairedDelimiter\floor{\lfloor}{\rfloor}
\renewcommand*{\thefootnote}{\fnsymbol{footnote}}
%\DeclareMathOperator*{\argmin}{arg\,min}
%\DeclareMathOperator*{\argmax}{arg\,max}
\def\naive{na\"{\i}ve\xspace}
\def\Naive{Na\"{\i}ve\xspace}
\makeatletter
\DeclareRobustCommand\onedot{\futurelet\@let@token\@onedot}
\def\@onedot{\ifx\@let@token.\else.\null\fi\xspace}
\def\eg{e.g\onedot} \def\Eg{E.g\onedot}
\def\ie{i.e\onedot} \def\Ie{I.e\onedot}
\def\cf{\emph{c.f}\onedot} \def\Cf{\emph{C.f}\onedot}
\def\etc{etc\onedot}
\def\vs{\emph{vs}\onedot}
\def\wrt{w.r.t\onedot}
\def\dof{d.o.f\onedot}
\def\etal{et al\onedot}
\def\aka{a.k.a\onedot}
\makeatother
%% comments
\definecolor{MyDarkBlue}{rgb}{0,0.08,1}
\definecolor{MyDarkGreen}{rgb}{0.02,0.6,0.02}
\definecolor{MyDarkRed}{rgb}{0.8,0.02,0.02}
\definecolor{MyDarkOrange}{rgb}{0.40,0.2,0.02}
\definecolor{MyPurple}{RGB}{111,0,255}
\definecolor{MyRed}{rgb}{1.0,0.0,0.0}
\definecolor{MyGold}{rgb}{0.75,0.6,0.12}
\definecolor{MyDarkgray}{rgb}{0.66, 0.66, 0.66}
\definecolor{turquoise}{cmyk}{0.65,0,0.1,0.3}
\newcommand{\jw}[1]{\textcolor{MyDarkGreen}{[Jiajun: #1]}}
\newcommand{\karen}[1]{\textcolor{red}{[Karen: {#1}]}}
\newcommand{\jiaman}[1]{\textcolor{orange}{[Jiaman: {#1}]}}
\newcommand{\xavi}[1]{\textcolor{green}{[Xavi: {#1}]}}
\newcommand{\new}[1]{\textcolor{blue}{#1}}
\newcommand{\revise}[1]{\textcolor{blue}{{#1}}}
\newcommand{\model}{EgoEgo\xspace}
\def\bR{\mathbb{R}}
\def\cL{\mathcal{L}}
\newcommand{\mysubsection}[1]{\vspace{-4pt}\subsection{#1}\vspace{-4pt}}
% \newcommand{\myparagraph}[1]{\vspace{-5pt}\paragraph{#1}}
\newcommand{\myparagraph}[1]{\noindent\textbf{#1}}
\newcommand{\myitem}{\vspace{-5pt}\item}
\newcommand{\mybf}[1]{\vspace{4pt}\noindent\textbf{#1}}
\newcommand{\mycell}[1]{\begin{tabular}[t]{@{}l@{}l}#1\end{tabular}}
\newcommand{\xhdr}[1]{{\noindent \bf #1}}
%\usepackage{titlesec}
%\titlespacing*{\section}{0pt}{0pt plus 2pt minus 2pt}{0pt plus 2pt minus 2pt}
%\titlespacing*{\subsection}{0pt}{0pt plus 1pt minus 1pt}{0pt plus 1pt minus 1pt}
\begin{abstract}
\begin{NoHyper}
\footnotetext{\footnotemark[2] indicates equal contribution.}
\end{NoHyper}
% \karen{Should we change the name to "Language-guided, Human-Object Interaction Synthesis"?}
% Synthesizing semantic-aware, continuous \karen{By "continuous", do you mean long-horizon?}\jiaman{yes, continuous means long-horizon.} human-object interaction is critical to simulate realistic human behaviors. In this work, we address the challenging problem of generating synchronized object motion and human motion guided by language descriptions and sparse object positions \karen{We do not assume that we have object waypoints as input, right? Didn't we generate them based on some heuristics? If so, we should not include it as input.} \jiaman{We use heristics and scenes to extract waypoints for the application part. But for quantitative evaluation, we are using gt object waypoints as input.}. This task serves as a core component for an AI system that generates continuous human-object interactions in 3D scenes driven by language commands \karen{This sentence is repeating similar information as the previous one. Remove it or move it to the end of the abstract to reiterate the contribution of this work.}. We develop a conditional diffusion model to generate object and human motions simultaneously where object motion is represented by global position and relative rotation with respect to the input object geometry \karen{Why do we talk in so much detail about the object representation? Is it the novel contribution we want to emphasize? I'd just state the input/output here. In the next sentence, I would start with "Naively applying diffusion models for human-object motion synthesis would result in bad things. We do XYZ to fix that.}. To improve the matching between the generated object motion and input object positions, we further incorporate sparse object keypoints into our loss functions as additional supervision. Moreover, to enhance the contact realism of hand and object, we leverage a couple of guidance terms in the sampling process of our trained diffusion model to further improve the generation quality. We also demonstrate the effectiveness of our module in 3D environments \karen{Don't we always demonstrate results in a 3D environment?}\jiaman{In our quantitative evaluations, our waypoints are from the ground truth motion, not extracted from 3D scenes.}. By extracting waypoints from 3D scenes, we demonstrate that our learned interaction module can be seamlessly applied to generate realistic, context-aware human-object interactions \karen{From the abstract, it seems that the main contribution to make this work is the waypoints of the object. Can we say that we found that object waypoints are crucial, but they do not need to be dense or precise? This finding enables us to directly apply many existing high-level planning algorithms for motion synthesis in long-horizon and complex scenarios}. \jiaman{revised in the following paragraph.}
% \jiaman{For title: 1. Synthesizing Human-Object Interactions from Language and Waypoints. 2. Controllable Human-Object Interaction Synthesis via Language and Waypoints.}
%Specifically, we employ sparse object waypoints as scene conditions since they can be effectively extracted from 3D scenes using high-level planning methods.
%Given a language description, initial object and human states, and sparse object waypoints,
Synthesizing semantic-aware, long-horizon, human-object interaction is critical to simulate realistic human behaviors. In this work, we address the challenging problem of generating synchronized object motion and human motion guided by language descriptions in 3D scenes.
We propose \textbf{C}ontrollable \textbf{H}uman-\textbf{O}bject \textbf{I}nteraction \textbf{S}ynthesis (CHOIS), an approach that generates object motion and human motion simultaneously using a conditional diffusion model given a language description, initial object and human states, and sparse object waypoints. While language descriptions inform style and intent, waypoints ground the motion in the scene and can be effectively extracted using high-level planning methods.
Naively applying a diffusion model fails to predict object motion aligned with the input waypoints and cannot ensure the realism of interactions that require precise hand-object contact and appropriate contact grounded by the floor. To overcome these problems, we introduce an object geometry loss as additional supervision to improve the matching between generated object motion and input object waypoints. In addition, we design guidance terms to enforce contact constraints during the sampling process of the trained diffusion model. %By extracting object waypoints from 3D scenes, we further demonstrate that our learned interaction module can be seamlessly applied to generate realistic, context-aware human-object interactions.
% \jw{I think we should be careful on the object waypoints. If we use them as input, maybe "language-guided" in the title is not totally precise.}
\end{abstract}
% We aim to address the problem of synthesizing long-horizon human-object interactions in 3D scenes guided by language.
% To solve this problem, we identify the key component of this problem, interaction synthesis conditioned on language, sparse waypoints of object. We introduce sparse object waypoints since the waypoints can be extracted from 3D scenes using high-level planning methods.
% To solve the key problem, we employ a conditional diffusion model. But naively apply a conditional diffusion model cannot generate good results. The generated object motion fails to match the input waypoints conditions, also, there are contact issues between hand-object, foot-floor, object-floor. To address these issue, we propose object keypoints loss to improve matching and guidance for test-time sampling.
\vspace{-4mm}
\section{Introduction}
\label{sec:intro}
% \jw{Please make another careful pass to fix the grammar mistakes, in particular on articles (a/an/the) and the incorrect use of punctuation marks around equations.}
Synthesizing human behaviors in 3D environments is critical for various applications in computer graphics, embodied AI, and robotics. Humans effortlessly navigate and engage within their surroundings, performing a plethora of tasks routinely. For example, drawing a chair closer to a desk to create a workspace, adjusting a floor lamp to cast the perfect glow, or neatly storing a suitcase. Each of these tasks requires precise coordination between the human, the object, and the surroundings. These tasks are also deeply rooted in purpose. Language serves as a powerful tool to articulate and convey these intentions. Synthesizing realistic human and object motion guided by language and scene context is the cornerstone of building an advanced AI systems that simulate continuous human behaviors in diverse 3D environments.
% \karen{Why is language important for the system we want to build? We should also argue that in this paragraph.} \jiaman{added}.
% This capability to realistically synthesize these interactions is crucial not only for animating virtual characters in VR/AR and computer games but also for providing training data to embodied AI systems where assistive robots can be trained in human-centric environments.
% \jw{The intro is also a bit too long. This prior work paragraph can be tightened and content can be moved to related work. The related work section itself can be tightened, too. This will give us the space needed for the new figure to show controllability based on different waypoints.}
% Prior work~\cite{tevet2022human,dabral2022mofusion,zhang2022motiondiffuse} has mostly focused on generating human motion from text,
% % primarily addressed text-to-motion tasks,
% neglecting the complexities of 3D scenes and their interactions. However, intelligent agents should do more than
% move following language descriptions.
% follow linguistic movement instructions.
% They should also effectively execute diverse daily tasks that involve interacting with various objects in 3D spaces.
While some existing works study the problem of human-scene interaction~\cite{hassan_samp_2021}, they are constrained to scenarios with static objects such as sitting on a chair, neglecting the highly dynamic interactions happening frequently in daily life. Recent advancements have been made in modeling dynamic human-object interactions, yet these approaches focus solely on smaller objects~\cite{ghosh2022imos,li2023task} or lack the ability to manipulate diverse objects~\cite{hassan2023synthesizing,xie2023hierarchical}. Manipulating diverse objects of larger size has been explored in recent work~\cite{wan2022learn,xu2023interdiff,li2023object}. However, these approaches rely on sequences of past interaction states or complete sequences of object motion, and are incapable of synthesizing both object motion and human motion from initial states alone. In this work, we aim to advance the field by focusing on synthesizing realistic interactions involving diverse objects of larger size from language and initial states.
% Recent advancements have been made in generating both object and human motion, yet the focus of interaction synthesis, as explored in ~\cite{ghosh2022imos,li2023task}, predominantly focused on small-sized objects where hand movements play a pivotal role. Conversely, other studies, such as ~\cite{hassan2023synthesizing,xie2023hierarchical} tackle the manipulation of large-sized objects. However, the policies they develop cannot generalize to diverse objects, thus restricting their applicability to a limited set of tasks.
% Manipulating diverse large-sized objects has been explored in ~\cite{wan2022learn,xu2023interdiff,li2023object}. However, these works~\cite{wan2022learn,xu2023interdiff} primarily rely on a sequence of past object and human states to predict future interactions, limiting their ability to generate long-term interactions based solely on initial states. OMOMO~\cite{li2023object}, while providing an approach to synthesize human motion from a complete object motion sequence, does not provide a solution for generating object motion.
% \karen{What about OMOMO?} \jiaman{added.}
% Developing a system that simulate continuous interactions is challenging and require a lot of components. This paragraph is trying to list all the important sub-tasks and clarify our main contribution is the core component.
Generating continuous human-object interactions from language descriptions within 3D scenes poses several challenges.
First, we need to generate object and human motion which is realistic and synchronized.
% the synchronized generation of object and human motion remains an unsolved problem.
The human hands should maintain appropriate contact with objects during interaction and object motion should maintain a causal relationship to human actions.
% It is crucial for human hands to maintain appropriate contact with objects during interactions, which requires precise coordination.
Second, 3D scenes are often cluttered with numerous objects, constraining the space of feasible motion trajectories. Thus, it is essential for interaction synthesis to accommodate for environment clutter, rather than operating under the assumption of an empty scene.
% Moreover, the system must effectively interpret language descriptions to guide interactions. These descriptions should provide clear directives to the agent about how and where to place objects within the scene
% \karen{The language challenge is not something we built a solution for, correct? If so, I'd not emphasize the language challenge here.}. \jiaman{correct. language is not our major contribution. removed.}
%Building such a sophisticated system requires integrating multiple components. The key component is an interaction synthesis module capable of generating synchronized object and human motion. Also, to be utilized in the 3D scenes, this module should be adeptly guided by sparse waypoints extracted from the 3D environment.
In this work, we focus on the key problem of synthesizing human-object interactions in 3D environments from natural language commands, generating object motion and human motion guided by language and sparse object waypoints.
% \karen{waypoints are not input.} \jiaman{waypoints are input when we evaluate on our FullBodyManipulation dataset since there's no 3D scene.}.
Starting with a language description outlining the desired human actions, a set of waypoints extracted from the environment, and an initial object and human state, our goal is to generate motions for both humans and objects. These motions should align with the directives specified in the language input, while also conforming to the environmental constraints defined by waypoint conditions derived from 3D scene geometry.
To achieve this,
% we introduce a robust object motion representation that combines a relative rotation matrix with global object positions.
% \karen{Is this new? Did OMOMO use that?} \jiaman{OMOMO didn't use this since OMOMO takes the whole sequence of object geometry as input, it didn't take relative rotation as input or predict the relative rotation.}.
we employ a conditional diffusion model to generate synchronized object and human motion simultaneously, conditioned on language descriptions, initial states, and sparse object waypoints. To improve the accuracy of the predicted object motion, we incorporate an object geometry loss during training. In addition, we devise guidance terms applied during the sampling process to improve the realism of the generated interaction. Furthermore, we demonstrate the effectiveness of our learned interaction synthesis module within a system that produces continuous realistic and context-aware interactions given language descriptions and 3D scenes.
% \jiaman{revised in the following paragraph.} To summarize, our work makes the following contributions. First, we define a problem that generates human-object interactions guided by language and waypoints which can serve as a core module in a complete system for human behavior simulation \karen{Do you mean that identifying the combination of language and object waypoints as informative and expressive condition for human-object motion synthesis is a contribution? I agree that object waypoints can be argued as contribution (see more comments in abstract), but we also need to argue that language is a good way to express human-object interaction (See the comment earlier).} \jiaman{Yes, I think we need to mention why we define the problem of generation interactions from waypoints. We design this interaction synthesis module to utilize it in a system that generate scene-aware interactions.}. Second, we propose an effective interaction synthesis model to generate both object motion and human motion.
% % \karen{The text below is just repeating the previous paragraph. Can be removed.}To ensure the generated object motion matches the input conditions, we introduce object keypoint loss during training to facilitate the object transformation prediction. Also, we devise a couple of guidance terms that can be effectively used during sampling to improve the quality of the generated results. Third, we integrate our learned interaction synthesis module into a pipeline that takes 3D scenes and language as input to synthesize continuous interactions in contextual environments.
% \karen{Also need a few sentences about evaluation.}
% \jiaman{added in the revised version.}
% \karen{After reading the current abstract/intro, I think our contributions should be: 1. We identify that the combination of language and object waypoints provides precise and expressive information for human-object motion synthesis. In addition, the object waypoints do not need to be dense nor precise, which allows us to utilize existing path planning algorithms to create long-horizon and complex scenarios. 2. Based on the above finding, we devise a method that synthesize human-object interaction guided by language and sparse waypoints of the object, using conditional diffusion model. 3. We further integrate our method into a pipeline that takes 3D scenes and language as input to synthesize long-horizon interactions in contextual environments. }.
To summarize, our work makes the following contributions. First, we identify that the combination of language and object waypoints provides precise and expressive information for human-object interaction synthesis. We show that object waypoints do not need to be dense or precise, which allows us to utilize existing path planning algorithms to generate sparse waypoints which represent long-horizon interactions in complex scenarios. Second, based on this finding, we devise a method that synthesizes human-object interaction guided by language and sparse waypoints of the object, using a conditional diffusion model. We demonstrate that our approach synthesizes realistic interactions on FullBodyManipulation dataset~\cite{li2023object}. In addition, the learned model can generalize to novel objects in 3D-FUTURE dataset~\cite{3dfuture}. Third, we integrate our method into a pipeline that synthesizes long-horizon environment-aware human-object interactions from 3D scenes and language input.
% in contextual environments.
\section{Related Work}
\label{sec:formatting}
\paragraph{Motion Synthesis from Language.}
With the development of large-scale high-quality motion capture datasets like AMASS~\cite{AMASS}, there has been a growing interest in generative human motion modeling. BABEL~\cite{punnakkal2021babel} and HumanML3D~\cite{Guo_2022_CVPR} further introduce action labels and language descriptions to enrich the mocap dataset, enabling the development of action-conditioned motion synthesis~\cite{petrovich2021action} and text-conditioned motion synthesis~\cite{Guo_2022_CVPR,petrovich2022temos,tevet2022motionclip}. Prior work has shown that VAE formulation is effective in generating diverse human motion from text~\cite{Guo_2022_CVPR,guo2022tm2t}. Recently, with the success of the diffusion model in this domain~\cite{chen2023executing,barquero2023belfusion,huang2023diffusion,raab2023single,shafir2023human,yuan2023physdiff,zhang2023tedi,tseng2022edge,li2023ego,li2023object,shi2023controllable}, extensive work has explored generating motion from text using conditioning~\cite{tevet2022human,dabral2022mofusion,zhang2022motiondiffuse,karunratanakul2023gmd}. In this work, we also take language descriptions as input to guide our generation. Instead of synthesizing human motion alone, we generate both object motion and human motion conditioned on the text.
\vspace{-4mm}
\paragraph{Motion Synthesis in 3D Scenes.}
With the advent of paired scene-motion data~\cite{prox,araujo2023circle,wang2022humanise,HPS,zheng2022gimo} and paired object-motion data~\cite{zhang2022couch,hassan_samp_2021}, approaches~\cite{wang2021synthesizing,wang2022towards,araujo2023circle,zhang2022couch,hassan_samp_2021,kulkarni2023nifty} have been developed to generate human interactions such as sitting on a chair and reaching a target position in 3D scenes. To populate human-object interactions without training on paired scene-motion data, path planning algorithms have been deployed to generate collision-free paths which then guide the human motion generation~\cite{hassan_samp_2021,mir2023generating,zhang2022wanderings,zhao2023synthesizing}. Another line of work leverages reinforcement learning frameworks to train scene-aware policies for synthesizing navigation and interaction motions in static 3D scenes~\cite{lee2023locomotion,xiao2023unified}. In this work, instead of focusing on static scenes or objects, we synthesize interactions with dynamic objects. Also, inspired by approaches that decompose scene-aware motion generation into path planning and goal-guided generation phases, we design an interaction synthesis module conditioned on sparse object waypoints that can be effectively integrated into a scene-aware synthesis pipeline.
% we aim to develop an effective module that can be integrated into a pipeline that synthesizes interactions in 3D scenes. Utilizing sparse object positions as conditional cues, our model is trained on datasets without explicit scenes, yet it is adept at synthesizing contextually rich interactions when combined with path-planning algorithms.
\vspace{-4mm}
\paragraph{Interaction Synthesis.}
The field of modeling dynamic human-object interactions has largely focused on hand motion synthesis~\cite{zhang2021manipnet,christen2022d,zheng2023cams}. Recently, with the advent of full-body motion datasets with hand-object interactions~\cite{GRAB:2020,fan2023arctic}, models~\cite{taheri2022goal,wu2022saga} have been developed to synthesize full-body motions preceding object grasping. Some recent studies predict object motion based on human movements~\cite{petrov2023object}, and others~\cite{ghosh2022imos,li2023task,braun2023physically} have taken this further by synthesizing both body and hand motion, subsequently applying optimization to predict object motion. However, these approaches focus on smaller objects where hand motion is the primary focus.
In terms of manipulating larger objects, some methods train reinforcement learning policies to synthesize box lifting and moving behaviors~\cite{hassan2023synthesizing,xie2023hierarchical,merel2020catch}, yet these models struggle to generalize to manipulation of diverse objects. Based on paired human-object motion data~\cite{bhatnagar22behave,wan2022learn,li2023object}, recent works predict interactions from a sequence of past interaction states~\cite{wan2022learn,xu2023interdiff} or an object motion sequence ~\cite{li2023object}, incapable of synthesizing interactions in 3D scenes solely from initial states. In this work, we generate synchronized object and human motion conditioned on sparse object waypoints, serving to ground the resulting trajectories in 3D scenes.
\section{Method}
%
\begin{figure*}[t!]
\begin{center}
\vspace{-5mm}
\includegraphics[width=\textwidth]{figures/Figure_method.pdf}
\end{center}
\vspace{-6mm}
\caption{\textbf{Method Overview.} Given an object geometry, we use the BPS representation to encode the geometry and an MLP to project the features into a low-dimensional vector. This feature vector is concatenated with masked pose states to form conditions for the denoising network. During sampling, we use analytical functions to compute gradients and perturb the generation to satisfy our defined constraints. }
% \jw{You can make G, S, pi, Linear, Transformer, etc all shorter, and you only need three "Add" circle instead of four in the middle. This way, the "masked condition representation" line can go right and then down, without first going up. I think it'll look much nicer. You also get some space.}
\label{fig:overview}
\vspace{-4mm}
\end{figure*}
% \xavi{The consistent 30-frame interval is confusing here. How about: Our goal is to generate long-range synchronized human and object motion given and initial human pose the object geometry and the object final pose. We then add the long-range as a third challenge and we address this using waypoints}
% Our goal is to simultaneously generate both object and human motion, conditioned on the object geometry, initial human pose and object pose, end positions of the object, object waypoints with a consistent 30-frame interval, and language inputs. Two primary challenges arise in this context: first, effectively representing object motion to facilitate testing on objects not seen during training; and second, ensuring the realism of contact between the human and object. To address these, we have developed an object representation that leverages relative rotation with respect to the object's input geometry, coupled with the global centroid position of the object. Additionally, to enhance the realism of the interactions, we incorporate several constraints as guidance during the sampling process of our diffusion model.
% We illustrate our approach in Figure~\ref{fig:overview}.
Our goal is to generate synchronized
% simultaneously generate both
object and human motion, conditioned on a language description, object geometry, initial object and human states, and sparse object waypoints. Two primary challenges arise in this context: first, modeling the complexity of synchronized object and human motion while also respecting the sparse condition signals; and second, ensuring the realism of contact between the human and object. To tackle the generation problem of complex interactions, we employ a conditional diffusion model to generate object motion and human motion at the same time. However, naively learning a conditional diffusion model to generate both object motion and human motion cannot ensure the precise contact between hand and object and the realism of the interaction. Thus, we incorporate several constraints as guidance during the sampling process of our trained diffusion model. We illustrate our approach in Figure~\ref{fig:overview}.
\subsection{Data Representation}
\paragraph{Object and Human Motion Representation.} We denote the human motion as $\bm{X} \in \mathbb{R}^{T \times D}$, where $T$ and $D$ represent the time steps and dimension of the human pose. $\bm{X}_t$, corresponding to the human pose at frame $t$, consists of global joint positions and 6D continuous rotations~\cite{zhou2019continuity}.
We adopt the widely used parametric human model, SMPL-X~\cite{smplx} to reconstruct the human mesh from the pose and shape parameters. To represent the object motion, we use two components: the global 3D position and the relative rotation.
% with respect to a reference coordinate frame from which we encode the object geometry.
% the input object geometry.
% \karen{I am not sure what "wrt object geometry" means. Do you mean "wrt the coordinate frame in which the geometry is defined"?}. \jiaman{the relative rotation is applied to the object vertices with respect to the coordinate frame in which the object geometry is initially defined. basically, at frame $t$, the object vertices (in global coordinate frame) can be computed by applying the relative rotation to the input object's vertices $\bm{V}_t = \bm{R}_{rel}\bm{V}_{input}$}
The global position is represented by the centroid of the object, while the relative rotation, denoted as $\bm{R}_\text{rel}$ at frame $t$, is expressed with respect to the input object's geometry $\bm{V}$ such that $\bm{V}_t = \bm{R}_\text{rel}\bm{V}$, where $\bm{V}_t$ represent the vertices of object at frame $t$. We denote the object motion by $\bm{O} \in \mathbb{R}^{T \times 12}$.
% we define O_t = {t_t, R_t}, where t_t is the global position and R_t is the rep... wrt ..... For a object's geometry V, V_t=R_t*V + t_t
\vspace{-4mm}
\paragraph{Object Geometry Representation.} We represent the object geometry using the Basis Point Set (BPS) representation~\cite{prokudin2019efficient}. Following prior work~\cite{li2023object}, we begin by sampling a set of basis points from a sphere with a 1-meter radius. Subsequently, for each sampled point, we calculate the minimum Euclidean distance to the nearest point on the object's mesh. Alongside this, we record the directional vectors from the basis points to their nearest neighbors. The resulting BPS representation is denoted as $\bm{G} \in \mathbb{R}^{1024 \times 3}$, representing 1024 sampled points each with a 3-dimensional vector indicating their spatial relationship to the object's surface.
\vspace{-4mm}
\paragraph{Input Condition Representation.}
We first use an MLP to project the object BPS representation $\bm{G}$ to a low-dimensional vector which is then broadcasted to each frame denoted as $\hat{\bm{G}} \in \mathbb{R}^{T \times 256}$ following~\cite{li2023object}. We then adopt a masked motion data representation denoted as $\bm{S} \in \mathbb{R}^{T \times (12 + D)}$ to represent the initial states and waypoint conditions. The initial state contains the human pose and object pose at the first frame.
The waypoint conditions consist of a series of 2D object positions for every 30 frames, and a 3D object position at the final frame. The remainder of $\bm{S}$ is padded with zeros. The encoded object geometry vector and the masked motion condition vector are then concatenated, serving as part of the input for our denoising network. For effectively integrating language conditions as input, we utilize CLIP~\cite{radford2021learning} as a text encoder to extract language embeddings.
% Simultaneously, an MLP is employed to embed the noise level $n$. The resultant embeddings from these two sources are combined to form a single embedding vector, denoted as $\bm{e}_{n}$.
\subsection{Interaction Synthesis Model}
\paragraph{Conditional Diffusion Model.} In our framework, we utilize a conditional diffusion model~\cite{ho2020denoising} to generate synchronized object and human motion. To improve the realism of hand-object interaction, our model also predicts contact labels $\bm{H} \in \mathbb{R}^{T \times 2}$ for both the left and right hands. These predicted contact labels play a crucial role in guiding the sampling process, ensuring more accurate and realistic hand-object contacts in the generated motion sequence. The complete data representation in our model is denoted as $\bm{\tau} = \{\bm{X}, \bm{O}, \bm{H}\}$, encapsulating motion and contact data.
The conditional signals of our model, denoted as $\bm{c}$, include initial states, sparse object waypoints, the object BPS representation, and language descriptions. The diffusion model consists of a forward diffusion process that progressively adds noise to the clean data $\bm{\tau}_{0}$ and a reverse diffusion process which is trained to reverse this process. The forward diffusion process introduces noise for $N$ steps formulated using a Markov chain,
\begin{align}
\label{eq:forward_diffusion_step}
q(\bm{\tau}_{n}|\bm{\tau}_{n-1}) := \mathcal{N}(\bm{\tau}_{n}; \sqrt{1-\beta_{n}}\bm{\tau}_{n-1}, \beta_{n}\bm{I}), \\
q(\bm{\tau}_{1:N}|\bm{\tau}_{0}) := \prod_{n=1}^{N} q(\bm{\tau}_{n}|\bm{\tau}_{n-1}),
\end{align}
where $\beta_{n}$ represents a fixed variance schedule and $\bm{I}$ is an identity matrix. Our goal is to learn a model $p_{\theta}$ to reverse the forward diffusion process,
\begin{equation}
\label{eq:reverse_diffusion_step}
p_{\theta}(\bm{\tau}_{n-1}|\bm{\tau}_{n}, \bm{c}) := \mathcal{N}(\bm{\tau}_{n-1}; \bm{\mu}_{\theta}(\bm{\tau}_n, n, \bm{c}), \bm{\Sigma}_{n}),
\end{equation}
where $\bm{\mu}_{\theta}$ denotes the predicted mean and $\bm{\Sigma}_{n}$ is a fixed variance.
Learning the mean can be re-parameterized as learning to predict the clean data representation $\bm{\tau}_0$. The objective~\cite{ho2020denoising} is defined as
\begin{equation}
\label{eq:loss}
\mathcal{L} = \mathbb{E}_{\bm{\tau}_0, n}||\hat{\bm{\tau}}_{\theta}(\bm{x}_{n}, n, \bm{c}) - \bm{\tau}_{0}||_{1}.
\vspace{-4mm}
\end{equation}
% \karen{Why do we use l-1 norm? Same question for Equation
% (5).}
% \jiaman{there's no specific reason for l-1 norm, l1-norm and l2-norm are both ok. I used l-1 because tried l1-norm first, also tried l2 at some point but didn't find improvement.}
% \vspace{-4mm}
\paragraph{Model Architecture.}
We employ a transformer architecture~\cite{transformer} as our denoising network. Our input consists of object geometry conditions $\hat{\bm{G}}$, masked motion conditions $\bm{S}$, and noisy data representation $\bm{\tau}_{n}$ at noise level $n$. The input is projected to a sequence of feature vectors using a linear layer. We employ an MLP to embed the noise level $n$. Then we combine the noise level embedding and the language embedding to form a single embedding vector denoted as $\bm{e}_{n}$. The embedding vector $\bm{e}_{n}$ has the same dimension as these feature vectors and is fed to the transformer along with these vectors. The final prediction $\hat{\bm{\tau}}_{0}$ is made by projecting the updated feature vectors of the transformer excluding the time step corresponding to the embedding $\bm{e}_{n}$. The interaction synthesis model is illustrated in Figure~\ref{fig:overview}.
\vspace{-4mm}
\paragraph{Object Geometry Loss.} During the training phase, we incorporate an additional loss to improve the object motion prediction. Utilizing the Basis Point Set (BPS) representation, we initially compute the nearest neighbor points on the object mesh in rest pose for each of the fixed set of points. From these, we sample 100 points out of the 1024 nearest neighbors to capture a rough outline of the object's shape. These selected points are defined as $\bm{K}_\text{rest} \in \mathbb{R}^{100 \times 3}$, representing our selected object vertices at rest pose.
At each time step in our model, the predicted object rotation (converted to relative rotation with respect to the object geometry in rest pose) and position are employed to calculate the corresponding positions of these selected vertices. This is represented by the following equation, where $\hat{\bm{R}}_t$ and $\hat{\bm{d}}_{t}$ denote the predicted rotation and translation of the object, and $\bm{K}_{t}$ refers to the ground truth vertices at time step $t$. The object geometry loss is computed as
\begin{equation}
\label{eq:loss_k}
\mathcal{L}_{\text{obj}} = \sum_{t=1}^{T}||\hat{\bm{R}_t}\bm{K}_\text{rest} + \hat{\bm{d}}_t - \bm{K}_t||_{1}.
\end{equation}
This loss function plays a critical role in guiding the model to accurately predict the transformation of the object.
\subsection{Guidance}
% \karen{It would be great if we could discuss why we decide to put Object Keypoints as loss in training rather a guidance like all other terms? In general, if we have any advice on how we decide what goes into loss and what goes to guidance that would be very nice.}
% \jiaman{I think guidance is used to refine the generation. If the model without guidance is not well-trained, adding guidance cannot improve the results. Here, the object keypoint loss is important for training to predict better object motion. Without this loss, the object position would be distant from the given start and end position, and sometimes leads to unrealistic object motion, which is hard to rectify using guidance during the sampling process. And I can think of two reasons that we use a lot of guidance instead of including these terms into training, one is training efficiency, some loss requires heavy computation which is not suitable for training, and the other reason I think is balancing loss weight is harder in training than inference. Since in training, to find proper weights we need to re-train the model with different weights. While using them as guidance, it's easier to try different weight for each guidance. }
During the training phase of our interaction synthesis model, there are no explicit contact constraints enforced in the losses. Incorporating loss terms such as hand-object contact loss, and object-floor penetration loss poses a challenge for training. First, these types of loss terms are computationally expensive and would slow down training significantly. Second, introducing more loss terms requires meticulously balancing different losses which usually necessitates re-training models with different settings. Instead, enforcing these constraints during test time is more flexible and makes it easier to select appropriate weights for different terms. Thus, to refine our generated interactions, we propose the application of guidance during the sampling process.
In the diffusion model framework, classifier guidance is commonly applied during test time to control the generation process in order to satisfy specific objectives or constraints. A typical approach to applying classifier guidance~\cite{dhariwal2021diffusion} is to perturb the noisy predicted mean at each denoising step. This is formulated as $\Tilde{\bm{\mu}} = \bm{\mu} - \alpha\bm{\Sigma}_{n}\nabla_{\bm{\mu}}F(\bm{\mu})$, where $\bm{\mu}$ denotes the predicted mean at denoising step $n$ defined by Equation~\ref{eq:reverse_diffusion_step}. $F$ represents a learned or analytical function that determines how much the predicted mean should be penalized and $\alpha$ represents the strength of the perturbation. This guidance computes the gradient with respect to the noisy mean, requiring $F$ to be trained on noisy data or a deterministic function designed for noisy data. Another approach is reconstruction guidance~\cite{ho2022video}, which has proven to be effective for controlling the generation process in prior work~\cite{rempe2023trace,karunratanakul2023gmd,kulkarni2023nifty}. Instead of perturbing the noisy mean, it perturbs the predicted clean data representation $\hat{\bm{\tau}}_{0}$ using the gradient with respect to the noisy input data representation $\bm{\tau}_{n}$. The process is formally represented as
\begin{equation}
\label{eq:guidance}
\Tilde{\bm{\tau}}_{0} = \hat{\bm{\tau}}_{0} - \alpha\bm{\Sigma}_{n}\nabla_{\bm{\tau}_n} F(\hat{\bm{\tau}}_{0}).
\end{equation}
% \jw{One thing that's missing is how the various Fs are integrated to be the F above. We need one more equation. And maybe we should rename $F$ to $F_\text{total}$ for better clarity.}
In this work, we leverage reconstruction guidance~\cite{ho2022video} in the sampling process as we empirically found it to be more stable. We define multiple analytical functions as guidance terms which we will introduce in the following sections.
% which can be either an analytical function or a learned model. This function is then employed to perturb the predicted clean data representation $\hat{\bm{\tau}}_{0}$ using the gradient with respect to the noisy input data representation $\bm{\tau}_{n}$. The process is formally represented as follows.
% \begin{equation}
% \label{eq:guidance}
% \Tilde{\bm{\tau}}_{0} = \hat{\bm{\tau}}_{0} - \alpha \nabla_{\bm{\tau}_n} F(\hat{\bm{\tau}}_{0})
% \end{equation}
% \karen{Why do we take gradient wrt $\tau_n$ but not $\hat{\tau}_0$?}
% \jiaman{The original classifier guidance is perturbing the noisy predicted mean which is defined as $\Tilde{\bm{\mu}} = \hat{\bm{\mu}} - \alpha \nabla_{\hat{\bm{\mu}}}F(\hat{\bm{\mu}})$. This guidance computes gradient with respect to the "noisy mean" which may be unstable. I tried a couple of experiments with this type of guidance and cannot make it work. What we're using now is called "reconstruction guidance". It's proposed in video diffusion model~\cite{ho2022video} and successfully applied in TRACE and PACE~\cite{rempe2023trace} and guided motion diffusion model~\cite{karunratanakul2023gmd}. In the reconstruction guidance, the function takes a predicted clean data representation as input and is found effective in prior work.}
\vspace{-4mm}
\paragraph{Hand Object Contact Guidance.}
We have implemented a specialized contact guidance function to improve the hand-object contact accuracy for frames generated by our model. This function is specifically designed to address cases where a noticeable distance exists between the hands and the object, thereby improving the realism and precision of the interaction.
The contact guidance function is defined as follows:
\begin{equation}
\label{eq:loss_contact}
F_{\text{contact}} = \bm{M}_{l}||\bm{J}_{l}-\bm{V}_{l}||_{1} + \bm{M}_{r}||\bm{J}_{r}-\bm{V}_{r}||_{1}.
\end{equation}
In this equation, $\bm{M}_{l}$ and $\bm{M}_{r}$ are binary masks for the left and right hand, respectively. These masks are derived from the predicted contact labels $\bm{H}$, with $\bm{M}_{l}, \bm{M}_{r} \in \mathbb{R}^{T \times 1}$ and are defined as $\bm{M}_{l}, \bm{M}_{r} = \left(\bm{H} > 0.95\right)$. This thresholding identifies frames where contact is likely to occur. $\bm{J}_{l}$, $\bm{J}_{r} \in \mathbb{R}^{T \times 3}$ represent the positions of the left and right hands. Meanwhile, $\bm{V}_{l}$, $\bm{V}_{r} \in \mathbb{R}^{T \times 3}$ denotes the nearest neighbor points on the object mesh to the respective hand positions.
\vspace{-4mm}
\paragraph{Feet-Floor Contact Guidance.}
When generating joint positions and rotations, our model operates without awareness of the body's shape. Consequently, using the SMPL-X model~\cite{smplx} with predicted root positions, joint rotations, and a test subject's specific body shape parameters to reconstruct the human mesh can sometimes lead to scenarios where the feet do not touch the floor. To rectify this, we implement a guidance term that encourages realistic feet-floor contact.
The joint positions of the left and right toes are represented as $\bm{J}_{l}$ and $\bm{J}_{r}$, respectively. We identify the supporting foot in each frame by comparing the z components of these two joints at each frame. We also introduce a threshold height $h = 0.02$ meters, which is determined from the analysis of foot height in the ground truth motion. The guidance term is defined as follows:
\begin{equation}
\label{eq:loss_feet_floor}
F_{\text{feet}} = ||min(\bm{J}_{l}^{z}, \bm{J}_{r}^{z}) - h||_{2}.
\end{equation}
%\karen{min operator is sub-differentiable. Is this going to cause an issue? How did you compute the gradient of min?} \jiaman{I'm using torch.minimum(a, b), in pytorch implementation, when a $<$ b, then the gradient for a is 1, when a $>$ b, the gradient for a is 0.}
This function computes the norm of the vertical difference between the lowest point of either toe and the threshold height $h$.
\vspace{-4mm}
\paragraph{Object-Floor Penetration Guidance.}
\begin{table*}[t!]
\small
\begin{center}
%\centering
\footnotesize{
%\resizebox{2\columnwidth}{!}{
\setlength{\tabcolsep}{4pt}
\begin{tabular}{lcccccccccccccc}
%\begin{tabular}{|c|c|c}
\toprule
& \multicolumn{3}{c}{Condition Matching} & \multicolumn{2}{c}{Human Motion} & \multicolumn{5}{c}{Interaction} & \multicolumn{4}{c}{GT Difference} \\
\cmidrule(lr){2-4}\cmidrule(lr){5-6}\cmidrule(lr){7-11}\cmidrule(lr){12-15}
Method & $T_{s}\downarrow$ & $T_{e}\downarrow$ & $T_{xy}\downarrow$ & $H_{feet}\downarrow$ & FS$\downarrow$ & $C_{prec}\uparrow$ & $C_{rec}\uparrow$ & $C_{F_1}\uparrow$ & $C_{\%}\uparrow$ & $P_{hand}\downarrow$ & MPJPE$\downarrow$ & $T_{root}\downarrow$ & $T_{obj}\downarrow$ & $O_{obj}\downarrow$ \\
\midrule
OMOMO~\cite{li2023object} & 0 & 0 & 0 & 7.21 & 0.41 & 0.68 & 0.56 & 0.57 & 0.54 & \textbf{0.51} & 21.73 & 36.62 & 17.12 & 1.21 \\
CHOIS w/o $L_{obj}$ & 5.76 & 14.16 & 8.44 & 6.55 & 0.40 & 0.75 & 0.50 & 0.55 & 0.43 & 0.66 & \textbf{14.34} & \textbf{21.97} & 15.53 & \textbf{0.98} \\
CHOIS w/o $F_\text{all}$ & 1.75 & 6.61 & \textbf{2.69} & 6.64 & 0.38 & 0.78 & 0.49 & 0.55 & 0.41 & 0.65 & 15.23 & 24.13 & \textbf{11.51} & 0.99 \\
CHOIS (ours) & \textbf{1.71} & \textbf{6.31} & 2.87 & \textbf{4.20} & \textbf{0.35} & \textbf{0.80} & \textbf{0.64} & \textbf{0.67} & \textbf{0.54} & 0.59 & 15.30 & 24.43 & 12.53 & 0.99 \\
\bottomrule
\end{tabular}
}
\end{center}
\vspace{-5mm}
\caption{\textbf{Interation synthesis} on the FullBodyManipulation dataset~\cite{li2023object}.}
\label{tab:single_window_cmp_seen}
\vspace{-4mm}
\end{table*}
\begin{table}[!t]
\centering
% \vspace{-5mm}
% \vspace{-2mm}
%\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
%\begin{adjustbox}{width=0.9\textwidth} {
\small
\footnotesize{
\setlength{\tabcolsep}{2pt}
\begin{tabular}{lccccccccc}
\toprule
& \multicolumn{3}{c}{Condition Matching} & \multicolumn{2}{c}{Human Motion} & \multicolumn{2}{c}{Interaction} \\
\cmidrule(lr){2-4}\cmidrule(lr){5-6}\cmidrule(lr){7-8}
& $T_{s}\downarrow$ & $T_{e}\downarrow$ & $T_{xy}\downarrow$ & $H_{feet}\downarrow$ & FS$\downarrow$ & $C_{\%}\uparrow$ & $P_{hand}\downarrow$ \\
\midrule
OMOMO~\cite{li2023object} & 0 & 0 & 0 & 6.39 & 0.43 & 0.40 & \textbf{0.10} \\
CHOIS w/o $L_{obj}$ & 6.70 & 13.73 & 7.99 & 5.68 & 0.41 & 0.36 & 0.30 \\
CHOIS w/o $F_\text{all}$ & 5.75 & 7.96 & \textbf{2.68} & 5.84 & 0.39 & 0.33 & 0.26 \\
CHOIS (ours) & \textbf{4.12} & \textbf{7.35} & 2.92 & \textbf{3.75} & \textbf{0.38} & \textbf{0.48} & 0.15 \\
\bottomrule
\end{tabular}
}
%}\end{adjustbox}
% \vspace{-0.6cm}
\vspace{-2mm}
\caption{\textbf{Interaction synthesis} on the 3D-FUTURE dataset~\cite{3dfuture}.
}
\label{tab:single_window_cmp_unseen}
\vspace{-4mm}
\end{table}
To address the issue of generated object states potentially penetrating the floor, we integrate an additional guidance function into the sampling process. Given that our floor is positioned at the plane where $z = 0$, we define the guidance term as follows:
\begin{equation}
\label{eq:loss_object_floor}
F_{\text{obj}} = ||min(\bm{V}^z, 0)||_{1},
\end{equation}
where $\bm{V}^z$ represents the z-coordinate of the object vertices.
During the inference phase, we apply multiple guidance concurrently defined as follows,
\begin{equation}
\label{eq:loss_all_guidance}
F_{\text{all}} = \lambda_{1}F_{\text{contact}} + \lambda_{2}F_{\text{feet}} + \lambda_{3}F_{\text{obj}},
\end{equation}
where $\lambda_{1}$, $\lambda_{2}$, $\lambda_{3}$ denote the loss weights for different terms. We apply the guidance in the last 10 denoising steps only since the prediction in the early steps is extremely noisy.
\section{Experiments}
We first introduce the datasets and evaluation metrics. Then we show comparisons of our proposed approach against the baselines. We further conduct a human perceptual study to complement our evaluation and ablation study to verify the effectiveness of our proposed guidance terms. Moreover, we demonstrate an application that generates long-term interactions conditioned on object waypoints extracted from 3D scenes.
\subsection{Datasets}
\noindent\textbf{The FullBodyManipulation dataset}~\cite{li2023object} consists of 10 hours of high-quality, paired object and human motion, including interaction with 15 different objects. However, our study does not encompass the generation of motion for articulated objects, leading us to exclude sequences related to two such objects (vacuum and mop). We employ this dataset both for training our interaction model and for evaluating the generated results. The training set comprises 15 subjects, with an additional 2 subjects designated for testing, adhering to the dataset partitioning used in OMOMO~\cite{li2023object}.
\vspace{2pt}
\noindent\textbf{The 3D-FUTURE dataset}~\cite{fu20213d} includes 3D models of various furniture items. From this dataset, we select 17 objects representing diverse types (such as chairs, tables, floor lamps, and boxes). This dataset serves to test our model's ability to generalize to objects it has not previously encountered. Given that the 3D-FUTURE dataset only includes 3D models, we integrate object position data from the testing set of the FullBodyManipulation dataset~\cite{li2023object} for evaluation.
% \paragraph{HumanML3D Dataset.} contains paired language descriptions and human motions. The human motion data contains subsets from AMASS dataset~\cite{}.
%\vspace{-4mm}
\subsection{Evaluation Metrics}
\noindent\textbf{Condition Matching Metric:} This metric calculates the Euclidean distance between the predicted and input object waypoints. It includes the start object position error $\left(T_{s}\right)$, end object position error $\left(T_{e}\right)$, and waypoint errors $\left(T_{xy}\right)$, all measured in centimeters (cm).
\vspace{2pt}
\noindent\textbf{Human Motion Quality Metric:} This metric encompasses the foot sliding score (FS) and foot heights $\left(H_{feet}\right)$. FS is the weighted average of accumulated translation in the xy plane, following prior work~\cite{he2022nemf}, measured in centimeters (cm). $H_{feet}$ assesses the height of the feet, also in centimeters.
\vspace{2pt}
\noindent\textbf{Interaction Quality Metric:} This metric assesses the accuracy of hand-object interactions, encompassing both contacts and penetrations. For contact accuracy, it employs precision $\left(C_{prec}\right)$, recall $\left(C_{rec}\right)$, and F1 score $\left(C_{F_1}\right)$ metrics following prior work~\cite{li2023object}. Additionally, it includes contact percentage $\left(C_{\%}\right)$, determined by the proportion of frames where contact is detected. To compute the penetration score $\left(P_{hand}\right)$, each vertex of the hand $V_i$ is used to query the precomputed object's Signed Distance Field (SDF). This process yields a corresponding distance value $d_i$ for each vertex. The penetration score is then derived by computing the average of the negative distance values (representing penetration), formalized as $\frac{1}{n}\sum_{i=1}^{n}|min(d_i, 0)|$, measured in centimeters (cm).
\vspace{2pt}
\noindent\textbf{Ground Truth (GT) Difference Metric:} This metric measures the deviation of generated results from the ground truth motion. It comprises the mean per-joint position error (MPJPE), translation error of the root joint $\left(T_{root}\right)$, and object position error $\left(T_{obj}\right)$, all computed using the Euclidean distance between the predicted and actual ground truth positions in centimeters (cm). Additionally, this metric includes the root joint orientation error $\left(O_{root}\right)$ and the object orientation error $\left(O_{obj}\right)$. These errors are calculated with the Frobenius norm of the rotational difference, formulated as $||\bm{R}_{pred}\bm{R}_{gt}^{-1} - \bm{I}||_{2}$ where $\bm{R}_{pred}$ and $\bm{R}_{gt}$ represent the predicted and ground truth rotation matrices respectively.
%\end{itemize}
% \begin{figure*}[t!]
% \begin{center}
% %\vspace{-5mm}
% \includegraphics[width=\textwidth]{figures/Figure_single_window_cmp.pdf}
% \end{center}
% \vspace{-6mm}
% \caption{Qualitative results comparisons.\jiaman{replace with unseen objects}}
% \label{fig:qualitative_res_unseen}
% \vspace{-2mm}
% \end{figure*}
\begin{figure*}[t!]
\begin{center}
%\vspace{-5mm}
\includegraphics[width=\textwidth]{figures/Figure_single_window_cmp_unseen.pdf}
\end{center}
\vspace{-6mm}
\caption{\textbf{Qualitative results.} The first two rows show the results on the FullBodyManipulation dataset~\cite{li2023object}, and the last row shows the results on the 3D-FUTURE dataset~\cite{3dfuture}.}
% \karen{The last text input is a bit unintuitive. If we say "Push the table forward", will you get very different target? Alternatively, can we use a different example since OMOMO seems to do a deecent job too?}
\label{fig:qualitative_res}
\vspace{-2mm}
\end{figure*}
\begin{figure}[t!]
\begin{center}
%\vspace{-2mm}
\includegraphics[width=\linewidth]{figures/Figure_human_study.pdf}
\end{center}
\vspace{-6mm}
\caption{\textbf{Results of human perceptual studies.} The numbers shown in the chart represent the percentage (\%).}
% \jw{the colors are too dark and thus the numbers are hard to see. And the colors are too similar.}
% \vspace{-10mm}
\label{fig:human_study}
\vspace{-5mm}
\end{figure}
\begin{table*}[t!]
\small
\begin{center}
%\centering
\footnotesize{
%\resizebox{2\columnwidth}{!}{
\setlength{\tabcolsep}{4pt}
\begin{tabular}{lcccccccccccccc}
%\begin{tabular}{|c|c|c}
\toprule
& \multicolumn{3}{c}{Condition Matching} & \multicolumn{2}{c}{Human Motion} & \multicolumn{5}{c}{Interaction} & \multicolumn{4}{c}{GT Difference} \\
\cmidrule(lr){2-4}\cmidrule(lr){5-6}\cmidrule(lr){7-11}\cmidrule(lr){12-15}
Method & $T_{s}\downarrow$ & $T_{e}\downarrow$ & $T_{xy}\downarrow$ & $H_{feet}\downarrow$ & FS$\downarrow$ & $C_{prec}\uparrow$ & $C_{rec}\uparrow$ & $C_{F_1}\uparrow$ & $C_{\%}\uparrow$ & $P_{hand}\downarrow$ & MPJPE$\downarrow$ & $T_{root}\downarrow$ & $T_{obj}\downarrow$ & $O_{obj}\downarrow$ \\
\midrule
CHOIS w/o $F_\text{contact}$ & \textbf{1.70} & 6.42 & 2.70 & \textbf{3.93} & \textbf{0.32} & 0.78 & 0.49 & 0.55 & 0.41 & 0.65 & 15.41 & \textbf{23.63} & \textbf{11.44} & 0.99 \\
CHOIS w/o $F_\text{feet}$ & 1.72 & 6.34 & 2.90 & 6.65 & 0.39 & 0.81 & \textbf{0.64} & 0.66 & \textbf{0.54} & \textbf{0.58} & 15.44 & 25.09 & 13.31 & 0.99 \\
CHOIS w/o $F_\text{all}$ & 1.75 & 6.61 & \textbf{2.69} & 6.64 & 0.38 & 0.78 & 0.49 & 0.55 & 0.41 & 0.65 & \textbf{15.23} & 24.13 & 11.51 & 0.99 \\
CHOIS (ours) & 1.71 & \textbf{6.31} & 2.87 & 4.20 & \textbf{0.35} & \textbf{0.80} & \textbf{0.64} & \textbf{0.67} & \textbf{0.54} & 0.59 & 15.30 & 24.43 & 12.53 & 0.99 \\
\bottomrule
\end{tabular}
}
\end{center}
\vspace{-5mm}
\caption{\textbf{Ablation study} on the FullBodyManipulation dataset~\cite{li2023object}. We measure the effect of different guidance terms in the human and object motion generation.}
\label{tab:ablation_study_seen}
\vspace{-4mm}
\end{table*}
\subsection{Results}
%\vspace{-3pt}
\paragraph{Baselines.}
As there is no prior work presenting a solution for our task, we adapt the most related work, OMOMO~\cite{li2023object}, to our problem setting in order to evaluate against a baseline. This method was designed for synthesizing human motion from a provided object motion trajectory. Since OMOMO~\cite{li2023object} requires a sequence of object states to generate full-body human poses, we implement a linear interpolation strategy for the object positions. This interpolation is based on the given start and end positions of the object, as well as predefined waypoints in the xy-plane. We also maintain a consistent object rotation, using the orientation from the initial frame throughout the entire sequence.
Additionally, we evaluate our approach CHOIS against two ablations: CHOIS w/o $L_\text{obj}$ and CHOIS w/o $F_\text{all}$. CHOIS w/o $L_\text{obj}$ is trained as a conditional diffusion model but does not include an additional object geometry loss. This variant allows us to understand the baseline performance of the diffusion model in a straightforward setup. In contrast, CHOIS w/o $F_\text{all}$ incorporates the object geometry loss in its training process but operates without guidance during inference. This approach lets us explore the effectiveness of object geometry loss during training while assessing the model's capability in the absence of guidance.
\begin{figure*}[t!]
\begin{center}
%\vspace{-5mm}
\includegraphics[width=\textwidth]{figures/Figure_seq_in_scene_application.pdf}
\end{center}
\vspace{-6mm}
\caption{\textbf{Long-term interaction synthesis.} Given language descriptions, a 3D scene with semantic labels, and initial human and object states, we synthesize long-term human-object interactions. The initial state is shown in green.}
\label{fig:qualitative_res_w_scene}
\vspace{-4mm}
\end{figure*}
\begin{table}[!t]
\centering
% \vspace{-5mm}
% \vspace{-2mm}
%\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
%\begin{adjustbox}{width=0.9\textwidth} {
\small
\footnotesize{
\setlength{\tabcolsep}{2pt}
\begin{tabular}{lccccccccc}
\toprule
& \multicolumn{3}{c}{Condition Matching} & \multicolumn{2}{c}{Human Motion} & \multicolumn{2}{c}{Interaction} \\
\cmidrule(lr){2-4}\cmidrule(lr){5-6}\cmidrule(lr){7-8}
& $T_{s}\downarrow$ & $T_{e}\downarrow$ & $T_{xy}\downarrow$ & $H_{feet}\downarrow$ & FS$\downarrow$ & $C_{\%}\uparrow$ & $P_{hand}\downarrow$ \\
\midrule
CHOIS w/o $F_\text{all}$ & \textbf{1.50} & \textbf{7.19} & \textbf{5.10} & 6.01 & \textbf{0.43} & 0.49 & 0.70 \\
CHOIS & 2.22 & 9.94 & 5.73 & \textbf{4.57} & 0.46 & \textbf{0.63} & \textbf{0.69} \\
\midrule
CHOIS$^{\ast}$ w/o $F_\text{all}$ & 6.29 & \textbf{9.39} & \textbf{5.26} & 4.77 & \textbf{0.39} & 0.42 & 0.55 \\
CHOIS$^{\ast}$ & \textbf{5.62} & 12.08 & 5.95 & \textbf{4.27} & 0.41 & \textbf{0.65} & \textbf{0.32} \\
\bottomrule
\end{tabular}
}
%}\end{adjustbox}
% \vspace{-0.6cm}
\vspace{-2mm}
\caption{\textbf{Long-term interaction synthesis results} on the FullBodyManipulation~\cite{li2023object} and 3D-FUTURE datasets~\cite{3dfuture}. $^{\ast}$ represents the results on the 3D-FUTURE dataset.
}
\label{tab:long_seq_cmp}
\vspace{-5mm}
\end{table}
\vspace{-4mm}
\paragraph{Results on the FullBodyManipulation Dataset.}
We evaluate our approach using objects from the FullBodyManipulation dataset~\cite{li2023object} as shown in Table~\ref{tab:single_window_cmp_seen}. Introducing object geometry loss notably improves the condition matching metric. Furthermore, adding guidance during inference leads to better contact accuracy, reduced hand-object penetration, and less foot floating. Note that OMOMO~\cite{li2023object} has zero deviation from the input object trajectory since it only predicts human motion and does not change the object motion input. We also showcase qualitative comparisons in Figure~\ref{fig:qualitative_res}.
Note that OMOMO's object motion is updated via linear interpolation and thus cannot follow the text prompt (row 2, lifting the table above the head).
\vspace{-4mm}
\paragraph{Results on the 3D-FUTURE Dataset.}
To test our model's ability to generalize to new objects, we conduct evaluations using the 3D-FUTURE dataset~\cite{3dfuture}. As shown in Table~\ref{tab:single_window_cmp_unseen}, our proposed method outperforms the baseline and the two ablations. We also provide qualitative results in Figure~\ref{fig:qualitative_res}.
% \paragraph{Results of Generation Guided by Waypoints from 3D Scenes.}
% We compare our approach in Table~\ref{tab:long_seq}. Applying guidance during inference effectively improves the contact and reduces the penetration with 3D scenes. We show qualitative results in Figure\jiaman{insert a figure to compare w/wo guidance in 3D scenes}.
% \begin{table*}[!t]
% \centering
% % \vspace{-5mm}
% % \vspace{-2mm}
% %\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
% %\begin{adjustbox}{width=0.9\textwidth} {
% \small
% % \footnotesize{
% \setlength{\tabcolsep}{4pt}
% \begin{tabular}{lcccccccccc}
% \toprule
% & $\mathbf{T}_{s}$ & $\mathbf{T}_{e}$ & $\mathbf{T}_{xy}$ & $\mathbf{H}_{feet}$ & FS & $\mathbf{C}_{\%}$ & $\mathbf{P}^{hand}$ & $\mathbf{P}^{SH}$ & $\mathbf{P}^{SO}$ \\
% \midrule
% Ours B & \textbf{1.50} & \textbf{7.19} & \textbf{5.10} & 6.01 & 0.43 & 0.49 & 0.70 & 0.43 & 0.12 \\
% Ours & 2.22 & 9.94 & 5.73 & \textbf{4.57} & \textbf{0.46} & \textbf{0.63} & \textbf{0.69} & \textbf{0.06} & \textbf{0.02} \\
% \midrule
% Ours B (Unseen Objects) & \textbf{6.29} & \textbf{9.39} & \textbf{5.26} & 4.77 & 0.39 & 0.42 & 0.55 & 0.28 & 0.28 \\
% Ours (Unseen Objects) & 5.62 & 12.08 & 5.95 & \textbf{4.27} & \textbf{0.41} & \textbf{0.65} & \textbf{0.14} & \textbf{0.01} & \textbf{0.023} \\
% \bottomrule
% \end{tabular}
% % }
% %}\end{adjustbox}
% % \vspace{-0.6cm}
% \vspace{-2mm}
% \caption{Long-term interaction synthesis on objects in FullBodyManipulation~\cite{li2023object} and 3DFuture~\cite{3dfuture}.
% }
% \label{tab:long_seq}
% \vspace{-2mm}
% \end{table*}
% \begin{table*}[!t]
% \centering
% % \vspace{-5mm}
% % \vspace{-2mm}
% %\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
% %\begin{adjustbox}{width=0.9\textwidth} {
% \small
% \begin{tabular}{lcccccccccc}
% \toprule
% & $\mathbf{T}_{start}$ & $\mathbf{T}_{end}$ & $\mathbf{T}_{xy}$ & $\mathbf{H}_{feet}$ & FS & $\mathbf{C}_{\%}$ & Collision & Scene-Human & Scene-Object \\
% \midrule
% Unseen Ours & 6.29 & 9.39 & 5.26 & 4.77 & 0.39 & 0.42 & 0.55 & 0.28 & 0.28 \\
% Unseen Ours w g & 5.62 & 12.08 & 5.95 & 4.27 & 0.41 & 0.65 & 0.14 & 0.01 & 0.023 \\
% \bottomrule
% \end{tabular}
% %}\end{adjustbox}
% % \vspace{-0.6cm}
% \vspace{-2mm}
% \caption{Interaction synthesis on long-term sequences for unseen objects.
% % \karen{Change to Hybrid (ours). It looks like our $O_{head}$ is not better than Droid-SLAM on real data. What's our explanation?}
% }
% \label{tab:long_seq_unseen}
% \vspace{-2mm}
% \end{table*}
\vspace{-4mm}
\paragraph{Human Perceptual Study.}
% \paragraph{Evaluation on Single-window Interaction Synthesis.}
We conduct two human perceptual studies to further complement the evaluation of our approach. The first study assesses the consistency between the generated interactions and the text input. The second study evaluates the overall quality of these generated interactions. For each of these studies, we generate 100 sequences using each method, including our CHOIS, the baseline, our ablations, and the ground truth. This results in a set of 400 pairs. We employ Amazon Mechanical Turk (AMT) for evaluation. Each sequence pair is reviewed by 10 different AMT workers. The results are illustrated in Figure~\ref{fig:human_study}. Given that OMOMO~\cite{li2023object} generates human motions based solely on interpolated object states and does not incorporate language conditions, our method demonstrates superior performance in aligning with text input. Moreover, our approach shows improvements over the CHOIS w/o $L_{\text{obj}}$, with both CHOIS w/o $F_\text{all}$ and CHOIS exhibiting comparable proficiency in text alignment. Regarding interaction quality, our method also surpasses all baseline and model ablations.
\subsection{Ablation Study}
We conduct an ablation study to validate the effectiveness of our proposed guidance terms. As shown in Table~\ref{tab:ablation_study_seen}, our hand-object contact guidance and feet-floor contact guidance are both critical. Without the hand-object contact guidance, the contact percentage degrades obviously. Without the feet-floor contact guidance, the height of the feet increases indicating there exists severe foot floating issues. We are not ablating object-floor penetration guidance as object-floor penetration issues are not common and this term is primarily designed for preventing penetration artifacts in qualitative results.
% text alignment
% Ours vs OMOMO preference rate:0.7194388777555111
% OMOMO vs Ours preference rate:0.25851703406813625
% Ours vs OMOMO equal rate:0.022044088176352707
% Ours vs Ours A (old model) preference rate:0.5771543086172345
% Ours A (old model) vs Ours preference rate:0.3807615230460922
% Ours vs Ours A (old model) equal rate:0.04208416833667335
% Ours vs Ours B (wo guidance) preference rate:0.468
% Ours B (wo guidance) vs Ours preference rate:0.384
% Ours vs Ours B (wo guidance) equal rate:0.148
% Ours vs GT preference rate:0.4690909090909091
% GT vs Ours preference rate:0.4690909090909091
% Ours vs GT equal rate:0.06181818181818182
% interaction quality
% Ours vs OMOMO preference rate:0.556
% OMOMO vs Ours preference rate:0.416
% Ours vs OMOMO equal rate:0.028
% Ours vs Ours A (old model) preference rate:0.53
% Ours A (old model) vs Ours preference rate:0.454
% Ours vs Ours A (old model) equal rate:0.016
% Ours vs Ours B (wo guidance) preference rate:0.5571142284569138
% Ours B (wo guidance) vs Ours preference rate:0.39478957915831664
% Ours vs Ours B (wo guidance) equal rate:0.04809619238476954
% Ours vs GT preference rate:0.4875
% GT vs Ours preference rate:0.4875
% Ours vs GT equal rate:0.025
% \begin{table}[!t]
% \centering
% % \vspace{-5mm}
% % \vspace{-2mm}
% %\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
% %\begin{adjustbox}{width=0.9\textwidth} {
% \small
% \begin{tabular}{lcccccccc}
% \toprule
% & OMOMO & Ours A & Ours B & GT \\
% \midrule
% Text Consistency & - & - & - & - \\
% Interaction & - & - & - & - \\
% Human Motion & - & - & - & - \\
% Object Motion & - & - & - & - \\
% \bottomrule
% \end{tabular}
% %}\end{adjustbox}
% % \vspace{-0.6cm}
% \vspace{-2mm}
% \caption{Preference rate in our human perceptual study.}
% \label{tab:human_study}
% \vspace{-4mm}
% \end{table}
% \paragraph{Evaluation on Long-term Interaction Synthesis.}
% \subsection{Ablation Study}
% \paragraph{Effects of Object Representation.}
% We study the effects of object representation in Tbale~\ref{tab:ablation_obj_rep}.
% \begin{table}[!t]
% \centering
% % \vspace{-5mm}
% % \vspace{-2mm}
% %\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
% %\begin{adjustbox}{width=0.9\textwidth} {
% \small
% \begin{tabular}{lccccccccc}
% \toprule
% & $\mathbf{T}_{start}$ & $\mathbf{T}_{end}$ & $\mathbf{T}_{xy}$ & $\mathbf{H}_{feet}$ & FS & $\mathbf{C}_{\%}$ & Collision \\
% \midrule
% Ours wo BPS & - & - & - & - & - \\
% Ours w rest BPS & - & - & - & - & - \\
% Ours w first BPS & - & - & - & - & - \\
% Ours w random BPS & - & - & - & - & - \\
% \bottomrule
% \end{tabular}
% %}\end{adjustbox}
% % \vspace{-0.6cm}
% \vspace{-2mm}
% \caption{Ablation study on object representation.}
% \label{tab:ablation_obj_rep}
% \vspace{-4mm}
% \end{table}
% % \paragraph{Effects of Sparse Keypoints.}
% \paragraph{Effects of Guidance Terms.}
% We study the effects of our proposed guidance terms in Table~\ref{tab:ablation_guidance}.
% \begin{table}[!t]
% \centering
% % \vspace{-5mm}
% % \vspace{-2mm}
% %\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}
% %\begin{adjustbox}{width=0.9\textwidth} {
% \small
% \begin{tabular}{lccccccccc}
% \toprule
% & $\mathbf{T}_{start}$ & $\mathbf{T}_{end}$ & $\mathbf{T}_{xy}$ & $\mathbf{H}_{feet}$ & FS & $\mathbf{C}_{\%}$ & Collision \\
% \midrule
% Ours & - & - & - & - & - \\
% Ours w contact & - & - & - & - & - \\
% Ours w contact + pen & - & - & - & - & - \\
% Ours w contact + pen + feet & - & - & - & - & - \\
% \bottomrule
% \end{tabular}
% %}\end{adjustbox}
% % \vspace{-0.6cm}
% \vspace{-2mm}
% \caption{Ablation study on guidance terms.}
% \label{tab:ablation_guidance}
% \vspace{-4mm}
% \end{table}
\subsection{Application}
\begin{figure}[t!]
\begin{center}
%\vspace{-2mm}
\includegraphics[width=\linewidth]{figures/Figure_diverse_waypoints.pdf}
\end{center}
\vspace{-6mm}
\caption{\textbf{Results of interaction synthesis} using the same text input but different waypoints. The initial state is in green. }
% \vspace{-10mm}
% \karen{Why use plural for "initial states"? Are there more than one? Can we use more visible colors on a yellow background? Or use gray to e the background color.}
\label{fig:same_text_diverse_waypoints}
\vspace{-5mm}
\end{figure}
This section presents a practical application of our method, enabling the synthesis of human-object interactions within 3D scenes, driven by language descriptions. We utilize 3D scenes from the Replica Dataset~\cite{replica}.
The process begins by composing language descriptions that specify the desired interactions, identifying both the objects involved and their intended positions. For example, the language description can be ``pull the floor lamp to be close to a shelf". We also define a set of primitive functions used to sample target 3D positions from 3D scenes. This set includes functions like sampling points on an object's surface or near it. GPT-3~\cite{brown2020language} is used to extract key information including the interaction object and target objects, and to select the appropriate primitive functions from our predefined function set. Combining the information with the semantic labels of the scene point cloud, we can determine the target 3D positions.
We leverage Habitat~\cite{habitat19iccv, szot2021habitat} to generate collision-free paths within the scene given the start and target object positions. However, as Habitat provides waypoints without corresponding time steps, we need to adapt these to our learned module. We apply heuristics to create waypoints at fixed intervals of 30 frames, which serve as the input conditions for our interaction synthesis model. An example of this application is shown in Figure~\ref{fig:qualitative_res_w_scene}, demonstrating how our learned interaction synthesis model
effectively synthesizes human-object motion following a description in a 3D scene.
% handles waypoints extracted from real 3D environments.
Table~\ref{tab:long_seq_cmp} includes a quantitative evaluation of the generated motion.
% We also conduct a quantitative evaluation as shown in Table~\ref{tab:long_seq_cmp}.
In addition, we showcase the results using the same text input but different waypoints in Figure~\ref{fig:same_text_diverse_waypoints}, demonstrating the effectiveness of the control using object waypoints.
\section{Conclusion}
In conclusion, our work addresses the problem of human-object interaction synthesis conditioned on language descriptions and sparse object waypoints. By employing a conditional diffusion model, we successfully generate object and human motions that are not only synchronized but also resonate with given language descriptions. We incorporate object geometry loss during training which significantly improves the performance of object motion generation. We also propose effective guidance terms used during the sampling process which enhance the realism of the generated results. Moreover, we demonstrate that our learned interaction module can be integrated into a pipeline that synthesizes long-term interactions given language and 3D scenes.
\paragraph{Acknowledgments.} This work is in part supported by the Stanford Institute for Human-Centered AI (HAI), NSF CCRI \#2120095, ONR MURI N00014-22-1-2740, and Meta. Part of the research was done during Jiaman Li's internship at FAIR, Meta.
\begin{thebibliography}{70}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Araujo et~al.(2023)Araujo, Li, Vetrivel, Agarwal, Gopinath, Wu, Clegg, and Liu]{araujo2023circle}
Joao~Pedro Araujo, Jiaman Li, Karthik Vetrivel, Rishi Agarwal, Deepak Gopinath, Jiajun Wu, Alexander Clegg, and C~Karen Liu.
\newblock Circle: Capture in rich contextual environments.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Barquero et~al.(2023)Barquero, Escalera, and Palmero]{barquero2023belfusion}
German Barquero, Sergio Escalera, and Cristina Palmero.
\newblock Belfusion: Latent diffusion for behavior-driven human motion prediction.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 2317--2327, 2023.
\bibitem[Bhatnagar et~al.(2022)Bhatnagar, Xie, Petrov, Sminchisescu, Theobalt, and Pons-Moll]{bhatnagar22behave}
Bharat~Lal Bhatnagar, Xianghui Xie, Ilya~A. Petrov, Cristian Sminchisescu, Christian Theobalt, and Gerard Pons-Moll.
\newblock Behave: Dataset and method for tracking human object interactions.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 15935--15946, 2022.
\bibitem[Braun et~al.(2023)Braun, Christen, Kocabas, Aksan, and Hilliges]{braun2023physically}
Jona Braun, Sammy Christen, Muhammed Kocabas, Emre Aksan, and Otmar Hilliges.
\newblock Physically plausible full-body hand-object interaction synthesis.
\newblock \emph{arXiv preprint arXiv:2309.07907}, 2023.
\bibitem[Brown et~al.(2020)Brown, Mann, Ryder, Subbiah, Kaplan, Dhariwal, Neelakantan, Shyam, Sastry, Askell, et~al.]{brown2020language}
Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared~D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et~al.
\newblock Language models are few-shot learners.
\newblock \emph{Advances in neural information processing systems}, 33:\penalty0 1877--1901, 2020.
\bibitem[Chen et~al.(2023)Chen, Jiang, Liu, Huang, Fu, Chen, and Yu]{chen2023executing}
Xin Chen, Biao Jiang, Wen Liu, Zilong Huang, Bin Fu, Tao Chen, and Gang Yu.
\newblock Executing your commands via motion diffusion in latent space.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 18000--18010, 2023.
\bibitem[Christen et~al.(2022)Christen, Kocabas, Aksan, Hwangbo, Song, and Hilliges]{christen2022d}
Sammy Christen, Muhammed Kocabas, Emre Aksan, Jemin Hwangbo, Jie Song, and Otmar Hilliges.
\newblock D-grasp: Physically plausible dynamic grasp synthesis for hand-object interactions.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 20577--20586, 2022.
\bibitem[Dabral et~al.(2023)Dabral, Mughal, Golyanik, and Theobalt]{dabral2022mofusion}
Rishabh Dabral, Muhammad~Hamza Mughal, Vladislav Golyanik, and Christian Theobalt.
\newblock Mofusion: A framework for denoising-diffusion-based motion synthesis.
\newblock In \emph{Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Dhariwal and Nichol(2021)]{dhariwal2021diffusion}
Prafulla Dhariwal and Alexander Nichol.
\newblock Diffusion models beat gans on image synthesis.
\newblock \emph{Advances in neural information processing systems}, 34:\penalty0 8780--8794, 2021.
\bibitem[Fan et~al.(2023)Fan, Taheri, Tzionas, Kocabas, Kaufmann, Black, and Hilliges]{fan2023arctic}
Zicong Fan, Omid Taheri, Dimitrios Tzionas, Muhammed Kocabas, Manuel Kaufmann, Michael~J. Black, and Otmar Hilliges.
\newblock {ARCTIC}: A dataset for dexterous bimanual hand-object manipulation.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Fu et~al.(2021{\natexlab{a}})Fu, Jia, Gao, Gong, Zhao, Maybank, and Tao]{3dfuture}
Huan Fu, Rongfei Jia, Lin Gao, Mingming Gong, Binqiang Zhao, Steve Maybank, and Dacheng Tao.
\newblock 3d-future: 3d furniture shape with texture.
\newblock \emph{International Journal of Computer Vision (IJCV)}, 129\penalty0 (12):\penalty0 3313--3337, 2021{\natexlab{a}}.
\bibitem[Fu et~al.(2021{\natexlab{b}})Fu, Jia, Gao, Gong, Zhao, Maybank, and Tao]{fu20213d}
Huan Fu, Rongfei Jia, Lin Gao, Mingming Gong, Binqiang Zhao, Steve Maybank, and Dacheng Tao.
\newblock 3d-future: 3d furniture shape with texture.
\newblock \emph{International Journal of Computer Vision}, 129:\penalty0 3313--3337, 2021{\natexlab{b}}.
\bibitem[Ghosh et~al.(2023)Ghosh, Dabral, Golyanik, Theobalt, and Slusallek]{ghosh2022imos}
Anindita Ghosh, Rishabh Dabral, Vladislav Golyanik, Christian Theobalt, and Philipp Slusallek.
\newblock Imos: Intent-driven full-body motion synthesis for human-object interactions.
\newblock In \emph{Eurographics}, 2023.
\bibitem[Guo et~al.(2022{\natexlab{a}})Guo, Zou, Zuo, Wang, Ji, Li, and Cheng]{Guo_2022_CVPR}
Chuan Guo, Shihao Zou, Xinxin Zuo, Sen Wang, Wei Ji, Xingyu Li, and Li Cheng.
\newblock Generating diverse and natural 3d human motions from text.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 5152--5161, 2022{\natexlab{a}}.
\bibitem[Guo et~al.(2022{\natexlab{b}})Guo, Zuo, Wang, and Cheng]{guo2022tm2t}
Chuan Guo, Xinxin Zuo, Sen Wang, and Li Cheng.
\newblock Tm2t: Stochastic and tokenized modeling for the reciprocal generation of 3d human motions and texts.
\newblock In \emph{European Conference on Computer Vision}, pages 580--597. Springer, 2022{\natexlab{b}}.
\bibitem[Guzov et~al.(2021)Guzov, Mir, Sattler, and Pons-Moll]{HPS}
Vladimir Guzov, Aymen Mir, Torsten Sattler, and Gerard Pons-Moll.
\newblock Human poseitioning system (hps): 3d human pose estimation and self-localization in large scenes from body-mounted sensors.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, 2021.
\bibitem[Hassan et~al.(2019)Hassan, Choutas, Tzionas, and Black]{prox}
Mohamed Hassan, Vasileios Choutas, Dimitrios Tzionas, and Michael~J Black.
\newblock Resolving 3d human pose ambiguities with 3d scene constraints.
\newblock In \emph{International Conference on Computer Vision (ICCV)}, pages 2282--2292, 2019.
\bibitem[Hassan et~al.(2021)Hassan, Ceylan, Villegas, Saito, Yang, Zhou, and Black]{hassan_samp_2021}
Mohamed Hassan, Duygu Ceylan, Ruben Villegas, Jun Saito, Jimei Yang, Yi Zhou, and Michael Black.
\newblock Stochastic scene-aware motion prediction.
\newblock In \emph{International Conference on Computer Vision (ICCV)}, pages 11354--11364, 2021.
\bibitem[Hassan et~al.(2023)Hassan, Guo, Wang, Black, Fidler, and Peng]{hassan2023synthesizing}
Mohamed Hassan, Yunrong Guo, Tingwu Wang, Michael Black, Sanja Fidler, and Xue~Bin Peng.
\newblock Synthesizing physical character-scene interactions.
\newblock In \emph{SIGGRAPH 2023 Conference Papers}, 2023.
\bibitem[He et~al.(2022)He, Saito, Zachary, Rushmeier, and Zhou]{he2022nemf}
Chengan He, Jun Saito, James Zachary, Holly Rushmeier, and Yi Zhou.
\newblock Nemf: Neural motion fields for kinematic animation.
\newblock \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2022.
\bibitem[Ho et~al.(2020)Ho, Jain, and Abbeel]{ho2020denoising}
Jonathan Ho, Ajay Jain, and Pieter Abbeel.
\newblock Denoising diffusion probabilistic models.
\newblock \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 33:\penalty0 6840--6851, 2020.
\bibitem[Ho et~al.(2022)Ho, Salimans, Gritsenko, Chan, Norouzi, and Fleet]{ho2022video}
Jonathan Ho, Tim Salimans, Alexey Gritsenko, William Chan, Mohammad Norouzi, and David~J Fleet.
\newblock Video diffusion models.
\newblock \emph{arXiv:2204.03458}, 2022.
\bibitem[Huang et~al.(2023)Huang, Wang, Li, Jia, Liu, Zhu, Liang, and Zhu]{huang2023diffusion}
Siyuan Huang, Zan Wang, Puhao Li, Baoxiong Jia, Tengyu Liu, Yixin Zhu, Wei Liang, and Song-Chun Zhu.
\newblock Diffusion-based generation, optimization, and planning in 3d scenes.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Karunratanakul et~al.(2023)Karunratanakul, Preechakul, Suwajanakorn, and Tang]{karunratanakul2023gmd}
Korrawe Karunratanakul, Konpat Preechakul, Supasorn Suwajanakorn, and Siyu Tang.
\newblock Gmd: Controllable human motion synthesis via guided diffusion models.
\newblock \emph{arXiv preprint arXiv:2305.12577}, 2023.
\bibitem[Kulkarni et~al.(2023)Kulkarni, Rempe, Genova, Kundu, Johnson, Fouhey, and Guibas]{kulkarni2023nifty}
Nilesh Kulkarni, Davis Rempe, Kyle Genova, Abhijit Kundu, Justin Johnson, David Fouhey, and Leonidas Guibas.
\newblock Nifty: Neural object interaction fields for guided human motion synthesis.
\newblock \emph{arXiv preprint arXiv:2307.07511}, 2023.
\bibitem[Lee and Joo(2023)]{lee2023locomotion}
Jiye Lee and Hanbyul Joo.
\newblock Locomotion-action-manipulation: Synthesizing human-scene interactions in complex 3d environments.
\newblock \emph{arXiv preprint arXiv:2301.02667}, 2023.
\bibitem[Li et~al.(2023{\natexlab{a}})Li, Liu, and Wu]{li2023ego}
Jiaman Li, Karen Liu, and Jiajun Wu.
\newblock Ego-body pose estimation via ego-head pose estimation.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 17142--17151, 2023{\natexlab{a}}.
\bibitem[Li et~al.(2023{\natexlab{b}})Li, Wu, and Liu]{li2023object}
Jiaman Li, Jiajun Wu, and C~Karen Liu.
\newblock Object motion guided human motion synthesis.
\newblock \emph{ACM Trans. Graph.}, 42\penalty0 (6), 2023{\natexlab{b}}.
\bibitem[Li et~al.(2023{\natexlab{c}})Li, Wang, Loy, and Dai]{li2023task}
Quanzhou Li, Jingbo Wang, Chen~Change Loy, and Bo Dai.
\newblock Task-oriented human-object interactions generation with implicit neural representations.
\newblock \emph{arXiv preprint arXiv:2303.13129}, 2023{\natexlab{c}}.
\bibitem[Mahmood et~al.(2019)Mahmood, Ghorbani, Troje, Pons-Moll, and Black]{AMASS}
Naureen Mahmood, Nima Ghorbani, Nikolaus~F Troje, Gerard Pons-Moll, and Michael~J Black.
\newblock Amass: Archive of motion capture as surface shapes.
\newblock In \emph{International Conference on Computer Vision (ICCV)}, pages 5442--5451, 2019.
\bibitem[{Manolis Savva*} et~al.(2019){Manolis Savva*}, {Abhishek Kadian*}, {Oleksandr Maksymets*}, Zhao, Wijmans, Jain, Straub, Liu, Koltun, Malik, Parikh, and Batra]{habitat19iccv}
{Manolis Savva*}, {Abhishek Kadian*}, {Oleksandr Maksymets*}, Yili Zhao, Erik Wijmans, Bhavana Jain, Julian Straub, Jia Liu, Vladlen Koltun, Jitendra Malik, Devi Parikh, and Dhruv Batra.
\newblock Habitat: {A} {P}latform for {E}mbodied {AI} {R}esearch.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, 2019.
\bibitem[Merel et~al.(2020)Merel, Tunyasuvunakool, Ahuja, Tassa, Hasenclever, Pham, Erez, Wayne, and Heess]{merel2020catch}
Josh Merel, Saran Tunyasuvunakool, Arun Ahuja, Yuval Tassa, Leonard Hasenclever, Vu Pham, Tom Erez, Greg Wayne, and Nicolas Heess.
\newblock Catch \& carry: reusable neural controllers for vision-guided whole-body tasks.
\newblock \emph{ACM Transactions on Graphics (TOG)}, 39\penalty0 (4):\penalty0 39--1, 2020.
\bibitem[Mir et~al.(2023)Mir, Puig, Kanazawa, and Pons-Moll]{mir2023generating}
Aymen Mir, Xavier Puig, Angjoo Kanazawa, and Gerard Pons-Moll.
\newblock Generating continual human motion in diverse 3d scenes.
\newblock \emph{arXiv preprint arXiv:2304.02061}, 2023.
\bibitem[Pavlakos et~al.(2019)Pavlakos, Choutas, Ghorbani, Bolkart, Osman, Tzionas, and Black]{smplx}
Georgios Pavlakos, Vasileios Choutas, Nima Ghorbani, Timo Bolkart, Ahmed A.~A. Osman, Dimitrios Tzionas, and Michael~J. Black.
\newblock Expressive body capture: 3d hands, face, and body from a single image.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 10975--10985, 2019.
\bibitem[Petrov et~al.(2023)Petrov, Marin, Chibane, and Pons-Moll]{petrov2023object}
Ilya~A Petrov, Riccardo Marin, Julian Chibane, and Gerard Pons-Moll.
\newblock Object pop-up: Can we infer 3d objects and their poses from human interactions alone?
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 4726--4736, 2023.
\bibitem[Petrovich et~al.(2021)Petrovich, Black, and Varol]{petrovich2021action}
Mathis Petrovich, Michael~J Black, and G{\"u}l Varol.
\newblock Action-conditioned 3d human motion synthesis with transformer vae.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 10985--10995, 2021.
\bibitem[Petrovich et~al.(2022)Petrovich, Black, and Varol]{petrovich2022temos}
Mathis Petrovich, Michael~J Black, and G{\"u}l Varol.
\newblock Temos: Generating diverse human motions from textual descriptions.
\newblock In \emph{European Conference on Computer Vision}, pages 480--497. Springer, 2022.
\bibitem[Prokudin et~al.(2019)Prokudin, Lassner, and Romero]{prokudin2019efficient}
Sergey Prokudin, Christoph Lassner, and Javier Romero.
\newblock Efficient learning on point clouds with basis point sets.
\newblock In \emph{International Conference on Computer Vision (ICCV)}, pages 4332--4341, 2019.
\bibitem[Punnakkal et~al.(2021)Punnakkal, Chandrasekaran, Athanasiou, Quiros-Ramirez, and Black]{punnakkal2021babel}
Abhinanda~R Punnakkal, Arjun Chandrasekaran, Nikos Athanasiou, Alejandra Quiros-Ramirez, and Michael~J Black.
\newblock Babel: Bodies, action and behavior with english labels.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 722--731, 2021.
\bibitem[Raab et~al.(2023)Raab, Leibovitch, Tevet, Arar, Bermano, and Cohen-Or]{raab2023single}
Sigal Raab, Inbal Leibovitch, Guy Tevet, Moab Arar, Amit~H Bermano, and Daniel Cohen-Or.
\newblock Single motion diffusion.
\newblock \emph{arXiv preprint arXiv:2302.05905}, 2023.
\bibitem[Radford et~al.(2021)Radford, Kim, Hallacy, Ramesh, Goh, Agarwal, Sastry, Askell, Mishkin, Clark, et~al.]{radford2021learning}
Alec Radford, Jong~Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, et~al.
\newblock Learning transferable visual models from natural language supervision.
\newblock In \emph{International conference on machine learning}, pages 8748--8763. PMLR, 2021.
\bibitem[Rempe et~al.(2023)Rempe, Luo, Bin~Peng, Yuan, Kitani, Kreis, Fidler, and Litany]{rempe2023trace}
Davis Rempe, Zhengyi Luo, Xue Bin~Peng, Ye Yuan, Kris Kitani, Karsten Kreis, Sanja Fidler, and Or Litany.
\newblock Trace and pace: Controllable pedestrian animation via guided trajectory diffusion.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 13756--13766, 2023.
\bibitem[Shafir et~al.(2023)Shafir, Tevet, Kapon, and Bermano]{shafir2023human}
Yonatan Shafir, Guy Tevet, Roy Kapon, and Amit~H Bermano.
\newblock Human motion diffusion as a generative prior.
\newblock \emph{arXiv preprint arXiv:2303.01418}, 2023.
\bibitem[Shi et~al.(2023)Shi, Wang, Jiang, and Dai]{shi2023controllable}
Yi Shi, Jingbo Wang, Xuekun Jiang, and Bo Dai.
\newblock Controllable motion diffusion model.
\newblock \emph{arXiv preprint arXiv:2306.00416}, 2023.
\bibitem[Straub et~al.(2019)Straub, Whelan, Ma, Chen, Wijmans, Green, Engel, Mur-Artal, Ren, Verma, et~al.]{replica}
Julian Straub, Thomas Whelan, Lingni Ma, Yufan Chen, Erik Wijmans, Simon Green, Jakob~J Engel, Raul Mur-Artal, Carl Ren, Shobhit Verma, et~al.
\newblock The replica dataset: A digital replica of indoor spaces.
\newblock \emph{arXiv preprint arXiv:1906.05797}, 2019.
\bibitem[Szot et~al.(2021)Szot, Clegg, Undersander, Wijmans, Zhao, Turner, Maestre, Mukadam, Chaplot, Maksymets, Gokaslan, Vondrus, Dharur, Meier, Galuba, Chang, Kira, Koltun, Malik, Savva, and Batra]{szot2021habitat}
Andrew Szot, Alex Clegg, Eric Undersander, Erik Wijmans, Yili Zhao, John Turner, Noah Maestre, Mustafa Mukadam, Devendra Chaplot, Oleksandr Maksymets, Aaron Gokaslan, Vladimir Vondrus, Sameer Dharur, Franziska Meier, Wojciech Galuba, Angel Chang, Zsolt Kira, Vladlen Koltun, Jitendra Malik, Manolis Savva, and Dhruv Batra.
\newblock Habitat 2.0: Training home assistants to rearrange their habitat.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2021.
\bibitem[Taheri et~al.(2020)Taheri, Ghorbani, Black, and Tzionas]{GRAB:2020}
Omid Taheri, Nima Ghorbani, Michael~J. Black, and Dimitrios Tzionas.
\newblock {GRAB}: A dataset of whole-body human grasping of objects.
\newblock In \emph{European Conference on Computer Vision (ECCV)}, 2020.
\bibitem[Taheri et~al.(2022)Taheri, Choutas, Black, and Tzionas]{taheri2022goal}
Omid Taheri, Vasileios Choutas, Michael~J Black, and Dimitrios Tzionas.
\newblock Goal: Generating 4d whole-body motion for hand-object grasping.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 13263--13273, 2022.
\bibitem[Tevet et~al.(2022)Tevet, Gordon, Hertz, Bermano, and Cohen-Or]{tevet2022motionclip}
Guy Tevet, Brian Gordon, Amir Hertz, Amit~H Bermano, and Daniel Cohen-Or.
\newblock Motionclip: Exposing human motion generation to clip space.
\newblock In \emph{European Conference on Computer Vision}, pages 358--374. Springer, 2022.
\bibitem[Tevet et~al.(2023)Tevet, Raab, Gordon, Shafir, Bermano, and Cohen-Or]{tevet2022human}
Guy Tevet, Sigal Raab, Brian Gordon, Yonatan Shafir, Amit~H Bermano, and Daniel Cohen-Or.
\newblock Human motion diffusion model.
\newblock In \emph{International Conference on Learning Representations (ICLR)}, 2023.
\bibitem[Tseng et~al.(2023)Tseng, Castellon, and Liu]{tseng2022edge}
Jonathan Tseng, Rodrigo Castellon, and C~Karen Liu.
\newblock Edge: Editable dance generation from music.
\newblock In \emph{Computer Vision and Pattern Recognition (CVPR)}, 2023.
\bibitem[Vaswani et~al.(2017)Vaswani, Shazeer, Parmar, Uszkoreit, Jones, Gomez, Kaiser, and Polosukhin]{transformer}
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan~N Gomez, {\L}ukasz Kaiser, and Illia Polosukhin.
\newblock Attention is all you need.
\newblock In \emph{Advances in Neural Information Processing Systems (NIPS)}, pages 5998--6008, 2017.
\bibitem[Wan et~al.(2022)Wan, Yang, Liu, Zhang, Jia, Choi, Pan, Theobalt, Komura, and Wang]{wan2022learn}
Weilin Wan, Lei Yang, Lingjie Liu, Zhuoying Zhang, Ruixing Jia, Yi-King Choi, Jia Pan, Christian Theobalt, Taku Komura, and Wenping Wang.
\newblock Learn to predict how humans manipulate large-sized objects from interactive motions.
\newblock \emph{IEEE Robotics and Automation Letters}, 7\penalty0 (2):\penalty0 4702--4709, 2022.
\bibitem[Wang et~al.(2021)Wang, Xu, Xu, Liu, and Wang]{wang2021synthesizing}
Jiashun Wang, Huazhe Xu, Jingwei Xu, Sifei Liu, and Xiaolong Wang.
\newblock Synthesizing long-term 3d human motion and interaction in 3d scenes.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 9401--9411, 2021.
\bibitem[Wang et~al.(2022{\natexlab{a}})Wang, Rong, Liu, Yan, Lin, and Dai]{wang2022towards}
Jingbo Wang, Yu Rong, Jingyuan Liu, Sijie Yan, Dahua Lin, and Bo Dai.
\newblock Towards diverse and natural scene-aware 3d human motion synthesis.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 20460--20469, 2022{\natexlab{a}}.
\bibitem[Wang et~al.(2022{\natexlab{b}})Wang, Chen, Liu, Zhu, Liang, and Huang]{wang2022humanise}
Zan Wang, Yixin Chen, Tengyu Liu, Yixin Zhu, Wei Liang, and Siyuan Huang.
\newblock Humanise: Language-conditioned human motion generation in 3d scenes.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2022{\natexlab{b}}.
\bibitem[Wu et~al.(2022)Wu, Wang, Zhang, Zhang, Hilliges, Yu, and Tang]{wu2022saga}
Yan Wu, Jiahao Wang, Yan Zhang, Siwei Zhang, Otmar Hilliges, Fisher Yu, and Siyu Tang.
\newblock Saga: Stochastic whole-body grasping with contact.
\newblock In \emph{European Conference on Computer Vision (ECCV)}, pages 257--274, 2022.
\bibitem[Xiao et~al.(2023)Xiao, Wang, Wang, Cao, Zhang, Dai, Lin, and Pang]{xiao2023unified}
Zeqi Xiao, Tai Wang, Jingbo Wang, Jinkun Cao, Wenwei Zhang, Bo Dai, Dahua Lin, and Jiangmiao Pang.
\newblock Unified human-scene interaction via prompted chain-of-contacts.
\newblock \emph{arXiv preprint arXiv:2309.07918}, 2023.
\bibitem[Xie et~al.(2023)Xie, Tseng, Starke, van~de Panne, and Liu]{xie2023hierarchical}
Zhaoming Xie, Jonathan Tseng, Sebastian Starke, Michiel van~de Panne, and C~Karen Liu.
\newblock Hierarchical planning and control for box loco-manipulation.
\newblock \emph{Symposium on Computer Animation (SCA)}, 2023.
\bibitem[Xu et~al.(2023)Xu, Li, Wang, and Gui]{xu2023interdiff}
Sirui Xu, Zhengyuan Li, Yu-Xiong Wang, and Liang-Yan Gui.
\newblock Interdiff: Generating 3d human-object interactions with physics-informed diffusion.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 14928--14940, 2023.
\bibitem[Yuan et~al.(2023)Yuan, Song, Iqbal, Vahdat, and Kautz]{yuan2023physdiff}
Ye Yuan, Jiaming Song, Umar Iqbal, Arash Vahdat, and Jan Kautz.
\newblock Physdiff: Physics-guided human motion diffusion model.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision}, pages 16010--16021, 2023.
\bibitem[Zhang et~al.(2021)Zhang, Ye, Shiratori, and Komura]{zhang2021manipnet}
He Zhang, Yuting Ye, Takaaki Shiratori, and Taku Komura.
\newblock Manipnet: neural manipulation synthesis with a hand-object spatial representation.
\newblock \emph{ACM Transactions on Graphics (ToG)}, 40\penalty0 (4):\penalty0 1--14, 2021.
\bibitem[Zhang et~al.(2022{\natexlab{a}})Zhang, Cai, Pan, Hong, Guo, Yang, and Liu]{zhang2022motiondiffuse}
Mingyuan Zhang, Zhongang Cai, Liang Pan, Fangzhou Hong, Xinying Guo, Lei Yang, and Ziwei Liu.
\newblock Motiondiffuse: Text-driven human motion generation with diffusion model.
\newblock \emph{arXiv preprint arXiv:2208.15001}, 2022{\natexlab{a}}.
\bibitem[Zhang et~al.(2022{\natexlab{b}})Zhang, Bhatnagar, Starke, Guzov, and Pons-Moll]{zhang2022couch}
Xiaohan Zhang, Bharat~Lal Bhatnagar, Sebastian Starke, Vladimir Guzov, and Gerard Pons-Moll.
\newblock Couch: Towards controllable human-chair interactions.
\newblock In \emph{European Conference on Computer Vision (ECCV)}, pages 518--535, 2022{\natexlab{b}}.
\bibitem[Zhang and Tang(2022)]{zhang2022wanderings}
Yan Zhang and Siyu Tang.
\newblock The wanderings of odysseus in 3d scenes.
\newblock In \emph{Conference on Computer Vision and Pattern Recognition (CVPR)}, pages 20481--20491, 2022.
\bibitem[Zhang et~al.(2023)Zhang, Liu, Aberman, and Hanocka]{zhang2023tedi}
Zihan Zhang, Richard Liu, Kfir Aberman, and Rana Hanocka.
\newblock Tedi: Temporally-entangled diffusion for long-term motion synthesis.
\newblock \emph{arXiv preprint arXiv:2307.15042}, 2023.
\bibitem[Zhao et~al.(2023)Zhao, Zhang, Wang, Beeler, and Tang]{zhao2023synthesizing}
Kaifeng Zhao, Yan Zhang, Shaofei Wang, Thabo Beeler, and Siyu Tang.
\newblock Synthesizing diverse human motions in 3d indoor scenes.
\newblock \emph{arXiv preprint arXiv:2305.12411}, 2023.
\bibitem[Zheng et~al.(2023)Zheng, Zheng, Fang, Liu, and Yi]{zheng2023cams}
Juntian Zheng, Qingyuan Zheng, Lixing Fang, Yun Liu, and Li Yi.
\newblock Cams: Canonicalized manipulation spaces for category-level functional hand-object manipulation synthesis.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, pages 585--594, 2023.
\bibitem[Zheng et~al.(2022)Zheng, Yang, Mo, Li, Yu, Liu, Liu, and Guibas]{zheng2022gimo}
Yang Zheng, Yanchao Yang, Kaichun Mo, Jiaman Li, Tao Yu, Yebin Liu, Karen Liu, and Leonidas Guibas.
\newblock Gimo: Gaze-informed human motion prediction in context.
\newblock In \emph{European Conference on Computer Vision (ECCV)}, 2022.
\bibitem[Zhou et~al.(2019)Zhou, Barnes, Lu, Yang, and Li]{zhou2019continuity}
Yi Zhou, Connelly Barnes, Jingwan Lu, Jimei Yang, and Hao Li.
\newblock On the continuity of rotation representations in neural networks.
\newblock In \emph{Computer Vision and Pattern Recognition (CVPR)}, 2019.
\end{thebibliography}