Modifying RL Policies with Imagined Actions: How Predictable Policies Can Enable Users to Perform Novel Tasks

Extracted source text · arXiv 2312.05991

%File: anonymous-submission-latex-2023.tex
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai23}  % DO NOT CHANGE THIS
\usepackage{times}  % DO NOT CHANGE THIS
\usepackage{helvet}  % DO NOT CHANGE THIS
\usepackage{courier}  % DO NOT CHANGE THIS
\usepackage[hyphens]{url}  % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm}  % DO NOT CHANGE THIS
\usepackage{natbib}  % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\frenchspacing  % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in} % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in} % DO NOT CHANGE THIS
%
% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
\usepackage{subfig}



\usepackage{algorithm}
\usepackage{algorithmic}

\usepackage{amsfonts}
%
% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
\usepackage{newfloat}
\usepackage{listings}
\DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
\lstset{%
	basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
	numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
	aboveskip=0pt,belowskip=0pt,%
	showstringspaces=false,tabsize=2,breaklines=true}
\floatstyle{ruled}
\newfloat{listing}{tb}{lst}{}
\floatname{listing}{Listing}
%
% Keep the \pdfinfo as shown here. There's no need
% for you to add the /Title and /Author tags.
\pdfinfo{
/TemplateVersion (2023.1)
}

% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference

\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.

% The file aaai23.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%

% Title

% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Modifying RL Policies with Imagined Actions: How Predictable Policies Can Enable Users to Perform Novel Tasks}
\author{
    %Authors
    % All authors must be in the same font size and format.
    Isaac Sheidlower, Reuben Aronson, Elaine Short
}
\affiliations{
    %Afiliations
    Tufts University, Department of Computer Science \\ \{isaac.sheidlower, reuben.aronson, elaine.short\}@tufts.edu
%
% See more examples next
}

%Example, Single Author, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it
\iffalse
\title{My Publication Title --- Single Author}
\author {
    Author Name
}
\affiliations{
    Affiliation\\
    Affiliation Line 2\\
    name@example.com
}
\fi

\iffalse
%Example, Multiple Authors, ->> remove \iffalse,\fi and place them surrounding AAAI title to use it

\fi




% REMOVE THIS: bibentry
% This is only needed to show inline citations in the guidelines document. You should not need it and can safely delete it.
\usepackage{bibentry}
% END REMOVE bibentry
\usepackage{color}

\newcommand{\para}{OUC}
 \newcommand{\parafull}{Override User Control}
 \newcommand{\algo}{IODA}
 \newcommand{\algofull}{Imaginary Out-of-Distribution Actions}

 \newcommand{\elaine}[1]{{\textcolor[rgb]{0.1,0.4,0.6}{[ESS: #1]}}}
\newcommand{\isaac}[1]{{\textcolor[rgb]{0.3,0.7,0.3}{[ISS: #1]}}}
\newcommand{\reuben}[1]{{\textcolor[rgb]{0.8,0.2,0.5}{[RMA: #1]}}}
 \newcommand{\delete}[1]{{\textcolor[rgb]{0.75,0.1,0}{\sout{#1}}}}

\begin{document}

\maketitle

\begin{abstract}
It is crucial that users are empowered to use the functionalities of a robot to creatively solve problems on the fly. A user who has access to a Reinforcement Learning (RL) based robot may want to use the robot's autonomy and their knowledge of its behavior to complete new tasks. One way is for the user to take control of some of the robot's action space through teleoperation while the RL policy simultaneously controls the rest. However, an out-of-the-box RL policy may not readily facilitate this. For example, a user's control may bring the robot into a failure state from the policy's perspective, causing it to act in a way the user is not familiar with, hindering the success of the user's desired task. In this work, we formalize this problem and present \algofull{}, \algo{}, an initial algorithm for addressing that problem and empowering user's to leverage their expectation of a robot's behavior to accomplish new tasks.
\end{abstract}

\section{Introduction}
As robots are increasingly in the hands of users, it is necessary to ensure that these users are empowered to use the robot and its functionality to accomplish the tasks they want. Ideally, one can deploy a robot with an exhaustive library of Reinforcement Learning (RL) policies that can autonomously perform any task the user wishes. However, this approach is impractical as users may wish to seamlessly use the robot for a task or in an environment that was not foreseen by the designers of the robot. In reality, the user will be provided with a limited set of policies and a means of controlling the robot, such as teleoperation. Given this, there is a need for methods that enable users to take advantage of these two types of control to perform new and emergent tasks.  

Consider the following illustrative example. Sally frequently uses her assistive robot arm to perform tasks around the house. Sally has used the robot arm's pick-and-place RL policy to move cups full of liquid many times. She knows how the robot will pick up the cup and steadily move it to a specified location. Now Sally wants to use the same functionality to water her flowers. To do this, she starts the autonomous policy and during execution rotates the robot's wrist to pour the water over her basin and water her flowers as the robot moves along it's path. Although in this example the interaction led to the desired result, an RL policy ``as-is'' may not facilitate this. For instance, if the RL policy was trained on a reward function such that if it spills liquid, it should stop moving along its path until the cup is upright. In this case, Sally may accidentally pour all her water onto one flower, since the robot did not continue along its path as she had expected. Similar problems exist in the autonomous driving domain, such as driving intentionally on the sidewalk or hitting an obstacle. 

These examples demonstrate potential problems of naively using an RL policy while the user controls part of the robots behavior. Since the user may have only seen successful examples of task execution, they may not know that their control signal can cause failure with respect to the robot's reward function, causing it to behave unexpectedly. Furthermore, that unexpected behavior may impede the user's ability to perform novel tasks or result in task failure from the user's perspective. This problem setting also allows the user to bring the policy ``out-of-distribution,'' which may result in undesirable behavior. There is a need to both better understand this type of interaction from a user's point of view and develop algorithms that act on RL policies to better facilitate the user completing their task. Our key insight is that whilst the user has partial control, the robot should act in a way similar to the behavior the user has seen before and is familiar with, allowing the user to leverage their prior knowledge about the robot's behavior to adapt it to new tasks.

In this paper, we formalize the aforementioned problem setting and present \algofull{} (\algo{}), an algorithm that modifies the state passed into an RL policy by projecting its current state to a previous state the user has seen: despite the robot's current real state, it chooses actions based on an \emph{imagined} state. \algo{} only modifies the state when the user brings the state out-of-distribution (OOD) with respect to states the user is familiar with/has seen before. IODA uses an OOD detector to determine when to modify the state, and then a projection function to determine the nearest state the user is familiar with to project on to. We study \algo{} in simulation and then discuss our future work studying how \algo{} can facilitate real users to accomplish new tasks. 

\section{Related Work}
Reinforcement learning (RL) is a common and effective way for robots to learn new tasks \cite{kaelbling_reinforcement_1996, kober_reinforcement_2014, ibarz_how_2021}. Much of the success of RL for robotics has come from Deep RL \cite{arulkumaran_brief_2017, nguyen_review_2019} and human-in-the-loop learning \cite{christiano_deep_2017, arzate_cruz_survey_2020, schrum_mind_2022}. However, these methods either assume that the task the user wishes to accomplish is known in advance or that a user is willing to spend time teaching a robot. These assumptions may limit the scope or applicability of any RL-based robot to one or a few tasks. While methods such as multitask reinforcement learning \cite{xie_lifelong_2022, shridhar_perceiver-actor_2023, walke_dont_2023}, large-language model skill grounding \cite{ichter_as_2023, driess_palm-e_2023, liang_code_2023}, and behavioral-diversity learning \cite{eysenbach_diversity_2018, osa_discovering_2022} have all partially addressed this problem by making robot's task repertoire larger and more diverse, they do not explicitly empower users with a method of controlling or collaborating with the robot other than through instruction.

In contrast to RL, teleoperation gives users complete control over a robot's behavior. However, the complete teleoperation of a complex robotic system can be challenging for users as there can be many degrees of freedom to control at once with limited feedback from the robot \cite{darvish_teleoperation_2023, moniruzzaman_teleoperation_2022}. A common approach to alleviate some of the burdens of teleoperation while maintaining a user's control-based empowerment is Shared Control (SC) \cite{li_continuous_2015, matsumoto_shared_2022, bustamante_toward_2021, dragan_policy-blending_2013}. SC often manifests itself as a blending of the user's and robot's control. We examine a case where the user has complete/teleoperation control over some parts of the robot while the others are fully autonomous. This is not unlike Shared Autonomy (SA) \cite{javdani_shared_2015, gopinath_human---loop_2017, selvaggio_autonomy_2021, fontaine_differentiable_2021}, where a user's control signal is interpreted as an indication of their desired goal. In SA, a typical assumption is that a user's goal is known in advance and can be represented in clear contrast to other goals in the environment. Although this assumption has been relaxed in some previous work by updating the potential set of goals \cite{zurek_situational_2021, fontaine_differentiable_2021, yoneda_noise_2023}, these approaches require a user to first demonstrate the task or a similar task without assistance. Instead, we focus on a user leveraging their creative problem-solving skills to partially control an otherwise fully autonomous robot to complete a novel task. In other words, the robot is not assisting or augmenting the user's control signal; rather, the user wants to make use of the robot's behavior as they understand it. Because of this, a robot's behavior should remain predictable under the partial control of the user. 

Legible robot motion can be defined as robot behavior that is easy for a human to predict and understand. A common way to generate legible motion in a goal-based robotic tasks is to model the user as having an internal cost function that is minimized when the robot's behavior saliently moves towards a given goal \cite{dragan_generating_2013, dragan_legibility_2013, faria_understanding_2021}. Another approach is to learn from humans through demonstrations or through feedback \cite{busch_learning_2017, bied_integrating_2020}. An important part of each of these works is that legibility and predictable robot behavior are in the context of the robot \emph{completing the task} and are often in real-time as opposed to pre-hoc or post-hoc explanations \cite{das_explainable_2021, cruz_explainable_2021, sakai_explainable_2022, paleja_utility_2021}. Predictability is also an important part of our work. We make and leverage the assumption that a robot's behavior is predictable if the user has seen similar behavior before. This assumption is not entirely dissimilar from robot-centric notions of out-of-distribution (OOD) states and behavior. 

OOD detection is an important problem in many robotic and machine learning tasks \cite{yang_generalized_2022, sun_out--distribution_2022}. Detecting when a scenario is OOD with respect to a robot's training data or what it has previously encountered may have implications about the robot's environment or performance, such as when an RL agent may behave sporadically or in an unexpected way when in a state it has never been in before \cite{lan_can_2023-1}. It has also been used to detect when a robot may require feedback from a person to help complete a novel task \cite{dass_pato_2023}. OOD in RL has recently been studied to detect when an agent is acting in a new MDP \cite{haider_out--distribution_2023}. We apply similar techniques to detect when the robot is in part of the environment that they would not otherwise act in. This can happen when a user is partially controlling a robot to perform a new task.


\section{Problem Setting}
Here we describe a problem setting in which a user is familiar with the autonomous execution of a task and wishes to partially control a robot during that execution to accomplish another task. In short, the user creates a plan to partially control the robot based on how they expect the robot to behave to accomplish a novel task. Thus, it may be critical that the robot behaves in a way that is predictable to the user, no matter where the robot may be in the state space. Given that the robot behaves in a user-predictable way, as opposed to sporadically or in an unfamiliar way, a user can perform various new tasks with relative ease and few surprises. 

In this setting, the user has seen the robot complete its task many times. We refer to this as a history of task ``rollouts.`` Based on this, we assume the following: when the robot is in a state the user has never seen before, the user expects that the robot will do the same thing as it would do in the ``closest`` state to its current state. Where ``closest`` is both problem and user specific, however, the intuition is that the robot will behave in a similar way in similar circumstances and that in novel circumstances, a user will project on to what they have seen before. This assumption temporarily constrains the problem space; however, it is a reasonable assumption in many robotic tasks. Thus, the problem can be defined as, for any given state unseen to the user, the robot should find a state that the user has seen before and act as if it were in that state.   

We will now define the original task the robot can complete autonomously and how this task is used to build up a user's expectation of the robot's behavior. Let task $orig$ be defined as an MDP with states $S\subset \mathbb{R}^n$, actions $A$, reward function $r: S \times A \rightarrow \mathbb{R}$, and transition function $T: S \times A \rightarrow S$, and discount factor $\gamma$. And there is a robot that has learned an optimal policy for the task denoted $\pi^*$. Let $D$ be defined as a history of rollouts under $\pi_{orig}^*$ that the user has seen. Then, let the user expectation of the robot's behavior given $D$ be:
\vspace{-1mm}
$$W_{D}:S \rightarrow S$$

Here, $W$ is a function that maps from the robot's current state to their anticipation of what the next state will be.

To this setting, we introduce partitioned control, where the user teleoperates one or more parts of the robot's actions. We separate the action space into two separate sets $A_U, A_R \subset A$; $A_U$ denotes the actions that the user can take and $A_R$ denotes the actions that the robot can take. We further assume that the user and robot action spaces are \emph{disjoint}; that is, $A_U \cap A_R = \{\mathbf{0}\}$. In other words, the user and the robot control different parts/axes of the action space. For example, if the robot is acting in Cartesian space, the user may take control over x-axis actions, or take control over the rotation of a specific joint. We denote the user's expectation of how the robot will act with their partial control signal as:
\vspace{-.9mm}
$$W_{D, U}:S \rightarrow S$$

where $U$ is the user control. For brevity, hereafter we refer to this only as $W.$ 

In order to make the robot behavior more predictable for the user, we want to adjust the behavior of the robot policy when it is outside the user's observation set $D$. The goal is to identify when the robot is in a novel state $s$ where there exists a state $s' \in D$ that leads to more predictable behavior. Formally, identify when $\exists s' \in D$ s.t.:
\vspace{-.9mm}
 \begin{equation}
     d(W(s), T(s, u \circ \pi^*(s))) \geq d(W(s), T(s, u \circ \pi^*(s')))
 \end{equation}

 where $d$ is a task-dependent distance metric between states, and $u\circ\pi^*(s)$ denotes the disjoint combination of the autonomous action of the robot and the user's teleoperation. When such a $s'$ is identified, the robot should act as if it were in $s'$. Specifically, we want to select a new proxy state $s'$ such that the user's \emph{predicted} state $W(s)$ is closer to the actual resultant state when simulating the policy in $s'$, $T(s, u \circ \pi^*(s'))$  than to the resultant state of running the policy directly $T(s, u \circ \pi^*(s))$. Lastly, in this setting, the true $W$ and the nature of the new task that the user wishes to accomplish are unknown. However, formalizing $W$ as such can be useful for modeling and/or simulating, creating a learning objective, or creating metrics to measure the success of algorithms applied to this problem. 

\section{\algofull{} (\algo{})}
In this section, we present \algofull{} (\algo{}), to facilitate a user to accomplish new tasks given a policy and a means of teleoperating an axis of robot behavior (the setting described in the previous section). Our key insight is that when the robot is acting in a region that greatly differs from what the user has seen before, the policy should act with imagined states that are as similar to the real state as possible while being ``in-distribution'' of what the user is familiar with and anticipates. For the rest of this section, unless otherwise specified, we refer to ``in/out of distribution'' states with respect to $D$. 

\begin{algorithm}[t]
\caption{\algofull{} \\ (\algo{})}
\label{alg:algorithm}
\textbf{Initialize:} Rollout history D \\
\textbf{Initialize:} OOD-state detector \\
\textbf{Initialize:} State $s$
 
\begin{algorithmic}[1] %[1] enables line numbers
\WHILE{not done}
\IF {s is OOD}
\STATE $s' \rightarrow \arg\min_{s'\in D}d(s,s')$ 
\STATE $a \rightarrow \pi^*(s')$
\ELSE 
\STATE $a \rightarrow \pi^*(s)$
\ENDIF
\STATE $u \rightarrow $ user's control signal
\STATE $s_{t+1} \rightarrow T(s, u\circ a)$
\ENDWHILE
\end{algorithmic}
\end{algorithm}

The complete \algo{} algorithm is presented in Algorithm 1. Here, we require that an OOD detector be trained on $D$. This is then used to detect ``novel'' states. While this technique is not new from a robot-centered perspective, it is also a human-modeling choice that draws an analogy between when an OOD detector outputs a state that is OOD and when a human may be projecting to a state they have seen in the past. It is also being used to determine when to search for a state that the robot policy should ``imagine'' it is in. 


\begin{figure*}[t]
  \centering
  \subfloat[IODA]{\includegraphics[width=0.33\textwidth]{figures/sim/ioda_test_v2.png}\label{fig:f1}} 
  \hfill
  \subfloat[Out-of-distribution, \\ no imagined state]{\includegraphics[width=0.33\textwidth]{figures/sim/noIoda_xpen.png}\label{fig:f2}}
  \hfill
  \subfloat[Must return to workspace, \\ no imagined state]{\includegraphics[width=0.33\textwidth]{figures/sim/noioda_ypen.png}\label{fig:f3}}
  \vspace{-.8mm}
  \caption{In a 2D goal navigation environment, a simulated user is trying to leverage an optimal policy to reach subgoals by controlling the x-axis of the robot whilst the policy controls the y-axis. These subgoals are outside the robot's workspace (highlighted in gray). Our algorithm \algo{} allows the user to seamlessly reach the subgoals.}
  \label{2dsim}
  \vspace{-4mm}
\end{figure*}

\section{Simulation Example}
In this section we demonstrate in a 2d continuous navigation task both the problem setting and the application of the \algo{} algorithm. In the original task (see Figure 1), the robot learned to navigate to any specified goal point from within its workspace (highlighted in gray in Figure 1). The reward function for the task is defined as $-|p_{agent}-p_{goal}|$, or the negative distance of the agent's position to the goal. The user then wishes to leverage the optimal behavior they have observed to control the x-axis actions to first guide the robot to an intermediate goal outside of the workspace (shown in Figure 1 as blue or green dots) and then to the primary goal: a novel task not represented by the robot's policy.

We train two RL agents to optimally solve two slightly different versions of the original task using the off-policy RL algorithm SAC \cite{haarnoja_soft_2019}. We use SAC as it has been shown to be relatively robust when acting out-of-distribution \cite{lan_can_2023-1}. In one version of the task, Figure 1, b, the RL algorithm was trained constrained to the gray workspace, simply being penalized for leaving it. In the other version, Figure 1, c, the agent is penalized if it is out of the workspace, and is further penalized by moving in the y-axis whilst it is out of the workspace. This encourages the agent to return to its workspace as quickly as possible before continuing the task. In both versions, the agent, when out of its workspace,  may behave in a way unpredictable to a user. As a user has only ever seen optimal rollouts, they may not be familiar with what happens when the robot ``fails`` or is out-of-distribution. 


In these environments, we collected 1000 rollouts of the optimal policies and trained Deep SVDD OOD detectors \cite{ruff_deep_2018} on the states of those rollouts. We choose $d$ to be the $L1$ distance between two states. Lastly, we substitute a human's control for an optimal x-position controller given the current x-position and subgoal location. As can be seen in Figure 1. \algo{} is the only condition in which the simulated user can reach all subgoals and then easily go to the primary goal. In Figure 1, b, the agent acted relatively sporadically when brought out-of-distribution, and could only reach both goals half the time.  In Figure 1, c, since the agent was trained not to move in the y-axis when outside of its workspace, the policy of the simulated user was incompatible with the agent's behavior. Furthermore, $D$ did not contain any indication that the robot would stop.

\section{Future Work and Discussion}
It is crucial that users are empowered to use the functionalities of a robot to solve creative and on-the-fly problems. We believe this can be achieved without necessarily modeling a user's internal goals or trying to account for everything a user may want in advance but rather by providing second-order tools that a user can use to adjust and freely modify agent behavior. In the case of reinforcement learning, policies often have failure cases or behavior that may not be seen by a user during day-to-day use cases, and this can inhibit a user's creative use of a policy to perform new tasks without having to explicitly teach the robot. By keeping track of not just the robot's experience but also the user's experience with the robot, we can reason about how to facilitate a user to leverage robot behavior they have seen before.

In this work, we presented a proof-of-concept problem setting and the \algo{} algorithm. This problem setting is important to examine and study to ensure end-users are empowered to perform novel tasks. In our follow-up work, we plan on expanding this setting. For example, in this work, we study the problem setting in a task where the notion of distances between states is fairly well defined and an agent's actions in those states are fairly predictable. Furthermore, we plan to empirically investigate our assumptions about how humans perceive an agent both when they partially control them and when acting independently. This investigation will be done through a user study where users can indicate predictions about an agent's behavior given what they have seen before. We then plan on a follow-up user study in significantly more complicated tasks than in the simulation environment, such as plant watering and kitchen organizing. The second study will examine a user's ability to perform these novel tasks given an RL policy and a means of teleoperation through \algo{} and other baselines. 

\section{Conclusion}
We proposed that users can simultaneously use partial teleoperation in conjunction with an RL policy to perform novel tasks. This is based on the claim that users anticipate a robot's behavior under their partial control given that they have seen and are familiar with how it behaves autonomously. Unfortunately, a robot's behavior may not always meet a user's expectation. Rather, when a user takes partial control, they may unintentionally bring the RL policy into failure states or ``out-of-distribution,'' both of which may cause unexpected robot behavior and hinder the success of the desired task. To address this, we proposed the \algo{} algorithm, which passes imaginary states to the RL policy that result in more predictable behavior. We demonstrated \algo{} in a simulation and plan to study both the problem setting and \algo{} in the hands of real users. 

% \bibliographystyle{ACM-Reference-Format} 

\bibliography{main}
\end{document}

\bigskip
\noindent Thank you for reading these instructions carefully. We look forward to receiving your electronic files!

\bibliography{aaai23}

\end{document}

\begin{thebibliography}{44}
\providecommand{\natexlab}[1]{#1}

\bibitem[{Arulkumaran et~al.(2017)Arulkumaran, Deisenroth, Brundage, and Bharath}]{arulkumaran_brief_2017}
Arulkumaran, K.; Deisenroth, M.~P.; Brundage, M.; and Bharath, A.~A. 2017.
\newblock A {Brief} {Survey} of {Deep} {Reinforcement} {Learning}.
\newblock \emph{IEEE Signal Processing Magazine}, 34(6): 26--38.
\newblock ArXiv:1708.05866 [cs, stat].

\bibitem[{Arzate~Cruz and Igarashi(2020)}]{arzate_cruz_survey_2020}
Arzate~Cruz, C.; and Igarashi, T. 2020.
\newblock A {Survey} on {Interactive} {Reinforcement} {Learning}: {Design} {Principles} and {Open} {Challenges}.
\newblock In \emph{Proceedings of the 2020 {ACM} {Designing} {Interactive} {Systems} {Conference}}, 1195--1209. Eindhoven Netherlands: ACM.
\newblock ISBN 978-1-4503-6974-9.

\bibitem[{Bied and Chetouani(2020)}]{bied_integrating_2020}
Bied, M.; and Chetouani, M. 2020.
\newblock Integrating an {Observer} in {Interactive} {Reinforcement} {Learning} to {Learn} {Legible} {Trajectories}.
\newblock In \emph{2020 29th {IEEE} {International} {Conference} on {Robot} and {Human} {Interactive} {Communication} ({RO}-{MAN})}, 760--767.
\newblock ISSN: 1944-9437.

\bibitem[{Busch et~al.(2017)Busch, Grizou, Lopes, and Stulp}]{busch_learning_2017}
Busch, B.; Grizou, J.; Lopes, M.; and Stulp, F. 2017.
\newblock Learning {Legible} {Motion} from {Human}–{Robot} {Interactions}.
\newblock \emph{International Journal of Social Robotics}, 9(5): 765--779.

\bibitem[{Bustamante et~al.(2021)Bustamante, Quere, Hagmann, Wu, Schmaus, Vogel, Stulp, and Leidner}]{bustamante_toward_2021}
Bustamante, S.; Quere, G.; Hagmann, K.; Wu, X.; Schmaus, P.; Vogel, J.; Stulp, F.; and Leidner, D. 2021.
\newblock Toward {Seamless} {Transitions} {Between} {Shared} {Control} and {Supervised} {Autonomy} in {Robotic} {Assistance}.
\newblock \emph{IEEE Robotics and Automation Letters}, 6(2): 3833--3840.
\newblock Conference Name: IEEE Robotics and Automation Letters.

\bibitem[{Christiano et~al.(2017)Christiano, Leike, Brown, Martic, Legg, and Amodei}]{christiano_deep_2017}
Christiano, P.; Leike, J.; Brown, T.~B.; Martic, M.; Legg, S.; and Amodei, D. 2017.
\newblock Deep reinforcement learning from human preferences.
\newblock Number: arXiv:1706.03741 arXiv:1706.03741 [cs, stat].

\bibitem[{Cruz et~al.(2021)Cruz, Dazeley, Vamplew, and Moreira}]{cruz_explainable_2021}
Cruz, F.; Dazeley, R.; Vamplew, P.; and Moreira, I. 2021.
\newblock Explainable robotic systems: understanding goal-driven actions in a reinforcement learning scenario.
\newblock \emph{Neural Computing and Applications}.

\bibitem[{Darvish et~al.(2023)Darvish, Penco, Ramos, Cisneros, Pratt, Yoshida, Ivaldi, and Pucci}]{darvish_teleoperation_2023}
Darvish, K.; Penco, L.; Ramos, J.; Cisneros, R.; Pratt, J.; Yoshida, E.; Ivaldi, S.; and Pucci, D. 2023.
\newblock Teleoperation of {Humanoid} {Robots}: {A} {Survey}.
\newblock ArXiv:2301.04317 [cs].

\bibitem[{Das, Banerjee, and Chernova(2021)}]{das_explainable_2021}
Das, D.; Banerjee, S.; and Chernova, S. 2021.
\newblock Explainable {AI} for {Robot} {Failures}: {Generating} {Explanations} that {Improve} {User} {Assistance} in {Fault} {Recovery}.
\newblock In \emph{Proceedings of the 2021 {ACM}/{IEEE} {International} {Conference} on {Human}-{Robot} {Interaction}}, {HRI} '21, 351--360. New York, NY, USA: Association for Computing Machinery.
\newblock ISBN 978-1-4503-8289-2.

\bibitem[{Dass et~al.(2023)Dass, Pertsch, Zhang, Lee, Lim, and Nikolaidis}]{dass_pato_2023}
Dass, S.; Pertsch, K.; Zhang, H.; Lee, Y.; Lim, J.; and Nikolaidis, S. 2023.
\newblock {PATO}: {Policy} {Assisted} {TeleOperation} for {Scalable} {Robot} {Data} {Collection}.
\newblock In \emph{Robotics: {Science} and {Systems} {XIX}}. Robotics: Science and Systems Foundation.
\newblock ISBN 978-0-9923747-9-2.

\bibitem[{Dragan and Srinivasa(2013{\natexlab{a}})}]{dragan_generating_2013}
Dragan, A.; and Srinivasa, S. 2013{\natexlab{a}}.
\newblock Generating {Legible} {Motion}.
\newblock In \emph{Robotics: {Science} and {Systems} {IX}}. Robotics: Science and Systems Foundation.
\newblock ISBN 978-981-07-3937-9.

\bibitem[{Dragan, Lee, and Srinivasa(2013)}]{dragan_legibility_2013}
Dragan, A.~D.; Lee, K.~C.; and Srinivasa, S.~S. 2013.
\newblock Legibility and predictability of robot motion.
\newblock In \emph{2013 8th {ACM}/{IEEE} {International} {Conference} on {Human}-{Robot} {Interaction} ({HRI})}, 301--308. Tokyo, Japan: IEEE.
\newblock ISBN 978-1-4673-3101-2 978-1-4673-3099-2 978-1-4673-3100-5.

\bibitem[{Dragan and Srinivasa(2013{\natexlab{b}})}]{dragan_policy-blending_2013}
Dragan, A.~D.; and Srinivasa, S.~S. 2013{\natexlab{b}}.
\newblock A policy-blending formalism for shared control.
\newblock \emph{The International Journal of Robotics Research}, 32(7): 790--805.
\newblock Publisher: SAGE Publications Ltd STM.

\bibitem[{Driess et~al.(2023)Driess, Xia, Sajjadi, Lynch, Chowdhery, Ichter, Wahid, Tompson, Vuong, Yu, Huang, Chebotar, Sermanet, Duckworth, Levine, Vanhoucke, Hausman, Toussaint, Greff, Zeng, Mordatch, and Florence}]{driess_palm-e_2023}
Driess, D.; Xia, F.; Sajjadi, M. S.~M.; Lynch, C.; Chowdhery, A.; Ichter, B.; Wahid, A.; Tompson, J.; Vuong, Q.; Yu, T.; Huang, W.; Chebotar, Y.; Sermanet, P.; Duckworth, D.; Levine, S.; Vanhoucke, V.; Hausman, K.; Toussaint, M.; Greff, K.; Zeng, A.; Mordatch, I.; and Florence, P. 2023.
\newblock {PaLM}-{E}: {An} {Embodied} {Multimodal} {Language} {Model}.
\newblock ArXiv:2303.03378 [cs].

\bibitem[{Eysenbach et~al.(2018)Eysenbach, Gupta, Ibarz, and Levine}]{eysenbach_diversity_2018}
Eysenbach, B.; Gupta, A.; Ibarz, J.; and Levine, S. 2018.
\newblock Diversity is {All} {You} {Need}: {Learning} {Skills} without a {Reward} {Function}.
\newblock Number: arXiv:1802.06070 arXiv:1802.06070 [cs].

\bibitem[{Faria, Melo, and Paiva(2021)}]{faria_understanding_2021}
Faria, M.; Melo, F.~S.; and Paiva, A. 2021.
\newblock Understanding {Robots}: {Making} {Robots} {More} {Legible} in {Multi}-{Party} {Interactions}.
\newblock In \emph{2021 30th {IEEE} {International} {Conference} on {Robot} \& {Human} {Interactive} {Communication} ({RO}-{MAN})}, 1031--1036.
\newblock ISSN: 1944-9437.

\bibitem[{Fontaine and Nikolaidis(2021)}]{fontaine_differentiable_2021}
Fontaine, M.~C.; and Nikolaidis, S. 2021.
\newblock Differentiable {Quality} {Diversity}.
\newblock Number: arXiv:2106.03894 arXiv:2106.03894 [cs].

\bibitem[{Gopinath, Jain, and Argall(2017)}]{gopinath_human---loop_2017}
Gopinath, D.; Jain, S.; and Argall, B.~D. 2017.
\newblock Human-in-the-{Loop} {Optimization} of {Shared} {Autonomy} in {Assistive} {Robotics}.
\newblock \emph{IEEE Robotics and Automation Letters}, 2(1): 247--254.
\newblock Conference Name: IEEE Robotics and Automation Letters.

\bibitem[{Haarnoja et~al.(2019)Haarnoja, Zhou, Hartikainen, Tucker, Ha, Tan, Kumar, Zhu, Gupta, Abbeel, and Levine}]{haarnoja_soft_2019}
Haarnoja, T.; Zhou, A.; Hartikainen, K.; Tucker, G.; Ha, S.; Tan, J.; Kumar, V.; Zhu, H.; Gupta, A.; Abbeel, P.; and Levine, S. 2019.
\newblock Soft {Actor}-{Critic} {Algorithms} and {Applications}.
\newblock \emph{arXiv:1812.05905 [cs, stat]}.

\bibitem[{Haider et~al.(2023)Haider, Roscher, Schmoeller~da Roza, and Günnemann}]{haider_out--distribution_2023}
Haider, T.; Roscher, K.; Schmoeller~da Roza, F.; and Günnemann, S. 2023.
\newblock Out-of-{Distribution} {Detection} for {Reinforcement} {Learning} {Agents} with {Probabilistic} {Dynamics} {Models}.
\newblock In \emph{Proceedings of the 2023 {International} {Conference} on {Autonomous} {Agents} and {Multiagent} {Systems}}, {AAMAS} '23, 851--859. Richland, SC: International Foundation for Autonomous Agents and Multiagent Systems.
\newblock ISBN 978-1-4503-9432-1.

\bibitem[{Ibarz et~al.(2021)Ibarz, Tan, Finn, Kalakrishnan, Pastor, and Levine}]{ibarz_how_2021}
Ibarz, J.; Tan, J.; Finn, C.; Kalakrishnan, M.; Pastor, P.; and Levine, S. 2021.
\newblock How to {Train} {Your} {Robot} with {Deep} {Reinforcement} {Learning}; {Lessons} {We}'ve {Learned}.
\newblock \emph{The International Journal of Robotics Research}, 40(4-5): 698--721.
\newblock ArXiv:2102.02915 [cs].

\bibitem[{Ichter et~al.(2023)Ichter, Brohan, Chebotar, Finn, Hausman, Herzog, Ho, Ibarz, Irpan, Jang, Julian, Kalashnikov, Levine, Lu, Parada, Rao, Sermanet, Toshev, Vanhoucke, Xia, Xiao, Xu, Yan, Brown, Ahn, Cortes, Sievers, Tan, Xu, Reyes, Rettinghouse, Quiambao, Pastor, Luu, Lee, Kuang, Jesmonth, Joshi, Jeffrey, Ruano, Hsu, Gopalakrishnan, David, Zeng, and Fu}]{ichter_as_2023}
Ichter, B.; Brohan, A.; Chebotar, Y.; Finn, C.; Hausman, K.; Herzog, A.; Ho, D.; Ibarz, J.; Irpan, A.; Jang, E.; Julian, R.; Kalashnikov, D.; Levine, S.; Lu, Y.; Parada, C.; Rao, K.; Sermanet, P.; Toshev, A.~T.; Vanhoucke, V.; Xia, F.; Xiao, T.; Xu, P.; Yan, M.; Brown, N.; Ahn, M.; Cortes, O.; Sievers, N.; Tan, C.; Xu, S.; Reyes, D.; Rettinghouse, J.; Quiambao, J.; Pastor, P.; Luu, L.; Lee, K.-H.; Kuang, Y.; Jesmonth, S.; Joshi, N.~J.; Jeffrey, K.; Ruano, R.~J.; Hsu, J.; Gopalakrishnan, K.; David, B.; Zeng, A.; and Fu, C.~K. 2023.
\newblock Do {As} {I} {Can}, {Not} {As} {I} {Say}: {Grounding} {Language} in {Robotic} {Affordances}.
\newblock In \emph{Proceedings of {The} 6th {Conference} on {Robot} {Learning}}, 287--318. PMLR.
\newblock ISSN: 2640-3498.

\bibitem[{Javdani, Srinivasa, and Bagnell(2015)}]{javdani_shared_2015}
Javdani, S.; Srinivasa, S.; and Bagnell, A. 2015.
\newblock Shared {Autonomy} via {Hindsight} {Optimization}.
\newblock In \emph{Robotics: {Science} and {Systems} {XI}}. Robotics: Science and Systems Foundation.
\newblock ISBN 978-0-9923747-1-6.

\bibitem[{Kaelbling, Littman, and Moore(1996)}]{kaelbling_reinforcement_1996}
Kaelbling, L.~P.; Littman, M.~L.; and Moore, A.~W. 1996.
\newblock Reinforcement {Learning}: {A} {Survey}.
\newblock ArXiv:cs/9605103.

\bibitem[{Kober and Peters(2014)}]{kober_reinforcement_2014}
Kober, J.; and Peters, J. 2014.
\newblock Reinforcement {Learning} in {Robotics}: {A} {Survey}.
\newblock In Kober, J.; and Peters, J., eds., \emph{Learning {Motor} {Skills}: {From} {Algorithms} to {Robot} {Experiments}}, Springer {Tracts} in {Advanced} {Robotics}, 9--67. Cham: Springer International Publishing.
\newblock ISBN 978-3-319-03194-1.

\bibitem[{Lan, Zhang, and Hsieh(2023)}]{lan_can_2023-1}
Lan, L.-C.; Zhang, H.; and Hsieh, C.-J. 2023.
\newblock Can {Agents} {Run} {Relay} {Race} with {Strangers}? {Generalization} of {RL} to {Out}-of-{Distribution} {Trajectories}.
\newblock ArXiv:2304.13424 [cs].

\bibitem[{Li et~al.(2015)Li, Tee, Chan, Yan, Chua, and Limbu}]{li_continuous_2015}
Li, Y.; Tee, K.~P.; Chan, W.~L.; Yan, R.; Chua, Y.; and Limbu, D.~K. 2015.
\newblock Continuous {Role} {Adaptation} for {Human}–{Robot} {Shared} {Control}.
\newblock \emph{IEEE Transactions on Robotics}, 31(3): 672--681.
\newblock Conference Name: IEEE Transactions on Robotics.

\bibitem[{Liang et~al.(2023)Liang, Huang, Xia, Xu, Hausman, Ichter, Florence, and Zeng}]{liang_code_2023}
Liang, J.; Huang, W.; Xia, F.; Xu, P.; Hausman, K.; Ichter, B.; Florence, P.; and Zeng, A. 2023.
\newblock Code as {Policies}: {Language} {Model} {Programs} for {Embodied} {Control}.
\newblock In \emph{2023 {IEEE} {International} {Conference} on {Robotics} and {Automation} ({ICRA})}, 9493--9500.

\bibitem[{Matsumoto and Riek(2022)}]{matsumoto_shared_2022}
Matsumoto, S.; and Riek, L.~D. 2022.
\newblock Shared {Control} in {Human} {Robot} {Teaming}: {Toward} {Context}-{Aware} {Communication}.
\newblock ArXiv:2203.10218 [cs].

\bibitem[{Moniruzzaman et~al.(2022)Moniruzzaman, Rassau, Chai, and Islam}]{moniruzzaman_teleoperation_2022}
Moniruzzaman, M.; Rassau, A.; Chai, D.; and Islam, S. M.~S. 2022.
\newblock Teleoperation methods and enhancement techniques for mobile robots: {A} comprehensive survey.
\newblock \emph{Robotics and Autonomous Systems}, 150: 103973.

\bibitem[{Nguyen and La(2019)}]{nguyen_review_2019}
Nguyen, H.; and La, H. 2019.
\newblock Review of {Deep} {Reinforcement} {Learning} for {Robot} {Manipulation}.
\newblock In \emph{2019 {Third} {IEEE} {International} {Conference} on {Robotic} {Computing} ({IRC})}, 590--595. Naples, Italy: IEEE.
\newblock ISBN 978-1-5386-9245-5.

\bibitem[{Osa, Tangkaratt, and Sugiyama(2022)}]{osa_discovering_2022}
Osa, T.; Tangkaratt, V.; and Sugiyama, M. 2022.
\newblock Discovering diverse solutions in deep reinforcement learning by maximizing state–action-based mutual information.
\newblock \emph{Neural Networks}, 152: 90--104.

\bibitem[{Paleja et~al.(2021)Paleja, Ghuy, Ranawaka~Arachchige, Jensen, and Gombolay}]{paleja_utility_2021}
Paleja, R.; Ghuy, M.; Ranawaka~Arachchige, N.; Jensen, R.; and Gombolay, M. 2021.
\newblock The {Utility} of {Explainable} {AI} in {Ad} {Hoc} {Human}-{Machine} {Teaming}.
\newblock In \emph{Advances in {Neural} {Information} {Processing} {Systems}}, volume~34, 610--623. Curran Associates, Inc.

\bibitem[{Ruff et~al.(2018)Ruff, Vandermeulen, Goernitz, Deecke, Siddiqui, Binder, Müller, and Kloft}]{ruff_deep_2018}
Ruff, L.; Vandermeulen, R.; Goernitz, N.; Deecke, L.; Siddiqui, S.~A.; Binder, A.; Müller, E.; and Kloft, M. 2018.
\newblock Deep {One}-{Class} {Classification}.
\newblock In \emph{Proceedings of the 35th {International} {Conference} on {Machine} {Learning}}, 4393--4402. PMLR.
\newblock ISSN: 2640-3498.

\bibitem[{Sakai and Nagai(2022)}]{sakai_explainable_2022}
Sakai, T.; and Nagai, T. 2022.
\newblock Explainable autonomous robots: a survey and perspective.
\newblock \emph{Advanced Robotics}, 36(5-6): 219--238.
\newblock Publisher: Taylor \& Francis \_eprint: https://doi.org/10.1080/01691864.2022.2029720.

\bibitem[{Schrum et~al.(2022)Schrum, Hedlund-Botti, Moorman, and Gombolay}]{schrum_mind_2022}
Schrum, M.~L.; Hedlund-Botti, E.; Moorman, N.; and Gombolay, M.~C. 2022.
\newblock {MIND} {MELD}: {Personalized} {Meta}-{Learning} for {Robot}-{Centric} {Imitation} {Learning}.
\newblock In \emph{Proceedings of the 2022 {ACM}/{IEEE} {International} {Conference} on {Human}-{Robot} {Interaction}}, {HRI} '22, 157--165. Sapporo, Hokkaido, Japan: IEEE Press.

\bibitem[{Selvaggio et~al.(2021)Selvaggio, Cognetti, Nikolaidis, Ivaldi, and Siciliano}]{selvaggio_autonomy_2021}
Selvaggio, M.; Cognetti, M.; Nikolaidis, S.; Ivaldi, S.; and Siciliano, B. 2021.
\newblock Autonomy in {Physical} {Human}-{Robot} {Interaction}: {A} {Brief} {Survey}.
\newblock \emph{IEEE Robotics and Automation Letters}, 6(4): 7989--7996.
\newblock Conference Name: IEEE Robotics and Automation Letters.

\bibitem[{Shridhar, Manuelli, and Fox(2023)}]{shridhar_perceiver-actor_2023}
Shridhar, M.; Manuelli, L.; and Fox, D. 2023.
\newblock Perceiver-{Actor}: {A} {Multi}-{Task} {Transformer} for {Robotic} {Manipulation}.
\newblock In \emph{Proceedings of {The} 6th {Conference} on {Robot} {Learning}}, 785--799. PMLR.
\newblock ISSN: 2640-3498.

\bibitem[{Sun et~al.(2022)Sun, Ming, Zhu, and Li}]{sun_out--distribution_2022}
Sun, Y.; Ming, Y.; Zhu, X.; and Li, Y. 2022.
\newblock Out-of-{Distribution} {Detection} with {Deep} {Nearest} {Neighbors}.
\newblock In \emph{Proceedings of the 39th {International} {Conference} on {Machine} {Learning}}, 20827--20840. PMLR.
\newblock ISSN: 2640-3498.

\bibitem[{Walke et~al.(2023)Walke, Yang, Yu, Kumar, Orbik, Singh, and Levine}]{walke_dont_2023}
Walke, H.~R.; Yang, J.~H.; Yu, A.; Kumar, A.; Orbik, J.; Singh, A.; and Levine, S. 2023.
\newblock Don’t {Start} {From} {Scratch}: {Leveraging} {Prior} {Data} to {Automate} {Robotic} {Reinforcement} {Learning}.
\newblock In \emph{Proceedings of {The} 6th {Conference} on {Robot} {Learning}}, 1652--1662. PMLR.
\newblock ISSN: 2640-3498.

\bibitem[{Xie and Finn(2022)}]{xie_lifelong_2022}
Xie, A.; and Finn, C. 2022.
\newblock Lifelong {Robotic} {Reinforcement} {Learning} by {Retaining} {Experiences}.
\newblock In \emph{Proceedings of {The} 1st {Conference} on {Lifelong} {Learning} {Agents}}, 838--855. PMLR.
\newblock ISSN: 2640-3498.

\bibitem[{Yang et~al.(2022)Yang, Zhou, Li, and Liu}]{yang_generalized_2022}
Yang, J.; Zhou, K.; Li, Y.; and Liu, Z. 2022.
\newblock Generalized {Out}-of-{Distribution} {Detection}: {A} {Survey}.
\newblock ArXiv:2110.11334 [cs].

\bibitem[{Yoneda et~al.(2023)Yoneda, Sun, Yang, Stadie, and Walter}]{yoneda_noise_2023}
Yoneda, T.; Sun, L.; Yang, a.~G.; Stadie, B.; and Walter, M. 2023.
\newblock To the {Noise} and {Back}: {Diffusion} for {Shared} {Autonomy}.
\newblock ArXiv:2302.12244 [cs].

\bibitem[{Zurek et~al.(2021)Zurek, Bobu, Brown, and Dragan}]{zurek_situational_2021}
Zurek, M.; Bobu, A.; Brown, D.~S.; and Dragan, A.~D. 2021.
\newblock Situational {Confidence} {Assistance} for {Lifelong} {Shared} {Autonomy}.
\newblock In \emph{2021 {IEEE} {International} {Conference} on {Robotics} and {Automation} ({ICRA})}, 2783--2789. Xi\'an, China: IEEE Press.

\end{thebibliography}