On some perturbation properties of nonsmooth optimization on Riemannian manifolds with applications
Extracted source text · arXiv 2308.06793
% !TeX spellcheck = en_US
\documentclass[final,onefignum,onetabnum]{siamart220329}
%% ------------------------------------------------------------------
%% Code used in examples, needed to reproduce
%% ------------------------------------------------------------------
%% Used for \set, used in an example below
\usepackage{braket,amsfonts}
%% Used in table example below
\usepackage{array}
%% Used in table and figure examples below
\usepackage[caption=false]{subfig}
%% Used for papers with subtables created with the subfig package
\captionsetup[subtable]{position=bottom}
\captionsetup[table]{position=bottom}
%% Used for PgfPlots example, shown in the "Figures" section below.
\usepackage{pgfplots}
%% Used for creating new theorem and remark environments
\newsiamthm{claim}{Claim}
\newsiamremark{remark}{Remark}
\newsiamremark{hypothesis}{Hypothesis}
\crefname{hypothesis}{Hypothesis}{Hypotheses}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage{fancyhdr}
\usepackage{extramarks}
\usepackage{amsfonts}
\usepackage{tikz}
\usepackage{algorithm}
%\usepackage{amsthm}
\usepackage[noend]{algpseudocode}
\usepackage{amssymb}
\usepackage{mathrsfs}
%\usepackage{algorithmicx}
\usepackage{stmaryrd}
\usepackage{color}
\usepackage{verbatim}
\usepackage{hyperref}
\hypersetup{hidelinks}
\usepackage{caption}
\usepackage{float}
\usepackage{cases,mathdots,colordvi,appendix,url,multirow,lscape}
\usepackage{geometry}
%% Algorithm style, could alternatively use algpseudocode
\usepackage{algorithmicx}
%% For figures
\usepackage{graphicx,epstopdf}
%% For referencing line numbers
\Crefname{ALC@unique}{Line}{Lines}
%% For creating math operators
\usepackage{amsopn}
\DeclareMathOperator{\Range}{Range}
%% ------------------------------------------------------------------
%% Macros for in-document examples. These are not meant to reused for
%% SIAM journal papers.
%% ------------------------------------------------------------------
\usepackage{xspace}
\usepackage{bold-extra}
\usepackage[most]{tcolorbox}
\newcommand{\BibTeX}{{\scshape Bib}\TeX\xspace}
\newcounter{example}
\colorlet{texcscolor}{blue!50!black}
\colorlet{texemcolor}{red!70!black}
\colorlet{texpreamble}{red!70!black}
\colorlet{codebackground}{black!25!white!25}
\newcommand\bs{\symbol{'134}} % print backslash in typewriter OT1/T1
\newcommand{\preamble}[2][\small]{\textcolor{texpreamble}{#1\texttt{#2 \emph{\% <- Preamble}}}}
\lstdefinestyle{siamlatex}{%
style=tcblatex,
texcsstyle=*\color{texcscolor},
texcsstyle=[2]\color{texemcolor},
keywordstyle=[2]\color{texemcolor},
moretexcs={cref,Cref,maketitle,mathcal,text,headers,email,url},
}
\tcbset{%
colframe=black!75!white!75,
coltitle=white,
colback=codebackground, % bottom/left side
colbacklower=white, % top/right side
fonttitle=\bfseries,
arc=0pt,outer arc=0pt,
top=1pt,bottom=1pt,left=1mm,right=1mm,middle=1mm,boxsep=1mm,
leftrule=0.3mm,rightrule=0.3mm,toprule=0.3mm,bottomrule=0.3mm,
listing options={style=siamlatex}
}
\newtcblisting[use counter=example]{example}[2][]{%
title={Example~\thetcbcounter: #2},#1}
\newtcbinputlisting[use counter=example]{\examplefile}[3][]{%
title={Example~\thetcbcounter: #2},listing file={#3},#1}
\DeclareTotalTCBox{\code}{ v O{} }
{ %fontupper=\ttfamily\color{texemcolor},
fontupper=\ttfamily\color{black},
nobeforeafter,
tcbox raise base,
colback=codebackground,colframe=white,
top=0pt,bottom=0pt,left=0mm,right=0mm,
leftrule=0pt,rightrule=0pt,toprule=0mm,bottomrule=0mm,
boxsep=0.5mm,
#2}{#1}
% Stretch the pages
\patchcmd\newpage{\vfil}{}{}{}
\flushbottom
%% ------------------------------------------------------------------
%% End of macros for in-document examples.
%% ------------------------------------------------------------------
%% ------------------------------------------------------------------
%% HEADING INFORMATION
%% ------------------------------------------------------------------
\begin{tcbverbatimwrite}{tmp_\jobname_header.tex}
\title{Strong variational sufficiency of nonsmooth optimization problems on Riemannian manifolds}
\author{Yuexin Zhou\thanks{Institute of Applied Mathematics, Academy of Mathematics and Systems Science, Chinese Academy of Sciences, Beijing, P.R. China, School of Mathematical Sciences, University of Chinese Academy of Science, Beijing, P.R. China.
(\email{zhouyuexin19@mails.ucas.ac.cn}).}
\and Chao Ding\thanks{Institute of Applied Mathematics, Academy of Mathematics and Systems Science, Chinese Academy of Sciences, Beijing, P.R. China, School of Mathematical Sciences, University of Chinese Academy of Science, Beijing, P.R. China.
(\email{dingchao@amss.ac.cn}).}
\and Yangjing Zhang\thanks{Institute of Applied Mathematics, Academy of Mathematics and Systems Science, Chinese Academy of Sciences, Beijing, P.R. China.
(\email{yangjing.zhang@amss.ac.cn}).}}
% Custom SIAM macro to insert headers
\headers{}{Yuexin Zhou, Chao Ding, and Yangjing Zhang}
\end{tcbverbatimwrite}
\input{tmp_\jobname_header.tex}
% Optional: Set up PDF title and authors
\ifpdf
\hypersetup{
pdftitle={Strong variational sufficiency of nonsmooth optimization problems on Riemannian manifolds},
pdfauthor={Yuexin Zhou, Chao Ding, and Yangjing Zhang}
}
\fi
%\input{rmvs_shared}
%
%% Optional PDF information
%\ifpdf
%\hypersetup{
% pdftitle={Strong variational sufficiency of nonsmooth optimization problems on Riemannian manifolds},
% pdfauthor={Yuexin Zhou, Chao Ding, and Yangjing Zhang}
%}
%\fi
%
\DeclareMathOperator*{\argmin}{argmin}
%\author{Yuexin Zhou\thanks{Institute of Applied Mathematics, Academy of Mathematics and Systems Science, Chinese Academy of Sciences, Beijing, P.R. China, School of Mathematical Sciences, University of Chinese Academy of Science, Beijing, P.R. China.},\quad Chenglong Bao\thanks{Yau Mathematical Sciences Center, Tsinghua University, China and Yanqi Lake Beijing Institute of Mathematical Sciences and Applications, China.},\quad Chao Ding\thanks{Institute of Applied Mathematics, Academy of Mathematics and System Sciences, Chinese Academy of Sciences, China.}}
\begin{document}
\maketitle
\begin{tcbverbatimwrite}{tmp_\jobname_abstract.tex}
\begin{abstract}
%The Riemannian augmented Lagrangian method (RALM) is proposed recently to solve the nonsmooth optimization problems on Riemannian manifolds. As demonstrated in previous works \cite{ZBDZ21, ZBD22}, the RALM consistently displays high efficiency, often demonstrating a fast local linear convergence rate. Nevertheless, the local convergence analysis the RALM under more realistic assumptions (specifically, without imposing the singletonness assumption on the multiplier) has yet to be fully explored. In this paper, we introduce the manifold variational sufficient condition and show that its strong version is equivalent to the manifold strong second-order sufficient condition (M-SSOSC) in some cases. More importantly, we formulate a local dual problem based on this condition and apply the Euclidean proximal point algorithm, consequently establishing the R-linear convergence rate of RALM. Furthermore, the nonsingularity of the generalized Hessian, which is crucial for the semismooth Newton method for solving the RALM subproblem is demonstrated under the M-SSOSC.
The Riemannian Augmented Lagrangian Method (RALM), a recently proposed algorithm for nonsmooth optimization problems on Riemannian manifolds, has consistently exhibited high efficiency as evidenced in prior studies \cite{ZBDZ21,ZBD22}. It often demonstrates a rapid local linear convergence rate. However, a comprehensive local convergence analysis of the RALM under more realistic assumptions, notably without the imposition of the uniqueness assumption on the multiplier, remains an uncharted territory. In this paper, we introduce the manifold variational sufficient condition and demonstrate that its strong version is equivalent to the manifold strong second-order sufficient condition (M-SSOSC) in certain circumstances. Critically, we construct a local dual problem based on this condition and implement the Euclidean proximal point algorithm, which leads to the establishment of the linear convergence rate of the RALM. Moreover, we illustrate that under suitable assumptions, the M-SSOSC is equivalent to the nonsingularity of the generalized Hessian of the augmented Lagrangian function, which is an essential attribute for the semismooth Newton-type methods.
\end{abstract}
\begin{keywords}
nonsmooth optimizations on Riemannian manifold, strong variational sufficiency, augmented Lagrangian method, rate of convergence
\end{keywords}
\begin{MSCcodes}
90C30, 90C46, 49J52, 65K05
\end{MSCcodes}
\end{tcbverbatimwrite}
\input{tmp_\jobname_abstract.tex}
%
% \keywords{nonsmooth optimizations on Riemannian manifold \and strong variational sufficiency \and augmented Lagrangian method \and rate of convergence}
% % \PACS{PACS code1 \and PACS code2 \and more}
% \subclass{ 90C30 \and 90C46 \and 49J52 \and 65K05 }
\section{Introduction}
This paper is concerned with the nonsmooth optimization problems on Riemannian manifolds in the following form:
\begin{equation}\label{eq:problem-mani}
\begin{array}{ll}
\min & f(x)+\theta(g(x)) \\
\text { s.t. }
& x\in \mathcal{M},
\end{array}
\end{equation}
where $\mathcal{M}$ is a connected Riemannian manifold, $f:\mathcal{M} \rightarrow \mathbb{R}$ and $g:\mathcal{M} \rightarrow \mathbb{Y}$ are continuously differentiable functions, $\mathbb{Y}$ is a Euclidean space equipped with a scalar
product $\langle\cdot, \cdot\rangle$ and its induced
norm $\|\cdot\|$, $\theta : \mathbb{Y}\rightarrow (-\infty, \infty]$ is a proper closed convex function. If $\theta$ is an indicator function of a closed convex set, then (\ref{eq:problem-mani}) is a constrained manifold optimization problem. Applications of (\ref{eq:problem-mani}) arise in various scenarios such as principal component analysis problems \cite{ZHT06}, low-rank %low
matrix completion problems \cite{CA16}, orthogonal dictionary learning problems \cite{SWW12,DH14} and compressed modes problems \cite{OLCO13}. One may refer to \cite{AH19,HLWY20} for more details.
Different algorithms have been designed for solving the manifold nonsmooth optimization problem (\ref{eq:problem-mani}), such as subgradient methods \cite{FO98,GH16}, proximal gradient methods \cite{CMMZ20,HW19,HW22}, alternating direction methods of multipliers \cite{KGB16,LO14} and proximal point methods (PPA) \cite{CDMS21,FO02}. In recent papers \cite{DP19,ZBDZ21}, the augmented Lagrangian method (ALM) has also been extended to solve the nonsmooth manifold optimization problem \eqref{eq:problem-mani}. By adding a perturbation parameter $u$, we can obtain the perturbed problem for (\ref{eq:problem-mani}):
\begin{equation}\label{eq:perturb-problem-mani}
\begin{array}{ll}
\min & \varphi(x,u):= f(x)+\theta(g(x)+u) \\
\text { s.t. } & x\in \mathcal{M}.
\end{array}
\end{equation}
%when $u=0$ this is exactly problem (\ref{eq:problem-mani}).
Problem \eqref{eq:perturb-problem-mani} is reduced to problem \eqref{eq:problem-mani} when $u=0$.
The Lagrangian function for (\ref{eq:problem-mani}) is
$$
l(x, y)=\inf _u\{\varphi(x, u)-\left\langle y,u\right\rangle \}=L(x, y)-\theta^*(y),
$$
where $L(x, y)=f(x)+\left\langle y, g(x)\right\rangle $ and $\theta^*$ is the conjugate function of $\theta$. Moreover, for $\rho > 0$, the augmented Lagrangian function for (\ref{eq:problem-mani}) is defined by
\begin{equation}\label{eq:al-mani}
l^{\rho}(x, y)=\inf _u\big\{\varphi(x, u)-\left\langle y ,u\right\rangle +\frac{\rho}{2}\|u\|^2\big\}.
\end{equation}
%Now suppose that $(\bar{x},\bar{y})$ is a first-order stationary point of (\ref{eq:perturb-problem-mani}).
The inexact Riemannian augmented Lagrangian method (RALM) proposed in \cite{DP19,ZBDZ21} for solving \eqref{eq:problem-mani} then takes the form of
\begin{equation}\label{eq:alm-mani}
\left\{\begin{array}{l}
\displaystyle x^{k+1} \approx \bar{x}^{k+1}:=\underset{x \in \mathcal{U}}{\operatorname{argmin}}\;
l^{\rho_k}\left(x, y^k\right), \\
\displaystyle y^{k+1}=y^k+\tilde{\rho}_{k} \nabla_y l^{\rho_k}\left(x^{k+1}, y^k\right),
\end{array}\right.
\end{equation}
where $\mathcal{U}$ is a subset of $\mathcal{M}$, $\tilde{\rho}_k > 0$ and $\rho_k>0$ are nondecreasing sequences satisfying $\tilde{\rho}_k \leq \rho_k$. As evidenced in \cite{ZBDZ21, ZBD22}, the RALM consistently displays high efficiency, often demonstrating a fast local linear convergence rate. Furthermore, when the semismooth Newton method is utilized to solve the augmented Lagrangian subproblems in \eqref{eq:alm-mani}, it has frequently been observed that for certain problem classes, the associated generalized Hessian of the augmented Lagrangian function remains consistently positive definite. Nevertheless, the fast local convergence rate the algorithm under more realistic assumptions (specifically, without imposing the singletonness assumption on the multiplier) and the characterization of the non-singularity of the generalized Hessian of the augmented Lagrangian function have yet to be fully explored.
%The classical Euclidean ALM is proposed by Hestenes \cite{H69} and Powell \cite{P69} for equality constrained problems and later extended to nonlinear programming (NLP) by Rockafellar \cite{R73}, and the convergence analysis of ALM under the Euclidean settings has been extensively studied for decades. The local linear convergence rate of ALM for NLP often requires the linear independence constraint qualification (LICQ) and the second-order sufficient condition (SOSC) in textbooks, e.g., \cite{B82,CGT00,WN99}. For non-convex non-polyhedral programs such as the nonlinear second-order cone programs and semidefinite programs, Liu and Zhang \cite{LZ08} and Sun et al. \cite{SSZ08} obtain the local linear convergence rate of ALM under the constraint non-degeneracy condition and the strong SOSC at Karush-Kuhn-Tucker (KKT) points, respectively. Recently, for the $\mathcal{ C}^2$-cone reducible constraints, Kanzow and Steck \cite{KS19} obtain the primal-dual linear convergence result under the strong Robinson constraint qualification (SRCQ) and SOSC. These analyses for the Euclidean ALM all rely on the uniqueness of the multipliers. For the RALM, the first convergence result is established in \cite{ZBDZ21} under the constant positive linear dependence constraint qualification and LICQ. Later, the local linear convergence rate analysis for the RALM is provided in \cite{ZBD22} under the manifold strict Robinson constraint qualification (M-SRCQ) and the manifold second-order sufficient condition (M-SOSC). The existing convergence results for the RALM still require a unique multiplier. However, we observe that the RALM can still perform well in certain cases even when the multiplier set is not a singleton. This phenomenon motivates us to consider alternative conditions to ensure the local linear convergence rate of RALM without requiring
%the uniqueness of the multipliers.
The classical Euclidean ALM, originally proposed by Hestenes \cite{H69} and Powell \cite{P69} for equality constrained problems, was later expanded to nonlinear programming (NLP) by Rockafellar \cite{R73}. Over the decades, the convergence analysis of ALM under the Euclidean setting has been extensively studied. In literature, it is posited that the local linear convergence rate of ALM for NLP requires both the linear independence constraint qualification (LICQ) and the second-order sufficient condition (SOSC) (see e.g., \cite{B82,CGT00,WN99}). For non-convex non-polyhedral problems, such as nonlinear second-order cone programs and semidefinite programs, Liu and Zhang \cite{LZ08} and Sun et al. \cite{SSZ08} obtain the local linear convergence rate of ALM under the constraint non-degeneracy condition and the strong SOSC at Karush-Kuhn-Tucker (KKT) points, respectively. More recently, for $\mathcal{ C}^2$-cone reducible constraints, Kanzow and Steck \cite{KS19} obtained the primal-dual linear convergence rate under the strong Robinson constraint qualification (SRCQ) and SOSC. These analyses for the Euclidean ALM all hinge on the uniqueness of the multipliers. Turning to the RALM, the first convergence result is established in \cite{ZBDZ21} under the constant positive linear dependence constraint qualification and LICQ. Subsequently, the local linear convergence rate for the RALM was provided in \cite{ZBD22} under the manifold strict Robinson constraint qualification (M-SRCQ) and the manifold second-order sufficient condition (M-SOSC). Nevertheless, existing convergence results for the RALM still necessitate a unique multiplier. Interestingly, we have observed that the RALM can still perform effectively in certain scenarios even when the multiplier set is not a singleton. This observation has driven us to explore alternative conditions that could ensure the local linear convergence rate of the RALM without requiring the uniqueness of the multipliers.
The recent studies \cite{R22-ext,WDZZ22} shed light on the possibility of relaxing the uniqueness assumption of the multipliers under the so-called (strong) variational sufficient condition. In \cite{R22}, Rockafellar introduces the concept of variational (strong) convexity, which requires the function value and the subdifferential of the non-convex function to be locally identical to a (strongly) convex function. By utilizing this property, the (strong) variational sufficient condition for optimizations in Euclidean setting is built up by requiring the perturbed augmented objective function to be variationally (strongly) convex with respect to a first-order stationary point. It is proven in \cite{R22} that the strong variational sufficient condition implies the local strong convexity of the augmented Lagrangian function and the augmented tilt stability. Furthermore, it is demonstrated that strong variational sufficiency is equivalent to the strong SOSC
if the function $\theta$ in \eqref{eq:problem-mani} (when the manifold $\mathcal{M}$ is taken as a Euclidean space) is a polyhedral function (\cite{R22}) or an indicator function of a second-order cone or semidefinite cone (\cite{WDZZ22}). By leveraging the strong convexity of the augmented Lagrangian function, one may be able to construct an augmented dual problem locally around a KKT pair and apply the PPA to this dual problem to achieve a local linear rate. Rockafellar \cite{R22-ext} then extends the classical results in \cite{R76}, which posit the equivalence of the dual PPA and primal ALM for convex problems, to obtain the local linear convergence rate of ALM under the strong variational sufficient condition. Inspired by this approach, our objective is to extend the (strong) variational sufficient condition to manifold optimization. The first challenge here is defining convexity on a manifold. It is widely understood that translating the concept of convexity from Euclidean settings to functions and sets on a Riemannian manifold is not straightforward, primarily because a standard `line segment' connecting two points $x$ and $y$ on a manifold cannot be represented by a convex combination of $x$ and $y$. To address this issue, we propose an alternative problem that is locally equivalent to \eqref{eq:problem-mani}. Suppose that $(\bar{x},\bar{y})$ is a stationary point of problem (\ref{eq:perturb-problem-mani}), as defined in \eqref{eq:first-opt-mani}. Let $R_{\bar{x}}$ denote the retraction and $T_{\bar{x}}\mathcal{M}$ denote the tangent space of $\mathcal{M}$ at $\bar{x}$ (see Section~\ref{sec:preliminary} for definitions). We locally transform the manifold problem (\ref{eq:problem-mani}) into an optimization problem on the tangent space $T_{\bar{x}}\mathcal{M}$ at $\bar{x}$ as follows:
\begin{equation}\label{eq:problem-tangent-1}
\begin{array}{ll}
\min & f\circ R_{\bar{x}}(\xi)+\theta(g\circ R_{\bar{x}}(\xi)) \\
\text { s.t. }
& \xi\in T_{\bar{x}}\mathcal{M}.
\end{array}
\end{equation}
This problem is locally equivalent to (\ref{eq:problem-mani}) if we set $x=R_{\bar{x}}(\xi)$. Compared with (\ref{eq:problem-mani}), problem \eqref{eq:problem-tangent-1} is more approachable due to $T_{\bar{x}}\mathcal{M}$ being a Euclidean space.
Consequently, we can now explore properties related to the convexity of (\ref{eq:problem-tangent-1}).
In this paper, we establish the ALM for solving (\ref{eq:problem-tangent-1}), which is locally equivalent to RALM (\ref{eq:alm-mani}) for solving (\ref{eq:problem-mani}). By assuming the variational sufficient condition for problem (\ref{eq:problem-tangent-1}) (a property we will refer to as the manifold variational sufficient condition), we are able to construct a local augmented dual problem in Euclidean space for (\ref{eq:problem-mani}).
Furthermore, we discover that the manifold strong variational sufficient condition is equivalent to the manifold strong second-order sufficient condition (M-SSOSC) under certain circumstances. This also implies that the manifold strong variational sufficient condition is independent of the choice of retractions. Consequently, by applying PPA to the local dual problem, we ultimately obtain R-linear convergence rate of the primal iterations of RALM and Q-linear rate of the multipliers sequence. Moreover, we show that the M-SSOSC is also equivalent to the non-singularity of the generalized Hessian of the augmented Lagrangian function, which ensures the efficiency of the semismooth Newton method when solving the subproblem of RALM.
%This implies that the strong variational sufficient condition is independent with the chosen retraction for constructing problem (\ref{eq:problem-tangent-1}), which enable us to define the strong variational sufficient condition for manifold optimization problem.
The rest of the paper is organized as follows. In Section \ref{sec:preliminary}, we revisit some fundamentals of smooth manifolds and nonsmooth analysis. In Section \ref{sec:equivalence}, we define the local equivalent problem for (\ref{eq:problem-mani}) in the tangent space and explore the relation of Lagrangian functions and first-order conditions between these two problems. The variational sufficient condition is discussed in Section \ref{sec:variational suff}. Moreover, the local duality under variational sufficient condition is given in this section. Section \ref{sec:convegence-alm} establishes the local convergence analysis of RALM. The semismooth Newton method for solving RALM subproblem and its local convergence rate are discussed in Section \ref{sec:semismooth}. The applications and numerical results are presented in Section \ref{sec:numerical}. Finally, we give our conclusion in Section \ref{sec:conclu}.
\section{Preliminaries and notations}\label{sec:preliminary}
We begin by introducing some basic concepts of manifolds that will be used in our discussion. Most of the properties referred to below can be found in the literature, specifically in \cite{AMS09,L13}.
Let $\mathcal{M}$ be an $n$-dimensional smooth manifold and $x \in \mathcal{M}$. $\mathfrak{F}_{x}(\mathcal{M})$ is defined as the set of all smooth real-valued functions on a neighborhood of $x$. The mapping $\xi_{x}$ from $\mathfrak{F}_{x}(\mathcal{M})$ to $\mathbb{R}$ such that there exists a curve $\gamma$ on $\mathcal{M}$ with $\gamma(0)=x$ satisfying $\xi_{x} f:=\dot{\gamma}(0)f:=\left.\dfrac{d(f(\gamma(t)))}{d t}\right|_{t=0}$ for all $f \in \mathfrak{F}_{x}(\mathcal{M})$ is called a tangent vector, and the tangent space $T_x\mathcal{M}$ is the set of all tangent vectors to $\mathcal{M}$ at $x$. If $\mathcal{M}$ is embedded in a Euclidean space $\mathbb{X}$, the normal space $N_x\mathcal{M}$ is defined as the orthogonal complement of $T_x\mathcal{M}$ in $\mathbb{X}$. The tangent bundle is defined as $T \mathcal{M}:=\bigcup_{x} T_{x} \mathcal{M}$, which is the set of all tangent vectors to $\mathcal{M}$. A map $V : \mathcal{M} \rightarrow T \mathcal{M} $ is called a vector field on $\mathcal{ M}$ if $V(x) \in T_{x}\mathcal{M}$ for all $x \in\mathcal{M}$.
Let $F: \mathcal{M} \rightarrow \mathbb{X}$ be a smooth mapping. The mapping $DF(x): T_{x} \mathcal{M} \rightarrow T_{F(x)} \mathbb{X}$ which is defined by $\left(D F(x)\xi_{x}\right) f:=\xi_{x}(f \circ F)$ for $\xi_{x} \in T_{x} \mathcal{M}$ and $f \in \mathfrak{F}_{F(x)}(\mathbb{X})$, is a linear mapping called the differential of $F$ at $x$. If $ \mathcal{M}$ is embedded in a Euclidean space, then $D F(x)$ is reduced to the classical definition of directional derivative, i.e., $\displaystyle D F(x)\xi_{x}=\lim _{t \rightarrow 0} \frac{F\left(x+t \xi_{x}\right)-F(x)}{t}$. To distinguish it from the Riemannian differential, we use $h'(x)\xi$ to represent the traditional directional derivative in the direction $\xi$ and $\nabla h(x)$ to be the Euclidean gradient of $h$.
Riemannian metric $\langle\cdot, \cdot\rangle_{x}$ is a smoothly varying inner product with respect to $x$ defined for tangent vectors. A differentiable manifold whose tangent spaces are endowed with Riemannian metrics is called a Riemannian manifold. When no confusions arise, we will use $\langle\cdot, \cdot\rangle$ instead of $\langle\cdot, \cdot\rangle_x$ for simplicity. The induced norm of this inner product is denoted by $\|\cdot\|$ with the subscript being omitted. Given $f \in \mathfrak{F}_{x}(\mathcal{M})$, the gradient of $f$ at $x$, denoted by $\operatorname{grad} f(x)$, is defined as the unique tangent vector that satisfying $ \langle\operatorname{grad} f(x), \xi\rangle:=\xi_{x} f$ for all $\xi \in T_{x} \mathcal{M}$.
The length of a curve $\gamma:[a, b] \rightarrow \mathcal{M}$ on a Riemannian manifold is defined by
$L(\gamma)=\int_{a}^{b} \sqrt{\langle\dot{\gamma}(t), \dot{\gamma}(t)\rangle} \mathrm{d} t$, and the Riemannian distance on $\mathcal{M}$ is given by $d(y, z):=\inf _{\gamma\in\Gamma} L(\gamma)$, where $\Gamma$ represents the set of all curves in $\mathcal{M}$ joining points $y$ and $z$. Then the set $\{y \in \mathcal{M} \mid d(x, y)<\delta\}$ is a neighborhood of $x$ with radius $\delta>0$. A geodesic is a curve on $\mathcal{ M }$ which locally minimizes the arc length. For every $\xi \in T_{x} \mathcal{M}$, there exists an interval $\mathcal{I}$ containing zero and a unique geodesic $\gamma(\cdot ; x, \xi): \mathcal{I} \rightarrow \mathcal{M}$ such that $\gamma(0)=x$ and $\dot{\gamma}(0)=\xi$. The mapping
$\operatorname{Exp}_{x}: T_{x} \mathcal{M} \rightarrow \mathcal{M}$, $\xi \mapsto \operatorname{Exp}_{x} (\xi)=\gamma(1 ; x, \xi)
$
is called the exponential mapping on $x \in \mathcal{M}$. A vector field $X$ is parallel along a smooth curve $\gamma$ if $\nabla_{\dot{\gamma}} X=0$. where $\nabla$ is the Riemannian connection on $\mathcal{M}$.
Given a smooth curve $\gamma$ and $\eta \in T_{\dot{\gamma}(0)} \mathcal{M}$, there exists a unique parallel vector field $X_\eta$ along $\gamma$ such that $X_\eta(0)=\eta$. We define the parallel transport along $\gamma$ to be $P_\gamma^{0 \rightarrow t} \eta:=X_\eta(t)$. When the geodesic from $p$ to $q$ is unique, denoted by $\gamma_{p q}$, we define $P_{p q}:=P_{\gamma_{p q}}^{0 \rightarrow 1}$.
A retraction on a manifold $\mathcal{M}$ is a smooth mapping $R$ from the tangent bundle $T \mathcal{M}$ onto $\mathcal{M}$ satisfying $R_x\left(0_x\right)=x$ and $
D R_x\left(0_x\right)=\mathrm{id}_{T_x \mathcal{M}}$. Let $R_x$ denote the restriction of $R$ to $T_x \mathcal{M}$. The Riemannian Hessian of $f \in \mathfrak{F}_x(\mathcal{M})$ at a point $x$ in $\mathcal{M}$ is defined as the (symmetric) linear mapping $\operatorname{Hess}f(x)$ from $T_x \mathcal{M}$ into itself satisfying $\operatorname{Hess}f(x) \xi=\nabla_{\xi} \operatorname{grad} f(x)$ for all $\xi \in T_x \mathcal{M}$. By \cite[Proposition 5.5.6]{AMS09}, if $ \operatorname{grad}f(x)=0$, then
\begin{equation}\label{eq:hess-tang-mani}
\operatorname{Hess} f(x)=\operatorname{Hess}\left(f \circ R_x\right)\left(0_x\right).
\end{equation}
Suppose that a subset $\mathcal{A} \subset \mathcal{M}$. The epigraph of a function $F: \mathcal{A} \rightarrow (-\infty,\infty]$ is defined as $ \operatorname{epi} F:=\{(x, \alpha) \in \mathcal{A} \times \mathbb{R} \mid F(x) \leq \alpha\} $. A proper function $F: \mathcal{A} \rightarrow (-\infty,\infty]$ is called lower semi-continuous (lsc) if $\operatorname{epi} F$ is closed. When considering the manifold $\mathcal{M}$ as a topology space, we can use the classical definition of lower semicontinuity of a function $F:\mathcal{M}\to (-\infty,\infty]$, that for any $x\in \mathcal{M}$, it holds that $\liminf _{y \rightarrow x} f(y) \geq f(x)$. It is not difficult to verify that this definition is equivalent to the epigraph definition if we consider $\mathcal{M}$ itself as a closed set of the topology space. The following definition of the Fr\'{e}chet subdifferential is taken from \cite[Corollary 4.5]{AFL05}.
\begin{definition}\label{def:subdiff-frechet}
Let $f: \mathcal{M} \rightarrow\mathbb{R}$ be a function defined on a Riemannian manifold, and $(\mathcal{U},\varphi)$ is a chart of $\mathcal{M}$. The Fr\'{e}chet subdifferential $\partial f(x)$ of $f$ at a point $x \in \operatorname{dom} f=\{x \in M: f(x)<\infty\}$ is defined as
$$
\begin{aligned}
\partial f(x) & =\big\{D \varphi(x)\zeta \mid \zeta \in \mathbb{R}^n, \liminf _{v \rightarrow 0} \frac{f \circ \varphi^{-1}(\varphi(x)+v)-f(x)-\langle\zeta, v\rangle}{\|v\|} \geq 0\big\} \\
& =\big\{D \varphi(x)\zeta \mid \zeta \in \partial\left(f \circ \varphi^{-1}\right)(\varphi(x))\big\}.
\end{aligned}
$$
\end{definition}
From this definition, one may deduce that $\partial f(x)=\partial\left(f \circ R _x\right)\left(0_x\right)$ for any given retraction $R$.
With the distance function $d(\cdot,\cdot)$ defined above, the Lipschitz property of functions can be extended to manifold. A function $f:\mathcal{M}\to\mathbb{R}$ is Lipschitz of rank $L>0$ in a set $\mathcal{U}$ if for any $y, z \in \mathcal{U}$, $|f(y)-f(z)| \leq Ld(y, z)$. If there exists a neighborhood $\mathcal{U}$ of $x\in\mathcal{ M}$ such that $f$ is Lipschitz of rank $L$ on $\mathcal{U}$, we say that $f$ is Lipschitz of rank $L$ at $x$; if for every $x\in\mathcal{ M}$, $f$ is Lipschitz of rank $L$ at $x$ for some $L>0$, then $f$ is said to be locally Lipschitz on $\mathcal{ M }$. The generalized directional derivative of a locally Lipschitz function $f$ at $x \in \mathcal{M}$ in the direction $v \in T_{x} \mathcal{M}$, is defined in \cite{HP11} as
$$
f^{\circ}(x ; v):=\limsup _{y \rightarrow x, t \downarrow 0} \dfrac{f \circ \varphi^{-1}(\varphi(y)+t D \varphi(x)v)-f \circ \varphi^{-1}(\varphi(y))}{t},
$$
where $(\mathcal{U}, \varphi)$ is a chart containing $x$. The definition of the generalized directional derivative implies that $f^{\circ}(x,v)=\left(f\circ R_x\right)^{\circ}(0_x,v)$ for any retraction.
The Clarke subdifferential of a locally Lipschitz function $f$ at $x \in \mathcal{M}$, denoted by $\partial_{C} f(x)$, is defined as
$$
\partial_{C} f(x)=\left\{\xi \in T_{x} \mathcal{M}\mid\langle\xi, v\rangle \leq f^{\circ}(x ; v) \text { for all } v \in T_{x} \mathcal{M}\right\}.
$$
According to \cite[Proposition 2.5]{HP11}, an equivalence can be established between the Clarke subdifferential of $f$ and the Clarke subdifferential of $f \circ R$ for any retraction $R$, as described below.
\begin{proposition}\label{pro:subdiff-mani-tangent}
Let $\mathcal{ M }$ be a Riemannian manifold and $x \in \mathcal{ M }$. Suppose that $f: \mathcal{ M }\to \mathbb{R}$ is Lipschitz near $x$ and $(\mathcal{U}, \varphi)$ is a chart at $x$. Then
$$
\partial_{C} f(x)=D \varphi(x)\left[\partial_{C}\left(f \circ \varphi^{-1}\right)(\varphi(x))\right].
$$
Therefore, we have $\partial_{C} f(x)=\partial_{C}\left(f \circ R _x\right)\left(0_x\right)$ for any retraction $R_x$.
\end{proposition}
As mentioned in the introduction, a useful tool for analyzing the local convergence of ALM is the variational sufficient condition. We first introduce the variational convexity proposed in \cite{R22}.
\begin{definition}\label{def:vari-conv}
Given a lsc function $F: \mathbb{R}^n \rightarrow(-\infty, \infty]$, the variational convexity of $F$ with respect to a pair $(\bar{w}, \bar{z}) \in \operatorname{gph}\partial F$ is said to hold if there exist open convex neighborhoods $\mathcal{W}$ of $\bar{w}$ and $\mathcal{Z}$ of $\bar{z}$ such that
there exists a proper lsc convex function $h \leq F$ on $\mathcal{W}$ such that
$$[\mathcal{W} \times \mathcal{Z}] \cap \operatorname{gph} \partial h=[\mathcal{W} \times \mathcal{Z}] \cap \operatorname{gph} \partial F$$
and, for $(w, z)$ belonging to this common set, $h(w)=F(w)$. If $h$ is strongly convex on $\mathcal{W}$, we say that $F$ is variationally strongly convex with respect to $(\bar{w},\bar{z})$.
\end{definition}
%If given
For a Euclidean optimization problem,
the variational sufficient condition is said to hold at a first-order stationary point if the augmented perturbed objective function $\varphi^{\rho}$ is variationally convex, with $\varphi^{\rho}(x, u)=\varphi(x, u)+\frac{\rho}{2}\|u\|^2$. The strong variational sufficient condition holds if $\varphi^{\rho}$ is variationally strongly convex.
\section{The localization problems and the Lagrangian functions}\label{sec:equivalence}
% We are now considering the general optimization problem on Riemannian manifold as follow:
% \begin{equation}\label{eq:problem-mani}
% \begin{array}{ll}
% \min & f(x)+\theta(g(x)) \\
% \text { s.t. }
% & x\in \mathcal{M},
%\end{array}
% \end{equation}
%where $\mathcal{M}$ is a connected complete Riemannian manifold, $f:\mathcal{M} \rightarrow \mathbb{R}$ and $g:\mathcal{M} \rightarrow \mathbb{Y}$ are continuously differentiable functions, $\mathbb{Y}$ is a Euclidean space equipped with a scalar
%product $\langle\cdot, \cdot\rangle$ and its induce norm $\|\cdot\|$, $\theta : \mathbb{Y}\rightarrow \overline{\mathbb{R}}$ is a proper closed convex function. If $\theta$ is an indicator function of a closed convex set, then (\ref{eq:problem-mani}) is equivalent to a constrained manifold optimization problem.
At a given point $x\in\mathcal{M}$, the optimization problem (\ref{eq:problem-mani}) can be locally transformed into an equivalent optimization problem on the tangent space $T_x\mathcal{M}$ at $x$ by employing the retraction $R_x : T_x\mathcal{M}\to \mathcal{M}$. This is made possible by the inverse function theorem \cite[Theorem 4.5]{L13}, which establishes that any retraction is a diffeomorphism within a neighborhood of $0_x$ in the tangent space $T_x\mathcal{M}$ for a general Riemannian manifold. The following definition given in \cite{HU17} defines the injectivity radius of a Riemannian manifold with respect to retraction, which aligns with the classical definition of injectivity radius \cite[Definition 10.19]{B20}, when the retraction is chosen as the exponential mapping.
\begin{definition} \label{def:inject-radius}
The injectivity radius of manifold $\mathcal{M}$ at a point $x$ with respect to retraction $R_x$, denoted by $r_{R_x}$, is the supremum over radii $r>0$ such that $R_x$ is defined and is a diffeomorphism on the open ball $B_x(r)=\left\{v \in \mathrm{T}_x \mathcal{M}:\|v\|<r\right\}$. By the inverse function theorem, $r_{R_x}>0$.
\end{definition}
Additionally, as outlined in \cite[Proposition 10.22]{B20}, we have $d(x, y)=\|v\|$ within the ball $B_x(r_{\operatorname{Exp}_x})$ if $y=\operatorname{Exp}_x(v)$.
% \begin{proposition}
%If $\|v\|<r_{\operatorname{Exp}_x}(x)$, the geodesic $c(t)=\operatorname{Exp}_x(t v)$ on the interval $[0,1]$ is the minimizing curve connecting $x$ to $y=\operatorname{Exp}_x(v)$. In particular, $d(x, y)=\|v\|_x$.
% \end{proposition}
To construct a problem that is locally equivalent to (\ref{eq:problem-mani}), we begin by introducing the following function defined on the tangent space at a given point $x\in\mathcal{M}$. Definition~\ref{def:function-tangent} on tangent space is different from the pullback function defined in \cite[Section 4]{AMS09}, where the latter is formulated as $f\circ R$ at each point on the manifold.
\begin{definition}\label{def:function-tangent}
Let $x\in\mathcal{M}$ and $r_{R_x}$ be the injectivity radius of $\mathcal{M}$ at $x$ with respect to $R_x$. For a given function $F:\mathcal{M}\to\mathbb{R}$, we define $F_{R_x}:T_x\mathcal{M}\to(-\infty,\infty]$ by
\begin{equation}\label{eq:function-tangent}
F_{R_x}(\xi)=\left\{\begin{array}{cl}
F\left(R_x \xi\right), & \xi \in B_x(r_{R_x}), \\
+\infty, & \xi \notin B_x(r_{R_x}).
\end{array}\right.
\end{equation}
\end{definition}
%\begin{remark}\label{rm:pullback}
% This definition on tangent space is different from the pullback function defined in \cite[Section 4]{AMS09}, where the latter is formulated as $f\circ R$ at each point on the manifold.
%\end{remark}
It is worth noting that the lower semicontinuity of $F$ is inherited by $F_{R_x}$ within the ball $B_x(r_{R_x})$. However, this continuity may not hold on %extend to
its boundary. Fortunately, this limitation does not affect %pose any issues for
our discussions since our focus is solely on the properties of $F_{R_x}$ within the ball $B_x(r_{R_x})$.
For a given point $\bar{x}$ and a retraction $R_{\bar{x}}$, applying (\ref{eq:function-tangent}) to $f$ and $g$, we obtain the following problem on tangent space $T_{\bar{x}}\mathcal{M}$ as
\begin{equation}\label{eq:problem-tangent}
\begin{array}{ll}
\min & f_{R_{\bar{x}}}(\xi)+\theta(g_{R_{\bar{x}}}(\xi)) \\
\text { s.t. }
& \xi\in T_{\bar{x}}\mathcal{M},
\end{array}
\end{equation}
where $f_{R_{\bar{x}}}$ and $g_{R_{\bar{x}}}$ are defined by (\ref{eq:function-tangent}). By Definitions \ref{def:inject-radius} and \ref{def:function-tangent}, problem \eqref{eq:problem-tangent} is locally equivalent to problem \eqref{eq:problem-mani} in the injectivity ball $B_{\bar{x}}(r_{R_{\bar{x}}})$ as we can build the one-to-one relationship $x = R_{\bar{x}}(\xi)$ for any $\xi\in B_{\bar{x}}(r_{R_{\bar{x}}})$.
%However, {\color{green}due to the function value becoming infinite outside this neighborhood, are justified in disregarding the non-equivalent part.}
The perturbed problem for (\ref{eq:problem-tangent}) can be written as
\begin{equation}\label{eq:perturb-problem-tangent}
\begin{array}{ll}
\min & \varphi_{R_{\bar{x}}}(\xi,u):= f_{R_{\bar{x}}}(\xi)+\theta(g_{R_{\bar{x}}}(\xi)+u) \\
\text { s.t. } & \xi\in T_{\bar{x}}\mathcal{M},
\end{array}
\end{equation}
when $u=0$ this is problem (\ref{eq:problem-tangent}). This problem is also locally equivalent to the manifold perturbed problem \eqref{eq:perturb-problem-mani} insider the ball $B_{\bar{x}}(r_{R_{\bar{x}}})$.
%For the corresponding $x \in \mathcal{M}$ for problem (\ref{eq:perturb-problem-mani}), we can define the critical cone $\mathcal{C}(x)$ of (\ref{eq:perturb-problem-mani}) at $x$ as
%\begin{equation}
%\mathcal{C}(x):=\left\{\xi \in T_x \mathcal{M} \mid Df(x)\xi+(\theta\circ g_1)^{\circ}(x ; \xi) \leq 0\right\}.
%\end{equation}
The Lagrangian function for (\ref{eq:problem-tangent}) is defined by
$$
l_{R_{\bar{x}}}(\xi, y)=\inf _u\{\varphi_{R_{\bar{x}}}(\xi, u)-\left\langle y ,u\right\rangle \}=L_{R_{\bar{x}}}(\xi , y)-\theta^*(y),
$$
where $L_{R_{\bar{x}}}(\xi, y):=f_{R_{\bar{x}}}(\xi)+\left\langle y, g_{R_{\bar{x}}}(\xi)\right\rangle $. For $\rho > 0$, the augmented Lagrangian function of (\ref{eq:problem-tangent}) is given by
\begin{equation}\label{eq:al-tangent}
l_{R_{\bar{x}}}^{\rho}(\xi, y)=\inf _u\big\{\varphi_{R_{\bar{x}}}(\xi, u)-\left\langle y ,u\right\rangle +\frac{\rho}{2}\|u\|^2\big\}.
\end{equation}
The augmented Lagrangian functions (\ref{eq:al-mani}) and (\ref{eq:al-tangent}) can be regraded as the Lagrangian functions of the augmented objective functions $\varphi^{\rho}(x, u)=\varphi(x, u)+\frac{\rho}{2}\|u\|^2$ and $\varphi_{R_{\bar{x}}}^{\rho}(\xi, u)=\varphi_{R_{\bar{x}}}(\xi, u)+\frac{\rho}{2}\|u\|^2$, respectively. Thus by definition, $-l^{\rho}(x,\cdot)$ and $-l_{R_{\bar{x}}}^{\rho}(\xi,\cdot)$ are the conjugate functions of $\varphi^{\rho}(x, \cdot)$ and $\varphi_{R_{\bar{x}}}^{\rho}(\xi, \cdot)$, respectively. Moreover, the lower semicontinuity of $\theta$ implies that $\varphi^{\rho}(x, \cdot)$ and $\varphi_{R_{\bar{x}}}^{\rho}(\xi, \cdot)$ are closed functions of $u$. Therefore, by \cite[12.2]{R70} we have
$$
\begin{array}{lcc}
&\varphi(x, u)=\sup _y\{l(x, y)+\left\langle y ,u\right\rangle\}, \;&\varphi^{\rho}(x, u)=\sup _y\left\{l^{\rho}(x, y)+\left\langle y ,u\right\rangle\right\}\\
\text{and}\quad & \varphi_{R_{\bar{x}}}(\xi, u)=\sup _y\{l_{R_{\bar{x}}}(\xi, y)+\left\langle y ,u\right\rangle\}, \;&\varphi_{R_{\bar{x}}}^{\rho}(\xi, u)=\sup _y\{l_{R_{\bar{x}}}^{\rho}(\xi, y)+\left\langle y ,u\right\rangle\}.
\end{array}
$$
For a given $\bar{x}\in\mathcal{M}$, we say $\bar{x}$ is a stationary point of the perturbed problem (\ref{eq:perturb-problem-mani}), if there exists $\bar{y}\in\mathbb{Y}$ such that
\begin{equation}\label{eq:first-opt-mani}
(0, \bar{y}) \in \partial \varphi(\bar{x}, 0).
\end{equation}
The following proposition elucidates the relationships between the first-order conditions for problems (\ref{eq:perturb-problem-mani}) and (\ref{eq:perturb-problem-tangent}).
\begin{proposition}\label{pro:first-order-cond}
Given $(\bar{x},\bar{y})$ and a retraction $R_{\bar{x}}$, the following statements are equivalent:
\begin{itemize}
\item [(i)] $(\bar{x},\bar{y})$ satisfies condition \eqref{eq:first-opt-mani} of problem (\ref{eq:perturb-problem-mani});
\item [(ii)] $(0_{\bar{x}},\bar{y})$ satisfies the following condition of problem (\ref{eq:perturb-problem-tangent}), i.e., %which is
\begin{equation}\label{eq:first-opt-tangent}
(0, \bar{y}) \in \partial \varphi_{R_{\bar{x}}}(0_{\bar{x}}, 0);
\end{equation}
\item [(iii)]For any $\rho>0$, $(\bar{x},\bar{y})$ satisfies $ (0, \bar{y}) \in \partial \varphi^{\rho}(\bar{x}, 0)$;
\item [(iv)]For any $\rho>0$, $(0_{\bar{x}},\bar{y})$ satisfies $(0_{\bar{x}},\bar{y})\in \partial \varphi_{R_{\bar{x}}}^{\rho}(0_{\bar{x}}, 0)$;
\item [(v)] $\operatorname{grad}_xl(\bar{x},\bar{y})=0$, $0\in\partial_y [-l](\bar{x},\bar{y})$, or $\operatorname{grad}_xL(\bar{x},\bar{y})=0$, $\bar{y}\in\partial \theta(g(\bar{x}))$;
\item [(vi)] $\nabla_{\xi}l_{R_{\bar{x}}}(0_{\bar{x}},\bar{y})=0$, $0\in\partial_y [-l_{R_{\bar{x}}}](0_{\bar{x}},\bar{y})$, or $\nabla_{\xi}L_{R_{\bar{x}}}(0_{\bar{x}},\bar{y})=0$, $\bar{y}\in\partial \theta(g_{R_{\bar{x}}}(0_{\bar{x}}))$ ;
\item [(vii)] $\operatorname{grad}_xl^{\rho}(\bar{x},\bar{y})=0$, $0\in\nabla_y l^{\rho}(\bar{x},\bar{y})$, or $\operatorname{grad}_xL(\bar{x},\bar{y})=0$, $\nabla \operatorname{env}_{\rho}\theta(g(\bar{x})+\rho^{-1}\bar{y})=\bar{y}$, where $\operatorname{env}_{\rho}\theta$ is the Moreau-Yosida regularization of $\theta$ defined by %that
$\operatorname{env}_{\rho}\theta(p):=\min_{y \in \mathbb{Y}} \theta(y)+\dfrac{\rho}{2}\|p-y\|^2$, for any $p$;
\item [(viii)] $\nabla_{\xi}l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},\bar{y})=0$, $0\in\nabla_y l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},\bar{y})$, or $\nabla_{\xi}L_{R_{\bar{x}}}(0_{\bar{x}},\bar{y})=0$, $\nabla \operatorname{env}_{\rho}\theta(g_{R_{\bar{x}}}(0_{\bar{x}})+\rho^{-1}\bar{y})=\bar{y}$.
\end{itemize}
\end{proposition}
\begin{proof}
The equivalence of (\ref{eq:first-opt-mani}) and (\ref{eq:first-opt-tangent}) is obtained by using Proposition \ref{pro:subdiff-mani-tangent}. While taking $u=0$, it is obvious that $\partial \varphi(\bar{x}, u)=\partial \varphi^{\rho}(\bar{x}, u)$ and $\partial \varphi_{R_{\bar{x}}}^{\rho}(0_{\bar{x}}, 0)=\partial \varphi_{R_{\bar{x}}}(0_{\bar{x}}, 0)$, which implies that $(i)\Longleftrightarrow(ii)\Longleftrightarrow(iii)\Longleftrightarrow(iv)$. By the chain rules of the subdifferential of manifold functions \cite[Proposition 1]{ZBD22}, we have
$$
\begin{aligned}
(v, y) \in \partial \varphi(x, u) & \Longleftrightarrow y \in \partial \theta(g(x)+u), \; v=\operatorname{grad}_x L(x, y) \\
& \Longleftrightarrow v \in \partial_x l(x, y), \; u \in \partial_y[-l](x, y).
\end{aligned}
$$
Moreover, Proposition \ref{pro:subdiff-mani-tangent} yields
$$
\begin{aligned}
(v, y) \in \partial \varphi(x, u) \Longleftrightarrow(v, y) \in \partial \varphi_{R_x}(0_x, u) & \Longleftrightarrow y \in \partial \theta(g_{R_x}(0_x)+u), \; v=\nabla_{\xi} l_{R_x}(0_x, y) \\
& \Longleftrightarrow v \in \partial_\xi l_{R_x}(0_x, y), \; u \in \partial_y[-l_{R_x}](0_x, y).
\end{aligned}
$$
Similarly, we can prove the equivalence between (i), (vii) and (viii).
% $$
% \begin{aligned}
% (v, y) \in \partial \varphi^{\rho}(x, u) &\Longleftrightarrow v=\operatorname{grad}_x l^{\rho}(x, y), \quad u=-\nabla_y l^{\rho}(x, y)\\ &\Longleftrightarrow
% v=\operatorname{grad}_x L(x, \eta) \text { and } u=\rho^{-1}(\eta-y) \text { for } \eta=\nabla \operatorname{env}_{\rho}\theta\left(g(x)+\rho^{-1} y\right),
% \end{aligned}
% $$
% and
% $$
% \begin{aligned}
% (v, y) \in \partial \varphi^{\rho}(x, u)&\Longleftrightarrow(v, y) \in \partial \varphi_{R_x}^{\rho}(0_x, u)\\
% &\Longleftrightarrow v=\nabla_{\xi} l_{R_x}^{\rho}(0_x, y), \quad u=-\nabla_y l_{R_x}^{\rho}(0_x, y)\\ &\Longleftrightarrow
% v=\nabla_{\xi} l_{R_x}(0_x, \eta) \text { and } u=\rho^{-1}(\eta-y) \text { for } \eta=\nabla \operatorname{env}_{\rho}\theta\left(g_{R_x}(0_x)+\rho^{-1} y\right).
% \end{aligned}
% $$
% Therefore, by taking $u=0$, the equivalence relationships hold for $(i)$, $(v)$, $(vi)$, $(vii)$ and $(viii)$.
\end{proof}
\section{Manifold variational sufficient condition}\label{sec:variational suff}
%In this section, we %are preparing to
%analyze the local convergence of Riemannian ALM without requiring the multipliers to be unique.
As mentioned in the introduction, the variational sufficient condition is closely related to the local maximal monotonicity of the augmented objective function and is crucial for the linear convergence analysis of ALM in Euclidean settings. In the Riemannian case, we first define the manifold (strong) variational sufficient condition through $\varphi^{\rho}_{R_{\bar{x}}}$ as follows.
\begin{definition}\label{def:vari-suff-mani}
The manifold (strong) variational sufficient condition for local optimality in (\ref{eq:perturb-problem-mani}) under retraction $R_{\bar{x}}$ is said to hold with respect to $(\bar{x},\bar{y})$ satisfying condition \eqref{eq:first-opt-mani} if the (strong) variational sufficiency condition holds for problem (\ref{eq:perturb-problem-tangent}) under $R_{\bar{x}}$.
% The manifold strong variational sufficient condition for (\ref{eq:perturb-problem-mani}) is said to hold if the strong variational sufficient condition for (\ref{eq:perturb-problem-tangent}) holds.
\end{definition}
We will show in Section \ref{subsection:SVS-charct} that this definition of manifold strong variational sufficiency is actually independent of the choice of retraction $R_{\bar{x}}$.
\subsection{Local augmented duality}
Inspired by \cite{R22} and \cite{R22-ext}, we will discuss the local augmented dual property for problem \eqref{eq:perturb-problem-mani} in this section. The following proposition on the equivalence between the manifold (strong) variational sufficiency and the augmented Lagrangian (strong) convexity follows directly from \cite[Theorem 1]{R22} and Proposition \ref{pro:subdiff-mani-tangent}.
\begin{proposition}\label{pro:vari-suff-al-conv}
Let $\bar{x}$ and $\bar{y}$ satisfy the condition (\ref{eq:first-opt-mani}). Given a retraction $R_{\bar{x}}$, the manifold (strong) variational sufficient condition with respect to $(\bar{x},\bar{y})$ for problem (\ref{eq:perturb-problem-mani}) holds at level $\bar{\rho}$ if and only if, there is a closed convex neighborhood $\mathcal{W} \times \mathcal{Y}$ of $(0_{\bar{x}}, \bar{y})$ such that $l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y)$ is locally (strongly) convex at $0_{\bar{x}}$ when $y \in \mathcal{Y}$ and concave in $y \in \mathcal{Y}$ when $\xi \in \mathcal{W}\subseteq B_{\bar{x}}(r_{R_{\bar{x}}})$. Then $(\bar{x}, \bar{y})$ is a saddle point of $l^{\bar{\rho}}(x, y)$ with respect to minimizing in $x \in R_{\bar{x}}(\mathcal{W})$ and maximizing in $y \in \mathcal{Y}$. Moreover, $l^{\rho}_{R_{\bar{x}}}(\xi, y)$ for every $\rho \geq \bar{\rho}$ enjoys those properties and $l^{\rho}(x, y)$ has $(\bar{x}, \bar{y})$ as a saddle point relative to $R_{\bar{x}}(\mathcal{W}) \times \mathcal{Y}$.
\end{proposition}
\begin{proof}
By applying \cite[Theorem 1]{R22} and Proposition \ref{pro:subdiff-mani-tangent} to (\ref{eq:perturb-problem-tangent}) at $(0_{\bar{x}},\bar{y})$, $l_{R_{\bar{x}}}^{\bar{\rho}}(\xi, y)$ is convex in $\xi \in \mathcal{W}$ when $y \in \mathcal{Y}$ as well as concave in $y \in \mathcal{Y}$ when $\xi \in \mathcal{W}$. %We are left
It remains to show that $(\bar{x}, \bar{y})$ is a saddle point of $l^{\bar{\rho}}(x, y)$ in $R_{\bar{x}}(\mathcal{W})\times\mathcal{Y}$, or $l^{\bar{\rho}}(x, y)$ attains its minimum at $\bar{x}$ in $R_{\bar{x}}(\mathcal{W})$. This is shown by
$$
l^{\bar{\rho}}\left( \bar{x},\bar{y}\right) =l_{R_{\bar{x}}}^{\bar{\rho}}\left( 0_{\bar{x}},\bar{y}\right) \leq l_{R_{\bar{x}}}^{\bar{\rho}}\left( R_{\bar{x}}^{-1}(x),\bar{y}\right) =l^{\bar{\rho}}(x,\bar{y})\quad\forall\; x\in R_{\bar{x}}(\mathcal{W}).
$$
Hence we complete the proof.
\end{proof}
If we require the strong variational sufficiency condition to be satisfied at $(0_{\bar{x}},\bar{y})$ for problem (\ref{eq:perturb-problem-tangent}) at level $\bar{\rho}$, by \cite[Theorem 2]{R22}, the augmented tilt stability holds at $0_{\bar{x}}$ for problem (\ref{eq:perturb-problem-tangent}). Additionally, we define the augmented tilt stability on manifold with respect to retraction.
\begin{definition}\label{def:augmented-tilt-stab-mani}
The manifold augmented tilt stability is said to hold at $\bar{x}\in\mathcal{M}$ with respect to retraction $R_{\bar{x}}$ if there is a neighborhood $\mathcal{V}$ of $0_{\bar{x}}$ such that the mapping
$$\small
(v, y) \mapsto \underset{x \in R_{\bar{x}}\left( \mathcal{W}\right) }{\operatorname{argmin}}\left\{l^{\rho}(x, y)-\left\langle v, R_{\bar{x}}^{-1}(x)\right\rangle \right\} \;\text{for}\;(v, y) \in \mathcal{V} \times \mathcal{Y}
$$
is single-valued and Lipschitz continuous. Here, $\mathcal{W}$ is the neighborhood defined in Definition \ref{def:vari-conv}.
\end{definition}
The next proposition is an augmented tilt characterization of manifold strong variational sufficiency.
\begin{proposition}\label{pro:vari-strong-suff-al-strong-conv}
Suppose the manifold strong variational sufficient condition holds at $(\bar{x},\bar{y})$ with respect to $R_{\bar{x}}$ for local optimality of (\ref{eq:perturb-problem-mani}) at level $\bar{\rho}$, then the manifold augmented tilt stability holds at $\bar{x}$ with respect to $R_{\bar{x}}$.
\end{proposition}
\begin{proof}
For $(v,y)\in\mathcal{V}\times\mathcal{Y}$, denote
$$h(v,y):=\underset{x \in R_{\bar{x}}\left( \mathcal{W}\right) }{\operatorname{argmin}}\left\{l^{\rho}(x, y)-\left\langle v, R_{\bar{x}}^{-1}(x)\right\rangle \right\}\quad \mbox{and} \quad h_{R_{\bar{x}}}(v, y) :=\underset{\xi \in \mathcal{W} }{\operatorname{argmin}}\left\{l_{R_{\bar{x}}}^{\rho}(\xi, y)-\left\langle v, \xi\right\rangle \right\}.$$
If $x\in h(v,y)$, then $R_{\bar{x}}^{-1}(x)$ is a minimizer of $l_{R_{\bar{x}}}^{\rho}(\xi, y)-\left\langle v, \xi\right\rangle$, implying that $h(v,y)\subseteq R_{\bar{x}}h_{R_{\bar{x}}}(v, y)$. The converse relation is also true, hence $h(v,y)=R_{\bar{x}} h_{R_{\bar{x}}}(v, y)$. Therefore, the single-valued and Lipschitz continuous properties of $h(v,y)$ is equivalent to $h_{R_{\bar{x}}}(v,y)$ being %have
single-valued and Lipschitz continuous. Thus, the augmented tilt stability will hold for $\bar{x}$ and $0_{\bar{x}}$ simultaneously. %Now
By \cite[Theorem 2]{R22}, we obtain the conclusion.
\end{proof}
In \cite{R22-ext}, the convergence of ALM is proved by applying local PPA to the local dual problem and using the convergence of PPA. Now by Propositions \ref{pro:vari-suff-al-conv} and \ref{pro:vari-strong-suff-al-strong-conv}, we are able to establish the local augmented dual problem for problem (\ref{eq:problem-mani}) under the manifold (strong) variational sufficiency. Assume that the manifold variational sufficient condition holds at the stationary point $(\bar{x},\bar{y})$ under retraction $R_{\bar{x}}$. Denote $\mathcal{S}$ as the set of all $(x,y)$ satisfying condition (\ref{eq:first-opt-mani}). %condition in Proposition \ref{pro:first-order-cond}.
By Proposition \ref{pro:vari-suff-al-conv}, there is a closed convex neighborhood $\mathcal{W} \times \mathcal{Y}$ of $(0_{\bar{x}}, \bar{y})$ such that $l_{R_{\bar{x}}}^{\bar{\rho}}(\xi, y)$ is convex in $\xi \in \mathcal{W}$ when $y \in \mathcal{Y}$ as well as concave in $y \in \mathcal{Y}$ when $\xi \in \mathcal{W}$, and $(\bar{x},\bar{y})$ is a saddle point of $l^{\bar{\rho}}(x,y)$ as well as $(0_{\bar{x}},\bar{y})$ is the saddle point of $l_{R_{\bar{x}}}^{\bar{\rho}}(\xi,y)$.
The choice of $\mathcal{W}$ and $\mathcal{Y}$ ensures that $\mathcal{S}\cap\left[\operatorname{int}R_{\bar{x}}(\mathcal{W})\times\operatorname{int}\mathcal{Y} \right]\neq \emptyset$.
%The convex-concave saddle points property of $l_{R_{\bar{x}}}^{\bar{\rho}}(\xi,y)$ corresponds to the local duality property that
%\begin{itemize}
% \item [(i)]$0_{\bar{x}}$ minimizes over $\xi \in \mathcal{W}$ the convex function $\sup_{y \in \mathbb{Y}} l_{R_{\bar{x}}}^{\bar{\rho}}(\xi,y)$;
% \item[(ii)] $\bar{y}$ maximizes over $y \in \mathbb{Y}$ the concave function $\inf_{\xi \in \mathcal{W}} l_{R_{\bar{x}}}^{\bar{\rho}}(\xi,y)$;
% \item[(iii)] the optimal values in these paired problems are equal.
%\end{itemize}
%Meanwhile, we are able to establish the local duality for manifold optimization problem as follows:
%\begin{itemize}
% \item [(i)]$\bar{x}$ minimizes over $x \in R_{\bar{x}}(\mathcal{W})$ the function $\sup_{y \in \mathbb{Y}} l^{\bar{\rho}}(x, y)$;
% \item[(ii)] $\bar{y}$ maximizes over $y \in \mathbb{Y}$ the concave function $\inf_{x \in R_{\bar{x}}(\mathcal{W})} l^{\bar{\rho}}(x, y)$;
% \item[(iii)] the optimal values in these paired problems are equal.
%\end{itemize}
To align % meet
with the traditional convex analysis of the primal and dual problems, we define the local perturbed objective function as in \cite{R22-ext}:
%\begin{equation}\label{eq:local-obj-tangent}
% \begin{array}{ll}
% \Phi_{R_{\bar{x}}}(\xi, u)=\sup_{y \in \mathbb{Y}}\left\{l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y)+\left\langle y , u\right\rangle \right\} \text { for } \xi \in \mathcal{W}, & \Phi_{R_{\bar{x}}}(\xi, u)=\infty \text { for }\xi \notin \mathcal{W}, \\
% \Psi_{R_{\bar{x}}}(v, y)=\inf_{\xi \in \mathcal{W}}\left\{l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y)-\left\langle v ,\xi \right\rangle \right\} \text { for } y \in \mathbb{Y}, & \Psi_{R_{\bar{x}}}(v, y)=-\infty \text { for } y \notin \mathbb{Y},
% \end{array}
%\end{equation}
%and
$$
\begin{array}{ll}
\Phi(x, u)=\sup_{y \in \mathcal{Y}}\left\{l^{\bar{\rho}}(x, y)+\left\langle y , u\right\rangle \right\} \;\text {for}\; x\in R_{\bar{x}}(\mathcal{W}), & \Phi(x, u)=\infty \;\text {for}\;x \notin
R_{\bar{x}}(\mathcal{W}), \\
\Psi(v, y)=\inf_{x \in R_{\bar{x}}(\mathcal{W})}\{l^{\bar{\rho}}(x, y)-\left\langle v ,R_{\bar{x}}^{-1}x \right\rangle \} \;\text {for}\; y \in \mathcal{Y}, & \Psi(v, y)=-\infty \;\text {for}\; y \notin \mathcal{Y}.
\end{array}
$$
The following definition establishes the local primal and dual problems for manifold optimization.
\begin{definition}
The associated local primal problem for (\ref{eq:perturb-problem-mani}) is defined as
\begin{equation}\label{eq:prime-mani-vs}
\min_{x\in R_{\bar{x}}(\mathcal{W})}\; F(x),\;\text{where}\; F(x)=\Phi(x, 0)=\sup_{y \in \mathcal{Y}}l^{\bar{\rho}}(x, y) \;\text{for}\;x\in R_{\bar{x}}(\mathcal{W}), \tag{$P$}
\end{equation}
while the local %augmented
dual problem is defined as
\begin{equation}\label{eq:dual-mani-vs}
\max_{y \in \mathcal{Y}}\; H(y),\;\text{where}\; H(y)=\Psi(0_{\bar{x}},y)=\inf_{x\in R_{\bar{x}}(\mathcal{W})} l^{\bar{\rho}}(x, y)\; \text{for}\;y \in \mathcal{Y}. \tag{$D$}
\end{equation}
\end{definition}
The primal and dual connections of the optimal values and solutions between (\ref{eq:prime-mani-vs}) and (\ref{eq:dual-mani-vs}) are given in the following theorem.
\begin{theorem}\label{thm:primal-dual-vs}
Let $(\bar{x},\bar{y})$ satisfy the condition \eqref{eq:first-opt-mani}. Suppose the manifold variational sufficient condition for local optimality holds at $(\bar{x},\bar{y})$ with respect to $R_{\bar{x}}$. Then the problems (\ref{eq:prime-mani-vs}) and (\ref{eq:dual-mani-vs}) are defined in the neighborhood $ R_{\bar{x}}(\mathcal{W}) \times \mathcal{Y}$ of $(\bar{x},\bar{y})$ and have optimal solutions with $\min (P)=\max (D)$, and
$$
\bar{x} \text { solves }(P) \quad \Longleftrightarrow \quad \bar{x} \text { minimizes in }(1.2) \text { relative to } R_{\bar{x}}(\mathcal{W}).
$$
Moreover, for a pair $(\bar{x}, y^*) \in \operatorname{int} R_{\bar{x}}(\mathcal{W}) \times \operatorname{int} \mathcal{Y}$, the following conditions
are equivalent and guarantee that $\bar{x}$ is locally optimal relative to $ R_{\bar{x}}(\mathcal{W})$ in (\ref{eq:perturb-problem-mani}) with the objective value $\varphi(\bar{x}, 0)$ agreeing with the optimal values in (\ref{eq:prime-mani-vs}) and (\ref{eq:dual-mani-vs}) as well as with $l^{\bar{\rho}}(\bar{x}, y^*)$ and $l^{\bar{\rho}}(\bar{x},\bar{y})$:
\begin{itemize}
\item [(a)] $\bar{x}$ minimizes in (\ref{eq:prime-mani-vs}) and $y^*$ maximizes in (\ref{eq:dual-mani-vs}),
\item [(b)] $(\bar{x}, y^*)$ is a saddle point of $l^{\bar{\rho}}$ on $ R_{\bar{x}}(\mathcal{W}) \times \mathcal{Y}$,
\item [(c)] $(\bar{x}, y^*)$ is a saddle point of $l^{\rho}$ on $ R_{\bar{x}}(\mathcal{W}) \times \mathcal{Y}$ for any $\rho\geq \bar{\rho}$.
\end{itemize}
\end{theorem}
\begin{proof}
Define the primal and dual problems for (\ref{eq:perturb-problem-tangent}) as follows:
$$
\min_{\xi\in\mathcal{W}}\; F_{R_{\bar{x}}}(\xi), \;\text{where}\; F_{R_{\bar{x}}}(\xi)=\Phi_{R_{\bar{x}}}(\xi, 0)=\sup_{y \in \mathcal{Y}} l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y) \text{ for }\xi\in\mathcal{W},
% \tag{$P1$}
$$
\begin{equation}\label{eq:dual-tangent}
\max_{y \in \mathcal{Y}}\; H_{R_{\bar{x}}}(y),\;\text{where}\; H_{R_{\bar{x}}}(y)=\Psi_{R_{\bar{x}}}(0_{\bar{x}},y)=\inf_{\xi\in\mathcal{W}} l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y) \text{ for }y \in \mathcal{Y}, \tag{$D1$}
\end{equation}
where $\Phi(R_{\bar{x}}\xi, u)=\Phi_{R_{\bar{x}}}(\xi, u)$ and $\Psi(v, y)=\Psi_{R_{\bar{x}}}(v, y)$.
By applying the proof of \cite[Theorem 2.1]{R22-ext} to the above problems at the point $( R_{\bar{x}}^{-1}\bar{x},y^*)$ we obtain the results.
\end{proof}
\subsection{Manifold strong variational sufficiency and second-order sufficient condition}\label{subsection:SVS-charct}
Another crucial aspect of strong variational sufficiency is its equivalence to the strong second-order sufficient condition (SSOSC) in Euclidean settings. In our case, we additionally unveil the correlation between the manifold strong variational sufficient condition and the manifold strong second-order sufficient condition (M-SSOSC).
For a differentiable function $f:\mathcal{M}\to \mathbb{R}$ with locally Lipschitz gradient and a given retraction $R_{\bar{x}}$, the Hessian bundle of $f_{R_{\bar{x}}}$ at $0_{\bar{x}}$ is defined as
$$
\overline{\nabla}^2 f_{R_{\bar{x}}}(0_{\bar{x}})=\left\{H \mid \exists \xi_k \rightarrow 0_{\bar{x}} \;\text {with}\; \nabla^{2} f_{R_{\bar{x}}}\left(\xi_k\right) \rightarrow H,\; \xi_k\in T_{\bar{x}}\mathcal{M}\right\}.
$$
Given the augmented Lagrangian function $ l_{R_{\bar{x}}}^{\rho}(\xi,y)$ of \eqref{eq:problem-tangent} %(\ref{eq:perturb-problem-tangent}),
for any matrix $H$ belonging to the Hessian bundle of $ l_{R_{\bar{x}}}^{\rho}(\xi,y)$, it can be separated into four parts as $H_{\xi\xi}$, $H_{\xi y}$, $H_{y\xi}$ and $H_{yy}$. Let $\overline{\nabla}_{\xi\xi}^2 l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},y):=\lbrace H_{\xi\xi}\mid H\in \overline{\nabla}^2 l_{R_{\bar{x}}}^{\rho}(\xi,y)\rbrace$. The critical cone of function $\theta$ and $g$ at $x$ and $y$ is defined by $\mathcal{C}_{\theta, g}(x, y):=\left\{d \in \mathbb{Y} \mid \theta^{\prime}(g(x) ; d)=\langle d, y\rangle\right\}$.
It is obvious that $\mathcal{C}_{\theta, g}(x, y)=\mathcal{C}_{\theta, g_{R_x}}(0_x, y)$ if $\mathcal{C}_{\theta, g_{R_x}}(0_x, y)=\left\{d \in \mathbb{Y} \mid \theta^{\prime}(g_{R_x}(0_x) ; d)=\langle d, y\rangle\right\}$. Using \cite[Theorem 3]{R22}, we are able to connect the following manifold strong second-order condition with the manifold strong variational sufficiency for (\ref{eq:perturb-problem-mani}).
\begin{theorem}\label{thm:strong-vari-suff-ssosc}
Let $\bar{x}$ and $\bar{y}$ satisfy the condition (\ref{eq:first-opt-mani}). The manifold strong variational sufficient condition with respect to $(\bar{x},\bar{y})$ under retraction $R_{\bar{x}}$ for (\ref{eq:perturb-problem-mani}) holds if and only if every matrix in $\overline{\nabla}_{\xi\xi}^2 l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},\bar{y})$ is positive-definite. Moreover, any $H_{\xi\xi}\in \overline{\nabla}_{\xi\xi}^2 l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},\bar{y})$ has the form of
\begin{equation}\label{eq:hess-bundle-pos-def-mani}
\operatorname{Hess}_{x} L(\bar{x}, \bar{y})+ Dg(\bar{x})^* G \operatorname{grad} g(\bar{x}) \;\text {for some}\; G \in \overline{\nabla}^2 \operatorname{env}_{\rho}\theta\left(g(\bar{x})+\rho^{-1} \bar{y}\right).
\end{equation}
If $\theta$ is a polyhedral convex function, then the manifold strong variational sufficient condition is equivalent to the following manifold strong second-order sufficient condition (M-SSOSC) at $(\bar{x},\bar{y})$:
\begin{equation}\label{eq:ssosc-mani-poly}
\left\langle\xi, \operatorname{Hess}_x L\left(\bar{x} ; \bar{y}\right) \xi\right\rangle>0 \quad \forall Dg(\bar{x})\xi \in \operatorname{aff}\mathcal{C}_{\theta,g}\left(\bar{x},\bar{y}\right) \backslash\{0\},
\end{equation}
where $\operatorname{aff}\mathcal{C}_{\theta,g}\left(\bar{x},\bar{y}\right)$ represents the affine hull of the critical cone $\mathcal{C}_{\theta,g}\left(\bar{x},\bar{y}\right)$.
Moreover, if $\theta$ is an indicator function of a second-order cone or positive semidefinite cone, then the manifold strong variational sufficient condition is equivalent to the M-SSOSC at $(\bar{x},\bar{y})$:
\begin{equation}\label{eq:ssosc-mani-cone}
\left\langle\xi, \operatorname{Hess}_x L\left(\bar{x};\bar{y} \right) \xi\right\rangle-\sigma\left(\bar{y}, \mathcal{T}_{\mathcal{K}}^2\left(g\left(\bar{x}\right), D g\left(\bar{x}\right) \xi\right)\right)>0 \quad \forall Dg(\bar{x})\xi \in \operatorname{aff}\mathcal{C}_{\theta,g}\left(\bar{x},\bar{y}\right) \backslash\{0\},
\end{equation}
where %for any $y \in \mathbb{Y}$,
$\sigma(y, \mathcal{D})$ is the support function of the set $\mathcal{D}$ at $y$ and $\mathcal{T}_{\mathcal{K}}^2(y,h)$ is the second-order tangent set of $\mathcal{D}$ at $y$ in direction $h$.
\end{theorem}
\begin{proof}
Applying \cite[Theorem 3]{R22}, all $H_{\xi\xi}\in \overline{\nabla}_{\xi\xi}^2 l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},\bar{y})$ are positive definite and of the form
$$
\nabla_{\xi\xi}^2 l_{R_{\bar{x}}}(0_{\bar{x}}, \bar{y})+ \nabla g_{R_{\bar{x}}}(0_{\bar{x}})^* G \nabla g_{R_{\bar{x}}}(0_{\bar{x}})\;\text {for some}\; G \in \overline{\nabla}^2 \operatorname{env}_{\rho}\theta\left(g_{R_{\bar{x}}}(0_{\bar{x}})+r^{-1} \bar{y}\right).
$$
Note that $\operatorname{grad}_x L(\bar{x},\bar{y})=0$. Therefore, by the definition of retraction and (\ref{eq:hess-tang-mani}), we know that the above form is equivalent to (\ref{eq:hess-bundle-pos-def-mani}). Furthermore, if $\theta$ is polyhedral convex, \cite[Theorem 4]{R22} shows that the strong variational sufficiency holds if and only if
$$
\left\langle\xi, \nabla_{\xi\xi}^2 l_{R_{\bar{x}}}\left(0_{\bar{x}}, \bar{y}\right) \xi\right\rangle>0 \quad \forall g^{\prime}_{R_{\bar{x}}}(0_{\bar{x}})\xi \in \operatorname{aff}\mathcal{C}_{\theta,g_{R_{\bar{x}}}}\left(0_{\bar{x}},\bar{y}\right) \backslash\{0\},
$$
which is equivalent to (\ref{eq:ssosc-mani-poly}) because $\nabla_{\xi\xi}^2 l_{R_{\bar{x}}}\left(0_{\bar{x}},\bar{y}\right)=\operatorname{Hess}_x L\left(\bar{x} ; \bar{y}\right)$, $g^{\prime}_{R_{\bar{x}}}(0_{\bar{x}})\xi=Dg(\bar{x})\xi$ and $\mathcal{C}_{\theta,g_{R_{\bar{x}}}}\left(0_{\bar{x}},\bar{y}\right)=\mathcal{C}_{\theta,g}\left(\bar{x},\bar{y}\right)$. If $\theta$ is an indicator function of a second-order cone or positive semidefinite cone, the manifold strong variational sufficiency is equivalent to (\ref{eq:ssosc-mani-cone}) by \cite{WDZZ22} and the above analysis. Therefore the proof is completed.
\end{proof}
\begin{remark}\label{rm:suff-independent}
Theorem \ref{thm:strong-vari-suff-ssosc} establishes that the matrices in the Hessian bundle of $l_{R_{\bar{x}}}^{\rho}(0_{\bar{x}},\bar{y})$ remain unaffected by the choice of retraction $R_{\bar{x}}$. Consequently, we observe that the strong variational sufficient condition is inherently independent of the retraction. This property allows us to consider it as an intrinsic characteristic of manifold optimization problems.
\end{remark}
\begin{remark}\label{re:suff-conv}
%It is known that the variational convexity can be viewed as a relaxation of local convexity.
Inspired by Remark \ref{rm:suff-independent}, we aim to establish a special local convexity property that is also independence of the choice of retraction. Given an lsc function $f:\mathcal{M}\rightarrow(-\infty, \infty]$, we say the function is retractional (strongly) convex at $x$ if for a retraction $R_x$, $f_{R_x}$ is locally (strongly) convex on $T_x\mathcal{M}$. Moreover, we say the function is retractional variationally (strongly) convex with respect to a pair $(x, z) \in \operatorname{gph}\partial f$ if for a retraction $R_x$ the Euclidean variationally (strongly) convexity of $f_{R_{x}}$ holds with respect to $(0_{x},z)\in \operatorname{gph}\partial f_{R_{x}}$ on $T_x\mathcal{M}\times T_x\mathcal{M}$. Remarkably, for a smooth function $f$, by \cite[Proposition 5.5.6]{AMS09} we observe that the definitions remain independent of the chosen retraction at a critical point $x$ (i.e., $\operatorname{grad}f(x)=0$) for the strong cases.
\end{remark}
%It is known that the exponential mapping is a second-order retraction, and all second-order retractions can be seen as the approximation of exponential mapping but are easier to calculate. \cite[Example 4.10]{AM12} give us two examples of the second-order retractions for embedded submanifold. Here we should introduce the definition of retractor.
%\begin{definition}[Retractor]\label{def:retractor}
%Let $\mathcal{M}$ be a $d$-dimensional submanifold of an $n$-dimensional Euclidean space $\mathcal{E}$. A retractor on $\mathcal{M}$ is a smooth mapping $D$ from the tangent bundle $TM$ into the Grassmann manifold $\operatorname{Gr}(n-d, \mathcal{E})$, whose domain contains a neighborhood of the zero section of $TM$, and such that, for all $x \in \mathcal{M}, D(x, 0)$ is transverse to $T_x \mathcal{M}$
%\end{definition}
%It is known that each retractor can define a retraction. If $D(x, u)=\mathrm{N}_{\mathcal{M}}(x)$, we call the relating retraction the orthographic retraction. If $D(x, u)=\mathrm{N}_{\mathcal{M}}\left(\mathrm{P}_{\mathcal{M}}(x+u)\right)$, then it defines the projective retraction. It follows from \cite[Example 4.10]{AM12} that The orthographic retraction and the projective retraction are second-order retractions.
%
%{\color{red}can use first-order retraction??at critical point? only assume the 2nd retraction in semismooth newton??}
%
%\begin{example}[Projective retraction on $St(n,p)=\{X\in \mathbb{R}^{n\times p}\mid X^{\top}X=I_p\}$]\label{ex:retra-proj-stie}
%
% For any $\xi$ such that $\|\xi\|<\sigma_m(\bar{X})$, the projection of $X+\xi$ onto $St(n,p)$ exists, is unique, and has the expression
% $$
% R(X,\xi)=\sum_{i=1}^p u_i v_i^{\top}
% $$
% given by a singular value decomposition of $X+\xi$. In other words, it is the $W$ of the polar decomposition $X+\xi =W S$.
%\end{example}
%
%
%\begin{example}[Orthographic retraction on $St(n,p)=\{X\in \mathbb{R}^{n\times p}\mid X^{\top}X=I_p\}$]\label{ex:retra-orth-stie}
%
% For any $\xi=X \Omega+X_{\perp} K$ where $\Omega$ is skew-symmetric and $K \in \mathbb{R}^{(n-p) \times p}$,
% $$
% R\left(X, X \Omega+X_{\perp} K\right)=X+X \Omega+X_{\perp} K+X S,
% $$
% where $S$ is symmetric and is the solution of the following equation:
% $$
% S^2+\left(I+\Omega^T\right) S+S(I+\Omega)+\Omega^T \Omega+K^T K=0.
% $$
%\end{example}
%
%\begin{example}[Projective retraction on $Fr(m, n, r):=\left\{X \in \mathbb{R}^{m \times n}: \operatorname{rank}(X)=r\right\}$]\label{ex:retra-proj-rank}
%
%For any $\xi$ such that $\|\xi\|<\sigma_r(X) / 2$, the projective retraction of $X+\xi$ onto $Fr(m, n, r)$ exists, is unique, and has the expression
%$$
%R(X,\xi)=\sum_{i=1}^r \sigma_i(X+\xi) u_i v_i^{\top}
%$$
%given by a singular value decomposition of $X+\xi=\sum_{i=1}^{\min\{m,n\}} \sigma_i(X+\xi) u_i v_i^{\top}$.
%\end{example}
%
%
%\begin{example}[Orthographic retraction on $Fr(m, n, r):=\left\{X \in \mathbb{R}^{m \times n}: \operatorname{rank}(X)=r\right\}$]\label{ex:retra-orth-rank}
%
%The orthographic retraction $R$ is given by
%$$
%\begin{aligned}
% R(X, \xi) & =U\left[\begin{array}{cc}
% \Sigma_0+A & C \\
% B & B\left(\Sigma_0+A\right)^{-1} C
% \end{array}\right] V^T \\
% & =U\left[\begin{array}{c}
% \Sigma_0+A \\
% B
% \end{array}\right]\left[\begin{array}{ll}
% I & \left(\Sigma_0+A\right)^{-1} C
% \end{array}\right] V^T
%\end{aligned}
%$$
%where $U, V, \Sigma_0, A, B$, and $C$ are obtained from the singular value decomposition of $X$ and $\xi$.
%\end{example}
\begin{remark}\label{rm:retracion-conv}
It seems that our definition of retractional convexity is quite similar to the retraction-convexity defined in \cite[Definition 3.2]{HW22}. However, there is a crucial distinction: retraction-convexity in \cite[Definition 3.2]{HW22} necessitates holding on a subset of the manifold, whereas our definition of retractional convexity is defined at a specific point on the manifold. It appears that requiring convexity to be held on a subset of the manifold might be unnecessary for the local convergence analysis around the stationary point.
\end{remark}
%\begin{remark}\label{rm:second-order-retraction}
% The locally convexity of a function $f$ is defined under a second-order retraction. This is because we want $f_{R_{x}}$ to inherit some of the second-order information of $f$ at $x$ so the second-order retraction is our best choice. In Proposition \ref{pro:local-conv-char} we will see that the locally strong convexity, which relies on the Hessian at $x$, is independent with the second-order retraction.
%\end{remark}
%In Remark \ref{rm:suff-convex}, we aim to construct a kind of convexity solely relies on the function itself, without requiring additional information. For the retractionly convexity. this is achieved by requiring the convexity for $f_{R}$ for all second-order retractions. However, for the strong version, we observe that the requirement can be relaxed to a arbitrary second-order retraction.
%
%
%\begin{proposition}\label{pro:local-conv-char}
% Given a lower semicontinuous (lsc) function $f:\mathcal{M}\to\mathbb{R}$ and a second-order retraction $R$, the following statements are true:
% \begin{itemize}
% \item [(i)] If $f$ is twice-differentiable near $x$ and retractionly convex at $x$, then
% $$
% \operatorname{Hess}f(x)\succeq 0.
% $$
% \item [(ii)] If $f$ is twice-differentiable near $x$, then $f$ is retractionly strongly convex at $x$ if and only if $\operatorname{Hess}f(x)\succ 0$.
% \item [(iii)] If $f$ is continuously differentiable near $x$ and $f_{R_x}$ is prox-regular for a second-order retraction $R_x$, then $f$ is retractionly strongly convex at $x$ if and only if for any second-order retraction $R_x$ and $H$ belongs to the following Hessian bundle (\ref{eq:hess-bundle-f}), $H$ is positive definite.
% \begin{equation}\label{eq:hess-bundle-f}
% \overline{\nabla}^2 f_{R_{x}}(0_{x})=\left\{H \mid \exists \xi_k \rightarrow 0_{x} \text { with } \nabla^{2} f_{R_{x}}\left(\xi_k\right) \rightarrow H,\; \xi_k\in T_{x}\mathcal{M}\right\}.
% \end{equation}
% \end{itemize}
%\end{proposition}
%\proof If $f$ is $C^2$ and retractionly convex at $x$, then for any second-order retraction $R_x$, we have
%$$
%\nabla^2 f_{R_x}(0_x)\succeq 0.
%$$
%By the definition of second-order retraction, we obtain $\operatorname{Hess}f(x)\succeq 0$.
%The 'only if' part of (ii) can be similarly obtained by using second-order characterization of $f_{R_x}$ at $0_x$. Now if $\operatorname{Hess}f(x)\succ 0$, for any second-order retraction $R_x$ we have $\nabla^2 f_{R_x}(0_x)\succ 0$, which implies that there exists a neighborhood $\mathcal{W}$ of $0_x$, such that for any $\xi\in\mathcal{M}$, $\nabla^2 f_{R_x}(\xi)$ is uniformly positive definite, i.e., there exists $\alpha>0$ such that for any $\xi\in\mathcal{M}$ and $v\in T_x\mathcal{M}$, $\left\langle v,\nabla^2 f_{R_x}(\xi)v \right\rangle\succeq \alpha\|v\|$. This is equivalent to have $f_{R_x}$ being strongly convex on $\mathcal{W}$. For a given second-order retraction $R_x$, the positive definiteness of the elements in $\overline{\nabla}^2 f_{R_{x}}(0_{x})$ is the standard conclusion in Euclidean settings. Now given two different second-order retraction $R_1$ and $R_2$ at $x$, it can be proved that if $f_{R_1}$ is $C^1$ and prox-regular, then $f_{R_2}$ is $C^1$ and prox-regular. We are left to show that the Hessian bundle (\ref{eq:hess-bundle-f}) of $f_{R_1}$ and $f_{R_2}$ are identical. In fact, for any $H\in \overline{\nabla}^2 f_{R_2}(0_{x})$, there exists $\{\xi_k\}$ converges to $0_x$, such that $\nabla^2 f_{R_2}(\xi_k)\to H$. Let $W_x:=R_2^{-1}R_1$, then we have
% $$
% \nabla^2f_{R_1}(\xi_k)= W^{\prime}_x(\xi_k)^* \nabla^2f_{R_2}(W_x(\xi_k)) \nabla W_x(\xi_k) + \nabla (W^{\prime}_x(\xi_k)^*)\nabla^2f_{R_2}(W_x(\xi_k)).
% $$
% It follows from the definition of $W_x$ that when $\xi_k\to 0_x$, $\nabla W_x(\xi_k)\to \operatorname{id}$ and $\nabla(W^{\prime}_x(\xi_k)^*)\to 0_x$. Therefore, if $\nabla^2 f_{R_2}(\xi_k)\to H$, then there exists a sequence $\{W_x(\xi_k)\}$ converge to $ 0_x$, such that $\nabla^2f_{R_1}(W_x(\xi_k))\to H$. By exchanging $R_1$ and $R_2$ it is shown that $\overline{\nabla}^2 f_{R_1}(0_{x})=\overline{\nabla}^2 f_{R_2}(0_{x})$. This implies that the hessian bundle is independent of the choice of second-order retraction.\qed
%
%{\color{red}Similar to retractionly strong convexity, the strong variational sufficiency is independent of the chosen second-order retraction and can be interpreted as a property of the manifold. This relationship is demonstrated by Theorem \ref{thm:strong-vari-suff-ssosc}.}
%
\section{Convergence analysis of Riemannian ALM}\label{sec:convegence-alm}
In this section, we analyze the local convergence of RALM. Suppose that the manifold variational sufficient condition under retraction $R_{\bar{x}}$ holds at level $\bar{\rho}$ with respect to $\bar{x}$ and $\bar{y}$ satisfying condition \eqref{eq:first-opt-mani}. Let $\mathcal{W}$ be given by Proposition~\ref{pro:vari-suff-al-conv}. We take $\mathcal{U}=R_{\bar{x}}(\mathcal{W})$ and $\tilde{\rho}_k=\rho_{k}-\bar{\rho}$. Then \eqref{eq:alm-mani} can be achieved by
\begin{equation}\label{eq:alm-tangent}
\left\{\begin{array}{l}
\xi^{k+1} \approx \bar{\xi}^{k+1}:=\underset{\xi \in \mathcal{W}}{\operatorname{argmin}}\; l_{R_{\bar{x}}}^{\rho_k}(\xi, y^k),\\
x^{k+1}=R_{\bar{x}}(\xi^{k+1}),\\
y^{k+1}=y^k+\tilde{\rho}_{k} \nabla_y l_{R_{\bar{x}}}^{\rho_k}(\xi^{k+1}, y^k).
\end{array}\right.
\end{equation}
The first iteration of \eqref{eq:alm-tangent} for finding $\xi^{k+1}$ can be considered as a traditional inexact ALM step in the tangent space, and the second iteration pulls $\xi^{k+1}$ back to manifold using retraction $R_{\bar{x}}$.
%
%Suppose that the manifold variational sufficient condition is satisfied at $(\bar{x},\bar{y})$ at level $\bar{\rho}$ in this section.
%If setting $\mathcal{U}=R_{\bar{x}}(\mathcal{W})$ and $\tilde{\rho}_k=\rho_{k}-\bar{\rho}$, (\ref{eq:alm-mani}) can be consider as applying the traditional inexact ALM in the tangent space and then pulling back to manifold using retraction $R_{\bar{x}}$, which is
%\begin{equation}\label{eq:alm-tangent}
% \left\{\begin{array}{l}
% \xi^{k+1} \approx \bar{\xi}^{k+1}:=\underset{\xi \in \mathcal{W}}{\operatorname{argmin}}\; l_{R_{\bar{x}}}^{\rho_k}\left(\xi, y^k\right),\\
% y^{k+1}=y^k+\tilde{\rho}_{k} \nabla_y l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right),\\
% x^{k+1}=R_{\bar{x}}(\xi^{k+1}).
% \end{array}\right.
%\end{equation}
We %shall
first apply PPA to the local dual problem (\ref{eq:dual-mani-vs}). Let the solution set of (\ref{eq:dual-mani-vs}) be
$
Z=\underset{y}{\operatorname{argmax}}\; H(y)=\underset{y}{\operatorname{argmax}}\; H_{R_{\bar{x}}}(y)
$.
Note that the dual problems (\ref{eq:dual-mani-vs}) and (\ref{eq:dual-tangent}) are %not only equivalent, but in fact,
identical %, as the objective function
($H(y)=H_{R_{\bar{x}}}(y)$ for any $y\in\mathcal{Y}$). Therefore, applying PPA to (\ref{eq:dual-mani-vs}) or (\ref{eq:dual-tangent}) is the same. % as employing it to (\ref{eq:dual-tangent}).
The PPA iteration is
\begin{equation}\label{eq:ppa}
\begin{aligned}
y^{k+1} \approx x^k\left(y^k\right) \text { with }x^k\left(y^k\right)&=\underset{y}{\operatorname{argmax}}\big\{H^k(y):=H(y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\big\}\\
&=\underset{y}{\operatorname{argmax}}\big\{ H_{R_{\bar{x}}}^k(y):= H_{R_{\bar{x}}}(y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\big\}.
\end{aligned}
\end{equation}
The proximal parameters $c_k$ satisfy $1\leq c_k\leq c_{\infty}\leq\infty$, the approximation is controlled by three stopping criteria
%of $y^{k+1}$ in (\ref{eq:ppa}) is given by
\begin{equation}\label{eq:ppa-app}
\left\|y^{k+1}-x^k\left(y^k\right)\right\| \leq\left\{\begin{array}{ll}
\text { (a) } & \varepsilon_k \\
\text { (b) } & \varepsilon_k \min \left\{1,\left\|y^{k+1}-y^k\right\|\right\} \\
\text { (c) } & \varepsilon_k \min \big\{1,\left\|y^{k+1}-y^k\right\|^2\big\},
\end{array}\right.
\end{equation}
in which the error parameters satisfy
\begin{equation}\label{eq:ppa-app-eps}
\varepsilon_k\in(0,1)\quad \mbox{and}\quad \sum_{k=0}^{\infty}\varepsilon_k=\sigma< \infty.
\end{equation}
The next theorem presents the convergence of local PPA, as established in \cite{R22-ext} and \cite{R21}.
\begin{theorem}\label{thm:converge-ppa}
Suppose that the manifold variational sufficient condition is satisfied at $(\bar{x},\bar{y})$ under retraction $R_{\bar{x}}$. Let the initial point $y^0$ and the value $\sigma$ in \eqref{eq:ppa-app-eps} satisfy the following closeness condition relative to $Z$:
\begin{equation}\label{eq:ppa-initial}
\exists\; \eta>\operatorname{dist}\left(y^0, Z\right)+\sigma \text { such that } \mathcal{Y} \supset\left\{y\mid\| y-y^0 \|<3 \eta\right\}.
\end{equation}
Then the sequence $\left\{y^k\right\}$ generated by the proximal point iterations (\ref{eq:ppa}) under (\ref{eq:ppa-app}a) will belong to $\operatorname{int}\mathcal{Y}$ and converge to a particular point $y^{*} \in Z$ in the ball $\left\{y\mid\| y-y^{*}_0 \|<\eta\right\} \subset \operatorname{int}\mathcal{Y}$,
where $y^{*}_0$ is the point of $Z$ closest to $y^0$. Moreover, neither $y^k$ nor $x^k\left(y^k\right)$ will leave that ball, and the dual objective values $H\left(y^k\right)$ will converge to the optimal value $H(y^{*})$ in (\ref{eq:dual-mani-vs}).
\end{theorem}
\begin{proof}
By \cite[11.48]{RW09}, $ H_{R_{\bar{x}}}$, or equally, $H$ is upper semicontinuous and concave in $y$, thus $\partial (-H)$ is maximal monotone \cite[12.17]{RW09}. By taking $T=\partial (-H)$ in \cite[Theorem 2.1]{R21} the results hold.
\end{proof}
From the convexity of $ l^{\bar{\rho}}_{R_{\bar{x}}}(\cdot, y)$ we know that
\begin{equation}\label{eq:local-min}
\underset{\xi \in \mathcal{W}}{\operatorname{argmin}}\; l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y)=\left\{\xi \mid-\nabla_{\xi} l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y) \in N_{\mathcal{W}}(\xi)\right\},
\end{equation}
where $N_{\mathcal{W}}(\xi)$ is the normal cone to $\mathcal{W}$ at $\xi$. %Now we consider the convergence of RALM.
For the approximate minimization in \eqref{eq:alm-mani} (where we take $\mathcal{U}=R_{\bar{x}}(\mathcal{W})$), three stopping criteria are given in \cite[(1.15)]{R22-ext} for the acceptability of $x^{k+1}$:
%The approximation of $\bar{x}^{k+1}$ is chosen as in \cite{R22-ext}, which are
\begin{equation}\label{eq:alm-app-mani}
\small \left(2 \tilde{\rho}_{k}%\rho_{k}
\left[l^{\rho_{k}}\left(x^{k+1}, y^k\right)-\inf _{ R_{\bar{x}}(\mathcal{W})} l^{\rho_{k}}\left(\cdot, y^k\right)\right]\right)^{1 / 2} \leq \begin{cases}\text { (a) } & \varepsilon_k \\ \text { (b) } & \varepsilon_k \min \left\{1,\left\|\tilde{\rho}_{k} \nabla_y l^{\rho_{k}}\left(x^{k+1}, y^k\right)\right\|\right\} \\ \text { (c) } & \varepsilon_k \min \big\{1,\left\|\tilde{\rho}_{k}\nabla_y l^{\rho_{k}}\left(x^{k+1}, y^k\right)\right\|^2\big\}.\end{cases}
\end{equation}
As a counterpart, three stopping criteria for the acceptability of $\xi^{k+1}$ in the approximate minimization in (\ref{eq:alm-tangent}) are
%To maintain the equivalence of (\ref{eq:alm-mani}) and (\ref{eq:alm-tangent}), the approximate error for $\bar{\xi}^{k+1}$ are also taken
\begin{equation}\label{eq:alm-app-tangent}
\small \left(2 \tilde{\rho}_{k}%\rho_{k}
\left[ l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right)-\inf _{\mathcal{W}} l_{R_{\bar{x}}}^{\rho_k}\left(\cdot, y^k\right)\right]\right)^{1 / 2} \leq \begin{cases}\text { (a}^{\prime}) & \varepsilon_k \\ \text { (b}^{\prime}) & \varepsilon_k \min \left\{1,\left\|\tilde{\rho}_{k} \nabla_y l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right)\right\|\right\} \\ \text { (c}^{\prime}) & \varepsilon_k \min \big\{1,\left\|\tilde{\rho}_{k}\nabla_y l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right)\right\|^2\big\}.\end{cases}
\end{equation}
The iteration \eqref{eq:alm-mani} under the stopping criteria \eqref{eq:alm-app-mani} is equivalent to the iteration \eqref{eq:alm-tangent} under the stopping criteria \eqref{eq:alm-app-tangent}.
\begin{theorem}\label{thm:converge-alm}
Suppose the manifold variational sufficient condition is satisfied at $(\bar{x},\bar{y})$ at level $\bar{\rho}$ under retraction $R_{\bar{x}}$, and the sets (\ref{eq:local-min}) are nonempty and bounded when $y\in\operatorname{int}\mathcal{Y}$. Let the RALM (\ref{eq:alm-mani}) be initiated with $y^0$ satisfying the conditions %{\color{red}\eqref{eq:ppa-initial}}
in Theorem \ref{thm:converge-ppa} and $\tilde{\rho}_k=\rho_{k}-\bar{\rho}$. With the stopping criterion (\ref{eq:alm-app-mani}a), error parameters $\varepsilon_k$ as in \eqref{eq:ppa-app-eps} and stepsizes $\tilde{\rho}
_k$ with $\tilde{\rho}_k \rightarrow \rho_{\infty}-\bar{\rho} \in(0, \infty]$, by the estimate
\begin{equation}\label{eq:estimation}
\left\|y^{k+1}-x^k\left(y^k\right)\right\|^2 \leq 2 \tilde{\rho}_k\big[l^{\rho_k}\left(x^{k+1}, y^k\right)-\inf _{ R_{\bar{x}}(\mathcal{W})} l^{\rho_k}\left(\cdot, y^k\right)\big],
\end{equation}
the sequence $\left\{y^k\right\}$ can be seen as being generated by PPA (\ref{eq:ppa}) with $c_k=\tilde{\rho}_k$ under the stopping criterion (\ref{eq:ppa-app}a) for the same error parameters $\varepsilon_k$. %with RALM.
Moreover, the sequence $\left\{x^k\right\}$ in $ R_{\bar{x}}(\mathcal{W})$ is bounded. %in (\ref{eq:perturb-problem-mani}).
Each of its accumulation points will be a solution $\bar{x}$ to (\ref{eq:prime-mani-vs}) and also a minimizer of (\ref{eq:perturb-problem-mani}) relative to $ R_{\bar{x}}(\mathcal{W})$. Therefore, $\bar{x}$ is locally optimal in (\ref{eq:prime-mani-vs}) if it belongs to
$\operatorname{int} R_{\bar{x}}(\mathcal{W})$.
Implementing RALM with stopping criterion (\ref{eq:alm-app-mani}b) or (\ref{eq:alm-app-mani}c)
corresponds to implementing PPA with (\ref{eq:ppa-app}b) or (\ref{eq:ppa-app}c), respectively.
% Instead of using (\ref{eq:alm-app-mani}a), applying the stopping criteria (\ref{eq:alm-app-mani}b) or (\ref{eq:alm-app-mani}c) to RALM corresponds to using the criteria (\ref{eq:ppa-app}b) or (\ref{eq:ppa-app}c) with PPA, respectively.
\end{theorem}
\begin{proof}
The proof is quite similar with the proof of \cite[Theorem 2.3]{R22-ext}.
%Given the RALM $k$-th iteration (\ref{eq:alm-mani}), it is corresponding to the ALM $k$-th iteration (\ref{eq:alm-tangent}).
Let the parameter $c_k$ in the $k$-th PPA iteration \eqref{eq:ppa} be $c_k=\tilde{\rho}_k$. Define $H_{R_{\bar{x}}}^{c_k}\left(y^k\right):=\max _y H_{R_{\bar{x}}}^k(y)$.
By definition we have
\begin{equation}\label{eq:hck-gradient}
\nabla H_{R_{\bar{x}}}^{c_k}\left(y^k\right)=c_k^{-1}\left[x^k\left(y^k\right)-y^k\right], \text { so that } x^k\left(y^k\right)=y^k+c_k \nabla H_{R_{\bar{x}}}^{c_k}\left(y^k\right).
\end{equation}
Define the convex-concave function
\begin{equation}\label{eq:ppa-lagrangian}
\hat{l}^k(\xi, y):= l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2 \text { for } \xi\in \mathcal{W} \text { and } y \in \mathcal{Y}.
\end{equation}
The $k$-th PPA iteration is therefore associated with the following primal and dual problems:
\begin{subequations}
\begin{align}
&\min_{\xi \in \mathcal{W}} F_{R_{\bar{x}}}^k(\xi), \text { where } F_{R_{\bar{x}}}^k(\xi):=\sup _{y \in \mathcal{Y}} \hat{l}^k(\xi, y)\tag{$P1$-$k$}\label{eq:prime-ppa-iter}, \\
&\max_{ y \in \mathcal{Y}} \big\{ \inf _{\xi \in \mathcal{W}} \hat{l}^k(\xi, y)= H_{R_{\bar{x}}}(y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\big\}. \tag{$D1$-$k$}\label{eq:dual-ppa-iter}
\end{align}
\end{subequations}
The unique solution of (\ref{eq:dual-ppa-iter}) is $x^k(y^k)$ %$P_{k}(y^k)$
with the optimal value $ H_{R_{\bar{x}}}^{c_k}(y^k)$.
Our assumption that the sets (\ref{eq:local-min}) are nonempty and bounded when $y \in \operatorname{int} \mathcal{Y}$
makes the convex functions $\hat{l}^{k}(\cdot, y)$ be level-bounded \cite[3.23]{RW09}, and that passes over to the functions $\hat{l}^k(\cdot, y)$, causing the convex objective function $ F_{R_{\bar{x}}}^k$ in (\ref{eq:prime-ppa-iter}) to be level-bounded as well.
The concave objective function in (\ref{eq:dual-ppa-iter}) is likewise
level-bounded from below %(from below instead of from above),
due to its quadratic term. Because of this,
optimal solutions to both (\ref{eq:prime-ppa-iter}) and (\ref{eq:dual-ppa-iter}) exist, characterized by forming saddle points in (\ref{eq:ppa-lagrangian}), and the optimal values in these problems agree \cite[11.40]{RW09}. Therefore, there exists $\tilde{\xi}^k \in \mathcal{W}$, such that
\begin{equation}\label{eq:ppa-iter-prime-dual}
\begin{aligned}
H_{R_{\bar{x}}}^{c_k}\left(y^k\right) &=\widehat{F}^k\left(\tilde{\xi}^k\right)=\hat{l}^k\left(\tilde{\xi}^k, x^k\left(y^k\right)\right)=\max _{y \in \mathcal{Y}} \hat{l}^k\left(\tilde{\xi}^k, y\right) \\
&=\max _{y \in \mathcal{Y}}\big\{ l^{\bar{\rho}}_{R_{\bar{x}}}\left(\tilde{\xi}^k, y\right)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\big\}.
\end{aligned}
\end{equation}
If it further holds that $x^k(y^k)\in \operatorname{int}\mathcal{Y}$, the concavity of $ l^{\bar{\rho}}_{R_{\bar{x}}}(\xi,y)$ in $y$ yields that
$$
\max _{y \in \mathcal{Y}}\big\{ l^{\bar{\rho}}_{R_{\bar{x}}}\left(\tilde{\xi}^k, y\right)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\big\}=\max _{y \in \mathbb{Y}}\big\{ l^{\bar{\rho}}_{R_{\bar{x}}}\left(\tilde{\xi}^k, y\right)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\big\} \;\text {if}\; x^k\left(y^k\right) \in \operatorname{int} \mathcal{Y},
$$
and the latter part equals to $ l_{R_{\bar{x}}}^{\rho_k}(\tilde{\xi}^k,y^k)$ by \cite[11.23]{RW09}. Now we obtain that
\begin{equation}\label{eq:hck-equ-lck}
\exists\; \tilde{\xi}^k \in \mathcal{W} \;\text {such that}\; H_{R_{\bar{x}}}^{c_k}\left(y^k\right)= l_{R_{\bar{x}}}^{\rho_k}\left(\tilde{\xi}^k, y^k\right) \;\text {if}\; x^k\left(y^k\right) \in \operatorname{int} \mathcal{Y}.
\end{equation}
For any $y\in\mathcal{Y}$ and $\xi\in \mathcal{W}$,
$H_{R_{\bar{x}}}(y)=\inf_{\xi^{\prime}\in\mathcal{W}} l^{\bar{\rho}}_{R_{\bar{x}}}(\xi^{\prime},y)\leq l^{\bar{\rho}}_{R_{\bar{x}}}(\xi,y)$.
Consequently,
\begin{equation}\label{eq:hck-leq-lck}
\begin{aligned}
H_{R_{\bar{x}}}^{c_k}\left(y\right)&=\max _{y^{\prime}}\big\{ H_{R_{\bar{x}}}(y^{\prime})-\frac{1}{2 \rho_k}\left\|y^{\prime}-y\right\|^2\big\}\\
&\leq \max _{y^{\prime}}\big\{ l^{\bar{\rho}}_{R_{\bar{x}}}(\xi,y^{\prime})-\frac{1}{2 c_k}\left\|y^{\prime}-y\right\|^2\big\}= l_{R_{\bar{x}}}^{\rho_k}\left(\xi, y\right).
\end{aligned}
\end{equation}
Combining (\ref{eq:hck-equ-lck}) with (\ref{eq:hck-leq-lck}), we know that if $x^k\left(y^k\right) \in \operatorname{int} \mathcal{Y}$,
$$
H_{R_{\bar{x}}}^{c_k}\left(y^k\right)=\min _{\xi \in \mathcal{W}} l_{R_{\bar{x}}}^{\rho_k}\left(\xi, y^k\right)=\min _{x \in R_{\bar{x}}(\mathcal{W})} l^{\rho_k}\left(x, y^k\right).
$$
Now consider the vectors $y^{k+1}$ in %ALM
(\ref{eq:alm-tangent}). By the concavity of $l^{\rho_k}\left(x^{k+1}, y\right)$ in $y$ we have
$$
l^{\rho_k}\left(x^{k+1}, y\right) \leq l^{\rho_k}\left(x^{k+1}, y^k\right)+\langle\nabla_y l^{\rho_k}\left(x^{k+1}, y^k\right) ,y-y^k\rangle \text { for all } y \in \mathbb{Y}.
$$
Moreover, by \cite[12.60]{RW09} and the definition of $ H_{R_{\bar{x}}}^{c_k}(y^k)$,
$$
H_{R_{\bar{x}}}^{c_k}(y) \geq H_{R_{\bar{x}}}^{c_k}\left(y^k\right)+\langle\nabla H_{R_{\bar{x}}}^{c_k}\left(y^k\right),y-y^k\rangle-\frac{1}{2 c_k}\left\|y-y^k\right\|^2 \text { for all } y \in \mathbb{Y}.
$$
Similar with (\ref{eq:hck-leq-lck}), the definition of $ H_{R_{\bar{x}}}^{c_k}(y)$ also implies $H_{R_{\bar{x}}}^{c_k}(y)\leq l^{\rho_k}(x,y)$ for any $ x\in R_{\bar{x}}(\mathcal{W})$ and $y\in \mathcal{Y}$. Therefore,
$$
H_{R_{\bar{x}}}^{c_k}\left(y^k\right)+\langle\nabla H_{R_{\bar{x}}}^{c_k}\left(y^k\right),y-y^k\rangle-\frac{1}{2 c_k}\left\|y-y^k\right\|^2 \leq l^{\rho_k}\left(x^{k+1}, y^k\right)+\langle\nabla_y l^{\rho_k}\left(x^{k+1}, y^k\right) ,y-y^k\rangle,
$$
which means
$
l^{\rho_k}\left(x^{k+1}, y^k\right)- H_{R_{\bar{x}}}^{c_k}\left(y^k\right) \geq\langle\nabla_y l^{\rho_k}\left(x^{k+1}, y^k\right)-\nabla H_{R_{\bar{x}}}^{c_k}\left(y^k\right),y-y^k\rangle-\frac{1}{2 c_k}\left\|y-y^k\right\|^2
$. The iterations (\ref{eq:alm-mani}) and (\ref{eq:hck-gradient}) yield that
$$
\nabla_y l^{\rho_k}\left(x^{k+1}, y^k\right)-\nabla H_{R_{\bar{x}}}^{c_k}\left(y^k\right)=c_k^{-1}\left[y^{k+1}-y^k\right]-c_k^{-1}\left[x^k\left(y^k\right)-y^k\right]=c_k^{-1}\left[y^{k+1}-x^k\left(y^k\right)\right].
$$
Therefore,
\begin{equation}\label{eq:alm-ppa-esti}
\begin{aligned}
&c_k\left[l^{\rho_k}\left(x^{k+1}, y^k\right)- H_{R_{\bar{x}}}^{c_k}\left(y^k\right)\right]\\
\geq &\max _{y}\big\{\left\langle y^{k+1}-x^k\left(y^k\right), y-y^k\right\rangle-\frac{1}{2}\|y-y^k\|^2\big\}=\frac{1}{2}\left\|y^{k+1}-x^k\left(y^k\right)\right\|^2.
\end{aligned}
\end{equation}
Theorem \ref{thm:converge-ppa} ensures that $x^k(y^k)\in \operatorname{int}\mathcal{Y}$ and $\{y^k\}$ will converge to a solution $y^*\in\operatorname{int}\mathcal{Y}$ if $y^0$ is chosen through (\ref{eq:ppa-initial}). Hence by (\ref{eq:hck-equ-lck}), (\ref{eq:alm-ppa-esti}) is the estimation (\ref{eq:estimation}). Moreover, the approximation (\ref{eq:alm-app-mani})(a,b,c) will lead to the PPA approximation (\ref{eq:ppa-app})(a,b,c).
We now consider the sequence $x^{k+1}$. Define $
F^k(x):=\sup_{y\in\mathcal{Y}}\; l(x,y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2$.
it is true that $F^k(x)= F_{R_{\bar{x}}}^k(\operatorname{Exp}
_{\bar{x}}^{-1}x)$, implying
$
\min_{\xi \in \mathcal{W}} F_{R_{\bar{x}}}^k(\xi)=\min_{x \in R_{\bar{x}}(\mathcal{W})} F^k(x)
$. Together with (\ref{eq:ppa-iter-prime-dual}) and (\ref{eq:hck-equ-lck}), we have
$
\min _{x \in R_{\bar{x}}(\mathcal{W})} F^k(x)=\min _{x \in R_{\bar{x}}(\mathcal{W})} l^{\rho_k}\left(x, y^k\right)= H_{R_{\bar{x}}}^{c_k}\left(y^k\right)
$.
Since $\xi^{k+1}= R_{\bar{x}}^{-1}x^{k+1}$ and $x^{k+1}$ is chosen under the stopping criterion (\ref{eq:alm-app-mani}a),
\begin{equation}\label{eq:Fk-leq-alpha}
\begin{aligned}
F^k\left(x^{k+1}\right)= F_{R_{\bar{x}}}^k(\xi^{k+1})&=\sup_{y\in\mathcal{Y}}\; l^{\bar{\rho}}_{R_{\bar{x}}}(\xi,y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\leq \max_{y\in\mathcal{Y}}\; l^{\bar{\rho}}_{R_{\bar{x}}}(\xi,y)-\frac{1}{2 c_k}\left\|y-y^k\right\|^2\\
& \leq l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right)= l^{\rho_k}\left(x^{k+1}, y^k\right)\leq \alpha_k:= H_{R_{\bar{x}}}^{c_k}\left(y^k\right)+\frac{\varepsilon_k^2}{2 c_k}.
\end{aligned}
\end{equation}
The definition of $ H_{R_{\bar{x}}}^{c_k}\left(y^k\right)$ implies $ H_{R_{\bar{x}}}\left(y^k\right) \leq H_{R_{\bar{x}}}^{c_k}\left(y^k\right) \leq \max H_{R_{\bar{x}}}$. By Theorem \ref{thm:converge-ppa},
$H_{R_{\bar{x}}}\left(y^k\right) \rightarrow \max H_{R_{\bar{x}}}$, hence
$
\alpha_k \rightarrow \alpha^*=\max (D_{\bar{x}})=\min (P_{\bar{x}})=\min (P)=\min _{x \in R_{\bar{x}}(\mathcal{W})} l^{\bar{\rho}}(x, y^*).
$
Moreover, from the definition of $F^k$ we have $F^k(x) \geq l^{\bar{\rho}}\left(x, y^k\right)$, which implies
$$
\begin{aligned}
\left\{x \in R_{\bar{x}}(\mathcal{W}) \mid F^k(x) \leq \alpha\right\} &\subset\left\{x \in R_{\bar{x}}(\mathcal{W}) \mid l^{\bar{\rho}}(x, y^*) \leq \alpha\right\}\\
&= R_{\bar{x}}\left\{\xi \in \mathcal{W} \mid l^{\bar{\rho}}_{R_{\bar{x}}}(\xi, y^*) \leq \alpha\right\}\; \text{for all}\; \alpha \in \mathbb{R},
\end{aligned}
$$
where sets on the right are bounded under the argmin assumption in \cite[3.23]{RW09}. From (\ref{eq:Fk-leq-alpha}), we have
$x^{k+1} \in\left\{x \in R_{\bar{x}}(\mathcal{W}) \mid l^{\bar{\rho}}(x, y^*) \leq \alpha\right\}$ for any $\alpha \geq \alpha_k$.
Thus, the sequence $\left\{x^k\right\}$ is bounded, and all its accumulation points belong to $\operatorname{argmin}_{ R_{\bar{x}}(\mathcal{W})} l^{\bar{\rho}}(\cdot, y^*)$.
\end{proof}
\begin{corollary}\label{cor:alm-converge-strong-convex}
Suppose the manifold strong variational sufficiency holds at $(\bar{x},\bar{y})$ under retraction $R_{\bar{x}}$. The sequence $\left\{x^k\right\}$ generated by RALM must converge to that local solution $\bar{x}$.
\end{corollary}
\begin{proof}
The manifold strong variational sufficiency holding at $(\bar{x},\bar{y})$ refers to the isolated minimizing property of $\bar{x}$. By Theorem \ref{thm:converge-alm}, $\{x^k\}$ will converge to the unique local solution $\bar{x}$.
\end{proof}
The local linear convergence rate of PPA iterations is obtained by \cite[Theorems 3.2 and 3.3]{R21} as follows.
\begin{theorem}\label{thm:ppa-liear-convergence}
In the circumstances of Theorem \ref{thm:converge-ppa} with stopping criterion (\ref{eq:ppa-app}b) to get $y^k \rightarrow \bar{y} \in Z=\operatorname{argmax}_{\mathcal{Y}} H_{R_{\bar{x}}}$, suppose
$\exists\, b>0, \lambda>0$, such that $ H_{R_{\bar{x}}}(y) \leq\left[\max _y H_{R_{\bar{x}}}\right]-b \operatorname{dist}^2(y, Z)$ when $\|y-\bar{y}\|<\lambda$. Then $\operatorname{dist}\left(y^k, Z\right) \rightarrow 0$ at the $Q$-linear rate $\rho=1 / \sqrt{1+b^2 \rho_{\infty}^2}$, which is 0 when $\rho_{\infty}=\infty$. If the still tighter stopping criterion (\ref{eq:ppa-app}c) is used, then $y^k \rightarrow \bar{y}$ at that Q-linear rate $\rho$.
\end{theorem}
A condition that ensures the fulfillment of the conditions stated in Theorem \ref{thm:ppa-liear-convergence} is provided in \cite[Theorem 4.2]{R22-ext}, and we now expand it to encompass our specific scenario in manifold optimization.
\begin{proposition}
Let $G=\partial \theta(g(\bar{x}))$ and $M=\left\{y \mid 0=\operatorname{grad}_x L(\bar{x}, y)=\operatorname{grad} f(\bar{x})+D g(\bar{x})^* y\right\}$, noting that $G=\left\lbrace y \mid \theta^*(y)=\theta^*(\bar{y})+\left\langle g(\bar{x}),y-\bar{y}\right\rangle \right\rbrace$. Suppose that $G$ is polyhedral, and there exist $ b_0>0$ and $\lambda_0>0$ such that, when $\left\| y-\bar{y}\right\| <\lambda_0$, it holds that
$$
\theta^*(y) \geq \theta^*(\bar{y})+D g(\bar{x})^*(y-\bar{y})+b_0 \operatorname{dist}^2(y, G).
$$
Furthermore, assume $\operatorname{grad} g(\bar{x}) \neq 0$ and denote $\beta(\operatorname{grad} g(\bar{x})):=\min \left\{\left\|Dg(\bar{x})^*\eta\right\| \mid \eta \in M^{\perp}, \left\| \eta \right\| =1\right\}$.
Then $\beta(\operatorname{grad} g(\bar{x}))>0$ and there exists $\kappa_{G,M}>0$ such that condition in Theorem \ref{thm:ppa-liear-convergence} holds for
$$
b=\frac{\kappa_{G, M}}{a_G+a_M}\quad \mbox{with}\quad a_G=b_0^{-1}+2 \bar{\rho} \quad \mbox{and}\quad a_M=\frac{2\left\|\operatorname{Hess} L(\bar{x}, \bar{y})+\bar{\rho} I\right\|}{\beta(\operatorname{grad} g(\bar{x}))^2}.
$$
\end{proposition}
\begin{proof}
Applying \cite[Theorem 4.2]{R22-ext} to problem (\ref{eq:problem-tangent}) and using $\operatorname{grad}_{x}L(\bar{x},y)=\nabla_{\xi}L_{R_{\bar{x}}}(0_{\bar{x}},y)$ and $\operatorname{Hess} L(\bar{x}, \bar{y})=\nabla_{\xi\xi} L_{R_{\bar{x}}}(0_{\bar{x}},y)$ we are able to obtain the conclusion.
\end{proof}
Now we assume that the manifold strong variational sufficiency holds at $(\bar{x},\bar{y})$ in the remaining %leaving
part of this section.
The locally strong convexity of $ l_{R_{\bar{x}}}^{\rho_k}(\cdot,y^k)$ implies that each RALM iteration \eqref{eq:alm-mani} has a unique solution, which %and
we denote by $\bar{x}^{k+1}$. Next we give %We are now considering
the local convergence rate of $\{x^k\}$ and $\{\bar{x}^k\}$ under the dual convergence.
\begin{theorem}%[ALM local convergence rate]
The convergence $y^k \rightarrow \bar{y} \in Z$ in the augmented Lagrangian method (\ref{eq:alm-mani}), as implemented in Theorem \ref{thm:converge-alm}, induces both $x^k \rightarrow \bar{x}$ and $\bar{x}^k \rightarrow \bar{x}$. If $\operatorname{dist}\left(y^k, Z\right) \rightarrow 0$ Q-linearly at a rate $\rho$ as $y^k \rightarrow \bar{y}$, then $\bar{x}^k \rightarrow \bar{x}$ R-linearly at that rate. Moreover, by requiring
\begin{equation}\label{eq:alm-stopping-vs}
\left\|\operatorname{grad}_x l^{\rho_k}\left(x^{k+1}, y^k\right)\right\| \leq c\left\|y^{k+1}-y^k\right\| \;\text{for some fixed}\; c,
\end{equation}
if $y^k \rightarrow \bar{y}$ Q-linearly at a rate $\rho$, then $x^k \rightarrow \bar{x}$ R-linearly at that rate.
\end{theorem}
\begin{proof}
By Corollary \ref{cor:alm-converge-strong-convex}, $x^k\to\bar{x}$. Let $\bar{\xi}^{k+1}$ denote the unique exact solution of the ALM subproblem in (\ref{eq:alm-tangent}) and $\bar{x}^{k+1}$ denote the exact solution of the RALM subproblem in (\ref{eq:alm-mani}), we have $\bar{x}^{k+1}=R_{\bar{x}}(\bar{\xi}^{k+1})$ and $ l_{R_{\bar{x}}}^{\rho_k}(\bar{\xi}^{k+1},y^k)=l^{\rho_k}(\bar{x}^{k+1},y^k)$. Since $\bar{\xi}^{k+1}$ minimize $ l_{R_{\bar{x}}}^{\rho_k}(\cdot,y^k)$ over $\mathcal{W}$,
$$
\left\langle \nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k}(\bar{\xi}^{k+1},y^k),0_{\bar{x}}-\bar{\xi}^{k+1}\right\rangle \geq 0.
$$
The strong convexity of $ l_{R_{\bar{x}}}^{\rho_k}(\cdot,y^k)$ gives us
$$
l_{R_{\bar{x}}}^{\rho_k}\left(0_{\bar{x}}, y^k\right) \geq l_{R_{\bar{x}}}^{\rho_k}\left(\bar{\xi}^{k+1}, y^k\right)+\left\langle \nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k}\left(\bar{\xi}^{k+1}, y^k\right) ,0_{\bar{x}}-\bar{\xi}^{k+1}\right\rangle +\frac{s}{2}\left\|0_{\bar{x}}-\bar{\xi}^{k+1}\right\|^2.
$$
Combining with (\ref{eq:hck-equ-lck}), we obtain
\begin{equation*}
\begin{aligned}
l^{\rho_k}\left(\bar{x}, y^k\right)= l_{R_{\bar{x}}}^{\rho_k}\left(0_{\bar{x}}, y^k\right)& \geq l_{R_{\bar{x}}}^{\rho_k}\left(\bar{\xi}^{k+1}, y^k\right)+\frac{s}{2}\left\|\bar{\xi}^{k+1}\right\|^2= H_{R_{\bar{x}}}^{c_k}(y^k)+\frac{s}{2}\left\|\bar{\xi}^{k+1}\right\|^2.
\end{aligned}
\end{equation*}
Moreover, $ H_{R_{\bar{x}}}^{c_k}(y^k)\to l^{\bar{\rho}}(\bar{x},\bar{y})$ through the proof of Theorem \ref{thm:converge-alm}, implying $l^{\rho_k}\left(\bar{x}^{k+1}, y^k\right)\to l^{\bar{\rho}}(\bar{x},\bar{y})$ and hence $\bar{x}^{k+1}\to\bar{x}$.
By the proof of \cite[Theorem 2]{R22}, the strong convexity of $ l_{R_{\bar{x}}}^{\rho}(\cdot,y)$ corresponds to the Lipschitz property with modulus $s^{-1}$ of the mapping $\lambda(v,y):=\argmin_{\xi\in\mathcal{W}}\{ l_{R_{\bar{x}}}^{\rho}(\xi,y)-\left\langle v,\xi\right\rangle \} $. Therefore,
$$
d(\bar{x}^{k+1},\bar{x})=\|\bar{\xi}^{k+1}\|\leq \dfrac{1}{s}\operatorname{dist}(y^k,Z),
$$
and the Q-linear convergence of $\{y^k\}$ means that $\bar{x}^{k}\to\bar{x}$ R-linearly.
Since $0_{\bar{x}}=\lambda(0,\bar{y})$ and $\xi^{k+1}=\lambda\left( \nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k}(\xi^{k+1},y^k),y^k\right) $, the Lipschitz property of $\lambda(v,y)$ yields
$$
d(x^{k+1},\bar{x})=\|\xi^{k+1}-0_{\bar{x}}\|\leq \dfrac{1}{s}\left( \|\nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k}(\xi^{k+1},y^k)\|+\|y^k-\bar{y}\|\right).
$$
Using the facts that $x^{k+1}= R_{\bar{x}}\xi^{k+1}$ and $\{\xi^k\}$ sequence is generated in the closed set $\mathcal{W}$, under the assumption (\ref{eq:alm-stopping-vs}), there exists a positive constant $L>0$, such that
\begin{equation}\label{eq:alm-stopping-tangent}
\left\|\nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k}(\xi^{k+1},y^k) \right\| \leq L\left\|\operatorname{grad}_x l^{\rho_k}\left(x^{k+1}, y^k\right)\right\| \leq Lc\left\|y^{k+1}-y^k\right\|.
\end{equation}
Therefore, we further have
$$
\begin{aligned}
s^2d^2(x^{k+1},\bar{x})&\leq L^2c^2\left\|y^{k+1}-y^k\right\|^2+ \left\|y^{k}-\bar{y}\right\|^2\\
&\leq L^2c^2\big( \dfrac{\left\|y^{k+1}-\bar{y}\right\|}{\left\|y^{k}-\bar{y}\right\|}+1\big) \left\|y^{k}-\bar{y}\right\|^2+\left\|y^{k}-\bar{y}\right\|^2,
\end{aligned}
$$
while the Q-linear convergence of $\{y^k\}$ implies there exists $0<r<1$, such that $
\displaystyle\limsup_{k}\dfrac{\left\|y^{k+1}-\bar{y}\right\|}{\left\|y^{k}-\bar{y}\right\|}=r$. Thus, $x^{k}$ converge to $ \bar{x}$ R-linearly.
\end{proof}
\begin{remark}\label{rm:srcq-sosc-ssosc}
In %paper
\cite{ZBD22}, a local primal-dual Q-linear convergence rate for RALM is obtained under the M-SRCQ and M-SOSC, which require the multipliers to be unique. Although our work under the manifold strong variational sufficient condition can only achieve the local primal R-linear convergence rate, it relaxes the constraint of being a singleton for the multiplier set, which provides theoretical guarantees for more real-world cases.
\end{remark}
\begin{remark}\label{rm:kkt-residue}
Assume that the manifold strong variational sufficient condition holds at $(\bar{x},\bar{y})$ under $R_{\bar{x}}$. It follows from \cite[Theorem 3.1]{R22-ext} that the approximate error for $\bar{\xi}^{k+1}$ in (\ref{eq:alm-app-tangent}) can be replaced by
$$
\sqrt{\widetilde{\rho}^k}\left\|\nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k} \left(\xi^{k+1}, y^k\right)\right\| \leq \begin{cases}\text { (a)} & \varepsilon^{\prime}_k \\ \text { (b}) & \varepsilon^{\prime}_k \min \left\{1,\left\|\tilde{\rho}_{k} \nabla_y l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right)\right\|\right\} \\ \text { (c}) & \varepsilon^{\prime}_k \min \big\{1,\left\|\tilde{\rho}_{k}\nabla_y l_{R_{\bar{x}}}^{\rho_k}\left(\xi^{k+1}, y^k\right)\right\|^2\big\},\end{cases}
$$
where $\varepsilon^{\prime}_k=\varepsilon_k\sqrt{s}$ as $s$ is the modulus of the strong convexity of $\nabla_{\xi} l_{R_{\bar{x}}}^{\rho_k}\left(\cdot, y^k\right)$. Moreover, from (\ref{eq:alm-stopping-tangent}) we can replace (\ref{eq:alm-app-mani}) to
\begin{equation}\label{eq:alm-app-mani-grad}
\sqrt{\widetilde{\rho}^k}\left\|\operatorname{grad}_x l^{\rho_k}\left(x^{k+1}, y^k\right) \right\|\leq \begin{cases}\text { (a) } & \varepsilon^{\prime\prime}_k \\ \text { (b) } & \varepsilon^{\prime\prime}_k \min \left\{1,\left\|\tilde{\rho}_{k} \nabla_y l^{\rho_{k}}\left(x^{k+1}, y^k\right)\right\|\right\} \\ \text { (c) } & \varepsilon^{\prime\prime}_k \min \big\{1,\left\|\tilde{\rho}_{k}\nabla_y l^{\rho_{k}}\left(x^{k+1}, y^k\right)\right\|^2\big\}.\end{cases}
\end{equation}
where $\varepsilon^{\prime\prime}_k=\varepsilon_k\sqrt{s}/L$ as $L$ is the local Lipschitz modulus given in (\ref{eq:alm-stopping-tangent}). The stopping criteria %using
in Theorem \ref{thm:converge-alm} can be replaced by (\ref{eq:alm-app-mani-grad}).
\end{remark}
Define the KKT residual mapping as
$
R(x, y):=\left\|\operatorname{grad}_x L(x , y)\right\|+\left\|g(x)-\operatorname{prox}_\theta\left(g(x)+y\right)\right\|
$. Under stopping criterion (\ref{eq:alm-app-mani-grad}b) we are able to obtain the following proposition, which implies that the KKT residual of (\ref{eq:problem-mani}) also converges R-linearly.
\begin{proposition}\label{pro:kkt-r-linear}
Suppose that the manifold strong variational sufficiency holds at $(\bar{x},\bar{y})$ at level $\bar{\rho}$ and the approximation error is chosen as (\ref{eq:alm-app-mani-grad}b). If $\sqrt{\widetilde{\rho}^k} \varepsilon_k<1$ for sufficiently large $k$, then there exists $L_g > 0$ such that
$$
R\left(x^{k+1}, y^{k+1}\right) \leq w^k \operatorname{dist}\left(Y^k, Z\right),
$$
where $w^k=\left(\varepsilon_k^{\prime\prime} \sqrt{\tilde{\rho}_k}+(1+\bar{\rho}+L_g \bar{\rho})\left(\tilde{\rho}_k\right)^{-1}\right)\left(1-\sqrt{\tilde{\rho}_k} \varepsilon_k\right)^{-1}$.
\end{proposition}
The proof of Proposition \ref{pro:kkt-r-linear} is similar with the proof of \cite[Proposition 4.7]{WDZZ22} with the replacement of projection with proximal mapping in our case, and we omit the details here.
\section{Semismooth Newton method for RALM subproblem}\label{sec:semismooth}
After assuming obtaining the linear convergence results of the RALM, one remaining issue is how to solve the subproblem of updating $x$ in (\ref{eq:alm-mani}) efficiently. In \cite{ZBDZ21} the authors propose a globalized semismooth Newton method on Riemannian manifold, which could be well-suited for our problem. To begin, we provide the definition in \cite{OF20} of generalized covariant derivative for vector field of manifold.
\begin{definition}
Let $X$ be a locally Lipschitz vector field on $\mathcal{M}$. The B-derivative is a set-valued map $\partial_B X: \mathcal{M} \rightrightarrows \mathcal{L}(T \mathcal{M})$ with
$$
\partial_B X(x):=\big\{H \in \mathcal{L}\left(T_x \mathcal{M}\right): \exists\left\{x^k\right\} \subset \mathcal{D}_X, \quad \lim _{k \rightarrow+\infty} x^k=x, \quad H=\lim _{k \rightarrow+\infty} \nabla X\left(x^k\right)\big\},
$$
where the last limit means that $\left\|\nabla X\left(x^k\right)\left[P_{x x^k} v\right]-P_{x x^k} H v\right\| \rightarrow 0$ for all $v \in T_x \mathcal{M}$. The Clarke generalized covariant derivative is a set-valued map $\partial X: \mathcal{M} \rightrightarrows \mathcal{L}(T \mathcal{M})$ such that $\partial X(x)$ is the convex hull of $\partial_B X(x)$.
\end{definition}
\begin{algorithm}[ht]
\small \caption{Globalized semismooth Newton method for solving (\ref{eq:alm-mani}) at $(x^j,y^j,\rho_{j})$}
\label{alg:ssn}
\hspace*{0.02in} {\bf Input:}
Choose $x^0 \in \mathcal{M}, \bar{\nu} \in(0,1]$ and let $\left\{\eta_k\right\} \subset \mathbb{R}_{+}$be a sequence converging to 0. Set $\mu \in(0,1 / 2), \delta \in(0,1), m_{\max } \in \mathbb{N}$, and $p, \beta_0, \beta_1>0$, and set $k:=0$.
\begin{algorithmic}[1]
\State Let $X^j(x^k):=\operatorname{grad}_xl^{\rho_{j}}\left(x^k,y^j\right)$. Choose $G^k \in \partial X^j(x^k)$ and find $V^k \in T_{x^k} \mathcal{M}$ such that
$$
\left\|\left(G^k+\omega_k I\right) V^k+X^j\left(x^k\right)\right\| \leq \tilde{\eta}_k,
$$
where $\omega_k:=\left\|X^j\left(x^k\right)\right\|^{\bar{\nu}}, \tilde{\eta}_k:=\min \big\{\eta_k,\left\|X^j\left(x^k\right)\right\|^{1+\bar{\nu}}\big\}$.
\State If $V^k$ is not a sufficient descent direction of $\varphi$, i.e. it does not satisfy
$$
\left\langle-X^j\left(x^k\right), V^k\right\rangle \geq \min \big\{\beta_0, \beta_1\left\|V^k\right\|^p\big\}\left\|V^k\right\|^2,
$$
then, we set $V^k$ to be $-X^j\left(x^k\right)$.
Next, find the minimum non-negative integer $m_k$ such that
$$
l^{\rho_{j}}\left(R_{x^k}\left(\delta^{m_k} V^k\right),y^j\right) \leq l^{\rho_{j}}\left(x^k,y^j\right)+\mu \delta^{m_k}\left\langle X\left(x^k\right), V^k\right\rangle.
$$
\State Set $x^{k+1}=R_{x^k}\left(\delta^{m_k} V^k\right)$.
\State Set $k=k+1$ and go to step 2.
\end{algorithmic}
\end{algorithm}
If a retraction $R$ additionally satisfies $
\left.\frac{\mathrm{D}^2}{\mathrm{~d} t^2} R(t \xi)\right|_{t=0}=0$ for any $\xi \in T_x \mathcal{M}$, where $\dfrac{\mathrm{D}^2}{\mathrm{~d} t^2} \gamma$ denotes acceleration of the curve $\gamma$, then we call it a second-order retraction. The second-order retraction can be interpreted as the approximation of exponential mappings and includes retractions such as the polar retraction for Stiefel manifold and the projective retraction for fixed rank manifold, see \cite[Exmaple 4.12]{AM12} for more details. In this section, we always assume that the retraction is second-order.
The globalized semismooth Newton method for solving RALM subproblem (\ref{eq:alm-mani}) is given in Algorithm \ref{alg:ssn}. Let $R_{\bar{x}}$ be a second-order retraction. Suppose that $\theta$ is a polyhedral convex function or is the indicator function of second-order cone or positive semidefinite cone, it follows Theorem \ref{thm:strong-vari-suff-ssosc} that the M-SSOSC is equivalent to the positive definiteness of the elements in Hessian bundle of $l^{\rho}_{R_{\bar{x}}}(0_{\bar{x}},\bar{y})$ for $\rho$ sufficiently large. Moreover, we shall show that this condition also is sufficient to the superlinear convergence of Algorithm \ref{alg:ssn} by employing following proposition.
\begin{proposition}\label{pro:equi-general-jacobian}
Let $(\bar{x}, \bar{y})$ satisfy the condition \eqref{eq:first-opt-mani}. Assume that the manifold $\mathcal{M}$ is an embedded submanifold, and retraction $R_{\bar{x}}$ is second-order, then we have
$$
\partial \nabla_{\xi}l_{R_{\bar{x}}}^{\rho^j}(0_{\bar{x}},\bar{y})=\partial X^j(\bar{x}),
$$
where $X^j(\bar{x})$ is given in Algorithm \ref{alg:ssn}.
\end{proposition}
\begin{proof}
Now for any $G\in \partial X^j(\bar{x})$, we claim that $G\in \partial \nabla_{\xi}l_{R_{\bar{x}}}^{\rho^j}(0_{\bar{x}},\bar{y})$, or equivalently,
$$
\lim_{\xi_k\to 0_{\bar{x}}} \left\|\nabla_{\xi\xi}^2l_{R_{\bar{x}}}^{\rho^j}\left(\xi^k\right)\left[ v\right]-Gv\right\|= 0.
$$
Since $l_{R_{\bar{x}}}^{\rho^j}(\xi)=l^{\rho^j}\circ R_{\bar{x}}(\xi)$, for any $v\in T_{\bar{x}}\mathcal{M}$ we have
\begin{equation}\label{eq:hess-euc-inpro}
\begin{aligned}
&\big\langle \nabla_{\xi\xi}^2l_{R_{\bar{x}}}^{\rho^j}\big(\xi^k\big)\big[ v\big],v \big\rangle \\
=& \dfrac{d^2}{dt^2}(l^{\rho^j}\circ R_{\bar{x}})(\xi+tv)\Big |_{t=0}=\dfrac{d}{dt}\big(\dfrac{d}{dt}l^{\rho^j}\left(R_{\bar{x}}(\xi+tv)\right)\big)\Big |_{t=0}\\
=&\dfrac{d}{dt}Dl^{\rho^j}\left(R_{\bar{x}}(\xi+tv)\right)\big[\dfrac{d}{dt}R_{\bar{x}}(\xi+tv)\big]\Big |_{t=0}\\
=&\Big\langle\dfrac{D}{dt}\operatorname{grad}l^{\rho^j}\big(R_{\bar{x}}(\xi+tv)\big),DR_{\bar{x}}(\xi)[v]\Big\rangle+\Big\langle\operatorname{grad}l^{\rho^j}\big(R_{\bar{x}}(\xi+tv)\big),\dfrac{D^2}{dt^2}R_{\bar{x}}(\xi+tv)\Big\rangle\Big |_{t=0}\\
=& \Big\langle\nabla\operatorname{grad}l^{\rho^j}(R_{\bar{x}}(\xi))\big[DR_{\bar{x}}(\xi)v\big],DR_{\bar{x}}(\xi)v\Big\rangle+\Big\langle\operatorname{grad}l^{\rho^j}\big(R_{\bar{x}}(\xi+tv)\big),\dfrac{D^2}{dt^2}R_{\bar{x}}(\xi+tv)\Big\rangle\Big |_{t=0}.
\end{aligned}
\end{equation}
By the definition of retraction and $\mathcal{M}$ is an embedded submanifold, for any $v\in T_{\bar{x}}\mathcal{M}$, we have
$DR_{\bar{x}}(\xi)v=D\operatorname{exp}_{\bar{x}}(\xi)v+O(\|\xi\|)$ when $\xi\to 0_{\bar{x}}$.
Therefore, the first term of the last equation in \eqref{eq:hess-euc-inpro} can be written as
\begin{equation}\label{eq:second-appro}
\begin{aligned}
&\left\langle\nabla\operatorname{grad}l^{\rho^j}(R_{\bar{x}}(\xi))\left[D\operatorname{exp}_{\bar{x}}(\xi)v\right],D\operatorname{exp}_{\bar{x}}(\xi)v\right\rangle+O(\|\xi\|)\\
=&\left\langle\nabla\operatorname{grad}l^{\rho^j}(R_{\bar{x}}(\xi))\left[P_{\bar{x}R_{\bar{x}}(\xi)}v\right],P_{\bar{x}R_{\bar{x}}(\xi)}v\right\rangle+O(\|\xi\|).
\end{aligned}
\end{equation}
Moreover, the right term of the last equation in \eqref{eq:hess-euc-inpro} equals to zero if $\xi_k\to 0_{\bar{x}}$ and $t\to 0$ as $R_{\bar{x}}$ is a second-order retraction. Thus, for any $v\in T_{\bar{x}}\mathcal{M}$,
$$
\begin{aligned}
\big\langle\nabla_{\xi\xi}^2l_{R_{\bar{x}}}^{\rho^j}\big(\xi^k\big)\big[ v\big]-Gv,v\big\rangle
= &\;\big\langle P_{\bar{x}R_{\bar{x}}(\xi^k)}\nabla_{\xi\xi}^2l_{R_{\bar{x}}}^{\rho^j}\big(\xi^k\big)\big[ v\big]-\operatorname{Hess}l^{\rho^j}\big(R_{\bar{x}}(\xi^k)\big)\big[P_{\bar{x}R_{\bar{x}}(\xi^k)} v\big],P_{\bar{x}R_{\bar{x}}(\xi^k)}v\big\rangle\\
&+\big\langle\operatorname{Hess}l^{\rho^j}\big(R_{\bar{x}}(\xi^k)\big)\big[ P_{\bar{x}R_{\bar{x}}(\xi^k)}v\big]-P_{\bar{x}R_{\bar{x}}(\xi^k)}Gv,P_{\bar{x}R_{\bar{x}}(\xi^k)}v\big\rangle.\\
\end{aligned}
$$
The first equality is obatined since the parallel transport is isometry. By (\ref{eq:hess-euc-inpro}), (\ref{eq:second-appro}) and $G\in \partial X^j(\bar{x})$, the above equation converges to 0 when $\xi^k\to 0_{\bar{x}}$. The arbitrary taken $v$ implies that
$$
\lim_{\xi_k\to 0_{\bar{x}}} \big\|\nabla_{\xi\xi}^2l_{R_{\bar{x}}}^{\rho^j}\big(\xi^k\big)\big[ v\big]-Gv\big\|= 0.
$$
Similarly, $\partial \nabla_{\xi}l_{R_{\bar{x}}}^{\rho^j}(0_{\bar{x}},\bar{y})\subseteq \partial X^j(\bar{x})$. This implies $\partial \nabla_{\xi}l_{R_{\bar{x}}}^{\rho^j}(0_{\bar{x}},\bar{y})=\partial X^j(\bar{x})$.
\end{proof}
%Under Proposition \ref{pro:equi-general-jacobian}, we are able to derive the following corollary.
\begin{theorem}\label{cor:vs-positive-semi}
Let $(\bar{x}, \bar{y})$ satisfy the condition \eqref{eq:first-opt-mani}. The manifold strong variational sufficient condition at $(\bar{x}, \bar{y})$ if and only if all elements in $\partial X^j(\bar{x})$ are positive definiteness.
\end{theorem}
\begin{proof}
By Theorem \ref{thm:strong-vari-suff-ssosc}, the M-SSOSC guarantee the positive definiteness of the elements in $\partial \nabla_{\xi}l_{R_{\bar{x}}}^{\rho^j}(0_{\bar{x}},\bar{y})$. Follows from Proposition \ref{pro:equi-general-jacobian} we have the conculsion.
\end{proof}
Now by Theorem \ref{cor:vs-positive-semi} and \cite[Theorem 4.3]{ZBDZ21} we obtain the following convergence result of Algorithm \ref{alg:ssn}.
\begin{proposition}\label{pro:ssn-convergence}
Suppose the manifold strong variational sufficient condition holds at a stationary point $(\bar{x}, \bar{y})$. Let $\left\{x^k\right\}$ be the sequence generated by Algorithm \ref{alg:ssn} and the retraction $R_{\bar{x}}$ to be second-order. Suppose there exists $\delta>0$ such that $\Omega:=\left\{x \in \mathcal{M}: \varphi(x) \leq \varphi\left(x^0\right)+\delta\right\}$ is compact. Denote $\hat{x}$ be any accumulation point of $\left\{x^k\right\}$. If $X^j$ is semismooth at $\hat{x}$ with order $\nu$ with respect to $ \partial X^j$, then we have $x^k \rightarrow \hat{x}$ as $k \rightarrow \infty$ and $\hat{x}$ is optimal for subproblem (\ref{eq:alm-mani}). Moreover, for sufficiently large $k$, it holds
$$
d\left(x^{k+1}, \hat{x}\right) \leq O\big(d\left(x^k, \hat{x}\right)^{1+\min \{\nu, \bar{\nu}\}}\big),
$$
where $\bar{\nu} \in(0,1]$ is the parameter defined in Algorithm \ref{alg:ssn}.
\end{proposition}
\begin{remark}\label{rm:ssosc-ssn-positive}
Based on Theorem \ref{cor:vs-positive-semi}, it can be concluded that the positive definiteness of the generalized Hessian of the augmented Lagrangian function is equivalent to the manifold strong variational sufficient condition of problem (\ref{eq:problem-mani}) at the stationary point. This relationship underscores the pivotal role played by manifold strong variational sufficiency in ensuring the efficiency of the semi-smooth Newton method for solving the RALM subproblem.
\end{remark}
\section{Applications and numerical experiments}\label{sec:numerical}
Based on Proposition \ref{pro:kkt-r-linear}, we are able to verify the convergence rate by using the KKT residual $R(x,y)$. In this section, we will present numerical experiments on fixed rank manifold and Stiefel manifold, and illustrate the local linear rate when the M-SSOSC is satisfied. All codes are implemented in Matlab (R2021b) and all the numerical experiments are run under a 64-bit MacOS on an Intel Cores i5 2.4GHz CPU with 16GB of memory.
\subsection{Robust matrix completion}
We are now considering the robust matrix completion (RMC) problem proposed in \cite{CA16}. For a given $A\in \mathbb{R}^{m\times n}$, let $g(X)=P_{\Omega}(X-A)$ and $\theta(\cdot)=\mu\|\cdot\|_{1}$. Here, $P_{\Omega}$ is the projector defined by $\left(P_{\Omega}(X)\right)_{i j}=X_{i j}$ if $(i, j) \in \Omega$ and $0$ otherwise. By setting $\mathcal{M}=Fr(m,n,r):=\{X\in \mathbb{R}^{m\times n}:\operatorname{rank}(X)=r\}$, we obtain the following robust matrix completion problem
\begin{equation}\label{eq:rmc-full}
\begin{array}{ll}
\min _{X \in \mathbb{R}^{m \times n}} & \left\|P_{\Omega}(X-A)\right\|_{1} \\
\text { s.t. } & X \in Fr(m,n,r).
\end{array}
\end{equation}
In comparison with the matrix completion using the Frobenius norm as an objective function, the $l_1$-norm is expected to due with an inexact data $A$ with some extreme outliers.
It is known in \cite{V13} that the tangent space of $\mathcal{M}=F(m,n,r)$ at a point $X=USV^{\top}$ is
$$\small
T_{X} \mathcal{M}=\left\{\left[\begin{array}{ll}
U & U_{\perp}
\end{array}\right]\left[\begin{array}{cc}
\mathbb{R}^{r \times r} & \mathbb{R}^{r \times(n-r)} \\
\mathbb{R}^{(m-r) \times r} & 0^{(m-r) \times(n-r)}
\end{array}\right]\left[\begin{array}{ll}
V & V_{\perp}
\end{array}\right]^{T}\right\}
$$
and the normal space is
$$\small
N_{X} \mathcal{M}=\left\{\left[\begin{array}{ll}
U & U_{\perp}
\end{array}\right]\left[\begin{array}{cc}
0^{r \times r} & 0^{r \times(n-r)} \\
0^{(m-r) \times r} & \mathbb{R}^{(m-r) \times(n-r)}
\end{array}\right]\left[\begin{array}{ll}
V & V_{\perp}
\end{array}\right]^{\top}\right\}.
$$
The projection to tangent space is given by $
\Pi_{X}(Y)=P_{U} Y P_{V}+P_{U}^{\perp} Y P_{V}+P_{U} Y P_{V}^{\perp}$.
The Lagrangian of (\ref{eq:rmc-full}) can be written as $L(X,y) = \left\langle P_{\Omega}(X-A), y\right\rangle$. Then the KKT condition takes the form of
\begin{equation}\label{eq:rmc-kkt}
\left\{\begin{array}{l}
\Pi_{X} P_{\Omega} (y) =0, \\
y \in \partial\| P_{\Omega}(X-A)\|_{1}.
\end{array}\right.
\end{equation}
By \cite[Section 3]{HLWY20}, the Hessian of a function $f$ on $Fr(m,n,r)$ at $X=U\Sigma V$ can be written as
$$
\operatorname{Hess}f(X)[\xi]=U \hat{M} V^{\top}+\hat{U}_p V^{\top}+U \hat{V}_p^{\top}\quad\forall \xi \in T_X Fr(m,n,r),
$$
where
$$\small
\begin{aligned}
\hat{M}&=M\left(\nabla^2 f(X)[\xi], X\right),\\
\hat{U}_p&=U_p\left(\nabla^2 f(X)[\xi] ; X\right)+P_U^{\perp} \nabla f(X) V_p(\xi ; X)/\Sigma,\\
\hat{V}_p&=V_p\left(\nabla^2 f(X)[\xi] ; X\right)+P_V^{\perp} \nabla f(X) U_p(\xi; X) / \Sigma,
\end{aligned}
$$
in which $M(Z ; X):=U^{\top} Z X$, $U_p(Z ; X)=P_U^{\perp} Z V$ and $V_p(Z ; X)=P_V^{\perp} Z^{\top} U$. While $\nabla_X L(\bar{X},\bar{y})=P_{\Omega}(\bar{y})$ and $\nabla_{XX}^2 L(\bar{X},\bar{y})=0$, it holds
$$
\begin{aligned}
\left\langle \xi, \operatorname{Hess}L(\bar{X},\bar{y})[\xi]\right\rangle &=\left\langle \xi,P_U^{\perp}P_{\Omega}(\bar{y})P_V^{\perp}\xi^{\top}U\Sigma^{-1}V^{\top}+U\Sigma^{-1}V^{\top}\xi^{\top}P_U^{\perp}P_{\Omega}(\bar{y})P_V^{\perp}\right\rangle\\
&=2\operatorname{tr}\left( \xi^{\top}P_U^{\perp}P_{\Omega}(\bar{y})P_V^{\perp}V^{\perp}\xi^{\top}U\Sigma^{-1}V^{\top}\right)=2\operatorname{tr}\left( \xi^{\top}P_{\Omega}(\bar{y})\xi^{\top}U\Sigma^{-1}V^{\top}\right),
\end{aligned}
$$
where the last equality is obtained by the KKT condition (\ref{eq:rmc-kkt}) that $\Pi_{\bar{X}}P_{\Omega}(\bar{y})=0$. Moreover, since $g(X)=P_{\Omega}(X-A)$, for any $\xi\in T_{X}\mathcal{M}$ we have $Dg(X)\xi=g^{\prime}(X)\xi=P_{\Omega}(\xi)$. We can further obtain that $\mathcal{C}_{\theta, g}(X, y)=\left\lbrace d\in\mathbb{R}^{m\times n}\mid \theta^{\prime}(P_{\Omega}(X-A) ; d) = \left\langle d,y \right\rangle \right\rbrace $, in which
$$\small
\theta^{\prime}(P_{\Omega}(X-A) ; d)
= \sum_{P_{\Omega}(X-A)_{ij}=0}|d_{ij}|+\sum_{P_{\Omega}(X-A)_{ij}>0}d_{ij}-\sum_{P_{\Omega}(X-A)_{ij}<0}d_{ij}.
$$
Therefore, the M-SSOSC for the RMC problem (\ref{eq:rmc-full}) at the KKT point $(\bar{X}, \bar{y})$ holds if for any $\xi\in T_{\bar{X}}\mathcal{M}$ satisfying $ P_{\Omega}(\xi)\in \operatorname{aff}\mathcal{C}_{\theta, g}(\bar{X}, \bar{y})\backslash\{0\}$,
$
\operatorname{tr}\left( \xi^{\top}P_{\Omega}(\bar{y})\xi^{\top}U\Sigma^{-1}V^{\top}\right)>0
$ holds.
We consider a basic example of problem (\ref{eq:rmc-full}), where $\Omega$ is the full index set. Let $U= \small\begin{bmatrix}
1& 0 &0 &0&0\\
0 &-\dfrac{\sqrt{2}}{2}&\dfrac{\sqrt{2}}{2}&0&0 \\
0 &\dfrac{\sqrt{2}}{2}&\dfrac{\sqrt{2}}{2}&0&0
\end{bmatrix}^{T}$, $V=\small \begin{bmatrix}
1& 0 &0 &0&0\\
0 &0.6&-0.8&0&0 \\
0 &0.8&0.6&0&0
\end{bmatrix}^{T}$ and $S=\small\begin{bmatrix}
1&0&0\\0&2&0\\0&0&3
\end{bmatrix}$. The observed matrix is set to $A=A_{\text{ex}}+E_{\text{out}}$, where $A_{\text{ex}}=USV^{T}$ is the assumed ground truth and $E_{\text{out}}$ is a matrix with random entries added only in the lower right $2\times 2$ submatrix. Since $A_{\text{ex}}$ is of rank $r=3$, $\bar{X}=A_{\text{ex}}$ is a solution of this problem. Consider if $(\bar{X},y)$ satisfies (\ref{eq:rmc-kkt}), then $y$ can be chosen as $y_{ij}=\operatorname{sgn}(E^{ij}_{\operatorname{out}})$. In this case, $\mathcal{ C}_{\theta,g}(\bar{X},y)=\{d\in\mathbb{R}^{m\times n}\mid d_{ij}\in \mathbb{R} \text{ if } E^{ij}_{\operatorname{out}}\neq 0, d_{ij}=0 \text{ if } E^{ij}_{\operatorname{out}}=0\}$, and $\operatorname{aff}\mathcal{ C}_{\theta,g}(\bar{X},y)=\mathcal{ C}_{\theta,g}(\bar{X},y)$. The nonzero position of $E_{\operatorname{out}}$ implies that only $0_{\bar{X}}$ can satisfy $P_{\Omega}(\xi)\in \operatorname{aff}\mathcal{C}_{\theta, g}(\bar{X}, y)$. Therefore, the M-SSOSC holds at $(\bar{X},y)$. Now we can apply the inexact RALM to this problem and obtain Figure \ref{fig:rmc-simple}, which is the variation of KKT residue.
\begin{figure}[h]
\centering
\includegraphics[scale=0.15]{figure/rmcfkkt.png}
\caption{KKT residues of the RMC problem generated by inexact RALM }
\label{fig:rmc-simple}
\end{figure}
\subsection{Compressed modes}
In this section, we will consider the compressed modes (CM) problem. Let $H$ be a discretization of the Hamilton operator, then the CM problem is formulated as follow (\cite{OLCO13}):
\begin{equation}\label{eq:cm}
\begin{array}{ll}
\min_{X \in \mathbb{R}^{n \times r}} & \operatorname{tr}(X^{\top}HX)+\mu\left\|X\right\|_{1} \\
\text { s.t. } & X \in \operatorname{St}(n,r).
\end{array}
\end{equation}
By setting $f(X)=\operatorname{tr}(X^{\top}HX)$, $\theta=\mu\|\cdot\|_1$, $g(X)=X$ and $\mathcal{M}= \operatorname{St}(n,r)=\{X|X^{\top}X=I_r\}$, this is of the form of (\ref{eq:problem-mani}).
The tangent space of $\mathcal{ M }=\operatorname{St}(n,r)$ at $X$ is
\begin{equation}\label{eq:stiefel-tangent}
T_X \mathcal{ M }=\left\{\xi \in \mathbb{R}^{n \times r}: X^{\top}\xi+\xi^{\top}X=0\right\},
\end{equation}
and the
the projection to tangent space is given by $\Pi_X Y=Y-X\operatorname{sym}(X^{\top}Y)$.
The Lagrangian of (\ref{eq:cm}) can be written as $L(X,y) =\operatorname{tr}(X^{\top}HX) +\left\langle X, y\right\rangle$. Then the KKT condition is
\begin{equation}\label{eq:cm-kkt}
\left\{\begin{array}{l}
\Pi_{X}( 2HX+y )=0, \\
y \in \mu\partial\|X\|_{1}.
\end{array}\right.
\end{equation}
The Euclidean gradient of $L(X,y)$ is $\nabla_{X} L(X,y)= 2HX+y$,
and for any $\xi\in T_X\mathcal{ M }$, the Euclidean Hessian of $L(X,y)$ is
$\nabla_{XX}^2 L(X,y)[\xi]= 2H\xi$.
Therefore, by \cite{HLWY20} the Riemannian gradient and Hessian of $L(X,y)$ can be computed as
$$
\begin{aligned}
\operatorname{grad}L(X,y) &= \Pi_{X}( 2HX+y ),\\
\operatorname{Hess} L(X,y)\xi &= \Pi_{X}(2H\xi-\xi \operatorname{sym}(X^{\top}\nabla_{X}L(X,y) ).
\end{aligned}
$$
Moreover, for any $\xi\in T_{w}\mathcal{ M }$, we have
\begin{equation}\label{eq:cm-hess-1}
\begin{aligned}
\left\langle \xi, \operatorname{Hess}L(w,y)\xi\right\rangle
&=\left\langle \xi,2H\xi-\xi \operatorname{sym}(X^{\top}\nabla_{X}L(X,y) \right\rangle \\
&=\left\langle \xi,2H\xi \right\rangle -\left\langle \xi,2\xi X^{\top}HX\right\rangle -\left\langle \xi,\xi \operatorname{sym}(X^{\top}y) \right\rangle.
\end{aligned}
\end{equation}
By the KKT condition (\ref{eq:cm-kkt}), we have $2HX+y\in N_X\mathcal{ M}$. Equivalently, there exists a symmetric matrix $S$, such that $2HX+y=XS$. Combining with (\ref{eq:cm-hess-1}) we can obtain
$$
\begin{aligned}
\left\langle \xi, \operatorname{Hess}L(w,y)\xi\right\rangle
%&=\left\langle \xi,2H\xi \right\rangle -\left\langle \xi,2\xi X^{\top}HX\right\rangle -\left\langle \xi,\xi \operatorname{sym}\left( X^{\top}y\right) \right\rangle\\
&= \left\langle \xi,2H\xi \right\rangle -\left\langle \xi,2\xi X^{\top}HX\right\rangle -\left\langle \xi,\xi \operatorname{sym}\left( X^{\top}(XS-2HX)\right) \right\rangle \\
%&= \left\langle \xi,2H\xi \right\rangle -\left\langle \xi,2\xi X^{\top}HX\right\rangle -\left\langle \xi,\xi S\right\rangle +\left\langle \xi,\xi 2X^{\top}HX\right\rangle\\
&=\operatorname{tr}(\xi^{\top}H\xi)-\operatorname{tr}(\xi^{\top}\xi S).
\end{aligned}
$$
We further obtain the critical cone of $\theta$ and $g$ as $\mathcal{ C}_{\theta,g}(X,y)=\{d\in \mathbb{R}^{n\times r}|\theta^{\prime}(X,d)=\left\langle d,y \right\rangle\}$, where
$$
\theta^{\prime}(X ; d)= \sum_{X_{ij}=0}|d_{ij}|+\sum_{X_{ij}>0}d_{ij}-\sum_{X_{ij}<0}d_{ij}.
$$
The affine hull of $\mathcal{ C}_{\theta,g}(X,y)$ is then given by
$$
\operatorname{aff}\mathcal{ C}_{\theta,g}(X,y)=\left\lbrace d\in \mathbb{R}^{n\times r}| d_{ij}=0\;\text{if}\; y_{ij}\neq \pm \mu ,\;d_{ij}\in \mathbb{R} \;\text{if}\; y_{ij}=\pm \mu\right\rbrace.
$$
Therefore, the M-SSOSC for CM problem at the KKT point $(\bar{X}, \bar{y})=(\bar{X}, -2H\bar{X}+\bar{X}\bar{S})$ holds if for any $\xi\in T_{\bar{X}}\mathcal{M}$ satisfying $ \xi\in \operatorname{aff}\mathcal{C}_{\theta, g}(\bar{X}, \bar{y})\backslash\{0\}$, $
\operatorname{tr}(\xi^{\top}H\xi)-\operatorname{tr}(\xi^{\top}\xi \bar{S})>0$ holds.
In \cite{ZBDZ21}, the authors consider setting the CM problem to solve Schrödinger equation of 1D free-electron model with periodic boundary condition:
\begin{equation}\label{eq:cm-50}
-\frac{1}{2} \Delta \phi(x)=\lambda \phi(x), \quad x \in[0,50]
\end{equation}
and numerically, they find that the smallest eigenvalue of $H_k \in \partial \operatorname{grad} L^{\rho_{k}}\left(X_k,y^k\right)$ is always larger than zero, which implies that the M-SSOSC may be satisfied in this case. Here we use a simple example to illustrate this conjecture.
Consider the Schrödinger equation of with boundary condition when $x\in [0,2]$. Discretize the domain$[0, 2]$ into $n=4$ nodes and let $H$ be the discretized version of $-\frac{1}{2} \Delta$. Then $H=-\small\begin{bmatrix}
-4& 2&0&2\\
2&-4&2&0\\
0&2&-4&2\\
2&0&2&-4
\end{bmatrix}$. For $r=2$, it can be verified that $\bar{X}=\small\begin{bmatrix}
0&0&\sqrt{2}/2&\sqrt{2}/2\\
\sqrt{2}/2&\sqrt{2}/2&0&0
\end{bmatrix}^{\top}$ is a stationary point of (\ref{eq:cm}), and one of the corresponding multiplier $\bar{y}$ is given by $\bar{y}=\mu\small\begin{bmatrix}
0&0&1&1\\
1&1&0&0
\end{bmatrix}^{\top}$. Moreover, if rewrite $\bar{y}=\bar{X}\bar{S}-2H\bar{X}$, then $\bar{S}=\small\begin{bmatrix}
-4+\sqrt{2}\mu&4\\
4&-4+\sqrt{2}\mu
\end{bmatrix}$. The affine hull of $\mathcal{ C}_{\theta,g}(\bar{X},\bar{y})$ is now written as
$
\operatorname{aff}\mathcal{ C}_{\theta,g}(\bar{X},\bar{y})=\left\lbrace \begin{bmatrix}
0&0&\xi_1&\xi_2\\
\xi_3&\xi_4&0&0
\end{bmatrix}^{\top}\Big|\xi_i\in \mathbb{R},i=1,2,3,4\right\rbrace
$.
For any $\xi\in \operatorname{aff}\mathcal{ C}_{\theta,g}(\bar{X},\bar{y})$ satisfying $\xi\in T_{\bar{X}}\mathcal{ M}$, by (\ref{eq:stiefel-tangent}) we further obtain $\xi=\small\begin{bmatrix}
0&0&\xi_1&-\xi_1\\
\xi_2&-\xi_2&0&0
\end{bmatrix}^{\top}$. Therefore, for any $\xi\in T_{\bar{X}}\mathcal{M}$ satisfying $ \xi\in \operatorname{aff}\mathcal{C}_{\theta, g}(\bar{X}, \bar{y})\backslash\{0\}$, we have
$$
\operatorname{tr}(\xi^{\top}H\xi)-\operatorname{tr}(\xi^{\top}\xi \bar{S})=12(\xi_1^2+\xi_2^2)-2(-4+\sqrt{2}\mu)(\xi_1^2+\xi_2^2)=(20-2\sqrt{2}\mu)(\xi_1^2+\xi_2^2).
$$
As long as $\mu<5\sqrt{2}$, the M-SSOSC is satisfied for this problem. From Corollary \ref{cor:vs-positive-semi}, the general Jacobian used in the semisooth newton is positve definite. Now setting $\mu=0.8$ and apply RALM, we obtain Figure \ref{fig:cm-simple}, which shows the linear rate of KKT residue.
\begin{figure}[ht]
\centering
\includegraphics[scale=0.15]{figure/cmkktsimple.png}
\caption{KKT residues of CM generated by inexact RALM }
\label{fig:cm-simple}
\end{figure}
For larger $n$ and $r$, it seems difficult to prove the positive definiteness of the general Hessian used in Algorithm \ref{alg:ssn}. However, The minimal eigenvalue is observed to be positive in \cite[Table 5]{ZBDZ21}.
We also consider back to problem (\ref{eq:cm-50}) with shifting $n$ nodes and similarly observe that the smallest eigenvalues of the general Hessians used in the semismooth Newton methods are positive.
\begin{table}[ht]
\centering
\caption{The minimum eigenvalue of the general Hessian which is used in Algorithm \ref{alg:ssn} for (\ref{eq:cm}).}
\label{table:cm}
\begin{tabular}{c c|c c c c c}
\toprule
n & r & \multicolumn{5}{c}{mimimum eigenvalue }\\
\midrule
200 & 20 &0.3049 &0.3238& 0.1926 &0.2811 &0.1447\\
\midrule
500 & 20 &0.2755& 0.1089 & 0.1173 & 0.0982 &0.1087 \\
\midrule
1000 & 20 & 0.0266 & 0.1658& 0.0026 & 0.1684 & 0.2055 \\
\bottomrule
\end{tabular}
\end{table}
\begin{figure}[htbp]
\centering
\subfloat[n=200, r=20]{\label{fig:cm200}\includegraphics[width=0.3\linewidth]{figure/cmkkt200.png}}
\subfloat[n=500, r=20]{\label{fig:cm500}\includegraphics[width=0.3\linewidth]{figure/cmkkt500.png}}
\subfloat[n=1000, r=20]{\label{fig:cm1000}\includegraphics[width=0.3\linewidth]{figure/cmkkt1000.png}}
% \subcaptionbox{\label{fig:cm200}}{\includegraphics[width=0.9\linewidth]{figure/cmkkt200.png}}\hfill
% \subcaptionbox{\label{fig:cm500}}{\includegraphics[width=0.9\linewidth]{figure/cmkkt500.png}}\hfill
% \subcaptionbox{\label{fig:cm1000}}{\includegraphics[width=0.9\linewidth]{figure/cmkkt1000.png}}
%% \begin{subfigure}[b]{0.325\linewidth}
%% \centering
%% \includegraphics[width=0.9\linewidth]{figure/cmkkt500.png}
%% \caption{n=500, r=20}
%% \label{fig:cm500}
%% \end{subfigure}
%% \centering
%% \begin{subfigure}[b]{0.325\linewidth}
%% \centering
%% \includegraphics[width=0.9\linewidth]{figure/cmkkt1000.png}
%% \caption{n=1000, r=20}
%% \label{fig:cm1000}
%% \end{subfigure}
\caption{The KKT residues of compressed modes problems generated by inexact RALM}
\label{fig:cmfigure}
\end{figure}
Table \ref{table:cm} indicates that the manifold strong variational sufficiency might be satisfied at the obtained points. Therefore, we consider applying the RALM and all settings follow \cite{ZBDZ21}. The results of variations in KKT residues are reported in Figure \ref{fig:cmfigure}\footnote{The code of using semismooth Newton based RALM to solve compressed modes problems is provided in the published paper \cite{ZBDZ21}}.
%and for any $\xi\in T_w\mathcal{ M }$, the Hessian of $L(w,y)$ can be computed as
%$$
%\{operatorname\}{Hess} L(w,y)\xi= 2\begin{bmatrix}
%(I-bb^{\top})^2(\Omega^{\top}\Omega \xi_1+\Omega^{\top}\xi_2)-\xi_1 b^{\top}(\Omega^{\top}\Omega b+\Omega^{\top}x)\\
%\Omega \xi_1-\xi_2
%\end{bmatrix}.
%$$
%\subsection{Finding a sparse vector in a subspace}
%In this section, we will consider find a sparse vector in a given subspace. Let $\Omega \in \mathbb{R}^{p \times n}$ be an orthonormal basis for a subspace $\mathcal{V}$. This problem is a special application of the orthogonal dictionary learning (ODL) model and can be formulated as follow (\cite{QSW14}):
% \begin{equation}\label{eq:dpcp}
%\min _{b, x:\|b\|_2=1}\|\Omega b-x\|_2^2+\lambda\|x\|_1
% \end{equation}
%By setting $w=(b^{\top},x^{\top})^{\top}$, $Y=[\Omega -I]$, $A=[0\; I]$, $f(w)=\|Yw\|_2^2$, $\theta=\lambda\|\cdot\|_1$, $g(X)=Aw$ and $\mathcal{ M}=S^{n-1}\times \mathbb{R}^p$, this is of the form of (\ref{eq:problem-mani}).
%
%
%The tangent space of $\mathcal{ M }=S^{n-1}\times \mathbb{R}^p$ at $w$ is
%$$
%T_w \mathcal{ M }=\left\{\xi=(\xi_1^{\top},\xi_2^{\top})^{\top} \in \mathbb{R}^{(n+p) \times 1}: b^{\top}\xi_1=0\right\},
%$$
%and the projection to tangent space is given by
%$$
%\Pi_w Y=\left( Y_1\left(I-b b^{\top}\right),Y_2^{\top}\right)^{\top}.
%$$
%The Lagrangian of (\ref{eq:dpcp}) can be written as $L(w,y) =\|Yw\|_2^2 +\left\langle Aw, y\right\rangle$. It is easy to see that the KKT condition is
%\begin{equation}\label{eq:dpcp-kkt}
%\left\{\begin{array}{l}
%\Pi_{w}( 2Y^{\top}Yw+A^{\top}y )=0, \\
%y \in \partial\|Aw\|_{1}.
%\end{array}\right.
%\end{equation}
%
%The gradient of $L(w,y)$ is given by
%$$
%\operatorname{grad}_w L(w,y)=\begin{bmatrix}
%(I-bb^{\top})(2Y^{\top}Yw+A^{\top}y)_1\\
%(2Y^{\top}Yw+A^{\top}y)_2
%\end{bmatrix}=\begin{bmatrix}
%2(I-bb^{\top})(\Omega^{\top}\Omega b+\Omega^{\top}x)\\
%2(\Omega b-x)+y
%\end{bmatrix},
%$$
%and for any $\xi\in T_w\mathcal{ M }$, the Hessian of $L(w,y)$ can be computed as
%$$
%\operatorname{Hess} L(w,y)\xi= 2\begin{bmatrix}
%(I-bb^{\top})^2(\Omega^{\top}\Omega \xi_1+\Omega^{\top}\xi_2)-\xi_1 b^{\top}(\Omega^{\top}\Omega b+\Omega^{\top}x)\\
%\Omega \xi_1-\xi_2
%\end{bmatrix}.
%$$
%Therfore, for any $\xi\in T_{w}\mathcal{ M }$, we have
%$$
%\begin{aligned}
%\left\langle \xi, \operatorname{Hess}L(w,y)\xi\right\rangle
%&=2\left\langle \xi_1,(I-bb^{\top})^2(\Omega^{\top}\Omega \xi_1+\Omega^{\top}\xi_2)-\xi_1 b^{\top}(\Omega^{\top}\Omega b+\Omega^{\top}x)\right\rangle+2\left\langle \xi_2,\Omega \xi_1-\xi_2 \right\rangle \\
%&=\xi_1^{\top}\Omega^{\top}\Omega \xi_1-\xi_1^{\top}\xi_1b^{\top}\Omega^{\top}(\Omega b-x)-\xi_2^{\top}\xi_2.
%\end{aligned}
%$$
%
% By Theorem \ref{thm:strong-vari-suff-ssosc}, the SSOSC condition can be interpret as for any $\xi\in T_{\bar{w}}\mathcal{M}$ and all $G \in \overline{\nabla}^2 \operatorname{env}_{\rho}\theta\left(A^{\top}\bar{w}+\rho^{-1} \bar{y}\right)$,
% $$
% \left\langle \xi,YGY^{\top}\xi \right\rangle +\xi_1^{\top}\Omega^{\top}\Omega \xi_1-\xi_1^{\top}\xi_1b^{\top}\Omega^{\top}(\Omega b-x)-\xi_2^{\top}\xi_2>0,
% $$
% or equivalently,
% $$
%\xi_1^{\top}\Omega^{\top}(G+I)\Omega \xi_1+\xi_2^{\top}(G-I)\xi_2 -\xi_1^{\top}\xi_1b^{\top}\Omega^{\top}(\Omega b-x)>0,
% $$
%
% Firstly, we will focus on applying RALM to a simple case on sphere. Consider a special dataset $U= \begin{bmatrix}
% 1& 1 &1 &0\\
% 0 &0&0&0 \\
% 0 &0&0&8
% \end{bmatrix}$. It is easy to find that the first three points are contained in the same subspace and the last column is an outlier needs to be excluded. By easy calculation we can obtain the optimal solution $\bar{x}=[0,1,0]^{\top}$ and a corresponding multiplier $\bar{y}=[0,0,0]^{\top}$. Hence, the affine hull of critical cone $\operatorname{aff}\mathcal{ C}_{\theta,g}(\bar{x},\bar{y})$ contains only zero, and the only $\xi\in T_{\bar{x}}\mathcal{M}$ which satisfied $Dg(\bar{x})\xi\in \operatorname{aff}\mathcal{ C}_{\theta,g}(\bar{x},\bar{y})$ is the zero tangent vector. This implies that the M-SSOSC is satisfied and we can apply the inexact ALM. It is shown in Figure \ref{fig:dpcp-simple} of the linear rate of KKT residues.
%\begin{figure}[h]
% \centering
% \includegraphics[scale=0.15]{figure/dpcpkkt.png}
% \caption{the KKT residues of DPCP generated by inexact ALM }
% \label{fig:dpcp-simple}
%\end{figure}
%
\section{Conclusion}\label{sec:conclu}
This paper investigates the local convergence of RALM without assuming the uniqueness of multiplier. We devise a local equivalent problem on tangent space and introduce the manifold variational sufficient condition. It is shown that manifold strong variational sufficient condition is equivalent to the M-SSOSC under certain circumstances. Under this condition, a local augmented dual problem is formulated, consequently establishing the linear convergence rate of RALM. Furthermore, we prove that general Hessians used in the semismooth Newton method for solving the RALM subproblem are positive definite when the manifold strong variational sufficiency holds. The numerical experiments on various applications demonstrate the linear convergence rate.
However, there remain several unresolved issues in Riemannian nonsmooth optimization. For instance, verifying the retractional convexity for manifold functions presents a significant challenge and is waiting for future study. Additionally, while it is understood that the primal proximal point algorithm is equivalent to the dual ALM in Euclidean settings, the relationship between these two algorithms remains unknown under the retactional convexity in a Riemannian setting.
\bibliographystyle{siamplain}
\bibliography{vsbib2}
\end{document}
\begin{thebibliography}{10}
\bibitem{AH19}
{\sc P.-A. Absil and S.~Hosseini}, {\em {A collection of nonsmooth Riemannian
optimization problems}}, Nonsmooth Optimization and Its Applications,
(2019), pp.~1--15.
\bibitem{AMS09}
{\sc P.-A. Absil, R.~Mahony, and R.~Sepulchre}, {\em {Optimization algorithms
on matrix manifolds}}, Princeton University Press, 2009.
\bibitem{AM12}
{\sc P.-A. Absil and J.~Malick}, {\em {Projection-like retractions on matrix
manifolds}}, SIAM Journal on Optimization, 22 (2012), pp.~135--158.
\bibitem{AFL05}
{\sc D.~Azagra, J.~Ferrera, and F.~L{\'o}pez-Mesas}, {\em {Nonsmooth analysis
and Hamilton--Jacobi equations on Riemannian manifolds}}, Journal of
Functional Analysis, 220 (2005), pp.~304--361.
\bibitem{B82}
{\sc D.~P. Bertsekas}, {\em {Constrained optimization and Lagrange multiplier
methods}}, Academic press, 2014.
\bibitem{B20}
{\sc N.~Boumal}, {\em {An introduction to optimization on smooth manifolds}},
Cambridge University Press, 2023.
\bibitem{CA16}
{\sc L.~Cambier and P.-A. Absil}, {\em {Robust low-rank matrix completion by
Riemannian optimization}}, SIAM Journal on Scientific Computing, 38 (2016),
pp.~S440--S460.
\bibitem{CDMS21}
{\sc S.~X. Chen, Z.~D. Deng, S.~Q. Ma, and A.~M.-C. So}, {\em {Manifold
proximal point algorithms for dual principal component pursuit and orthogonal
dictionary learning}}, IEEE Transactions on Signal Processing, 69 (2021),
pp.~4759--4773.
\bibitem{CMMZ20}
{\sc S.~X. Chen, S.~Q. Ma, A.~M.-C. So, and T.~Zhang}, {\em {Proximal gradient
method for nonsmooth optimization over the Stiefel manifold}}, SIAM Journal
on Optimization, 30 (2020), pp.~210--239.
\bibitem{CGT00}
{\sc A.~R. Conn, N.~I.~M. Gould, and P.~L. Toint}, {\em {Trust region
methods}}, SIAM, 2000.
\bibitem{OF20}
{\sc F.~R. de~Oliveira and O.~P. Ferreira}, {\em {Newton method for finding a
singularity of a special class of locally Lipschitz continuous vector fields
on Riemannian manifolds}}, Journal of Optimization Theory and Applications,
185 (2020), pp.~522--539.
\bibitem{DH14}
{\sc L.~Demanet and P.~Hand}, {\em {Scaling law for recovering the sparsest
element in a subspace}}, Information and Inference: A Journal of the IMA, 3
(2014), pp.~295--309.
\bibitem{DP19}
{\sc K.~K. Deng and Z.~Peng}, {\em {An inexact augmented Lagrangian method for
nonsmooth optimization on Riemannian manifold}}, arXiv preprint
arXiv:1911.09900, (2019).
\bibitem{FO98}
{\sc O.~P. Ferreira and P.~R. Oliveira}, {\em {Subgradient algorithm on
Riemannian manifolds}}, Journal of Optimization Theory and Applications, 97
(1998), pp.~93--104.
\bibitem{FO02}
{\sc O.~P. Ferreira and P.~R. Oliveira}, {\em {Proximal point algorithm on
Riemannian manifolds}}, Optimization, 51 (2002), pp.~257--270.
\bibitem{GH16}
{\sc P.~Grohs and S.~Hosseini}, {\em {$\varepsilon$-subgradient algorithms for
locally Lipschitz functions on Riemannian manifolds}}, Advances in
Computational Mathematics, 42 (2016), pp.~333--360.
\bibitem{H69}
{\sc M.~R. Hestenes}, {\em {Multiplier and gradient methods}}, Journal of
optimization theory and applications, 4 (1969), pp.~303--320.
\bibitem{HP11}
{\sc S.~Hosseini and M.~R. Pouryayevali}, {\em {Generalized gradients and
characterization of epi-Lipschitz sets in Riemannian manifolds}}, Nonlinear
Analysis: Theory, Methods \& Applications, 74 (2011), pp.~3884--3895.
\bibitem{HU17}
{\sc S.~Hosseini and A.~Uschmajew}, {\em {A Riemannian gradient sampling
algorithm for nonsmooth optimization on manifolds}}, SIAM Journal on
Optimization, 27 (2017), pp.~173--189.
\bibitem{HLWY20}
{\sc J.~Hu, X.~Liu, Z.~Wen, and Y.~X. Yuan}, {\em A brief introduction to
manifold optimization}, Journal of the Operations Research Society of China,
8 (2020), pp.~199--248.
\bibitem{HW19}
{\sc W.~Huang and K.~Wei}, {\em {An extension of fast iterative
shrinkage-thresholding algorithm to Riemannian optimization for sparse
principal component analysis}}, Numerical Linear Algebra with Applications,
29 (2022), p.~e2409.
\bibitem{HW22}
{\sc W.~Huang and K.~Wei}, {\em Riemannian proximal gradient methods},
Mathematical Programming, 194 (2022), pp.~371--413.
\bibitem{KS19}
{\sc C.~Kanzow and D.~Steck}, {\em {Improved local convergence results for
augmented Lagrangian methods in $\mathcal{C}^2$-cone reducible constrained
optimization}}, Mathematical Programming, 177 (2019), pp.~425--438.
\bibitem{KGB16}
{\sc A.~Kovnatsky, K.~Glashoff, and M.~M. Bronstein}, {\em {MADMM: a generic
algorithm for non-smooth optimization on manifolds}}, in European Conference
on Computer Vision, Springer, 2016, pp.~680--696.
\bibitem{LO14}
{\sc R.~J. Lai and S.~Osher}, {\em {A splitting method for orthogonality
constrained problems}}, Journal of Scientific Computing, 58 (2014),
pp.~431--449.
\bibitem{L13}
{\sc J.~M. Lee}, {\em {Introduction to Smooth manifolds}}, Springer, 2003.
\bibitem{LZ08}
{\sc Y.~J. Liu and L.~W. Zhang}, {\em {Convergence of the augmented Lagrangian
method for nonlinear optimization problems over second-order cones}}, Journal
of optimization theory and applications, 139 (2008), pp.~557--575.
\bibitem{WN99}
{\sc J.~Nocedal and S.~J. Wright}, {\em {Numerical optimization}}, Springer,
1999.
\bibitem{OLCO13}
{\sc V.~Ozoli{\c{n}}{\v{s}}, R.~J. Lai, R.~Caflisch, and S.~Osher}, {\em
{Compressed modes for variational problems in mathematics and physics}},
Proceedings of the National Academy of Sciences, 110 (2013),
pp.~18368--18373.
\bibitem{P69}
{\sc M.~J. Powell}, {\em {A method for nonlinear constraints in minimization
problems}}, Optimization, (1969), pp.~283--298.
\bibitem{R70}
{\sc R.~T. Rockafellar}, {\em {Convex analysis}}, vol.~18, Princeton university
press, 1970.
\bibitem{R73}
{\sc R.~T. Rockafellar}, {\em {A dual approach to solving nonlinear programming
problems by unconstrained optimization}}, Mathematical programming, 5 (1973),
pp.~354--373.
\bibitem{R76}
{\sc R.~T. Rockafellar}, {\em {Augmented Lagrangians and applications of the
proximal point algorithm in convex programming}}, Mathematics of operations
research, 1 (1976), pp.~97--116.
\bibitem{R21}
{\sc R.~T. Rockafellar}, {\em {Advances in convergence and scope of the
proximal point algorithm}}, J. Nonlinear and Convex Analysis, 22 (2021),
pp.~2347--2375.
\bibitem{R22}
{\sc R.~T. Rockafellar}, {\em {Augmented Lagrangians and hidden convexity in
sufficient conditions for local optimality}}, Mathematical Programming,
(2022), pp.~1--36.
\bibitem{R22-ext}
{\sc R.~T. Rockafellar}, {\em {Convergence of augmented Lagrangian methods in
extensions beyond nonlinear programming}}, Mathematical Programming, (2022),
pp.~1--46.
\bibitem{RW09}
{\sc R.~T. Rockafellar and R.~J.-B. Wets}, {\em {Variational analysis}},
vol.~317, Springer Science \& Business Media, 2009.
\bibitem{SWW12}
{\sc D.~A. Spielman, H.~Wang, and J.~Wright}, {\em {Exact recovery of
sparsely-used dictionaries}}, in Conference on Learning Theory, JMLR Workshop
and Conference Proceedings, 2012, pp.~37--1.
\bibitem{SSZ08}
{\sc D.~F. Sun, J.~Sun, and L.~W. Zhang}, {\em {The rate of convergence of the
augmented Lagrangian method for nonlinear semidefinite programming}},
Mathematical Programming, 114 (2008), pp.~349--391.
\bibitem{V13}
{\sc B.~Vandereycken}, {\em {Low-rank matrix completion by Riemannian
optimization}}, SIAM Journal on Optimization, 23 (2013), pp.~1214--1236.
\bibitem{WDZZ22}
{\sc S.~W. Wang, C.~Ding, Y.~J. Zhang, and X.~Zhao}, {\em {Strong Variational
Sufficiency for Nonlinear Semidefinite Programming and its Implications}},
arXiv preprint arXiv:2210.04448, (2022).
\bibitem{ZBDZ21}
{\sc Y.~Zhou, C.~Bao, C.~Ding, and J.~Zhu}, {\em {A semismooth Newton based
augmented Lagrangian method for nonsmooth optimization on matrix manifolds}},
Mathematical Programming, 201 (2023), pp.~1--61.
\bibitem{ZBD22}
{\sc Y.~X. Zhou, C.~L. Bao, and C.~Ding}, {\em On the robust isolated calmness
of a class of nonsmooth optimizations on riemannian manifolds and its
applications}, arXiv preprint arXiv:2208.07518, (2022).
\bibitem{ZHT06}
{\sc H.~Zou, T.~Hastie, and R.~Tibshirani}, {\em Sparse principal component
analysis}, Journal of computational and graphical statistics, 15 (2006),
pp.~265--286.
\end{thebibliography}