Dynamic Tensor Product Regression
Extracted source text · arXiv 2210.03961
\def\isarxiv{1} %%%NeurIPS version, we comment this line
\ifdefined\isarxiv
\documentclass[11pt]{article}
\else
\documentclass[nohyperref]{article}
\usepackage[final]{neurips_2022}
\usepackage{xcolor}
\fi
\usepackage{amsmath}
\usepackage{amsthm}
\allowdisplaybreaks
\usepackage{amssymb}
\usepackage{algorithm}
\usepackage{subfig}
%\usepackage{color}
%\usepackage[dvipsnames]{xcolor}
%\usepackage{graphicx}
\usepackage{wrapfig,epsfig}
\usepackage{epstopdf}
\usepackage{url}
\usepackage{graphicx}
\usepackage{color}
\usepackage{epstopdf}
\usepackage{algpseudocode}
\usepackage{scrextend}
\usepackage[T1]{fontenc}
\usepackage{bbm}
\usepackage{comment}
%\usepackage{authblk}
\usepackage{multicol}
\usepackage{dsfont}
\usepackage{mathtools}
\usepackage{enumitem}
\usepackage{thmtools,thm-restate}
%%% print refs in table of contents
\let\C\relax
\usepackage{tikz}
\usepackage{hyperref}
%%% arxiv don't allow this.
%\hypersetup{colorlinks=true,citecolor=red,linkcolor=blue}
\definecolor{mydarkblue}{rgb}{0,0.08,0.45}
\hypersetup{colorlinks=true, citecolor=mydarkblue,linkcolor=mydarkblue}
\usetikzlibrary{arrows}
\ifdefined\isarxiv
\usepackage[margin=1in]{geometry}
\linespread{1}
\else
\fi
\graphicspath{{./figs/}}
\definecolor{b2}{RGB}{51,153,255}
\definecolor{mygreen}{RGB}{80,180,0}
\newcommand{\Zhao}[1]{\textcolor{b2}{[Zhao: #1]}}
\newcommand{\Lichen}[1]{\textcolor{red}{[Lichen: #1]}}
\newcommand{\Aravind}[1]{\textcolor{orange}{[Aravind: #1]}}
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{conjecture}[theorem]{Conjecture}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{condition}[theorem]{Condition}
\newtheorem{hypothesis}[theorem]{Hypothesis}
\newtheorem{question}[theorem]{Question}
\newtheorem{notation}[theorem]{Notation}
%\newtheorem{remark}[theorem]{Remark}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{open}[theorem]{Open Problem}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{task}[theorem]{Task}
%\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
%%% some writing rules
%% Writing rule for creating tags.
%% Tags :
%% Theorem \ref{thm:bla_bla}
%% Lemma \ref{lem:bla_bla}
%% Claim \ref{cla:bla_bla}
%% Corollary \ref{cor:bla_bla}
%% Fact \ref{fac:bla_bla}
%% Definition \ref{def:bla_bla}
%% Section \ref{sec:bla_bla}
%% Subsection \ref{sub:bla_bla}
%% Equation \ref{eq:bla_bla}
\newcommand{\wh}{\widehat}
\newcommand{\wt}{\widetilde}
\newcommand{\ov}{\overline}
\newcommand{\eps}{\varepsilon}
\renewcommand{\epsilon}{\varepsilon}
\renewcommand{\phi}{\varphi}
\newcommand{\B}{\mathsf{B}}
\newcommand{\A}{\mathsf{A}}
\newcommand{\N}{\mathcal{N}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}}
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\volume}{\mathrm{volume}}
\newcommand{\RHS}{\mathrm{RHS}}
\newcommand{\LHS}{\mathrm{LHS}}
\renewcommand{\i}{\mathbf{i}}
\renewcommand{\tilde}{\wt}
\renewcommand{\hat}{\wh}
\renewcommand{\bar}{\ov}
\renewcommand{\d}{\mathrm{d}}
\newcommand{\poly}{\mathrm{poly}}
\newcommand{\tr}{\mathrm{tr}}
\newcommand{\Tmat}{{\cal T}_{\mathrm{mat}}}
\newcommand{\nnz}{\mathrm{nnz}}
\newcommand{\diag}{\mathrm{diag}}
\newcommand{\new}{\mathrm{new}}
\newcommand{\tw}{\mathrm{tw}}
%%%%notations from BSS12 proofs
\renewcommand{\S}{\mathcal{S}}
\newcommand{\U}{\mathcal{U}}
\newcommand{\id}{\mathrm{id}}
\newcommand{\davg}{d_{\mathsf{avg}}}
\newcommand{\dmax}{d_{\mathsf{max}}}
\newcommand{\lmax}{\lambda_{\mathsf{max}}}
\newcommand{\lmin}{\lambda_{\mathsf{min}}}
\newcommand{\deltaL}{\delta_{L}}
\newcommand{\deltaU}{\delta_{U}}
\newcommand{\epsL}{\epsilon_{L}}
\newcommand{\epsU}{\epsilon_{U}}
\DeclareMathOperator{\rank}{rank}
\newcommand{\lo}{\mathrm{lo}}
\newcommand{\hi}{\mathrm{hi}}
\DeclareMathOperator{\pr}{Pr}
\newcommand{\E}{\mathbb{E}}
\newcommand{\Var}{\mathbf{Var}}
\newcommand{\vect}{\mathrm{vec}}
\newcommand{\T}{\mathcal{T}}
%\newcommand{\sd}{\mathrm{s_d}}
\DeclareMathOperator{\sd}{\mathtt{sd}}
\DeclareMathOperator{\opt}{OPT}
%%%%% matrix construction %%%%%
\newcommand {\mat} [1] {\left[\begin{array}{#1}}
\newcommand {\rix} {\end{array}\right]}
%%%Footnote without mark
\makeatletter
\def\blfootnote{\gdef\@thefnmark{}\@footnotetext}
\makeatother
\begin{document}
\ifdefined\isarxiv
\title{Dynamic Tensor Product Regression\footnote{A preliminary version of this paper appeared in the proceedings of 36th Conference on Neural Informal Processing Systems (NeurIPS 2022).}}
\author{Aravind Reddy\thanks{\texttt{aravind.reddy@cs.northwestern.edu}. Northwestern University.}
\and
Zhao Song\thanks{\texttt{zsong@adobe.com}. Adobe Research.}
\and Lichen Zhang\thanks{\texttt{lichenz@mit.edu}. MIT.}
}
\date{}%\today}
\else
\title{Dynamic Tensor Product Regression}
\author{Aravind Reddy\thanks{\texttt{aravind.reddy@cs.northwestern.edu}. Northwestern University.}
\And
Zhao Song\thanks{\texttt{zsong@adobe.com}. Adobe Research.}
\And Lichen Zhang\thanks{\texttt{lichenz@mit.edu}. MIT. (Author names in alphabetical order, equal contribution)}
}
\maketitle
\iffalse
\author{%
David S.~Hippocampus\thanks{Use footnote for providing further information
about author (webpage, alternative address)---\emph{not} for acknowledging
funding agencies.} \\
Department of Computer Science\\
Cranberry-Lemon University\\
Pittsburgh, PA 15213 \\
\texttt{hippo@cs.cranberry-lemon.edu} \\
% examples of more authors
% \And
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email} \\
% \AND
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email} \\
% \And
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email} \\
% \And
% Coauthor \\
% Affiliation \\
% Address \\
% \texttt{email} \\
}
\fi
\fi
\ifdefined\isarxiv
\begin{titlepage}
\maketitle
\begin{abstract}
\input{abstract}
\end{abstract}
\thispagestyle{empty}
\end{titlepage}
\newpage
\else
%\maketitle
\begin{abstract}
\input{abstract}
\end{abstract}
\fi
\input{intro}
\input{relat}
\input{prelim}
\input{tech}
\input{algorithm}
\input{splines}
\input{low-rank-approximation}
\input{conclusion}
\input{acknowledgements}
%%%%temporary
\ifdefined\isarxiv
\addcontentsline{toc}{section}{References}
\bibliographystyle{alpha}
\bibliography{ref}
\else
\bibliography{ref}
\bibliographystyle{alpha}
\fi
\ifdefined\isarxiv
\else
\input{checklist}
\fi
\newpage
\appendix
\input{app_preli}
\input{appendix-tree}
\input{appendix-splines}
\input{appendix-low-rank}
\end{document}
In this work, we initiate the study of \emph{Dynamic Tensor Product Regression}. One has matrices $A_1\in \mathbb{R}^{n_1\times d_1},\ldots,A_q\in \mathbb{R}^{n_q\times d_q}$ and a label vector $b\in \mathbb{R}^{n_1\ldots n_q}$, and the goal is to solve the regression problem with the design matrix $A$ being the tensor product of the matrices $A_1, A_2, \dots, A_q$ i.e. $\min_{x\in \mathbb{R}^{d_1\ldots d_q}}~\|(A_1\otimes \ldots\otimes A_q)x-b\|_2$. At each time step, one matrix $A_i$ receives a sparse change, and the goal is to maintain a sketch of the tensor product $A_1\otimes\ldots \otimes A_q$ so that the regression solution can be updated quickly. Recomputing the solution from scratch for each round is very slow and so it is important to develop algorithms which can quickly update the solution with the new design matrix. Our main result is a dynamic tree data structure where any update to a single matrix can be propagated quickly throughout the tree. We show that our data structure can be used to solve dynamic versions of not only Tensor Product Regression, but also Tensor Product Spline regression (which is a generalization of ridge regression) and for maintaining Low Rank Approximations for the tensor product.
\section{Introduction}
The task of fitting data points to a line is well-known as the least-squares regression problem~\cite{s81}, which has a wide range of applications in signal processing~\cite{rgy78}, convex optimization~\cite{b15}, network routing~\cite{m13_flow,m16}, and training neural networks~\cite{bpsw21,szz21}. In this work, we study a generalized version of least-squares regression where the design matrix $A$ is a tensor product of $q$ smaller matrices $A_1\otimes A_2\otimes \ldots \otimes A_q$.
Tensor products have been extensively studied in mathematics and the physical sciences since their introduction more than a century ago by Whitehead and Russell in their Principia Mathematica~\cite{wr1912}. They have been shown to have a humongous number of applications in several areas of mathematics like applied linear algebra and statistics~\cite{van2000ubiquitous}. In particular, machine learning applications include image processing~\cite{nagy2006kronecker}, multivariate data fitting~\cite{golubVanLoan2013Book}, and natural language processing ~\cite{panahi2021shapeshifter}. Furthermore, regression problems involving tensor products arise in surface fitting and multidimensional density smoothing~\cite{eilers2006multidimensional}, and structured blind deconvolution problems~\cite{oh2005}, among several other applications, as discussed in~\cite{dssw18,djssw19}.
Solving tensor product regression in $\ell_2$ norm~\cite{djssw19, ffg22} to a $(1\pm\epsilon)$ precision takes time $\wt O(\sum_{i=1}^q\nnz(A_i)+\poly(dq/(\delta\epsilon)))$, where each matrix $A_i\in \R^{n_i\times d_i}$ and $d=d_1d_2\ldots d_q$, and $\nnz(A)$ denotes the number of nonzero entries of $A$. However, these algorithms are inherently \emph{static}, meaning that even if there is a sparse (or low-rank) update to a \textit{single} design matrix $A_i$, it needs to completely recompute the new solution from scratch. In modern machine learning applications, such static algorithms are not practical due to the fact that data points are always evolving and recomputing the solution from scratch every time is computationally too expensive. Hence, it is important to develop efficient \emph{dynamic algorithms} that can adaptively update the solution. For example, graphs for real-world network data are modeled as the tensor product of a large number of smaller graphs~\cite{leskovec2010kronecker}. Many important problems on these graphs can be solved by regression of the adjacency and Laplacian matrices of these graphs~\cite{st04}. Real-world network data is always time-evolving and so it is crucial to develop dynamic algorithms for solving regression problems where the design matrix is a tensor product of a large number of smaller matrices. Hence, we ask the following question:
\begin{center}
{\it Can we design a dynamic data structure for tensor product regression that can handle updates to the design matrix and quickly updates the solution?}
\end{center}
We provide a positive answer to the above question.
At the heart of our data structure is a binary tree that can maintain a succinct sketch of $A_1\otimes \ldots \otimes A_q$ and supports an update to one of the matrices quickly. Such tree structures have been useful in designing efficient sketching algorithms for tensor products of vectors~\cite{akk+20,swyz21}. However, the goal of these works has been in reducing the sketching dimension from an exponential dependence on $q$ to a polynomial dependence on $q$. Their results can be directly generalized to solving tensor product regression \emph{statically} but not dynamically. In this work, we build upon the tree structure to solve the Dynamic Tensor Product Regression problem and other related dynamic problems like Dynamic Tensor Spline Regression and Dynamic Tensor Low Rank Approximation. Our key observation is that updating the entire tree is efficient when a single leaf of the tree gets an update: specifically, we only need to update the nodes which fall on the path from the leaf to the root.
\paragraph{Technical Contributions.}
\begin{itemize}
\item We design a dynamic tree data structure \textsc{DynamicTensorTree} that maintains a succinct representation of the tensor product $A_1\otimes \ldots \otimes A_q$ and supports efficient updates to any of the $A_i$'s.
\item Consequently, we develop a dynamic algorithm for solving Tensor Product Regression, when one of the matrices $A_i$'s is updated and we need to output an estimate of the new solution to the regression problem quickly.
\item We also show that we can use our \textsc{DynamicTensorTree} data-structure in a fast dynamic algorithm to solve the Tensor Product Spline Regression problem, which is a generalization of the classic Ridge Regression problem.
\item We also initiate the study of \emph{Dynamic Tensor Low Rank Approximation}, where the goal is to maintain a low rank approximation (LRA) of the tensor product with dynamic updates, and show how we can use our \textsc{DynamicTensorTree} data structure to solve this problem.
\end{itemize}
\paragraph{Roadmap.}
In section~\ref{sec:related_work}, we first discuss some related work. Then, we provide notation, some background for our work, and the main problem formulation in section~\ref{sec:preliminaries}. In section~\ref{sec:technical_overview}, we provide a technical overview of our paper. Following that, we provide our dynamic tree data structure in section~\ref{sec:tree_data_structure}. We then show how it can be used for Dynamic Tensor Product Regression, Dynamic Tensor Spline Regression, and Dynamic Tensor Low Rank Approximation in section~\ref{sec:applications}. We end with our conclusion and discuss some future directions in section~\ref{sec:conclusion}.
\section{Related Work}\label{sec:related_work}
\paragraph{Sketching.}
Sketching techniques have many applications in numerical linear algebra, such as linear regression, low-rank approximation \cite{cw13,nn13,mm13,bw14,rsw16,swz17,hlw17,alszz18,bbb+19,ivww19,swz19_neurips1,swz19_neurips2,mw21,wy22,cstz22}, distributed problems \cite{wz16,bwz16}, reinforcement learning \cite{wzd+20,ssx21}, projected gradient descent \cite{xss21}, training over-parameterized neural networks \cite{syz21,szz21}, tensor decomposition \cite{swz19_soda}, clustering \cite{emz21}, cutting plane method \cite{jlsw20}, generative adversarial networks \cite{xzz18}, recommender systems \cite{rrs+22}, and linear programming \cite{lsz19,jswz21,sy21}.
\paragraph{Dynamic Least-squares Regression.}
A dynamic version of the ordinary least-squares regression problem (without a tensor product design matrix) was recently studied in~\cite{jpw22}. In their model, at each iteration, a new row is prepended to the design matrix $A$ and a new coordinate is prepended to the vector $b$. To efficiently maintain the necessary parts of the design matrix and the solution, they make use of a fast leverage score maintenance data structure by \cite{cmp20} and achieve a running time of $\wt O(\nnz(A^{(T)})+\poly(d,\epsilon^{-1}))$ where $T$ is the number of time steps.
\iffalse
There are several major differences between their work and ours. First is that our model is more general than their model -- we allow \textit{updates} which can also model insertions (by maintaining zero entries at the beginning) and deletions (by providing an update which is the negative value of the corresponding data) whereas they only allow insertions. Second is that they do not consider tensor product inputs and so computing the entire tensor product first and then applying their algorithm as-is becomes prohibitively expensive. We note that naively forming the tensor product $A_1\otimes \ldots \otimes A_q$ already takes $O(\prod_{i=1}^q n_id_i)$ time, while the input size is only $O(\sum_{i=1}^q n_i d_i)$ which is much smaller than $O(\prod_{i=1}^q n_id_i)$. Third is that we also consider the problems of spline regression and maintaining low-rank approximations.
\fi
\paragraph{Tensor/Kronecker Product Problems.}
Many machine learning problems involve tensor/Kronecker product computations such as, regression \cite{hlw17,dssw18,djssw19}, low-rank approximation \cite{swz19_soda}, fast kernel computation \cite{swyz21}, semi-definite programming \cite{jklps20,hjstz21}, and training over-parameterized neural networks \cite{bpsw21,szz21}. Apart from solving problems which directly involve tensor/Kronecker products, another line of research focuses on constructing polynomial kernels efficiently so that the computation of various kernel problems can be improved~\cite{akk+20,swyz21}.
\section{Preliminaries}\label{sec:preliminaries}
\subsection{Notation}\label{subsec:notation}
For any natural number $n$, we use $[n]$ to denote the set $\{1,2,\dots,n\}$. We use $\E[\cdot]$ to denote expectation of a random variable if it exists. We use $\mathbf{1}[\cdot]$ and $\Pr[\cdot]$ to denote the indicator function and probability of an event respectively. For any natural numbers $a,b,c$, we use $\Tmat(a,b,c)$ to denote the time it takes to multiply two matrices of sizes $a \times b$ and $b \times c$. For a vector $x$, we use $\|x\|_2$ to denote its $\ell_2$ norm. For a matrix $A$, we use $\|A\|_F$ to denote its Frobenius norm. For a matrix $A$, we use $A^\top$ to denote the transpose of $A$. Given two matrices $A\in \R^{n_1\times d_1}$ and $B\in \R^{n_2\times d_2}$, we use $A\otimes B$ to denote their tensor product (which is also referred to as Kronecker product), i.e., $(A\otimes B)_{i_1+(i_2-1)\cdot n_1,j_1+(j_2-1)\cdot d_1}=A_{i_1,j_1}\cdot B_{i_2,j_2}$. For example, for $2 \times 2$ matrices $A$ and $B$, \[\begin{bmatrix} a_{11} & a_{12}\\ a_{21} & a_{22} \end{bmatrix} \otimes B = \begin{bmatrix} a_{11} B & a_{12} B\\ a_{21} B & a_{22} B \end{bmatrix} = \begin{bmatrix} a_{11}b_{11} & a_{11}b_{12} & a_{12}b_{11} & a_{12}b_{12}\\ a_{11}b_{21} & a_{11}b_{22} & a_{12}b_{21} & a_{12}b_{22} \\ a_{21}b_{11} & a_{21}b_{12} & a_{22}b_{11} & a_{22}b_{12} \\ a_{21}b_{21} & a_{21}b_{22} & a_{22}b_{21} & a_{22}b_{22} \end{bmatrix}\]
We use $\bigotimes$ to denote tensor product of more than two matrices, i.e., $\bigotimes_{i=1}^q A_i = A_1 \otimes A_2 \otimes \cdots \otimes A_q$. For non-negative real numbers $a$ and $b$, we use $a= (1\pm \epsilon)b$ to denote $(1-\epsilon) \cdot b \leq a \leq (1+\epsilon) \cdot b$. For a real matrix $A$, we use $\sigma_i(A)$ to denote its $i$-th largest singular value. For any function $f$, we use $\wt{O}(f)$ to denote $O( f \poly ( \log f ) )$.
\iffalse
\subsection{Background}\label{subsec:background}
In this section, we provide definitions for some terms used throughout our paper.
%We begin with the definitions of {\it Statistical Dimension}, {\it Oblivious Subspace Embeddings} (OSE), and {\it Approximate Matrix Product} property.
\begin{definition}[Statistical Dimension]\label{def:SD}
For any $\lambda \geq 0$ and any positive semidefinite matrix $M \in \R^{n \times n}$, the $\lambda$-statistical dimension of $M$ is
\begin{align*}
\sd_{\lambda}(M) &\coloneqq \tr[ M(M+\lambda I_n)^{-1} ]
\end{align*}
\end{definition}
\begin{definition}[Oblivious Subspace Embedding~\cite{s06,akk+20}]\label{def:OSE}
Let $\epsilon,\delta\in (0,1), \mu > 0,$ and $d,n\geq 1$ be integers. An $(\epsilon,\delta,\mu,d,n)$-Oblivious Subspace Embedding (OSE) is a distribution over $m\times d$ matrices with the guarantee that for any fixed matrix $A\in \R^{d\times n}$ with $\sd_{\lambda}(A^\top A) \leq \mu$, we have for any vector $x\in \R^n$, with probability at least $1-\delta$,
\begin{align*}
\|\Pi Ax\|_2+\lambda \|x\|_2=(1\pm\epsilon) (\|Ax\|_2+\lambda \|x\|_2) .
\end{align*}
\end{definition}
\begin{definition}[Approximate Matrix Product]\label{def:amp}
Given $\epsilon,\delta>0$, we say that a distribution ${\cal D}$ over $m\times d$ matrices $\Pi$ has the $(\epsilon,\delta)$-approximate matrix product property if for every $C,D\in \R^{d\times n}$, with probability at least $1-\delta$,
\begin{align*}
\|C^\top \Pi^\top \Pi D-C^\top D\|_F \leq & ~ \epsilon \|C\|_F\|D\|_F.
\end{align*}
\end{definition}
%We will define two key sketching matrices that will serve the crux of our data structure.
\begin{definition}[\textsc{OSNAP} matrix~\cite{nn13}]
\label{def:osnap}
For every sparsity parameter $s$, target dimension $m$, and positive integer $d$, the OSNAP transform with sparsity $s$ is defined as
\begin{align*}
S_{r,j} = & ~ \frac{1}{\sqrt s} \cdot \delta_{r,j}\cdot \sigma_{r,j},
\end{align*}
for all $r\in [m]$, $j\in [d]$, where $\sigma_{r,j}\in \{\pm 1\}$ are independent and uniform Rademacher random variables and $\delta_{r,j}$ are Bernoulli random variables satisfying
\begin{itemize}
\item For every $i\in [d]$, $\sum_{r\in [m]}\delta_{r,i}=s$. In other words, every column has exactly $s$ nonzero entries.
\item For all $r\in [m]$ and all $i\in [d]$, $\E[\delta_{r,i}]=s/m$.
\item The $\delta_{r,i}$'s are negatively correlated: $\forall\ T\subset [m]\times [d],\ \E[\prod_{(r,i)\in T}\delta_{r,i}]\leq \prod_{(r,i)\in T} \E[\delta_{r,i}]=(\frac{s}{m})^{|T|}$.
\end{itemize}
\end{definition}
\begin{definition}[\textsc{TensorSRHT}~\cite{akk+20}]
\label{def:TensorSRHT}
We define the \textsc{TensorSRHT} $S : \R^d \times \R^d \rightarrow \R^m$ as $S = \frac{1}{\sqrt m} P \cdot (H D_1\times H D_2)$, where each row of $P\in \{0,1\}^{m\times d}$ contains only one $1$ at a random coordinate, one can view $P$ as a sampling matrix. $H$ is a $d \times d$ Hadamard matrix, and $D_1, D_2$ are two $d\times d$ independent diagonal matrices with diagonals that are each independently set to be a Rademacher random variable (uniform in $\{-1,1\}$).
\end{definition}
For completeness, we define other sketching matrices such as {\sc CountSketch}~\cite{ccf02}, {\sc SRHT}~\cite{ldfu13} and {\sc TensorSketch} \cite{p13} in Appendix~\ref{sec:appendix-prelim}.
\fi
\subsection{Problem Formulation}\label{subsec:problem_formulation}
In this section, we define our task. We start with the \emph{static version} of tensor product regression.
\begin{definition}[Static version]
In the $\ell_2$ Tensor Product Regression problem, we are given matrices $A_1, A_2, \dots, A_q$ where $A_i \in \R^{n_i \times d_i}$ and $b \in \R^{n_1 n_2 \dots n_q}$. Let us define $n \coloneqq n_1n_2\dots n_q$ and $d \coloneqq d_1d_2\dots d_q$. The goal is to output $x \in \R^{d}$ such that we minimize the objective: \[\|(A_1 \otimes A_2 \otimes\dots \otimes A_q) x - b\|_2\]
\end{definition}
Our dynamic version of $\ell_2$ Tensor Product Regression involves updating one of the matrices $A_i$ with an update matrix $B$. We formally define the problem as follows:
\begin{definition}[Dynamic version]
In the Dynamic $\ell_2$ Tensor Product Regression problem, we are given matrices $A_1,A_2,\ldots,A_q$ where $A_i\in \R^{n_i\times d_i}$, a sequence $\{(i_1,B_1),(i_2,B_2),\ldots,(i_T,B_T)\}$ with $i_t \in [q], B_t \in \R^{n_{i_t}\times d_{i_t}}$ for all $t \in [T]$, and the goal is to design a data structure with the following procedures:
\begin{itemize}
\item {\sc Initialize}: The data structure initializes $A_1,\ldots, A_q$ and maintains an approximation to $\bigotimes_{i=1}^q A_i$.
\item {\sc Update}: At time step $t$, given an index $i_t$ and an update matrix $B_t$, the data-structure needs to update $A_{i_t} \leftarrow A_{i_t}+B_t$, and maintain an approximation to $A_1\otimes \ldots \otimes (A_{i_t}+B_t)\otimes \ldots \otimes A_q$.
\item {\sc Query}: The data structure outputs a vector $\wh x\in \R^{d}$ such that
\begin{align*}
\|(\bigotimes_{i=1}^q A_i)\wh x-b\|_2 = (1\pm\epsilon) \min_{x\in \R^{d}}~\|(\bigotimes_{i=1}^q A_i) x-b\|_2.
\end{align*}
\end{itemize}
\end{definition}
Note that although we don't discuss updates to the vector $b$ in our model, they are easy to implement in our data-structure. In addition to the standard regression problem, we also consider the spline regression (which is a generalization of ridge regression) and low rank approximation problems. We will provide the definitions for these problems in their respective sections.
\section{Technical Overview}\label{sec:technical_overview}
The two main design considerations for our data structure are as follows: 1).\ We want a data structure that stores a \emph{sketch} of the tensor product, this has the advantage of having a smaller storage space and a faster time to form the tensor product. 2).\ The update time to one of the matrices $A_i$ should be fast, ideally we want to remove the linear dependence on $q$. With these two goals in mind, we introduce a dynamic tree data structure that satisfies both of them simultaneously, building on the sketch of \cite{akk+20}. Specifically, we use each leaf of the tree to represent $S_i A_i$, a sketch of the base matrix that only has $m$ rows instead of $n$ rows, where $m$ is proportional to the small dimension $d$. Each internal node represents a sketch of the Tensor product of its two children. To speed up computation, we adapt tensor-typed sketching matrices to avoid the need to explicitly form the Tensor product of the two children. The root of the tree will store a sketch for the tensor product $\bigotimes_{i=1}^q A_i$.
We point out that in order for this tree to work, we need to apply an \emph{independent} sketching matrix to each tree node. Since there are $2q-1$ nodes in total, we will still have a dependence on $q$ in the initialization phase (which will be dominated by the $d$ term). However, during the update phase, our tree has a clear advantage: if one of the leaves has been updated, we only need to propagate the change along the path to root. Since the height of the tree is at most $O(\log q)$, we remove the linear dependence on $q$ in the update phase. This is the key insight for obtaining a faster update time using our dynamic data structure. Moreover, the correctness follows naturally from the guarantee of the tree: as shown in~\cite{akk+20}, the tree itself is an Oblivious Subspace Embedding (OSE) for matrices of proper size, hence, it also preserves the subspace after updating one of the leaves. Figure~\ref{fig:tree} is an example tree for tensor product of 4 matrices.
\begin{figure*}[!ht]
\centering
\begin{tikzpicture}[scale = 0.9,level distance=1.5cm,
level 1/.style={sibling distance=6cm},
level 2/.style={sibling distance=3cm},
level 3/.style={sibling distance=1.5cm}]
\node[circle, draw](z) {$S^1_1$}
child{node[circle, draw]{$S^2_1$}
child{node[circle, draw]{$T^3_1$}}
child{node[circle, draw]{$T^3_2$}}
}
child{node[circle, draw]{$S^2_2$}
child{node[circle, draw]{$T^3_3$}}
child{node[circle, draw]{$T^3_4$}}
};
\end{tikzpicture}
\caption{A tree for sketching the tensor product of 4 matrices $A_1, A_2, A_3$ and $A_4$. The leaves use sketches of type $T_{\mathrm{base}}$ and internal nodes use sketches of type $S_{\mathrm{base}}$. The algorithm starts to apply $T_i$ to each matrix $A_i$, and then propagates them up the tree.}
\label{fig:tree}
\end{figure*}
With this data structure, we provide efficient algorithms for dynamic Tensor product regression, dynamic Tensor Spline regression, and dynamic Tensor low rank approximation. For all these problems, we do not explicitly maintain the solutions, rather, we use our tree to quickly update a sketch of the design matrix. In the query phase (when we need to actually solve the regression problems), we use this sketch of updated design matrix to quickly compute the solution.
\section{Dynamic Tree Data Structure}\label{sec:tree_data_structure}
In this section, we introduce our data structure, which contains {\sc Initialize} and {\sc Update} procedures. We'll define task-specific {\sc Query} procedures later. We defer proofs in this section to Appendix~\ref{sec:appendix-tree_data_structure}. We start with an outline of Algorithm~\ref{alg:tree_static}.
%\subsection{Initialization}
\begin{algorithm*}[!t]\caption{Our dynamic tree data structure}\label{alg:tree_static}
\begin{algorithmic}[1]
\State {\bf data structure} {\sc DynamicTensorTree} \Comment{Theorem~\ref{thm:tree_init}}
\State
\State {\bf members}
\State \hspace{4mm} $J_{k,\ell}\in \R^{m \times d^{2^\ell}}$ for $0\leq \ell\leq \log q$ and $0\leq k\leq q/2^\ell-1$
%where $m = \Omega()$
\Comment{$\ell$ is the level of the tree.}
%\State \hspace{4mm} $\wt b \in \R^{n}$
%\Lichen{Aravind, can you move this $\wt b$ to Algorithm 2?}
\State {\bf end members}
\State
\Procedure{Initialize}{$A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}, m \in \mathbb{N}_+, C_{\mathrm{base}}, T_{\mathrm{base}}$}
\For{$\ell=0\to \log q$}
\For{$k=0\to q/2^\ell-1$}
\If{$\ell=0$}
\State Choose a $C_k \in \R^{m \times n_k}$ from $C_{\mathrm{base}}$. \Comment{$C_{\mathrm{base}}$ can be {\sc CountSketch}, {\sc SRHT} or {\sc OSNAP}.}
\State $J_{k,\ell}\gets C_k A_k$
\Else
\State Choose a $T_{k,\ell} \in \R^{m \times m^2}$ from $T_{\mathrm{base}}$. \Comment{$T_{\mathrm{base}}$ can be {\sc TensorSketch} or {\sc TensorSRHT}.}
\State $J_{k,\ell} \gets T_{k,\ell}(J_{2k,\ell-1} \otimes J_{2k+1,\ell-1})$ \Comment{Sketch of tensor product of its children}
\EndIf
\EndFor
\EndFor
%\State Let $S \in \R^{m \times n^q}$ be the sketching matrix formed by the tree
%\State $\wt b \gets S b$
\EndProcedure
\State
\Procedure{Update}{$i\in [q], B\in \R^{n_i\times d_i}$}
\For{$\ell=0\to \log(q)$}
\If{$\ell=0$}
\State $J_{i,0}^{\mathrm{new}}\gets C_i B$
\State $J_{i,0}\gets J_{i,0} + J_{i,0}^{\mathrm{new}}$
\Else
%\State $J_{i/2,1} \gets J_{i/2,1} + T_{i/2,0}((C_i B) \otimes J_{i+1,0})$
\State $J_{\lceil i/2^{\ell} \rceil,\ell}^{\mathrm{new}} \gets T_{k,\ell} (J_{\lceil i/2^{\ell-1} \rceil,\ell-1}^{\mathrm{new}} \otimes J_{\lceil i/2^{\ell-1} \rceil + 1,\ell-1}) $ \Comment{Sibling is to the left also sometimes}
\State $J_{\lceil i/2^{\ell} \rceil,\ell} \gets J_{\lceil i/2^{\ell} \rceil,\ell} + J_{\lceil i/2^{\ell} \rceil,\ell}^{\mathrm{new}}$ \Comment{Sketch of tensor product of its children}
\EndIf
\EndFor
%\State $L_0 \gets (J_{0,\log(q)}^\top J_{0,\log(q)})^{-1}$
\EndProcedure
\end{algorithmic}
\end{algorithm*}
%\subsection{Running Time of Our Dynamic Tree}
\textbf{Outline of Tree:} For simplicity, let us assume that the number of matrices $q$ is a power of $2$. Also, let us assume that all the matrices are of the same dimension $n_1 \times d_1$. The output is a matrix of dimension $m \times d$ where $m = \poly(q,d,\epsilon^{-2})$. First, we apply $q$ independent base sketch matrices (such as {\sc CountSketch} (Def.~\ref{def:cs}), {\sc SRHT} (Def.~\ref{def:srht})) $\{T_i \in \R^{m \times n_1}, i \in [q]\}$ to each of the matrices $A_1, A_2, \dots, A_q$. After this step, all the leaf nodes are of size $m \times d_1$. Then, we have a tree of height $\log q$. Let us use $\ell = 0 \to \log q$ to represent the level of the tree nodes where $\ell = 0$ represents the leaves and $\ell = \log q$ represents the root node. Let us use $J_{k,\ell}\in \R^{n\times d^{2^\ell}}$ to denote the matrix stored at the node $k, \ell$. For leaf nodes, we choose base sketches $T_k\in \R^{m\times n_1}$ and compute $J_{k, 0} = T_k A_k$. At each internal node, we use a sketch for tensor-typed inputs (such as {\sc TensorSketch} (Def.~\ref{def:TensorSketch}) or {\sc TensorSRHT} (Def.~\ref{def:TensorSRHT})), apply it to its two children. The sketching matrix $S_i\in \R^{m\times m^2}$, and it can be applied to its two inputs fast, obtaining a running time of $\wt O(m)$ instead of $O(m^2)$. We inductively compute the tree all the way up to the root for initialization. Note that at the root, we end up with a matrix of size $m\times d$, which is significantly smaller than the tensor product $n\times d$.
The running time guarantees of Algorithm~\ref{alg:tree_static}, which we will prove in Appendix~\ref{sec:appendix-tree_data_structure}, are as follows:
\begin{theorem}[Running time of our main result. Informal version of Theorem~\ref{thm:app:tree_init}]\label{thm:tree_init}
There is an algorithm (Algorithm~\ref{alg:tree_static}) that has the following procedures:
\begin{itemize}
%\item The space storage is
\item \textsc{Initialize}($A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2},\dots, A_q \in \R^{n_q \times d_q}, m\in \mathbb{N}_+, C_{\mathrm{base}}, T_{\mathrm{base}}$): Given matrices $A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2}, \dots, A_q \in \R^{n_q \times d_q},$, a sketching dimension $m$, families of base sketches $C_{\mathrm{base}}$ and $T_{\mathrm{base}}$, the data-structure pre-processes in time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd)$.
\item {\sc Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given a matrix $B$ and an index $i\in [q]$, the data structure updates the approximation to $A_1\otimes \ldots \otimes (A_i+B)\otimes \ldots \otimes A_q$ in time $\wt O(\nnz(B) + md)$.%\footnote{$\omega$ is the exponent of the current matrix multiplication time~\cite{w12,aw21}.}
%\item \textsc{Query}: Query outputs an approximate solution $\hat{x}$ to the Tensor product regression problem where $\|A\hat{x} - b\|_2 \leq (1 + \epsilon) \|Ax^* - b\|_2$ with probability at least $1 - \delta$ in time ?? where $x^*$ is an optimal solution to the Tensor product regression problem i.e. $x^* = \arg \min_{x \in \R^d} \|Ax - b\|_2$.
\end{itemize}
\end{theorem}
\begin{remark}
In our data structure, we only pay the linear $q$ term in {\sc Initialization} (which will be dominated by the term $d$), since we are maintaining a tree with $2q-1$ nodes, each node corresponds to a matrix of $m$ rows. In the {\sc Update} phase, we shave off the linear dependence on $q$. We also want to point out that the sketching dimension $m$ is typically at least linear on $q$, however, it only depends on $q$, the small dimension $d$ and the precision parameter $\epsilon$, hence it is far more efficient to use this sketch representation compared to the original tensor product. Also, note that though we only design a data structure for $q$ being a power of 2, it is not hard to generalize to $q$ being an arbitrary positive integer via the technique introduced in~\cite{swyz21}. We only incur a $\log q$ factor by doing so.
\end{remark}
The approximation guarantee for our dynamic tree, which follows from \cite{akk+20} and for which we provide a brief sketch in Appendix~\ref{sec:appendix-tree_data_structure} are as follows:
\begin{theorem}[Approximation guarantee of our dynamic tree. Informal version of Theorem~\ref{thm:app:tree_init}]\label{thm:tree_ds_approx}
Let $\Pi^q$ denote the sketching matrix generated by the dynamic tree, we then have that:
\begin{align*}
\|\Pi^q (\bigotimes_{i=1}^q A_i)x\|_2 = & ~ (1\pm\epsilon) \|(\bigotimes_{i=1}^q A_i)x\|_2.
\end{align*}
\end{theorem}
\paragraph{Choices of parameter $m$:}
An important parameter to be chosen is the sketching dimension $m$. Intuitively, it should depend polynomially on $\epsilon^{-1}, d, q$. We list them in the following table.
\begin{table}[!ht]
\centering
\begin{tabular}{|l|l|l|l|}
\hline
$C_{\rm base}$ & $T_{\rm base}$ & $m({\rm dim})$ & {\bf Init time} \\ \hline
{\sc CountSketch} & {\sc TensorSketch} & $\epsilon^{-2} q\cdot {\rm dim}^2\cdot (1/\delta)$ & $\sum_{i=1}^q \nnz(A_i)$ \\ \hline
{\sc OSNAP} & {\sc TensorSRHT} & $\epsilon^{-2}q^4 \cdot {\rm dim}\cdot\log(1/\delta)$ & $\sum_{i=1}^q \nnz(A_i)$ \\ \hline
{\sc OSNAP} & {\sc TensorSRHT} & $\epsilon^{-2}q\cdot {\rm dim}^2\cdot \log(1/\delta)$ & $\sum_{i=1}^q \nnz(A_i)$ \\ \hline
\end{tabular}
\vspace{2mm}
\caption{How different choices of $C_{\rm base}$ and $T_{\rm base}$ affect the sketching dimension $m({\rm dim})$ and initialization time that depends on the choice of sketch. We note that the sketching dimension $m$ is a function of ${\rm dim}$, the fundamental dimension of the problem. For example, ${\rm dim}=d$ for tensor product regression and ${\rm dim}=k$ for $k$-low rank approximation. The sketching dimension corresponds to the problem of computing an $(\epsilon, \delta, {\rm dim}, d, n)$-OSE.}
\label{tab:diff_sketch}
\end{table}
\paragraph{Robustness against an Adaptive Adversary:}
An often encountered issue in dynamic algorithms is the presence of an \emph{adaptive adversary} \cite{bkm+22}. Consider a sequence of update pairs $\{(i, B_i) \}_{i=1}^T$ in which the adversary can choose the pair $(i, B_i)$ based on the output of our data structure on $(i-1, B_{i-1})$. Such an adversary model naturally captures many applications, in which the data structure is used in an \emph{iterative process}, and the input to the data structure depends on the output from last iteration.
We will now describe how our data structure can be extended to handle an adaptive adversary. We store all initial factor matrices $A_1,\ldots,A_q$. During an update, suppose we are given a pair $(i, B_i)$. Instead of using the sketching matrices correspond to the $i$-th leaf during initialization, we instead choose fresh sketching matrices along the path from the leaf to the root. Since the only randomness being used is the sketching matrices along the path, by using fresh randomness, our data structure works against adaptive adversary.
Such modification does not affect the update time too much: during each update, we need to apply the newly-generated base sketch $C_i^{\new}$ to both $A_i$ and $B$, incurring a time of $\wt O(\nnz(A_i)+\nnz(B))$. Along the path, the data structure repeatedly apply $T_{k, \ell}^{\new}$ to its two children, which takes $\wt O(md)$ time. The overall time is thus
\begin{align*}
\wt O(\nnz(A_i)+\nnz(B)+md).
\end{align*}
Thus, in later applications of our data structure, we present the runtime without this modification.
\section{Faster Dynamic Tensor Product Algorithms with Dynamic Tree}
\label{sec:applications}
In this section, we instantiate the dynamic tree data structure introduced in section~\ref{sec:tree_data_structure} for three applications: dynamic tensor product regression, dynamic tensor spline regression, and dynamic tensor low rank approximation.
\subsection{Dynamic Tensor Product Regression}\label{sec:dtpregression}
We present an algorithm (Algorithm.~\ref{alg:kronecker_reg}) for solving the dynamic Tensor product regression problem. Our algorithm uses the dynamic tree data structure to maintain a sketch of the design matrix and updates it efficiently. Additionally, we maintain a sketch of the label vector $b$ via $\Pi^q b$. During the query phase, we output the solution using the sketch of the tensor product design matrix and the sketch of the label vector. We have the following guarantees, the proof of which we defer to Appendix~\ref{sec:appendix-dtpregression}:
\begin{theorem}[Informal version of Theorem~\ref{thm:app:dynamic_k_reg}]\label{thm:dynamic_k_reg}
There is an algorithm (Algorithm~\ref{alg:kronecker_reg}) for dynamic Tensor product regression problem with the following procedures:
\begin{itemize}
\item \textsc{Initialize}($A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2},\dots, A_q \in \R^{n_q \times d_q}, m\in \mathbb{N}_+, C_{\mathrm{base}}, T_{\mathrm{base}}, b\in \R^{n_1\ldots n_q}$): Given matrices $A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2}, \dots, A_q \in \R^{n_q \times d_q},$, a sketching dimension $m$, families of base sketches $C_{\mathrm{base}}$, $T_{\mathrm{base}}$ and a label vector $b\in \R^{n_1\ldots n_q}$, the data-structure pre-processes in time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd + m\cdot \nnz(b))$.
\item {\sc Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given a matrix $B$ and an index $i\in [q]$, the data structure updates the approximation to $A_1\otimes \ldots \otimes (A_i+B)\otimes \ldots \otimes A_q$ in time $\wt O(\nnz(B) + md)$.
\item \textsc{Query}: Query outputs an approximate solution $\hat{x}$ to the Tensor product regression problem where $\|\bigotimes_{i=1}^q A_i\hat{x} - b\|_2 = (1 \pm \epsilon) \|\bigotimes_{i=1}^q A_ix^* - b\|_2$ with probability at least $1 - \delta$ in time $O(md^{\omega-1}+d^{\omega})$ where $x^*$ is an optimal solution to the Tensor product regression problem i.e. $x^* = \arg \min_{x \in \R^d} \|\bigotimes_{i=1}^q A_ix - b\|_2$.
\end{itemize}
\end{theorem}
To realize the input sparsity time initialize and update time, we can use the combination of {\sc OSNAP} and {\sc TensorSRHT}, which gives a sketching dimension
\begin{align*}
m = & ~ \epsilon^{-2} q d^2 \log(1/\delta).
\end{align*}
Hence, the update time is $\wt O(\nnz(B)+\epsilon^{-2}q d^{3}\log(1/\delta))$. Compared to the static algorithm~\cite{djssw19}, we note that their algorithm is suitable to be modified into dynamic setting, with a running time of $\wt O(\nnz(A_i)+\nnz(B)+\epsilon^{-2}(q+d)d/\delta))$, their algorithm cannot accommodate general dynamic setting due to the $1/\delta$ dependence in running time. Suppose the length dynamic sequence is at least $d$, then one requires the failure probability to be at most $\frac{1}{d}$ for union bound. In such scenario, the algorithm of~\cite{djssw19} is strictly worse.~\cite{ffg22} improves the dependence on $d$, yielding a subquadratic update time in the form of $\wt O((\nnz(A_i)+\nnz(B)+\epsilon^{-2}q^2 d_i^\omega+\epsilon^{-1}d^{2-\theta})\log(1/\delta))$. While their algorithm is more efficient in solving the regression problem, their approach does not scale with the statistical dimension of the problem, and is geared only towards regression, whereas our data structure also handles low rank approximation.
\subsection{Dynamic Tensor Spline Regression}\label{sec:spline_regression}
Spline regression models are a well studied class of regression models~\cite{marsh2001spline}. In particular, Spline regression problems involving tensor products arise in the context of B-Splines and P-Splines~\cite{eilers1996flexible}. For a very brief but relevant introduction to Spline regression, we refer the reader to~\cite{dssw18}. In this section, we first define the Spline regression problem as it pertains to our sketching application. Algorithm~\ref{alg:kronecker_spline}, which shows how we can use our {\sc DynamicTensorTree} data structure to solve a dynamic version of the Spline regression problem is provided in Appendix~\ref{sec:appendix-splines}.
\begin{definition}[Spline Regression]
In the Spline regression problem, given a design matrix $A \in \R^{m \times n}$, a target vector $x \in \R^{n}$, a regularization matrix $L \in \R^{p \times d}$, and a non-negative penalty coefficient $\lambda \in \R$, the goal is to optimize
\begin{equation}\label{eq:spline_regression}
\min_{x \in \R^n} \|Ax-b\|_2^2 + \lambda \cdot \|Lx\|_2^2
\end{equation}
\end{definition}
Notice that the classic ridge regression problem is a special case of spline regression (when $L = I$). In the {\sc Tensor Spline Regression} problem, we want to solve the Spline regression problem where the design matrix $A$ can be decomposed as the tensor product of multiple smaller matrices i.e. $A = A_1 \otimes \dots \otimes A_q$. Our {\sc DynamicTensorTree} data structure can be used to solve a dynamic version of {\sc Tensor Spline Regression}. In our dynamic model, at each time step, the update is of the form $i \in [q], B \in \R^{n_i \times d_i}$, and the data structure has to update $A_i \gets A_i + B$.
Before introducing our main result, we define an important metric to measure the rank of a Spline regression:
\begin{definition}[Statistical dimension of Splines]
For matrix $A\in \R^{n\times d}$ and $L\in \R^{p\times d}$ with ${\rm rank}(L)=p$ and ${\rm rank}\left(\begin{bmatrix}
A \\
L
\end{bmatrix} \right)=d$. Given the generalized singular value decomposition~\cite{golubVanLoan2013Book} of $(A, L)$ such that
$
A = U \begin{bmatrix}
\Sigma & 0_{p\times (n-p)} \\
0_{(n-p)\times p} & I_{d-p}
\end{bmatrix}R Q^\top,
L = V\begin{bmatrix}
\Omega & 0_{p\times (n-p)}
\end{bmatrix} RQ^\top,
$
where $\Sigma={\rm diag}(\sigma_1,\ldots,\sigma_p), \Omega={\rm diag}(\mu_1,\ldots,\mu_p)\in \R^{p\times p}$ are diagonal matrices. Finally, let $\gamma_i=\sigma_i/\mu_i$ for $i\in [p]$.
The statistical dimension of the Spline is defined as
$\sd_{\lambda}(A, L) = \sum_{i=1}^p 1/(1+\lambda/\gamma_i^2)+d-p$.
\end{definition}
The statistical dimension of the Spline can be much smaller than $d$, and we will show that the sketching dimension depends on the statistical dimension instead of $d$.
Let $x^*$ denote an optimal solution of the Spline regression problem, i.e.
\begin{align*}
x^* = \underset{x \in \R^n}{\arg\min} \|Ax - b\|_2^2 + \lambda \cdot \|Lx\|_2^2
\end{align*}
and let $\opt \coloneqq \|Ax^* - b\|_2^2 + \lambda \cdot \|Lx^*\|_2^2$. We can compute $x^*$ given $A, b, L, \lambda$ by $x^* = (A^\top A + \lambda L^\top L)^{-1} A^{\top} b$.
\begin{theorem}[Guarantees for {\sc DTSRegression}. Informal version of Theorem~\ref{thm:app:spline}]\label{thm:spline}
There exists an algorithm (Algorithm~\ref{alg:kronecker_spline}) with the following procedures:
\begin{itemize}
\item \textsc{Initialize}$(A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q})$: Given matrices $A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$, the data structure pre-processes in time
\begin{align*}
\wt O(\sum_{i=1}^q \nnz(A_i)+ qmd + m \cdot \nnz(b)).
\end{align*}
\item \textsc{Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given an index $i\in [q]$ and an update matrix $B\in \R^{n\times d}$, the data structure updates in time $\wt O(\nnz(B) + md)$.
\item \textsc{Query}: Let $A$ denote the tensor product $\bigotimes_{i=1}^q A_i$. Query outputs a solution $\wt x \in \R^n$ with constant probability such the $\wt x$ is an $\epsilon$ approximate to the Tensor Spline Regression problem i.e.
\begin{align*}
\|A \wt x - b\|_2^2 + \lambda \cdot \|L \wt x\|_2^2 \leq (1+ \epsilon) \cdot \opt
\end{align*}
where $\opt = \min_{x \in \R^n} \|Ax - b\|_2^2 + \lambda \cdot \|L x\|_2^2$. Query takes time $md^{(\omega - 1)} + pd^{(\omega - 1)} + d^{\omega}$.
\end{itemize}
\end{theorem}
We defer the proof of the above theorem to Appendix~\ref{sec:appendix-splines}.
The sketching dimension $m$ here depends on the statistical dimension of the Spline, which is at most $d$. To realize the above runtime, one can pick {\sc OSNAP} and {\sc TensorSHRT}, which gives $m=\epsilon^{-1}q \sd^2_{\lambda}(A, L) \log(1/\delta)$ and an update time $\wt O(\nnz(B)+\epsilon^{-1} q\sd^2_{\lambda}(A, L) d\log(1/\delta))$. The improved dependence on $\epsilon^{-1}$ comes from using approximate matrix product directly, instead of OSE.
\subsection{Dynamic Tensor Low Rank Approximation}\label{sec:low_rank_approximation}
In this section, we consider a problem studied in~\cite{djssw19}: given matrices $A_1,\ldots,A_q$, the goal is to compute a low rank approximation for the tensor product matrix $A_1\otimes \ldots \otimes A_q \in \R^{n\times d}$, specifically, a rank-$k$ approximation $B\in \R^{n\times d}$ such that
\begin{align*}
\|B-A\|_F\leq (1+\epsilon) \mathrm{OPT}_k,
\end{align*}
where $\mathrm{OPT}_k=\min_{\mathrm{rank}\text{-}k~A'}\|A'-A\|_F$ and $A=\bigotimes_{i=1}^q A_i$. The~\cite{djssw19} algorithm works as follows: they pick $q$ independent {\sc CountSketch} matrices~\cite{ccf02} with $O(\epsilon^{-2} qk^2)$ rows, then apply each sketch matrix $S_i$ to $A_i$. After that, they form the tensor product $M=\bigotimes_{i=1}^q S_i A_i$ and run SVD on $M=U\Sigma V^\top$. Finally, they output $B=AU_k^\top U_k$ in factored form (as matrices $A_1,\ldots,A_q,U_k$), where $U_k\in \R^{k\times d}$ denotes the top $k$ right singular vectors. By doing so, they achieve an overall running time of
\begin{align*}
O(\sum_{i=1}^q \nnz(A_i)+\epsilon^{-2q}q^q k^{2q} d^{\omega-1})
\end{align*}
Two central issues around their algorithm are 1).\ exponential dependence on parameters $\epsilon^{-1}, q, k$ and 2).\ the algorithm is inherently static, so a natural way to make it dynamic is to apply $S_i$ to the update $B$, reforming the tensor product of sketches and compute the SVD.
We address these two issues simultaneously by using our tree data structure. For the sketching dimension, we observe it is enough to design a sketch that is $(\epsilon, \delta, k, d, n)$-OSE and $(\epsilon/k, \delta)$-approximate matrix product, therefore, we can set sketching dimension $m=\epsilon^{-2}q k^2 \log(1/\delta)$. As we will show later, this gives a much improved running time. By using the dynamic tree, we also provide a dynamic algorithm.
\begin{theorem}[Guarantees for {\sc LowRankMaintenance}. Informal version of Theorem~\ref{thm:app:low_rank}]\label{thm:low_rank}
There exists an algorithm (Algorithm~\ref{alg:kronecker_low_rank}) that has the following procedures
\begin{itemize}
\item \textsc{Initialize}$(A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q})$: Given matrices $A_1\in \R^{n_1\times d},\ldots,A_q\in \R^{n_q\times d_q}$, the data structure processes in time
\begin{align*}
\wt O(\sum_{i=1}^q \nnz(A_i)+qmd).
\end{align*}
\item \textsc{Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given an index $i\in [q]$ and an update matrix $B\in \R^{n_i\times d_i}$, the data structure updates the approximation for $A_1\otimes \ldots \otimes (A_i+B)\otimes \ldots \otimes A_q$ in time
\begin{align*}
\wt O(\nnz(B)+md).
\end{align*}
\item \textsc{Query}: Let $A$ denote the tensor product $\bigotimes_{i=1}^q A_i$. The data structure outputs a rank-$k$ approximation $C$ such that
\begin{align*}
\|C-A\|_F \leq & ~ (1+\epsilon)\min_{\mathrm{rank}\text{-}k~A'}~\|A'-A\|_F.
\end{align*}
The time to output $C$ is $\wt O(md^{\omega-1})$.
\end{itemize}
\end{theorem}
By choosing $m=\epsilon^{-2}qk^2\log(1/\delta)$, we can provide good guarantees for low rank approximation. This gives
\begin{itemize}
\item Initialization time in $\wt O(\sum_{i=1}^q \nnz(A_i)+\epsilon^{-2}q dk^2\log(1/\delta))$;
\item Update time in $\wt O(\nnz(B)+\epsilon^{-2}qdk^2\log(1/\delta))$;
\item Query time in $\wt O(\epsilon^{-2}qd^{\omega-1}k^2\log(1/\delta))$.
\end{itemize}
Without using fast matrix multiplication, our algorithm improves upon the prior state-of-the-art~\cite{djssw19} on all parameters even in the static setting. %To the best of our knowledge, our tensor product low rank approximation algorithm is the fastest, both in static and dynamic settings. %Need to be careful here, and need to compare with the new NeurIPS'22 paper.
\section{Conclusion \& Future Directions}\label{sec:conclusion}
In this work, we initiate the study of the Dynamic Tensor Regression, Dynamic Tensor Spline Regression, and Dynamic Tensor Low-Rank Approximation problems, and design a tree data structure {\sc DynamicTensorTree} which can be used to provide fast dynamic algorithms for these problems. The problems we study are very interesting and there are a lot of exciting future directions, some of which we discuss below. First, it is unclear whether the algorithms we proposed are optimal and so having lower bounds for the update times would be an important direction. Second, we have focused exclusively on regression problems with the $\ell_2$ norm whereas solving Dynamic Tensor Regression and Dynamic Tensor Spline Regression with respect to the more robust $\ell_1$-norm as well as for more general $p$-norms are also very appealing.
\section*{Acknowledgments}
We would like to thank all the NeurIPS 2022 reviewers of our paper and also the ICML 2022 reviewers who reviewed an earlier version of this work. Most of this work was done while LZ was at CMU. AR was supported in-part by NSF grants CCF-1652491, CCF-1934931, and CCF-1955351 during the preparation of this manuscript.
\section*{\centering{Appendix}}
\paragraph{Roadmap.}
In this appendix, we provide omitted details and proofs for our main paper. In particular, section~\ref{sec:appendix-prelim} of our appendix corresponds to section~\ref{sec:preliminaries} of our main paper, section~\ref{sec:appendix-tree_data_structure} to section~\ref{sec:tree_data_structure}, section~\ref{sec:appendix-dtpregression} to section~\ref{sec:dtpregression}, section~\ref{sec:appendix-splines} to section~\ref{sec:spline_regression}, and section~\ref{sec:appendix-low-rank} to section~\ref{sec:low_rank_approximation}.
\section{Background} \label{sec:appendix-prelim}
In this section, we provide definitions for some terms used in the main paper and rest of the appendix.
\begin{definition}[Statistical Dimension]\label{def:SD}
For any $\lambda \geq 0$ and any positive semidefinite matrix $M \in \R^{n \times n}$, the $\lambda$-statistical dimension of $M$ is
\begin{align*}
\sd_{\lambda}(M) &\coloneqq \tr[ M(M+\lambda I_n)^{-1} ]
\end{align*}
\end{definition}
\begin{definition}[Oblivious Subspace Embedding~\cite{s06,akk+20}]\label{def:OSE}
Let $\epsilon,\delta\in (0,1), \mu > 0,$ and $d,n\geq 1$ be integers. An $(\epsilon,\delta,\mu,d,n)$-Oblivious Subspace Embedding (OSE) is a distribution over $m\times d$ matrices with the guarantee that for any fixed matrix $A\in \R^{d\times n}$ with $\sd_{\lambda}(A^\top A) \leq \mu$, we have for any vector $x\in \R^n$, with probability at least $1-\delta$,
\begin{align*}
\|\Pi Ax\|_2+\lambda \|x\|_2=(1\pm\epsilon) (\|Ax\|_2+\lambda \|x\|_2) .
\end{align*}
\end{definition}
\begin{definition}[Approximate Matrix Product]\label{def:amp}
Given $\epsilon,\delta>0$, we say that a distribution ${\cal D}$ over $m\times d$ matrices $\Pi$ has the $(\epsilon,\delta)$-approximate matrix product property if for every $C,D\in \R^{d\times n}$, with probability at least $1-\delta$,
\begin{align*}
\|C^\top \Pi^\top \Pi D-C^\top D\|_F \leq & ~ \epsilon \|C\|_F\|D\|_F.
\end{align*}
\end{definition}
%We will define two key sketching matrices that will serve the crux of our data structure.
\begin{definition}[\textsc{OSNAP} matrix~\cite{nn13}]
\label{def:osnap}
For every sparsity parameter $s$, target dimension $m$, and positive integer $d$, the OSNAP transform with sparsity $s$ is defined as
\begin{align*}
S_{r,j} = & ~ \frac{1}{\sqrt s} \cdot \delta_{r,j}\cdot \sigma_{r,j},
\end{align*}
for all $r\in [m]$, $j\in [d]$, where $\sigma_{r,j}\in \{\pm 1\}$ are independent and uniform Rademacher random variables and $\delta_{r,j}$ are Bernoulli random variables satisfying
\begin{itemize}
\item For every $i\in [d]$, $\sum_{r\in [m]}\delta_{r,i}=s$. In other words, every column has exactly $s$ nonzero entries.
\item For all $r\in [m]$ and all $i\in [d]$, $\E[\delta_{r,i}]=s/m$.
\item The $\delta_{r,i}$'s are negatively correlated: $\forall\ T\subset [m]\times [d],\ \E[\prod_{(r,i)\in T}\delta_{r,i}]\leq \prod_{(r,i)\in T} \E[\delta_{r,i}]=(\frac{s}{m})^{|T|}$.
\end{itemize}
\end{definition}
\begin{definition}[\textsc{TensorSRHT}~\cite{akk+20}]
\label{def:TensorSRHT}
We define the \textsc{TensorSRHT} $S : \R^d \times \R^d \rightarrow \R^m$ as $S = \frac{1}{\sqrt m} P \cdot (H D_1\times H D_2)$, where each row of $P\in \{0,1\}^{m\times d}$ contains only one $1$ at a random coordinate, one can view $P$ as a sampling matrix. $H$ is a $d \times d$ Hadamard matrix, and $D_1, D_2$ are two $d\times d$ independent diagonal matrices with diagonals that are each independently set to be a Rademacher random variable (uniform in $\{-1,1\}$).
\end{definition}
\iffalse
Here, we will define some sketches which we use throughout our paper: Subsampled Randomized Hadamard Transform ({\sc SRHT})~\cite{ldfu13}, Count Sketch ({\sc CountSketch})~\cite{ccf02} and Tensor Count Sketch ({\sc TensorSketch})~\cite{p13,pp13}.
Next, we introduce \textsc{CountSketch} and \textsc{TensorSketch}, which are important sketching matrices we will be using across our paper.
\fi
\begin{definition}[\textsc{CountSketch} matrix~\cite{ccf02}]\label{def:cs}
Let $h : [n] \rightarrow [b]$ be a random $2$-wise independent hash function and $\sigma : [n] \rightarrow \{-1,+1\}$ be a random $4$-wise independent hash function. Then $R\in\R^{b\times n}$ is a \textsc{CountSketch} matrix if we set $R_{h(i),i} = \sigma(i)$ for all $i\in[n]$ and all the other entries of $R$ to zero.
\end{definition}
\begin{definition}[\textsc{TensorSketch} matrix~\cite{p13,pp13}]\label{def:TensorSketch}
Let $h_1,h_2:[m] \rightarrow [s]$ be 3-wise independent hash functions, also let $\sigma:[m] \rightarrow \{\pm 1\}$ be a 4-wise independent random sign function. The degree two \textsc{TensorSketch} transform, $S:\R^m\times \R^m\rightarrow \R^s$ is defined as follows: for any $i,j\in [m]$ and $r\in [s]$,
\begin{align*}
S_{r,(i,j)} = & ~ \sigma_1(i)\cdot \sigma_2(j)\cdot \mathbf{1}[h_1(i)+h_2(j)=r~\textnormal{mod}~s].
\end{align*}
\end{definition}
\begin{definition}[Subsampled Randomized Hadamard Transform ({\sc SRHT}) matrix \cite{ldfu13}]\label{def:srht}
We say $R \in \R^{b\times n}$ is a subsampled randomized Hadamard transform matrix\footnote{In this case, we require $\log{n}$ to be an integer.} if it is of the form $R = \sqrt{n/b} \cdot SHD$, where $S \in \R^{b \times n}$ is a random matrix whose rows are $b$ uniform samples (without replacement) from the standard basis of $\R^n$, $H \in \R^{n \times n}$ is a normalized Walsh-Hadamard matrix, and $D \in \R^{n \times n}$ is a diagonal matrix whose diagonal elements are i.i.d. Rademacher random variables.
\end{definition}
\section{Proofs for Dynamic Tree Data Structure (section ~\ref{sec:tree_data_structure})}\label{sec:appendix-tree_data_structure}
In this section, we provide proofs for the running time and approximation guarantees of our dynamic tree data structure (Algorithm~\ref{alg:tree_static}).
\subsection{Running Time of Algorithm~\ref{alg:tree_static}}
\begin{theorem}[Formal version of Theorem~\ref{thm:tree_init}]\label{thm:app:tree_init}
There is an algorithm (Algorithm~\ref{alg:tree_static}) that has the following procedures:
\begin{itemize}
\item \textsc{Initialize}($A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2},\dots, A_q \in \R^{n_q \times d_q}, m\in \mathbb{N}_+, C_{\mathrm{base}}, T_{\mathrm{base}}$): Given matrices $A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2}, \dots, A_q \in \R^{n_q \times d_q}$, a sketching dimension $m$, families of base sketches $C_{\mathrm{base}}$ and $T_{\mathrm{base}}$, the data-structure pre-processes in time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd)$ where $d = d_1 d_2 \dots d_q$.
\item {\sc Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given a matrix $B$ and an index $i\in [q]$, the data structure updates the approximation to $A_1\otimes \ldots \otimes (A_i+B)\otimes \ldots \otimes A_q$ in time $\wt O(\nnz(B) + md)$.
\end{itemize}
\end{theorem}
Note: We will assume that $n_1 = n_2 = \dots = n_q$ and $d_1 = d_2 = \dots = d_q$ for the following computations. To modify our {\sc DynamicTensorTree} to work for the non-uniform case, we only need to choose the base sketches at the leaves of the appropriate dimension and also have different dimensions for the internal nodes.
\begin{proof}
{\bf Proof of \textsc{Initialization}}. The initialization part has $2$ key steps - applying the $T_{\mathrm{base}}$ sketches to the leaves and applying $S_{\mathrm{base}}$ sketches to each of the internal nodes.
\begin{itemize}
\item Applying a {\sc CountSketch} matrix $C_i:\R^{m \times n_i}$ to any matrix $A_i: \R^{n_i \times d_i}$ takes input sparsity time $O(\nnz(A_i))$. Therefore, the total time for this step is $O(\sum_{i = 1}^q \nnz(A_i))$. If {\sc OSNAP} is used, this is true up to polylogarithmic factors.
\item For all internal nodes, let us consider the time of computing one node at level $\ell \in [\log_2 q]$ using {\sc TensorSRHT}. Consider the computation of $J_{k,\ell} \in \R^{m\times d_1^{2^\ell}}$, we pick a {\sc TensorSRHT} $T_{k,\ell}\in \R^{m\times m^2}$ and apply it to $J_{2k,\ell-1}\otimes J_{2k+1,\ell-1}$. This can be viewed as computing the tensor product of column vectors of $J_{2k,\ell-1}$ and $J_{2k+1,\ell-1}$: each column of $J_{k,\ell}$ is a tensor product of two columns of its two children after applying $T_{k,\ell}$. Each column takes time $O(m \log m)$ and the total number of columns of $J_{k,\ell}$ is $d_1^{2^\ell}$. Hence, the time of forming $J_{k,\ell}$ is $O(md_1^{2^\ell}\log m)$.
\item For each level, there are $q/2^\ell$ nodes, so $O(q/2^\ell md_1^{2^\ell})$ time in total for level $\ell$.
\item Sum over all levels for $\ell\in [\log q]$, we have
\begin{align*}
m \log m (\sum_{\ell=1}^{\log q} \frac{q}{2^\ell} d_1^{2^\ell}) \leq & ~ m \log m (\sum_{\ell=1}^{\log q} \frac{q}{2^\ell} d_1^{q}) \\
= & ~ md\log m(\sum_{\ell=1}^{\log q} \frac{q}{2^\ell}) \\
\leq & ~ 2q md\cdot\log m.
\end{align*}
%\item Applying a {\sc TensorSketch} or {\sc TensorSRHT} $T_{k,\ell} \in \R^{m \times m^2}$ to $J_{2k,\ell-1} \otimes J_{2k+1,\ell-1}$ takes time $O(d_1^{2^{\ell}}(\nnz(J_{2k,\ell-1}) + \nnz(J_{2k+1,\ell-1})) + 2 d_1^{2^{\ell}} m)$. Since $J_{2k,\ell-1}, J_{2k+1,\ell-1} \in \R^{m \times d_1^{2^{\ell - 1}}}$, $\nnz(J_{2k,\ell-1}) + \nnz(J_{2k+1,\ell-1}) \leq 2md_1^{2^{\ell-1}}$. Thus, the time for this step is $O(2md_1^{3 \cdot 2^{\ell-1}} + 2m d_1^{3 \cdot 2^{\ell-1}} \log(m))$ for $\ell\in [\log_2 q]$.
%\item Therefore, for one level $\ell$, it takes time $\wt O (qmd_1^{3\cdot 2^{\ell-1}}/2^{\ell})$ because there are $q/2^{\ell}$ nodes in one level and each node takes time $\wt O(md_1^{3 \cdot 2^{\ell - 1}})$. This is at most $\wt O(md_1^{3q/2})$ which is $\wt O(md^{3/2})$ where $d = d_1^q$.
%\item Since there are $\log(q)$ levels in the entire tree, it takes a total of time $\wt O(md^{3/2} \log(q))$. Since $d = d_1^q > q$, this time is $\wt O(md \log (d))$ which is further $\wt O(md^{3/2})$.
\item Thus, the total time for initialization is
\begin{align*}
\wt O(\sum_{i=1}^q \nnz(A_i) + qmd).
\end{align*}
\end{itemize}
\iffalse
\Lichen{My calculation:
For each internal node at level $\ell$, its corresponding matrix is of size $m\times d_1^{2^\ell}$, this means that to form it, one can view it as computing $d^{2^\ell}$ tensor product of vectors. For each tensor product, it takes $O(m\log m)$ time, so in total, $(d_1^{2^\ell}m\log m)$ time. For each level, there are $q/2^\ell$ nodes, this means that the total time for a level is $O(q/2^\ell d_1^{2^\ell}m\log m)$. Sum over all levels, we have
\begin{align*}
\sum_{\ell=1}^{\log q} q/2^\ell d_1^{2^\ell} \leq & ~ \sum_{\ell=1}^{\log q} q/2^\ell d_1^q \\
\leq & ~ q\cdot (\frac{1-1/q}{1/2}) \cdot d_1^q \\
\leq & ~ 2q\cdot d_1^q \\
= & ~ O(q d_1^q).
\end{align*}
Hence, the total time is $O(qm d_1^q \log m)=O(qmd \log m)$.
}
\fi
{\bf Proof of \textsc{Update}}. We note that \textsc{Update} can be decomposed into the following parts:
\begin{itemize}
\item Apply \textsc{CountSketch} $C_i$ to the update matrix $B$, which takes $O(\nnz(B))$ time. If {\sc OSNAP} or {\sc SRHT} are used, then this is true up to polylogarithmic factors.
\item Updating the leaf node takes $O(md_i)$ time because $C_i B \in \R^{m\times d_i}$.
\item For each level $\ell\in [\log_2 q]$, we first compute the update tree by applying \textsc{TensorSketch} $T_{k,\ell} \in \R^{m \times m^2}$ to $J_{2k,\ell-1} \otimes J_{2k+1,\ell-1}$ which takes time $O(md_1^{2^\ell} \log m)$.
\item We also need to update \textbf{one} internal node for each level, which takes $\wt O(md_1^{2^{\ell}})$ time. This is at most $O(md \log m)$ for the root. Since there are $\log q$ levels, the total time for updating all internal nodes is also $\wt O(md)$.
\iffalse
\item We also compute $J_0^\top J_0$. This takes time $O(\Tmat(d,m,d))$.
\item We then compute $(J_0^\top J_0)^{-1}$. This takes time $\Tmat(d,d,d)$.
\fi
\end{itemize}
Hence, the total runtime of \textsc{Update} is
\begin{align*}
& ~ \wt O(\nnz(B)+ m d_1 + md ) \\
= & ~ \wt O(\nnz(B) + md).
\end{align*}
%Here, $\omega$ is the exponent of the current matrix multiplication time~\cite{w12,aw21}. The last step follows from $\omega<2.5$.
\end{proof}
\subsection{Approximation Guarantee of Algorithm~\ref{alg:tree_static}}
In this section, we provide the approximation guarantee of Algorithm~\ref{alg:tree_static}.
\begin{theorem}\label{thm:app:tree_approximation_guarantee}
Let $A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$ be the input matrices, $m\in \mathbb{N}_+$ be the sketching dimension, let $C_{\mathrm{base}}, T_{\mathrm{base}}$ be appropriate family of sketching matrices. Let $\Pi^q$ be the sketching matrix generated by Algorithm~\ref{alg:tree_static}. Then for any $x\in \R^{d}$, we have that with probability at least $1-\delta$,
\begin{align*}
\|\Pi^q \bigotimes_{i=1}^q A_i x\|_2 = & ~ (1\pm\epsilon) \|\bigotimes_{i=1}^q A_i x\|_2,
\end{align*}
if
\begin{itemize}
\item $C_{\mathrm{base}}$ is {\sc CountSketch}, $T_{\mathrm{base}}$ is {\sc TensorSketch} and $m=\Omega(\epsilon^{-2}qd^2 1/\delta)$.
\item $C_{\mathrm{base}}$ is {\sc OSNAP}, $T_{\mathrm{base}}$ is {\sc TensorSRHT} and $m=\wt \Omega(\epsilon^{-2}q^4 d\log(1/\delta))$.
\item $C_{\mathrm{base}}$ is {\sc OSNAP}, $T_{\mathrm{base}}$ is {\sc TensorSRHT} and $m=\wt \Omega(\epsilon^{-2}qd^2 \log(1/\delta))$.
\end{itemize}
\end{theorem}
First we will discuss the theorems of \cite{akk+20} which lead to our approximation guarantees and then discuss the specifics for each choice of base sketches seperately.
\begin{lemma}[Theorem 1 of~\cite{akk+20}]\label{lem:cs_tensorsketch}
For every positive integers $n,q,d$, every $\epsilon,\sd_\lambda>0$, there exists a distribution on linear sketches $\Pi^q\in \R^{m\times d^q}$ such that: 1).\ If $m=\Omega(\epsilon^{-2}q\sd_\lambda^2)$, then $\Pi^q$ is an $(\epsilon,1/10,\sd_\lambda,d^q,n)$-OSE (Def.~\ref{def:OSE}). 2).\ If $m=\Omega(\epsilon^{-2}q)$, then $\Pi^q$ has the $(\epsilon,1/10)$-approximate matrix product (Def.~\ref{def:amp}).
\end{lemma}
\begin{lemma}[Theorem 2 of~\cite{akk+20}]\label{lem:osnap_tensorsrht}
For every positive integers $n,q,d$, every $\epsilon,\sd_\lambda>0$, there exists a distribution on linear sketches $\Pi^q\in \R^{m\times d^q}$ such that: 1).\ If $m=\wt \Omega(\epsilon^{-2}q\sd_\lambda^2)$, then $\Pi^q$ is an $(\epsilon,1/\poly(n),\sd_\lambda,d^q,n)$-OSE (Def.~\ref{def:OSE}). 2).\ If $m=\wt \Omega(\epsilon^{-2}q)$, then $\Pi^q$ has the $(\epsilon,1/\poly(n))$-approximate matrix product (Def.~\ref{def:amp}).
\end{lemma}
\begin{lemma}[Theorem 3 of~\cite{akk+20}]\label{lem:srht_tensorsrht}
For every positive integers $n,q,d$, every $\epsilon,\sd_\lambda>0$, there exists a distribution on linear sketches $\Pi^q\in \R^{m\times d^q}$ which is an $(\epsilon,1/\poly(n),\sd_\lambda,d^q,n)$-OSE (Def.~\ref{def:OSE}), provided that the integer $m$ satisfies $m = \wt \Omega(q^4 \sd_{\lambda}/\epsilon^2)$
\end{lemma}
We note that the 3 results due to~\cite{akk+20} are actually stronger than what we need here, since the statements say that they are subspace embedding for \emph{all} matrices of proper size. Also, when the statistical dimension $\sd_\lambda$ is smaller than the rank of the design matrix, the sketching dimension is even smaller, which is important in our Spline regression application.
%\Lichen{Please try to fill in}\\
We then proceed to prove the individual guarantees for different choice of base sketches item by item.
\begin{itemize}
\item By Lemma~\ref{lem:cs_tensorsketch}, we know that in order for $\Pi^q$ to be an OSE, we will require $m=\Omega(\epsilon^{-2}qd^2 1/\delta)$. Note that the statistical dimension $\sd_\lambda$ is $d$ in this case. We point out that the dependence on success probability is $1/\delta$, this means that we can not obtain a high probability version. This is the main reason that Lemma~\ref{lem:cs_tensorsketch} only provides an OSE with constant probability.
\item By Lemma~\ref{lem:srht_tensorsrht}, we need $m=\wt \Omega(\epsilon^{-2}q^4 d\log(1/\delta))$. We get a nearly linear dependence on $d$ in the expense of worse dependence on $q$.
\item By Lemma~\ref{lem:osnap_tensorsrht}, we will require $m=\wt \Omega(\epsilon^{-2}qd^2 \log(1/\delta))$. In this scenario, the dependence on $\delta$ is $\log(1/\delta)$, hence we can aim for a high probability guarantee for our OSE.
\end{itemize}
\section{Proofs for Dynamic Tensor Product Regression (section \ref{sec:dtpregression})}
\label{sec:appendix-dtpregression}
\begin{algorithm}[!ht]\caption{Dynamic Tensor Product Regression}
\label{alg:kronecker_reg}
\begin{algorithmic}[1]
\State {\bf data structure} {\sc DTRegression}
\State
\State {\bf members}
\State \hspace{4mm} $\textsc{DynamicTensorTree DTT} $
\State \hspace{4mm} $A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$
\State \hspace{4mm} $\wt b\in \R^{m}$
\State {\bf end members}
\State
\Procedure{Initialize}{$A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}, m,C_{\mathrm{base}},T_{\mathrm{base}}, b\in \R^n$}
\State $\textsc{DTT.Initialize}(A_1,\ldots,A_q, m,C_{\mathrm{base}},T_{\mathrm{base}})$
\State $A_1\gets A_1,\ldots,A_q\gets A_q$
\State Let $\Pi_q \in \R^{m\times n}$ be the sketching matrix corresponds to {\sc DTT}
\State $\wt b\gets \Pi_q b$
\EndProcedure
\State
\Procedure{Query}{$ $}
\State $M\gets \textsc{DTT.}J_{0,0}$
\State $\wt x \gets \arg \min_{x \in \R^n} \|Mx - \wt b\|_2^2$
\State \Return $x$
\EndProcedure
\end{algorithmic}
\end{algorithm}
\begin{theorem}[Formal version of Theorem~\ref{thm:dynamic_k_reg}]\label{thm:app:dynamic_k_reg}
There is a dynamic data structure for dynamic Tensor product regression problem with the following procedures:
\begin{itemize}
\item \textsc{Initialize}($A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2},\dots, A_q \in \R^{n_q \times d_q}, m\in \mathbb{N}_+, C_{\mathrm{base}}, T_{\mathrm{base}}, b\in \R^{n_1\ldots n_q}$): Given matrices $A_1 \in \R^{n_1 \times d_1}, A_2 \in \R^{n_2 \times d_2}, \dots, A_q \in \R^{n_q \times d_q},$, a sketching dimension $m$, families of base sketches $C_{\mathrm{base}}$, $T_{\mathrm{base}}$ and a label vector $b\in \R^{n_1\ldots n_q}$, the data-structure pre-processes in time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd+m\cdot \nnz(b))$.
\item {\sc Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given a matrix $B$ and an index $i\in [q]$, the data structure updates the approximation to $A_1\otimes \ldots \otimes (A_i+B)\otimes \ldots \otimes A_q$ in time $\wt O(\nnz(B) + md)$.
\item \textsc{Query}: Query outputs an approximate solution $\hat{x}$ to the Tensor product regression problem where $\|\bigotimes_{i=1}^q A_i\hat{x} - b\|_2 = (1 \pm \epsilon) \|\bigotimes_{i=1}^q A_ix^* - b\|_2$ with probability at least $1 - \delta$ in time $O(md^{\omega-1}+d^{\omega})$ where $x^*$ is an optimal solution to the Tensor product regression problem i.e. $x^* = \arg \min_{x \in \R^d} \|\bigotimes_{i=1}^q A_ix - b\|_2$.
\end{itemize}
\end{theorem}
\begin{proof}
{\bf Proof of \textsc{Initialization}}. The initialization part has $3$ steps:
\begin{itemize}
\item Initializing DTT takes time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd)$ from Theorem~\ref{thm:app:tree_init}.
\item Initializing all the matrices $A_i, i \in [q]$ takes time $\wt O(\sum_{i=1}^q \nnz(A_i))$.
\item Computing $\hat b$ takes time $\wt O(m \cdot \nnz(b))$.
\end{itemize}
Therefore, the total time for initialization is
\begin{align*}
& ~ \wt O(\sum_{i=1}^q \nnz(A_i) + qmd + \sum_{i=1}^q \nnz(A_i) + m \cdot \nnz(b) )\\
= & ~ \wt O(\sum_{i=1}^q \nnz(A_i) + qmd + m \cdot \nnz(b))
\end{align*}
{\bf Proof of \textsc{Update}}. We just update our DTT data structure. This takes time $\wt O(\nnz(B) + md^{3/2})$ from Theorem~\ref{thm:app:tree_init}.
{\bf Proof of \textsc{Query} time}. We can view the Query part as having the following $5$ steps:
\begin{itemize}
\item Initializing $M$ takes time $\wt O(md)$ since $M \in \R^{m \times d}$.
\item $M^\top (\wt b)$, this takes $O(md)$ time because we are multiplying a $d \times m$ matrix with a $m \times 1$ vector.
\item $M^\top \cdot M$, this step takes $\Tmat(d,m,d)$ time as we are multiplying an $d \times m$ matrix with an $m \times d$ matrix.
\item $(M^\top M)^{-1}$ this step takes $\Tmat(d,d,d)$ time as it is inverting $M^\top M$ which is of size $d \times d$.
\item $(M^\top M)^{-1} (M^\top \wt b)$. In this step, we are multiplying $(M^\top M)^{-1}$ which is a $d \times d$ matrix with a vector $M^\top \wt b$ of size $d$. This takes time $O(d^2)$.
\end{itemize}
Overall, the total running time of $\textsc{Query}$ is
\begin{align*}
& ~ \wt O(md + md + \Tmat(d,m,d) + \Tmat(d,d,d) + d^2) \\
= & ~ \wt O(md + md^{(\omega - 1)} + d^{\omega} + d^2) \\
= & ~ \wt O(md^{(\omega - 1)} + d^{\omega})
\end{align*}
{\bf Proof of Correctness for \textsc{Query}}. The proof of correctness for Query follows from Theorem~\ref{thm:app:tree_approximation_guarantee} identically to the proof of Theorem 3.1 in \cite{dssw18}.
\end{proof}
\section{Omitted details and proofs for Dynamic Tensor Spline Regression (section~\ref{sec:spline_regression})}\label{sec:appendix-splines}
To specify the sketching dimension $m$ needed , we use the following definition of {\it Statistical Dimension} for Splines~\cite{dssw18}:
\begin{definition}[Statistical Dimension for Splines]
For the Spline regression problem specified by Eq.~\ref{eq:spline_regression}, the statistical dimension is defined as $\sd_{\lambda}(A,L) = \sum_{i}1/(1 + \lambda/\gamma_i^2) + d - p$. Here $\{\gamma_i, i \in [p]\}$ are the {\it generalized singular values} of $(A,L)$ which are defined as follows.
\end{definition}
\begin{definition}[Generalized singular values]
For matrices $A \in \R^{n \times d}$ and $L \in \R^{p \times d}$ such that $\rank(L) = p$ and $\rank\left(\mat{c} A \\ L \rix\right) = d$, the generalized singular value decomposition (GSVD)~\cite{golubVanLoan2013Book} of $(A,L)$ is given by the pair of factorizations $A=U\begin{bmatrix}\Sigma& {0}_{p\times (d-p)}\cr { 0}_{(d-p)\times p}& I_{d-p}
\end{bmatrix}RQ^\top \quad \hbox{and} \quad
L=V\begin{bmatrix}\Omega &{
0}_{p \times (d-p)}\end{bmatrix}RQ^\top,
$
where $U \in \R^{n\times d}$ has orthonormal columns,
$V \in \R^{p \times p}$, $Q\in \R^{d \times d} $ are orthogonal, $R\in\mathbb{R}^{d\times d}$ is
upper triangular and nonsingular, and $\Sigma$ and $\Omega$ are
$p\times p$ diagonal matrices:
$\Sigma=\diag(\sigma_1,\sigma_2,\ldots,\sigma_{p})$ and
$\Omega=\diag(\mu_1,\mu_2,\ldots,\mu_p)$ with
$
0 \leq \sigma_1 \leq \sigma_2 \leq \ldots \leq \sigma_p < 1
\quad \hbox{and} \quad
1 \geq \mu_1 \geq \mu_2 \geq \ldots \geq \mu_p>0,
$
satisfying $\Sigma^\top \Sigma+\Omega^\top \Omega=I_{p}$. The {\em generalized singular values $\gamma_i$} of $(A,L)$ are defined by the ratios $\gamma_i=\sigma_i/\mu_i$ for $i \in [p]$.
\end{definition}
We prove that for the spline regression, our sketching matrix can have even smaller dimension. The following is due to~\cite{dssw18}.
\begin{lemma}
\label{lem:dssw18_1}
Let $x^*=\arg\min_{x\in \R^d} \|Ax-b\|_2^2+\lambda \|Lx\|_2^2$, $A\in \R^{n\times d}$ and $b\in \R^n$. Let $U_1\in \R^{n\times d}$ be the first $n$ rows of an orthogonal basis for $\begin{bmatrix}
A \\
\sqrt{\lambda} L
\end{bmatrix}\in \R^{(n+p)\times d}$. Let sketching matrix $S\in \R^{m\times n}$ have a distribution such that with probability $1-\delta$,
\begin{itemize}
\item $\|U_1^\top S^\top SU_1-U_1^\top U_1\|_2 \leq 1/4$,
\item $\|U_1^\top(S^\top S-I)(b-Ax^*)\|_2 \leq \sqrt{\epsilon~\mathrm{OPT}/2}$.
\end{itemize}
Let $\wt x$ denote $\arg\min_{x\in \R^d} \|S(Ax-b)\|_2^2+\lambda \|Lx\|_2^2$. Then with probability at least $1-\delta$,
\begin{align*}
\|A\wt x-b\|_2^2 + \lambda \|L\wt x\|_2^2 = & ~ (1\pm\epsilon)\mathrm{OPT}.
\end{align*}
\end{lemma}
\begin{lemma}
\label{lem:dssw18_2}
For $U_1$ as in Lemma~\ref{lem:dssw18_1}, we have
\begin{align*}
\|U_1\|_F^2 = & ~ \sd_{\lambda}(A, L).
\end{align*}
\end{lemma}
\begin{algorithm}[!ht]\caption{Dynamic Tensor Spline Regression}
\label{alg:kronecker_spline}
\begin{algorithmic}[1]
\State {\bf data structure} {\sc DTSRegression}
\State
\State {\bf members}
\State \hspace{4mm} $\textsc{DynamicTensorTree DTT} $
\State \hspace{4mm} $A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$
\State \hspace{4mm} $\wt b\in \R^{m}$
\State {\bf end members}
\State
\Procedure{Initialize}{$A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}, m,C_{\mathrm{base}},T_{\mathrm{base}}, b\in \R^n$}
\State $\textsc{DTT.Initialize}(A_1,\ldots,A_q, m,C_{\mathrm{base}},T_{\mathrm{base}})$
\State $A_1\gets A_1,\ldots,A_q\gets A_q$
\State Let $\Pi_q \in \R^{m\times n}$ be the sketching matrix corresponds to {\sc DTT}
\State $\wt b\gets \Pi_q b$
\EndProcedure
\State
\Procedure{Query}{$ $}
\State $M\gets \textsc{DTT.}J_{0,0}$
\State $\wt x \gets \arg \min_{x \in \R^n} \|Mx - \wt b\|_2^2 + \|Lx\|_2^2$
\State \Return $x$
\EndProcedure
\end{algorithmic}
\end{algorithm}
We are now ready to prove the main theorem.
\begin{theorem}[Formal version of Theorem~\ref{thm:spline}]\label{thm:app:spline}
There exists an algorithm (Algorithm~\ref{alg:kronecker_spline}) with the following procedures:
\begin{itemize}
\item \textsc{Initialize}$(A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q})$: Given matrices $A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$, the data structure pre-processes in time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd + m \cdot \nnz(b))$.
\item \textsc{Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given an index $i\in [q]$ and an update matrix $B\in \R^{n\times d}$, the data structure updates in time $\wt O(\nnz(B) + md)$.
\item \textsc{Query}: Let $A$ denote the Tensor product $\bigotimes_{i=1}^q A_i$. Query outputs a solution $\wt x \in \R^n$ with constant probability such the $\wt x$ is an $\epsilon$ approximate to the Tensor Spline Regression problem i.e. $\|A \wt x - b\|_2^2 + \lambda \cdot \|L \wt x\|_2^2 \leq (1+ \epsilon) \cdot \opt$ where $\opt = \min_{x \in \R^n} \|Ax - b\|_2^2 + \lambda \cdot \|L x\|_2^2$. Query takes time $\wt O(md^{(\omega - 1)} + pd^{(\omega - 1)} + d^{\omega})$.
\end{itemize}
Moreover, we have that if
\begin{itemize}
\item $C_{\mathrm{base}}$ is {\sc CountSketch}, $T_{\mathrm{base}}$ is {\sc TensorSketch}, then $m = \Omega(\epsilon^{-1}q{\sd_{\lambda}^2(A,L)} 1/\delta)$.
\item $C_{\mathrm{base}}$ is {\sc OSNAP}, $T_{\mathrm{base}}$ is {\sc TensorSRHT}, then $m=\wt \Omega(\epsilon^{-1}q^4 \sd_{\lambda}(A,L)\log(1/\delta))$.
% \item $C_{\mathrm{base}}$ is {\sc OSNAP}, $T_{\mathrm{base}}$ is {\sc TensorSRHT}, then $m=\wt \Omega(\epsilon^{-2}q\sd_{\lambda}^2(A,L) \log(1/\delta))$.
\end{itemize}
\end{theorem}
\begin{proof}
{\bf Proof of Correctness for \textsc{Query}}.
To prove that it is enough to use sketching whose dimension depends on $\sd_{\lambda}(A, L)$, we just need to show that the sketching matrix related to the dynamic tree satisfies the two guarantees of Lemma~\ref{lem:dssw18_1}. We will use approximate matrix product (Def.~\ref{def:amp}) to prove this guarantee. By Lemma~\ref{lem:dssw18_2}, we have that $\|U_1\|_F^2=\sd_{\lambda}(A, L)$, and consider an $(\sqrt{\epsilon}/\sd_{\lambda}(A, L),\delta)$-approximate matrix product sketching, we have that
\begin{align*}
\|U_1^\top S^\top S U_1-U_1^\top U_1\|_2 \leq & ~ \|U_1^\top S^\top S U_1-U_1^\top U_1\|_2 \\
\leq & ~ \sqrt{\epsilon} \|U_1\|_F^2/\sd_{\lambda}(A, L) \\
= & ~ \sqrt{\epsilon},
\end{align*}
setting $\epsilon\leq 1/2$, we obtain desired result. For the second part, we set $C=U_1$ and $D=b-Ax^*$, then
\begin{align*}
\|U_1^\top (S^\top S-I)(b-Ax^*b) \|_2 \leq & ~ \|U_1^\top (S^\top S-I)(b-Ax^*b) \|_F \\
\leq & ~ \sqrt{\epsilon}/\sd_{\lambda}(A, L)\cdot \|U_1\|_F \|b-Ax^*\|_2 \\
= & ~ \sqrt{\epsilon/\sd_{\lambda}(A,L)}~\sqrt{{\rm OPT}} \\
\leq & ~ \sqrt{\epsilon~{\rm OPT}/2}.
\end{align*}
This means that to obtain good approximation for Spline regression, it is enough to employ $(\sqrt{\epsilon}/\sd_{\lambda}(A, L), \delta)$-approximate matrix product. The dimension follows from Lemma~\ref{lem:cs_tensorsketch} and~\ref{lem:osnap_tensorsrht}.
We note that even though the definition of statistical dimension in~\cite{akk+20} and ours is different, our definition is more general and the~\cite{akk+20} definition can be viewed as picking $L^\top L$ to be the identity matrix. Hence, one can easily generalize the conclusion of Lemma~\ref{lem:cs_tensorsketch},~\ref{lem:osnap_tensorsrht} and~\ref{lem:srht_tensorsrht} to our definition of statistical dimension via the argument used in~\cite{anw17,dssw18,akk+20}.
{\bf Proof of \textsc{Initialization}}. The initialization part of Algorithm~\ref{alg:kronecker_spline} is identical to the initialization part of Algorithm~\ref{alg:kronecker_reg}. Therefore, the time for initialization is $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd + m \cdot \nnz(b))$ from Theorem~\ref{thm:app:dynamic_k_reg}.
{\bf Proof of \textsc{Update}}. We just update our DTT data structure. This takes time $\wt O(\nnz(B) + md)$ from Theorem~\ref{thm:app:tree_init}.
{\bf Proof of \textsc{Query} time}. As mentioned in Section~\ref{sec:spline_regression}, we can compute $\wt x \gets \arg \min_{x \in \R^n} \|Mx - \wt b\|_2^2 + \|Lx\|_2^2$ by $\wt x \gets (M^\top M + \lambda L^\top L)^{-1} M^{\top} \wt b$. We can view Query as consisting of the following $5$ steps:
\begin{itemize}
\item Initializing $M$ takes time $\wt O(md)$ since $M \in \R^{m \times d}$.
\item $M^\top (\wt b)$, this takes $O(md)$ time because we are multiplying a $d \times m$ matrix with a $m \times 1$ vector.
\item $M^\top \cdot M$, this step takes $\Tmat(d,m,d)$ time as we are multiplying a $d \times m$ matrix with an $m \times d$ matrix.
\item $L^\top L$, this step takes $\Tmat(d,p,d)$ time as we are multiplying a $d \times p$ matrix with a $p \times d$ matrix.
\item $M^\top M + \lambda L^\top L$, this takes $O(d^2)$ time as we are adding two $d \times d$ size matrices.
\item $(M^\top M + \lambda L^\top L)^{-1}$, this step takes $\Tmat(d,d,d)$ time as it is inverting a matrix of size $d \times d$.
\item $(M^\top M + \lambda L^\top L)^{-1} (M^\top \wt b)$. In this step, we are multiplying $(M^\top M + \lambda L^\top L)^{-1}$ which is a $d \times d$ matrix with a vector $M^\top \wt b$ of size $d$. This takes time $O(d^2)$.
\end{itemize}
Overall, the total running time of $\textsc{Query}$ is
\begin{align*}
& ~ \wt O(md + md + \Tmat(d,m,d) + \Tmat(d,p,d) + d^2 + \Tmat(d,d,d) + d^2) \\
= & ~ \wt O(md + md^{(\omega - 1)} + pd^{(\omega - 1)} + d^{\omega} + d^2) \\
= & ~ \wt O(md^{(\omega - 1)} + pd^{(\omega - 1)} + d^{\omega})
\end{align*}
\end{proof}
\section{Omitted details and proofs for Dynamic Tensor Low Rank Approximation (section~\ref{sec:low_rank_approximation})}\label{sec:appendix-low-rank}
For low rank approximation, we show it is enough to design sketching matrix whose dimension depends on $k$ instead of $d$.
\begin{definition}[Projection Cost Preserving Sketch]
Let $A\in \R^{n\times d}$, we say a sketching matrix $S\in \R^{m\times n}$ is a $k$ \emph{Projection Cost Preserving Sketch} (PCPSketch if for any orthogonal projection $P\in \R^{d\times d}$ of rank $k$, we have
\begin{align*}
(1-\epsilon) \|A-AP\|_F^2 \leq \|SA-SAP\|_F^2+c\leq (1+\epsilon) \|A-AP\|_F^2
\end{align*}
where $c\geq 0$ is some fixed constant independent of $P$, but may depend on $A$ and $SA$.
\end{definition}
For low rank approximation, we note that the optimal low rank approximation for $A$ is projecting onto the top $k$ left singular vectors. Hence, if we can prove that the sketching corresponds to the dynamic tree is a $k$-PCPSketch, then we are done. We will require a few technical tools.
\begin{lemma}[Theorem 12 of~\cite{cem+15}]
\label{lem:cem_1}
Let $A\in \R^{n\times d}$. Suppose $S\in \R^{m\times n}$ is an $(\epsilon, \delta, k, d, n)$-OSE and $(\epsilon/\sqrt{k},\delta)$-approximate matrix product, then $S$ is a $k$-PCPSketch for $A$.
\end{lemma}
\begin{algorithm}[!ht]\caption{Our low rank approximation algorithm}
\label{alg:kronecker_low_rank}
\begin{algorithmic}[1]
\State {\bf data structure} {\sc LowRankMaintenance}
\State
\State {\bf members}
\State \hspace{4mm} $\textsc{DynamicTensorTree DTT} $
\State \hspace{4mm} $A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$
\State {\bf end members}
\State
\Procedure{Initialize}{$A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q}$}
\State $\textsc{DTT.Initialize}(A_1,\ldots,A_q)$
\State $A_1\gets A_1,\ldots,A_q\gets A_q$
\EndProcedure
\State
\Procedure{Update}{$i\in [q], B\in \R^{n_i\times d_i}$}
\State $\textsc{DTT.Update}(i, B)$
\EndProcedure
\State
\Procedure{Query}{}
\State $M\gets \textsc{DTT.}J_{0,0}$
\State Compute SVD of $M$ such that $M=U\Sigma V^\top$
\State $U_k\gets \text{top $k$ right singular vectors of $M$}$
\State \Return $A_1\otimes \ldots \otimes A_q U_k^\top U_k$ in factored form
\EndProcedure
\end{algorithmic}
\end{algorithm}
\begin{theorem}[Formal version of Theorem~\ref{thm:low_rank}]\label{thm:app:low_rank}
There exists an algorithm (Algorithm~\ref{alg:kronecker_low_rank}) that has the following procedures
\begin{itemize}
\item \textsc{Initialize}$(A_1\in \R^{n_1\times d_1},\ldots,A_q\in \R^{n_q\times d_q})$: Given matrices $A_1\in \R^{n\times d},\ldots,A_q\in \R^{n\times d}$, the data structure processes in time $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd + m \cdot \nnz(b))$.
\item \textsc{Update}$(i\in [q], B\in \R^{n_i\times d_i})$: Given an index $i\in [q]$ and an update matrix $B\in \R^{n\times d}$, the data structure updates the approximation for $A_1\otimes \ldots \otimes (A_i+B)\otimes \ldots \otimes A_q$ in time $\wt O(\nnz(B) + md))$.
\item \textsc{Query}: Let $A$ denote the tensor product $\bigotimes_{i=1}^q A_i$. The data structure outputs a rank-$k$ approximation $C$ such that
\begin{align*}
\|C-A\|_F \leq & ~ (1+\epsilon)\min_{\mathrm{rank}-k~A'}~\|A'-A\|_F.
\end{align*}
The time to output $C$ is $\wt O(md^{\omega-1})$.
\end{itemize}
Moreover, we have that if
\begin{itemize}
\item $C_{\mathrm{base}}$ is {\sc CountSketch}, $T_{\mathrm{base}}$ is {\sc TensorSketch} and $m=\Omega(\epsilon^{-2}qk^2 1/\delta)$.
% \item $C_{\mathrm{base}}$ is {\sc SRHT}, $T_{\mathrm{base}}$ is {\sc TensorSRHT} and $m=\wt \Omega(\epsilon^{-2}q^4 d\log(1/\delta))$.
\item $C_{\mathrm{base}}$ is {\sc OSNAP}, $T_{\mathrm{base}}$ is {\sc TensorSRHT} and $m=\wt \Omega(\epsilon^{-2}qk^2 \log(1/\delta))$.
\end{itemize}
\end{theorem}
\begin{proof}
{\bf Proof of Correctness for \textsc{Query}}.
We first show that if we have a $k$-PCPSketch for $A$, then we can compute a matrix $C$ that is a good approximation to rank-$k$ low rank approximation. Recall that for any projection matrix, we have
\begin{align*}
(1-\epsilon) \|AP-A\|_F^2 \leq \|SAP-SA\|_F^2+c \leq (1+\epsilon) \|AP-A\|_F^2,
\end{align*}
we need to design the projection $P$. Let $SA=U\Sigma V^\top$ be the singular value decomposition of $SA$, and let $U_k\in \R^{k\times d}$ be the top $k$ left singular vectors of $SA$. Set $P=U_k^\top U_k$, we note that $SAU_k^\top U_k$ is the optimal rank-$k$ approximation for $SA$. On the other hand, let $Q\in \R^{d\times d}$ be the projection such that $AQ$ is the optimal rank-$k$ approximation for $A$. We have
\begin{align*}
\|SAU_k^\top U_k-SA\|_F^2+c \leq \|SAQ-SA\|_F^2+c
= (1\pm\epsilon) \min_{\text{rank-$k$ projection}~P}\|AP-A\|_F^2.
\end{align*}
By Lemma~\ref{lem:cem_1}, to obtain a $k$-PCPSketch, we need $(\epsilon,\delta,k,d,n)$-OSE and $(\epsilon/\sqrt{k},\delta)$-approximate matrix product, choosing dimension according to Lemma~\ref{lem:cs_tensorsketch} and~\ref{lem:osnap_tensorsrht} gives the desired result.
{\bf Proof of \textsc{Initialization}}. The initialization part of Algorithm~\ref{alg:kronecker_spline} is identical to the initialization part of Algorithm~\ref{alg:kronecker_reg}. Therefore, the time for initialization is $\wt O(\sum_{i=1}^q \nnz(A_i) + qmd + m \cdot \nnz(b))$ from Theorem~\ref{thm:app:dynamic_k_reg}.
{\bf Proof of \textsc{Update}}. We just update our DTT data structure. This takes time $\wt O(\nnz(B) + md)$ from Theorem~\ref{thm:app:tree_init}.
{\bf Proof of \textsc{Query} time}. We can view Query as consisting of the following $5$ steps:
\begin{itemize}
\item Initializing $M$ takes time $\wt O(md)$ since $M \in \R^{m \times d}$.
\item Computing the SVD of $M$ takes time $O(md^{\omega-1})$ since $M$ is $m \times d$ matrix and $m\geq d$.
\item $U_k$ can be obtained directly from the SVD computation so this step doesn't take any additional time.
\item Once the algorithm has $U_k$, it can terminate as it outputs $(A_1,...,A_q,U_k)$ in factored form.
\end{itemize}
Overall, the total running time of $\textsc{Query}$ is
\begin{align*}
\wt O(md + md^{\omega-1}))
= & ~ \wt O(md^{\omega-1}). \qedhere
\end{align*}
\end{proof}
\newcommand{\etalchar}[1]{$^{#1}$}
\begin{thebibliography}{WZD{\etalchar{+}}20}
\bibitem[ACW17]{anw17}
Haim Avron, {Kenneth L.} Clarkson, and {David P.} Woodruff.
\newblock Sharper bounds for regularized data fitting.
\newblock In {\em Approximation, Randomization, and Combinatorial Optimization.
Algorithms and Techniques - 20th International Workshop, APPROX 2017 and 21st
International Workshop, RANDOM 2017}, August 2017.
\bibitem[AKK{\etalchar{+}}20]{akk+20}
Thomas~D Ahle, Michael Kapralov, Jakob~BT Knudsen, Rasmus Pagh, Ameya
Velingker, David~P Woodruff, and Amir Zandieh.
\newblock Oblivious sketching of high-degree polynomial kernels.
\newblock In {\em Proceedings of the Fourteenth Annual ACM-SIAM Symposium on
Discrete Algorithms (SODA)}, pages 141--160, 2020.
\bibitem[ALS{\etalchar{+}}18]{alszz18}
Alexandr Andoni, Chengyu Lin, Ying Sheng, Peilin Zhong, and Ruiqi Zhong.
\newblock Subspace embedding and linear regression with orlicz norm.
\newblock In {\em International Conference on Machine Learning (ICML)}, pages
224--233. PMLR, 2018.
\bibitem[BBB{\etalchar{+}}19]{bbb+19}
Frank Ban, Vijay Bhattiprolu, Karl Bringmann, Pavel Kolev, Euiwoong Lee, and
David~P. Woodruff.
\newblock A ptas for $\ell_p$-low rank approximation.
\newblock In {\em Proceedings of the Thirtieth Annual ACM-SIAM Symposium on
Discrete Algorithms}, SODA '19, page 747–766, 2019.
\bibitem[BKM{\etalchar{+}}22]{bkm+22}
Amos Beimel, Haim Kaplan, Yishay Mansour, Kobbi Nissim, Thatchaphol Saranurak,
and Uri Stemmer.
\newblock Dynamic algorithms against an adaptive adversary: Generic
constructions and lower bounds.
\newblock In {\em Proceedings of the 54th Annual ACM SIGACT Symposium on Theory
of Computing}, pages 1671--1684, 2022.
\bibitem[BPSW21]{bpsw21}
Jan van~den Brand, Binghui Peng, Zhao Song, and Omri Weinstein.
\newblock Training (overparametrized) neural networks in near-linear time.
\newblock In {\em ITCS}, 2021.
\bibitem[Bub15]{b15}
S{\'e}bastien Bubeck.
\newblock Convex optimization: Algorithms and complexity.
\newblock {\em Foundations and Trends{\textregistered} in Machine Learning},
8(3-4):231--357, 2015.
\bibitem[BW14]{bw14}
Christos Boutsidis and David~P Woodruff.
\newblock Optimal cur matrix decompositions.
\newblock In {\em Proceedings of the 46th Annual ACM Symposium on Theory of
Computing (STOC)}, 2014.
\bibitem[BWZ16]{bwz16}
Christos Boutsidis, David~P Woodruff, and Peilin Zhong.
\newblock Optimal principal component analysis in distributed and streaming
models.
\newblock In {\em Proceedings of the forty-eighth annual ACM symposium on
Theory of Computing (STOC)}, pages 236--249, 2016.
\bibitem[CCFC02]{ccf02}
Moses Charikar, Kevin Chen, and Martin Farach-Colton.
\newblock Finding frequent items in data streams.
\newblock In {\em International Colloquium on Automata, Languages, and
Programming}, pages 693--703. Springer, 2002.
\bibitem[CEM{\etalchar{+}}15]{cem+15}
Michael~B Cohen, Sam Elder, Cameron Musco, Christopher Musco, and Madalina
Persu.
\newblock Dimensionality reduction for k-means clustering and low rank
approximation.
\newblock In {\em Proceedings of the forty-seventh annual ACM symposium on
Theory of computing}, pages 163--172, 2015.
\bibitem[CMP20]{cmp20}
Michael~B. Cohen, Cameron Musco, and Jakub Pachocki.
\newblock Online row sampling.
\newblock {\em Theory Comput.}, 16:Paper No. 15, 25, 2020.
\bibitem[CSTZ22]{cstz22}
Sitan Chen, Zhao Song, Runzhou Tao, and Ruizhe Zhang.
\newblock Symmetric sparse boolean matrix factorization and applications.
\newblock In {\em 13th Innovations in Theoretical Computer Science Conference
(ITCS 2022)}. Schloss Dagstuhl-Leibniz-Zentrum f{\"u}r Informatik, 2022.
\bibitem[CW13]{cw13}
Kenneth~L. Clarkson and David~P. Woodruff.
\newblock Low rank approximation and regression in input sparsity time.
\newblock In {\em Proceedings of the 45th Annual ACM Symposium on Theory of
Computing (STOC)}, 2013.
\bibitem[DJS{\etalchar{+}}19]{djssw19}
Huaian Diao, Rajesh Jayaram, Zhao Song, Wen Sun, and David Woodruff.
\newblock Optimal sketching for kronecker product regression and low rank
approximation.
\newblock {\em Advances in neural information processing systems},
32:4737--4748, 2019.
\bibitem[DSSW18]{dssw18}
Huaian Diao, Zhao Song, Wen Sun, and David Woodruff.
\newblock Sketching for kronecker product regression and p-splines.
\newblock In {\em International Conference on Artificial Intelligence and
Statistics}, pages 1299--1308. PMLR, 2018.
\bibitem[EM96]{eilers1996flexible}
Paul~HC Eilers and Brian~D Marx.
\newblock Flexible smoothing with b-splines and penalties.
\newblock {\em Statistical science}, 11(2):89--121, 1996.
\bibitem[EM06]{eilers2006multidimensional}
Paul~HC Eilers and Brian~D Marx.
\newblock Multidimensional density smoothing with p-splines.
\newblock In {\em Proceedings of the 21st international workshop on statistical
modelling}, 2006.
\bibitem[EMZ21]{emz21}
Hossein Esfandiari, Vahab Mirrokni, and Peilin Zhong.
\newblock Almost linear time density level set estimation via dbscan.
\newblock In {\em AAAI}, 2021.
\bibitem[FFG22]{ffg22}
Matthew Fahrbach, Thomas Fu, and Mehrdad Ghadiri.
\newblock Subquadratic kronecker regression with applications to tensor
decomposition.
\newblock {\em arXiv preprint arXiv:2209.04876}, 2022.
\bibitem[GVL13]{golubVanLoan2013Book}
Gene~H. Golub and Charles~F. Van~Loan.
\newblock {\em Matrix computations}.
\newblock Johns Hopkins Studies in the Mathematical Sciences. Johns Hopkins
University Press, Baltimore, MD, 2013.
\bibitem[HJS{\etalchar{+}}21]{hjstz21}
Baihe Huang, Shunhua Jiang, Zhao Song, Runzhou Tao, and Ruizhe Zhang.
\newblock Solving sdp faster: A robust ipm framework and efficient
implementation, 2021.
\bibitem[HLW17]{hlw17}
Jarvis Haupt, Xingguo Li, and David~P Woodruff.
\newblock Near optimal sketching of low-rank tensor regression.
\newblock In {\em ICML}, 2017.
\bibitem[IVWW19]{ivww19}
Pitor Indyk, Ali Vakilian, Tal Wagner, and David~P Woodruff.
\newblock Sample-optimal low-rank approximation of distance matrices.
\newblock In {\em Conference on Learning Theory}, pages 1723--1751. PMLR, 2019.
\bibitem[JKL{\etalchar{+}}20]{jklps20}
Haotian Jiang, Tarun Kathuria, Yin~Tat Lee, Swati Padmanabhan, and Zhao Song.
\newblock A faster interior point method for semidefinite programming.
\newblock In {\em FOCS}, 2020.
\bibitem[JLSW20]{jlsw20}
Haotian Jiang, Yin~Tat Lee, Zhao Song, and Sam Chiu-wai Wong.
\newblock An improved cutting plane method for convex optimization,
convex-concave games and its applications.
\newblock In {\em STOC}, 2020.
\bibitem[JPW22]{jpw22}
Shunhua Jiang, Binghui Peng, and Omri Weinstein.
\newblock Dynamic least-squares regression.
\newblock {\em arXiv preprint arXiv:2201.00228}, 2022.
\bibitem[JSWZ21]{jswz21}
Shunhua Jiang, Zhao Song, Omri Weinstein, and Hengjie Zhang.
\newblock Faster dynamic matrix inverse for faster lps.
\newblock In {\em STOC}. arXiv preprint arXiv:2004.07470, 2021.
\bibitem[LCK{\etalchar{+}}10]{leskovec2010kronecker}
Jure Leskovec, Deepayan Chakrabarti, Jon Kleinberg, Christos Faloutsos, and
Zoubin Ghahramani.
\newblock Kronecker graphs: an approach to modeling networks.
\newblock {\em Journal of Machine Learning Research}, 11(2), 2010.
\bibitem[LDFU13]{ldfu13}
Yichao Lu, Paramveer Dhillon, Dean~P Foster, and Lyle Ungar.
\newblock Faster ridge regression via the subsampled randomized hadamard
transform.
\newblock In {\em Advances in neural information processing systems}, pages
369--377, 2013.
\bibitem[LSZ19]{lsz19}
Yin~Tat Lee, Zhao Song, and Qiuyi Zhang.
\newblock Solving empirical risk minimization in the current matrix
multiplication time.
\newblock In {\em COLT}, 2019.
\bibitem[Mad13]{m13_flow}
Aleksander Madry.
\newblock Navigating central path with electrical flows: From flows to
matchings, and back.
\newblock In {\em 2013 IEEE 54th Annual Symposium on Foundations of Computer
Science (FOCS)}, pages 253--262. IEEE, 2013.
\bibitem[Mad16]{m16}
Aleksander Madry.
\newblock Computing maximum flow with augmenting electrical flows.
\newblock In {\em 2016 IEEE 57th Annual Symposium on Foundations of Computer
Science (FOCS)}, pages 593--602. IEEE, 2016.
\bibitem[MC01]{marsh2001spline}
L.C. Marsh and D.R. Cormier.
\newblock {\em Spline Regression Models}.
\newblock Number 137 in Quantitative Applications in the Social Sciences. SAGE
Publications, 2001.
\bibitem[MM13]{mm13}
Xiangrui Meng and Michael~W Mahoney.
\newblock Low-distortion subspace embeddings in input-sparsity time and
applications to robust linear regression.
\newblock In {\em Proceedings of the forty-fifth annual ACM symposium on Theory
of computing (STOC)}, pages 91--100, 2013.
\bibitem[MW21]{mw21}
Arvind~V. Mahankali and David~P. Woodruff.
\newblock Optimal $\ell_1$ column subset selection and a fast ptas for low rank
approximation.
\newblock In {\em Proceedings of the Thirty-Second Annual ACM-SIAM Symposium on
Discrete Algorithms}, SODA '21, page 560–578, 2021.
\bibitem[NK06]{nagy2006kronecker}
James~G Nagy and Misha~Elena Kilmer.
\newblock Kronecker product approximation for preconditioning in
three-dimensional imaging applications.
\newblock {\em IEEE Transactions on Image Processing}, 15(3):604--613, 2006.
\bibitem[NN13]{nn13}
Jelani Nelson and Huy~L Nguy{\^e}n.
\newblock Osnap: Faster numerical linear algebra algorithms via sparser
subspace embeddings.
\newblock In {\em Proceedings of the 54th Annual IEEE Symposium on Foundations
of Computer Science (FOCS)}, 2013.
\bibitem[OY05]{oh2005}
S.~Oh, S.~Kwon and J.~Yun.
\newblock A method for structured linear total least norm on blind
deconvolution problem.
\newblock {\em Applied Mathematics and Computing}, 19:151--164, 2005.
\bibitem[Pag13]{p13}
Rasmus Pagh.
\newblock Compressed matrix multiplication.
\newblock {\em ACM Trans. Comput. Theory}, 5(3), aug 2013.
\bibitem[PP13]{pp13}
Ninh Pham and Rasmus Pagh.
\newblock Fast and scalable polynomial kernels via explicit feature maps.
\newblock In {\em Proceedings of the 19th ACM SIGKDD international conference
on Knowledge discovery and data mining}, pages 239--247, 2013.
\bibitem[PSA21]{panahi2021shapeshifter}
Aliakbar Panahi, Seyran Saeedi, and Tom Arodz.
\newblock Shapeshifter: a parameter-efficient transformer using factorized
reshaped matrices.
\newblock {\em Advances in Neural Information Processing Systems}, 34, 2021.
\bibitem[RGlY78]{rgy78}
L.~R. Rabiner, B.~Go~ld, and C.~K. Yuen.
\newblock Theory and application of digital signal processing.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
8(2):146--146, 1978.
\bibitem[RRS{\etalchar{+}}22]{rrs+22}
Aravind Reddy, Ryan~A Rossi, Zhao Song, Anup Rao, Tung Mai, Nedim Lipka, Gang
Wu, Eunyee Koh, and Nesreen Ahmed.
\newblock One-pass algorithms for map inference of nonsymmetric determinantal
point processes.
\newblock In {\em International Conference on Machine Learning}, pages
18463--18482. PMLR, 2022.
\bibitem[RSW16]{rsw16}
Ilya Razenshteyn, Zhao Song, and David~P. Woodruff.
\newblock Weighted low rank approximations with provable guarantees.
\newblock In {\em Proceedings of the Forty-Eighth Annual ACM Symposium on
Theory of Computing}, STOC '16, page 250–263, 2016.
\bibitem[Sar06]{s06}
Tam{\'{a}}s Sarl{\'{o}}s.
\newblock Improved approximation algorithms for large matrices via random
projections.
\newblock In {\em Proceedings of 47th Annual {IEEE} Symposium on Foundations of
Computer Science (FOCS)}, 2006.
\bibitem[SSX21]{ssx21}
Anshumali Shrivastava, Zhao Song, and Zhaozhuo Xu.
\newblock Sublinear least-squares value iteration via locality sensitive
hashing.
\newblock {\em arXiv preprint arXiv:2105.08285}, 2021.
\bibitem[ST04]{st04}
Daniel~A Spielman and Shang-Hua Teng.
\newblock Nearly-linear time algorithms for graph partitioning, graph
sparsification, and solving linear systems.
\newblock In {\em Proceedings of the thirty-sixth annual ACM symposium on
Theory of computing}, pages 81--90, 2004.
\bibitem[Sti81]{s81}
Stephen~M. Stigler.
\newblock {Gauss and the Invention of Least Squares}.
\newblock {\em The Annals of Statistics}, 9(3):465 -- 474, 1981.
\bibitem[SWYZ21]{swyz21}
Zhao Song, David~P. Woodruff, Zheng Yu, and Lichen Zhang.
\newblock Fast sketching of polynomial kernels of polynomial degree.
\newblock In {\em ICML}, 2021.
\bibitem[SWZ17]{swz17}
Zhao Song, David~P Woodruff, and Peilin Zhong.
\newblock Low rank approximation with entrywise $\ell_1$-norm error.
\newblock In {\em Proceedings of the 49th Annual Symposium on the Theory of
Computing (STOC)}, 2017.
\bibitem[SWZ19a]{swz19_neurips1}
Zhao Song, David Woodruff, and Peilin Zhong.
\newblock Average case column subset selection for entrywise $\ell_1$-norm
loss.
\newblock {\em Advances in Neural Information Processing Systems (NeurIPS)},
32:10111--10121, 2019.
\bibitem[SWZ19b]{swz19_neurips2}
Zhao Song, David Woodruff, and Peilin Zhong.
\newblock Towards a zero-one law for column subset selection.
\newblock {\em Advances in Neural Information Processing Systems},
32:6123--6134, 2019.
\bibitem[SWZ19c]{swz19_soda}
Zhao Song, David~P Woodruff, and Peilin Zhong.
\newblock Relative error tensor low rank approximation.
\newblock In {\em SODA}. arXiv preprint arXiv:1704.08246, 2019.
\bibitem[SY21]{sy21}
Zhao Song and Zheng Yu.
\newblock Oblivious sketching-based central path method for solving linear
programming problems.
\newblock In {\em 38th International Conference on Machine Learning (ICML)},
2021.
\bibitem[SYZ21]{syz21}
Zhao Song, Shuo Yang, and Ruizhe Zhang.
\newblock Does preprocessing help training over-parameterized neural networks?
\newblock {\em Advances in Neural Information Processing Systems (NeurIPS)},
34, 2021.
\bibitem[SZZ21]{szz21}
Zhao Song, Lichen Zhang, and Ruizhe Zhang.
\newblock Training multi-layer over-parametrized neural network in subquadratic
time.
\newblock {\em arXiv preprint arXiv:2112.07628}, 2021.
\bibitem[VL00]{van2000ubiquitous}
Charles~F Van~Loan.
\newblock The ubiquitous kronecker product.
\newblock {\em Journal of computational and applied mathematics},
123(1-2):85--100, 2000.
\bibitem[WR12]{wr1912}
Alfred~North Whitehead and Bertrand Russell.
\newblock {\em Principia mathematica}, volume~2.
\newblock Cambridge University Press, 1912.
\bibitem[WY22]{wy22}
David~P Woodruff and Taisuke Yasuda.
\newblock Improved algorithms for low rank approximation from sparsity.
\newblock In {\em Proceedings of the 2022 Annual ACM-SIAM Symposium on Discrete
Algorithms (SODA)}, pages 2358--2403. SIAM, 2022.
\bibitem[WZ16]{wz16}
David~P Woodruff and Peilin Zhong.
\newblock Distributed low rank approximation of implicit functions of a matrix.
\newblock In {\em 2016 IEEE 32nd International Conference on Data Engineering
(ICDE)}, pages 847--858. IEEE, 2016.
\bibitem[WZD{\etalchar{+}}20]{wzd+20}
Ruosong Wang, Peilin Zhong, Simon~S Du, Russ~R Salakhutdinov, and Lin~F Yang.
\newblock Planning with general objective functions: Going beyond total
rewards.
\newblock In {\em Annual Conference on Neural Information Processing Systems
(NeurIPS)}, 2020.
\bibitem[XSS21]{xss21}
Zhaozhuo Xu, Zhao Song, and Anshumali Shrivastava.
\newblock Breaking the linear iteration cost barrier for some well-known
conditional gradient methods using maxip data-structures.
\newblock {\em Advances in Neural Information Processing Systems (NeurIPS)},
34, 2021.
\bibitem[XZZ18]{xzz18}
Chang Xiao, Peilin Zhong, and Changxi Zheng.
\newblock Bourgan: generative networks with metric embeddings.
\newblock In {\em Proceedings of the 32nd International Conference on Neural
Information Processing Systems (NeurIPS)}, pages 2275--2286, 2018.
\end{thebibliography}