Generalization Performance of Transfer Learning: Overparameterized and Underparameterized Regimes

Extracted source text · arXiv 2306.04901

\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage[numbers]{natbib}
\usepackage{bm}
\usepackage{dsfont}

\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{bbm}

\usepackage{color}
\usepackage{hyperref}
\usepackage{url}
\hypersetup{
    colorlinks,
    linkcolor={red!80!black},
    citecolor={blue!90!black},
    urlcolor={blue!90!black}
}

\usepackage[margin=1in]{geometry}
\usepackage{amsthm}
\usepackage{xcolor}
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{proposition}{Proposition}
\newtheorem{assumption}{Assumption}
\newtheorem{remark}{Remark}

\usepackage[capitalize,noabbrev]{cleveref}
\crefname{equation}{Eq.}{Eqs.}
\crefname{figure}{Fig.}{Figs.}
\crefname{assumption}{Assumption}{Assumptions}

% \title{Generalization Performance Analysis on Transfer Learning with Partial Task Similarity}
\title{Generalization Performance of Transfer Learning: Overparameterized and Underparameterized Regimes}
\date{June 1, 2023}

\newcommand*\samethanks[1][\value{footnote}]{\footnotemark[#1]}
\author{Peizhong Ju\thanks{Department of ECE, 
 The Ohio State University. Email: \texttt{\{ju.171, lin.4282, liang.889\}@osu.edu}} \and Sen Lin\samethanks[1] \and  Mark S. Squillante\thanks{Mathematical Sciences Department, IBM Research, USA. Email: \texttt{mss@us.ibm.com}} \and Yingbin Liang\samethanks[1] \and Ness B. Shroff\thanks{Department of ECE and CSE, The Ohio State University. Email: \texttt{shroff.11@osu.edu}}}


\input{newDefine.tex}

\begin{document}

\maketitle

\input{content.tex}


\bibliographystyle{plain}
\bibliography{refs}
% \bibliographystyle{apalike}

\clearpage
\newpage

\appendix
\input{appendix.tex}
\end{document}

\usepackage{dsfont}
\usepackage{bm}
% \usepackage{cases}
\usepackage{empheq}
\newcommand{\TODO}{{\color{red}TODO}}
\newcommand{\normTwo}[1]{\left\|#1\right\|}
\newcommand{\abs}[1]{\left\lvert#1\right\rvert}
\DeclareMathOperator{\eig}{Eig}

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\E}{\mathop{\mathbb{E}}}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\diag}{diag}
\newcommand{\defeq}{\coloneqq}

\newcommand\numberthis{\stepcounter{equation}\tag{\theequation}}
\newcommand{\iMatrix}[1]{\mathbf{I}_{#1}}

\newcommand{\sx}{\hat{\bm{x}}}
\newcommand{\sz}[1]{\hat{\bm{z}}_{(#1)}}
\newcommand{\sw}[1]{\hat{\bm{w}}_{(#1)}}
\newcommand{\sq}[1]{\hat{\bm{q}}_{(#1)}}
\newcommand{\ww}[1]{\bm{w}_{(#1)}}
\newcommand{\qq}[1]{\bm{q}_{(#1)}}
\newcommand{\featureSpace}[1]{\hat{\mathcal{S}}_{(#1)}}
\newcommand{\commonSpace}{\hat{\mathcal{S}}_{\text{co}}}
\newcommand{\commonSpaceSelected}{\mathcal{S}_{\text{co}}}
\newcommand{\featureSpaceSelected}[1]{\mathcal{S}_{(#1)}}
\newcommand{\pCommon}{p}
\newcommand{\pCommonThres}{t}
\newcommand{\pSpecial}[1]{p_{(#1)}}
\newcommand{\nSample}[1]{n_{(#1)}}
\newcommand{\XStacked}[1]{\mathbf{X}_{(#1)}}
\newcommand{\ZStacked}[1]{\mathbf{Z}_{(#1)}}
\newcommand{\yStacked}[1]{\bm{y}_{(#1)}}

\newcommand{\wbar}{\bar{\bm{w}}}
\newcommand{\qbar}{\bar{\bm{q}}}

\newcommand{\trainErr}[1]{\mathcal{L}_{(#1)}^{\text{train}}(\wbar,\qbar)}
\newcommand{\modelErr}{\mathcal{L}}

\newcommand{\wOne}{\bm{w}_{(1)}}
\newcommand{\wOneEx}{\bm{w}_{(1),e}}
\newcommand{\wTwoEx}{\bm{w}_{(2),e}}
\newcommand{\qOne}{\bm{q}_{(1)}}
\newcommand{\qOneEx}{\bm{q}_{(1),e}}
\newcommand{\nOne}{n_{(1)}}
\newcommand{\eOne}{\bm{\epsilon}_{(1)}}
\newcommand{\sigmaOne}{\sigma_{(1)}}
\newcommand{\yOne}{\bm{y}_{(1)}}

\newcommand{\wTwo}{\bm{w}_{(2)}}
\newcommand{\qTwo}{\bm{q}_{(2)}}
\newcommand{\nTwo}{n_{(2)}}
\newcommand{\eTwo}{\bm{\epsilon}_{(2)}}
\newcommand{\eTilde}{\tilde{\bm{\epsilon}}}
\newcommand{\sigmaTwo}{\sigma_{(2)}}
\newcommand{\yTwo}{\bm{y}_{(2)}}

\newcommand{\pSum}{\overline{p}}
\newcommand{\nSum}{\overline{n}}

\newcommand{\wTilde}{\tilde{\bm{w}}_{(1)}}
\newcommand{\wTildeEx}{\tilde{\bm{w}}_{(1),e}}
\newcommand{\qTildeOne}{\tilde{\bm{q}}_{(1)}}
\newcommand{\qTilde}{\tilde{\bm{q}}_{(2)}}
\newcommand{\qHat}{\hat{\bm{q}}_{(2)}}
\newcommand{\wTildeTwo}{\tilde{\bm{w}}_{(2)}}
\newcommand{\wHatTwo}{\hat{\bm{w}}_{(2)}}

\newcommand{\UU}{\mathbf{U}_{(1)}}
\newcommand{\XX}{\mathbf{X}_{(1)}}
\newcommand{\ZZ}{\mathbf{Z}_{(1)}}
\newcommand{\UUU}{\mathbf{U}_{(2)}}
\newcommand{\XXX}{\mathbf{X}_{(2)}}
\newcommand{\ZZZ}{\mathbf{Z}_{(2)}}
\newcommand{\ZFake}{\mathbf{Z}_{\text{fake}}}
\newcommand{\ZTrue}{\hat{\mathbf{Z}}_{(1)}}
\newcommand{\DD}{\mathbf{D}}
\newcommand{\GG}{\mathbf{G}}

\newcommand{\myProj}[1]{#1 (#1^T#1)^{-1}#1^T}
\newcommand{\PP}{\mathbf{P}}

\newcommand{\myMatrix}[1]{\left[\begin{smallmatrix}#1\end{smallmatrix}\right]}
\newcommand{\myMa}[1]{\left[\begin{matrix}#1\end{matrix}\right]}


% notations for transferring samples
\newcommand{\VV}{\mathbf{V}}
\newcommand{\xiTerm}{\bm{\xi}}
\newcommand{\sumNoise}{\sigma_{\Sigma}^2}

\newcommand{\EwOneDiff}{\mathcal{L}_{\text{co}}}
\newcommand{\EwOneDiffNoNoise}{\mathcal{L}_{\text{co}}^{\text{noiseless}}}
\newcommand{\EClipping}{\mathcal{L}_{\text{specific}}}
\newcommand{\oRatio}{r}
\newcommand{\similarity}{\delta}
\newcommand{\bNoise}{b_{\text{noise}}}

\newcommand{\RangeKSimilar}{\left[\frac{\left(\nOne-2\sqrt{\nOne\ln \nOne}\right)\similarity^2}{\left(\sqrt{\pCommon}+\sqrt{\nSum}+\sqrt{2\ln \nOne}\right)^2},\ \frac{\left(\nOne+2\sqrt{\nOne\ln \nOne}+2\ln \nOne\right)\similarity^2}{\left(\sqrt{\pCommon}-\sqrt{\nSum}-\sqrt{2\ln \nOne}\right)^2}\right]}
\newcommand{\KBias}{K_{\text{bias}}}
\newcommand{\KNoise}{K_{\text{noise}}}
\newcommand{\KSimilar}{K_{\text{similarity}}}

\newcommand{\TBias}{T_{\text{bias}}}
\newcommand{\TVar}{T_{\text{var}}}


\newcommand{\kTilde}{\tilde{k}}
\newcommand{\bigO}{\tilde{\mathcal{O}}}
\input{abstract.tex}
\input{introduction.tex}
\input{related_work}
\input{system_model.tex}
\input{results_main.tex}
\input{comparison.tex}
\input{conclusion.tex}
\begin{abstract}
Transfer learning is a useful technique for achieving improved performance and reducing training costs by leveraging the knowledge gained from source tasks and applying it to target tasks. Assessing the effectiveness of transfer learning relies on understanding the similarity between the ground truth of the source and target tasks. In real-world applications, tasks often exhibit partial similarity, where certain aspects are similar while others are different or irrelevant. To investigate the impact of partial similarity on transfer learning performance, we focus on a linear regression model with two distinct sets of features: a common part shared across tasks and a task-specific part. Our study explores various types of transfer learning, encompassing two options for parameter transfer. By establishing a theoretical characterization on the error of the learned model, we compare these transfer learning options, particularly examining how generalization performance changes with the number of features/parameters in both underparameterized and overparameterized regimes. Furthermore, we provide practical guidelines for determining the number of features in the common and task-specific parts for improved generalization performance. For example, when the total number of features in the source task's learning model is fixed, we show that it is more advantageous to allocate a greater number of redundant features to the task-specific part rather than the common part. Moreover, in specific scenarios, particularly those characterized by high noise levels and small true parameters, sacrificing certain \emph{true} features in the common part in favor of employing more \emph{redundant} features in the task-specific part can yield notable benefits.
% Transfer learning has emerged as a valuable approach for achieving improved performance and reducing training costs by leveraging knowledge gained from source tasks and applying it to target tasks. Assessing the effectiveness of transfer learning relies on understanding the similarity between the ground truth of the source and target tasks. In real-world applications, tasks often exhibit partial similarity, where certain aspects are similar while others are different or irrelevant. To investigate the impact of partial similarity on transfer learning performance, we focus on a linear regression model with two distinct sets of features: a common part shared across tasks and a task-specific part. Our study explores various types of transfer learning, encompassing options for transferring parameters and samples. By establishing theoretical upper bounds on the error of the learned model, we compare these transfer learning options, particularly examining how generalization performance changes concerning the number of features/parameters in both underparameterized and overparameterized regimes. Furthermore, we provide practical guidelines for determining the number of features in the common and task-specific parts. Through analytical and numerical analysis, we demonstrate the relative tightness of our upper bound. 
% Transfer learning has been shown to be effective to achieve better performance while saving training costs through transferring the learning resources/results of the source task(s) to the target task. To measure the usefulness of transfer learning, a critical factor is the similarity between the ground truth of the source task(s) and the ground truth of the target task. For most real-world applications, people do not have the exact same or totally different source and target tasks. Instead, most of the tasks are similar only in certain parts while totally different/irrelevant in other parts, i.e., partial similarity among tasks. To understand how such partial similarity affects the performance of transfer learning, we consider a linear regression model with two parts of features for each task: the common part and the task-specific part. Under this setup, we study different types of transfer learning, including different options for transferring parameters and transferring samples. By providing theoretical upper bounds on the error of the learned model, we make comparisons among these different options of transfer learning, especially on how the generalization performance changes with respect to the number of features/parameters for both the underparameterized and overparameterized regime. We also provide guidance on how to choose the number of features for the common part and the task-specific part. Analytical and numerical evidence shows that our upper bound is relatively tight.
\end{abstract}
\section{Introduction}

Transfer learning is a powerful technique that enhances the learning performance of a target task by leveraging knowledge from a related source task \citep{pan2010survey}. 
There are two main categories of transfer learning: parameter transfer and sample transfer. In parameter transfer, the learned parameters from the source task are directly copied to the target task's learning model. In sample transfer, training samples from the source task are integrated into the target task's dataset and contribute to its training process.
Comparing these two methods, sample transfer can provide additional valuable information and allow for preprocessing of the transferred samples to better align them with the target task, while parameter transfer offers significant savings in training costs and thus is very helpful for models with a large number of parameters such as deep neural networks (DNNs).

Despite the proven effectiveness of transfer learning with DNNs in various real-world applications, a comprehensive theoretical understanding of its performance remains under-explored. DNNs are typically overparameterized, allowing them to fit all training samples while maintaining relatively good generalization performance. This behavior challenges our understanding of the classical bias-variance trade-off. Recent studies have explored the phenomenon of "double-descent" or "benign overfitting" in certain linear regression setups, where the test error descends again in the overparameterized region, shedding light on this mystery. However, most of the existing literature focuses on single-task learning. The existence of a similar phenomenon in transfer learning, even in the simple linear regression setting, remains insufficiently explored. The additional transfer process in transfer learning makes the analysis of the generalization performance in the underparameterized and overparameterized regimes considerably more complex.
Furthermore, quantifying task similarity necessitates the development of appropriate analytical methods to establish a connection with the generalization performance of transfer learning.

In this paper, we investigate the generalization performance of transfer learning in linear regression models under both the underparameterized and overparameterized regimes. Unlike the existing literature that characterizes task similarity solely based on the distance between true parameters of the source and target tasks, we propose a more comprehensive framework that considers the similarity not only in terms of parameter distance but also in terms of feature sets. Specifically, we partition the feature space into a common part and a task-specific part. This setup enables us to analyze how the number of parameters in different parts influences the generalization performance of the target task. By characterizing the generalization performance, we offer insightful findings on transfer learning. For instance, when the total number of features in the source task's learning model is fixed, our analysis reveals the advantage of \emph{allocating more redundant features to the task-specific part rather than the common part}. Additionally, in specific scenarios characterized by high noise levels and small true parameters, \emph{sacrificing certain true features in the common part 
in favor of employing more redundant features in the task-specific part can yield notable benefits}.


% Transfer learning aims to improve the learning performance of a target task by  transferring the knowledge of a source task (or some source tasks) to the target task. Due to the similarity between the source task and the target task, transferring the data and/or the learned model of the source task to the target task is believed to be beneficial for most of cases. 
% There are two categories of transfer learning: one is transferring parameters and another is transferring samples. For the method of transferring parameters, the learned parameters in the source task are copied by the learning model of the target task. 
% For the method of transferring samples, the training samples of the source task are transferred to the dataset of the target task and will participate in the training process of the target task.
% For comparison between these two methods, transferring parameters can save the training cost a lot, while transferring samples may contain more useful information and can do preprocessing on the transferred samples (e.g. discard the dissimilar parts) to fit the target task better.

% Transfer learning with deep neural networks (DNNs) is proven to be very effective in many real-world applications, yet a complete theoretical understanding of its performance is still insufficient. DNNs usually are so overparameterized that can fit all the training samples yet still have a relatively good generalization performance, which seems to defy the classical understanding of the bias-variance trade-off. To understand such a mystery, some recent studies discover and study a phenomenon called ``double-descent'' or ``benign overfitting'' in linear regression. Later on, such a phenomenon is also studied in the neural tangent kernel (NTK), a model which approximates a shallow and wide neural network. However, since most of the existing literature is on single-task learning, whether/when a similar phenomenon exists in transfer learning is still insufficient even in the simple linear regression. Since transfer learning has an additional transferring process compared with single-task learning, the generalization performance analysis in the underparameterized and overparameterized regime becomes much more complicated.
% Meanwhile, the quantification of the task similarity also requires suitable analysis methods to make a connection to the generalization performance of transfer learning.

% In this paper, we study the generalization performance of transfer learning in both the underparameterized and overparameterized regime in a linear regression model. Different from the existing literature where task similarity is simply characterized as the distance between the true parameters of the source and target tasks, we consider a more general setup where the similarity is not only characterized by the distance of the parameters but also by the set of features. Specifically, we divide the feature space into two parts, the common part, and the task-specific part. Such a setup enables us to analyze how the numbers of parameters for different parts affect the generalization performance of the target task. By providing a relatively tight upper bound, we show some interesting insights on transfer learning. For example, when the total number of features in the source task's learning model is fixed, we show that it is more advantageous to allocate a greater number of redundant features to the task-specific part rather than the common part. Furthermore, in specific scenarios, particularly those characterized by high noise levels and small true parameters, sacrificing certain \emph{true} features in the common part in favor of employing more \emph{redundant} features in the task-specific part can yield notable benefits.

% \TODO






\subsection{Related Work}

``Benign overfitting'' and ``double-descent'' have been discovered and studied for overfitted solutions in single-task linear regression. Some works have explored double-descent with minimum $\ell_2$-norm overfitted solutions \citep{belkin2018understand, belkin2019two,bartlett2019benign,hastie2019surprises,muthukumar2019harmless} or minimum $\ell_1$-norm overfitted solutions \citep{mitra2019understanding,ju2020overfitting}, while employing simple features such as Gaussian or Fourier features.
In recent years, other studies have investigated overfitted generalization performance by utilizing features that approximate shallow neural networks. For example, researchers have explored random feature (RF) models \citep{mei2019generalization}, two-layer neural tangent kernel (NTK) models \citep{arora2019fine,satpathi2021dynamics,ju2021generalization}, and three-layer NTK models \citep{ju2022generalization}. Note that all of these studies have focused solely on a single task.

% Depending on the application scenarios, there are some learning techniques such as meta-learning and continual learning that are also regarded as some kinds of transfer learning. There exists some literature about the theoretical analysis of these learning techniques.

There are only a limited number of studies on the theoretical analysis of transfer learning. \cite{lampinen2018an} investigates
the generalization dynamics in transfer learning by multilayer linear networks using a student-teacher scenario where the teacher network generates data for the student network, which is different from our setup where the data of the source task and the target task are independently generated by their own ground truth. \cite{dhifallah2021phase} focuses on the problem of when transfer learning is beneficial using the model of the single-layer perceptron. \cite{gerace2022probing} studies a binary classification problem by transfer learning of the first layer in a two-layer neural network. However, both \cite{dhifallah2021phase} and \cite{gerace2022probing} include an explicit regularization term in their models, which prevents overfitting. There are also some recent studies of transfer learning on linear models \cite{bastani2021predicting,li2022transfer,tian2022transfer,li2023estimation,tripuraneni2020theory,zhang2022class, lin2022transfer}.  For example, \cite{bastani2021predicting} and \cite{li2022transfer} investigate estimation and prediction in high-dimensional linear models. \cite{tian2022transfer} and \cite{li2023estimation} further extend the setup to high-dimensional generalized linear models.
\cite{tripuraneni2020theory} considers the case where  source and  target tasks  share a common and low-dimensional linear representation. \cite{lin2022transfer} studies transfer learning in a functional linear regression where the similarity between source and target tasks is measured using the Reproducing Kernel Hilbert Spaces norm.
\cite{zhang2022class} provides minimax bounds on the generalization performance but does not overfit the training data. In particular, none of these studies have considered the task similarity structure of interest in this paper, nor investigated the generalization performance in both overparameterized and underparameterized regimes.



The most related work to ours is \cite{dar2022double}. Specifically, \cite{dar2022double} studies the double descent phenomenon in transfer learning, which is also our focus in this paper. However, \cite{dar2022double} does not consider an explicit separation of the feature space by the common part and the task-specific part like we do in this paper. As we will show, such a separation in the system model enables us to analyze the double descent phenomenon under different options for transfer learning, including two options for parameter transfer and two options for data transfer. In contrast, \cite{dar2022double} only studies one option of parameter transfer. Therefore, our analysis is quite different from \cite{dar2022double}.

% \TODO

% related to heterogeneous transfer learning \citep{day2017survey}. \TODO
\section{System Model}





\subsection{Ground truth}\label{subsec.ground_truth}
% Consider $m$ training (source) tasks and one test (target) task. For simplicity of notations, we index the test task as the $(m+1)$-th task.
Consider one training (source) task 
%(as the first task) 
and one test (target) task,
%(as the second task).
respectively
referred to as the first and second task from now on.
We consider a linear model for each task; i.e., for the $i$-th task with $i\in \{1\text{ (source)},2 \text{ (target)}\}$, samples are generated by
\begin{align}\label{eq.ground_truth_original}
    y_{(i)}=\sx^T\sw{i}+\sz{i}^T\sq{i}+\epsilon_{(i)} ,
\end{align}
where $\hat{\bm{x}}\in \mathds{R}^{s}$ denotes the value of the features that correspond to the similar/common parameters $\sw{i}\in \mathds{R}^{s}$, $\sz{i}\in \mathds{R}^{s_{(i)}}$ denotes the value of the features that correspond to the task-specific parts $\sq{i}\in \mathds{R}^{s_{(i)}}$, and $\epsilon_{(i)}\in \mathds{R}$ denotes the noise. Here, $s$ denotes the number of common features and $s_{(i)}$ denotes the number of $i$-th task-specific features.
% We index all possible features by $1,2,\cdots$. 
Let $\featureSpace{i}$ denote the set of features  corresponding to $\sz{i}$ and
%let 
$\commonSpace$
%denote
the set of features corresponding to $\sx$ (so their cardinality $\abs{\featureSpace{i}}=s_i$ and $\abs{\commonSpace}=s$). 

% As a starting point, in this paper, we focus only on the situation where $m=1$, i.e., there is only one source task and one target task.


{\bf Representative motivating example:} In real-world applications, many tasks actually have such a partial similarity structure. For example, for image recognition tasks, some low-level features are common (e.g., skin texture of animals, the surface of a machine) among different tasks even if the objectives of those tasks are completely different (e.g., classifying cat and airplane, or classifying dog and automobile). These low-level features are usually captured by convolutional layers in DNNs, while the remaining parts of the DNNs (e.g., full-connected layers) are used to extract task-specific features. Even for a simple linear regression model, a theoretical explanation of the effect of common features and task-specific features on the generalization performance of transfer learning may provide useful insights on designing more suitable real-world transfer learning model structures (e.g., how many neurons to use in convolutional layers of DNNs to extract common low-level features to transfer).


% Here $\sw{i}$ can be different for different $i$. However, for the transfer learning scenario, they must be similar to each other, i.e., $\|\sw{i}-\sw{j}\|$ shouldn't be too large.


\subsection{Feature selection for learning}
From the learner's point of view, the true feature sets $\commonSpace$ and $\featureSpace{i}$ are usually unknown for many real-world applications. Therefore, in order to make sure that all true features are included, a common strategy is choosing more features to ensure the coverage of the true features, which is expressed by the following assumption.


\begin{assumption}\label[assumption]{as.no_missing_features}
$\commonSpace\subseteq \commonSpaceSelected$ and $\featureSpace{i}\subseteq \featureSpaceSelected{i}$ for all $i\in \{1,2\}$, where $\commonSpaceSelected$ denotes the set of selected features for the common part, and $\featureSpaceSelected{i}$ denotes the set of selected task-specific features.
\end{assumption}

Define $\pCommon\defeq \abs{\commonSpaceSelected}$ and $\pSpecial{i}\defeq \abs{\featureSpaceSelected{i}}$.
Let $\tilde{\bm{w}}\in \mathbf{R}^{\pCommon}$ denote the parameters to learn the common part and $\tilde{\bm{q}}_{(i)}\in \mathds{R}^{\pSpecial{i}}$ the parameters to learn the $i$-th task's specific part.




With \cref{as.no_missing_features}, we construct $\ww{i}\in \mathds{R}^{\pCommon}$ (corresponding to $\commonSpaceSelected$) from $\sw{i}$ (corresponding to $\commonSpace$) by filling zeros in the positions of the redundant features (corresponding to $\commonSpaceSelected\setminus \commonSpace$). We similarly construct $\qq{i}\in \mathds{R}^{\pSpecial{i}}$ from $\sq{i}$. Thus, \cref{eq.ground_truth_original} can be alternatively expressed as
\begin{align}\label{eq.ground_truth_extended}
    y_{(i)}=\bm{x}^T\ww{i} + \bm{z}_{(i)}^T\qq{i} + \epsilon_{(i)},
\end{align}
where $\bm{x}\in \mathds{R}^{\pCommon}$ are the features of $\commonSpace$ and $\bm{z}_{(i)}\in \mathds{R}^{\pSpecial{i}}$ are the features of $\featureSpaceSelected{i}$. Notice that the ground truth (i.e., input and output) does not change with $\pCommon$ or $\pSpecial{i}$ (since it only changes how many additional zeros are added). 
% Therefore, we can analyze how different numbers of selected features/parameters affect the performance for a given ground truth.

For analytical tractability, we adopt Gaussian features and noise, which is formally stated by the following assumption.
\begin{assumption}\label[assumption]{as.Gaussian}
All features follow \emph{i.i.d.} standard Gaussian $\mathcal{N}(0, 1)$. The noise also follows Gaussian distribution. Specifically, $\epsilon_{(1)}\sim \mathcal{N}\left(0, \sigmaOne^2\right)$ and $\epsilon_{(2)}\sim \mathcal{N}\left(0,\sigmaTwo^2\right)$.
\end{assumption}

\begin{remark}\label[remark]{remark.missing_features}
If there exist some missing features in $\commonSpaceSelected$ and $\featureSpaceSelected{i}$ (i.e., \cref{as.no_missing_features} is not satisfied), then the effect of these missing features is the same as the noise since we adopt \emph{i.i.d.} Gaussian features. Thus, our methods and results still hold
% to the case where \cref{as.no_missing_features} is violated 
by redefining $\sigmaOne^2$ and $\sigmaTwo^2$ as the total power of the noise and the missing features, i.e., $\sigma_{(i)}^2\gets \sigma_{(i)}^2 + \normTwo{\sw{i}^{\text{missing}}}^2+ \normTwo{\sq{i}^{\text{missing}}}^2$ where $\sw{i}^{\text{missing}}$ and $\sq{i}^{\text{missing}}$ denote the sub-vectors for the missing features of $\sw{i}$ and $\sq{i}$, respectively.
\end{remark}

\subsection{Training samples and training losses}

Let $\nSample{i}$ denote the number of training samples for task $i\in \{1,2\}$.
We stack these $\nSample{i}$ samples as matrices/vectors $\XStacked{i}\in \mathds{R}^{\pCommon\times \nSample{i}}$, $\ZStacked{i}\in \mathds{R}^{\pSpecial{i}\times \nSample{i}}$, $\yStacked{i}\in \mathds{R}^{\nSample{i}}$, where the $j$-th column of $\XStacked{i}$, the $j$-th column of $\ZStacked{i}$, and the $j$-th element of $\yStacked{i}$ correspond to $(\bm{x},\bm{z}_{(i)},y_{(i)})$ in \cref{eq.ground_truth_extended} of the $j$-th training sample.
Now \cref{eq.ground_truth_extended} can be written into a matrix equation for training samples:
\begin{align}\label{eq.ground_truth_matrix}
    \bm{y}_{(i)}=\XStacked{i}^T\ww{i}+\ZStacked{i}^T\qq{i}+\bm{\epsilon}_{(i)},
\end{align}
where $\bm{\epsilon}_{(i)}\in \mathds{R}^{\nSample{i}}$ is the stacked vector that consists of the noise in the output of each training sample (i.e., $\epsilon_{(i)}$ in \cref{eq.ground_truth_extended}).


We use mean squared error (MSE) as the training loss for the $i$-th task with the learner's parameters $\wbar, \qbar$ as
\begin{align*}
    \trainErr{i}\defeq \frac{1}{\nSample{i}}\normTwo{\bm{y}_{(i)}-\XStacked{i}^T\wbar - \ZStacked{i}^T\qbar}^2.
\end{align*}

% We stack the features $\bm{x}$ and $\bm{z}_{(i)}$ for the input of each one of $\nSample{i}$ samples as matrices $\mathbf{X}_{(i)}\in \mathds{R}^{\pCommon\times \nSample{i}}$ and $\mathbf{Z}_{(i)}\in \mathds{R}^{p_2\times \nSample{i}}$, respectively. We stack the output  as a vector $\bm{y}_{(i)}\in \mathds{R}^{\nSample{i}}$.
% Notice that $\pCommon$ may not equal to $s$ and $p_2$ may not equal to $s_{(i)}$, because $\pCommon$ and $p_2$ are chosen without knowing $s$ or $s_{(i)}$.







% \begin{figure}[h!]
%     \centering
%     \includegraphics[width=5in]{example.png}
%     \caption{An example of partial task similarity. Both users receive similar interference from base station A, but have different signal sources. The tasks could be optimizing the settings of each signal-source base station (i.e., B and C) to optimize the SNR.}
%     \label{fig:my_label}
% \end{figure}


\subsection{Options of parameter transfer}\label{subsec.model_transfer_parameters}

% For simplicity, we consider a minimum transfer learning scenario where there is only one training task, i.e., $m=1$. 
The process of transfer learning by transferring parameters consists of three steps: \textbf{step~1}) train for the source task using samples $(\XStacked{1},\ZStacked{1};\yStacked{1})$; \textbf{step~2}) extract and transfer the learned parameters of the common features; and \textbf{step~3}) determine/train the parameters for the target task using its own samples $(\XStacked{2},\ZStacked{2};\yStacked{2})$ based on the transferred parameters in step~2.

Step~1 is similar to a classical single-task linear regression. 
The training process will converge to a solution $\wTilde,\qTildeOne$ that minimizes this training loss, i.e., $(\wTilde,\qTildeOne)\defeq \argmin_{\wbar,\qbar} \trainErr{1}$. When $\pCommon+\pSpecial{1}>\nOne$ (overparameterized), there exist multiple solutions that can make the training loss zero (with probability $1$). In this situation, we will choose the one with the smallest $\ell_2$-norm $(\wTilde,\qTildeOne)$ which is defined as the solution of the following optimization problem:
\begin{align*}
    \min_{\wbar,\qbar}\quad &\normTwo{\wbar}^2+\normTwo{\qbar}^2\quad \text{subject to }\quad \XX^T\wbar + \ZZ^T\qbar= \yOne.
\end{align*}
We are interested in this minimum $\ell_2$-norm solution among all overfitted solutions because it corresponds to the convergence point of stochastic gradient descent (SGD) or gradient descent (GD) training with zero initial point (see proof in \cref{le.min_l2_solution}). 

Steps~2 and 3 jointly determine the learned result for the target task $\wTildeTwo$ and $\qTilde$. In this paper, we analyze two possible options differentiated by the usage of the transferred common part $\wTilde$.

\textbf{Option A:} We directly copy the learned result, i.e., $\wTildeTwo \defeq \wTilde$. For the training of the target task, only the task-specific parameters are trained. In other words, $\qTilde\defeq \argmin_{\qbar}\trainErr{2}$ when underparameterized. When $\pSpecial{i}>\nTwo$ (overparameterized), there exist multiple solutions that can make the training loss zero. We then define $\qTilde$ as the minimum $\ell_2$-norm overfitted solution, i.e., $\qTilde$ is defined as the solution of the following optimization problem:
\begin{align*}
    \min_{\qbar}&\normTwo{\qbar}^2\quad \text{ subject to }\quad \XXX^T\wTilde + \ZZZ^T\qTilde = \yTwo.
\end{align*}

\textbf{Option B:} We only use the learned common part as an initial training point of $\wTildeTwo$. In this option, both $\wTildeTwo$ and $\qTilde$ are determined by the training of the source task. Specifically, $(\wTildeTwo,\qTilde)\defeq \argmin_{\wbar,\qbar}\trainErr{2}$ when underparameterized. When $\pCommon+\pSpecial{2}>\nTwo$, there are multiple solutions that can make $\mathcal{L}_{(2)}(\wbar,\qbar)=0$.
We then define $(\wTildeTwo,\qTilde)$ as the convergence point of SGD/GD starting from $(\wbar=\wTilde,\qbar=\bm{0})$. Indeed, $(\wTildeTwo,\qTilde)$ corresponds to the smallest $\ell_2$-norm of the difference between the result and the initial point (see proof in \cref{le.min_l2_solution}):
\begin{align*}
    \min_{\wbar,\qbar}\quad  \normTwo{\wbar-\wTilde}^2+\normTwo{\qbar}^2\quad    \text{subject to }\quad \XXX^T\wbar +\ZZZ^T\qbar = \yTwo.
\end{align*}






\subsection{Performance evaluation}

The primary goal is to evaluate the generalization performance of the solution for the target task. Specifically, we define the \emph{model error} as 
\begin{align}
    \modelErr\defeq\normTwo{\wTildeTwo-\wTwo}^2+\normTwo{\qTilde-\qTwo}^2.\label{eq.def_model_err}
\end{align}
It can be proven that the model error $\modelErr$ is the expected test loss on noiseless test samples. To make our results in the following sections concise, we define 
\begin{alignat}{2}
    &\EwOneDiff\defeq \E_{\XX,\ZZ,\eOne}\normTwo{\wTwo-\wTilde}^2\qquad &&\text{(transferring error)},\label{eq.def_Lco}\\
    &\EwOneDiffNoNoise\defeq \EwOneDiff\big|_{\sigmaOne=\bm{0}} &&\text{(transferring error when $\sigmaOne=\bm{0}$)},\\
    &\similarity\defeq \normTwo{\wTwo-\wOne}&&\text{(similarity on common features)},\label{eq.def_similarity}\\
    &\oRatio\defeq 1-\frac{\nOne}{\pCommon+\pSpecial{1}}&&\text{(overparameterized ratio in step 1)}.\nonumber
\end{alignat}
Intuitively, $\EwOneDiff$ describes how well the common part learned from the source task estimates the target task's common part, $\similarity$ reflects the similarity between the common parts of the source task and the target task, and $\oRatio$ can be regarded as  the overparameterization ratio in step~1 introduced in \cref{subsec.model_transfer_parameters}.



% \begin{enumerate}
%     \item Train $m$ source tasks to minimize the training loss $\mathcal{L}_1(\tilde{\bm{w}}, \tilde{\bm{q}})$. 
%     \item Copy the common parameters to the target task.
%     \item There are two options: OPT 1) Train the special part of the target task to minimize the training loss $\mathcal{L}_2(\tilde{\bm{w}}, \tilde{\bm{q}}_{(m+1)})$. In other words, 
%     \begin{align*}
%         \min_{\tilde{\bm{q}}_{(m+1)}}\mathcal{L}_2(\tilde{\bm{w}}, \tilde{\bm{q}}_{(m+1)}).
%     \end{align*}
%     OPT 2) Train the common part and the special part together, but use the training tasks' result as an initial point.
% \end{enumerate}

% For a training result $\tilde{\bm{w}}, \tilde{\bm{q}}=(\tilde{\bm{q}}_{(1)}, \tilde{\bm{q}}_{(2)},\cdots,\tilde{\bm{q}}_{(m)}),\tilde{\bm{q}}_{(m+1)}$, those losses are defined as
% \begin{align*}
%     &\mathcal{L}_1(\tilde{\bm{w}}, \tilde{\bm{q}})\defeq \sum_{i=1}^m \normTwo{\bm{y}_{(i)}-\mathbf{X}_{(i)}^T \tilde{\bm{w}}-\mathbf{Z}_{(i)}^T\tilde{\bm{q}}_{(i)}}^2,\\
%     &\mathcal{L}_2(\tilde{\bm{w}}, \tilde{\bm{q}}_{(m+1)})\defeq \normTwo{\bm{y}_{(m+1)}-\mathbf{X}_{(m+1)}^T \tilde{w}-\mathbf{Z}_{(i)}^T\tilde{\bm{q}}_{(m+1)}}^2.
% \end{align*}

% \subsection{Transfer samples}

% Train all data (from both source tasks and the target task) together:
% \begin{align*}
%     \min_{\tilde{\bm{w}}, \tilde{\bm{q}}, \tilde{\bm{q}}_{(m+1)}}\sum_{i=1}^{m+1} \normTwo{\bm{y}_{(i)}-\mathbf{X}_{(i)}^T \tilde{\bm{w}}-\mathbf{Z}_{(i)}^T\tilde{\bm{q}}_{(i)}}^2
% \end{align*}

\section{Main Results for Parameter Transfer}
For the scheme of transferring parameters (\cref{subsec.model_transfer_parameters}), we will establish three theorems corresponding to the performance of the transferring error\footnote{The error caused by the transferred parameters. The precise definition is given in \cref{eq.def_Lco}.}, the model error of Option~A, and the model error of Option~B, respectively.
% In order to estimate $\modelErr$, we need to first quantify the performance of step 1. 
% Notice that $\EwOneDiff$ depends on the similarity between the common parts of the source task and the target task, which is reflected by 
% We define $\similarity$ as the distance between $\wOne$ and $\wTwo$, 


\begin{theorem}[transferring error]\label{thm.w_err}
The transferring error is given by 
\begin{empheq}[left={\EwOneDiff=\empheqlbrace}]{alignat=2}
& \EwOneDiffNoNoise+\bNoise,\quad && \text{ for } \pCommon+\pSpecial{1}>\nOne+1,\label{eq.result_1}\\
& \similarity^2+\underbrace{\frac{\pCommon\sigmaOne^2}{\nOne-\left(\pCommon+\pSpecial{1}\right)-1}}_{\text{Term O1}},\quad  && \text{ for } \nOne>\pCommon+\pSpecial{1}+1,\label{eq.result_2}
\end{empheq}
where $0\leq \EwOneDiffNoNoise\leq \min\limits_{i=1,2,3} \overline{b}_i^2$, and
\begin{align}
    &\textstyle\overline{b}_1\defeq \similarity+\sqrt{\oRatio\left(\normTwo{\wOne}^2+\normTwo{\qOne}^2\right)},\label{eq.def_b1}\\
    &\textstyle\overline{b}_2\defeq \normTwo{\wTwo}+\sqrt{1-\oRatio}\normTwo{\wOne}+\sqrt{\min\{\oRatio,1-\oRatio\}}\normTwo{\qOne},\label{eq.def_b2}\\
    &\textstyle\overline{b}_3\defeq \sqrt{r}\normTwo{\wOne}+\similarity+\sqrt{\min\{\oRatio,1-\oRatio\}}\normTwo{\qOne},\label{eq.def_b3}\\
    &\bNoise\defeq \frac{\pCommon}{\pCommon+\pSpecial{1}}\cdot \frac{\nOne \sigmaOne^2}{\pCommon+\pSpecial{1}-\nOne- 1}.\label{eq.def_bNoise}
\end{align}
% When $\nOne>\pCommon+\pSpecial{1}+1$, we have
% \begin{align}
%     \EwOneDiff = \similarity^2+\frac{\pCommon\sigmaOne^2}{\nOne-\left(\pCommon+\pSpecial{1}\right)-1}.
% \end{align}
\end{theorem}
% \begin{proof}
% See Appendix~\ref{app.proof_w_err}.
% \end{proof}


% We have the following theorems for Option~A and Option~B:

\begin{theorem}[Option A]\label{thm.target_task_err} For Option~A, we must have
\begin{empheq}[left={\E[\modelErr]=\empheqlbrace}]{alignat=2}
\textstyle
    &\underbrace{\EwOneDiff+\frac{\nTwo \left(\EwOneDiff+\sigmaTwo^2\right)}{\pSpecial{2}-\nTwo-1}}_{\text{Term A1}}+\underbrace{\left(1-\frac{\nTwo}{\pSpecial{2}}\right)\normTwo{\qTwo}^2}_{\text{Term A2}}, \quad && \text{ for } \pSpecial{2}>\nTwo+1,\label{eq.result_3}\\
    &\EwOneDiff+\frac{\pSpecial{2}\left(\EwOneDiff+\sigmaTwo^2\right)}{\nTwo-\pSpecial{2}-1}, && \text{ for } \nTwo>\pSpecial{2}+1.\label{eq.result_4}
\end{empheq}
% When $\pSpecial{2}>\nTwo+1$, we must have
% \begin{align}
%     \E[\modelErr]=\underbrace{\EwOneDiff+\frac{\nTwo \left(\EwOneDiff+\sigmaTwo^2\right)}{\pSpecial{2}-\nTwo-1}}_{\text{Term A1}}+\underbrace{\left(1-\frac{\nTwo}{\pSpecial{2}}\right)\normTwo{\qTwo}^2}_{\text{Term A2}}.\label{eq.result_3}
% \end{align}
% When $\nTwo>\pSpecial{2}+1$, we must have
% \begin{align}
%   \E[\modelErr]=  \EwOneDiff+\frac{\pSpecial{2}\left(\EwOneDiff+\sigmaTwo^2\right)}{\nTwo-\pSpecial{2}-1}\label{eq.result_4}
% \end{align}
\end{theorem}
% \begin{proof}
% See Appendix~\ref{app.proof_target_task_err}.
% \end{proof}

\begin{theorem}[Option B]\label{thm.opt2}
For Option~B, we must have 
%$\E[\modelErr]$
\begin{empheq}[
%left={=\empheqlbrace}]{alignat=2}
left={\E[\modelErr]=\empheqlbrace}]{alignat=2}
\textstyle
    &\underbrace{\textstyle\left(1-\frac{\nTwo}{\pCommon+\pSpecial{2}}\right)\left(\EwOneDiff+\normTwo{\qTwo}^2\right)}_{\text{Term B1}}+\underbrace{\textstyle\frac{\nTwo\sigmaTwo^2}{\pCommon+\pSpecial{2}-\nTwo -1}}_{\text{Term B2}},\quad && \text{ for } \pCommon+\pSpecial{2}>\nTwo + 1,\label{eq.result_5}\\
    &\frac{(\pCommon+\pSpecial{2})\sigmaTwo^2}{\nTwo-(\pCommon+\pSpecial{2})-1}, && \text{ for } \nTwo>\pCommon+\pSpecial{2}+1.\label{eq.result_6}
\end{empheq}
% When $\pCommon+\pSpecial{2}>\nTwo + 1$, we have
% \begin{align}
%     \E[\modelErr] =& \underbrace{\left(1-\frac{\nTwo}{\pCommon+\pSpecial{2}}\right)\left(\EwOneDiff+\normTwo{\qTwo}^2\right)}_{\text{Term B1}}+\underbrace{\frac{\nTwo\sigmaTwo^2}{\pCommon+\pSpecial{2}-\nTwo -1}}_{\text{Term B2}}.\label{eq.result_5}
% \end{align}
% When $\nTwo>\pCommon+\pSpecial{2}+1$, we have
% \begin{align}
%     \E[\modelErr] =\frac{(\pCommon+\pSpecial{2})\sigmaTwo^2}{\nTwo-(\pCommon+\pSpecial{2})-1}.\label{eq.result_6}
% \end{align}
\end{theorem}
The proofs of \cref{thm.w_err,thm.target_task_err,thm.opt2} are given in \cref{app.proof_w_err,app.proof_target_task_err,app.proof_opt2}, respectively. \cref{thm.w_err,thm.target_task_err,thm.opt2} provide some interesting insights, which we now discuss in \cref{subsec.similar,subsec.A,subsec.B}.

\begin{figure}[t!]
    \centering
    \includegraphics[width=0.7\textwidth]{figs/change_p.eps}
    \caption{Generalization performance of transfer learning under different setups, where $s=s_1=s_2=5$, $\nOne=100$, $\nTwo=50$, $\wOne=\wTwo$. Each point is the average of $100$ random runs. Other settings for each subfigure: (a) $\normTwo{\qTwo}=\normTwo{\wOne}=1$; (b) $\normTwo{\wOne}=\normTwo{\qOne}=1$; (c) $\normTwo{\sigmaOne}=1$, $\normTwo{\sigmaTwo}=0.2$, $\normTwo{\qOne}=\normTwo{\qTwo}=\normTwo{\wOne}=0.1$.}
    \label{fig.change_p}
\end{figure}


\subsection{Common insights for Options~A and B}\label{subsec.similar}

\noindent\textbf{(1) Benign overfitting w.r.t. $\pSpecial{1}$ needs large $\sigmaOne$.} For the overparameterized regime result in \cref{eq.result_1} of \cref{thm.w_err}, when $\sigmaOne$ is large, the term $\bNoise$ (defined in \cref{eq.def_bNoise}) dominates $\EwOneDiff$ and is monotone decreasing w.r.t. $\pSpecial{1}$. When $\pSpecial{1}\to \infty$, we have $\bNoise\to 0$. In contrast, for the underparameterized regime result in \cref{eq.result_2}, Term~O1 (noise effect) is always larger than $\frac{\pCommon \sigmaOne^2}{\nOne}$, which can be worse than that of the overparameterized regime when $\pSpecial{1}$ is sufficiently large. By \cref{thm.target_task_err,thm.opt2}, we know that $\modelErr$ decreases when $\EwOneDiff$ decreases. Therefore, in the situation of large $\sigmaOne$, increasing $\pSpecial{1}$ in the overparameterized regime (of step~1) w.r.t. $\pSpecial{1}$ can reduce the generalization error, which implies the existence of benign overfitting.

We also numerically verify the impact of $\sigmaOne$ on the benign overfitting in \cref{fig.change_p}(a), where we plot the empirical average of $\modelErr$ w.r.t. $\pSpecial{1}$. The two curves of $\sigmaOne=3$ with markers ``$\times$'' descend in the overparameterized regime ($\pSpecial{1}>80$) and can be lower than their values in the underparameterized regime. In contrast,  the two curves of $\sigmaOne=0.1$ with markers ``$+$'' increase in most parts of the overparameterized regime and are higher than the underparameterized regime. Such a contrast indicates the benign overfitting w.r.t. $\pSpecial{1}$ needs large $\sigmaOne$.

\noindent\textbf{(2) Benign overfitting w.r.t. $\pSpecial{2}$ needs large $\sigmaTwo$.} For \cref{eq.result_4} (underparameterized regime of Option~A), $\E[\modelErr]$ is always larger than $\EwOneDiff(1+\frac{1}{\nTwo})$. In contrast, for \cref{eq.result_3} (overparameterized regime of Option~A), when $\sigmaTwo$ is much larger than $\normTwo{\qTwo}^2$, then Term~A2 is negligible and Term~A1 dominates. In this situation, $\E[\modelErr]$ is monotone decreasing w.r.t. $\pSpecial{2}$ and will approaches $\EwOneDiff$ when $\pSpecial{2}\to \infty$. In other words, benign overfitting exists. Similarly, by \cref{thm.opt2}, benign overfitting exists when $\sigmaTwo^2$ is much larger than $\EwOneDiff+\normTwo{\qTwo}^2$.

In \cref{fig.change_p}(b), the two curves with markers ``$\triangledown$'' denote the model error of Option~A and Option~B when $\sigmaTwo$ is large ($\sigmaTwo=2$). They have a descending trend in the entire overparameterized regime. In contrast, the two curves with markers ``$+$'', which denote the model error for the situation of small $\sigmaTwo$ ($\sigmaTwo=0.2$), only decrease w.r.t. $\pSpecial{2}$ at the beginning of the overparameterized regime, while increasing thereafter.

\noindent\textbf{(3) A descent floor w.r.t. $\pSpecial{2}$ sometimes exists.} For \cref{eq.result_3} of Option~A, Term~A1 is monotone decreasing w.r.t. $\pSpecial{2}$, while Term~A2 is monotone increasing w.r.t. $\pSpecial{2}$. When $\pSpecial{2}$ is a little larger than $\nTwo$, the denominator $\pSpecial{2}-\nTwo-1$ in Term~A1 is close to zero, and thus Term~A1 dominates and causes $\E[\modelErr]$ to be decreasing w.r.t. $\pSpecial{2}$. When $\pSpecial{2}$ gradually increases to infinity, $\E[\modelErr]$ will approach $\EwOneDiff+\normTwo{\qTwo}^2$. By calculating $\partial \E[\modelErr] / \partial \pSpecial{2}$, we can tell that if $\EwOneDiff+\sigmaTwo^2<\normTwo{\qTwo}^2$, in the overparameterized regime, $\E[\modelErr]$ will first decrease and then increase, which implies a descent floor (by \cref{le.derivative_3} in \cref{subsec.derivative}). Similarly, by calculating $\partial \E[\modelErr] / \partial \pSpecial{2}$ for \cref{eq.result_5} of Option~B, if $\sigmaTwo^2<\EwOneDiff+\normTwo{\qTwo}^2$, in the overparameterized regime, $\E[\modelErr]$ will have a descent floor w.r.t. $\pSpecial{2}$ (by \cref{le.derivative_5} in \cref{subsec.derivative}).
An interesting observation related to the descent floor is that \emph{the condition of the existence of the descent floor is different for Option~A and Option~B, where Option~A needs small $\EwOneDiff$ but Option~B needs large $\EwOneDiff$}.

In \cref{fig.change_p}(b), we see that both  curves with markers ``$+$'' have a descent floor in the overparameterized regime.  In contrast, for the two curves with markers ``$\times$'' where $\sigmaOne$ is large, only Option~B has a descent floor while Option~A does not. Since large $\sigmaOne$ implies large $\EwOneDiff$, such a difference confirms that the descent floor of Option~A needs small $\EwOneDiff$ while the one of Option~B needs large $\EwOneDiff$.


\noindent\textbf{(4) The effect of $\qOne$ is negligible when heavily or slightly  overparameterized in step~1.} The effect of $\qOne$ on $\modelErr$ is through $\EwOneDiffNoNoise$. By \cref{eq.def_b1,eq.def_b2,eq.def_b3,eq.result_1}, the coefficient of $\normTwo{\qTwo}$ is $\min\{\oRatio,1-\oRatio\}$. When heavily overparameterized in step~1, we have $\pCommon+\pSpecial{1}\gg \nOne$ and thus $\oRatio \approx 0$. When slightly overparameterized in step~1, we have $\pCommon+\pSpecial{1}\approx \nOne$ and thus $\oRatio \approx 1$. In both situations, we have the coefficient $\min\{\oRatio,1-\oRatio\}\approx 0$, which implies that the effect of $\qOne$ is negligible when heavily or slightly  overparameterized in step~1.

In \cref{fig.change_p}(a), we compare two curves with markers ``$\triangle$'' (for large $\qOne$ that $\normTwo{\qOne}=5$) against two curves with markers ``$+$'' (for small $\qOne$ that $\normTwo{\qOne}=1$). We observe for both Option~A and Option~B that the curves with markers ``$\triangle$'' overlap the curves with markers ``$+$'' at the beginning and the latter part of the overparameterized regime. This phenomenon validates the implication (4) which is inferred from the factor $\min\{\oRatio, 1-\oRatio\}$ in \cref{eq.def_b2,eq.def_b3}.

\subsection{Insights for Option A}\label{subsec.A}


\noindent\textbf{(A1) Benign overfitting w.r.t. $\pSpecial{2}$ is easier to observe with small knowledge transfer.} In the underparameterized regime, by \cref{eq.result_4}, $\E[\modelErr]$ is at least $\EwOneDiff+\frac{\EwOneDiff+\sigmaTwo^2}{\nTwo}$. In contrast, for the overparameterized regime, when $\EwOneDiff$ is large, Term~A1 of \cref{eq.result_3} dominates $\E[\modelErr]$. When $\pSpecial{2}$ increases to $\infty$, Term~A1 will decrease to $\EwOneDiff$. Notice that large $\EwOneDiff$ implies small knowledge transfer from the source task to the target task. Thus, \emph{benign overfitting w.r.t. $\pSpecial{2}$ appears when knowledge transfer is small}.

In \cref{fig.change_p}(b), we let the ground-truth parameters be very small compared with the noise level, so the error $\modelErr$ in \cref{fig.change_p} is mainly from noise. The blue curve with markers ``$\times$'' has larger $\sigmaOne$ (with $\sigmaOne=3$) compared with the blue curve with markers ``$\triangledown$'' (with $\sigmaOne=0.1$), and consequently, larger $\EwOneDiff$ and smaller knowledge transfer. We observe from \cref{fig.change_p}(b) that the blue curve with markers ``$\times$'' descends w.r.t. $\pSpecial{2}$ in the entire overparameterized regime, while the blue curve with markers ``$\triangledown$'' descends at the beginning of the overparameterized regime and ascends in the remainder of the overparameterized regime. Such a phenomenon validates the insight (A1).


\noindent\textbf{(A2) Larger $\pCommon$ is not always good to reduce the noise effect when overparameterized.} By \cref{thm.target_task_err,thm.w_err}, we know that the direct effect of $\pCommon$ on noise in the overparameterized regime is only through the term $\bNoise$ in $\EwOneDiff$. By checking the sign of $\frac{\partial \bNoise}{\partial \pCommon}$, we can prove that $\bNoise$ increases w.r.t. $\pCommon$ when $\pCommon^2< \pSpecial{1}(\pSpecial{1}-\nOne-1)$, and decreases when $\pCommon^2 > \pSpecial{1}(\pSpecial{1}-\nOne-1)$ (see calculation details in \cref{le.non_monotone_p} in \cref{subsec.derivative}).

In \cref{fig.change_p}(c), the blue curve with markers ``$\triangleright$'' depicts how the model error $\modelErr$ of Option~A changes w.r.t. $\pCommon$ in the overparameterized regime ($\pCommon+\pSpecial{1}>\nOne$). This curve first increases and then decreases, which validates the insight (A2).

\subsection{Insights for Option B}\label{subsec.B}


\noindent\textbf{(B1) Benign overfitting w.r.t. $\pSpecial{2}$ is easier to observe with large knowledge transfer and small target task-specific parameters.} In \cref{eq.result_5}, small $\EwOneDiff+\normTwo{\qTwo}^2$ implies that Term~B2 dominates the value of $\E[\modelErr]$. As we explained previously in (2) of \cref{subsec.similar}, benign overfitting exists in this situation. Meanwhile, small $\EwOneDiff$ and $\normTwo{\qTwo}$ imply large knowledge transfer and small target task-specific parameters, respectively.

In \cref{fig.change_p}(b), the orange curve with markers ``$\triangleleft$'' denotes the model error $\modelErr$ of Option~B w.r.t. $\pSpecial{2}$ when $\sigmaOne$ and $\qTwo$ are small, i.e., large knowledge transfer and small target task-specific parameters. Compared with the orange curve with markers ``$\times$'', this curve descends in the entire overparameterized regime and can achieve a lower value than that of the underparameterized regime. This phenomenon validates the insight (B1).

\noindent\textbf{(B2) Multiple descents of noise effect when increasing $\pCommon$ in the overparameterized regime.} Different from Option~A where $\pCommon$ only affects the consequence of the noise in the source task (since no $\pCommon$ appears in \cref{eq.result_3} except $\EwOneDiff$), for \cref{eq.result_5} of Option~B, we see that $\pCommon$ not only affects $\EwOneDiff$ but also Term~B2, which implies that $\pCommon$ relates to the noise effect in both the source task and the target task. Specifically, the trend of $\E[\modelErr]$ w.r.t. $\pCommon$ is determined by $(1-\frac{\nTwo}{\pCommon+\pSpecial{2}})\bNoise$ and Term~B2 in \cref{eq.result_5}. In (A2) of \cref{subsec.A}, we show that $\bNoise$ sometimes first increases and then decreases. The factor $1-\frac{\nTwo}{\pCommon+\pSpecial{2}}$ is monotone increasing w.r.t. $\pCommon$. Term~B2 in \cref{eq.result_5} is monotone decreasing w.r.t. $\pCommon$. Thus, the overall noise effect may have multiple descents w.r.t. $\pCommon$.

In \cref{fig.change_p}(c), the orange curve with markers ``$\triangleright$'' provides an example of how the model error $\modelErr$ of Option~B behaves in the overparameterized regime. We see that this curve has multiple descents, which validates the insight (B2).


% \section{Comparison}
\section{Further Discussion}
% In order to make a comprehensive comparison among different transfer learning methods, besides the aforementioned two options for transferring parameters, we introduce another transfer learning method that transfers samples (instead of parameters) in \cref{subsec.transfer_samples}. We then compare all three methods in \cref{subsec.compare}.




\subsection{Which option performs better in the overparameterized regime?}\label{subsec.compare}

% Now we are ready to compare the generalization performance of three methods of transfer learning: Option~A, Option~B, and transferring samples. We focus on the overparameterized situation.

\textbf{(C1)} First, by comparing the coefficients of $\EwOneDiff$ in \cref{eq.result_3} and \cref{eq.result_5}, we know that the effect of the error in step one deteriorates in the model error $\modelErr$ of Option~A (since the coefficient of $\EwOneDiff$ in \cref{eq.result_3} is larger than $1$), whereas this is mitigated in the model error of Option~B (since the coefficient of $\EwOneDiff$ in \cref{eq.result_5} is smaller than $1$). \textbf{(C2)} Second, by comparing the coefficients of $\normTwo{\qTwo}^2$ and $\sigmaTwo^2$ in \cref{eq.result_3,eq.result_5} under the same $\pCommon$ and $\pSpecial{2}$, we know that Option~B is worse to learn $\qTwo$ but is better to reduce the noise effect of $\sigmaTwo$ than Option~A (since $1-\frac{\nTwo}{\pSpecial{2}}<1-\frac{\nTwo}{\pCommon+\pSpecial{2}}$ and $\frac{\nTwo}{\pSpecial{2}-\nTwo-1}>\frac{\nTwo}{\pCommon+\pSpecial{2}-\nTwo-1}$). \textbf{(C3)} Third, by letting $\pSpecial{2}\to \infty$ in \cref{eq.result_3,eq.result_5}, the model error $\modelErr$ of both Option~A and Option~B approaches the same value $\EwOneDiff+\normTwo{\qTwo}^2$.

{\bf Intuitive Comparison of Options A and B:} An intuitive explanation of the reason for these differences is that Option~B does train the common part learned by the source task but Option~A does not. Thus, Option~B should do better to learn the common part. At the same time, since Option~B uses more  parameters ($\pCommon+\pSpecial{2}$) than Option~A ($\pCommon$) to learn the target task's samples, the noise effect  is spread among more parameters in Option~B than in Option~A, and thus Option~B can mitigate the noise better than Option~A. However, those additional $\pCommon$ parameters interfere with the learning of $\qTwo$ since those $\pCommon$ parameters correspond to the features of the common part $\commonSpace$, not the target task-specific features $\featureSpace{2}$, which implies that Option~B is worse in learning $\qTwo$ than Option~A.

In \cref{fig.change_p}(b), when overparameterized (i.e., $\pSpecial{2}>50$ for Option~A, and $\pSpecial{2}>30$ for Option~B), Option~A is slightly better than Option~B around $\pSpecial{2}=70$ under the situation ``$\sigmaOne=0.1$, $\sigmaTwo=0.2$, $\normTwo{\qTwo}=1$'' (i.e., the two curves with markers ``$+$''). Notice that this situation has the smallest $\sigmaOne,\sigmaTwo$ and the largest $\normTwo{\qTwo}$. Thus, insights (C1),(C2) are verified. Besides, in \cref{fig.change_p}(b), in every situation, the curves of Option~A and Option~B overlap when $\pSpecial{2}$ is very large, which validates insight (C3).



\subsection{The common part or the task-specific part?}

% In the classical single-task linear regression result,  overparameterization is better when the single-to-noise ratio (SNR) is low (i.e., the noise level is very large).
% \footnote{This is because in \cref{eq.single_task}, the noise term in the overparameterized regime $\frac{\nSum \sigmaTwo^2}{\pCommon-\nSum-1}\to 0$ when $\pCommon\to \infty$, while the noise term in the underparameterized regime $\frac{\pCommon\sigmaTwo^2}{\nSum-\pCommon-1}$ is at least $\frac{\sigmaTwo^2}{\nSum}$.}. 
% However, in our case, 
Since there are multiple kinds of features and multiple signals and noise, it is valuable to discuss how to choose the number of parameters for each part with the fixed total number of parameters. We have the following results.

% a question is choose the number of parameters for each part (e.g., when and where to overparameterize) is not that simple. Under our system model, we have the following conclusion:

% In previous sections, we have shown that different options for transferring parameters have different preferences on overparameterization. 

% In this subsection, we will derive a result on this issue.

% In transfer learning, $\nTwo$ is usually very small (i.e., we only have a limited number of training samples for the target task).

\noindent\textbf{When the total number of parameters is fixed, it is better to use more parameters on the task-specific parts.} Specifically, we have the following proposition:
\begin{proposition}\label[proposition]{prop.best_p}
When $\pCommon + \pSpecial{1}=C$ is fixed, $\EwOneDiff$ is monotone increasing with respect to $\pCommon$. Therefore, in order to minimize $\EwOneDiff$ when \cref{as.no_missing_features} is assured, the best choice is $\pCommon =  s$, $\pSpecial{1}=C-s$.
\end{proposition}



% \cref{prop.best_p} is also consistent with the upper bound shown in \cref{thm.w_err}. Specifically, in the overparameterized regime, all terms except $\bNoise$ in the expression of $\EwOneDiff$ do not change with $\pCommon$ when $\pCommon+\pSpecial{1}$ is fixed. By \cref{eq.def_bNoise}, we can see that $\bNoise$ decreases when $\pCommon$ decreases, which is consistent with \cref{prop.best_p}. \cref{prop.best_p} is proven in \cref{app.proof_best_p}.

\textbf{Sometimes it is even better to sacrifice certain \emph{true} features in the common part in favor of employing more \emph{redundant} features in the task-specific part.}
We still consider the case of fixed $\pCommon + \pSpecial{1}=C$.
In certain situations (especially when the noise level is large and some true parameters are very small), it is better to make $\pCommon$ even smaller than $s$, i.e., it is better to violate \cref{as.no_missing_features} deliberately (in contrast to \cref{remark.missing_features} where \cref{as.no_missing_features} is violated unconsciously). 
We now construct an example of this situation. Let $\normTwo{\qOne}^2=0$, $\normTwo{\wTwo}+\normTwo{\wOne}=1$ (so $ \overline{b}_2^2\leq 1$ by \cref{eq.def_b2}). Suppose there are only 2 true common features (i.e., $s=2$) and $C>\nOne+1$. If we do not violate \cref{as.no_missing_features}, then by \cref{prop.best_p}, the best choice is to let $\pCommon=2$. By \cref{thm.w_err} we know that $\EwOneDiff$ is at least $\mathcal{Q}_1\defeq \frac{2}{C}\cdot \frac{\nOne \sigmaOne^2}{C - \nOne - 1}$ (since $\EwOneDiffNoNoise\geq 0$). In contrast, if we violate \cref{as.no_missing_features} deliberately by sacrificing one true common feature with parameter value 0.1 for the source task and value 0 for the target task, then the only effect is enlarging the source task's noise level by $\sigmaOne^2\gets \sigmaOne^2 + 0.1^2$. Thus, by \cref{thm.w_err}, we know that $\EwOneDiff$ is at most $\mathcal{Q}_2\defeq 1 + \frac{1}{C}\cdot \frac{\nOne (\sigmaOne^2+0.1^2)}{C - \nOne - 1}$ (since $\overline{b}_2^2 \leq 1$). We can easily find a large enough $\sigmaOne^2$ to make $\mathcal{Q}_1>\mathcal{Q}_2$, which leads to our conclusion.







% \noindent\textbf{(2) Option~A and Option~B have different preferences on the overparameterization.}
% \TODO

\section{Conclusion}

Our study on transfer learning in linear regression models provides valuable insights into the generalization performance of the target task. We propose a comprehensive framework that considers task similarity in terms of both parameter distance and feature sets. Our analysis characterizes the double descent of transfer learning for two different options of parameter transfer. Further investigation reveals that allocating more redundant features to the task-specific part, rather than the common part, can enhance performance when the total number of features is fixed. Moreover, sometimes sacrificing true features in the common part in favor of employing more redundant features in the task-specific part can yield notable benefits, especially in scenarios with high noise levels and small numbers of true parameters. These findings contribute to a better understanding of transfer learning and offer practical guidance for designing effective transfer learning approaches.

There are some interesting directions for future work. 
% First, since we have studied the situation of only one source task, it will be meaningful to extend our analysis to the situation of multiple source tasks. 
First, we can use our current framework of partial similarity to analyze the performance of sample transfer. Second, going beyond the linear models of Gaussian features, we can use models that are closer to actual DNNs (such as neural tangent kernel models) to study the generalization performance of overfitted transfer learning.



\begin{center}
\textbf{\large Supplemental Material}
\end{center}

\input{useful_lemmas.tex}
\input{proof_w_err.tex}
\input{proof_target_task_err.tex}
\input{proof_opt2.tex}
\input{proof_best_p.tex}

% \input{transfer_samples.tex}
% \input{proof_transfer_samples.tex}
\section{Some useful lemmas}\label{app.useful_lemmas}
We first introduce some useful lemmas that will be used in the proofs of our main results.
\begin{lemma}\label[lemma]{le.min_l2_solution}
Consider $n$ training samples and $p$ features. The stacked training input is $\mathbf{X}\in \mathds{R}^{p\times n}$ and the corresponding output is $\bm{y}\in \mathds{R}^n$. If GD/SGD on linear regression with mean-square-error converges to zero (i.e., overfitted), then the convergence point is the solution that has the minimum $\ell_2$-norm of the change of parameters from the initial point.
\end{lemma}
\begin{proof}
Let $\bm{a}$ denote the parameters to train and $\bm{a}_0$ the initial point. Since GD/SGD is used in a linear model, then in each iteration, the change of the parameters must be in the column space of $\mathbf{X}$. To realize this, we can simply calculate the gradient for the $i$-th training sample as
\begin{align*}
    \frac{\partial (y_i-\bm{x}_i^T\bm{a})^2}{\partial \bm{a}}=-2(y_i-\bm{x}_i^T\bm{a})\bm{x}_i,
\end{align*}
where $\bm{x}_i$ is the $i$-th column of $\mathbf{X}$ and $y_i$ is the $i$-th element of $\bm{y}$. As we can see, the gradient is always parallel to one column of $\mathbf{X}$, which implies that the overall change of the parameters is still in the column space of $\mathbf{X}$. Thus, we can always find $\bm{b}\in \mathds{R}^p$ such that
\begin{align}\label{eq.temp_011505}
    \bm{a}-\bm{a}_0=\mathbf{X}\bm{b}.
\end{align}
For the convergence point that makes the training loss become zero, we have
\begin{align}\label{eq.temp_011506}
    \mathbf{X}^T\bm{a} = \bm{y}.
\end{align}
Substituting Eq.~\eqref{eq.temp_011505} into Eq.~\eqref{eq.temp_011506}, we have
\begin{align*}
    &\mathbf{X}^T\left(\bm{a}_0+\mathbf{X}\bm{b}\right) = \bm{y}\\
    \implies & \bm{b}=(\mathbf{X}^T\mathbf{X})^{-1}\left(\bm{y}-\mathbf{X}^T\bm{a}_0\right)\\
    \implies & \bm{a}=\bm{a}_0+\mathbf{X}(\mathbf{X}^T\mathbf{X})^{-1}\left(\bm{y}-\mathbf{X}^T\bm{a}_0\right),
\end{align*}
which is exactly the minimum $\ell_2$-norm solution of the following linear problem:
\begin{align*}
    \min_{\bm{a}}\quad & \normTwo{\bm{a}-\bm{a}_0}\\
    \text{subject to} \quad &\mathbf{X}^T(\bm{a}-\bm{a}_0)=\bm{y}-\mathbf{X}^T\bm{a}_0.
\end{align*}
The constraint is also equivalent to $\XX^T\bm{a}=\bm{y}$. The result of this lemma thus follows.
\end{proof}


\begin{lemma}[Cauchy-Schwarz inequality on random vectors]\label[lemma]{le.Cauchy}
Consider any two random vectors $\bm{a}, \bm{b}\in \mathds{R}^d$. We must have
\begin{align}
    \abs{\E [\bm{a}^T\bm{b}]}\leq \sqrt{\E[\normTwo{\bm{a}}^2]\cdot \E[\normTwo{\bm{b}}^2]}.\label{eq.temp_bias_1}
\end{align}
Consequently, we have
\begin{align}
    \E[\normTwo{\bm{a}+\bm{b}}^2]\leq \left(\sqrt{\E[\normTwo{\bm{a}}^2]}+\sqrt{\E[\normTwo{\bm{b}}^2]}\right)^2,\label{eq.temp_bias_2}
\end{align}
and
\begin{align}
    \E[\normTwo{\bm{a}+\bm{b}}^2]\geq \left(\sqrt{\E[\normTwo{\bm{a}}^2]}-\sqrt{\E[\normTwo{\bm{b}}^2]}\right)^2.\label{eq.temp_bias_3}
\end{align}
\end{lemma}
\begin{proof}
\cref{eq.temp_bias_1} directly follows by Cauchy-Schwarz inequality after defining the inner product on the set of random vectors using the expectation of their inner product:
\begin{align*}
    \langle \bm{a},\ \bm{b} \rangle_{\mu} \defeq \int \bm{a}^T\bm{b}\ d\mu=\E [\bm{a}^T\bm{b}],
\end{align*}
where $\mu(\cdot)$ denotes the probability measure. We can easily verify the following properties of $\langle \cdot,\cdot\rangle_{\mu}$:
\begin{align*}
    &\langle \bm{a},\ \bm{b}\rangle_{\mu}=\langle \bm{b},\ \bm{a}\rangle_{\mu},\text{  (conjugate symmetry);}\\
    &\langle\lambda_1\bm{a}+\lambda_2\bm{b},\ \bm{c}\rangle_{\mu}=\lambda_1 \langle\bm{a},\ \bm{c}\rangle_{\mu} + \lambda_2 \langle\bm{b},\ \bm{c}\rangle_{\mu}, \text{ (linearity);}\\
    &\langle\bm{a},\ \bm{a}\rangle_{\mu}>0 \text{ for all $\bm{a}\neq \bm{0}$. (positive-definiteness).}
\end{align*}
Thus, $\langle \cdot,\cdot\rangle_{\mu}$ defines a valid inner product space. It remains to prove \cref{eq.temp_bias_2} and \cref{eq.temp_bias_3}. To that end, we have
\begin{align*}
    \E [\normTwo{\bm{a}+\bm{b}}^2]=&\E [\normTwo{\bm{a}}^2]+\E [\bm{a}^T\bm{b}]+2\E[\normTwo{\bm{b}}^2]\\
    \leq & \E [\normTwo{\bm{a}}^2]+2\abs{\E [\bm{a}^T\bm{b}]}+\E[\normTwo{\bm{b}}^2]\\
    \leq & \E [\normTwo{\bm{a}}^2]+2\sqrt{\E[\normTwo{\bm{a}}^2]\cdot \E[\normTwo{\bm{b}}^2]}+\E[\normTwo{\bm{b}}^2]\text{ (by \cref{eq.temp_bias_1})}\\
    =& \left(\sqrt{\E[\normTwo{\bm{\bm{a}}}^2]}+\sqrt{\E[\normTwo{\bm{b}}^2]}\right)^2.
\end{align*}
Similarly, we have
\begin{align*}
    \E [\normTwo{\bm{a}+\bm{b}}^2]=&\E [\normTwo{\bm{a}}^2]+\E [\bm{a}^T\bm{b}]+2\E[\normTwo{\bm{b}}^2]\\
    \geq & \E [\normTwo{\bm{a}}^2]-2\abs{\E [\bm{a}^T\bm{b}]}+\E[\normTwo{\bm{b}}^2]\\
    \geq & \E [\normTwo{\bm{a}}^2]-2\sqrt{\E[\normTwo{\bm{a}}^2]\cdot \E[\normTwo{\bm{b}}^2]}+\E[\normTwo{\bm{b}}^2]\text{ (by \cref{eq.temp_bias_1})}\\
    =& \left(\sqrt{\E[\normTwo{\bm{a}}^2]}-\sqrt{\E[\normTwo{\bm{b}}^2]}\right)^2.
\end{align*}
The result of this lemma therefore follows.
\end{proof}

The result of the following lemma can be found in the literature (e.g., \cite{belkin2020two}).
\begin{lemma}\label[lemma]{le.bias}
Consider a random matrix $\mathbf{K}\in \mathds{R}^{p\times n}$ where $p$ and $n$ are two positive integers and $p>n+1$. Each element of $\mathbf{K}$ is \emph{i.i.d.} according to standard Gaussian distribution. For any fixed vector $\bm{a}\in \mathds{R}^p$, we must have
\begin{align*}
    &\E\normTwo{\left(\iMatrix{p}-\mathbf{K}\left(\mathbf{K}^T\mathbf{K}\right)^{-1}\mathbf{K}^T\right)\bm{a}}^2 = \left(1-\frac{n}{p}\right)\normTwo{\bm{a}}^2,\\
    &\E\normTwo{\mathbf{K}\left(\mathbf{K}^T\mathbf{K}\right)^{-1}\mathbf{K}^T\bm{a}}^2 = \frac{n}{p}\normTwo{\bm{a}}^2.
\end{align*}
Further, when $p\geq 16$, we have
\begin{align*}
    &\Pr\left\{\normTwo{\left(\iMatrix{p}-\mathbf{K}\left(\mathbf{K}^T\mathbf{K}\right)^{-1}\mathbf{K}^T\right)\bm{a}}^2\leq \frac{p-n +2\sqrt{(p-n)\ln p}+2\ln p}{p-2\sqrt{p\ln p}}\normTwo{\bm{a}}^2\right\}\geq 1 - \frac{2}{p}.\\
    &\Pr\left\{\normTwo{\mathbf{K}\left(\mathbf{K}^T\mathbf{K}\right)^{-1}\mathbf{K}^T\bm{a}}^2\leq \frac{n +2\sqrt{n\ln n}+2\ln n}{p-2\sqrt{p\ln p}}\normTwo{\bm{a}}^2\right\}\geq 1 - \frac{1}{p} - \frac{1}{n}.
\end{align*}
\end{lemma}
\begin{proof}
Define $\mathbf{P}_0\defeq \mathbf{K}\left(\mathbf{K}^T\mathbf{K}\right)^{-1}\mathbf{K}^T$. By the definition of $\mathbf{P}_0$, we have $\mathbf{P}_0\mathbf{P}_0=\mathbf{P}_0$. Thus, $\mathbf{P}_0$ is a projection that projects a $p$-dim vector to a $n$-dim subspace (i.e., the column space of $\mathbf{K}$). Since each element of $\mathbf{K}$ is \emph{i.i.d.} following standard Gaussian distribution, we can conclude that $\mathbf{P}_0$ has rotational symmetry. The rest of the calculation and proof is similar to that of Proposition~3 of \cite{jutheoretical}. Here we give an intuitive explanation of why $\E\normTwo{(\iMatrix{p}-\mathbf{P}_0)\bm{a}}^2=\left(1-\frac{n}{p}\right)\normTwo{\bm{a}}^2$.
The rotational symmetry implies that the projection on $\bm{a}$ makes the change of the squared norm proportional to the dimension of the original space and the subspace, i.e.,  $E\normTwo{\mathbf{P}_0\bm{a}}^2=\frac{n}{p}\normTwo{\bm{a}}^2$ and consequently, $\E\normTwo{\bm{a}-\mathbf{P}_0\bm{a}}^2=\left(1-\frac{n}{p}\right)\normTwo{\bm{a}}^2$.
% A more rigorous proof can be shown by integration (e.g., Lemma~16 of \citet{jutheoretical}), which is omitted here. 
\end{proof}

\begin{lemma}\label[lemma]{le.IW}
Consider a random matrix $\mathbf{K}\in \mathds{R}^{a\times b}$ where $a>b+1$. Each element of $\mathbf{K}$ is \emph{i.i.d.} following standard Gaussian distribution $\mathcal{N}(0,1)$. Consider three Gaussian random vectors $\bm{\alpha},\bm{\gamma}\in\mathbf{R}^a$ and $\bm{\beta}\in\mathbf{R}^b$ such that $\bm{\alpha}\sim \mathcal{N}(\bm{0}, \sigma_{\alpha}^2\iMatrix{a})$, $\bm{\gamma}\sim \mathcal{N}(\bm{0}, \diag(d_1^2,d_2^2,\cdots,d_a^2))$, and $\bm{\beta}\sim \mathcal{N}(\bm{0}, \sigma_{\beta}^2\iMatrix{b})$. Here $\mathbf{K}$, $\bm{\alpha}$, $\bm{\gamma}$, and $\bm{\beta}$ are independent of each other. We then must have
\begin{align}
    &\E \left[(\mathbf{K}^T\mathbf{K})^{-1}\right]=\frac{\iMatrix{b}}{a-b-1},\label{eq.temp_010301}\\
    &\E \normTwo{\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta}}^2=\frac{b\sigma_{\beta}^2}{a-b-1},\label{eq.temp_010302}\\
    &\E \normTwo{(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}}^2=\frac{b\sigma_{\alpha}^2}{a-b-1},\label{eq.temp_010303}\\
    & \E \normTwo{(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\gamma}}^2=\frac{b\sum_{i=1}^a d_i^2}{a(a-b-1)}.\label{eq.temp_010320}
\end{align}
\end{lemma}
\begin{proof}
By the definition of $\mathbf{K}$, we know that $(\mathbf{K}^T\mathbf{K})^{-1}$ follows inverse-Wishart distribution, and thus Eq.~\eqref{eq.temp_010301} follows. (The expression and derivation of the mean of any inverse-Wishart matrix can be found in the literature, e.g., \cite{mardia1979jt}.) It remains to prove Eq.~\eqref{eq.temp_010302} and Eq.~\eqref{eq.temp_010303}. To that end, we obtain
\begin{align*}
    \E \normTwo{\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta}}^2=&\E \left[\left(\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta}\right)^T\left(\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta}\right)\right]\\
    =&\E \left[\bm{\beta}^T(\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta} \right]\\
    =&\E_{\bm{\beta}}\left[\bm{\beta}^T\E_{\mathbf{K}}[(\mathbf{K}^T\mathbf{K})^{-1}]\bm{\beta}\right]\text{ (since $\mathbf{K}$ and $\bm{\beta}$ are independent)}\\
    =&\frac{1}{a-b-1}\E\left[\bm{\beta}^T\bm{\beta}\right]\text{ (by Eq.~\eqref{eq.temp_010301})}\\
    % =&\E \left[\Tr\left( \bm{\beta}^T(\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta} \right)\right]\\
    % =&\E \left[\Tr\left((\mathbf{K}^T\mathbf{K})^{-1}\bm{\beta}\bm{\beta}^T\right)\right]\text{ (by trace trick that $\Tr(\mathbf{AB})=\Tr(\mathbf{BA})$)}\\
    % =&\E_{\mathbf{K}}\left[\Tr\left((\mathbf{K}^T\mathbf{K})^{-1}\E_{\bm{\beta}}[\bm{\beta}\bm{\beta}^T]\right)\right]\text{ (since $\mathbf{K}$ and $\bm{\beta}$ are independent)}\\
    % =&\sigma_{\beta}^2\E \left[\Tr\left((\mathbf{K}^T\mathbf{K})^{-1}\right)\right]\text{ (since $\E_{\bm{\beta}}[\bm{\beta}\bm{\beta}^T]=\sigma_{\beta}^2\iMatrix{b}$ because $\bm{\beta}\sim \mathcal{N}(\bm{0}, \sigma_{\beta}^2\iMatrix{b})$)}\\
    % =&\sigma_{\beta}^2\Tr\left(\E[(\mathbf{K}^T\mathbf{K})^{-1}]\right)\\
    =&\frac{b\sigma_{\beta}^2}{a-b-1}\text{ (since $\E_{\bm{\beta}}[\bm{\beta}^T\bm{\beta}]=b\sigma_{\beta}^2$ because $\bm{\beta}\sim \mathcal{N}(\bm{0}, \sigma_{\beta}^2\iMatrix{b})$)},
\end{align*}
and
\begin{align*}
    &\E \normTwo{(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}}^2\\
    =&\E \left[\left((\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}\right)^T\left((\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}\right)\right]\\
    =&\E \left[\bm{\alpha}^T\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}\right]\\
    =&\E \left[\Tr\left(\bm{\alpha}^T\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}\right)\right]\\
    =&\E \left[\Tr\left(\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\bm{\alpha}\bm{\alpha}^T\right)\right]\text{ (by trace trick that $\Tr(\mathbf{AB})=\Tr(\mathbf{BA})$)}\\
    =&\E_{\mathbf{K}}\left[\Tr\left(\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\E_{\bm{\alpha}}[\bm{\alpha}\bm{\alpha}^T]\right)\right]\text{ (since $\mathbf{K}$ and $\bm{\alpha}$ are independent)}\numberthis \label{eq.temp_012601}\\
    =&\sigma_{\alpha}^2 \E \left[\Tr\left(\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\right)\right]\text{ (since $\E_{\bm{\alpha}}[\bm{\alpha}\bm{\alpha}^T]=\sigma_{\alpha}^2\iMatrix{a}$ because $\bm{\alpha}\sim \mathcal{N}(\bm{0}, \sigma_{\alpha}^2\iMatrix{a})$)}\\
    =&\sigma_{\alpha}^2 \E \left[\Tr\left((\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\mathbf{K}\right)\right]\text{ (by trace trick that $\Tr(\mathbf{AB})=\Tr(\mathbf{BA})$))}\\
    =&\sigma_{\alpha}^2\Tr\left(\E[(\mathbf{K}^T\mathbf{K})^{-1}]\right)\\
    =&\frac{b\sigma_{\alpha}^2}{a-b-1}\text{ (by Eq.~\eqref{eq.temp_010301})}.
\end{align*}
Thus, we have proven \cref{eq.temp_010302} and \cref{eq.temp_010303}. It remains to prove \cref{eq.temp_010320}. To that end, we define $\bm{e}_i$ as the $i$-th standard basis. Define $\mathbf{M}_j\in \mathds{R}^{a\times a}$ as the permutation matrix that switches the first element with the $j$-th element for $j=2,3,\cdots,a$. Since $\mathbf{M}_j$ is a permutation matrix, we have $\mathbf{M}_j^T\mathbf{M}_j=\iMatrix{a}$.
Thus, for all $j=2,3,\cdots,a$, we obtain
{
\newcommand{\KK}{\mathbf{K}}
\newcommand{\KKK}{\hat{\mathbf{K}}}
\newcommand{\MM}{\mathbf{M}_j}
\begin{align*}
    (\KK^T\KK)^{-1}\KK^T\bm{e}_j=(\KK^T\MM^T\MM\KK)^{-1}\KK^T\MM^T\MM\bm{e}_j=(\KKK^T\KKK)^{-1}\KKK^T\bm{e}_1,
\end{align*}
where $\KKK\defeq \MM\KK$. By definition of $\KK$, we know that $\KKK$ and $\KK$ have the same distribution. Thus, we have
\begin{align}
    \E \normTwo{(\KK^T\KK)^{-1}\KK^T\bm{e}_j}^2 = \E \normTwo{(\KK^T\KK)^{-1}\KK^T\bm{e}_1}^2 \text{ for all }j=2,3,\cdots,b.\label{eq.temp_012602}
\end{align}
We then derive
\begin{align*}
    &\E \normTwo{(\KK^T\KK)^{-1}\KK^T\bm{\gamma}}^2\\
    =&\E_{\mathbf{K}}\left[\Tr\left(\mathbf{K}(\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\E_{\bm{\gamma}}[\bm{\gamma}\bm{\gamma}^T]\right)\right]\text{ (similar to \cref{eq.temp_012601})}\\
    =&\E\left[\Tr\left(\KK(\KK^T\KK)^{-1}(\KK^T\KK)^{-1}\KK\diag(d_1^2,\cdots,d_a^2)\right)\right]\text{ (by the definition of $\bm{\gamma}$)}\\
    =&\sum_{i=1}^a\E\left[\Tr\left(\KK(\KK^T\KK)^{-1}(\KK^T\KK)^{-1}\KK\diag(0,\cdots,0,\underbrace{d_i^2}_{\text{i-th element}},0,\cdots,0)\right)\right]\\
    =&\sum_{i=1}^a d_i^2\E \normTwo{(\KK^T\KK)^{-1}\KK^T\bm{e}_i}^2\text{ (similar to \cref{eq.temp_012601})}\\
    =& \frac{\sum_{j=1}^a d_j^2}{a}\sum_{i=1}^a\E \normTwo{(\KK^T\KK)^{-1}\KK^T\bm{e}_i}^2\text{ (by \cref{eq.temp_012602})}\\
    =& \frac{\sum_{j=1}^a d_j^2}{a}\sum_{i=1}^a \E\left[\Tr\left(\KK(\KK^T\KK)^{-1}(\KK^T\KK)^{-1}\KK\diag(0,\cdots,0,\underbrace{1}_{\text{i-th element}},0,\cdots,0)\right)\right]\\
    &\text{ (similar to \cref{eq.temp_012601})}\\
    =&\frac{\sum_{j=1}^a d_j^2}{a} \E\left[\Tr\left(\KK(\KK^T\KK)^{-1}(\KK^T\KK)^{-1}\KK\right)\right]\\
    =&\frac{\sum_{j=1}^a d_j^2}{a} \E \left[\Tr\left((\mathbf{K}^T\mathbf{K})^{-1}(\mathbf{K}^T\mathbf{K})^{-1}\mathbf{K}^T\mathbf{K}\right)\right]\text{ (by trace trick that $\Tr(\mathbf{AB})=\Tr(\mathbf{BA})$)}\\
    =&\frac{\sum_{j=1}^a d_j^2}{a}\Tr\left(\E[(\mathbf{K}^T\mathbf{K})^{-1}]\right)\\
    =&\frac{b\sum_{j=1}^a d_j^2}{a(a-b-1)}\text{ (by Eq.~\eqref{eq.temp_010301})}.
\end{align*}
We have therefore proven \cref{eq.temp_010320}, and
}
the result of this lemma follows.
\end{proof}


\subsection{Calculation of derivative}\label{subsec.derivative}


\begin{lemma}\label[lemma]{le.derivative_3}
Consider the expression of $\E[\modelErr]$ in \cref{eq.result_3} (overparameterized regime of Option~A). When $\EwOneDiff+\sigmaTwo^2\geq \normTwo{\qTwo}^2$, $\E[\modelErr]$ is monotone decreasing w.r.t. $\pSpecial{2}$. When $\EwOneDiff+\sigmaTwo^2 < \normTwo{\qTwo}^2$, we have
\begin{align*}
    \frac{\partial\ \E[\modelErr]}{\partial \pSpecial{2}}\begin{cases}
        \leq 0, \text{ if }\pSpecial{2}\in \left(\nTwo+1,\ \frac{\nTwo+1}{1-\frac{\sqrt{\EwOneDiff+\sigmaTwo^2}}{\normTwo{\qTwo}}}\right],\\
        >0, \text{ if }\pSpecial{2}\in \left( \frac{\nTwo+1}{1-\frac{\sqrt{\EwOneDiff+\sigmaTwo^2}}{\normTwo{\qTwo}}},\ \infty\right).
    \end{cases}
\end{align*}
\end{lemma}
\begin{proof}
We obtain
\begin{align*}
    \frac{\partial\ \E[\modelErr]}{\partial \pSpecial{2}}=&-\frac{\nTwo\left(\EwOneDiff+\sigmaTwo^2\right)}{(\pSpecial{2}-\nTwo-1)^2}+\frac{\nTwo}{\pSpecial{2}^2}\normTwo{\qTwo}^2\\
    =&\frac{\nTwo\normTwo{\qTwo}^2}{(\pSpecial{2}-\nTwo-1)^2}\left(-\frac{\EwOneDiff+\sigmaTwo^2}{\normTwo{\qTwo}^2}+\left(1-\frac{\nTwo+1}{\pSpecial{2}}\right)^2\right).
\end{align*}
The result of this lemma thus follows by checking the sign of the above expression.
\end{proof}

\begin{lemma}\label[lemma]{le.derivative_5}
Define
\begin{align*}
    \pCommonThres\defeq \frac{\nTwo+1}{1-\frac{\sigmaTwo}{\sqrt{\EwOneDiff+\normTwo{\qTwo}^2}}}.
\end{align*}
Consider the expression of $\E[\modelErr]$ in \cref{eq.result_5}. When $\sigmaTwo^2\geq \EwOneDiff+\normTwo{\qTwo}^2$ or $\pCommon \geq \pCommonThres$, $\E[\modelErr]$ is monotone decreasing w.r.t. $\pSpecial{2}$ in the overparameterized regime. When $\sigmaTwo^2 < \EwOneDiff+\normTwo{\qTwo}^2$ and $\pCommon <\pCommonThres$, we have
\begin{align*}
    \frac{\partial\ \E[\modelErr]}{\partial \pSpecial{2}}\begin{cases}
        \leq 0, \text{ if }\pSpecial{2}\in \left(\nTwo+1,\ \pCommonThres-\pCommon\right],\\
        >0, \text{ if }\pSpecial{2}\in \left( \pCommonThres-\pCommon,\ \infty\right).
    \end{cases}
\end{align*}
\end{lemma}
\begin{proof}
We obtain
\begin{align*}
    \frac{\partial\ \E[\modelErr]}{\partial \pSpecial{2}}=& \frac{\nTwo}{(\pCommon + \pSpecial{2})^2}\left(\EwOneDiff+\normTwo{\qTwo}^2\right)-\frac{\nTwo\sigmaTwo^2}{(\pCommon+\pSpecial{2}-\nTwo-1)^2},\\
    =&\frac{\nTwo\left(\EwOneDiff+\normTwo{\qTwo}^2\right)}{(\pCommon+\pSpecial{2}-\nTwo-1)^2}\left(\left(1 - \frac{\nTwo+1}{\pCommon+\pSpecial{2}}\right)^2-\frac{\sigmaTwo^2}{\EwOneDiff+\normTwo{\qTwo}^2}\right).
\end{align*}
The result of this lemma thus follows by checking the sign of the above expression.
\end{proof}

\begin{lemma}\label[lemma]{le.non_monotone_p}
When $\pCommon+\pSpecial{1}>\nOne + 1$, we have
\begin{align*}
    \frac{\partial\ \bNoise}{\partial\ \pCommon}\begin{cases}
    <0, &\text{when }\pCommon^2 > \pSpecial{1}\left(\pSpecial{1}-\nOne-1\right),\\
    \geq 0, &\text{otherwise}.
    \end{cases}
\end{align*}
\end{lemma}
\begin{proof}
By \cref{eq.def_bNoise}, we obtain
\begin{align*}
    \frac{\nOne\sigmaOne^2}{\bNoise} = \left(1+\frac{\pSpecial{1}}{\pCommon}\right)\left(\pCommon+\pSpecial{1}-\nOne-1\right).
\end{align*}
In order to determine the sign of $\frac{\partial\ \bNoise}{\partial\ \pCommon}$, it is equivalent to check the sign of $-\frac{\partial (\nOne\sigmaOne^2 / \bNoise)}{\partial \pCommon}$. To that end, we have
\begin{align*}
    -\frac{\partial (\nOne\sigmaOne^2 / \bNoise)}{\partial \pCommon} = &\frac{\pSpecial{1}}{\pCommon^2}\left(\pCommon+\pSpecial{1}-\nOne-1\right)-1-\frac{\pSpecial{1}}{\pCommon}\\
    =&\frac{\pSpecial{1}\left(\pSpecial{1}-\nOne-1\right)-\pCommon^2}{\pCommon^2}.
\end{align*}
The result of this lemma therefore follows.
\end{proof}

Let $\lambda_{\min}(\cdot)$ and $\lambda_{\min}(\cdot)$ denote the minimum and maximum singular value of a matrix.
\begin{lemma}[Corollary 5.35 of \cite{vershynin2010introduction}]\label[lemma]{le.singular_Gaussian}
    Let $\mathbf{A}$ be an $N_1\times N_2$ matrix ($N_1>N_2$) whose entries are independent standard normal random variables. Then for every $t\geq 0$, with probability at least $1-2\exp(-t^2/2)$, one has
    \begin{align*}
        \sqrt{N_1}-\sqrt{N_2}-t\leq \lambda_{\min}(\mathbf{A})\leq \lambda_{\max}(\mathbf{A})\leq \sqrt{N_1}+\sqrt{N_2}+t.
    \end{align*}
\end{lemma}

\begin{lemma}[stated on pp. 1325 of \cite{laurent2000adaptive}]\label[lemma]{le.chi_bound}
Let $U$ follow $\chi^2$ distribution with D
degrees of freedom. For any positive x, we have
\begin{align*}
    &\Pr\left\{U-D\geq 2\sqrt{Dx}+2x\right\}\leq e^{-x},\\
    &\Pr\left\{D-U\geq 2\sqrt{Dx}\right\}\leq e^{-x}.
\end{align*}
By the union bound, we thus obtain
\begin{align*}
    \Pr\left\{U\in \left[D-2\sqrt{Dx},\ D+2\sqrt{Dx}+2x\right]\right\}\geq 1 - 2e^{-x}.
\end{align*}
\end{lemma}

\newcommand{\mK}{\mathbf{K}}
\newcommand{\mU}{\mathbf{U}}
\newcommand{\mQ}{\mathbf{Q}}
\newcommand{\mD}{\mathbf{D}}
\begin{lemma}\label[lemma]{le.norm_min_eig}
Let $\mK\in \mathds{R}^{p\times n}$ where $n > p$. We have
\begin{align*}
    \normTwo{(\mK\mK^T)^{-1}\mK\bm{a}}^2 \leq \frac{\normTwo{\bm{a}}^2}{\min\eig(\mK\mK^T)}=\frac{\normTwo{\bm{a}}^2}{\lambda_{\min}^2(\mK)}.
\end{align*}
\end{lemma}
\begin{proof}
Apply the singular value decomposition on $\mK$ as $\mK = \mU \mD \mQ^T$, where $\mU \in \mathds{R}^{p\times p}$ and $\mQ \in \mathds{R}^{n\times n}$ are the orthonormal matrices, and $\mD \in \mathds{R}^{p\times n}$ is a  rectangular diagonal matrix whose diagonal elements are $d_1,d_2,\cdots, d_p$. Then, we obtain
\begin{align*}
    &\mK \mK^T = \mU \mD \mQ^T \mQ \mD^T \mU^T = \mU \mD \mD^T \mU^T =\mU \diag(d_1^2, d_2^2,\cdots, d_p^2) \mU^T,\\
    &(\mK \mK^T)^{-1}=\mU \diag(d_1^{-2},d_2^{-2},\cdots,d_p^{-2})\mU^T,
\end{align*}
and
\begin{align*}
    \normTwo{(\mK \mK^T)^{-1}\mK \bm{a}}^2 =& \bm{a}^T \mK^T (\mK \mK^T)^{-1} (\mK \mK^T)^{-1} \mK \bm{a}\\
    =&\bm{a}^T \mQ \mD^T \diag(d_1^{-4},d_2^{-4},\cdots, d_p^{-4})\mD \mQ^T\bm{a}\\
    =& \bm{a}^T \mQ \diag(d_1^{-2},d_2^{-2},\cdots,d_p^{-2},0,\cdots,0)\mQ^T \bm{a}.
\end{align*}
Notice that $\normTwo{\mQ^T \bm{a}}=\normTwo{\bm{a}}$. Therefore, we have
\begin{align*}
    \normTwo{(\mK \mK^T)^{-1}\mK \bm{a}}^2 \leq & \max \left\{d_1^{-2},d_2^{-2},\cdots,d_p^{-2},0,\cdots,0\right\}\cdot \normTwo{\mQ^T \bm{a}}^2\\
    =& \min\{d_1,d_2,\cdots,d_p\}^{-2}\cdot \normTwo{\bm{a}}^2\\
    =& \frac{\normTwo{\bm{a}}^2}{\lambda_{\min}^2(\mK)}.
\end{align*}
\end{proof}

\section{Proof of Theorem~\ref{thm.w_err}}\label{app.proof_w_err}

Define
\begin{align*}
    &\UU\defeq \left[\begin{smallmatrix}\XX\\\ZZ\end{smallmatrix}\right]\in \mathds{R}^{(\pCommon+\pSpecial{1})\times \nOne},\numberthis \label{eq.def_U}\\
    &\DD\defeq \left[\begin{smallmatrix}
        \iMatrix{\pCommon} &\bm{0}\\
        \bm{0} &\bm{0}
    \end{smallmatrix}\right]\in \mathds{R}^{(\pCommon+\pSpecial{1})\times (p_1+p_2)},\numberthis \label{eq.def_D}\\
    &\GG\defeq \iMatrix{p_1+p_2}-\DD= \left[\begin{smallmatrix}
        \bm{0} &\bm{0}\\
        \bm{0} &\iMatrix{\pCommon}
    \end{smallmatrix}\right]\in \mathds{R}^{(p_1+p_2)\times (p_1+p_2)}.\numberthis \label{eq.def_G}
\end{align*}
When $p_1+p_2>\nOne$ (overparameterized), we have (with probability 1)
\begin{align}
    \wTildeEx = \DD\UU \left(\UU^T\UU\right)^{-1}\left(\XX^T \wOne+\ZZ^T \qOne+\eOne\right).\label{eq.wOne_solution_over}
\end{align}

\subsection*{The overparameterized situation}

We define
\begin{align}\label{eq.def_PP}
    \PP\defeq \UU(\UU^T\UU)^{-1}\UU^T.
\end{align}
Notice that
\begin{align}\label{eq.PP_is_proj}
    \PP\PP=\PP,\text{ and }\PP^T=\PP,
\end{align}
i.e., $\PP$ is a projection to the column space of $\UU$.






Define the extended vector of $\wOne$, $\wTwo$, and $\wTilde$ as
\begin{align}\label{eq.def_wex}
    \wOneEx\defeq \left[\begin{smallmatrix}
        \wOne\\
        \bm{0}
    \end{smallmatrix}\right]\in \mathds{R}^{\pCommon+\pSpecial{1}}, \quad \wTwoEx\defeq \left[\begin{smallmatrix}
        \wTwo\\
        \bm{0}
    \end{smallmatrix}\right]\in \mathds{R}^{\pCommon+\pSpecial{1}}, \quad \wTildeEx\defeq \myMatrix{\wTilde\\\bm{0}}\in \mathds{R}^{\pCommon+\pSpecial{1}},
\end{align}
respectively.
Similarly, define the extended vector of $\qOne$ as
\begin{align}\label{eq.def_qex}
    \qOneEx \defeq \left[\begin{smallmatrix}
        \bm{0}\\
        \qOne
    \end{smallmatrix}\right]\in \mathds{R}^{\pCommon+\pSpecial{1}}.
\end{align}
By Eq.~\eqref{eq.wOne_solution_over} and \cref{eq.def_wex}, we have
\begin{align*}
    \normTwo{\wTwo-\wTilde}^2=&\normTwo{\wTwoEx -\DD\UU \left(\UU^T\UU\right)^{-1}\left(\XX^T \wOne+\ZZ^T \qOne+\eOne\right)}^2\\
    =& \normTwo{\wTwoEx-\DD\UU \left(\UU^T\UU\right)^{-1}\left(\UU^T\myMatrix{\wOne\\\qOne}+\eOne\right)}^2\text{ (by \cref{eq.def_U})}.
\end{align*}
By Assumption~\ref{as.Gaussian} (noise has zero mean), we obtain
\begin{align*}
    \E_{\eOne}\left\langle \wTwoEx - \DD\UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne},\ \DD\UU(\UU^T\UU)^{-1}\eOne \right\rangle = 0,
\end{align*}
and therefore conclude
\begin{align}
    \E_{\eOne}\normTwo{\wTwo - \wTilde}^2= &\underbrace{\normTwo{\wTwoEx - \DD\UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}}^2}_{\text{Term 1}}+\underbrace{\normTwo{\DD\UU(\UU^T\UU)^{-1}\eOne}^2}_{\text{Term 2}}.\label{eq.temp_012401}
\end{align}




The following lemma shows the expected value of Term~2 of \cref{eq.temp_012401}.
\begin{lemma}\label[lemma]{le.core_noise}
We have
\begin{align*}
    \E_{\UU,\bm{\epsilon}}[\text{Term 2 of \cref{eq.temp_012401}}]=\frac{\pCommon}{\pCommon+\pSpecial{1}}\cdot \frac{\nOne \sigmaOne^2}{\pCommon+\pSpecial{1}-\nOne - 1}.
\end{align*}
\end{lemma}
\begin{proof}
    See \cref{app.proof_core_noise}.
\end{proof}


It remains to estimate Term~1 of \cref{eq.temp_012401}. The following lemma aims to achieve a relatively precise estimation when $\oRatio\left(\normTwo{\wOne}^2+\normTwo{\qOne}^2\right)$ is relatively small with respect to $\similarity$.
\begin{lemma}\label[lemma]{le.core_bias_1}
We have
\begin{align*}
    \left[\similarity-\sqrt{\oRatio\left(\normTwo{\wOne}^2+\normTwo{\qOne}^2\right)}\right]_+^2\leq \E[\text{Term 1 in \cref{eq.temp_012401}}]\leq \overline{b}_1^2 .
\end{align*}
\end{lemma}
\begin{proof}
    See \cref{app.proof_core_bias_1}.
\end{proof}



However, \cref{le.core_bias_1} may be loose in some other cases, which requires some finer estimation on Term~1 of \cref{eq.temp_012401} for those cases.
Define
\begin{align}
    &A\defeq \wTwoEx-\wOneEx,\label{eq.def_A}\\
    &B\defeq \wOneEx- \DD\UU(\UU^T\UU)^{-1}\XX^T\wOne,\label{eq.def_B}\\
    &C\defeq \DD\UU(\UU^T\UU)^{-1}\ZZ^T\qOne.\label{eq.def_C}
\end{align}
We then obtain
\begin{align}
    \text{Term 1 in \cref{eq.temp_012401}}= \normTwo{ A+ B + C}^2\label{eq.temp_012407}
\end{align}
The following lemmas concern the estimation of $A$, $B$, and $C$.
\begin{lemma}\label[lemma]{le.core_bias_2}
We have
\begin{align*}
    \left[\normTwo{\wTwo}-\sqrt{1-r}\normTwo{\wOne}\right]_+^2\leq\E\left[\normTwo{A+B}^2\right]\leq \left(\normTwo{\wTwo}+\sqrt{1-r}\normTwo{\wOne}\right)^2 .
\end{align*}
\end{lemma}
\begin{proof}
    See \cref{app.proof_core_bias_2}
\end{proof}



\begin{lemma}\label[lemma]{le.term_1}
We have
\begin{align*}
    &\left[1-\frac{2\nOne}{\pCommon+\pSpecial{1}}\right]_+\cdot\normTwo{\wOne}^2\leq \E \left[\normTwo{B}^2\right] \leq \oRatio\normTwo{\wOne}^2.
\end{align*}
% Here $[\cdot]^+$ denotes $\max\{0,\cdot\}$.
% We thus have $\normTwo{B}^2$ converges in probability to $\normTwo{\wOne}^2$ as $\pCommon+\pSpecial{1}\to \infty$, i.e.,
% \begin{align*}
%     &\normTwo{B}^2\overset{P}{\to}\normTwo{\wOne}^2.
% \end{align*}
\end{lemma}
\begin{proof}
    See \cref{app.proof_term_1}.
\end{proof}




\begin{lemma}\label[lemma]{le.term_2}
We have
\begin{align*}
    \E \left[\normTwo{C}^2\right] \leq \min\left\{\oRatio, 1-\oRatio\right\}\normTwo{\qOne}^2.
\end{align*}
\end{lemma}
\begin{proof}
    See \cref{app.proof_term_2}.
\end{proof}

We therefore obtain
\begin{align*}
    &\E[\text{Term 1 of \cref{eq.temp_012401}}] \\
    =& \E [\normTwo{A+B+C}^2]\text{ (by \cref{eq.temp_012407})}\\
    \leq & \left(\sqrt{\E [\normTwo{A+B}^2]}+\sqrt{\E[\normTwo{C}^2]}\right)^2\text{ (by \cref{le.Cauchy})}\numberthis \label{eq.temp_012410}\\
    \leq & \left(\normTwo{\wTwo}+\sqrt{1-r}\normTwo{\wOne}+\sqrt{\min\{\oRatio,1-\oRatio\}}\normTwo{\qOne}\right)^2\text{ (by \cref{le.core_bias_2} and \cref{le.term_2})}\\
    =&\overline{b}_2^2,\numberthis \label{eq.temp_012501}
\end{align*}
and
\begin{align*}
    &\E[\text{Term 1 of \cref{eq.temp_012401}}] \\
    =& \E [\normTwo{A+B+C}^2]\text{ (by \cref{eq.temp_012407})}\\
    \geq & \left[\sqrt{\E [\normTwo{A+B}^2]}-\sqrt{\E[\normTwo{C}^2]}\right]_+^2\text{ (by \cref{le.Cauchy})}\numberthis \label{eq.temp_012411}\\
    \geq & \left[\normTwo{\wTwo}-\sqrt{1-r}\normTwo{\wOne}-\sqrt{\min\{\oRatio,1-\oRatio\}}\normTwo{\qOne}\right]_+^2\text{ (by \cref{le.core_bias_2} and \cref{le.term_2})}.
\end{align*}

We also have
\begin{align*}
    &\E[\text{Term 1 of \cref{eq.temp_012401}}]\\
    \leq &\left(\sqrt{\E [\normTwo{A+B}^2]}+\sqrt{\E[\normTwo{C}^2]}\right)^2\text{ (by \cref{eq.temp_012410})}\\
    \leq & \left(\sqrt{\E[\normTwo{A}^2]}+\sqrt{\E[\normTwo{B}^2]}+\sqrt{\E[\normTwo{C}^2]}\right)^2\text{ (by \cref{le.Cauchy})}\\
    \leq & \left(\similarity+\sqrt{\oRatio}\normTwo{\wOne}+\sqrt{\min\{\oRatio,1-\oRatio\}}\normTwo{\qOne}\right)^2\text{ (by \cref{eq.def_similarity}, \cref{le.term_1}, and \cref{le.term_2})}\\
    =&\overline{b}_3^2,\numberthis \label{eq.temp_012502}
\end{align*}
and
\begin{align*}
    &\E[\text{Term 1 of \cref{eq.temp_012401}}]\\
    \geq &\left[\sqrt{\E [\normTwo{A+B}^2]}-\sqrt{\E[\normTwo{C}^2]}\right]_+^2\text{ (by \cref{eq.temp_012411})}\\
    \geq & \left[\sqrt{\E[\normTwo{B}^2]}-\sqrt{\E[\normTwo{A}^2]}-\sqrt{\E[\normTwo{C}^2]}\right]_+^2\text{ (by \cref{le.Cauchy})}\\
    \geq & \left[\sqrt{\oRatio}\normTwo{\wOne}-\similarity-\sqrt{\min\{\oRatio,1-\oRatio\}}\normTwo{\qOne}\right]_+^2\text{ (by \cref{eq.def_similarity}, \cref{le.term_1}, and \cref{le.term_2})}.
\end{align*}

By \cref{le.core_bias_1}, \cref{eq.temp_012501}, and \cref{eq.temp_012502}, we therefore conclude
\begin{align}
    \E[\text{Term 1 of Eq.~\eqref{eq.temp_012401}}]\leq \min_{i=1,2,3}\overline{b}_i^2.\label{eq.temp_012503}
\end{align}
By \cref{eq.temp_012503}, \cref{le.core_noise}, and \cref{eq.temp_012401}, we thus have \cref{eq.result_1}.



% \begin{lemma}
% We have
% \begin{align*}
%     \abs{\E [\text{Term 4 of Eq.~\eqref{eq.decomp}}]}\leq 2\sqrt{\E [\text{Term 1 of Eq.~\eqref{eq.decomp}}]\cdot \E [\text{Term 2 of Eq.~\eqref{eq.decomp}}]}
% \end{align*}
% \end{lemma}
% \begin{proof}
% This result directly follows by Lemma~\ref{le.Cauchy}.
% \end{proof}


\subsection*{The underparameterized situation}
When $\pCommon+\pSpecial{1}<\nOne$, the solution that minimizes the training error is given by
\begin{align*}
    \wTildeEx = &\DD\left(\UU\UU^T\right)^{-1}\UU\yOne\\
    =& \DD\left(\UU\UU^T\right)^{-1}\UU\left(\XX^T\wOne + \ZZ^T\qOne + \eOne\right)\\
    =&\DD\left(\UU\UU^T\right)^{-1}\UU\left(\UU^T\myMatrix{\wOne\\\qOne} + \eOne\right)\\
    =& \wOneEx + \DD \left(\UU\UU^T\right)^{-1}\UU\eOne\text{ (by \cref{eq.def_wex} and \cref{eq.def_D})}.
\end{align*}
Thus, we have
\begin{align}\label{eq.temp_012412}
    \normTwo{\wTwo-\wTilde}^2=\normTwo{(\wTwoEx-\wOneEx)-\DD \left(\UU\UU^T\right)^{-1}\UU\eOne}^2.
\end{align}
By \cref{as.Gaussian}, we know that $\eOne$ is independent of $\UU$ and has zero mean. We therefore obtain
\begin{align}\label{eq.temp_012413}
    \E_{\eOne} \left\langle\wTwoEx-\wTildeEx,\ \DD \left(\UU\UU^T\right)^{-1}\UU\eOne\right\rangle=0.
\end{align}
Recalling the definition of $\similarity$ in \cref{eq.def_similarity}, by \cref{eq.temp_012412} and \cref{eq.temp_012413}, we thus have
\begin{align}\label{eq.temp_012415}
    \E \normTwo{\wTwo-\wTilde}^2=\similarity^2+\E\normTwo{\DD \left(\UU\UU^T\right)^{-1}\UU\eOne}^2.
\end{align}
It remains to estimate $\E\normTwo{\DD \left(\UU\UU^T\right)^{-1}\UU\eOne}^2$. To that end, we derive
\begin{align*}
    &\E\normTwo{\DD \left(\UU\UU^T\right)^{-1}\UU\eOne}^2\\
    =&\E \left[\left(\DD \left(\UU\UU^T\right)^{-1}\UU\eOne\right)^T\DD \left(\UU\UU^T\right)^{-1}\UU\eOne\right]\\
    =&\E\Tr\left(\left(\DD \left(\UU\UU^T\right)^{-1}\UU\eOne\right)^T\DD \left(\UU\UU^T\right)^{-1}\UU\eOne\right)\\
    =&\E \Tr\left(\DD \left(\UU\UU^T\right)^{-1}\UU\eOne\eOne^T\UU^T(\UU\UU^T)^{-1}\DD\right)\\
    &\text{ (by trace trick that $\Tr(\mathbf{AB})=\Tr(\mathbf{BA})$)}\\
    =&\E_{\UU}\Tr\left(\DD \left(\UU\UU^T\right)^{-1}\UU\E_{\eOne}[\eOne\eOne^T]\UU^T(\UU\UU^T)^{-1}\DD\right)\\
    =&\sigmaOne^2\E\Tr\left(\DD(\UU\UU^T)^{-1}\DD\right)\text{ ($\E[\eOne\eOne^T]=\sigmaOne^2\iMatrix{\nOne}$ by Assumption~\ref{as.Gaussian})}\\
    =&\Tr\left(\DD\E[(\UU\UU^T)^{-1}]\DD\right)\\
    =&\frac{\pCommon}{\pCommon+\pSpecial{1}}\cdot\frac{(\pCommon+\pSpecial{1})\sigmaOne^2}{\nOne-(\pCommon+\pSpecial{1})-1}\text{ (by Lemma~\ref{le.IW})}\\
    =&\frac{\pCommon\sigmaOne^2}{\nOne-(\pCommon+\pSpecial{1})-1}.\numberthis \label{eq.temp_012414}
\end{align*}
Substituting \cref{eq.temp_012414} into \cref{eq.temp_012415}, we therefore obtain
\begin{align*}
    \E \normTwo{\wTwo-\wTilde}^2=\similarity^2+\frac{\pCommon\sigmaOne^2}{\nOne-(\pCommon+\pSpecial{1})-1},
\end{align*}
i.e., we have \cref{eq.result_2}.

Combining our proof for the overparameterized situation and the underparameterized situation, \cref{thm.w_err} thus follows.


%----------------------------------------
% The rest of this file is for the proofs
%----------------------------------------

% \subsection{Geometric Illustration for the Proofs}

\begin{figure}[t]
    \centering
    \includegraphics[width=0.5\textwidth]{figs/transfer_learning.eps}
    \caption{Geometric illustration of some vectors and projections used in the proof.}
    \label{fig.projection}
\end{figure}

\renewcommand{\arraystretch}{1.5}
\begin{table}[t]
\centering
\begin{tabular}{| c | c |} 
 \hline
 Notation in Fig.~\ref{fig.projection} & Meaning \\
 \hline
 plane $\alpha$ & column space of $\UU$\\
 \hline
 plane $\beta$ & subspace spanned by $\bm{e}_1,\cdots,\bm{e}_{\pCommon}$\\
 \hline
 vector $\overrightarrow{\mathrm{OA}}$ & $\wOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{OB}}$ & $\PP\wOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{OD}}$ & $\DD\PP\wOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{DA}}$ & $\wOneEx-\DD\PP\wOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{OE}}$ & $\langle\wOneEx,\ \DD\PP\wOneEx\rangle \cdot \wOneEx/\normTwo{\wOneEx}^2$\\
 \hline
 vector $\overrightarrow{\mathrm{EA}}$ & $\wOneEx-\langle\wOneEx,\ \DD\PP\wOneEx\rangle \cdot \wOneEx/\normTwo{\wOneEx}^2$\\
 \hline
\end{tabular}
\caption{Interpretation of notations in Fig.~\ref{fig.projection} for Lemma~\ref{le.term_1}.}
\label{table.notation_in_proj_fig}
\end{table}


\renewcommand{\arraystretch}{1.5}
\begin{table}[t]
\centering
\begin{tabular}{| c | c |} 
 \hline
 Notation in Fig.~\ref{fig.projection} & Meaning \\
 \hline
 plane $\alpha$ & column space of $\UU$\\
 \hline
 plane $\beta$ & subspace spanned by $\bm{e}_{\pCommon+1},\cdots,\bm{e}_{\pCommon+\pSpecial{1}}$\\
 \hline
 vector $\overrightarrow{\mathrm{OA}}$ & $\qOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{OB}}$ & $\PP\qOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{OD}}$ & $\GG\PP\qOneEx$\\
 \hline
 vector $\overrightarrow{\mathrm{DB}}$ & $\DD \PP \qOneEx$\\
 \hline
\end{tabular}
\caption{Interpretation of notations in Fig.~\ref{fig.projection} for Lemma~\ref{le.term_2}.}
\label{table.notation_in_proj_fig_term_2}
\end{table}




\input{tightness.tex}

\subsection{Proof of Lemma~\ref{le.core_noise}}\label{app.proof_core_noise}
\begin{proof}
We first prove that $\UU(\UU^T\UU)^{-1}\eOne$ has rotation symmetry. Consider any rotation $\mathbf{S}\in \mathsf{SO}(\pCommon+\pSpecial{1})$ where $\mathsf{SO}(\pCommon+\pSpecial{1})\in \mathds{R}^{(\pCommon+\pSpecial{1})\times(\pCommon+\pSpecial{1})}$ denotes all rotations in $(\pCommon+\pSpecial{1})$-dimensional space. We have
\begin{align*}
    \mathbf{S}\UU(\UU^T\UU)^{-1}\eOne =& \mathbf{S}\UU(\UU^T\mathbf{S}^{-1}\mathbf{S}\UU)^{-1}\eOne\\
    =& \mathbf{S}\UU(\UU^T\mathbf{S}^T\mathbf{S}\UU)^{-1}\eOne\text{ ($\mathbf{S}^{-1}=\mathbf{S}^T$ since $\mathbf{S}$ is a rotation)}\\
    =&(\mathbf{S}\UU)((\mathbf{S}\UU)^T\mathbf{S}\UU)^{-1}\eOne.
\end{align*}
By Assumption~\ref{as.Gaussian}, we know that $\mathbf{S}\UU$ has the same distribution as $\UU$. Thus, the rotated vector $\mathbf{S}\UU(\UU^T\UU)^{-1}\eOne$ has the same distribution as the original vector $\UU(\UU^T\UU)^{-1}\eOne$. This implies that
\begin{align*}
    \E ([\UU(\UU^T\UU)^{-1}\eOne]_1)^2=\E ([\UU(\UU^T\UU)^{-1}\eOne]_2)^2=\cdots=\E ([\UU(\UU^T\UU)^{-1}\eOne]_{\pCommon+\pSpecial{1}})^2,
\end{align*}
where $[\cdot]_i$ denotes the $i$-th element of the vector. We therefore obtain
\begin{align*}
    \E \normTwo{\DD\UU(\UU^T\UU)^{-1}\eOne}^2 = &\sum_{i=1}^{\pCommon}\E ([\UU(\UU^T\UU)^{-1}\eOne]_i)^2\\
    =&\frac{\pCommon}{\pCommon+\pSpecial{1}}\E\normTwo{\UU(\UU^T\UU)^{-1}\eOne}^2\\
    =& \frac{\pCommon}{\pCommon+\pSpecial{1}}\cdot \frac{\nOne\sigmaOne^2}{\pCommon+\pSpecial{1}-\nOne-1}\text{ (by Lemma~\ref{le.IW})}.
\end{align*}
The result of this lemma thus follows.
\end{proof}

\subsection{Proof of Lemma~\ref{le.core_bias_1}}\label{app.proof_core_bias_1}
\begin{proof}
By \cref{eq.def_D} and \cref{eq.def_wex}, we have
\begin{align}
    \wTwoEx=\DD\myMatrix{\wTwo\\\qOne}=\DD\myMatrix{\wTwo-\wOne\\\bm{0}}+\DD\myMatrix{\wOne\\\qOne}=\myMatrix{\wTwo\\\qOne}=\DD\myMatrix{\wTwo-\wOne\\\bm{0}}+\DD\myMatrix{\wOne\\\qOne}.\label{eq.temp_012402}
\end{align}
Thus, we obtain
\begin{align*}
    &\E[\text{Term 1 of \cref{eq.temp_012401}}]\\
    =&\E\normTwo{\DD\left(\myMatrix{\wTwo-\wOne\\\bm{0}}+\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}\right)}^2\text{ (by \cref{eq.temp_012402})}\\
    \leq & \E\normTwo{\myMatrix{\wTwo-\wOne\\\bm{0}}+\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}}^2\text{ (since $\normTwo{\DD \bm{a}}\leq \normTwo{\bm{a}}$ for all $\bm{a}\in \mathds{R}^{\pCommon+\pSpecial{1}}$)}\\
    \leq & \left(\similarity+\sqrt{\E\normTwo{\left(\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}\right)}^2}\right)^2\text{ (by \cref{le.Cauchy} and \cref{eq.def_similarity})}\\
    =&\left(\similarity+\sqrt{\oRatio\left(\normTwo{\wOne}^2+\normTwo{\qOne}^2\right)}\right)^2\text{ (by \cref{le.bias})}\\
    =&\overline{b}_1^2.\numberthis \label{eq.temp_012405}
\end{align*}
We also have
\begin{align*}
    &\E[\text{Term 1 of \cref{eq.temp_012401}}]\\
    =&\E\normTwo{\myMatrix{\wTwo-\wOne\\\bm{0}}+\DD\left(\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}\right)}^2\text{ (by \cref{eq.temp_012402})}\\
    \geq & \left(\similarity-\sqrt{\E\normTwo{\DD\left(\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}\right)}^2}\right)^2\text{ (by \cref{le.Cauchy} and \cref{eq.def_similarity})}\\
    \geq & \left(\left[\similarity-\sqrt{\E\normTwo{\DD\left(\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}\right)}^2}\right]^+\right)^2\\
    \geq &\left[\similarity-\sqrt{\E\normTwo{\myMatrix{\wOne\\\qOne}- \UU(\UU^T\UU)^{-1}\UU^T\myMatrix{\wOne\\\qOne}}^2}\right]_+^2\text{ (since $\normTwo{\DD \bm{a}}\leq \normTwo{\bm{a}}$ for all $\bm{a}\in \mathds{R}^{\pCommon+\pSpecial{1}}$)}\\
    =&\left[\similarity-\sqrt{\oRatio\left(\normTwo{\wOne}^2+\normTwo{\qOne}^2\right)}\right]_+^2\text{ (by \cref{le.bias})}.\numberthis \label{eq.temp_012406}
\end{align*}
By \cref{eq.temp_012405} and \cref{eq.temp_012406}, the result of this lemma thus follows.
\end{proof}

\subsection{Proof of Lemma~\ref{le.core_bias_2}}\label{app.proof_core_bias_2}
\begin{proof}
We have
\begin{align*}
    \normTwo{A+B}^2=&\normTwo{\wTwoEx-\DD\UU(\UU^T\UU)^{-1}\XX^T\wOne}^2\text{ (by \cref{eq.def_A} and \cref{eq.def_B})}\\
    =&\normTwo{\wTwoEx-\DD\UU(\UU^T\UU)^{-1}\UU^T\wOneEx}^2 \text{ (by \cref{eq.def_wex} and \cref{eq.def_U})}.\numberthis \label{eq.temp_012408}
\end{align*}
Thus, we obtain
\begin{align*}
    \E \normTwo{A+B}^2\leq  &\left(\normTwo{\wTwo}+\sqrt{\E\normTwo{\DD\UU(\UU^T\UU)^{-1}\UU^T\wOneEx}^2}\right)^2\text{ (by \cref{eq.temp_012408} and \cref{le.Cauchy})}\\
    \leq & \left(\normTwo{\wTwo}+\sqrt{\E\normTwo{\UU(\UU^T\UU)^{-1}\UU^T\wOneEx}^2}\right)^2\text{ (since $\normTwo{\DD \bm{a}}\leq \normTwo{\bm{a}}$ for all $\bm{a}\in \mathds{R}^{\pCommon+\pSpecial{1}}$)}\\
    = & \left(\normTwo{\wTwo}+\sqrt{1-r}\normTwo{\wOne}\right)^2\text{ (by \cref{le.bias})}.
\end{align*}
We also have
\begin{align*}
    \E \normTwo{A+B}^2\geq  &\left(\normTwo{\wTwo}-\sqrt{\E\normTwo{\DD\UU(\UU^T\UU)^{-1}\UU^T\wOneEx}^2}\right)^2\text{ (by \cref{eq.temp_012408} and \cref{le.Cauchy})}\\
    \geq & \left[\normTwo{\wTwo}-\sqrt{\E\normTwo{\UU(\UU^T\UU)^{-1}\UU^T\wOneEx}^2}\right]_+^2\text{ (since $\normTwo{\DD \bm{a}}\leq \normTwo{\bm{a}}$ for all $\bm{a}\in \mathds{R}^{\pCommon+\pSpecial{1}}$)}\\
    = & \left[\normTwo{\wTwo}-\sqrt{1-r}\normTwo{\wOne}\right]_+^2\text{ (by \cref{le.bias})}.
\end{align*}
The result of this lemma thus follows.
\end{proof}

\subsection{Proof of Lemma~\ref{le.term_1}}\label{app.proof_term_1}
\begin{proof}
We first prove the upper bound in this lemma.
By the definition of $\wOneEx$ in Eq.~\eqref{eq.def_wex}, we obtain
\begin{align}
    &\XX^T\wOne = \UU^T\wOneEx,\quad \DD\wOneEx = \wOneEx.\label{eq.temp_122201}
\end{align}
Notice that for any vector $\bm{a}\in \mathds{R}^{\pCommon+\pSpecial{1}}$, we have $\normTwo{\DD \bm{a}}\leq \normTwo{\bm{a}}$.
Thus, we conclude
\begin{align}
    \normTwo{\wOneEx - \PP\wOneEx}\geq & \normTwo{\DD\left(\wOneEx - \PP\wOneEx\right)}.\label{eq.temp_122202}
\end{align}
By Eq.~\eqref{eq.temp_122201}, we have
\begin{align}
    \DD\left(\wOneEx - \PP\wOneEx\right)=\wOneEx -\DD \UU(\UU^T\UU)^{-1}\XX^T\wOne.\label{eq.temp_122203}
\end{align}
By Eq.~\eqref{eq.temp_122202} and Eq.~\eqref{eq.temp_122203}, we thus obtain
\begin{align}
    \normTwo{\wOneEx - \DD\UU(\UU^T\UU)^{-1}\XX^T\wOne}\leq \normTwo{\wOneEx - \PP\wOneEx}.\label{eq.temp_011101}
\end{align}
We further have
\begin{align*}
    &\E\normTwo{\wOneEx - \DD\UU(\UU^T\UU)^{-1}\XX^T\wOne}^2\\
    \leq & \E \normTwo{\wOneEx - \PP\wOneEx}^2\text{ (by Eq.~\eqref{eq.temp_011101})}\\
    = & \left(1-\frac{\nOne}{\pCommon+\pSpecial{1}}\right) \normTwo{\wOneEx}^2\text{ (by Lemma~\ref{le.bias})}\\
    = & \left(1-\frac{\nOne}{\pCommon+\pSpecial{1}}\right) \normTwo{\wOne}^2\text{ (since $\normTwo{\wOne}=\normTwo{\wOneEx}$ by Eq.~\eqref{eq.def_wex})}.
\end{align*}
The upper bound in this lemma thus holds. It remains to prove the lower bound. To that end, we define
\begin{align}
    &\bm{a}\defeq\wOneEx-\frac{\langle\DD\PP\wOneEx,\ \wOneEx\rangle}{\normTwo{\wOneEx}^2}\wOneEx\text{ (corresponds to $\overrightarrow{\mathrm{OE}}$ in Fig.~\ref{fig.projection})},\label{eq.temp_011201}\\
    &\bm{b}\defeq\frac{\langle\DD\PP\wOneEx,\ \wOneEx\rangle}{\normTwo{\wOneEx}^2}\wOneEx-\DD\PP\wOneEx\text{ (corresponds to $\overrightarrow{\mathrm{BE}}$ in Fig.~\ref{fig.projection})}.\label{eq.temp_011202}
\end{align}
Notice that
\begin{align}
    \bm{a}^T\bm{b}=\langle\DD\PP\wOneEx,\ \wOneEx\rangle-\langle\DD\PP\wOneEx,\ \wOneEx\rangle-\frac{\langle\DD\PP\wOneEx,\ \wOneEx\rangle^2}{\normTwo{\wOneEx}^2}+\frac{\langle\DD\PP\wOneEx,\ \wOneEx\rangle^2}{\normTwo{\wOneEx}^2}=0.\label{eq.temp_011203}
\end{align}
Thus, we obtain
\begin{align*}
    &\normTwo{\wOneEx-\DD\PP\wOneEx}^2\\
    =&\normTwo{\bm{a}+\bm{b}}^2\text{ (by Eq.~\eqref{eq.temp_011201} and Eq.~\eqref{eq.temp_011202})}\\
    =&\normTwo{\bm{a}}^2+\normTwo{\bm{b}}^2\text{ (by Eq.~\eqref{eq.temp_011203})}\\
    \geq &\normTwo{\bm{a}}^2\\
    =&\normTwo{\wOneEx}^2+\frac{\langle\DD\PP\wOneEx,\ \wOneEx\rangle^2}{\normTwo{\wOneEx}^2}-2\langle\DD\PP\wOneEx,\ \wOneEx\rangle\\
    \geq & \normTwo{\wOneEx}^2-2\langle\DD\PP\wOneEx,\ \wOneEx\rangle.\numberthis \label{eq.temp_011301}
\end{align*}
We also have
\begin{align*}
    \langle\DD\PP\wOneEx,\ \wOneEx\rangle=&\wOneEx^T\DD\PP\wOneEx\\
    =&\wOneEx^T\PP\wOneEx\text{ (since $\wOneEx^T\DD=\wOneEx^T$ by Eq.~\eqref{eq.temp_122201})}\\
    =&\wOneEx\PP^T\PP\wOneEx\text{ (by Eq.~\eqref{eq.PP_is_proj})}\\
    =&\normTwo{\PP\wOneEx}^2.\numberthis \label{eq.temp_011302}
\end{align*}
By Eq.~\eqref{eq.temp_011301} and Eq.~\eqref{eq.temp_011302}, we further obtain
\begin{align*}
    \E\normTwo{\wOneEx-\DD\PP\wOneEx}^2 \geq&  \normTwo{\wOneEx}^2-2\E \normTwo{\PP\wOneEx}^2\\
    =& \left(1-\frac{2\nOne}{\pCommon+\pSpecial{1}}\right)\normTwo{\wOneEx}^2\text{ (by Lemma~\ref{le.bias})}\\
    =& \left(1 - \frac{2\nOne}{\pCommon+\pSpecial{1}}\right)\normTwo{\wOne}^2\text{ (by Eq.~\eqref{eq.def_wex})}.
\end{align*}
Therefore, we have proven the lower bound in this lemma. In summary, the result of this lemma thus holds.
\end{proof}

\subsection{Proof of Lemma~\ref{le.term_2}}\label{app.proof_term_2}
\begin{proof}
By Eq.~\eqref{eq.def_G} and Eq.~\eqref{eq.def_qex}, we have
\begin{align}\label{eq.temp_011401}
    \ZZ^T\qOne = \UU^T\qOneEx.
\end{align}
By Eq.~\eqref{eq.def_G}, we conclude
\begin{align}\label{eq.temp_011404}
    \GG^T\GG=\GG,\quad \GG\qOneEx =\GG^T\qOneEx= \qOneEx.
\end{align}
By Eq.~\eqref{eq.temp_011404} and Eq.~\eqref{eq.PP_is_proj}, we obtain
\begin{align}\label{eq.temp_011405}
    \qOneEx^T\PP^T\GG^T\GG\PP\qOneEx=\qOneEx^T\PP^T\GG\PP^T\qOneEx, \quad \qOneEx^T\GG\PP\qOneEx = \qOneEx^T\PP\qOneEx.
\end{align}
By Eq.~\eqref{eq.PP_is_proj}, we have
\begin{align}\label{eq.temp_011406}
    \qOneEx^T\PP^T\PP\qOneEx = \qOneEx^T\PP\qOneEx.
\end{align}
Define
\begin{align}
    &\bm{a}\defeq\PP\qOneEx-\frac{\langle\PP\qOneEx,\ \qOneEx\rangle}{\normTwo{\qOneEx}^2}\qOneEx,\text{ (corresponds to $\overrightarrow{\mathrm{EB}}$ in Fig.~\ref{fig.projection})}\label{eq.temp_011402}\\
    &\bm{b}\defeq\PP\qOneEx-\GG\PP\qOneEx\text{ (corresponds to $\overrightarrow{\mathrm{DB}}$ in Fig.~\ref{fig.projection})}.\label{eq.temp_011403}
\end{align}
Notice that
\begin{align*}
    &(\bm{a}-\bm{b})^T\bm{b}\\
    =&\qOneEx^T\PP^T\PP\qOneEx-\qOneEx^T\PP^T\GG\PP\qOneEx-\frac{\langle\PP\qOneEx,\ \qOneEx\rangle}{\normTwo{\qOneEx}^2}\qOneEx^T\PP\qOneEx\\
    &+\frac{\langle\PP\qOneEx,\ \qOneEx\rangle}{\normTwo{\qOneEx}^2}\qOneEx^T\GG\PP\qOneEx\\
    &+\qOneEx^T\PP^T\PP\qOneEx - 2\qOneEx^T\GG\PP\qOneEx + \qOneEx^T\PP^T\GG^T\GG\PP\qOneEx \text{ (by Eq.~\eqref{eq.temp_011402} and Eq.~\eqref{eq.temp_011403})}\\
    =&0\text{ (by Eq.~\eqref{eq.temp_011405} and Eq.~\eqref{eq.temp_011406}).}\numberthis \label{eq.temp_011407}
\end{align*}
Thus, we obtain
\begin{align*}
    &\normTwo{C}^2\\
    =&\normTwo{\DD\PP\qOneEx}^2\text{ (by Eq.~\eqref{eq.def_PP} and Eq.~\eqref{eq.temp_011401})}\\
    =&\normTwo{\PP\qOneEx-\GG\PP\qOneEx}^2\text{ (since $\GG=\iMatrix{\pCommon+\pSpecial{1}}-\DD$ by Eq.~\eqref{eq.def_G})}\\
    =&\normTwo{\bm{b}}^2\text{ (by Eq.~\eqref{eq.temp_011403})}\\
    \leq & \normTwo{\bm{a}-\bm{b}}^2 + \normTwo{\bm{b}}^2\\
    =&  \normTwo{\bm{a}-\bm{b}}^2 + \normTwo{\bm{b}}^2+2(\bm{a}-\bm{b})^T\bm{b}\text{ (by Eq.~\eqref{eq.temp_011407})}\\
    =&\normTwo{\bm{a}-\bm{b}+\bm{b}}^2\\
    =&\normTwo{\bm{a}}^2.\numberthis \label{eq.temp_011503}
\end{align*}
It remains to estimate the norm of $\bm{a}$. To that end, we have
\begin{align*}
    \normTwo{\bm{a}}^2 =& \normTwo{\PP\qOneEx-\frac{\qOneEx^T\PP\qOneEx}{\normTwo{\qOneEx}^2}\qOneEx}^2\text{ (by Eq.~\eqref{eq.temp_011402})}\\
    =&\normTwo{\PP\qOneEx}^2-\frac{\left(\qOneEx^T\PP\qOneEx\right)^2}{\normTwo{\qOneEx}^2}\text{ (expand and merge like terms)}\\
    =&\normTwo{\PP\qOneEx}^2-\frac{\normTwo{\PP\qOneEx}^4}{\normTwo{\qOneEx}^2}\text{ (by Eq.~\eqref{eq.temp_011406})}\\
    =&\normTwo{\qOneEx}^2\cdot \frac{\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2}\cdot \left(1-\frac{\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2}\right).\numberthis \label{eq.temp_011501}
\end{align*}
Since $\PP$ is a projection by Eq.~\eqref{eq.PP_is_proj}, we know $\normTwo{\qOneEx}\geq \normTwo{\PP\qOneEx}$. Then, we obtain
\begin{align}\label{eq.temp_011502}
    \frac{\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2}\leq 1,\quad \left(1-\frac{\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2}\right)\leq 1.
\end{align}
We thus conclude
\begin{align*}
    \normTwo{\bm{a}}^2\leq& \min\left\{\frac{\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2},\ 1-\frac{\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2}\right\}\normTwo{\qOneEx}^2\text{ (by Eq.~\eqref{eq.temp_011501} and Eq.~\eqref{eq.temp_011502})}.
\end{align*}
Therefore, we have
\begin{align*}
    \E\left[\normTwo{C}^2\right]\leq & \E\normTwo{\bm{a}}^2 \text{ (by Eq.~\eqref{eq.temp_011503})}\\
    \leq & \min\left\{\frac{\E\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2},\ 1-\frac{\E\normTwo{\PP\qOneEx}^2}{\normTwo{\qOneEx}^2}\right\}\normTwo{\qOneEx}^2\\
    \leq &\min\left\{\left(1-\frac{\nOne}{\pCommon+\pSpecial{1}}\right),\ \frac{\nOne}{\pCommon+\pSpecial{1}}\right\}\normTwo{\qOneEx}^2 \text{ (by Lemma~\ref{le.bias})}\\
    =& \min\left\{\left(1-\frac{\nOne}{\pCommon+\pSpecial{1}}\right),\ \frac{\nOne}{\pCommon+\pSpecial{1}}\right\}\normTwo{\qOne}^2 \text{ (by Eq.~\eqref{eq.def_qex})}.
\end{align*}
The result of this lemma thus follows.
\end{proof}
\subsection{Tightness of Eq.~\texorpdfstring{\eqref{eq.result_1}}{(8)}}
% The impact of noise $\eOne$ to $\EwOneDiff$ is characterized by $b_{\text{noise}}$ and is precise. It remains to discuss the tightness of the bias estimation, i.e., the impact of $\wOne$, $\wTwo$, and $\qOne$. 



All our estimations except $\EwOneDiffNoNoise$ are precise. 
\begin{figure}[t]
    \centering
    \includegraphics[width=0.8\textwidth]{figs/tightness.eps}
    \caption{The curves of $\EwOneDiffNoNoise$ with respect to $\pCommon$ in the overparameterized region, where $\nOne=100$, $\normTwo{\wOne}=1$, and $\pSpecial{1}=20$. Each data point is the average of $20$ runs with different random seeds (randomness is on input features). Some other settings for each subfigure: (a) $\wOne=\wTwo$ and $\normTwo{\qOne}=1$; (b) $\wOne=-\wTwo$, and $\normTwo{\qOne}=1$; (c) $\wOne=\wTwo$ and $\normTwo{\qOne}=5$; (d) $\wOne=-\wTwo$, and $\normTwo{\qOne}=5$.}\label{fig.tightness}
\end{figure}
It remains to check the tightness of \cref{eq.result_1}, i.e.,  $\min_{i=1,2,3}\overline{b}_i^2$. Here  $\overline{b}_1$, $\overline{b}_2$, and $\overline{b}_3$ are for different situations. 

First, when $\oRatio$ and $\similarity$ are small, $\overline{b}_1$ is the smallest. In this case, $\overline{b}_1\leq \overline{b}_3$ because $\sqrt{\oRatio\left(\normTwo{\wOne}^2 + \normTwo{\qOne}^2\right)}\leq \sqrt{\oRatio}\normTwo{\wOne}+\sqrt{r}\normTwo{\qOne}$; and $\overline{b}_1\leq \overline{b}_2$ because $\oRatio=\normTwo{\wTwo-\wOne}\leq \normTwo{\wTwo}+\normTwo{\wOne}\approx \normTwo{\wTwo}+\normTwo{\wOne}$ when $\oRatio$ is small. In \cref{fig.tightness}(a)(c) (where $\similarity=0$), the curves of $\overline{b}_1^2$ (blue dashed curves) are the closest ones to the empirical value of $\EwOneDiffNoNoise$ (black curves with markers ``$+$'') in the near overparameterized regime.

Second, if only $\similarity$ is small but $\oRatio$ is large, then $\overline{b}_3$ is the smallest. In this case, $\overline{b}_3\leq \overline{b}_1$ because the coefficient of $\normTwo{\qOne}$ in $\overline{b}_3$ approaches $0$ when $\oRatio\to 1$ while that in $\overline{b}_1$ approaches $1$. For this reason, the difference between $\overline{b}_3$ and $\overline{b}_1$ can be observed more easily when $\normTwo{\qOne}$ is large. In \cref{fig.tightness}(a)(c), we can see that the curves of $\overline{b}_3^2$ are the closest ones to the actual value of $\EwOneDiffNoNoise$ when $\pCommon$ is large (i.e., $\oRatio$ is large). As expected, the difference between $\overline{b}_3$ and $\overline{b}_1$ in \cref{fig.tightness}(b) is larger and easier to observe compared to that in \cref{fig.tightness}(a).

Third, when $\similarity$ is large, $\overline{b}_2$ can sometimes be the smallest since $\wTwo$ and $\wOne$ are treated separately in the expression of $\overline{b}_2$ to get a more precise estimation. In \cref{fig.tightness}(b)(d) (where $\similarity$ is very large), the curves of $\overline{b}_2^2$ (red dashed ones) are the closest ones to the curve of $\EwOneDiffNoNoise$ (black curves with the markers ``$+$'').




\section{Proof of Theorem~\ref{thm.target_task_err}}\label{app.proof_target_task_err}
% {\color{red}Specify $\E$ meaning.}
\begin{proof}
When $\pSpecial{2}>\nTwo +1$, we have
\begin{align*}
    \qTilde = &\ZZZ(\ZZZ^T\ZZZ)^{-1}\left(\yTwo-\XXX^T\wTilde\right)\\
    =&\ZZZ(\ZZZ^T\ZZZ)^{-1}\left(\XXX^T\left(\wTwo - \wTilde\right)+\ZZZ^T\qTwo+\eTwo\right).
\end{align*}
Define
\begin{align}
    &\bm{a}\defeq\left(\iMatrix{\pSpecial{2}} - \myProj{\ZZZ}\right)\qTwo \in \mathds{R}^{\pSpecial{2}},\label{eq.temp_010201}\\
    &\bm{b}\defeq\left( \ZZZ(\ZZZ^T\ZZZ)^{-1}\left(\XXX^T\left(\wTwo - \wTilde\right)+\eTwo\right)\right)\in \mathds{R}^{\pSpecial{2}}.\label{eq.temp_010601}
\end{align}
Thus, we obtain
\begin{align}\label{eq.temp_010103}
    \qTwo-\qTilde = \bm{a}+\bm{b}.
\end{align}
By Assumption~\ref{as.Gaussian}, we conclude
\begin{align}\label{eq.temp_010101}
    &\XXX^T(\wTwo - \wTilde)+\eTwo\sim \mathcal{N}\left(\bm{0},\ \left(\normTwo{\wTwo - \wTilde}^2+\sigmaTwo^2\right)\iMatrix{\nTwo}\right).
\end{align}
Thus, we have
\begin{align*}
    \E_{\XXX,\eTwo}[\bm{b}] =& \E_{\XXX,\eTwo}\left[\left( \ZZZ(\ZZZ^T\ZZZ)^{-1}\left(\XXX^T\left(\wTwo - \wTilde\right)+\eTwo\right)\right)\right]\text{ (by Eq.~\ref{eq.temp_010601})}\\
    =& \ZZZ(\ZZZ^T\ZZZ)^{-1}\E_{\XXX,\eTwo}\left[\left( \left(\XXX^T\left(\wTwo - \wTilde\right)+\eTwo\right)\right)\right]\\
    &\text{ (since $\XXX$, $\ZZZ$, and $\eTwo$ are independent by Assumption~\ref{as.Gaussian})}\\
    = &\bm{0}\text{ (by Eq.~\eqref{eq.temp_010101}).}
\end{align*}
We therefore obtain
\begin{align}\label{eq.temp_010102}
    \E_{\ZZZ,\XXX,\eTwo}[\bm{a}^T\bm{b}] = \E_{\ZZZ}\left[\bm{a}^T \E_{\XXX,\eTwo}[\bm{b}]\right] = 0.
\end{align}
By Eq.~\eqref{eq.temp_010103} and Eq.~\eqref{eq.temp_010102}, we have
\begin{align*}
    &\E\normTwo{\qTwo-\qTilde}^2 \\
    =& \E\normTwo{\bm{a}}^2+\E\normTwo{\bm{b}}^2\\
    =&\left(1-\frac{\nTwo}{\pSpecial{2}}\right)\normTwo{\qTwo}^2+\E\normTwo{\bm{b}}^2\text{ (by Eq.~\eqref{eq.temp_010201} and Lemma~\ref{le.bias})}\\
    =&\left(1-\frac{\nTwo}{\pSpecial{2}}\right)\normTwo{\qTwo}^2+\frac{\nTwo\left(\normTwo{\wTwo - \wTilde}^2+\sigmaTwo^2\right)}{\pSpecial{2}-\nTwo-1}\text{ (by Eq.~\eqref{eq.temp_010101} and  Lemma~\ref{le.IW})}\numberthis \label{eq.temp_010602}
\end{align*}
When $\pSpecial{2}+1< \nTwo$, we obtain
\begin{align*}
    \qTilde =& \left(\ZZZ\ZZZ^T\right)^{-1}\ZZZ\left(\yTwo - \XXX^T\wTilde\right)\\
    =& \left(\ZZZ\ZZZ^T\right)^{-1}\ZZZ \left(\XXX^T\left(\wTwo - \wTilde\right)+\ZZZ^T\qTwo +\eTwo\right)\\
    =& \qTwo +\left(\ZZZ\ZZZ^T\right)^{-1}\ZZZ \left(\XXX^T\left(\wTwo - \wTilde\right)+\eTwo\right),
\end{align*}
and therefore we have
\begin{align*}
    \E\normTwo{\qTwo - \qTilde}^2=&\E\normTwo{\left(\ZZZ\ZZZ^T\right)^{-1}\ZZZ \left(\XXX^T\left(\wTwo - \wTilde\right)+\eTwo\right)}^2\\
    =&\frac{\pSpecial{2}\left(\normTwo{\wTwo - \wTilde}^2+\sigmaTwo^2\right)}{\nTwo-\pSpecial{2}-1}\text{ (by Eq.~\eqref{eq.temp_010101} and Lemma~\ref{le.IW}).}\numberthis \label{eq.temp_010603}
\end{align*}
By Eq.~\eqref{eq.temp_010602} and Eq.~\eqref{eq.temp_010603}, the result of this proposition thus follows.
\end{proof}
\section{Proof of Theorem~\ref{thm.opt2}}\label{app.proof_opt2}
% {\color{red}Specify $\E$ meaning.}

Define
\begin{align*}
    \UUU\defeq \myMatrix{\XXX\\\ZZZ}\in \mathds{R}^{(\pCommon+\pSpecial{2})\times \nTwo}.
\end{align*}
\subsection{Overfitted situation}
We first consider the overfitted solution, i.e., $\pCommon+\pSpecial{2}>\nTwo$. The overfitted solution that corresponds to the convergence of GD/SGD is given by
\begin{align*}
    \min_{\wHatTwo,\qHat} \quad & \normTwo{\wHatTwo-\wTilde}^2 + \normTwo{\qHat}^2\numberthis \label{op.option_2_solution}\\
    \text{subject to}\quad & \yTwo = \XXX^T\wHatTwo +\ZZZ^T\qHat.
\end{align*}

The solution of \eqref{op.option_2_solution} is as follows:
\begin{align*}
    \left[\begin{smallmatrix}\wTildeTwo-\wTilde\\
    \qTilde\end{smallmatrix}\right]=&\UUU(\UUU^T\UUU)^{-1}\left(\yTwo-\XXX^T\wTilde\right)\\
    =&\UUU(\UUU^T\UUU)^{-1}\left(\XXX^T(\wTwo-\wTilde)+\ZZZ^T\qTwo+\eTwo\right)\text{ (by Eq.~\eqref{eq.ground_truth_matrix})}\\
    =&\UUU(\UUU^T\UUU)^{-1}\eTwo + \UUU(\UUU^T\UUU)^{-1}\UUU^T\left[\begin{smallmatrix}\wTwo-\wTilde\\
    \qTilde\end{smallmatrix}\right].\numberthis \label{eq.temp_011601}
\end{align*}
By Assumption~\ref{as.Gaussian}, we know that $\eTwo$ has zero mean and is independent of $\UUU$. Thus, we have
\begin{align}\label{eq.temp_011602}
    \left\langle\left(\iMatrix{\pCommon+\pSpecial{2}}-\UUU(\UUU^T\UUU)^{-1}\UUU^T\right)\left[\begin{smallmatrix}\wTwo-\wTilde\\
    \qTwo\end{smallmatrix}\right],\ \UUU(\UUU^T\UUU)^{-1}\eTwo\right\rangle = 0.
\end{align}
We therefore obtain
\begin{align*}
    &\E\left[\normTwo{\wTwo -\wTildeTwo}^2 + \normTwo{\qTwo - \qTilde}^2\right]\\
    =&\E\normTwo{\left[\begin{smallmatrix}\wTwo-\wTilde\\
    \qTwo\end{smallmatrix}\right]-\left[\begin{smallmatrix}\wTildeTwo-\wTilde\\
    \qTilde\end{smallmatrix}\right]}^2\\
    =& \E \normTwo{\left(\iMatrix{\pCommon+\pSpecial{2}}-\UUU(\UUU^T\UUU)^{-1}\UUU^T\right)\left[\begin{smallmatrix}\wTwo-\wTilde\\
    \qTwo\end{smallmatrix}\right]+\UUU(\UUU^T\UUU)^{-1}\eTwo}^2\text{ (by Eq.~\eqref{eq.temp_011601})}\\
    =&\E \normTwo{\left(\iMatrix{\pCommon+\pSpecial{2}}-\UUU(\UUU^T\UUU)^{-1}\UUU^T\right)\left[\begin{smallmatrix}\wTwo-\wTilde\\
    \qTwo\end{smallmatrix}\right]}^2+\E\normTwo{\UUU(\UUU^T\UUU)^{-1}\eTwo}^2\text{ (by Eq.~\eqref{eq.temp_011602})}\\
    =&\left(1-\frac{\nTwo}{\pCommon+\pSpecial{2}}\right)\normTwo{\begin{smallmatrix}\wTwo-\wTilde\\
    \qTwo\end{smallmatrix}}^2+\frac{\nTwo\sigmaTwo^2}{\pCommon+\pSpecial{2}-\nTwo-1}\text{ (by Lemma~\ref{le.bias} and Lemma~\ref{le.IW})}\\
    =& \left(1-\frac{\nTwo}{\pCommon+\pSpecial{2}}\right)\left(\normTwo{\wTwo-\wTilde}^2+\normTwo{\qTwo}^2\right)+\frac{\nTwo\sigmaTwo^2}{\pCommon+\pSpecial{2}-\nTwo -1}.
\end{align*}

\subsection{Underparameterized situation}

The solution that minimizes the training loss is given by
\begin{align*}
    \left[\begin{smallmatrix}\wTildeTwo\\
    \qTilde\end{smallmatrix}\right]=&(\UUU\UUU^T)^{-1}\UUU\yTwo\\
    =&(\UUU\UUU^T)^{-1}\UUU\left(\UUU^T\left[\begin{smallmatrix}\wTwo\\
    \qTwo\end{smallmatrix}\right]+\eTwo\right)\text{ (by Eq.~\eqref{eq.ground_truth_matrix})}\\
    =&\left[\begin{smallmatrix}\wTwo\\
    \qTwo\end{smallmatrix}\right] + (\UUU\UUU^T)^{-1}\UUU\eTwo.
\end{align*}
Thus, we have
\begin{align*}
    \E \left[\normTwo{\wTwo -\wTildeTwo}^2 + \normTwo{\qTwo - \qTilde}^2\right] = &\normTwo{(\UUU\UUU^T)^{-1}\UUU\eTwo}^2\\
    =&\frac{(\pCommon+\pSpecial{2})\sigmaTwo^2}{\nTwo-(\pCommon+\pSpecial{2})-1}\text{ (by Lemma~\ref{le.IW})}.
\end{align*}

\section{Proof of Proposition~\ref{prop.best_p}}\label{app.proof_best_p}
\begin{proof}
We use $[\cdot]_i$ to denote the $i$-th element of a vector.
Then we have
\begin{align*}
    &\normTwo{\wTwo-\wTilde}^2 \\
    =& \sum_{i=1}^p \left([\wTwo]_i - [\wTilde]_i\right)^2\ \text{ (since $\wTwo,\wTilde\in \mathds{R}^{\pCommon}$)}\\
    =& \sum_{i\in \commonSpace} \left([\wTwo]_i-[\wTilde]_i\right)^2 + \sum_{i\in \commonSpaceSelected\setminus\commonSpace}[\wTilde]_i^2\ \text{ (by the definition of $\wTwo$)}.\numberthis \label{eq.temp_041701}
\end{align*}
We thus obtain
\begin{align*}
    \EwOneDiff=&\E \normTwo{\wTwo-\wTilde}^2\ \text{ (by \cref{eq.def_Lco})}\\
    =&\E \sum_{i\in \commonSpace} \left([\wTwo]_i-[\wTilde]_i\right)^2 + \E\sum_{i\in \commonSpaceSelected\setminus\commonSpace}[\wTilde]_i^2 \ \text{ (by \cref{eq.temp_041701})}.
\end{align*}
Since $\pCommon+\pSpecial{1}=C$ is fixed and all features are \emph{i.i.d.} Gaussian (\cref{as.Gaussian}), the distribution of $\myMa{\wTilde\\\qTildeOne}\in \mathds{R}^C$ is the same for different $\pCommon$. 
Notice that since \cref{as.no_missing_features} is assured, the term $\E \sum_{i\in \commonSpace} \left([\wTwo]_i-[\wTilde]_i\right)^2$ does not change with respect to $\pCommon$.
In contrast, the term $\E \sum\limits_{i\in \commonSpaceSelected\setminus\commonSpace}[\wTilde]_i^2$ is monotone increasing with respect to $\pCommon$ (since $\commonSpaceSelected\setminus\commonSpace$includes more fake features when  $\pCommon$ is larger). The result of \cref{prop.best_p} thus follows.
\end{proof}
\begin{thebibliography}{10}

\bibitem{arora2019fine}
Sanjeev Arora, Simon Du, Wei Hu, Zhiyuan Li, and Ruosong Wang.
\newblock Fine-grained analysis of optimization and generalization for
  overparameterized two-layer neural networks.
\newblock In {\em International Conference on Machine Learning}, pages
  322--332, 2019.

\bibitem{bartlett2019benign}
Peter~L Bartlett, Philip~M Long, G{\'a}bor Lugosi, and Alexander Tsigler.
\newblock Benign overfitting in linear regression.
\newblock {\em Proceedings of the National Academy of Sciences}, 2020.

\bibitem{bastani2021predicting}
Hamsa Bastani.
\newblock Predicting with proxies: Transfer learning in high dimension.
\newblock {\em Management Science}, 67(5):2964--2984, 2021.

\bibitem{belkin2019two}
Mikhail Belkin, Daniel Hsu, and Ji~Xu.
\newblock Two models of double descent for weak features.
\newblock {\em arXiv preprint arXiv:1903.07571}, 2019.

\bibitem{belkin2020two}
Mikhail Belkin, Daniel Hsu, and Ji~Xu.
\newblock Two models of double descent for weak features.
\newblock {\em SIAM Journal on Mathematics of Data Science}, 2(4):1167--1180,
  2020.

\bibitem{belkin2018understand}
Mikhail Belkin, Siyuan Ma, and Soumik Mandal.
\newblock To understand deep learning we need to understand kernel learning.
\newblock In {\em International Conference on Machine Learning}, pages
  541--549, 2018.

\bibitem{dar2022double}
Yehuda Dar and Richard~G Baraniuk.
\newblock Double double descent: on generalization errors in transfer learning
  between linear regression tasks.
\newblock {\em SIAM Journal on Mathematics of Data Science}, 4(4):1447--1472,
  2022.

\bibitem{dhifallah2021phase}
Oussama Dhifallah and Yue~M Lu.
\newblock Phase transitions in transfer learning for high-dimensional
  perceptrons.
\newblock {\em Entropy}, 23(4):400, 2021.

\bibitem{gerace2022probing}
Federica Gerace, Luca Saglietti, Stefano~Sarao Mannelli, Andrew Saxe, and Lenka
  Zdeborov{\'a}.
\newblock Probing transfer learning with a model of synthetic correlated
  datasets.
\newblock {\em Machine Learning: Science and Technology}, 3(1):015030, 2022.

\bibitem{hastie2019surprises}
Trevor Hastie, Andrea Montanari, Saharon Rosset, and Ryan~J Tibshirani.
\newblock Surprises in high-dimensional ridgeless least squares interpolation.
\newblock {\em arXiv preprint arXiv:1903.08560}, 2019.

\bibitem{jutheoretical}
Peizhong Ju, Yingbin Liang, and Ness Shroff.
\newblock Theoretical characterization of the generalization performance of
  overfitted meta-learning.
\newblock In {\em The Eleventh International Conference on Learning
  Representations}, 2023.

\bibitem{ju2020overfitting}
Peizhong Ju, Xiaojun Lin, and Jia Liu.
\newblock Overfitting can be harmless for basis pursuit, but only to a degree.
\newblock {\em Advances in Neural Information Processing Systems},
  33:7956--7967, 2020.

\bibitem{ju2021generalization}
Peizhong Ju, Xiaojun Lin, and Ness~B Shroff.
\newblock On the generalization power of overfitted two-layer neural tangent
  kernel models.
\newblock {\em arXiv preprint arXiv:2103.05243}, 2021.

\bibitem{ju2022generalization}
Peizhong Ju, Xiaojun Lin, and Ness~B Shroff.
\newblock On the generalization power of the overfitted three-layer neural
  tangent kernel model.
\newblock {\em arXiv preprint arXiv:2206.02047}, 2022.

\bibitem{lampinen2018an}
Andrew~K. Lampinen and Surya Ganguli.
\newblock An analytic theory of generalization dynamics and transfer learning
  in deep linear networks.
\newblock In {\em International Conference on Learning Representations}, 2019.

\bibitem{laurent2000adaptive}
Beatrice Laurent and Pascal Massart.
\newblock Adaptive estimation of a quadratic functional by model selection.
\newblock {\em Annals of Statistics}, pages 1302--1338, 2000.

\bibitem{li2022transfer}
Sai Li, T~Tony Cai, and Hongzhe Li.
\newblock Transfer learning for high-dimensional linear regression: Prediction,
  estimation and minimax optimality.
\newblock {\em Journal of the Royal Statistical Society Series B: Statistical
  Methodology}, 84(1):149--173, 2022.

\bibitem{li2023estimation}
Sai Li, Linjun Zhang, T~Tony Cai, and Hongzhe Li.
\newblock Estimation and inference for high-dimensional generalized linear
  models with knowledge transfer.
\newblock {\em Journal of the American Statistical Association}, pages 1--12,
  2023.

\bibitem{lin2022transfer}
Haotian Lin and Matthew Reimherr.
\newblock Transfer learning for functional linear regression with structural
  interpretability.
\newblock {\em arXiv preprint arXiv:2206.04277}, 2022.

\bibitem{mardia1979jt}
KV~Mardia.
\newblock Jt kent. and j. m. bibby.
\newblock {\em Multivariate Analysis}, 1979.

\bibitem{mei2019generalization}
Song Mei and Andrea Montanari.
\newblock The generalization error of random features regression: Precise
  asymptotics and double descent curve.
\newblock {\em arXiv preprint arXiv:1908.05355}, 2019.

\bibitem{mitra2019understanding}
Partha~P Mitra.
\newblock Understanding overfitting peaks in generalization error: Analytical
  risk curves for $ l_2 $ and $ l_1 $ penalized interpolation.
\newblock {\em arXiv preprint arXiv:1906.03667}, 2019.

\bibitem{muthukumar2019harmless}
Vidya Muthukumar, Kailas Vodrahalli, and Anant Sahai.
\newblock Harmless interpolation of noisy data in regression.
\newblock In {\em 2019 IEEE International Symposium on Information Theory
  (ISIT)}, pages 2299--2303. IEEE, 2019.

\bibitem{pan2010survey}
Sinno~Jialin Pan and Qiang Yang.
\newblock A survey on transfer learning.
\newblock {\em IEEE Transactions on knowledge and data engineering},
  22(10):1345--1359, 2010.

\bibitem{satpathi2021dynamics}
Siddhartha Satpathi and R~Srikant.
\newblock The dynamics of gradient descent for overparametrized neural
  networks.
\newblock In {\em Learning for Dynamics and Control}, pages 373--384. PMLR,
  2021.

\bibitem{tian2022transfer}
Ye~Tian and Yang Feng.
\newblock Transfer learning under high-dimensional generalized linear models.
\newblock {\em Journal of the American Statistical Association}, pages 1--14,
  2022.

\bibitem{tripuraneni2020theory}
Nilesh Tripuraneni, Michael Jordan, and Chi Jin.
\newblock On the theory of transfer learning: The importance of task diversity.
\newblock {\em Advances in neural information processing systems},
  33:7852--7862, 2020.

\bibitem{vershynin2010introduction}
Roman Vershynin.
\newblock Introduction to the non-asymptotic analysis of random matrices.
\newblock {\em arXiv preprint arXiv:1011.3027}, 2010.

\bibitem{zhang2022class}
Xuhui Zhang, Jose Blanchet, Soumyadip Ghosh, and Mark~S Squillante.
\newblock A class of geometric structures in transfer learning: Minimax bounds
  and optimality.
\newblock In {\em International Conference on Artificial Intelligence and
  Statistics}, pages 3794--3820. PMLR, 2022.

\end{thebibliography}