Toward Efficient Online Scheduling for Distributed Machine Learning Systems
Extracted source text ยท arXiv 2108.02917
% !TEX root = ../ML_Networking_TNSE20.tex
\begin{abstract}
%Thanks to the rise of machine learning (ML) and its wide range of applications,
Recent years have witnessed a rapid growth of distributed machine learning (ML) frameworks, which exploit the massive parallelism of computing clusters to expedite ML training.
However, the proliferation of distributed ML frameworks also introduces many unique technical challenges in computing system design and optimization.
In a networked computing cluster that supports a large number of training jobs, a key question is how to design efficient scheduling algorithms to allocate workers and parameter servers across different machines to minimize the overall training time.
Toward this end, in this paper, we develop an online scheduling algorithm that jointly optimizes resource allocation and locality decisions.
Our main contributions are three-fold: i) We develop a new analytical model that considers both resource allocation and locality;
ii) Based on an equivalent reformulation and observations on the worker-parameter server locality configurations, we transform the problem into a mixed packing and covering integer program, which enables approximation algorithm design;
iii) We propose a meticulously designed approximation algorithm based on randomized rounding and rigorously analyze its performance.
Collectively, our results contribute to the state of the art of distributed ML system optimization and algorithm design.
\end{abstract}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Proof of Lemma~\ref{lem_ILP}} \label{appdx:lem_ILP1}
\begin{proof}
Consider the probabilities of the following ``bad'' events: 1) $\c^{\top} \hat{\x} > \frac{3G_\delta}{\delta} \c^{\top}\bar{\x}$; 2) $\exists i$ such that $(\A\hat{\x})_{i} < a_{i}$; and 3) $\exists i$ such that $(\B\hat{\x})_{i} > b_{i}$.
Note that events 2) and 3) can be equivalently rewritten as: 2') $\exists i$ such that $\mathbb{E}\{(\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} < W_{a}\}$ and 3') $\exists i$ such that $\mathbb{E}\{(\B\hat{\x})_{i} \frac{W_{b}}{b_{i}} > W_{b}\}$.
Since $\mathbb{E}\{ \hat{\x} \} \!=\! \x' \!=\! G_\delta\bar{\x}$, by linearity of expectation, we have:
\begin{align}
\label{eqn_exp1}&\mathbb{E}\{\c^{\top} \hat{\x} \} = \c^{\top} \mathbb{E}\{ \hat{\x} \} = \c^{\top} G_\delta \bar{\x} = G_\delta \c^{\top}\bar{\x}, \\
%
\label{eqn_exp2}& \mathbb{E}\left\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \right\} = G_\delta \mathbb{E} \left\{ (\A\bar{\x})_{i}\frac{W_{a}}{a_{i}} \right\} \geq G_\delta W_{a}, \\
%
\label{eqn_exp3}& \mathbb{E}\left\{ (\B\hat{\x})_{i} \frac{W_{b}}{b_{i}} \right\} = G_\delta \mathbb{E} \left\{ (\B\bar{\x})_{i}\frac{W_{b}}{b_{i}} \right\} \leq G_\delta W_{b}.
\end{align}
Then, by the Markov inequality and ~(\ref{eqn_exp1}), we can obtain the probability $
\mathrm{Pr} \left\{ \c^{\top} \hat{\x} > \frac{3G_\delta}{\delta} \c^{\top} \bar{\x} \right\} \leq \frac{\delta}{3}$.
%
Next, we note that each $\hat{x}_{j}$ can be viewed as a sum of independent random variables in $[0,1]$ as follows:
The fixed part of $\lfloor x'_{j} \rfloor$ is a sum of $\lfloor x'_{j} \rfloor$ random variables with value 1 with probability 1.
Then, we have that $(\B\hat{\x})_{i} \frac{W_b}{b_{i}} = (\sum_{j}[\B]_{ij}\hat{x}_{j}) \frac{W_b}{b_{i}}$ is also a sum of independent random variables in $[0,1]$.
Using the Chernoff bound, we have
\begin{align*}
\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} \frac{W_{b}}{b_{i}} > (1+\epsilon) G_\delta W_{b} \Big\} \leq \exp \Big( -\epsilon^{2} \frac{G_\delta W_{b}}{3} \Big).
\end{align*}
%Using Chernoff bound, we have $\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq (1-\epsilon) GW_{a} \} \leq \exp(-\epsilon^{2}\frac{GW_{a}}{2} )$.
Setting $(1+\epsilon)G_\delta \!=\!1$, i.e., $\epsilon \!=\! \frac{1}{G_\delta}\!-\!1$, we have:
\begin{align} \label{eqn_chernoff_bnd1_1}
%\mathrm{Pr} \left\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq W_{a} \right\} \leq \exp\left(- \Big(1-\frac{1}{G}\Big)^{2} G \frac{W_{a}}{2} \right).
\mathrm{Pr} \left\{ (\B\hat{\x})_{i} \frac{W_b}{b_{i}} > W_b \right\} \leq \exp\left(- \Big(\frac{1}{G_\delta}-1\Big)^{2} G_\delta \frac{W_b}{3} \right). \!\!\!
\end{align}
Forcing $\exp\!\left(- (\frac{1}{G_\delta}\!-\!1)^{2} G_\delta \frac{W_b}{3} \right) \!=\! \frac{\delta}{3r}$ and solving $G_\delta$, we have:
\begin{align} \label{eqn_L_def_1}
G_\delta \triangleq 1 + \frac{3\ln(3r/\delta)}{2W_b}\! -\!\sqrt{ \bigg(\frac{3\ln(3r/\delta)}{2W_b} \bigg)^{2} + \frac{3\ln(3r/\delta)}{W_b} }. \!\!\!
\end{align}
Using (\ref{eqn_exp2}), the Chernoff bound, and following similar arguments, we have:
\begin{align*}
\mathrm{Pr} \Big\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq (1-\epsilon) G_\delta W_{a} \Big\} \leq \exp \Big(-\epsilon^{2}\frac{G_\delta W_{a}}{2} \Big).
\end{align*}
Forcing $\exp\left( -\epsilon^{2} \frac{G_\delta W_a}{2} \right)=\frac{\delta}{3m}$ and solving for $\epsilon$, we have $\epsilon = (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$.
It follows that
\begin{align*}
\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_a}{a_{i}} \leq ( 1 \!-\! (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}} )G_\delta W_a\} \leq \frac{\delta}{3m},
\end{align*}
which implies that:
\begin{align} \label{eqn_chernoff_bnd2_1}
\!\!\! \mathrm{Pr} \left\{ (\A\hat{\x})_{i} \!\leq\! a_i \left( 1 \!-\! \sqrt{\frac{2}{G_\delta W_a}\ln\Big(\frac{3m}{\delta}\Big)} \right)G_\delta, \exists i \right\} \leq \frac{\delta}{3m}. \!\!\!\!\!
\end{align}
By using union bound and (\ref{eqn_chernoff_bnd1_1}) and (\ref{eqn_chernoff_bnd2_1}), we have that events 1)--3) occur with probability less than $\frac{\delta}{3} + m\cdot\frac{\delta}{3m} + r\cdot\frac{\delta}{3r} = \delta$,
and the proof is complete.
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Proof of Lemma~\ref{lem_ILP2}} \label{appdx:lem_ILP2}
\begin{proof}
% {\em 2) When $G>1$: }
Similar to the case when $0<G_\delta\leq 1$, we can have the expectation equations for the bad cases as in Eqns.~(\ref{eqn_exp1})-(\ref{eqn_exp3}).
We also can view each $\hat{x}_j$ as a sum of independent random variables in $[0,1]$ in the same way.
Then, we have that $(\A\hat{\x})_{i} \frac{W_a}{a_{i}} = (\sum_{j}[\A]_{ij}\hat{x}_{j}) \frac{W_a}{a_{i}}$ is also a sum of independent random variables in $[0,1]$.
% Using Chernoff bound, we have $\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_a{a_i} \leq (1-\epsilon) GW_a \}
% \leq \exp( -\epsilon^{2} \frac{GW_a}{2} )$.
Using the Chernoff bound, we have that:
\begin{align*}
\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq (1-\epsilon) G_\delta W_{a} \} \leq \exp(-\epsilon^{2}\frac{G_\delta W_{a}}{2} ).
\end{align*}
Setting $(1-\epsilon)G_\delta \!=\!1$, i.e., $\epsilon = 1-\frac{1}{G_\delta}$, we have:
\begin{align} \label{eqn_chernoff_bnd1}
\mathrm{Pr} \left\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq W_{a} \right\} \leq \exp\left(- \Big(1-\frac{1}{G_\delta}\Big)^{2} G_\delta \frac{W_{a}}{2} \right). \!\!\!
% \mathrm{Pr} \left\{ (\B\hat{\x})_{i} \frac{W_b}{b_{i}} > W_b \right\} \leq \exp\left(- \Big(\frac{1}{G}-1\Big)^{2} G \frac{W_b}{3} \right).
\end{align}
Forcing $\exp\!\left(- (1-\frac{1}{G_\delta})^{2} G_\delta \frac{W_a}{2} \right) \!=\! \frac{\delta}{3m}$ and solving $G_\delta$, we have:
\begin{align} \label{eqn_L_def}
G_\delta \triangleq 1 + \frac{\ln(3m/\delta)}{W_a}+\sqrt{ \bigg(\frac{\ln(3m/\delta)}{W_a} \bigg)^{2} + \frac{2\ln(3m/\delta)}{W_a} }. \!\!\!
\end{align}
Using (\ref{eqn_exp2}), the Chernoff bound, and following similar arguments, we have:
\begin{align*}
\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} \frac{W_b}{b_i} > (1+\epsilon) G_\delta W_b \Big\} \leq \exp \Big( -\epsilon^{2}\frac{G_\delta W_b}{3} \Big).
\end{align*}
Forcing $\exp\left( -\epsilon^{2} \frac{G_\delta W_b}{3} \right) = \frac{\delta}{3r}$ and solving for $\epsilon$, we have $\epsilon = (\frac{3}{G_\delta W_b}\ln(\frac{3r}{\delta}))^{\frac{1}{2}}$.
It follows that
\begin{align*}
\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} \frac{W_b}{b_i} > ( 1 + (\frac{3}{G_\delta W_b}\ln(\frac{3r}{\delta}))^{\frac{1}{2}} )G_\delta W_b \Big\} \leq \frac{\delta}{3r}, \end{align*}
which implies that:
\begin{align} \label{eqn_chernoff_bnd2}
\!\!\! \mathrm{Pr} \left\{ (\B\hat{\x})_{i} > b_i \left( 1+ \sqrt{\frac{3}{G_\delta W_b}\ln\Big(\frac{3r}{\delta}\Big)} \right)G_\delta, \exists i \right\} \leq \frac{\delta}{3r}. \!\!\!\!\!
\end{align}
By using union bound and (\ref{eqn_chernoff_bnd1}) and (\ref{eqn_chernoff_bnd2}), we have that events 1)--3) occur with probability less than $\frac{\delta}{3} + m\cdot\frac{\delta}{3m} + r\cdot\frac{\delta}{3r} = \delta$,
and the proof is complete.
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Proof of the Competitive Ratio} \label{appdx:thm_Competitive}
We use $OPT$ as the optimal objective value of Problem~R-DMLRS, which is also the optimum to Problem~D-R-DMLRS.
We let $\hat{\pi}_{i}$ denote the approximate schedule obtained by Algorithm~2, which inexactly solves Problem D-R-DMLRS.
Let $P_{i}$ and $D_{i}$ be the primal and dual objective values of Problems R-DMLRS and D-RMLRS after determining the schedule $\hat{\pi}_{i}$ in Algorithm~1.
Let $P_0$ and $D_0$ be the initial values of Problems R-DMLRS and D-RMLRS, respectively, where $P_0=0$ and $D_0=\sum_{t\in\mathcal{T}}\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}P_h^r[0]C_h^r$.
We also let $P_I$ and $D_i$ be the final primal and dual objective values returned by Algorithm~1.
We present our main result in Lemma~\ref{lem:result}.
\begin{lem}
\label{lem:result}
If there exists constants $\epsilon\geq 1$, $G_\delta> 0$ and $\delta\in(0,1]$ such that $P_i-P_{i-1}\geq \frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}), \forall i\in\mathcal{I}$, and if $P_0=0$ and $D_0\leq \frac{1}{2}OPT$, then Algorithm~1 is $\frac{6G_\delta\epsilon}{\delta}$-competitive.
\end{lem}
\begin{proof}[Proof of Lemma~\ref{lem:result}]
Since $P_I=\sum\limits_{i\in\mathcal{I}}(P_i-P_{i-1})$, and $D_I-D_0=\sum\limits_{i\in\mathcal{I}}(D_i-D_{i-1})$, we can have:
\begin{align*}
P_I&=\sum_{i\in\mathcal{I}}(P_i-P_{i-1})\geq \frac{\delta/3G_\delta}{\epsilon}\sum_{i\in\mathcal{I}}(D_i-D_{i-1})\\
&=\frac{\delta/3G_\delta}{\epsilon}(D_I-D_0).
\end{align*}
By weak duality theorem~\cite{Bertsekas99:Nonlinear}, we have
\begin{align*}
D_I\geq OPT\geq P_I.
\end{align*}
Thus, it follows that:
\begin{align*}
&D_I-D_0\geq \frac{1}{2}OPT,\\
&P_I\geq \frac{\delta/3G_\delta}{\epsilon}(D_I-D_0)\geq \frac{\delta/3G_\delta}{2\epsilon}OPT,
\end{align*}
and the proof is complete.
\end{proof}
Next, following similar arguments in~\cite{Buchbinder09,Bao18:ML_INFOCOM},
we introduce the relationship between the cost and resource consumption before and after processing one job.
Let $p_h^{r,i}[t]$ be the unit cost of type-$r$ resource on server $h$ at time $t$ after handling job $i$.
Let $\rho_h^{r,i}[t]$ be the amount of type-$r$ resource allocated to jobs on server $h$ at time $t$ after processing the job $i$.
For ease of our subsequent analysis, we now define the following allocation-cost relation that is implied by Algorithm~1:
\begin{defn} \label{defn:alloc_cost}
The allocation-cost relationship for Algorithm~1 with $\epsilon>1, G_\delta> 0$ and $\delta\in(0,1]$ is
\begin{align*}
&p_h^{r,i-1}[t](\rho_h^{r,i}[t]-\rho_h^{r,i-1}[t])\geq \frac{\delta/3G_\delta C_h^r}{\epsilon}(p_h^{r,i}[t]-p_h^{r,i-1}[t]),\\
&\hspace{2in}\forall i\in\mathcal{I},h\in\mathcal{H},r\in\mathcal{R}.
\end{align*}
\end{defn}
The allocation-cost relationship shows that the cost in each time slot for scheduling a new job is bounded by the increase of term $C_h^rp_h^r[t]$ in Problem~D-R-DMLRS, and the possible increment introduced by randomized rounding in Algorithm~4.
This is ensured by the update of the price function and the rounding scheme, respectively.
\begin{lem}
\label{lem:relation}If the allocation-cost relationship holds for $\epsilon\geq 1, G_\delta> 0$ and $\delta\in(0,1]$, then Algorithm~1 ensures $P_i-P_{i-1}\geq \frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}),\forall i\in\mathcal{I}$.
\end{lem}
\begin{proof}[Proof of Lemma~\ref{lem:relation}]
For any job $i\in\mathcal{I}$, if job $i$ is rejected, then we have $P_i-P_{i-1}=D_i-D_{i-1}=0$ according to Problems~R-DMLRS and D-R-DMLRS, the inequality must hold.
If job $i$ is accepted with schedule $\pi_i$, i.e., $x_{\pi_i}=1$, then the increment value of the primal objective value $P_i$ is
\begin{align*}
P_i-P_{i-1}=u_i(t_{\pi_i}-a_i).
\end{align*}
Since $x_{\pi_i}=1$, according to Algorithm~1, the constraint~(\ref{eqn_dualconstr}) is binding.
Then, we can have
\begin{align*}
&u_i(t_{\pi_i}-a_i)\\
&=\lambda_i+\sum_{t\in\mathcal{T}(\pi_i)}\sum_{h\in\mathcal{H}(\pi_i[t])}\sum_{r\in\mathcal{R}}(\alpha_i^rw_{ht}^{\pi_i}+\beta_i^rs_{ht}^{\pi_i})p_h^r[t]\\
&=\lambda_i+\sum_{t\in\mathcal{T}(\pi_i)}\sum_{h\in\mathcal{H}(\pi_i[t])}\sum_{r\in\mathcal{R}}p_h^r[t](\rho_h^{r,i}[t]-\rho_h^{r,i-1}[t]).
\end{align*}
Similarly, we can have the increment value of the dual objective value $D_i$ as follows:
\begin{align*}
D_i-D_{i-1}\!=\!\lambda_i+\sum_{t\in\mathcal{T}(\pi_i)}\sum_{h\in\mathcal{H}(\pi_i[t])}\sum_{r\in\mathcal{R}}(p_h^{r,i}[t]-p_h^{r,i-1}[t])C_h^r.
\end{align*}
Summing up the allocation-cost relationship over all $t\in\mathcal{T}(\pi_i),h\in\mathcal{H}(\pi_i[t]),r\in\mathcal{R}$, we have
\begin{align*}
P_i-P_{i-1}&\geq\frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}-\lambda_i)+\lambda_i\\
&=\frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1})+(1-\frac{\delta/3G_\delta}{\epsilon})\lambda_i.
\end{align*}
As $\lambda_i\geq0$, $\epsilon, G_\delta> 0$ and $\delta\in(0,1]$, we have
\begin{align*}
P_i-P_{i-1}\geq\frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}).
\end{align*}
This completes the proof of Lemma~\ref{lem:relation}.
\end{proof}
For specific $h\in\mathcal{H},r\in\mathcal{R}$, we define $\epsilon_h^r$ as the corresponding parameter in the allocation-cost relationship for any job $i\in\mathcal{I}$ and any time slot $t\in\mathcal{T}$.
Then, it holds that $\epsilon=\max_{h,r}\{\epsilon_h^r\}$.
Without loss of generality, we assume that the resource demand of each worker or parameter server is much smaller compared to the capacity of that resource on one server, i.e., $\alpha_i^r\ll C_h^r, \beta_i^r\ll C_h^r$.
This is common in real-world machine learning system as large percentage of resources in the whole server.
As $\rho_h^r[t]$ increases from 0 to $C_h^r$, then we can claim that $d\rho_h^r[t]=\rho_h^{r,i}-\rho_h^{r,i-1}$, and derive a differential version of the allocation-cost relationship, which is defined as follows:
\begin{defn} \label{defn:diff_alloc_cost}
The differential allocation-cost relationship for Algorithm~1 with $\epsilon_h^r\geq 1$ is
\begin{align*}
p_h^r[t]d\rho_h^r[t]\geq \frac{C_h^r}{\epsilon_h^r}dp_h^r[t],\forall t\in\mathcal{T},h\in\mathcal{H},r\in\mathcal{R}.
\end{align*}
\end{defn}
Next we show that a feasible $\epsilon_h^r$ satisfies the differential allocation-cost relationship with price function $p_h^r[t]$ defined in (\ref{eqn_cost_update_fn}).
\begin{lem}\label{lem:diff_alloc_cost}
$\epsilon_h^r=\ln\frac{U^r}{L}$, and the price function defined in (\ref{eqn_cost_update_fn}) satisfy the differential allocation-cost relationship.
\end{lem}
\begin{proof}[Proof of Lemma~\ref{lem:diff_alloc_cost}]
The derivation of the marginal cost function is
\begin{align*}
dp_h^r[t]=p_h^{r'}(\rho_h^r[t])d\rho_h^r[t]=L(\frac{U^r}{L})^{\frac{\rho_h^r[t]}{C_h^r}}\ln(\frac{U^r}{L})^{\frac{1}{C_h^r}}d\rho_h^r[t].
\end{align*}
The differential allocation-cost relationship is
\begin{align*}
L(\frac{U^r}{L})^{\frac{\rho_h^r[t]}{C_h^r}}d\rho_h^r[t]\geq \frac{C_h^r}{\epsilon_h^r}L(\frac{U^r}{L})^{\frac{\rho_h^r[t]}{C_h^r}}\ln(\frac{U^r}{L})^{\frac{1}{C_h^r}}d\rho_h^r[t],
\end{align*}
which holds for $\epsilon_h^r\geq \ln(\frac{U^r}{L})$.
Then, we can set $\epsilon\!=\!\max\limits_{r\in\mathcal{R}}(1,\ln(\frac{U^r}{L}))$, which satisfies the differential allocation-cost relationship.
This completes the proof.
\end{proof}
With the aforementioned lemmas, we are now in a position to prove Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}.
Note that the Theorems provide probabilistic guarantees.
We first analyze the performance ratio, which is followed by the probabilistic feasibility discussion for both cases.
\begin{proof}[Proof of Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}]
According to Lemma~\ref{lem:diff_alloc_cost}, the marginal cost function used in Algorithm~1 satisfies the differential allocation-cost relationship with $\epsilon=\max_r(1,\ln\frac{U^r}{L})$.
Since the resource demand in a job $i$ is much smaller than the capacity, we can derive
\begin{align*}
&d\rho_h^r[t]=\rho_h^{r,i}-\rho_h^{i-1}[t], \\
&dp_h^r[t]=p_h^{r'}(\rho_h^r[t])(\rho_h^{r,i}[t]-\rho_h^{r,i-1}[t])=p_h^{r,i}[t]-p_h^{r,i-1}[t].
\end{align*}
So, the differential allocation-cost relationship in Definition~\ref{defn:diff_alloc_cost} implies the allocation-cost relationship in Definition~\ref{defn:alloc_cost} holds for $\epsilon=\max_r(1,\ln\frac{U^r}{L})$.
According to Algorithm~1, we note that
\begin{align*}
\frac{1}{\mu}\leq \frac{\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})}{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}},\forall i\in\mathcal{I}.
\end{align*}
Then, the minimum amount of overall resource consumption of job $i$ can be computed as:
\begin{multline*}
\frac{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}}{\mu}\\
\leq \ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r}).
\end{multline*}
Then, it follows that:
\begin{align*}
&D_0=\sum_{t,h,r}LC_h^r\\
&=\sum_{t,h,r}\min_{i\in\mathcal{I},\pi_i\in\Pi_i}\frac{1/(2\mu) u_{i}(t_{\pi_i}-a_{i})C_h^r}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}\\
&=\frac{T\sum_{h,r}C_h^r}{2\mu}\min_{i,\pi_i}\frac{u_{i}(t_{\pi_i}-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}\\
&\leq \frac{1}{2}\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})\\
&\min_{i\in\mathcal{I},\pi_i\in\Pi_i}\frac{u_{i}(t_{\pi_i}-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}, \forall i\in\mathcal{I}.\\
&\overset{(a)}{\leq}\frac{1}{2}\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})\\
&\frac{u_{i}(t_{\pi_i}-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}\\
&\leq \frac{1}{2}u_i(t_{\pi_i}-a_i)\overset{(b)}{\leq} \frac{1}{2}OPT,
\end{align*}
where (a) follows by selecting $(i,\pi)=\arg\min_{i\in\mathcal{I},\pi_i\in\Pi_i}u_i(t_{\pi_i}-a_i)$, and (b) follows from the assumption that the offline optimal solution accepts at least one job, which is reasonable in real-world machine learning system.
Then we have $OPT\geq \min_{i,\pi}u_i(t_{\pi_i}-a_i)$.
According to Lemmas~\ref{lem:result} and \ref{lem:relation}, the competitive ratio is proved.
Recall that the randomized rounding algorithm is a key component in our algorithm.
Toward this end, we show the probability of obtaining a feasible solution with the proved competitive ratio.
Here, we consider the following two cases:
{\em 1) } When $0<G_\delta\leq 1$ (Theorem~\ref{thm_Competitive_1}): According to Theorem~\ref{thm_Alg4_1}, the probability of violating the cover constraint is no greater than $\delta/3$ at each randomized rounding iteration.
Recall that our Algorithm~1 runs a predetermined number of $S$ iterations to find a feasible integer solution.
Thus the probability that no feasible integer solution returned after $S$ iterations rounding is at most $(\delta/3)^S$.
It then follows that the probability of at least one feasible integer solution found is at least $1-(\delta/3)^S$.
Moreover, the number of states $(t,v)$ in the dynamic programming for each job $i$ is $O(TK_iE_i)$.
Therefore, with probability greater than $(1-(\delta/3)^S)^{T_iK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible integer solution with the proved competitive ratio.
{\em 2) } When $G_\delta>1$ (Theorem~\ref{thm_Competitive_2}): According to Theorem~\ref{thm_Alg4_2}, the probability of violating the packing constraint is no greater that $\delta/3(HR+1)$ at each randomized rounding iteration.
Following the similar arguments in {\em 1)}, we can show that with probability greater than $(1-(\delta/3(HR+1))^S)^{T_iK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible integer solution with the proved competitive ratio, and the proof is complete.
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
%\label{bio}
%\vspace{-.5in}
\begin{IEEEbiography}[{\includegraphics[width=1in, height=1.25in]{Figures/Menglu_Bio1.jpg}}]
{Menglu Yu} received her B.S. degree from the Department of Electrical and Information Engineering at Hunan University, China in 2014.
She is currently pursuing her Ph.D. degree in the Department of Computer Science at Iowa State University.
Her primary research interests include optimization for distributed machine learning systems and data centers, as well as network optimization.
\end{IEEEbiography}
%\vspace{-.5in}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/Liu_Bio.eps}}]
{Jia Liu} (S'03--M'10--SM'16) is an Assistant Professor in the Department of Electrical and Computer Engineering at The Ohio State University, where he joined in Aug. 2020.
He received his Ph.D. degree from the Department of Electrical and Computer Engineering at Virginia Tech in 2010.
From Aug. 2017 to Aug. 2020, he was an Assistant Professor in the Department of Computer Science at Iowa State University.
His research areas include theoretical machine learning, control and optimization for stochastic networks, and optimization for data analytics infrastructure and cyber-physical systems.
Dr. Liu is a senior member of IEEE and a member of ACM. He has received numerous awards at top venues, including IEEE INFOCOM'19 Best Paper Award, IEEE INFOCOM'16 Best Paper Award, IEEE INFOCOM'13 Best Paper Runner-up Award, IEEE INFOCOM'11 Best Paper Runner-up Award, and IEEE ICC'08 Best Paper Award.
Dr. Liu is a recipient of the NSF CAREER Award in 2020. He is a recipient of the Google Faculty Research Award in 2020.
He is also a winner of the LAS Award for Early Achievement in Research from the College of Liberal Arts and Sciences at Iowa State University in 2020, and the Bell Labs President Gold Award in 2001.
His research is supported by NSF, AFOSR, AFRL, and ONR.
\end{IEEEbiography}
%\vspace{-.5in}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/wu.png}}]
{Chuan Wu} received her B.Engr. and M.Engr. degrees in 2000 and 2002 from the Department of Computer Science and Technology, Tsinghua University, China, and her Ph.D. degree in 2008 from the Department of Electrical and Computer Engineering, University of Toronto, Canada. Since September 2008, Chuan Wu has been with the Department of Computer Science at the University of Hong Kong, where she is currently a Professor. Her current research is in the areas of cloud computing, distributed machine learning systems and algorithms, and intelligent elderly care technologies. She is a senior member of IEEE, a member of ACM, and served as the Chair of the Interest Group on Multimedia services and applications over Emerging Networks (MEN) of the IEEE Multimedia Communication Technical Committee (MMTC) from 2012 to 2014. She is an associate editor of IEEE Transactions on Cloud Computing, IEEE Transactions on Multimedia, ACM Transactions on Modeling and Performance Evaluation of Computing Systems, and IEEE Transactions on Circuits and Systems for Video Technology. She was the co-recipient of the best paper awards of HotPOST 2012 and ACM e-Energy 2016.
\end{IEEEbiography}
%
%\vfill
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/BoJi_Bio.jpg}}]{Bo Ji}(S'11-M'12-SM'18)
received his B.E. and M.E. degrees in Information Science and Electronic Engineering from Zhejiang University, Hangzhou, China, in 2004 and 2006, respectively, and his Ph.D. degree in Electrical and Computer Engineering from The Ohio State University, Columbus, OH, USA, in 2012. Dr. Ji is an Associate Professor in the Department of Computer Science at Virginia Tech, Blacksburg, VA, USA. Prior to joining Virginia Tech, he was an Associate Professor in the Department of Computer and Information Sciences and a faculty member of the Center for Networked Computing at Temple University, where he was an Assistant Professor from July 2014 to June 2020. He was also a Senior Member of the Technical Staff with AT\&T Labs, San Ramon, CA, from January 2013 to June 2014. His research interests are in the modeling, analysis, control, optimization, and learning of computer and network systems, such as wired and wireless networks, large-scale IoT systems, high performance computing systems and data centers, and cyber-physical systems. He currently serves on the editorial boards of the IEEE/ACM Transactions on Networking, IEEE Internet of Things Journal, and IEEE Open Journal of the Communications Society. Dr. Ji is a senior member of the IEEE and a member of the ACM. He is a National Science Foundation (NSF) CAREER awardee (2017) and an NSF CISE Research Initiation Initiative (CRII) awardee (2017). He is also a recipient of the IEEE INFOCOM 2019 Best Paper Award.
\end{IEEEbiography}
%
%
%\vspace{-5in}
\begin{IEEEbiography}
[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/Bentley_Bio.jpg}}]
{Elizabeth S. Bentley} has a B.S. degree in Electrical Engineering from Cornell University, a M.S. degree in Electrical Engineering from Lehigh University, and a Ph.D. degree in Electrical Engineering from University at Buffalo.
She was a National Research Council Post-Doctoral Research Associate at the Air Force Research Laboratory in Rome, NY.
Currently, she is employed by the Air Force Research Laboratory in Rome, NY, performing in-house research and development in the Networking Technology branch.
Her research interests are in cross-layer optimization, wireless multiple-access communications, wireless video transmission, modeling and simulation, and directional antennas/directional networking.
\end{IEEEbiography}
%\documentclass[9.5pt,journal]{IEEEtran}
%\documentclass[conference,compsoc]{IEEEtran}
\documentclass[10pt, journal,compsoc]{IEEEtran}
% *** CITATION PACKAGES ***
%
\ifCLASSOPTIONcompsoc
% IEEE Computer Society needs nocompress option
% requires cite.sty v4.0 or later (November 2003)
\usepackage[nocompress]{cite}
\else
% normal IEEE
\usepackage{cite}
\fi
% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
\else
\fi
\pdfoutput=1
\usepackage{ifpdf}
\usepackage{cite}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{mathrsfs}
\usepackage[dvips,dvipdf]{graphicx}
\usepackage{subfigure}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage{multirow}
\usepackage{bbm}
\usepackage{arydshln}
%\usepackage{flushend}
\usepackage{url}
\usepackage{pict2e,picture} % picture
\usepackage{tikz} % tikz
\usepackage{pifont} % ding
\usepackage{enumitem}
\usepackage{mathtools}
\usepackage{algpseudocode}
%\usepackage{tabularx}
\usepackage{comment}
\usepackage{xcolor}
%\usepackage{tabu}
%\usepackage{subfig}
\usepackage{caption}
\DeclarePairedDelimiter{\ceil}{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}
\bibliographystyle{IEEEtran}
%\include{Complete.bib}
\input{Supporting_Preambles/symbols_commands}
%set up for the captions
%\captionsetup[figure]{position=bottom,justification=centering,width=.85\textwidth,labelfont=bf,font=small}
%\captionsetup[figure]{position=bottom,calcwidth=.85\linewidth,font=scriptsize}
% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
\title{Toward Efficient Online Scheduling for Distributed Machine Learning Systems}
\author{Menglu~Yu,
Jia~Liu~\IEEEmembership{Senior Memeber,~IEEE},
Chuan~Wu~\IEEEmembership{Senior Memeber,~IEEE},
Bo~Ji~~\IEEEmembership{Senior Memeber,~IEEE}
Elizabeth~S.~Bentley~\IEEEmembership{Member,~IEEE}% <-this % stops a space
%\thanks{
%This work has been supported by NSF grants CNS-1527078, 1514260, 1446582, 1409336, 1012700, WiFiUS-1456806, ECCS-1444026, 1232118; ONR grant N00014-15-1-2166; ARO grant W911NF-14-1-0368; DTRA grants HDTRA 1-14-1-0058, 1-15-1-0003, AFRL VFRP'15 award; DARPA grant HROOll-15-C-0097and QNRF grant NPRP 7-923-2-344.
%DISTRIBUTION STATEMENT A: Approved for Public Release; distribution unlimited 88ABW-2015-5989 on Dec. 14, 2015.}
\thanks{
Menglu Yu is with the Department of Computer Science, Iowa State University, Ames, IA 50011, USA (e-mail: mengluy@iastate.edu).}
\thanks{
Jia Liu is with the Department of Electrical and Computer Engineering, The Ohio State University, Columbus, OH 43210, USA (e-mail: liu@ece.osu.edu).}
\thanks{
Bo Ji is with the Department of Computer Science, Virginia Tech, Blacksburg, VA 24061, USA (e-mail: boji@vt.edu).}
\thanks{
Chuan Wu is with the Department of Computer Science, The University of Hong Kong, Hong Kong (e-mail: cwu@cs.hku.hk).}
\thanks{
Elizabeth S. Bentley is with the Air Force Research Laboratory, Information Directorate, Rome, NY 13441, USA (e-mail: elizabeth.bentley.3@us.af.mil).}
\thanks{
Digital Object Identifier xx.xxxx/XXXX.xxxx.xxxxx.}%
}
%\author{Menglu Yu$^{\dag}$ \mbox{\hspace{0.4cm}} Jia Liu$^{\mathsection}$ \mbox{\hspace{0.4cm}} Chuan Wu$^{\mathparagraph}$ \mbox{\hspace{0.4cm}} Bo Ji$^{\ddag}$ \mbox{\hspace{0.4cm}} Elizabeth Bentley$^{*}$
%\IEEEcompsocitemizethanks{\IEEEcompsocthanksitem $^{\dag}$Department of Computer Science, Iowa State University
%\IEEEcompsocthanksitem $^{\mathsection}$Department of Electrical and Computer Engineering, The Ohio State University
%\IEEEcompsocthanksitem $^{\mathparagraph}$Department of Computer Science, University of Hong Kong
%\IEEEcompsocthanksitem $^{\ddag}$Department of Computer Science, Virginia Tech
%\IEEEcompsocthanksitem $^{*}$Air Force Research Laboratory, Information Directorate
%}}
%\author{Paper ID: 1570578306}
\IEEEtitleabstractindextext{%
%\begin{abstract}
\input{Abstract/Abstract}
%\end{abstract}
% Note that keywords are not normally used for peerreview papers.
\begin{IEEEkeywords}
Online resource scheduling, distributed machine learning, approximation algorithm%up to 5
\end{IEEEkeywords}}
\maketitle
\IEEEdisplaynontitleabstractindextext
\IEEEpeerreviewmaketitle
%\IEEEraisesectionheading{\section{Introduction}\label{sec:introduction}}
%\input{Abstract/Abstract}
\input{Sec1_Intro/Sec1_Intro}
\input{Sec2_Related/Sec2_Related}
\input{Sec3_Model/Sec3_Model}
\input{Sec4_Algorithm/Sec4_Algorithm}
\input{Sec5_Numerical/Sec5_Numerical}
\input{Sec6_Conclusion/Sec6_Conclusion}
%\appendices
%\input{Appdx_A/Appdx_A}
%\input{Appdx_B/Appdx_B}
%\input{Appdx_C/Appdx_C}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\bibliography{IEEEabrv,./BIB/Complete,./BIB/HeavyBall,./BIB/Kevin_All,./BIB/CSMA}
%\clearpage
\bibliography{IEEEabrv,./BIB/JobScheduling,./BIB/DMLF,./BIB/Kevin_All,./BIB/Complete,./BIB/CSMA}
\input{Bio/Bio}
%\begin{IEEEbiography}
%[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/Liu_Bio.eps}}]
%{Jia Liu} (S'03--M'10--SM'16) is an Assistant Professor in the Dept. of Electrical and Computer Engineering at The Ohio State University, where he joined in Aug. 2020.
%He received his Ph.D. degree from the Dept. of Electrical and Computer Engineering at Virginia Tech in 2010.
%From Aug. 2017 to Aug. 2020, he was an Assistant Professor in the Dept. of Computer Science at Iowa State University.
%
%Prior to that, he was a Postdoctoral Researcher and subsequently a Research Assistant Professor from Feb. 2010 to Jul. 2017, both in the ECE department at OSU.
%
%His research areas include theoretical machine learning, control and optimization for stochastic networks, and optimization for data analytics infrastructure and cyber-physical systems.
%
%Dr. Liu is a senior member of IEEE and a member of ACM. He has received numerous awards at top venues, including IEEE INFOCOM'19 Best Paper Award, IEEE INFOCOM'16 Best Paper Award, IEEE INFOCOM'13 Best Paper Runner-up Award, IEEE INFOCOM'11 Best Paper Runner-up Award, and IEEE ICC'08 Best Paper Award.
%
%Dr. Liu is a recipient of the NSF CAREER Award in 2020.
%
%He is a recipient of the Google Faculty Research Award in 2020 and the Bell Labs President Gold Award in 2001. His research is supported by NSF, AFOSR, AFRL, and ONR.
%\end{IEEEbiography}
\clearpage
\appendices
\input{Appdx_A/Appdx_A}
\input{Appdx_B/Appdx_B}
\input{Appdx_C/Appdx_C}
\end{document}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Introduction} \label{sec:intro}
Fueled by the rapid growth of data analytics and machine learning (ML) applications, recent years have witnessed an ever-increasing hunger for computing power.
However, with hardware capability no longer advancing at the pace of the Moore's law, it has been widely recognized that a viable solution to sustain such computing power needs is to exploit {\em parallelism} at both machine and chip scales.
Indeed, the recent success of deep neural networks (DNN)
%(a revival of artificial neural networks with a much larger number of layers in depth)
is enabled by the use of distributed ML frameworks, which exploit the massive parallelism over computing clusters with a large number of GPUs.
These distributed ML frameworks have significantly accelerated the training of DNN for many applications (e.g., image and voice recognition, natural language processing, etc.).
To date, prevailing distributed ML frameworks include TensorFlow~\cite{TensorFlow}, MXNet~\cite{MXNet}, PyTorch~\cite{Paszke19:Pytorch}, Caffe~\cite{Caffe}, to name just a few.
However, the proliferation of distributed ML frameworks also introduces many unique technical challenges on large-scale computing system design and network resource optimization.
%A {\em common} feature underlying distributed ML frameworks is
Particularly, due to the decentralized nature, at the heart of distributed learning system optimization lies the problem of scheduling ML jobs and resource provisioning across different machines to minimize the total training time.
Such scheduling problems involve dynamic and combinatorial %virtual-machine-based
worker and parameter server allocations, which are inherently NP-hard.
Also, the allocations of workers and parameter servers should take {\em locality} into careful consideration, since co-located workers and parameter servers can avoid costly network communication overhead.
However, locality optimization adds yet another layer of difficulty in scheduling algorithm design.
Exacerbating the problem is the fact that the future arrival times of training jobs at an ML computing cluster are hard to predict,
which necessitates {\em online} algorithm design without the knowledge of future job arrivals.
So far in the literature, there remains a lack of holistic theoretical studies that address all the aforementioned challenges.
Most of the existing scheduling schemes are based on simple heuristics without performance guarantee (see Section~\ref{sec:Related} for detailed discussions).
This motivates us to fill this gap and pursue efficient online scheduling designs for distributed ML resource optimization, which offer {\em provable} performance guarantee.
The main contribution of this paper is that we develop an online scheduling algorithmic framework that {\em jointly} yields resource scheduling and locality optimization decisions with strong competitive ratio performance.
Further, we reveal interesting insights on how distributed ML frameworks affect online resource scheduling optimization.
Our main technical results are summarized as follows:
\vspace{-.01in}
\begin{list}{\labelitemi}{\leftmargin=1em \itemindent=-0.0em \itemsep=.2em}
\item By abstracting the architectures of prevailing distributed ML frameworks, we formulate an online resource scheduling optimization problem that: i) models the training of ML jobs based on the parameter server (PS) architecture and stochastic gradient descent (SGD) method; and ii) explicitly takes {\em locality} optimization into consideration.
We show that, due to the heterogeneous internal (between virtual machines or containers) and external (between physical machines) communications, the locality-aware scheduling problem contains {\em non-deterministic} constraints and is far more complex compared to the existing works that are locality-oblivious (see, e.g., ~\cite{Chun16:Dolphin,Bao18:ML_INFOCOM}).
\item To solve the locality-aware scheduling problem, we
develop an equivalent problem reformulation to enable subsequent developments of online approximation algorithms.
Specifically, upon carefully examining the locality configurations of worker-server relationships, we are able to transform the original problem to a special-structured integer nonlinear program with mixed cover/packing-type constraints, and
% which further entails
the low-complexity approximation algorithm design with provable performance can be further entailed.
\item To tackle the integer nonlinear problem with mixed cover/packing-type constraints, we propose an approximation algorithm based on a meticulously designed randomized rounding scheme and then rigorously prove its performance.
We note that the results of our randomized rounding scheme are general and could be of independent theoretical interest.
Finally, by putting all algorithmic designs together, we construct a primal-dual online resource scheduling (PD-ORS) scheme, which has an overall competitive ratio that only {\em logarithmically} depends on ML job characteristics (e.g., required epochs, training samples).
\end{list}
Collectively, our results contribute to a comprehensive and fundamental understanding of distributed machine learning system optimization.
The remainder of this paper is organized as follows.
In Section~\ref{sec:Related}, we review the literature to put our work in comparative perspectives.
Section~\ref{sec:model_formulation} introduces the system model and problem formulation.
Section~\ref{sec:alg} presents our algorithms and their performance analysis.
Section~\ref{sec:numerical} presents numerical results and Section~\ref{sec:conclusion} concludes this paper.
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Related Work} \label{sec:Related}
As mentioned in Section~\ref{sec:intro}, due to the high computational workload of ML applications, many distributed ML frameworks (e.g., TensorFlow~\cite{TensorFlow}, MXNet~\cite{MXNet}, PyTorch~\cite{Paszke19:Pytorch}, Caffe~\cite{Caffe}) have been proposed to leverage modern large-scale computing clusters.
% to leverage modern computing platforms and offer massive parallelism.
% e.g., TensorFlow~\cite{TensorFlow}, MXNet~\cite{MXNet}, Cognitive ToolKit (CNTK)~\cite{CNTK}, Caffe~\cite{Caffe}, etc.
A common distributed training architecture implemented in these distributed ML frameworks is the PS architecture~\cite{Li14:ML_Static_OSDI,Chilim14:ML_Static_OSDI}, which employs multiple workers and PSs (implemented as virtual machines or containers) to collectively train a global ML model.
Coupled with the {\em iterative} ML training based on stochastic gradient descent (SGD), the interactions between machines in the distributed ML cluster are significantly different from those in traditional cloud computing platforms (e.g., MapReduce~\cite{Dean08:MapReduce} and Dryad~\cite{Isard07:Dryad} and references therein).
For example, a MapReduce job usually partitions the input data into independent chunks, which are then processed by the {\it map} step in a parallel fashion.
The output of the maps are then fed to the {\it reduce} step to be aggregated to yield the final result.
Clearly, the data flows in MapReduce are a ``one-way traffic", which is unlike those iterative data flows in ML training jobs whose completions highly depend on the ML job's convergence property.
%However, machine learning algorithms typically require a large number of iterations to optimize its parameters using optimization method such as SGD.}
%Although each iteration can be expressed as a MapReduce job, each job must reload data from the disk, which could lead to a significant performance degradation.
As a result, existing job scheduling algorithms for cloud systems are not suitable for distributed ML frameworks.
%~\cite{Curino14:Rayon_SoCC,Vavilapalli13:YARN_SoCC,Boutin14:Apollo_OSDI,Huang15:CORA_INFOCOM,Chen17:Maxmin_INFOCOM,Azar15:CloudSch_EC,Lucier13:CloudSch_SPAA,Zhou17:CloudSch_TON,Zhang17:CloudSch_TON,Xiao14:CloudSch_TC,Tumanov16:CloudSch_EuroSys}
Among distributed ML system studies, most of the early attempts (see, e.g., ~\cite{Li14:ML_Static_OSDI,Chilim14:ML_Static_OSDI} and references therein) only considered static allocation of workers and parameter servers.
To our knowledge, the first work on understanding the performance of distributed ML frameworks is~\cite{Yan15:MLFramework_KDD}, where Yan {\em et al.} developed analytical models to quantify the impacts of model-data partitioning and system provisioning for DNN.
Subsequently, Chun {\em et al.} ~\cite{Chun16:Dolphin} developed heuristic dynamic system reconfiguration algorithms to allocate workers and parameter servers to minimize the runtime, but without providing optimality guarantee.
The first dynamic distributed scheduling algorithm with optimality guarantee was reported in ~\cite{Sun17:ML_MILP}, where Sun {\em et al.} used standard mixed integer linear program (MILP) solver to dynamically compute the worker-parameter server partition solutions.
Due to NP-hardness of the MILP, the scalability of this approach is limited. %cannot be scaled up to handle large-size distributed ML systems in practice.
The most recent work~\cite{Zhang20:onlinescheduling} proposed an online scheduling algorithm to schedule synchronous training jobs in ML clusters with the goal to minimize the weighted completion time.
However, the consecutive time slots were allocated for each training job, and the numbers of workers and parameter servers could not be adjusted.
Another line of the research is to leverage the learning-based approach to do the resource scheduling.
There are a number of recent works using deep reinforcement learning (DRL) for resource allocation, device placement, and video streaming.
For example, Mao {\it et al.}~\cite{Mao16:DRL} and Chen {\it et al.}~\cite{Chen17:DRL} designed a multi-resource cluster scheduler using DRL with the goal to minimize average job slowdown.
The proposed scheduler picks one or more of the waiting jobs in the queue and allocate to machines at each time slot, and the resource demand of each job is unknown until after its arrival.
Later, Mao {\it et al.}~\cite{Mao18:graph-based,Mao19:learningscheduling} used DRL to heuristically train scheduling policies for graph-based parallel jobs by setting both parallelism level and execution order.
Meanwhile, Mirhoseini {\it et al.}~\cite{Mirhoseini18:hierarchicalmodel, Mirhoseini17:deviceplacement} utilized DRL to design a model for efficient placement of computational graphs onto hardware devices, aiming at minimize the running time of each individual TensorFlow job.
Although various performance gains have been empirically reported, these DRL-based studies do not offer optimality performance guarantee due to the lack of theoretical foundation of DRL as of today.
\begin{figure*}[t!]
\centering
\begin{minipage}[t]{0.3\textwidth}
\centering
\vspace{-1.26in}
\includegraphics[height=0.14\textheight]{Figures/fig_Distr_ML1.eps}
\vspace{-.03in}
\caption{Illustration of distributed training with the PS architecture.}
\label{fig:DistrML}
\end{minipage}
%
\hspace{0.01\columnwidth}%
%
\begin{minipage}[t]{0.3\textwidth}
\centering
\vspace{-1.3in}
\includegraphics[width=0.23\textheight]{Figures/fig_ML_Workflow1.eps}
\vspace{-.15in}
\caption{The workflow of iterative training.}
\label{fig:MLWorkflow}
\end{minipage}
%
\hspace{0.01\columnwidth}%
%
\begin{minipage}[t]{0.3\textwidth}
\centering
\includegraphics[width=0.16\textheight]{Figures/fig_Colocate1.eps}
\vspace{.06in}
\caption{Colocated parameter servers and workers on physical machines.}
\label{fig:Colocate}
\end{minipage}
\vspace{-.1in}
\end{figure*}
%\smallskip
The most relevant work to ours is ~\cite{Bao18:ML_INFOCOM}, where Bao {\em et al.} developed an online primal-dual approximation algorithm, OASiS, to solve the scheduling problem for distributed ML systems.
% by employing a similar primal-dual algorithmic framework.
Our work differs from~\cite{Bao18:ML_INFOCOM} in the following key aspects: 1) In \cite{Bao18:ML_INFOCOM}, the workers and parameter servers are allocated on two {\em strictly separated} sets of physical machines, i.e., {\em no} worker and parameter server can share the same physical machine, which significantly simplified the underlying optimization problem. In this work, we consider the cases that workers and parameter servers can be {\em co-located} on the same physical machine, which is the common practice in existing ML systems (see, e.g., ~\cite{MXNet1,MXNet2}). Such co-location can significantly reduce inter-server communication, expedite training, and improve resource utilization efficiency between workers and parameter servers.
However, as will be shown later, the co-location setting leads to an integer {\em non-convex} optimization problem with {\em non-deterministic} constraints, which is much harder and necessitates new algorithm design.
%To this end, we propose a low-complexity approximation algorithm to solve the problem and rigorously prove its performance.
%More importantly, we consider the setting with co-located workers and parameter servers because
2) Ref.~\cite{Bao18:ML_INFOCOM} advocates dynamic worker number adjustment, but does {\em not} guarantee the same global batch size across the training iterations.
According to recent literature~\cite{Goyal17:SGD}, maintaining a consistent global batch size is important for ensuring convergence of DNN training, when the worker number varies. We ensure a consistent global batch size in our model.
We note that the co-location setting was considered in~\cite{Peng18:ML_EuroSys}. However, the scheduling algorithm therein is a heuristic and does not provide performance guarantee. This motivates us to develop new algorithms with provable performance to fill this gap.
% !TEX root = ../ML_Networking_TNSE20.tex
\section{System Model and Problem Formulation} \label{sec:model_formulation}
In this section, we first provide a quick overview on the architecture of distributed ML frameworks to familiarize readers with the necessary background.
Then, we will introduce our analytical models for ML jobs and resource constraints, as well as the overall problem formulation.
% \begin{figure}
% \centering
% % \vspace{-1.26in}
% \includegraphics[width=0.3\textheight]{Figures/fig_Distr_ML1.eps}
% % \vspace{-.03in}
% \caption{Illustration of distributed training with the PS architecture.}
% \label{fig:DistrML}
% \end{figure}
\smallskip
{\bf 1) Distributed Machine Learning: A Primer.} %\label{subsec:distr_ML}
As illustrated in Fig.~\ref{fig:DistrML},
the key components of a PS-based distributed ML system include parameter servers, workers, and the training dataset, which are usually implemented over a connected computing cluster that contains multiple physical machines.
The training dataset of an ML job is stored in distributed data storage (e.g., HDFS~\cite{HDFS}) and usually divided into equal-sized data chunks.
Each data chunk further contains multiple equal-sized mini-batches.
%
%\begin{figure}[t!]
%\centering
%\begin{minipage}[t]{0.24\textwidth}
%\centering
%\vspace{-1.1in}
%\includegraphics[height=0.12\textheight]{Figures/fig_Distr_ML1.eps}
%\vspace{-.08in}
%\caption{An illustration of a distributed machine learning framework.}
%\label{fig:DistrML}
%\end{minipage}
%%
%\hspace{0.001\columnwidth}%
%%
%\begin{minipage}[t]{0.24\textwidth}
%\centering
%\includegraphics[width=0.14\textheight]{Figures/fig_Colocate1.eps}
%\vspace{-.06in}
%\caption{Colocated parameter servers and workers on physical machines.}
%\label{fig:Colocated}
%\end{minipage}
%\vspace{-.2in}
%\end{figure}
%
To date, one of the most widely adopted training algorithms in distributed ML frameworks is the {\em stochastic gradient descent method} (SGD)~\cite{Dean:2012:LSD}. %thanks to its low-complexity~\cite{????}.
With SGD, the interactions between workers and parameter servers are illustrated in Fig~\ref{fig:MLWorkflow}.
A worker is loaded with the DNN model (we focus on data parallel training) with current values of the model parameters (e.g., the weights of a DNN) and retrieves a new data chunk from the data storage.
In each training iteration, a worker processes one mini-batch from its data chunk to compute gradients (i.e., directions and magnitudes of parameter changes).\footnote{
As an example, in a DNN model, gradients can be computed by the well-known ``back-propagation'' approach.
}
Upon finishing a mini-batch, the worker sends the gradients to the parameter servers, receives updated global parameters, and then continues with the next training iteration to work on the next mini-batch.
%After finishing the current data chunk, the worker will repeat the same process on a new data chunk.
On the parameter server side, parameters are updated as: $\w[k] = \w[k-1] + \alpha_{k} \g[k]$, where $\w[k]$, $\alpha_{k}$, and $\g[k]$ denote the parameter values, step-size, and stochastic gradient in the $k$-th update, respectively.
%\begin{figure}[t!]
%\centering
%\includegraphics[width=0.76\columnwidth]{Figures/fig_ML_Workflow.eps}
%\vspace{-.05in}
%\caption{Workflow of the asynchronous stochastic gradient descent method (Async-SGD) in a distributed machine learning framework.}
%\label{fig:MLWorkflow}
%\vspace{-.3in}
%\end{figure}
%\begin{figure}[t!]
%\centering
%\begin{minipage}[t]{0.33\textwidth}
%\centering
%%\vspace{-1.1in}
%\includegraphics[height=0.102\textheight]{Figures/fig_Colocate1.eps}
%\vspace{-.2in}
%\caption{The workflow of Async-SGD on a distributed ML system.}
%\label{fig:MLWorkflow}
%\end{minipage}
%%
%\hspace{0.001\columnwidth}%
%%
%\begin{minipage}[t]{0.33\textwidth}
%\centering
%\vspace{-.9in}
%\includegraphics[width=0.18\textheight]{Figures/fig_locality1.eps}
%\vspace{-.07in}
%\caption{$b_{i}(h,p)$'s value is locality-dependent (internal or external).}
%\label{fig_bhp}
%\end{minipage}
%\vspace{-.2in}
%\end{figure}
%We can further see from Fig.~\ref{fig:MLWorkflow} that the training progress at different workers is {\em not} synchronized: Each parameter server updates its parameters without coordinating with other parameter servers, hence the name Async-SGD.
%It has been shown that Async-SGD achieves the {\em same} $O(1/\sqrt{k})$ convergence rate as its synchronous counterpart~\cite{lian2015asynchronous,huo2017asynchronous}, while avoiding technical complexities such as maintaining a common clock, slow machine(s) bottlenecks, periodic spikes of information exchanges and congestions, etc.
%Thanks to these salient features, Async-SGD is particularly suitable for distributed ML systems.
%For further details of Async-SGD, we refer interested readers to the excellent overview by XXX~\cite{???}.
%Upon understanding the basics of Async-SGD, we are now in a position to model jobs in distributed ML systems and facilitate scheduling algorithm design.
\smallskip
{\bf 2) Modeling of Learning Jobs:}
We consider a time-slotted system.
%, where time-slots are indexed by 0,1,2,$\dots$.
The scheduling time-horizon is denoted as $\mathcal{T}$ with $|\mathcal{T}|=T$.
We use $\mathcal{I}$ to represent the set of training jobs and let $a_{i}$ denote the arrival time-slot of job $i \in \mathcal{I}$.
As shown in Fig.~\ref{fig:Colocate}, parameter servers and workers %are usually implemented on virtual machines and
could spread over multiple physical machines.
We let $\mathcal{H}$ represent the set of physical machines.
For each job $i$, we use $w_{ih}[t], s_{ih}[t] \geq 0$ to represent the allocated numbers of workers and parameter servers on machine $h \in \mathcal{H}$ in each time-slot $t \geq a_{i}$, respectively.
Further, we let $ \mathcal{P}_{i}[t] \!\triangleq\! \{h \!\in\! \mathcal{H}|s_{ih}[t] \!>\! 0\}$ and $ \mathcal{W}_{i}[t] \!\triangleq\! \{h \!\in\! \mathcal{H}| w_{ih}[t] \!>\! 0\}$ denote the sets of physical machines that contain parameter servers and workers for job $i$ in time-slot $t$, respectively.
We use a binary variable $x_{i}\in \{0,1\}$ to indicate whether job $i$ is admitted ($x_{i} =1$) or not ($x_{i}=0$).
%We use $\tau_{i}$ to denote the training time for each sample of job $i$, computed by the time for training one mini-batch divided by the number of samples per mini-batch.
We use $\tau_{i}$ to denote the training for each sample of job $i$.
%Admission Control is common in cloud management systems and jobs get rejected can be queued or resubmitted later[citation].
We let $b_{i}(h,p)$ denote the data rate of the link between a worker for job $i$ (on machine $h$) and a parameter server (on machine $p$).
Each worker or parameter server is exclusively assigned to some job $i$, and $b_i(h,p)$ is reserved and decided by the user upon job submission, which is common to ensure the data transfer performance~\cite{Bao18:ML_INFOCOM}.
Note that the value of $b_{i}(h,p)$ is {\em locality-dependent} where the slowest worker will become the bottleneck since we focus on Bulk-Synchronous-Parallel (BSP) scheme ~\cite{Cheatham05:BSP}.
Specifically, we have:
\begin{align*}
b_{i}(h,p) =
\begin{cases}
b_{i}^{(i)}, & \text{if $h=p$}, \\
b_{i}^{(e)}, & \text{otherwise},
\end{cases}
\end{align*}
where $b_{i}^{(i)}$ and $b_{i}^{(e)}$ denote the internal and external communication link rates, respectively.
For example, as shown in Fig.~\ref{fig:Colocate}, since Job 1's worker $\mathrm{W}_{4}$ and parameter server $\mathrm{PS}_{2}$ are both on the same machine, they communicate at the internal link rate $b_{1}^{(i)}$.
On the other hand, since Job 1's worker $\mathrm{W}_{3}$ and parameter server $\mathrm{PS}_{2}$ are on different physical machines, they communicate at the external link rate $b_{1}^{(e)}$.
In practice, it usually holds that $b_{i}^{(e)} \ll b_{i}^{(i)}$.
%Usually in practice, parameters of the same job are evenly divided among its parameter servers (e.g., if three parameter servers are allocated for a job, then each will possess one third of the parameters).
%However, to reduce the implementation complexity of workers, the parameters divisions are made transparent to the workers.
Next, we calculate the amount of time for a worker on machine $h$ to process a sample.
We use $F_{i}$ to denote the global batch size of job $i$, which is a fixed constant across all time-slots.\footnote{We note that this fixed global batch size requirement is {\em compliant} with the standard SGD implementation~\cite{Bazaraa_Sherali_Shetty_93:NLP} and important for ensuring convergence~\cite{Goyal17:SGD}.
In contrast, the global batch size in some existing works on dynamic ML resource allocation (e.g.,~\cite{Bao18:ML_INFOCOM}) could be time-varying, which necessitates time-varying dynamic learning rate adjustments to offset correspondingly and further complicates the SGD implementation.
}
%\footnote{To address the problem of time-varying global batch size, one approach is to adjust learning rate in each time-slot.}
We assume $F_i$ is equally divided among workers, i.e., the local batch size at each worker is: $F_i/\sum_{h'\in\mathcal{H}}w_{ih'}[t]$.\footnote{ Most distributed ML frameworks (e.g., Tensorflow~\cite{Keras}) set the same local batch size to each worker for the distributed training.}
%(hence, number of samples in a mini-batch of a worker may vary with time $t$.
We assume symmetric link speed in both directions between a worker and a PS.
Let $g_{i}$ denote the size of gradients/parameters of job $i$.
% and download do not distinguish the sending and receiving gradients/parameters time.
%Then, in time-slot $t$, the roundtrip transmission time of job $i$ between a parameter server on machine $p$ and a worker on machine $p$ can be computed as:
%\begin{align}
%\frac{g_{i}(s_{ip}[t]/\sum_{h'\in\mathcal{H}} s_{ih'}[t])}{b_i(h,p)}
%\end{align}
Then, from a worker's perspective, to push gradients to and pull updated parameters from the PSs for job $i$,
the combined uplink/downlink communication time can be computed as:
$(2{g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t])/(\min_{p\in \mathcal{P}_i[t]}b_i(h,p)})$,
where the numerator term $g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t]$ follows from the assumption of even parameter distribution among the PSs, and the denominator is due to the fact that push/pull time is decided by the slowest link among all connections from the worker to all PSs (i.e., $\min_{p\in P_i[t]}b_i(h,p)$).
Hence, the average computation and communication time to process a sample on machine $h\in\mathcal{W}_i[t]$ for job $i$ in time slot $t$ can be computed as:
\begin{align*}
\underbrace{\tau_{i}}_{\substack{\mathrm{Training \,\, time} \\ {\mathrm{per \,\, sample}}}} \!\!\! + \underbrace{ \bigg(\frac{2g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t]}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)} \bigg) \bigg/ \bigg(\frac{F_i}{\sum_{h'\in\mathcal{H}}w_{ih'}[t]} \bigg) }_{\mathrm{Communication \,\, time \,\, per \,\, sample}}.
\end{align*}
%amount of time to train a sample on machine $h$ and then communicate the result with its parameter server(s).
%\textcolor{red}{The communication time is determined by the external (i.e, $b_i(h,p)=b_i^{(e)}$, if it exists) physical machine whose number of parameter servers related to that worker is largest. We note that $\frac{s_{ih}[t]}{\sum_{h\in\mathcal{H}}s_{ih}[t]}=1$ under internal communication.}
Recall that we focus on the BSP scheme, where all workers are synchronized before they proceed to the next iteration.
In other words, the total number of samples trained on machine $h\in\mathcal{W}_i[t]$ for job $i$ in time slot $t$ is determined by the slowest link among all connections from all workers to all PS (i.e., $\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)$).
It then follows that the {\em number of samples trained} on machine $h\in\mathcal{W}_i[t]$ for job $i$ in time-slot $t$ can be computed as:
\begin{align} \label{eqn_NumTrainedSamples}
\frac{w_{ih}[t]}{\tau_{i} + \Big( \frac{2g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t]}{\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)} \Big) \Big/ \Big( \frac{F_i}{\sum_{h'\in\mathcal{H}}w_{ih'}[t]}\Big)}.
\end{align}
Note that in practice, ML users usually specify a fixed ratio between worker number and PS number (e.g., often 1:1 \footnote{In practice, the ratio between numbers and parameter servers are specified by the user upon the job's submission (e.g., 1:1 in Kubernetes~\cite{Kubernetes}).}) when launching their training jobs to ensure appropriate coordinations between workers and PSs in terms of channel bandwidth, memory allocation, etc.
To model this practice, we define the ratio of worker number to PS number for each job $i$ as:
\begin{align}\label{eqn_WorkerPSRatio}
\gamma_i \triangleq \frac{\sum_{h'\in\mathcal{H}}w_{ih'}[t]}{\sum_{h'\in\mathcal{H}}s_{ih'}[t]}, \quad \forall i, t.
\end{align}
With $\gamma_{i}$, we can rewrite Eq.~(1) as:
\begin{align*}
\frac{w_{ih}[t]}{\tau_{i} +\frac{\gamma_i}{F_i} \frac{2g_i}{\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)}}.
\end{align*}
Suppose that, for job $i$, there are $K_{i}$ data samples in its training dataset.
%, which is estimated based on the open datasets from Pathmind ~\cite{pathmind}. \kliu{What is pathmind?}
In practice, $K_i \gg F_i$.
In ML systems, an epoch is defined as a round of training that exhausts all data samples.
We let $E_{i}$ denote the number of epochs needed by job $i$.
In this paper, we assume that
the epoch of each job is predetermined.
% in a certain range and can be known a prior.
This is because it is often difficult to estimate the required number of epochs for SGD-type methods' convergence.
Therefore, most SGD-type algorithms in practice stop after a fixed number of iterations (i.e., fixed number of epochs, see, e.g., ~\cite{Wang18:SpiderBoost} and references therein) to avoid excessive training delay.
% Moreover, the early stopping strategy is commonly used (e.g., scikit-learn). In such cases, we estimate the least number of iterations that can generalize well to unseen data based on the validation set.
%We let $F_i$ denote the global batch size where assumed to be equally processed by workers.
Then, the total number of samples to be processed for job $i$ over the entire training process is $E_iK_i$.
To make sure that there are sufficient workers allocated for job $i$ over the entire training horizon, we have:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} \label{eqn_min_workers_minibatch}
%&\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ih}[t]}{\tau_{i} + \frac{2g_i}{\min_{p\in \mathcal{P}_{i}[t]}b_{i}(h,p)}}\geq x_{i} E_{i}K_{i}M_{i},\forall i \in \mathcal{I}.
\!\!\!\!\!\! \sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ih}[t]}{\tau_{i} +\frac{\gamma_i}{F_i} \frac{2g_i}{\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)}} \!\geq\! x_{i} E_{i}K_{i},\forall i \in \mathcal{I}. \!\!\!\!
\end{align}
We note that, with co-located workers and parameter servers on each machine, Eq.~(\ref{eqn_min_workers_minibatch}) is {\em non-deterministic} due to the existence of the $\min\{\cdot\}$ operator.
As will be shown later, this non-determistic constraint makes the scheduling design far more complicated than related works~\cite{Li14:ML_Static_OSDI,Chilim14:ML_Static_OSDI,Chun16:Dolphin,Bao18:ML_INFOCOM}.
To model the fact that the largest number of assigned concurrent workers is no more than the global batch size %the number of $F_i$ in each time slot
(otherwise, some workers will be idle), we have:
\begin{align} \label{eqn_max_workers_datachunk}
\sum_{h \in \mathcal{H}}w_{ih}[t] \leq x_{i} F_i, \quad \forall i \in \mathcal{I}, a_{i} \leq t \leq T.
\end{align}
%\smallskip
{\bf 3) Resource Constraint Modeling:}
We let $\mathcal{R}$ denote the set of resources (e.g., CPU/GPU, memory, storage, etc.).
Let $\alpha_{i}^{r}$ and $\beta_{i}^{r}$ be the amount of type-$r$ resource required by a worker and a parameter server for job $i$, respectively.
Let $C_{h}^{r}$ be the capacity of type-$r$ resource on machine $h$.
To ensure the resources do not exceed type-$r$'s limit, we have:
\begin{align}
\label{capacity}\sum_{i \in \mathcal{I}}(\alpha_{i}^{r}w_{ih}[t]+\beta_{i}^{r}s_{ih}[t])\leq C_{h}^{r}, \forall t \in \mathcal{T}, r \in \mathcal{R}, h \in \mathcal{H}.
\end{align}
%In a distributed ML system,
%%the total number of bandwidth of workers should not be greater than the total number of bandwidth of parameter servers.
%%In other words,
%the parameter servers should not be the bottleneck during gradient/parameter exchanges.
%To this end, we let $B_{i}$ denote the communication data rate of job $i$'s parameter server.
%Then, we have:
%\begin{align} \label{eqn_worker_ps_bandwidth}
%\sum_{h \in \mathcal{W}_{i}[t]} \sum_{p \in \mathcal{P}_{i}[t]} \!\!\! w_{ih}[t]b_{i}(h,p) \leq \!\!\!\sum_{h\in \mathcal{P}_{i}[t]} \!\!\!s_{ih}[t]B_{i},\forall i\in \mathcal{I},t\in \mathcal{T}. \!\!\!
%%\sum_{h \in \mathcal{H}} \sum_{p \in \mathcal{H}} \mathbb{E} [ w_{ih}[t] ]b_{i}(h,p) \leq \sum_{h\in \mathcal{H}} \mathbb{E} [ s_{ih}[t] ] B_{i}, \forall i\in \mathcal{I}, \forall t \in \mathcal{T}. \!\!\!
%\end{align}
%In practice, the number of parameter servers is upper bounded by the number of workers in each time slot for any job.
%This can be modeled as follows:
%\begin{align} \label{eqn_workers_ps_bound}
%\sum_{h\in \mathcal{P}_{i}[t]} s_{ih}[t] \leq \sum_{h\in \mathcal{W}_{i}[t]} w_{ih}[t], \quad \forall i\in \mathcal{I},t\in \mathcal{T}.
%\end{align}
Note that for job $i$, its completion time $\tilde{t}_{i}$ corresponds to the latest time-slot where there remain some active workers allocated for it.
Therefore, we have:
\begin{align} \label{eqn_completion_time}
\tilde{t}_{i}=\arg \max_{t \in \mathcal{T}}\bigg\{\sum_{h \in \mathcal{H}} w_{ih}[t]>0 \bigg\}, \quad \forall i \in \mathcal{I}.
\end{align}
To ensure that no workers and parameter servers are allocated before job $i$'s arrival, we have:
\begin{align}
\label{yz}&w_{ih}[t]=s_{ih}[t]=0, \quad \forall i \in \mathcal{I}, h \in \mathcal{H}, t<a_{i}.
\end{align}
{\bf 4) Objective Function and Problem Statement:}
Let $u_{i}(\tilde{t}_{i} - a_{i})$ be the utility function for job $i$, which is non-increasing with respect to the training time $\tilde{t}_{i} - a_{i}$.
The utility functions could play the role of various performance metrics based on job completion times (e.g., fairness).
%That is, the utility received by a job does not increase if its completion time is long.
In this paper, our goal is to maximize the overall utility for all jobs.
Putting all constraints and the objective function together, the offline (with knowledge of $a_{i}$, $\forall i$) distributed ML resource scheduling problem (DMLRS) can be formulated as:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} %\label{eqn_offline_formulation}
\textbf{DMLRS: } & \underset{\x,\w,\s}{\text{Maximize }} \sum_{i \in \mathcal{I}}x_{i} u_{i}(\tilde{t}_{i} - a_{i}) \nonumber\\
& \text{subject to } \text{Constraints (\ref{eqn_min_workers_minibatch}) -- (\ref{yz})}. \nonumber
\end{align}
Problem DMLRS is an integer nonlinear program, which is NP-hard in general~\cite{Hochbaum97}.
Also, Problem DMLRS involves two {\em non-deterministic} constraints in (\ref{eqn_min_workers_minibatch}) and (\ref{eqn_completion_time}), which are not amenable for conventional optimization techniques.
Moreover, the arrivals $\{a_{i}, \forall i\}$ are often unknown in practice, which necessitates {\em online optimization}.
Overcoming these challenges constitutes the rest of this paper.
To conclude this section, we summarize the key notation used in this paper in Table~\ref{table:notation} for easy reference.
\begin{table}[t!]
\centering
\caption{Notation.}
\label{table:notation}
\vspace{-.1in}
\begin{tabular}{| c | l |}
\hline
$\mathcal{I/T}$ & The set of jobs$/$System timespan \\ \hline
% $\mathcal{T}$ & System timespan \\ \hline
$\tilde{t}_{i}/a_{i}$ & Completion time of job $i$ $/$Arrival time of job $i$ \\ \hline
% $a_{i}$ & Arrival time of job $i$ \\ \hline
$u_{i}(\cdot)/K_{i}$ & Job $i$'$s$ utility function$/$Number of samples in $i$ \\ \hline
$\mathcal{R/H}$ & The set of resource types$/$The set of machines \\ \hline
% $K_{i}$ & \# of samples in $i$ \\ \hline
$E_{i}/F_{i}$ &\# of training epochs/Global batch size for job $i$ \\ \hline
% $M_{i}$ & Number of mini-batches in a data chunk for job $i$ \\ \hline
% $F_{i}$ & Global batch size for job $i$ \\ \hline
% $\mathcal{H}$ & Set of physical machines\\ \hline
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
$x_{i}$ & Admission decision variable to accept job $i$ or not \\ \hline
$C_{h}^{r}$ & Capacity of type-$r$ resource on server $h$ \\ \hline
$\alpha_{i}^{r}$ & Type-$r$ resource required by a worker in job $i$ \\ \hline
$\beta_{i}^{r}$ & \begin{tabular}[c]{@{}l@{}l@{}}Type-$r$ resource required by a PS in job $i$\end{tabular} \\ \hline
$w_{ih}[t]$ & Number of workers of job $i$ on server $h$ in $t$ \\ \hline
$s_{ih}[t]$ & Number of PSs of job $i$ on server $h$ in $t$ \\ \hline
%$B_{i}$ & Bandwidth offered by a parameter server of job $i$; \\ \hline
$b_{i}(h,p)$ &\begin{tabular}[c]{@{}l@{}l@{}}Bandwidth consumed by a worker of job $i$, where \\ $b_{i}(h,p) =b_{i}^{(e)}$, if $h\neq p$ or $b_{i}^{(i)}$, otherwise. \end{tabular} \\ \hline
$\tau_{i}$ & Time to train a sample for job $i$ \\ \hline
$g_{i}$ &\begin{tabular}[c]{@{}l@{}}Size of gradients and parameters for job $i$\end{tabular} \\ \hline
$\mathcal{W}_{i}[t]$ & \begin{tabular}[c]{@{}l@{}}Set of physical machines containing workers\\ for job $i$ in $t$\end{tabular} \\ \hline
$\mathcal{P}_{i}[t]$ & \begin{tabular}[c]{@{}l@{}}Machines containing parameter servers\\ for job $i$ in $t$\end{tabular} \\ \hline
$x_{\pi_i}$ & \begin{tabular}[c]{@{}l@{}} Binary decision variable to select schedule $\pi$ \\for job $i$ or not\end{tabular} \\ \hline
$\tilde t_{\pi_i}$ & The completion time slot of job $i$ with schedule $\pi$\\ \hline
$w_{ht}^{\pi_i}$ & \# of workers on server $h$ in $t$ for job $i$ in schedule $\pi$\\ \hline
$s_{ht}^{\pi_i}$ & \# of PSs on server $h$ for schedule $\pi$ in $t$ \\ \hline
$\Pi_{i}$ & Set of all feasible schedules for job $i$\\ \hline
$\gamma_i$ & The ratio of worker number to PS number for job $i$\\ \hline
$\rho_h^r[t]$ & Allocated type-$r$ resource on machine $h$ in time $t$ \\ \hline
$Q_h^r(\cdot)$ & Price function for type-$r$ resource on machine $h$\\ \hline
\end{tabular}
% \end{tabularx}
\vspace{-.1in}
\end{table}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Online Scheduling Algorithm Design} \label{sec:alg}
In this section, we structure the key components of our online scheduling algorithm design for solving Problem~DMLRS into three steps from Sections~\ref{subsec:reformulation} to \ref{subsec:best_schedule}.
We state our main theoretical performance results in Section~\ref{subsec:performance_analysis}.
\subsection{Reformulation for Non-Deterministic Constraint (\ref{eqn_completion_time})} \label{subsec:reformulation}
The first challenge in solving Problem~DMLRS stems from the non-deterministic ``argmax'' structure in constraint (\ref{eqn_completion_time}).
%In the literature, a commonly used technique to handle argmax-type constraints is via an equivalent reformulation that enumerates all possible schedules in the system (see, e.g., \cite{Bao18:ML_INFOCOM}).
To address this challenge, we let $\Pi_{i}$ be the set of all feasible schedules for job $i \in \mathcal{I}$ that satisfy constraints (\ref{eqn_min_workers_minibatch}), (\ref{eqn_max_workers_datachunk}).
%and (\ref{eqn_worker_ps_bandwidth}).
Each schedule $\pi_{i} \in \Pi_{i}$ is defined by the numbers of workers $w_{ht}^{\pi_i}$ and parameter servers $s_{ht}^{\pi_i}$ allocated for job $i$ on machine $h$ in each time-slot $t$, i.e., $\pi_i \!\triangleq\! \{ w_{ht}^{\pi_i}, s_{ht}^{\pi_i}, \forall t \!\in\! \mathcal{T}, h \!\in\! \mathcal{H} \}$.
Note that $w_{ht}^{\pi_i}$ and $s_{ht}^{\pi_i}$ are constants, not to be confused with decision variables $w_{ih}[t]$ and $s_{ih}[t]$.
We define a binary variable $x_{\pi_i} \!\in\! \{0,1\}$ that is equal to 1 if job $i$ is admitted and scheduled under schedule $\pi_{i}$, or 0, otherwise. %, $\forall i \in \mathcal{I}, \pi \in \Pi_{i}$.
Clearly, due to the combinatorial nature, $|\Pi_{i}|$ is exponential.
We let $\tilde{t}_{\pi_i}$ denote job $i$'s completion time under schedule $\pi_i$.
%{\color{red}We also let $w_{ih}^{\pi}[t]$ and $z_{ih}^{\pi}[t]$) denote the number of workers and parameter servers specified by schedule $\pi$ on machine $h \in \mathcal{H}$ in $t \in \mathcal{T}$ for job $i \in \mathcal{I}$, respectively.}
Then, one can equivalently reformulate Problem DMLRS as:
\begin{align} %\label{eqn_reformulation}
& \textbf{R-DMLRS:} \nonumber \\
& \underset{\x}{\text{Maximize }}\sum_{i \in \mathcal{I}} \sum_{\pi_{i} \in \Pi_{i}} x_{\pi_i} u_{i}(\tilde{t}_{\pi_i} - a_{i}) \nonumber \\
%&- \gamma\max \left\{0, x_{\pi_i} E_{i}K_{i}M_{i} -\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ht}^{\pi_i}}{\tau_i + \frac{2g_i}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)}} \right\} \nonumber\\
%& \nonumber \\
\label{eqn_rsc_limit}& \text{subject to } \sum_{i\in \mathcal{I}} \sum_{\pi_i \in \Gamma(t,h)} (\alpha_{i}^{r} w_{ht}^{\pi_i} + \beta_{i}^{r} s_{ht}^{\pi_i}) x_{\pi_i} \leq C_{h}^{r}, \nonumber \\
%
&\hspace{1in} \forall t \in \mathcal{T}, r\in \mathcal{R}, h \in \mathcal{H}, \\
\label{eqn_sch_select} &\hspace{.6in} \sum_{\pi_i \in \Pi_{i}} x_{\pi_i} \leq 1, \quad \forall i \in \mathcal{I}, \\
%%
&\hspace{.6in} x_{\pi_i} \in \{0,1\}, \quad \forall i \in \mathcal{I}, \pi_i \in \Pi_{i}, \nonumber
\end{align}
\noindent where we use $\Gamma(t,h)$ to represent the set of feasible schedules that use machine $h$ to deploy workers or parameter servers in time-slot $t$.
Constraint (\ref{eqn_rsc_limit}) guarantees that, in any time-slot $t$ and on any machine $h$, the total amount of consumed type-$r$ resources will not exceed the capacity limit $C_{h}^{r}$.
Constraint (\ref{eqn_sch_select}) ensures that, for each job $i$, at most one feasible schedule from $\Pi_{i}$ will be selected.
%
Note that Problem R-DMLRS is an integer linear program (ILP) and a feasible solution to Problem R-DMLRS has a corresponding feasible solution to the original Problem DMLRS, % with the same objective value,
and vice versa.
Notice that the non-deterministic constraint (\ref{eqn_completion_time}) {\em no longer exists} in Problem~R-DMLRS.
Further, if relaxing binary $x_{\pi_i}$-variables to real-valued, Problem R-DMLRS is a linear program (LP).
However, it remains difficult to solve Problem~R-DMLRS since it has an {\em exponential} number of $x_{\pi_i}$-variables due to the combinatorial nature of feasible schedules.
We will address this challenge in the next subsection.
%Therefore, directly solving Problem~R-DMLRS remains intractable.
%In what follows, we adopt a primal-dual online algorithmic framework, which is an effective approach to address this kind of challenge in the literature (see, e.g., ~\cite{Buchbinder09,Bao18:ML_INFOCOM}).
\subsection{An Online Primal-Dual Framework for R-DMLRS}
In what follows, we adopt a primal-dual online algorithmic framework to reduce the number of binary variables, which is an effective approach to address this kind of challenge in the literature (see, e.g., \cite{Buchbinder09,Bao18:ML_INFOCOM}).
Note that, in the dual of Problem~R-DMLRS, the number of dual variables is polynomial.
Meanwhile, although there are an exponential number of constraints in the dual problem, one only needs to be concerned with the set of active (binding) constraints, which are easier to deal with.
%
To see this, we associate dual variables $p_{h}^{r}[t] \geq 0$, $\forall t\in \mathcal{T}$, $h \in \mathcal{H}$, $r \in \mathcal{R}$ and $\lambda_{i} > 0$, $i \in \mathcal{I}$, with (\ref{eqn_rsc_limit}) and (\ref{eqn_sch_select}), respectively.
Then, following the standard procedure of dualization (relaxing the integrality constraints), we obtain the following dual problem:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} \label{eqn_reformulation_dual}
& \textbf{D-R-DMLRS:} \nonumber \\
& \underset{\vlambda,\p}{\text{Minimize }} && \hspace{-.3in} \sum_{i \in \mathcal{I}} \lambda_{i} + \sum_{t\in\mathcal{T}}\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}p_{h}^{r}[t] C_{h}^{r} & \\
& \text{subject to} && \hspace{-.3in} \lambda_{i} \geq u_{i}(\tilde{t}_{\pi_i} - a_{i}) - \sum_{t \in \mathcal{T}(\pi_i)} \sum_{h \in \mathcal{H}(\pi_i[t])} \sum_{r \in \mathcal{R}} (\alpha_{i}^{r} w_{ht}^{\pi_i} \nonumber\\
\label{eqn_dualconstr} & && \hspace{-.05in} + \beta_{i}^{r} s_{ht}^{\pi_i}) p_{h}^{r}[t], \quad \forall i \in \mathcal{I}, \pi_i \in \Pi_{i}, \\
& && \hspace{-.3in} p_{h}^{r}[t] \geq 0, \quad \forall t \in \mathcal{T}, h \in \mathcal{H}, r \in \mathcal{R}, \nonumber\\
& && \hspace{-.3in} \lambda_{i} \geq 0, \quad \forall i \in \mathcal{I}, \nonumber
\end{align}
\noindent where $\mathcal{T}(\pi_i)$ denotes the time-slots utilized by schedule $\pi_i$ and $\mathcal{H}(\pi_i[t])$ denotes the set of machines containing workers and/or parameter servers under $\pi_i$ in time-slot $t$.
Here, $p_{h}^{r}[t]$ can be viewed as the price for type-$r$ resource in time $t$.
Then, the right-hand-side (RHS) of (\ref{eqn_dualconstr}) can be interpreted as job utility minus overall resource cost of job $i$ using schedule $\pi_i$.
Thus $\lambda_{i}\geq 0$ can be viewed as the payoff of admitting job $i$ with $\pi_i$.
Next, we examine the properties of Problem D-R-DMLRS.
To minimize (\ref{eqn_reformulation_dual}), we tend to reduce $\lambda_{i}$ and $p_{h}^{r}[t]$ until they hit zero.
However, as $\lambda_{i}$ and $p_{h}^{r}[t]$ decrease, the left-hand-side (LHS) and RHS of (\ref{eqn_dualconstr}) decreases and increases, respectively (the term
%\begin{multline*}
$u_{i}(\tilde{t}_{\pi_i}-a_{i})$
%%
%\gamma\max \left\{0, x_{\pi_i} E_{i}K_{i}M_{i} -\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ht}^{\pi_i}}{\tau_i + \frac{2g_i}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)}} \right\}
%\end{multline*}
in the RHS of (\ref{eqn_dualconstr}) is a constant given $\pi_i$).
Therefore, $\lambda_{i}$ will drop to a value $\lambda_{i}^{*}$, which is equal to maximum of the RHS of (\ref{eqn_dualconstr}) achieved by some schedule $\pi_{i}^{*}$ and dual price $p_{h}^{r*}[t]$, i.e.,
\begin{align*}
&\lambda_{i}^{*} \!=\! u_{i}(\tilde{t}_{\pi_i^*}\!-\!a_{i})
%
%&-\! \gamma\max \left\{0, x_{\pi_i^*} E_{i}K_{i}M_{i}\!-\!\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ht}^{\pi_i}}{\tau_i + \frac{2g_i}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)}} \right\}\nonumber\\
%
\!-\!\!\sum_{t \in \mathcal{T}(\pi_i^*)} \! \sum_{h \in \mathcal{H}(\pi_i^{*}[t])} \! \sum_{r \in \mathcal{R}} (\alpha_{i}^{r} w_{ht}^{\pi_i^{*}} \!+\! \beta_{i}^{r} s_{ht}^{\pi_i^{*}}) p_{h}^{r*}[t].
\end{align*}
This optimality structural insight implies that Problem~D-R-DMLRS is equivalent to finding an optimal schedule $\pi_i^{*}$ and dual price $p_{h}^{r*}[t]$ to maximize the RHS of (\ref{eqn_dualconstr}).
The above insights motivate the following primal-dual-based algorithm as shown in Algorithm~1.
% which motivates the following primal-dual~\cite{Buchbinder09} online resource scheduling (PD-ORS) framework in Algorithm~1:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~1:} {Primal-Dual Online Resource Scheduling (PD-ORS).}
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let $w_{ih}[t]=0$, $s_{ih}[t]=0$, $\forall i,t,h$. Let $\rho_{h}^{r}[t] = 0$, $\forall h,r,t$. Choose some appropriate initial values for $p_{h}^{r}[0]$.
\end{enumerate}
\noindent {\bf Main Loop:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Upon the arrival of job $i$, determine a schedule $\pi_{i}^{*}$ to maximize the RHS of (\ref{eqn_dualconstr}) and its corresponding payoff $\lambda_{i}$ using {\bf Algorithm~2} ({\em to be specified}).
%
\item[3.] If $\lambda_{i} > 0$, set $x_{i} = 1$.
Set $w_{ih}[t]$ and $s_{ih}[t]$ according to schedule $\pi_{i}^{*}$, $\forall t \in \mathcal{T}(\pi_{i}^{*})$, $h\in\mathcal{H}(\pi_{i}^{*}[t])$.
\begin{itemize}
\item Update $\rho_{h}^{r}[t] \leftarrow \rho_{h}^{r}[t] + \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r}s_{ih}[t]$, $\forall t \in \mathcal{T}(\pi_{i}^{*})$, $h\in\mathcal{H}(\pi_{i}^{*}[t])$, $r \in \mathcal{R}$.
\item Update $p_{h}^{r}[t] = Q_{h}^{r}(\rho_{h}^{r}[t])$, $\forall t \in \mathcal{T}(\pi_{i}^{*})$, $h\in\mathcal{H}(\pi_{i}^{*}[t])$, $r \in \mathcal{R}$.
Schedule job $i$ based on $\pi_{i}^{*}$ and go to Step 2.
\end{itemize}
%
\item[4.] If $\lambda_{i} \leq 0$, set $x_{i} = 0$ and reject job $i$ and go to Step 2.
\end{enumerate}
\smallskip
\hrule
\medskip
The intuition of Algorithm~1 is as follows:
By the complementary slackness condition of the Karush-Kuhn-Tucker (KKT) conditions~\cite{Bazaraa_Sherali_Shetty_93:NLP}, the primal constraint (\ref{eqn_sch_select}) must be tight when dual variable $\lambda_{i} > 0$, which implies that $x_{i} =1$ (Step 3) in Problem DMLRS.
Otherwise, if $\lambda_{i} = 0$, then the RHS of (\ref{eqn_dualconstr}) is non-positive, meaning the utility is low compared to the cost of resource consumption under schedule $\pi_{i}^{*}$.
Therefore, we should reject job $i$ ($x_{i} = 0$ in Step 4).
However, in order for the PD-ORS algorithm to work, two challenging components need to be specified: the schedule $\pi_{i}^{*}$ and how to update the cost function $Q_{h}^{r}(\cdot)$\footnote{
Here, we note that the ``cost function'' $Q_{h}^{r}(\cdot)$ is interpreted from servers' perspective rather than jobs' perspective.
Specifically, higher cost means servers allocated more resources to jobs, which implies higher utility for the jobs since jobs receive more resources from servers.}
In what follows, we will first focus on designing $Q_{h}^{r}(\cdot)$ and defer the challenging problem of finding $\pi_{i}^{*}$ to Section~\ref{subsec:best_schedule}.
For the design of $Q_{h}^{r}(\cdot)$,
%we adopt the existing approach in the literature ~\cite{Buchbinder09,Bao18:ML_INFOCOM} and
consider the following choice of $Q_{h}^{r}(\cdot)$:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} \label{eqn_cost_update_fn}
Q_{h}^{r}(\rho_{h}^{r}[t]) = L(U^{r}/L)^{\frac{\rho_{h}^{r}[t]}{C_{h}^{r}}},
\end{align}
where constants $U^{r}$, $\forall r$, and $L$ are defined as follows:
%$\textcolor{red}{U^{r} \triangleq \gamma\max_{i\in\mathcal{I}}\frac{u_{i}(\ceil{E_{i}M_{i}\tau_{i}}-a_{i})}{\alpha_{i}^{r}+\beta_{i}^{r}}?}$
\begin{align}
\label{eqn_Ur} & \!\!\!\!\!\! U^{r} \!\triangleq\!\max_{i\in\mathcal{I}}\frac{u_{i}(\ceil{\frac{E_iK_i}{F_i}(\tau_{i}+2g_{i}\gamma_i/(b_i^{(i)}F_i))}-a_{i})}{\alpha_{i}^{r}+\beta_{i}^{r}},\forall r\in\mathcal{R},\!\!\!\!\!\! \\
%
\label{eqn_L} & \!\!\!\!\!\! L \triangleq \min_{i\in\mathcal{I}}\frac{1/(2\mu) u_{i}(T-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_i^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}. \!\!\!\!\!\!
\end{align}
The scaling factor $\mu$ in the definition of $L$ satisfies as follows:
\begin{align*}
\frac{1}{\mu}\leq \frac{\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})}{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}},\forall i\in\mathcal{I}.
\end{align*}
% $\frac{1}{\mu}\leq \frac{\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})}{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}},\forall i\in\mathcal{I}$.
Here, $U^{r}$ is the maximum unit-resource job utility to deploy workers and parameter servers with type-$r$ resource .
Here, $u_{i}(\ceil{\frac{E_iK_i}{F_i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(i)}F_i))}-a_{i})$ is the largest utility job $i$ can achieve by using the maximum number of {\em co-located} workers and parameter servers (hence communicating rate is $b_{i}^{(i)}$) at all times during all $E_{i}$ epochs, so that $\ceil{\frac{E_iK_i}{F_i}(\tau_{i}\!+\!2g_{i}\gamma_i/(b_{i}^{(i)}F_i))}\!-\!a_{i}$ is the earliest possible job completion time.
Similarly, $L$ represents the minimum unit-time unit-resource job utility among all jobs, with $u_{i}(T-a_{i})$ being the smallest utility for job $i$, and workers and parameter servers communicate at small external rate $b_{i}^{(e)}$.
We use $\rho_h^r[t]$ to denote the allocated amount of type-$r$ resource to machine $h$ for (future) time slot $t$.
% \textcolor{red}{Under external communication case, we have $\frac{s_{ih}[t]}{\sum_{h\in\mathcal{H}}s_{ih}[t]}<1$, and by relaxing it to 1, we can get the lower bound of $L$.
% We use $\rho_h^r[t]$ to denote the allocated amount of type-$r$ resource to machine $h$ at time $t$.}
The intuition behind the $Q_{h}^{r}(\cdot)$ function is as follows:
%(proofs follow similarly from ~\cite{Buchbinder09,Bao18:ML_INFOCOM} and are omitted due to space limitation):
i) At $t\!=\!0$, $\rho_{h}^{r}[0]\!=\!0, \forall h\in\mathcal{H},r\in\mathcal{R}$.
Hence, the price $p_{h}^{r}[0]=L$ is the lowest, $\forall h,r$, and any job can be admitted;
ii) As allocated resources increases, the price increases exponentially fast to reject early coming jobs with low utility and to reserve resources for later arrived jobs with higher utility;
iii) When some type-$r$ resource is exhausted, i.e., $\rho_{h}^{r}[t] = C_{h}^{r}, \exists r\in\mathcal{R}$, $Q_{h}^{r}[C_{h}^{r}]=U^{r}$ and no job that requires type-$r$ resources will be admitted since the $U^{r}$ is the highest price.
As will be shown later, this price function leads to a logarithmically scaling competitive ratio in online algorithm design.
Note that computing the price function in Algorithm~1 requires the information of constants $U^r$, $L$, which can usually be estimated empirically based historical data.
Here, we point out a few interesting insights on the design choices of the cost function in Eq.~\eqref{eqn_cost_update_fn}.
Note that $U^r$ and $L$ are defined in Eqns~(\ref{eqn_Ur}) and (\ref{eqn_L}), respectively.
Here, we intentionally choose $U^r$ to be dependent on $r$ and $L$ to be independent on $L$ due to the following reasons:
First, the rationality of choosing an upper bound $U^r$ that varies with different resource types is to ensure that when some type-$r$ resource is exhausted, no more jobs that require type-$r$ resource should be allocated.
In other words, when the allocated amount of type-$r$ resource reaches the capacity of physical machine $h$, i.e., $\rho_h^r[t]=C_n^r, \exists r\in\mathcal{R}$, the price $p_h^r=Q_h^r(C_h^r)$ should reach the upper bound $U^r$, indicating that any job that requires type-$r$ resource will not be allocated to $h$.
However, jobs that do not require type-$r$ resource should still be able to be scheduled on machine $h$.
For example, jobs that do not require GPU can still be placed on a machine with no available GPUs.
Second, the reason that we choose the lower bound $L$ to be independent of any resource type $r$ is to yield a larger ratio of $\frac{U^r}{L}$.
%to admit jobs incoming at $t=0$. Also, as the allocated resources increases, the price increases exponentially (see Eqn.~(\ref{eqn_cost_update_fn})).
The larger the ratio of $\frac{U^r}{L}$ is, the greater the price will be.
Intuitively, the ratio $\frac{U^r}{L}$ can be interpreted as the scheduling ``uncertainty,'' which increases as the ratio gets larger, implying the price function reacts to the accumulative resource consumption more aggressively.
Thus, choosing $L$ to be independent of resource type $r$ allows the price function $Q_h^r(\cdot)$ to react more aggressively to the accumulative allocated resource amount.
%Later, we show on Page~10, Theorems~5 and 6 that our choice have a good log performance.
We note that one can also choose the lower bound to be dependent on resource type $r$ by replacing $L$ with $L^r$.
By doing so, the log-scaling theoretical competitive ratio in Theorem~\ref{thm_Competitive_1} still holds and the proof in Appendix~\ref{appdx:thm_Competitive} only needs to be slightly updated with the new notation $L^r$.
However, the empirical performance of using $L^r$ as lower bound is worse since the price function reacts less aggressively to the accumulative allocated resources.
\begin{figure*}[t!]
\centering
\subfigure[{$|\mathcal{P}_{i}[t]|\ne 1$.}]{%
\label{fig:IntExt1}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_1.eps}}%
\hspace{6pt}%
\subfigure[{$|\mathcal{P}_{i}[t]|=1,|\mathcal{W}_{i}[t]|\ne 1$.}]{%
\label{fig:IntExt2}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_2.eps}}%
\hspace{6pt}%
\subfigure[{$|\mathcal{P}_{i}[t]|\!\!=\!\!|\mathcal{W}_{i}[t]|\!\!=\!\!1,\mathcal{P}_{i}[t]\!\!\ne\!\!\mathcal{W}_{i}[t]$.}]{%
\label{fig:IntExt3}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_3.eps}}%
\hspace{6pt}%
\subfigure[{$|\mathcal{P}_{i}[t]|\!\!=\!\!|\mathcal{W}_{i}[t]|\!\!=\!\!1,\mathcal{P}_{i}[t]\!\!=\!\!\mathcal{W}_{i}[t]$.}]{%
\label{fig:IntExt4}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_4.eps}}%
\vspace{-.08in}
\caption{Values of $\frac{2g_{i}}{\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)}$ under various settings of $\mathcal{P}_{i}[t]$ and $\mathcal{W}_{i}[t]$.
% Clearly,
Here,
$\frac{2g_{i}}{\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)} = 2\frac{g_{i}}{b_{i}^{(i)}}$ if and only if in (d).}%
\label{fig:IntExt}%
\vspace{-.15in}
\end{figure*}
\vspace{-.1in}
\subsection{Determining Schedule $\pi_{i}^{*}$ in Step 2 of Algorithm 1} \label{subsec:best_schedule}
Now, consider the problem of finding a schedule $\pi_{i}^{*}$ in Step 2 of Algorithm~1 to maximize the RHS of (\ref{eqn_dualconstr}).
First, we note that {\em any} schedule for job $i$ has a unique completion time $\tilde{t}_{i}$, including the maximizer $\pi_{i}^{*}$ for (\ref{eqn_dualconstr}).
Hence, the problem of finding the maximum RHS of (\ref{eqn_dualconstr}) can be decomposed as:
%This key subproblem can be stated as follows:
\begin{align} \label{eqn_ProbFindSch}
\!\!\!\underset{{\tilde{t}_{i}}}{\text{Max}} \left\{
\begin{array}{l}
\underset{{ \w,\s }}{\text{Max}} \,\, u_{i}(\tilde{t}_{i} \!-\! a_{i}) \!-\! \sum\limits_{t\in\mathcal{T}}\sum\limits_{h\in \mathcal{H}}\sum\limits_{r\in\mathcal{R}} p_{h}^{r}[t] \\
%
\hspace{1.1in} \times (\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]) \\
%
\text{s.t. }\ \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t] \leq \hat{C}_{h}^{r}[t], \\
%
\hspace{1.1in} \forall t\in \mathcal{T},r\in\mathcal{R},h\in\mathcal{H}, \\
%
\hspace{.3in} \text{Constraints (\ref{eqn_min_workers_minibatch})(\ref{eqn_max_workers_datachunk})(\ref{yz}) for } x_{i}=1,
\end{array}
\right\}, \!\!\!\!\!
\end{align}
\noindent where $\hat{C}_{h}^{r}[t] \triangleq C_{h}^{r} - \rho_{h}^{r}[t]$.
Note that in the inner problem, $u_{i}(\tilde{t}_{i} \!-\! a_{i})$ is a constant for any given $\tilde{t}_{i}$.
Thus, the inner problem can be simplified as:
\begin{align}
\label{eqn_obj_given_t} \underset{\w,\s}{\text{Minimize }} & \sum_{t \in [a_{i}, \tilde{t}_{i}]} \sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}} p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]) \\
%
\label{eqn_workload_coupling} \text{subject to } & \sum_{t\in[a_{i},\tilde{t}_{i}]} \sum_{h\in\mathcal{H}} \frac{w_{ih}[t]}{\tau_{i}+\frac{\gamma_i}{F_i} \frac{2g_{i}}{\min_{p\in \mathcal{P}_{i}[t],h'\in\mathcal{W}_i[t]}b_{i}(h',p)}}\geq V_{i}, \!\!\! \\
%
\label{eqn_rsrc_limit} & \alpha_{i}^{r} w_{ih}[t] \!+\! \beta_{i}^{r} s_{ih}[t] \!\leq\! \hat{C}_{h}^{r}[t], \forall r,h, \forall t \!\in\! [a_{i},\tilde{t}_{i}], \!\!\!\\
&\text{Constraint (\ref{eqn_max_workers_datachunk}) for all } t\in[a_{i},\tilde{t}_{i}],\nonumber
\end{align}
where $V_{i} \triangleq E_{i} K_{i}$ represents the total training workload
(total count of samples trained, where a sample is counted $E_i$ times if trained for $E_i$ times).
Note that in Problem~(\ref{eqn_obj_given_t}), the only coupling constraint is (\ref{eqn_workload_coupling}).
This observation inspires a dynamic programming approach to solve Problem (\ref{eqn_obj_given_t}).
Consider the problem below if training workload at time $t$ is known (denoted as $V_{i}[t]$):
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
\label{eqn_obj_given_t2} \underset{w_{ih}[t],s_{ih}[t],\forall h}{\text{Minimize }} & \hspace{.1in} \sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}} p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]) \\
%
\label{eqn_nondeterministic} \text{subject to } & \hspace{.1in} \sum_{h\in\mathcal{H}} \frac{w_{ih}[t]}{\tau_{i}+\frac{\gamma_i}{F_i} \frac{2g_{i}}{\min_{p\in \mathcal{P}_{i}[t],h'\in\mathcal{W}_i[t]}b_{i}(h',p)}}\geq V_{i}[t], \\
%
& \hspace{.1in} \text{Constraints (\ref{eqn_max_workers_datachunk})(\ref{eqn_rsrc_limit}) for the given } t.\nonumber
\end{align}
Let $\Theta(\tilde{t}_{i},V_{i})$ and $\theta(t,V_{i}[t])$ denote the optimal values of Problems~(\ref{eqn_obj_given_t}) and (\ref{eqn_obj_given_t2}), respectively.
Then, Problem~(\ref{eqn_obj_given_t}) is equivalent to the following dynamic program:
\begin{align} \label{eqn_2d_dp}
\Theta(\tilde{t}_{i},V_{i}) = \min_{v\in[0,V_{i}]} \left\{ \theta(\tilde{t}_{i},v) + \Theta(\tilde{t}_{i}-1,V_{i}-v) \right\}.
\end{align}
We find the optimal workload $v$ to be completed in time slot $\tilde{t}_{i}$ by enumerating it from 0 to $E_iK_i$, and the remaining workload $E_iK_i-v$ will be carried out in time span $[a_i,\tilde{t}_{i}-1]$.
The optimal workload would be the schedule with minimum costs, i.e., the objective function of Problem~(\ref{eqn_obj_given_t2}) is minimum.
Then we proceed to the next time slot $\tilde{t}_{i}-1$ with the workload $E_iK_i-v$, which is the same as finding the optimal schedule and cost as the last time slot $\tilde{t}_{i}$ except in a smaller scale.
Then, by enumerating all $\tilde{t}_{i} \in [a_{i}, T]$ and solving the dynamic program $\Theta(\tilde{t}_{i},V_{i})$ in (\ref{eqn_2d_dp}) for every choice of $\tilde{t}_{i}$, we can solve Problem (\ref{eqn_ProbFindSch}) and determine the optimal schedule $\pi_{i}^{*}$.
We summarize this procedure in Algorithms~2 and 3:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~2:} Determine $\pi_{i}^{*}$ in Step 2 of Algorithm 1.
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let $\tilde{t}_{i}\!=\!a_{i}$. Let $\lambda_{i} \!=\! 0$, $\pi_{i}^{*} \!=\! \varnothing$, $w_{ih}[t] \!=\! s_{ih}[t] \!=\! 0$, $\forall i,t,h$.
\end{enumerate}
\noindent {\bf Main Loop:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Compute $\Theta(\tilde{t}_{i},V_{i})$ in (\ref{eqn_2d_dp}) using {\bf Algorithm 3}.
Denote the resulted schedule as $\pi_{i}$.
Let $\lambda'_{i} = u_{i}(\tilde{t}_{i}-a_{i}) - \Theta(\tilde{t}_{i},V_{i})$.
If $\lambda'_{i} > \lambda_{i}$, let $\lambda_{i} \leftarrow \lambda'_{i}$ and $\pi_{i}^{*} \leftarrow \pi_{i}$.
%
\item[3.] Let $\tilde{t}_{i} \leftarrow \tilde{t}_{i} + 1$. If $\tilde{t}_{i} > T$, stop; otherwise, go to Step 2.
\end{enumerate}
\smallskip
\hrule
\medskip
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~3:} Solving $\Theta(\tilde{t}_{i},V_{i})$ by Dynamic Programming.
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let {\em cost-min} = $\infty$, $\pi_{i} = \varnothing$, and $v = 0$.
\end{enumerate}
\noindent {\bf Main Loop:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Compute $\theta(\tilde{t}_{i}, v)$ using {\bf Algorithm~4} ({\em to be specified}).
Denote the resulted cost and schedule as {\em cost-v} and $\hat{\pi}_{i}$.
\item[3.] Compute $\Theta(\tilde{t}_{i}-1,V_{i}-v)$ by calling {\bf Algorithm~3} itself.
Denote the resulted cost and schedule as {\em cost-rest} and $\tilde{\pi}_{i}$.
%
\item[4.] If $\text{{\em cost-min}} > \text{{\em cost-v}} + \text{{\em cost-rest}}$ then $\text{{\em cost-min}} = \text{{\em cost-v}} + \text{{\em cost-rest}}$ and let $\pi_{i} \leftarrow \hat{\pi}_{i} \cup \tilde{\pi}_{i}$.
%
\item[5.] Let $v \leftarrow v+1$. If $v>V_{i}$ stop; otherwise go to Step 2.
\end{enumerate}
\smallskip
\hrule
\medskip
% \textcolor{red}{Note that the number of states $(t,v)$ in the dynamic programming for each job $i$ is $O(TK_iE_i)$, and the time complexity to execute dynamic programming function is $O(K_iE_i)$, thus we can have the $O(TK_i^2E_i^2)$ time complexity for dynamic programming since we save all the intermediate results to avoid re-computation.}
%{\em 2) Solving $\theta(t,v)$ (i.e., Problem (\ref{eqn_obj_given_t2}))}:
In Algorithm~3, however, how to compute $\theta(t,v)$ in Step 2 (i.e., Problem~(\ref{eqn_obj_given_t2})) is yet to be specified.
A challenge in solving (\ref{eqn_obj_given_t2}) is the {\em non-deterministic} constraint in (\ref{eqn_nondeterministic}), where $b_{i}(h,p)$ can be either $b_{i}^{(i)}$ or $b_{i}^{(e)}$.
%However, the workload constraint (\ref{eqn_nondeterministic}) is {\em non-deterministic} because $b_{i}(h,p)$ can take values either $b_{i}^{(i)}$ or $b_{i}^{(e)}$.
Therefore, we need to handle both cases.
To this end, we observe the following basic fact about $\frac{2g_{i}}{\min_{p \in \mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)}$ (also illustrated in Fig.~\ref{fig:IntExt}) as stated in Fact~\ref{fact1}.
We omit the proof of this fact due to its simplicity, which is illustrated in Fig.~\ref{fig:IntExt}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{fact} \label{fact1}
The function $(2g_{i}/\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)) = 2g_{i}/b_{i}^{(i)}$ if and only if $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]| =1$ and $\mathcal{P}_{i}[t] = \mathcal{W}_{i}[t]$; otherwise, $(2g_{i}/\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)) = 2g_{i}/b_{i}^{(e)}$.
\end{fact}
%\smallskip
With Fact~\ref{fact1}, we now consider the following two cases:
\smallskip
{\em Case 1): $b_{i}^{(i)}$ (Internal Communication):}
In Case 1), Fact~\ref{fact1} implies that Problem~(\ref{eqn_obj_given_t2}) reduces to a {\em single-machine} problem (i.e., discarding $\sum_{h\in\mathcal{H}}\{\cdot\}$ in (\ref{eqn_obj_given_t2}) and (\ref{eqn_nondeterministic})).
Note further that if we temporarily ignore the workload-coupling constraint (\ref{eqn_nondeterministic}) and use the worker-PS ratio in (\ref{eqn_WorkerPSRatio}), Problem~(\ref{eqn_obj_given_t2}) can be {\em decoupled} across resources and simplified as:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\begin{equation}\label{linear}
%\sum_{r\in\mathcal{R}}
%\begin{Bmatrix*}[l]
%&\hspace{-.1in} \text{Min} &\hspace{-.1in} p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t])\\
%%&\hspace{-.1in}&\hspace{-.1in}+ \gamma\max \{0, V_i[t]M_{i}\!-\!\frac{w_{ih}[t]}{\tau_i + \frac{2g_i}{b_i^{(i)}}}\}, \\
%&\hspace{-.1in} \text{s.t.} &\hspace{-.1in} \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]\leq \hat{C}_{h}^{r}[t], \\
%&&\hspace{-.1in}w_{ih}[t] = \gamma_is_{ih}[t],\\
%&&\hspace{-.1in} \text{Constraint }(\ref{eqn_max_workers_datachunk}) \text{ for given } r,h
%\end{Bmatrix*}, \!\!\!\!
%\end{equation}
%and it can be further simplified as follows:
\begin{equation}\label{linear}
\sum_{r\in\mathcal{R}}
\begin{Bmatrix*}[l]
&\hspace{-.1in} \text{Min} &\hspace{-.1in}p_{h}^{r}[t]s_{ih}[t](\alpha_{i}^{r} \gamma_i + \beta_{i}^{r} )\\
%&\hspace{-.1in}&\hspace{-.1in}+ \gamma\max \{0, V_i[t]M_{i}\!-\!\frac{w_{ih}[t]}{\tau_i + \frac{2g_i}{b_i^{(i)}}}\}, \\
&\hspace{-.1in} \text{s.t.} &\hspace{-.1in} s_{ih}[t] (\alpha_{i}^{r} \gamma_i+ \beta_{i}^{r})\leq \hat{C}_{h}^{r}[t], \\
&&\hspace{-.1in} \text{Constraint }(\ref{eqn_max_workers_datachunk}) \text{ for given } r,h
\end{Bmatrix*}, \!\!\!\!
\end{equation}
where each summand in (\ref{linear}) is an integer linear program (ILP) having a trivial solution $w_{ih}[t]=s_{ih}[t]=0$, $\forall h \in \mathcal{H}$.
However, $w_{ih}[t]=0$, $\forall h\in\mathcal{H}$, clearly violates the workload constraint ($\ref{eqn_nondeterministic}$).
Thus, when (\ref{linear}) is optimal, there should be {\em exactly one} machine $h' \in\mathcal{H}$ with $w_{ih'}[t]\geq 1$ and {\em exactly one} machine $h'' \in\mathcal{H}$ with $s_{ih''}[t] \geq 1$.
This observation shows that the optimal solution of (\ref{eqn_obj_given_t2}) {\em tends to favor} $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]|=1$ if the workload constraint (\ref{eqn_nondeterministic}) is not binding.
%which matches the internal case condition in Fact~\ref{fact1}.
Notice that the workload constraint (\ref{eqn_nondeterministic}) in Case 1) becomes $\gamma_is_{ih}[t] \geq V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(i)}F_i})$.
This implies the following simple solution:
We can first sort each physical machine $h$ according to $\sum_{r\in\mathcal{R}}p_h^r[t](\alpha_i^r\gamma_i+\beta_i^r)$ and calculate the minimum number of $s_{ih}[t]=V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(i)}F_i})/\gamma_i$ from the workload constraint.
The last step is to check if the machine satisfy the resource capacity constraint (\ref{eqn_rsrc_limit}) and constraint (\ref{eqn_max_workers_datachunk}).
%{\color{red}This observation suggests that, for the ``$b_{i}^{(i)}$'' case, we should check the workload constraint (\ref{eqn_nondeterministic}) on each machine one by one (i.e., ensuring $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]|=1$ and $\mathcal{P}_{i}[t] = \mathcal{W}_{i}[t]$). }
%In this setting, the workload constraint (\ref{eqn_nondeterministic}) becomes $\gamma_is_{ih}[t] \geq V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(i)}F_i})$.
%After checking all machines, choose, if any, the machine $h$ that satisfies (\ref{eqn_nondeterministic}) and has the lowest cost.
If so, we return the schedule $(w_{ih}[t],s_{ih}[t])$ and the corresponding cost value.
\smallskip
{\em Case 2): $b_{i}^{(e)}$ (External Communication):}
For those settings that do not satisfy $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]| =1$ and $\mathcal{P}_{i}[t] = \mathcal{W}_{i}[t]$, Fact~\ref{fact1} indicates that parameter servers and workers are communicating at external rate $b_{i}^{(e)}$.
In this case, the workload constraint (\ref{eqn_nondeterministic}) simply becomes: $\sum_{h \in \mathcal{H}} w_{ih}[t] \geq V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i})$.
%For convenience, we let $m_{w}[t] \triangleq \ceil{V_{i}[t]M_{i}(\tau_{i}+2g_{i}/b_{i}^{(e)})}$ and $m_{p}[t] \triangleq \ceil{m_{w}[t] b_{i}^{(e)}/B_{i}}$ represent the minimum required numbers of workers and parameter servers given workload $V_{i}[t]$, respectively.
Then, we can rewrite Problem~(\ref{eqn_obj_given_t2}) as:
\begin{align}
\label{eqn_obj_given_t3} \underset{w_{ih}[t],s_{ih}[t],\forall h}{\text{Minimize }} & \hspace{.1in} \sum_{h\in\mathcal{H}} p_{h}^{w}[t] w_{ih}[t] + p_{h}^{s}[t] s_{ih}[t] \\
%
\label{eqn_packing} \text{subject to } & \hspace{.1in} \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t] \leq \hat{C}_{h}^{r}[t], \,\, \forall h,r,\\
%
\label{eqn_packing2} &\hspace{.1in} \sum\nolimits_{h\in\mathcal{H}} w_{ih}[t] \leq F_i,\\
\label{eqn_cover1} &\hspace{.1in} \sum\nolimits_{h\in\mathcal{H}} w_{ih}[t] \geq V_{i}[t] \bigg( \tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i} \bigg),
%\label{eqn_cover2} &\hspace{.1in} \sum\nolimits_{h\in\mathcal{H}} s_{ih}[t] \geq m_{p}[t],
\end{align}
where $p_{h}^{w}[t] \triangleq \sum_{r\in\mathcal{R}} p_{h}^{r}[t] \alpha_{i}^{r}$ and $p_{h}^{s}[t] \triangleq \sum_{r\in\mathcal{R}} p_{h}^{r}[t] \beta_{i}^{r}$ denote the aggregated prices of all resources of allocating workers and PSs on machine $h$ in time $t$, respectively.
%Let denote $c_i\triangleq 1/(\tau_i\!+\! \frac{2g_i}{b_i^{(i)}})$, we can have the following equivalent problem under infeasible case:
%\begin{align}
%\label{eqn_obj_given_t4} \underset{w_{ih}[t],s_{ih}[t],\forall h}{\text{Minimize }} & \hspace{.1in} V_i[t]M_{i}\!+\!\sum_{h\in\mathcal{H}}( p_{h}^{w}[t]-c_i) w_{ih}[t]\!+\!p_{h}^{s}[t] s_{ih}[t]\\
%\label{feasible_packing} \text{subject to } & \hspace{.1in} \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t] \leq \hat{C}_{h}^{r}[t], \,\, \forall h,r,
%\end{align}
Unfortunately, Problem~(\ref{eqn_obj_given_t3}) is a highly challenging integer programming problem with generalized packing and cover type constraints (i.e., integer variables rather than 0-1 variables) in (\ref{eqn_packing})--(\ref{eqn_cover1}), respectively,
which is clearly NP-Hard.
% Therefore, Problem~(\ref{eqn_obj_given_t3}) is a generalized mixed cover/packing problem, which is clearly NP-Hard.
Also, it is well-known that there are {\em no} polynomial time approximation schemes (PTAS) even for the basic set-cover and bin-packing problems unless P = NP~\cite{Hochbaum97}.
In what follows, we will pursue an instance-dependent constant ratio approximation scheme to solve Problem~(\ref{eqn_obj_given_t3}) in this paper.
%\textbf{this appears conflicting with the non-ptas claim above}.
To this end, we propose a randomized rounding scheme:
First, we solve the linear programming relaxation of Problem~(\ref{eqn_obj_given_t3}).
Let $\{ \bar{w}_{ih}[t], \bar{s}_{ih}[t], \forall h,t \}$ be the fractional optimal solution.
We let $\delta\in(0,1]$ be a parameter.
Let $G_\delta$ be a constant {(the notation $G_\delta$ signifies that $G_{\delta}$ is dependent on $\delta$) to be defined later, and let $w'_{ih}[t] \!=\! G_\delta\bar{w}_{ih}[t], s'_{ih}[t] \!=\! G_\delta\bar{s}_{ih}[t]$, $\forall h,t$.
Then, we randomly round $\{ w'_{ih}[t], s'_{ih}[t], \forall h,t \}$ to obtain an integer solution as follows:
\begin{align}
\label{eqn_rounding_w} w_{ih}[t] &= \begin{cases}
\lceil w'_{ih}[t] \rceil, & \hspace{-.1in} \text{with probability } w'_{ih}[t] - \lfloor w'_{ih}[t] \rfloor,\\
\lfloor w'_{ih}[t] \rfloor, & \hspace{-.1in} \text{with probability } \lceil w'_{ih}[t] \rceil - w'_{ih}[t],
\end{cases} \!\!\!\\
%
\label{eqn_rounding_s} s_{ih}[t] &= \begin{cases}
\lceil s'_{ih}[t] \rceil, & \hspace{-.1in} \text{with probability } s'_{ih}[t] - \lfloor s'_{ih}[t] \rfloor,\\
\lfloor s'_{ih}[t] \rfloor, & \hspace{-.1in} \text{with probability } \lceil s'_{ih}[t] \rceil - s'_{ih}[t].
\end{cases}
\end{align}
We will later prove in Theorem~\ref{thm_Alg4_1} (when $0<G_\delta\leq 1)$ and Theorem~\ref{thm_Alg4_2} (when $G_\delta>1$) that the approximation ratio of this randomized rounding scheme in (\ref{eqn_rounding_w})-(\ref{eqn_rounding_s}) enjoys a ratio that is independent on the problem size.
%\textbf{so the ratio is not constant?}.
\smallskip
Lastly, summarizing results in Cases 1) -- 2) yields the following approximation algorithm for solving Problem~(\ref{eqn_obj_given_t2}):
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~4:} Solving $\theta(t,v)$ (i.e., Problem (\ref{eqn_obj_given_t2})).
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let $w_{ih}[t] \!=\! s_{ih}[t] \!=\! 0$, $\forall h$. Let $h\!=\!1$.
% Pick some $G$.
Pick some $\delta\in(0,1]$.
Let $G_\delta$ be defined as in Eq.~(\ref{eqn_Thm3_G}) or Eqn~(\ref{eqn_Thm4_G}).
Let $D \!=\! \lceil v(\tau_{i} + 2g_{i}\gamma_i/(b_{i}^{(i)}F_i)) \rceil$.
Let $h^* = \varnothing$. Let {\em cost-min}$=\infty$.
Choose some integer $S \geq 1$.
%Let $S = (1 - \delta/3)^{-1}$ if $G_\delta$ is defined as in Eq.~\eqref{eqn_Thm3_G} or $S = (1 - \delta/3m)^{-1}$ if $G_\delta$ is defined as in Eq.~\eqref{eqn_Thm4_G}.
Let $iter\leftarrow1$.
\end{enumerate}
\noindent {\bf Handling Internal Communication:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Sort machines in $\mathcal{H}$ according to $\sum_{r\in\mathcal{R}} p_h^r[t](\alpha_i^r\gamma_i+\beta_i^r)$ in non-decreasing order into $h_1,h_2,...,h_H$.
\item[3.] Calculate the minimum number of $s_{ih}[t]=V_i[t] \Big(\tau_i+\frac{2g_i\gamma_i}{b_i^{(i)}F_i} \Big)/\gamma_i$.
\item[4.] If Constraint (\ref{eqn_max_workers_datachunk}) is not satisfied, go to Step 7.
\item[5.] If Constraint (\ref{eqn_packing}) is not satisfied, go to Step 7.
\item[6.] Return {\em cost-min}$\sum_{r\in\mathcal{R}} p_h^r[t]s_{ih}[t](\alpha_i^r\gamma_i+\beta_i^r)$ and $h^* = h$.
\item[7.] Let $h\leftarrow h+1$. If $h>H$, stop; otherwise, go to Step 2.
%\item[2.] If constraint (\ref{eqn_packing}) is not satisfied, go to Step 7.
%
%\item[3.] If $w_{ih}[t] < D$, $w_{ih}[t] \leftarrow w_{ih}[t]+1$ and go to Step 2.
%
%\item[4.] If $w_{ih}[t] > K_{i}$, go to Step 7.
%
%%\item[5.] If $w_{ih}[t] b_{i}^{(i)} \!\!>\!\! s_{ih}[t] B_{i}$, $s_{ih}[t] \!\leftarrow\! s_{ih}[t]\!+\!1$ and go to Step 2.
%
%\item[6.] If {\em cost-min} $>p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}^{r}[t])$, then let {\em cost-min}$=p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}^{r}[t])$ and $h^* = h$.
%
%\item[7.] Let $h\leftarrow h+1$. If $h>H$, stop; otherwise, go to Step 1.
\end{enumerate}
\noindent {\bf Handling External Communication:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[8.] Solve the linear programming relaxation of Problem~(\ref{eqn_obj_given_t3}).
%with $m_{w}[t] \!\!=\!\! \ceil{v M_{i}(\tau_{i} \!+\! 2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}$.
%, $m_{p}[t] \!=\! \ceil{m_{w}[t] b_{i}^{(e)}/B_{i}}$.
Let $\{ \bar{w}_{ih}[t], \bar{s}_{ih}[t], \forall h,t \}$ be the fractional optimal solution.
\item[9.] Let $w'_{ih}[t] = G_\delta\bar{w}_{ih}[t], s'_{ih}[t] = G_\delta\bar{s}_{ih}[t]$, $\forall h,t$.
\item[10.] Generate an integer solution $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$ following the randomized rounding scheme in (\ref{eqn_rounding_w})--(\ref{eqn_rounding_s}).
%\item[11.] If $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$ is infeasible, go to Step 10.
\item[11.] If $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$ is infeasible or $iter< S$, then $iter\leftarrow iter+1$,
go to Step 10.
\end{enumerate}
\noindent {\bf Final Step:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[12.] Compare the solutions between internal and external cases. Pick the one with the lowest cost among them and return the cost and the corresponding schedule $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$.
\end{enumerate}
\smallskip
\hrule
\medskip
In the internal communication part of Algorithm~4, we first sort the machines and then check each machine one by one (Step 2).
We calculate the minimum number of $s_{ih}[t]$ needed to satisfy the learning workload demand $D$ (Step 3).
If Constraint (\ref{eqn_max_workers_datachunk}) is satisfied (Step 4), we further check the resource capacity constraint (\ref{eqn_rsrc_limit}) (Step 5).
%{\color{red}If the resource capacity constraint (\ref{eqn_packing}) is satisfied (Step 2), we increase workers to satisfy the learning workload demand $D$ (Step 3) and also increase parameter servers accordingly (Step 5).}
%{\color{red}If we detect a machine with lower cost, we update the cost and schedule accordingly (Step 6).}
If we detect a machine with all above constraints satisfied, we return the cost and schedule accordingly (Step 6).
After exploring one machine, we move on to the next one as long as it is not the last machine (Step 7).
The external communication part is based on LP relaxation (Step 8) and randomized rounding (Step 9-12).
Note that the randomized rounding will find at most $S$ integer feasible solutions (Step 12).
Finally, we choose the lowest cost among the solutions from the internal and external communication parts (Step 13).
\subsection{Theoretical Performance Analysis} \label{subsec:performance_analysis}
We now examine the competitive ratio of our PD-ORS algorithm.
Note that the key component in PD-ORS is our proposed randomized rounding scheme in (\ref{eqn_rounding_w})--(\ref{eqn_rounding_s}), which is in turn the foundation of Algorithm~1.
Thus, we first prove the following results regarding the randomized rounding algorithm.
Consider an integer program with generalized cover/packing constraints: $\min \{ \c^{\top} \x : \A\x \geq \a, \B\x \leq \b, \x \in \mathbb{Z}_{+}^{n} \}$, where $\A \in \mathbb{R}_{+}^{m\times n}$, $\B \in \mathbb{R}_{+}^{r\times n}$, $\a \in \mathbb{R}_{+}^{m}$, $\b \in \mathbb{R}_{+}^{r}$, and $\c \in \mathbb{R}_{+}^{n}$.
Let $\bar{\x}$ be a fractional optimal solution.
Consider the randomized rounding scheme: Let $\x' = G_\delta\bar{\x}$ for some $G_\delta$ (to be specified).
Randomly round $\x'$ to $\hat{\x} \in \mathbb{Z}_{+}^{n}$ as:
$\hat{x}_{j} = \lceil x'_{j} \rceil$ w.p. $x'_{j} - \lfloor x'_{j} \rfloor$ and $\hat{x}_{j} = \lfloor x'_{j} \rfloor$ o.w.
Note that in the rounding process, if $G_\delta > 1$ ($G_\delta \in (0,1]$), the packing (cover) constraint is prone to be violated and the cover (packing) constraint is easier to be satisfied.
Hence, depending on which constraint is more preferred to be feasible, we consider two cases with respect to $G_\delta$.
1) $0<G_\delta\leq1$: We have the following approximation result (see proof in Appendix~\ref{appdx:lem_ILP1}):
\begin{lem}[Rounding] \label{lem_ILP}
Let $W_{a} \!\!\triangleq\!\! \min \{ a_{i}/[\A]_{ij}\!: [\A]_{ij} \!\!>\!\! 0 \}$ and $W_{b} \!\!\triangleq\!\! \min \{ b_{i}/[\B]_{ij}\!: [\B]_{ij} \!\!>\!\! 0 \}$.
Let $\delta \in (0,1]$ be a given constant and define $G_\delta$ as:
\begin{align*}
G_\delta\triangleq1+ \frac{3\ln(3r/\delta)}{2W_b} - \sqrt{ \bigg(\frac{3\ln(3r/\delta)}{2W_b} \bigg)^{2} + \frac{3\ln(3r/\delta)}{W_b} }.
\end{align*}
Then, with probability greater than $1-\delta$, $\hat{\x}$ achieves a cost at most $\frac{3G_\delta}{\delta}$ times the cost of $\bar{\x}$.
Meanwhile, $\hat{\x}$ satisfies
% and $\hat{\x}$ satisfies
$\mathrm{Pr}\Big\{ (\A\hat{\x})_{i} \leq a_{i} (1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}})G_\delta, \exists i \Big\} \leq \frac{\delta}{3m}$.
\end{lem}
%\begin{figure}[t]
% \centering
% \includegraphics[width=0.9\columnwidth]{Figures/GdeltaStudy.eps}
% \caption{The feasibility study.}
% \label{fig:Gdelta}
%\end{figure}
\begin{rem}[Discussions on Feasibility]
{\em
An insightful remark of Lemma~\ref{lem_ILP} is in order.
Note that, theoretically, the expression $1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$ in Lemma~\ref{lem_ILP} could become negative.
In this case, the last probabilistic inequality in Lemma~\ref{lem_ILP} trivially holds and is not meaningful in characterizing the feasibility, even though the inequality remains valid.
In order for the probabilistic statement to be meaningful in characterizing the feasibility of the integer linear program, we solve for $\delta$ by enforcing $1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}} > 0$, which yields $\delta\geq3m /e^\frac{G_\delta W_a}{2}$.
On the other hand, we prefer $\delta$ to be small since it bounds the feasibility violation probability and approximation ratio achievability in Lemma~\ref{lem_ILP}.
Hence, the smaller the value of $3m /e^\frac{G_\delta W_a}{2}$, the less restrictive the condition $\delta\geq3m /e^\frac{G_\delta W_a}{2}$ is.
%($m$ becomes 1 and $W_a$ becomes $W_1$ in Theorem~\ref{thm_Alg4_1}).
% However, it will not impact the competitive ratio of Algorithm~1 since we conduct rounding until it finds the feasible solution.
%However, we note that the results in Lemma~\ref{lem_ILP} and Theorem~\ref{thm_Alg4_1} are still valid even though they are not useful in characterizing the feasibility when it is negative.
\begin{figure}[t]
\centering
\includegraphics[width=0.9\columnwidth]{Figures/GdeltaStudy.eps}
\caption{The feasibility study.}
\label{fig:Gdelta}
\end{figure}
To gain a deeper understanding on how restrictive the condition $\delta\geq 3m /e^\frac{G_\delta W_a}{2}$ is, we conduct a case study and the results are illustrated in Fig.~\ref{fig:Gdelta}.
Here, we let RHS $\triangleq3m /e^\frac{G_\delta W_a}{2}$ for convenience.
We vary $\delta$ from $0.02$ to $0.1$.
Clearly, the left-hand-side (LHS) of the condition is the $45^{\circ}$ straight line.
In order for the condition to hold, the curve of RHS should fall under this straight line.
Based on typical computing cluster parameters, we set $W_b$ to $15$, and set $r\triangleq RH+1$ to $401$ ($R=4, H=100$).
We can see from Fig.~\ref{fig:Gdelta} that as $W_a$ increases, the curve of RHS crosses the dashed line of LHS at a smaller $\delta$-value.
This means that, the larger the value of $W_a$, the easier for $1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$ to become positive.
Hence, the probabilistic feasibility characterization in Lemma~\ref{lem_ILP} is useful for typical system parameters in practice.
% Moreover,
% since $G\!\lessapprox\!1$, and
% $m$ becomes 1 and $W_a$ becomes $W_1$ in Theorem~\ref{thm_Alg4_1},
% which in practice cannot be small:
% due to the slow growth rate of $\ln(\frac{3}{\delta})$, the RHS becomes negative most likely when $W_a$ is small.
% However, $W_a$ (becomes $W_2$ in Theorem~2) in practice cannot be small:
% It can be seen in Theorem~2 that $W_2$ depends on the global batch size $F_i$ and several ratios between the system resource capacities and unit resource demands.
% which represents the number of workers should be allocated to the accepted job in each time slot.
% In practice, the value of $W_1$ is at least in 10s when typical system parameters are plugged in, implying that its positivity usually holds.
}
\end{rem}
2) $G_\delta>1$: We have the following approximation result (see proof in Appendix~\ref{appdx:lem_ILP2}):
\begin{lem}[Rounding] \label{lem_ILP2}
Let $W_{a}\triangleq \min \{ a_{i}/[\A]_{ij}: [\A]_{ij}> 0 \}$ and $W_{b}\triangleq \min \{ b_{i}/[\B]_{ij}: [\B]_{ij}> 0 \}$.
Let $\delta \in (0,1]$ be a given constant and define $G_\delta$ as:
\begin{align*}
G_\delta \triangleq 1 + \frac{\ln(3m/\delta)}{W_a} + \sqrt{ \bigg(\frac{\ln(3m/\delta)}{W_a} \bigg)^{2} + \frac{2\ln(3m/\delta)}{W_a} }.
\end{align*}
Then, with probability greater than $1-\delta$, $\hat{\x}$ achieves a cost at most $\frac{3G_\delta}{\delta}$ times the cost of $\bar{\x}$.
Meanwhile, $\hat{\x}$ satisfies
% and $\hat{\x}$ satisfies
$\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} > b_{i} (1+ (\frac{3}{G_\delta W_b}\ln(\frac{3r}{\delta}))^{\frac{1}{2}})G_\delta, \exists i \Big\} \leq \frac{\delta}{3r}$.
\end{lem}
Several important remarks for Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} are summarized as follows:
\begin{list}{\labelitemi}{\leftmargin=1em \itemindent=-0.0em \itemsep=.2em}
\item[i)] Note that Alg.~4 is a randomized algorithm.
Therefore, its performance is also characterized probabilistically, and $\delta$ is used for such probabilistic characterization.
Here, it means that with probability $1 - \delta$, one achieves an approximation ratio at most $\frac{3G_\delta}{\delta}$ and a probabilistic feasibility guarantee as stated in Lemma~\ref{lem_ILP} when $0<G_\delta\leq 1$ and Lemma~\ref{lem_ILP2} when $G_\delta>1$.
% Recall that $G_\delta$ is defined as in Eqns~(\ref{eqn_L_def_1}) and (\ref{eqn_L_def}), based on which we can conclude that as $\delta$ increases, $G$ will also increase.
% However, the growth rate of $G_\delta$ is much smaller compared to that of $\delta$ due to the log functions.
% Thus $\delta$ is the dominance to control the performance ratio.
In other words, the statements mean that the probability of getting a better approximation ratio is smaller under randomized rounding (i.e., better approximation ratio $\Rightarrow$ smaller $\frac{3G_\delta}{\delta}$ $\Rightarrow$ larger $\delta$ $\Rightarrow$ smaller probability $1-\delta$).
%(the smaller the value of $\frac{3G}{\delta}$, the better the approximation ratio).
That is, the trade-off between the approximation ratio value and its achieving probability is quantified by $\delta$.
% The probability parameter $\delta$ controls the trade-off between approximation ratio and efficiency in finding a feasible rounding solution:
A larger $\delta$ implies a smaller approximation ratio, but the probability of obtaining a feasible solution of this ratio is also smaller (i.e., more rounds of rounding needed).
%This implies taking more rounds of rounding.
Interestingly, for $\delta=1$, Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} indicate that there is still {\em non-zero} probability to achieve an approximation ratio not exceeding $3G_\delta$.
\item[ii)] Note that if we pick $G_\delta\in(0,1]$, the approximation ratio $\frac{3G_\delta}{\delta}$ decreases (the smaller the approximation ratio, the better) as $\delta$ increases based on Eqn.~(\ref{eqn_L_def_1}) in Appendix~\ref{appdx:lem_ILP1}.
However, the growth rate of $G_\delta$ is slower compared to that of $\delta$ due to the $\log$ operator.
On the other hand, if we pick $G_\delta>1$, $\frac{3G_\delta}{\delta}$ decreases as $\delta$ increases according to Eqn.~(\ref{eqn_L_def}) in Appendix~\ref{appdx:lem_ILP2}.
Therefore, the approximation ratio is ultimately controlled by parameter $\delta$.
Also, the theoretical approximation ratio $\frac{3G_\delta}{\delta}$ is conservative.
%(due to the uses of Markov inequality and union bound in our proof).
Our numerical studies show that the approximation ratio performance in reality is much smaller than $\frac{3G_\delta}{\delta}$.
\item[iii)] The probabilistic guarantee of the cover constraint ($\A\x \geq \a$) when $0<G_\delta\leq 1$ and packing constraint ($\B\x \leq \b$) when $G_\delta>1$, is unavoidable and due to the fundamental hardness of satisfying both cover and packing constraints, which are of {\em conflicting} nature: Any strategy trying to better satisfy the packing constraints (multiplying a $G_\delta$-factor with $G_\delta\in (0,1]$) may increase the probability of violating the cover constraints, and the probability of violating the packing constraints may be increased otherwise.
However, the probabilistic bound here is for worst case and may be pessimistic.
\item[iv)] The results in Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} are in fact applicable for general ILP with mixed cover/packing constraints.
Hence, the results and their insights in Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} could be of independent theoretical interest.
\end{list}
By specializing Lemma~\ref{lem_ILP} and Lemma~\ref{lem_ILP2} with parameters in Problem~(\ref{eqn_obj_given_t3}), we have the following approximation results for Algorithm 4.
The first result corresponds to the case where the feasibility of the resource constraint (packing) is more favored, i.e., $0<G_\delta\leq1$:
%
%\begin{figure}[t]
% \centering
% \includegraphics[width=.9\columnwidth]{Figures/jobFixedshaded}
% \caption{Total utility with increasing number of machines.}\label{fig:jobFixed}
% \vspace{-.1in}
%\end{figure}
%
%\begin{figure}[t]
% \centering
% \includegraphics[width=0.9\columnwidth]{Figures/machineFixedshaded}
% \caption{Total utility with increasing number of jobs.} \label{fig:machineFixed}
% \vspace{-.1in}
%\end{figure}
\begin{thm}[Approximation Performance of Alg.~4 When Resource Constraint Feasibility is Favored]\label{thm_Alg4_1}
Let $W_{1}\!\triangleq\! V_{i}[t] \big( \tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i}\big)$, $W_{2}\!\triangleq\!\min\{ F_i, \hat{C}_{h}^{r}[t]/\alpha_{i}^{r}, \hat{C}_{h}^{r}[t]/\beta_{i}^{r}, \forall r,h \}$, and
% $W_3\triangleq \ln(3(RH+1)/\delta)$.
$\delta \in (0,1]$. Define $G_\delta$ as:
\begin{align} \label{eqn_Thm3_G}
&\nonumber G_\delta\!\triangleq\! 1 \!+\! \frac{3\ln(3(RH\!+\!1)/\delta)}{2W_2}\\
&\!-\! \sqrt{ \bigg(\frac{3\ln(3(RH+1)/\delta)}{2W_2} \bigg)^{2} \!+\! \frac{3\ln(3(RH\!+\!1)/\delta)}{W_2} }.
\end{align}
Then, with probability greater than $1\!-\!\delta$, Algorithm~4 obtains a schedule $\{ w_{ih}[t], s_{ih}[t], \forall t,h \}$ that has an approximation ratio at most $\frac{3G_\delta}{\delta}$ with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_cover1})} \!\leq\! W_1(1- (\frac{2}{G_\delta W_1}\ln(\frac{3}{\delta}))^{\frac{1}{2}})G_\delta, \exists i\} \leq \frac{\delta}{3}$.
\end{thm}
% Note that $1- (\frac{2}{GW_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$ could be negative in Lemma~\ref{lem_ILP}.
% However, the results in Lemma~\ref{lem_ILP} and Theorem~\ref{thm_Alg4_1} are still valid even though they are not useful in characterizing the feasibility when it is negative.
% Moreover, since $G\!\lessapprox\!1$ and due to the slow growth rate of $\ln(\frac{3}{\delta})$, it becomes negative most likely when $W_a$ is small.
% However, $W_a$ (becomes $W_2$ in Theorem~\ref{thm_Alg4_1}) in practice cannot be small: It can be seen in Theorem~\ref{thm_Alg4_1} that $W_2$ depends on the global batch size $F_i$ and several ratios between the system resource capacities and unit resource demands.
% These values are in the range of 10s to 100s when typical system parameters are plugged in, implying that the positivity usually holds in practice.
The next result corresponds to the case where the feasibility of the workload constraint (cover) is more favored, i.e., $G_\delta > 1$:
\begin{thm}[Approximation Performance of Alg.~4 When Workload Constraint Feasibility is Favored]\label{thm_Alg4_2}
Let $W_{1}\triangleq V_{i}[t] \big( \tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i}\big)$, $W_{2} \triangleq \min\{ F_i, \hat{C}_{h}^{r}[t]/\alpha_{i}^{r}, \hat{C}_{h}^{r}[t]/\beta_{i}^{r}, \forall r,h \}$,
% $W_3\triangleq \ln(3(RH+1)/\delta)$.
and $\delta \in (0,1]$. Define $G_\delta$ as:
\begin{align} \label{eqn_Thm4_G}
&G_\delta \triangleq 1 + \frac{\ln(3/\delta)}{W_1} + \sqrt{ \bigg(\frac{\ln(3/\delta)}{W_1} \bigg)^{2} + \frac{2\ln(3/\delta)}{W_1} }.
\end{align}
Then, with probability greater than $1-\delta$, Algorithm~4 obtains a schedule $\{ w_{ih}[t], s_{ih}[t], \forall t,h \}$ that has an approximation ratio at most $\frac{3G_\delta}{\delta}$ with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_packing})} > \hat{C}_h^r[t] G_\delta(1+(\frac{3}{G_\delta W_2}\ln(\frac{3(HR+1)}{\delta}))^{\frac{1}{2}})\} \leq \frac{\delta}{3(HR+1)}$.
\end{thm}
Note that Eqn.~(\ref{eqn_packing2}) is guaranteed in practice since the number of samples is typically far more than the number of workers.
The competitive ratio of our online algorithm is the worst-case upper bound of the ratio of the overall utility of admitted jobs devided by the offline optimal solution of Problem~DMLRS to the total utility
of admitted jobs achieved by Algorithm~1 in the overall time horizon.
Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2} follow directly from Lemmas~\ref{lem_ILP} and \ref{lem_ILP2}, respectively, and we omit the proof here for brevity.
Based on these results, we can establish the overall competitive ratio for Algorithm~1 as follows.
\begin{thm}[Competitive Ratio of Alg~1 when $0\!<\!G_\delta\!\leq\! 1$]\label{thm_Competitive_1}
Let $\delta$, $G_\delta$ and $W_1$ be as defined in Theorem~\ref{thm_Alg4_1}.
Let $U^{r}$ and $L$ be as defined in (\ref{eqn_Ur}) and (\ref{eqn_L}), respectively.
Then, with probability greater than $(1-(\delta/3)^S)^{TK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible solution that is $\frac{6G_\delta}{\delta} \max_{r\in\mathcal{R}} (1,\ln\frac{U^{r}}{L})$--competitive.
% with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_cover1})} \!\leq\! W_1(1- (\frac{2}{G_\delta W_1}\ln(\frac{3}{\delta}))^{\frac{1}{2}})G_\delta, \exists i\} \leq \frac{\delta}{3}$.
\end{thm}
\begin{thm}[Competitive Ratio of Alg~1 when $G_\delta>1$]\label{thm_Competitive_2}
Let $\delta$, $G_\delta$ and $W_2$ be as defined in Theorem~\ref{thm_Alg4_2}.
Let $U^{r}$ and $L$ be as defined in (\ref{eqn_Ur}) and (\ref{eqn_L}), respectively.
Then, with probability greater than $(1-(\delta/3(HR+1))^S)^{TK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible solution that is $\frac{6G_\delta}{\delta} \max_{r\in\mathcal{R}} (1,\ln\frac{U^{r}}{L})$--competitive.
% with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_packing})} >\hat{C}_h^r[t]G_\delta(1+(\frac{3}{G_\delta W_2})^{\frac{1}{2}})\} \leq \frac{\delta}{3(HR+1)}$.
\end{thm}
It is worth pointing out that in Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}, the feasibility achieving probability values, i.e., $(1-(\delta/3)^S)^{TK_iE_i}$ and $(1-(\delta/3(HR+1))^S)^{TK_iE_i}$, can controlled by choosing appropriate values of $\delta$ and $S$ (i.e., rounds of rounding) to offset the impact of total number of DP iterations $TK_i E_i$.
The smaller $\delta$ and the larger $S$ are, the higher the feasibility achieving probability.
Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2} can be proved by weak duality and the approximation results in Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2}.
We provide a proof in Appendix~\ref{appdx:thm_Competitive}.
% due to space limitation.
\begin{thm}[Polynomial Running Time]\label{thm_Time}
% {\color{red}{Let $\delta$ be as defined in Theorems~\ref{thm_Alg4_1} when $0<G_\delta\leq1$ and \ref{thm_Alg4_2}} when $G_\delta>1$}.
%Let $\delta\in(0,1)$.
By combining Algorithms~1--4, % {\color{red}{with probability greater than $1-\delta$}},
the time complexity of PD-ORS is
% $O(\frac{1}{\delta}TK_{i}^{2}E_{i}^{2}H^{4})$,
$O(\sum_{i=1}^{|\mathcal{I}|} T K_i^2 E_{i}^{2} (H^{3} + S) )$,
which is polynomial.
% which is straightforward to prove by considering dynamic programming and randomized rounding.
\end{thm}
% \textcolor{red}{The proof can be found in Appendix~\ref{appdx:thm_Time}}
% The key components of Theorem~\ref{thm_Time} includes dynamic programming and randomized rounding.
\begin{proof}
When solving $\theta(t,v)$ using Algorithm~4, it takes $O(H\log H)$ iterations to sort machines in internal communication case under each time slot $t$ and looping all machines to calculate the minimum number $s_{ih}[t]$ takes $O(H)$.
Thus, it takes $(H\log H)$ time for the internal communication part in Algorithm~4.
For the external communication part in Algorithm~4, solving the LP relaxation of Problem~\eqref{eqn_obj_given_t3} can be approximately bounded $O(H^3)$ if we use a polynomial time LP solver (e.g., Vaidya's algorithm~\cite{Vaidya87:LP}).
According to Algorithm~4, the rounding time is proportional to $S$.
Hence, the running time for the external communications part is upper bounded by $O(|\mathcal{H}|^3 + S)$.
Combining the discussions above, the running time complexity of Algorithm~4 is $O(H\log H + H^3 + S) = O(H^3 + S)$.
%Hence it takes ${\color{red}O(H\log H+K_iE_iH)}$ to compute all $\theta(t,v),\forall v\in[0,K_iE_i]$.
Moreover, the number of states $(t,v)$ is $O(TK_iE_i)$ in the dynamic programming (DP) for each job $i$, and the time complexity of executing DP is $O(K_iE_i)$.
Thus, the time complexity is $O(TK_i^2E_i^2)$ in DP.
In Algorithm~1, the number of steps in the main loop is equal to the number of jobs.
Therefore, the overall running time complexity can be computed as $O(\sum_{i=1}^{|\mathcal{I}|} T K_i^2 E_{i}^{2} (H^{3} + S) )$.
%Therefore, it takes $O(TH\log H+TK_iE_iH+TK_i^2E_i^2+THV)=O(TK_iE_iH+TK_i^2E_i^2)$ for job $i$ to make a scheduling decision, and combined with randomized rounding, we establish the time complexity.
%{\color{red}{Note that $\delta$ quantifies the trade-off between the approximation ratio value of Alg.~4 and its achieving probability.
%That is, the better the approximation ratio, the smaller its achieving probability.
%Moreover, the approximation ratio value of Alg.~4 (a subroutine of PD-ORS) directly affects the overall running time of PD-ORS: The better the approximation ratio in each round of Step~2 of Alg.~1, the fewer number of iterations are needed on average.
%% in the main loop in Alg.~1 on average.
%Therefore, the time complexity of PD-ORS probabilistically depends on the subroutine Alg.~4, which can be quantified by $\delta$.}}
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Numerical Results}
\label{sec:numerical}
In this section, we conduct simulation studies to evaluate the efficacy of our proposed PD-ORS algorithm.
We test an ML system with jobs parameters generated uniformly at random from the following intervals: $E_{i} \in [50,200]$, $K_{i} \in [20000,500000]$,
% \textbf{the total number of samples is a bit too small},
$g_{i} \in [30,575]$ MB, $\tau_{i} \in [10^{-5},10^{-4}]$
%\textbf{it should be smaller since it is the time for computing one sample only}
time slots, $\gamma_i\in[1,10]$,
%\textbf{the max. ratio can be much smaller, e.g., 10},
$F_i\in[1,200]$. %all generated uniformly at random.
%The ML system running for $T=20-50$ time slots with $H=20$--$100$.
%$b_{i} \in [100\text{Mbps}, 4\text{Gbps}]$, and $B_{i}=[4\text{Gbps},20\text{Gbps}]$, all generated uniformly at random.
We consider four types of resources: GPU, CPU, memory, and storage.
For fair comparisons, following similar settings in~\cite{Li14:OSDI}~\cite{Chilimbi14:OSDI}~\cite{Sun17:Dorm}, we set resource demand of each worker as follows: 0--4 GPUs, 1--10 vCPUs, 2--32 GB memory, and 5--10GB storage.
We set resource demand of each parameter server as follows: 1--10 vCPUs, 2--32GB memory and 5-10GB storage.
%The resource capacities of each physical machine is set by following EC2 GPU instances P2 and G3 randomly~\cite{AmazonEC2Instances}.
The resource capacity of each physical machine is set roughly 18 times of the resource demands of a worker/PS following EC2 C5n instances~\cite{AmazonEC2Instances}. %that is, we set the capacity of each machine is relatively larger compared to each worker's/parameter server's demand of each job $i$.
We set the job arrival pattern according to the Google Cluster data~\cite{Reiss12:googlecluster}, but with normalized job arrival rates in alternating time-slots as follows: the arrival rates are $1/3$ and $2/3$ in odd and even time-slots, respectively.
% \textbf{clarify how the job rate is varied}.
% \textcolor{red}{For fair comparisons, we adopt the sigmoid utility function ~\cite{Huang15:CORA,Bao18:ML_INFOCOM}, as $f_i(t-a_i)=\frac{\theta_1}{1+e^{\theta_2(t-a_i-\theta_3)}}$, where $\theta_1\in[1,100]$, $\theta_2\in[0,6]$ and $\theta_3\in[1,15]$.}
For fair comparisons, we adopt the Sigmoid utility function ~\cite{Huang15:CORA,Bao18:ML_INFOCOM}: $u_i(t-a_i)=\frac{\theta_1}{1+e^{\theta_2(t-a_i-\theta_3)}}$, where $\theta_1\in[1,100]$ indicates the priority of job $i$, $\theta_2$ indicates how critical the time is for the job $i$, and $\theta_3\in[1,15]$ is the estimated target completion time.
We set $\theta_2=0$ for time-insensitive jobs (10\% of jobs), $\theta_2\in[0.01,1]$ for time-sensitive jobs (55\% of jobs) and $\theta_2\in[4,6]$ for time-critical jobs (35\% of jobs).
% For fair comparisons, we adopt the same sigmoid utility in ~\cite{Huang15:CORA,Bao18:ML_INFOCOM}.
%\textbf{reorder the figures according to their order of appearance in the main text}
%\begin{figure}[t!]
% \centering
% \includegraphics[width=.9\columnwidth]{Figures/jobFixedshaded}
% \caption{Total utility with increasing number of machines.}\label{fig:jobFixed}
% \vspace{-.1in}
%\end{figure}
%
%\begin{figure}[t!]
% \centering
% \includegraphics[width=0.9\columnwidth]{Figures/machineFixedshaded}
% \caption{Total utility with increasing number of jobs.} \label{fig:machineFixed}
% \vspace{-.1in}
%\end{figure}
We first compare our PD-ORS algorithm with three baseline job scheduling policies: (1) {\em FIFO} in Hadoop and Spark~\cite{Zaharia10:USENIX}, where the jobs are processed in the order of their arrival times. In our setting, the fixed number of workers (parameter servers) is between 1 to 30,
(2) {\em Dominant Resource Fairness Scheduling} (DRF) in Yarn~\cite{Vavilapalli13:Yarn} and Mesos~\cite{Hindman11:Mesos}, where the jobs are scheduled based on their dominant resource share in the cloud to achieve its max-min fairness.
The number of workers and parameter servers are computed and allocated dynamically,
and (3) {\em Dorm}~\cite{Sun17:Dorm}, where the numbers of workers (parameter servers) are computed and placed by an MILP resource utilization maximization problem with fairness and adjustment overhead constraints.
Workers and parameter servers are placed in a round-robin fashion on available machines in Baselines (1) and (2).
% {\color{red}{(1) FIFO: it is used in Hadoop and Spark [cite] and the jobs are processed in order of their arrival times.
% In our setting, the fixed number of workers (parameter servers) is between 1 to 30.
% (2) Dominant Resource Fairness Scheduling (DRF): it is used in Yarn[cite] and Mesos [~\cite] and jobs are scheduled based on their dominant resource share in the cloud to achieve its max-min fairness.
% The number of workers (parameter servers) are computed and allocated dynamically.
% (3) Dorm[cite]: The numbers of workers (parameter servers) are computed and placed by an MILP resource utilization maximization problem with fairness and adjustment overhead constraints.
% We employ a round-robin approach to place workers and parameter servers on available machines in Baselines (1) and (2).}}
The comparison results are shown in Figs.~\ref{fig:jobFixed} and~\ref{fig:machineFixed}.
In Fig.~\ref{fig:jobFixed}, we set $T=20$ and $I=50$, while in Fig.~\ref{fig:machineFixed}, we set $T=20$ and $H=100$.
% \textbf{give the following settings in producing the figures, including number of jobs in Fig.~\ref{fig:jobFixed}, number of machines in Fig ~\ref{fig:machineFixed}, time slots}.
We can see that PD-ORS significantly outperforms other policies and the gains in total utility becomes more pronounced as the numbers of jobs and machines increase.
%\textbf{but the figures show the total utility of PD-ORS is smallest. which shadowed bar is the PD-ORS case? possibly change the markers to the bars more distinghuishable}
%\begin{figure*}[t!]
%%\centering
%\begin{minipage}{0.45\textwidth}
%%\centering
%%\vspace{-1.26in}
%\includegraphics[width=1\textwidth]{Figures/jobFixedshaded}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed}
%\end{minipage}
%%
%\hspace{0.1\columnwidth}%
%%
%\begin{minipage}{0.45\textwidth}
%%\centering
%%\vspace{-1.3in}
%\includegraphics[width=1\textwidth]{Figures/machineFixedshaded}
%%\vspace{.15in}
%\caption{Total utility with increasing number of jobs.} \label{fig:machineFixed}
%\end{minipage}
%\end{figure*}
%
%
%\begin{figure*}[t!]
%\begin{minipage}{0.49\textwidth}
%\vspace{-.2in}
% \includegraphics[width=1\textwidth]{Figures/colocation.eps}
% \caption{Utility comparison between PD-ORS and OASiS.} \label{fig:UtilityComp}
%\end{minipage}
%%
%\hspace{0.02\columnwidth}%
%%
%\begin{minipage}{0.5\textwidth}
% \includegraphics[width=1\textwidth]{Figures/G2.eps}
%\vspace{-.25in}
% \caption{Impact of pre-rounding gain factor $G_\delta$ on competitive ratio.}\label{fig:Factor_ApprRatio}
%\end{minipage}
%\end{figure*}
\begin{figure*}[t!]
%\centering
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.26in}
\includegraphics[width=1\textwidth]{Figures/jobFixedshaded.png}
%\vspace{-.03in}
\caption{Total utility with increasing number of machines (synthetic data).}\label{fig:jobFixed}
\end{minipage}
%
\hspace{0.1\columnwidth}%
%
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.3in}
\includegraphics[width=1\textwidth]{Figures/machineFixedshaded.png}
%\vspace{.15in}
\caption{Total utility with increasing number of jobs (synthetic data).} \label{fig:machineFixed}
\end{minipage}
\hspace{0.1\columnwidth}%
%
\begin{minipage}{0.3\textwidth}
%\centering
\vspace{.05in}
\includegraphics[width=1.1\textwidth]{Figures/colocation.eps}
%\vspace{.15in}
\caption{Utility comparison between PD-ORS and OASiS with increasing number of jobs.} \label{fig:UtilityComp_1}
\end{minipage}
\end{figure*}
\begin{figure*}[t!]
\begin{minipage}{0.3\textwidth}
\vspace{-.2in}
\includegraphics[width=1.11\textwidth]{Figures/actual_time.eps}
%\vspace{-.25in}
\caption{Median of actual training time comparison.} \label{fig:actual_time}
\end{minipage}
\hspace{0.1\columnwidth}%
\begin{minipage}{0.3\textwidth}
\vspace{-.3in}
\includegraphics[width=1.1\textwidth]{Figures/ratio.eps}
\caption{Competitive ratio.} \label{fig:ratio}
\end{minipage}
\hspace{0.1\columnwidth}%
\begin{minipage}{0.3\textwidth}
\vspace{-.2in}
\includegraphics[width=1.1\textwidth]{Figures/G2.eps}
\caption{Impact of pre-rounding gain factor $G_\delta$ on competitive ratio.}\label{fig:Factor_ApprRatio}
\end{minipage}
\end{figure*}
%\begin{figure}[h]
%\centering
% \includegraphics[width=.4\textwidth]{Figures/G2.eps}
%%\vspace{-.25in}
% \caption{Impact of pre-rounding gain factor $G_\delta$ on competitive ratio.}\label{fig:Factor_ApprRatio}
%\end{figure}
\begin{figure*}[t!]
%\centering
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.26in}
\includegraphics[width=1\textwidth]{Figures/jobFixed_response.png}
%\vspace{-.03in}
\caption{Total utility with increasing number of machines (Google cluster data trace).}\label{fig:jobFixed_response}
\end{minipage}
%
\hspace{0.1\columnwidth}%
%
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.3in}
\includegraphics[width=1\textwidth]{Figures/serverFixed_response.png}
%\vspace{.15in}
\caption{Total utility with increasing number of jobs (Google cluster data trace).} \label{fig:serverFixed_response}
\end{minipage}
\hspace{0.1\columnwidth}%
\begin{minipage}{0.3\textwidth}
%\vspace{-.2in}
\includegraphics[width=1.11\textwidth]{Figures/servervaried_1.eps}
%\vspace{-.2in}
\caption{Total utility with increasing number of machines [T=80, I=100, (10\%, 55\% , 35\%)].}\label{fig:jobFixed1}
\end{minipage}
\end{figure*}
\begin{figure*}[t!]
%\centering
%\begin{minipage}{0.22\textwidth}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%\end{minipage}
\begin{minipage}{0.3\textwidth}
\includegraphics[width=1.11\textwidth]{Figures/servervaried_2.eps}
%\vspace{.15in}
\caption{Total utility with increasing number of machines [T=80, I=100, (30\%, 69\% , 1\%)].} \label{fig:jobFixed2}
\end{minipage}
\hspace{0.1\columnwidth}%
\begin{minipage}{0.3\textwidth}
%\vspace{-.15in}
\includegraphics[width=1.11\textwidth]{Figures/jobvaried_1.eps}
\caption{Total utility with increasing number of jobs [T=80, H=30, (10\%, 55\% , 35\%)].} \label{fig:serverFixed1}
\end{minipage}
\hspace{0.1\columnwidth}%
\begin{minipage}{0.3\textwidth}
%\vspace{-.15in}
\includegraphics[width=1.11\textwidth]{Figures/jobvaried_2.eps}
%\vspace{-.2in}
\caption{Total utility with increasing number of jobs [T=80, H=30, (30\%, 69\% , 1\%)].}\label{fig:serverFixed2}
\end{minipage}
\end{figure*}
Next, we compare our PD-ORS algorithm with the OASiS algorithm in~\cite{Bao18:ML_INFOCOM}, which is also a dynamic scheduling scheme. %, which is the most related work to ours and the state-of-the-art of scheduling for distributed ML systems.
As mentioned earlier, the key difference in OASiS is that parameter servers and workers are located on two {\em strictly} separated sets of machines (i.e., no co-located workers and PSs).
Here, we let $H=100$ and $T=20$.
%{\color{blue} Let $T=80$ and $I=100$ in Fig.~\ref{fig:UtilityComp_2}.}
For OASiS, half of the machines host parameter servers and the other half host workers.
For fair comparisons, both algorithms adopt the same Sigmoid utility function.
The comparison results are shown in Fig.~\ref{fig:UtilityComp_1}.
We can see that PD-ORS outperforms OASiS by allowing co-located parameter servers and workers.
%\textbf{better discuss why the more jobs there are, the better the improvement is}
We can see from Fig.~\ref{fig:UtilityComp_1} that the performance gap between PD-ORS and OASiS widens as the number of jobs increases, which implies that PD-ORS is more scalable.
This is due to the advantage afforded by colocation of workers and parameter servers, which allows each physical machine to be fully utilized.
On the other hand, the strict separation of workers and parameter servers in OASiS may lead to the inability of placing workers on server-side machines, should there be available resources or vice verse.
%Next, we investigate the impact of $b_{i}^{(i)}/{b_{i}^{(e)}}$ on total utility value and the results are shown in Fig.~\ref{fig:IntExt_Ratio}.
%In this experiment, we let $H=20$, $T=50$, and vary $b_{i}^{(i)}/{b_{i}^{(e)}}$ from $1$ to $60$.
%We can see that the total utility rises rapidly as $b_{i}^{(i)}/{b_{i}^{(e)}}$ increases initially, which shows that PD-ORS reacts aggressively to a large $b_{i}^{(i)}/{b_{i}^{(e)}}$-value.
%On the other hand, the increase of total utility becomes more gradual when $b_{i}^{(i)}/{b_{i}^{(e)}}$ is greater than 30.
%This is because beyond this point, most jobs already have a large number of co-located parameter servers and workers.
%, which results in marginal performance improvements.
Next, we investigate the actual training time (completion time - arrival time) under different methods, where $T=80$, $H=30$ and $I=100$.
The median of the actual training time is shown in Fig.~\ref{fig:actual_time}.
Here, we simply set its training time to $T$ (i.e., 80) if the job cannot be finished within the scheduling time span $\mathcal{T}$.
%As we can see from the figure that only PD-ORS and OASiS are able to schedule all jobs within the time horizon.
As we can see from Fig.~\ref{fig:actual_time}, PD-ORS outperforms other scheduling policies, i.e., it has the smallest median time.
Also, due to the co-location advantage of PD-ORS, its median time is smaller compared to OASiS, where workers and parameter servers are placed in strictly separated sets of machines.
We expect that the difference between PD-ORS and OASiS will become more noticeable as the number of machines or the capacity of each machine increases since it will allow more co-location placements.
Next, we demonstrate the competitive ratio of our algorithm PD-ORS, which is the ratio between the total job utility of the offline optimal solution and the total job utility achieved by PD-ORS.
Recall that Problem~DMLRS is a non-convex problem with constraints (e.g., Eq.~(\ref{eqn_NumTrainedSamples})) that are not amenable to be directly solved by conventional optimization techniques. To obtain its offline optimum, all possible combinations of $w_{ih}[t], s_{ih}[t], \forall i, h, t$ need to be considered, which is time prohibitive.
%Although the optimum is hard to obtain and the actual performance ratio can not be derived, for online scheduling algorithms, competitive ratio is typically provided instead (e.g.,~[6]).''
Thus, we limit the number of jobs $I$ to 10 and time span $T$ to 10, and the result is shown in Fig.~\ref{fig:ratio}.
As we can see from the figure, the performance ratio is between 1.0 to 1.4, indicating that our proposed algorithm PD-ORS has a good competitive ratio performance.
Lastly, we examine the performance of the randomized rounding scheme in Algorithm~4, which is the key of PD-ORS.
We evaluate the rounding performance in terms of the ratio between the optimal total utility and the total utility obtained by our algorithm.
The optimal utility is computed using the Gurobi optimization solver.
We let $H=100$, $I=50$, $T=20$.
We vary the pre-rounding gain factor $G_{\delta}$ (Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2}) from $0.2$ to $1.2$.
The results are shown in Fig.~ \ref{fig:Factor_ApprRatio}. %, respectively.
The packing constraints are easier to satisfy with a smaller $G_\delta$, while the cover constraints are prone to be violated as $G_\delta$ gets smaller.
In our experiments, if the total rounds of randomized rounding before we find an integer feasible solution exceeds a preset threshold (e.g, $5000$), we will discard the corresponding job.
Theorem~\ref{thm_Alg4_1} suggests that there is a trade-off: if we set $G_\delta$ to be close to one to pursue a better total utility result, the rounding time could be large to obtain a feasible solution.
As $G_\delta$ increases, the probability of violating the packing constraints increases, meaning we need to have more rounding attempts to obtain an integer feasible solution.
However, according to our numerical experiences, if the machine's resource capacity is relatively large compared to the jobs' resource demands per worker/PS, the number of rounding attempts is small and not sensitive to the variation of $G_\delta$.
On the other hand, as $G_\delta$ decreases, the probability of violating the cover constraint increases.
However, in practice, the model usually converges with fewer number of iterations than the pre-defined training epochs since the required number of epochs is usually overestimated~\cite{Jeon18:multi-tenant}.
In other words, the violation of the cover constraint in one iteration may be acceptable.
As we can see from Fig.~\ref{fig:Factor_ApprRatio}, the best approximation ratio value is achieved when $G_\delta=1$.
This is because if $G_\delta$ approaches $0$, it implies that $\delta$ decreases at a larger rate (cf. Eq.~(\ref{eqn_Thm3_G})), resulting the increment of the performance ratio.
On the other hand, if $G_\delta$ goes to infinity, it implies $\delta$ decreases (cf. Eq.~\eqref{eqn_Thm4_G}), resulting in a much faster increment of the performance ratio.
%xx \textbf{explain why by pointing to earlier section} .
%This is because packing constraints are easier to satisfy with a smaller $G$.
%When $G>1.006$, we can see that times of rounding start to increase.
%This is because packing constraints are prone to be violated as $G$ gets large.
%We note that, in all cases, the times of rounding are less than $1000$, for which the runtime is short (especially given the size of the search space is $2^{50}$).
Also, we can see from Fig.~\ref{fig:Factor_ApprRatio} that the performance ratios for all choices of $G_\delta$ are much better than the theoretical bounds in Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2}, which shows that the approximation ratio is much tighter than the worse-case bound suggested in Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}.
%%%%%%%%%%%%%%additional experiments results%%%%%%%%%%%%%%
%Thanks for your question.
%In the experiment, we set job arrival pattern according to the Google cluster data with varied arrival rates.
%Our goal was to spread jobs kinda evenly across the entire scheduling time horizon $\mathcal{T}$, which potentially would include all kinds of jobs, i.e., jobs that arrive early/later in the time span with small/large utility, to show our proposed algorithm performance.
%On the other hand, if we simulate job arrivals exactly based on timestamps recorded in the trace by scaling down the original job trace, traffic bustiness may occur and the experiments results may lack of representative due to its dependence on the selected ``snippets" from the trace.
%
%In addition, based on google cluster trace, the job arrival pattern is shown in Fig.~\ref{fig:pattern}, where x-axis is the time (seconds) and y-axis is the number of submitted jobs (see ref. [1] below).
%We can see from the figure that the distribution of job submissions has a repetitive and fractal pattern, which indicates a periodicity of 10, 100, 1000 and 10000.
%Note that every 10 minutes, there exists 1 minute absence of job submissions, and every 100 minutes, there exists 10 minutes absence of job submission, and so on so forth.
%However, such absence does not mean no job arrives in the data center but that no job is released by the global scheduler [1].
%Thus the job arrives somehow evenly spread over the entire time span, and our arrival rates design roughly follow the pattern.
%However, we agree that since we follow the google trace, we can simulate exactly follow its arrivals.
%We also include the comparison results in Figs.~\ref{fig:jobFixed_response} - \ref{fig:jobFixed_ratio}.
%Here, we set $T=80$, $I=100$, and we increase the number of machines from 10 to 50.
%And we set $T=80$, $H=30$, and we increase the number of jobs from 20 to 100.
%As we can see from the figures, our algorithm PD-ORS outperforms other scheduling policies.
%Similarly, due to the advantage of co-location in PD-ORS, it achieves more total job utility compared to OASiS.
%\begin{figure*}[h]
%\centering
%\begin{minipage}{0.3\textwidth}
%%\centering
%%\vspace{-1.26in}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_response.eps}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed_response}
%\end{minipage}
%%
%%\hspace{0.1\columnwidth}%
%%
%\begin{minipage}{0.3\textwidth}
%%\centering
%%\vspace{-1.3in}
%\includegraphics[width=1\textwidth]{response/Figures/jobvaried_response.eps}
%%\vspace{.15in}
%\caption{Total utility with increasing number of jobs.} \label{fig:serverFixed_response}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.2in}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%\end{minipage}
%\end{figure*}
%
%\begin{figure*}[h]
%\centering
%%\begin{minipage}{0.22\textwidth}
%%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%%\vspace{-.03in}
%%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_2.eps}
%%\vspace{.15in}
%\caption{Total utility with increasing number of machines.} \label{fig:jobFixed2}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_1.eps}
% \caption{Total utility with increasing number of jobs.} \label{fig:serverFixed1}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_2.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing number of jobs.}\label{fig:serverFixed2}
%\end{minipage}
%\end{figure*}
%\begin{figure*}[t!]
%\begin{minipage}{0.5\textwidth}
%%\vspace{-.2in}
% \includegraphics[width=1\textwidth]{Figures/jobvaried_ratio_response.eps}
% \caption{Total utility with increasing number of jobs.} \label{fig:serverFixed_ratio}
%\end{minipage}
%%
%%\hspace{0.02\columnwidth}%
%%
%\begin{minipage}{0.5\textwidth}
% \includegraphics[width=1\textwidth]{Figures/servervaried.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing number of machines.}\label{fig:jobFixed_ratio}
%\end{minipage}
%\end{figure*}
Next, we show further experimental results with real-world data traces.
%In the experiment, we set job arrival pattern according to the Google cluster data with varied arrival rates.
%Our goal was to spread jobs kinda evenly across the entire scheduling time horizon $\mathcal{T}$, which potentially would include all kinds of jobs, i.e., jobs that arrive early/later in the time span with small/large utility, to show our proposed algorithm performance.
We first compare our PD-ORS algorithm with baseline scheduling algorithms, where we follow job arrivals exactly based on timestamps recorded in the Google Cluster data~\cite{Reiss12:googlecluster} by scaling down the original job trace (i.e., a ``snippet'' of the trace).
Here, we set $T=80$, $I=100$ and $H=30$.
%We increase the number of machines from 10 to 50 with $I=100$.
%And we increase the number of jobs from 20 to 100 with $H=30$.
The comparison results are shown in Figures~\ref{fig:jobFixed_response} - \ref{fig:serverFixed_response}.
%Here, we set $T=80$, $I=100$, and we increase the number of machines from 10 to 50.
%And we set $T=80$, $H=30$, and we increase the number of jobs from 20 to 100.
Similarly, as we can see from the figures, our algorithm PD-ORS outperforms other scheduling policies.
In addition, due to the co-location advantage of PD-ORS, it achieves more total job utility compared to OASiS.
In the previous experiments, we have set the portions for time-insensitive jobs, time-sensitive jobs and time-critical jobs to 10\%, 55\% and 35\%, respectively, which follows the default setting in~\cite{Bao18:ML_INFOCOM} for fair comparison.
Theoretically, the larger the portion of time-sensitive and time-critical jobs is, the better the performance is in terms of job utility compared to other scheduling policies.
Based on the Google trace analysis~\cite{Minet18:google_trace}, there are four categories of scheduling class of a job to indicate the latency sensitivity of the job, where we label class 0 as time-insensitive, Classes 1 and 2 as time-sensitive, and Class 3 as time-critical.
%The portion of each class is depicted in Table~\ref{table:class}.
%We can observe from the results that most jobs are time critical.
%Here, we label class 0 as time-insensitive, class 1 and 2 as time-sensitive, and class 3 as time-critical.
In order to follow the practical setting in the trace, we roughly set the ratio to 30\%, 69\% and 1\%.
We set $T=80$.
The number of machines increases from $10$ to $50$ with the number of job fixed to $100$, and the number of jobs increases from $20$ to $100$ with the number of machines fixed at $30$.
We let Figs.~\ref{fig:jobFixed1} and \ref{fig:serverFixed1} follow the previous ratio setting (i.e., 10\%, 55\% and 35\%), and Figs.~\ref{fig:jobFixed2} and \ref{fig:serverFixed2} follow the revised ratio setting (i.e., 30\%, 69\% and 1\%).
We examine the utility gain compared to OASiS, where it is normalized.
We present our experimental results in Figs.~\ref{fig:jobFixed1} -- \ref{fig:serverFixed2}.
%In Figs.~\ref{fig:jobFixed1} and \ref{fig:jobFixed2}, we set $T=80, I=100$, and increases number of servers from 10 to 50.
%In Figs.~\ref{fig:serverFixed1} and \ref{fig:serverFixed2}, we set $T=80, H=30$, and increases number of jobs from 20 to 100.
%We can see that similar results are obtained.
We can see from the figures that as the portion of critical jobs decreases by 34\%, the utility gain becomes smaller.
That is, the advantage of our proposed algorithm PD-ORS becomes less prominent.
We note that although in theory we can compare our PD-ORS algorithm (a special case with utility function $u(x)=x$) with Optimus in~\cite{Peng18:ML_EuroSys} (which also takes co-location into consideration), it is not straightforward to do so in practice.
Optimus requires an offline stage to estimate the $\theta$-parameters of the speed function $f(p_j, w_j)$ (cf. $\theta_{0}$--$\theta_{3}$ in [\!\!\cite{Peng18:ML_EuroSys}, Eq.~(3)] for asynchronous training and $\theta_{0}$--$\theta_{4}$ for synchronous training in [\!\!\cite{Peng18:ML_EuroSys}, Eq.~(4)]).
Estimating these parameters requires specific hardware and software packages that are not available in our current experimental environment.
Due to the above computing resource limitations and time constraints, we are unable to conduct experiments to directly compare PD-ORS and Optimus in this work.
Also, our focus in this work is on scheduling algorithmic designs for deep learning training with main contributions being on the theoretical aspects (proving rigorous scheduling performance guarantees).
Implementing our PD-ORS algorithm in a similar testbed environment and having a comparison with Optimus is very interesting and will be our next step in future studies.
%\begin{figure*}[h]
%\centering
%%\begin{minipage}{0.22\textwidth}
%%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%%\vspace{-.03in}
%%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_2.eps}
%%\vspace{.15in}
%\caption{Total utility with increasing number of machines.} \label{fig:jobFixed2}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_1.eps}
% \caption{Total utility with increasing no. of jobs.} \label{fig:serverFixed1}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_2.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing no. of jobs.}\label{fig:serverFixed2}
%\end{minipage}
%\end{figure*}
%\begin{figure*}[h]
%\centering
%\begin{minipage}{0.4\textwidth}
%%\vspace{-.2in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_1.eps}
% \caption{Total utility with increasing no. of jobs.} \label{fig:serverFixed1}
%\end{minipage}
%%
%%\hspace{0.02\columnwidth}%
%%
%\begin{minipage}{0.4\textwidth}
%%\centering
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_2.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing no. of jobs.}\label{fig:serverFixed2}
%\end{minipage}
%\end{figure*}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Conclusion}
\label{sec:conclusion}
In this paper, we investigated online resource scheduling for distributed machine learning jobs in a shared computing cluster.
We considered the most general setting that workers and parameter servers can be co-located on the same set of physical machines.
We showed that this problem can be formulated as a challenging integer nonlinear programming problem with {\em non-deterministic} constraints.
We developed an efficient online scheduling algorithm with competitive ratio guarantee.
Our main contributions are three-fold:
i) We developed a new analytical model that jointly considers resource locality and allocation;
ii) Through careful examinations of worker-server configuration relationships, we resolved the locality ambiguity in the model and reduce the problem to a mixed cover/packing integer program that entails low-complexity approximation algorithm design;
iii) We proposed a meticulously designed randomized rounding algorithm to solve the mixed cover/packing integer program and rigorously established its approximation ratio guarantee.
Collectively, our results expand the theoretical frontier of online optimization algorithm design for resource optimization in distributed machine learning systems.
%>From snoeyink%cs.ubc.ca@cs.Technion.AC.IL Wed Nov 3 08:18:27 1993
%Received: from csmail.cs.technion.ac.il by math.tau.ac.il (5.67/math.tau-930921)
%id AA17625; Wed, 3 Nov 93 08:18:23 +0200
%Received: from csc (csc.cs.technion.ac.il) by csmail.cs.Technion.AC.IL (4.1/3.14)
%id AA14244; Wed, 3 Nov 93 01:19:10 IST
%Received: from csa by csc (4.1/3.14)
%id AA24688; Wed, 3 Nov 93 01:19:48 IST
%Received: from grolsch.cs.ubc.ca by csa (4.1/3.14)
%id AA20525; Wed, 3 Nov 93 01:11:28 IST
%Return-Path: <snoeyink@cs.ubc.ca>
%Received: from cascade.cs.ubc.ca by grolsch.cs.ubc.ca with SMTP id AA13516
%(5.65c/IDA-1.3.5 for alon@cs.Technion.AC.IL); Tue, 2 Nov 1993 13:56:34 -0800
%Date: Tue, 2 Nov 1993 13:56:34 -0800
%From: Jack Snoeyink <snoeyink%cs.ubc.ca@cs.Technion.AC.IL>
%Message-Id: <199311022156.AA13516@grolsch.cs.ubc.ca>
%Received: by cascade.cs.ubc.ca (4.1/SMI-4.1)
%id AA21873; Tue, 2 Nov 93 13:56:33 PST
%To: alon@cs.Technion.AC.IL
%In-Reply-To: Alon Efrat's message of Sun, 31 Oct 93 23:38:48 IST <9310312138.AA19880@csc>
%Subject: LaTex Macros
%Status: RO
%Content-Length: 19065
%X-Lines: 638
%
%
%I'm in oberwolfach w/ micha, so I';ll just send you this and answer
%your questions on it when I get back to vancouver next week...
%
%o Jack
%_/\_.
%(')>-(`) snoeyink@cs.ubc.ca
%----figbox.tex----
% 12 Oct 90 LaTeX figure boxing macros by Jack Snoeyink
% w/ ideas from John Hershberger
% and code from Tom Rokicki
%%%%%%%%%%%%%%%%%
%
% This file contains two classes of macros for dealing with figures in
% Latex documents:
% \figbox (and its relatives) take a box containing a figure or table
% or whatever, caption it, and either place it on the page and wrap the
% text around it or let it float (as usual).
% \boxeps and \boxps take encasulated or plain postscript from a file
% and put it in a box for use by \figbox
%
%%%%%%%%%%%%%%%%
%
% The most important macro for dealing with boxes is:
%
% \figbox: caption a box as a figure or table and place it in the margin
% (a marginal box) or float it (a floating box).
% Wide marginal boxes are promoted to one column floats; wide one
% column floats are promoted to two column floats.
%
% Usage: \figbox [*htblr] {box} {style} {label} {caption}
%
% The optional argument default is [htb]. Other choices
% * two column (must come first),
% h here, t top, b bottom (as in latex),
% l left, r right (marginal boxes that do not float)
%
% NB: Left and right marginal boxes will not overlap in height.
%
% Two column figures default to htb unless some other
% options follow the *.
% They can also be specified by style {figure*} or {table*}.
%
% box is a box containing a figure or table.
% style is the caption style {figure}, {figure*}, {table}, or {table*}.
% label is the label assigned.
% if empty, the figure will not be labeled.
% caption is of the form {}, {Caption} or {[short caption] Long caption},
% where the short caption goes into the list of figures/tables.
% if empty, the figure will not be captioned or labeled.
%
% The lower level macro \captionbox, described much later, can be used
% to combine two or more captioned figures into a single marginal or
% floating box.
%
%%%%%%%%%%%%%%%%
%
% options defined by macros are: \captionstyle \boxcaptionstyle
% \maxfigfraction \figboxhang \figboxmargin
%
% \captionstyle is executed for all floating box captions
% \boxcaptionstyle is executed for all marginal box captions.
% Default \raggedright
% \maxfigfraction is the maximum fraction of a column that can be taken up
% by a marginal box before it gets converted to a float.
% Default .6
% dimen \figboxhang is the distance a marginal box hangs into the margin.
% Default 0pt
% dimen \figboxmargin is the width of space between the text and a
% marginal box.
% Default .15in
%
%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%
%
% The most important macros for turning postscript into boxes are:
%
% \boxeps: Form a box containing encapsulated postscript from a file at a
% given scale.
% \boxps: Form a box containing postscript from a file at a given scale,
% width, and height.
%
%
% Usage: \boxeps[local commands] {file} {scale}
% \boxps[local commands] {file} {scale} {width} {height}
%
% The optional local commands are executed after scaling. As described
% below, they are useful for setting origins, non-uniform scaling, etc?
% file is the filename in the \figuredirectory and without the .ps
% extension.
% scale is a real number: 1 to use the drawn size. 0.9 to use 90% of
% drawn size. Can be changed by local commands. (see below)
% width and height are dimensions of the box produced by \boxps.
%
%%%%%%%%%%%%%%%%
%
% options: \DVIscaling \figuredirectory \missingfigure \boxer
% local commands: \horizscale \vertscale
%
% \DVIscaling is the number the dvi to ps converter considers a unit
% scaling transformation. Default: 1 for dvi2ps --- 100 for
% some of Rokicki's versions of dvips.
%
% \globalscaling is the scaling applied to every picture --- useful
% for making double size figures for journals. Default: 1
%
% \figuredirectory is the pathname where figure postscript files are kept.
% Default: ./figure
%
% \missingfigure{file} is executed when the file is not found.
% Default: \def\missingfigure#1{\hbox{Missing figure #1.ps}}
%
% \boxer is the command used by \boxps to make a box of the desired size
% containing the figure. Possible commands depend on the origin of
% the figure, and include \llboxer \lcboxer \ulboxer and \ccboxer.
% (ll = lower left, ul = upper left, cc = dead center, etc.)
% \boxer can be defined in the local commands or globally.
%
% Local commands \horizscale, and \vertscale can scale the picture
% horizontally and vertically by different factors. \scale
% scales the picture uniformly. Scales accumulate.
%
%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%
%
% Default options
%
\def\captionstyle{}
\def\boxcaptionstyle{\raggedright}
\def\maxfigfraction{.6}% --- max frac of column for marg figure
\newdimen\figboxmargin% --- margin between text and picture
\figboxmargin = .15in
\newdimen\figboxhang% --- amount to hang picture into margin
\figboxhang = 0pt
\def\DVIscaling{1}% --- for dvi2ps
\def\globalscaling{1}% --- for scaling everything
\def\figuredirectory{./figures}%--- where to find the .ps files
\let\boxer=\llboxer% --- default origin in lower left for non-
% encapsulated postscript figures
\def\missingfigure#1{\hbox{Missing figure #1.ps}}
%%%%%%%%%%%%%%%%
%
% Use latex internals
%
\catcode `\@=11
\special{! /@scaleunit 1 def }
\newbox\figurebox
\def\figbox{% [place] box style label caption
\@ifnextchar[{\figboxaux}{\figboxaux[htb]}}
%
% Parse position parameters and try appropriate placement
%
\long\def\figboxaux[#1#2]#3#4#5#6{% --- want left margin figure?
\writepict{{#3}{#4}{#5}{#6}}% --- save figure if journalpicts
\setbox\figurebox\hbox{#3}%
\if l#1\tryleftbox{#4}{#5}{#6}%
\else% --- want right margin figure?
\if r#1\tryrightbox{#4}{#5}{#6}%
\else% --- want one or two column figure?
\if *#1\checktwocoloptions#2]{\box\figurebox}{#4*}{#5}{#6}%
\else\tryonecol[#1#2]{#4}{#5}{#6}%
\fi
\fi
\fi\ignorespaces}
%
% Try a placement; promote if too wide
%
\long\def\tryleftbox#1#2#3{% --- does left margin figure fit?
\ifdim\wd\figurebox>\maxfigfraction\columnwidth \tryonecol[htb]{#1}{#2}{#3}%
\else\leftbox{\captionbox{\box\figurebox}{#1}{#2}{#3}}\fi}
\long\def\tryrightbox#1#2#3{% --- does right margin figure fit?
\ifdim\wd\figurebox>\maxfigfraction\columnwidth \tryonecol[htb]{#1}{#2}{#3}%
\else\rightbox{\captionbox{\box\figurebox}{#1}{#2}{#3}}\fi}
\def\checktwocoloptions{% --- make sure twocol has htb or p
\@ifnextchar]{\floatbox[htb}{\floatbox[}}
\long\def\tryonecol[#1]#2#3#4{% --- does one column figure fit?
\ifdim\wd\figurebox>\columnwidth \floatbox[#1]{\box\figurebox}{#2*}{#3}{#4}%
\else\floatbox[#1]{\box\figurebox}{#2}{#3}{#4}\fi}
%
% Do the work for a floating box.
%
% \floatbox[place]{box} {style} {label} {caption}
%
\long\def\floatbox[#1]#2#3#4#5{%
\begin{#3}[#1]
\hbox to \hsize{\hfil#2\hfil}
\captionandlabel{#3}{#4}{#5}
\end{#3}
}
%
% Make a caption the width of the box with given style.
%
%\captionbox {box} {style} {label} {[optshortcap] caption}
%
% options: \boxcaptionstyle
%
%%%%%%%%%%%%%%%%
%
% This can be used with box commands (\hbox, \vbox, \mbox, and the like),
% along with \figbox with an empty caption, to place two or more captioned
% boxes in a single marginal or floating box.
%
% e.g. \figbox{\hbox{
% \captionbox{ \boxeps{fig1}1} {figure} {f1} {a figure}
% \hskip 1in
% \captionbox{ \boxeps{tab1}1} {table} {t1} {a table}
% }} {figure} {ignored-label} {}
%
\long\def\captionbox#1#2#3#4{% box style label {[optshortcap] caption}
\setbox\figurebox\hbox{#1}%
\parbox[t]{\wd\figurebox}{%
\bigskip\box\figurebox
\let\captionstyle=\boxcaptionstyle
\captionandlabel{#2}{#3}{#4}
\bigskip
}}
% --- caption and label if label is nonempty
\def\captionandlabel#1#2#3{% --- {style} {label} {caption}
\def\testit{#3}%
\ifx\testit\empty\else% --- if non empty caption
\writecapt{{#1}{#2}{#3}}% --- save for journal figs
\captypeunstarred#1*.% --- remove optional stars
\getcaption#3\endc@ption% --- parse the caption
\def\testit{#2}% --- check for empty label
\ifx\testit\empty\else\label{#2}\fi
\fi}
\def\captypeunstarred#1*#2.{% --- remove two-column star from style
\def\@captype{#1}}
%
% Parse the caption of the form {caption} or {[short] long caption}
%
\def\getcaption{\@ifnextchar[{\getcaptwo}{\getcapone}}
\long\def\getcapone#1\endc@ption{\caption[#1]%
{\def\baselinestretch{1}\Large\normalsize\captionstyle\ignorespaces #1}}
\long\def\getcaptwo[#1]#2\endc@ption{\caption[#1]%
{\def\baselinestretch{1}\Large\normalsize\captionstyle\ignorespaces #2}}
%
% Do the right thing with an already captioned marginal box
%
% \leftbox{box}
% \rightbox{box}
%
% options: \figboxmargin \figboxhang \figuredirectory
\newdimen\figboxht
\newcount\figboxn
\newcount\figboxlines% --- picture height in number of lines
\newdimen\figboxwid% --- picture width
\newif\ifisleftbox
\long\def\leftbox#1{%
\setbox\figurebox\hbox{#1}\global\isleftboxtrue
\startmarginbox
\vadjust{\smash{\rlap{\hskip\hsize\hskip\figboxhang
\llap{\raise.7\baselineskip\box\figurebox\hskip\rightskip}}}}%
\endmarginbox%
}
\long\def\rightbox#1{%
\setbox\figurebox\hbox{#1}\global\isleftboxfalse
\startmarginbox
\smash{\llap{\raise.7\baselineskip\box\figurebox\hskip\figboxmargin}}%
\endmarginbox%
}
\def\startmarginbox{%
\ifvmode\passpict\let\endmarginbox=\indent
\else\message{WARNING: marginbox in not in vmode}\hfilneg\ \passpict
\let\endmarginbox=\relax\fi
%
\figboxht=\dp\figurebox% actual height
\advance\figboxht by 1.3\baselineskip% round up
\vskip\figboxht\penalty-300\vskip-\figboxht% filbreak trick
\divide\figboxht by\baselineskip% how many lines?
\global\figboxlines=\figboxht
%
\global\figboxwid=\wd\figurebox
\global\advance\figboxwid by \figboxmargin
\global\advance\figboxwid by -\figboxhang
\setmypar\noindent}
%
%
% Useful macros
%
% for lengthening (and shortening) picture space
%
\def\addlines#1{\global\advance\figboxlines by #1\myparshape}
\def\zerolines{\origpar\global\figboxlines=0\myparshape}
%
% for passing any marginal figures
%
\def\passpict{\par\ifnum\figboxlines>1\vskip\figboxlines\baselineskip
\zerolines\fi}
\def\emptybox#1#2{\hbox to #1{\vbox to #2{\vss}\hss}}
%%%%%%%%%%%%%%%%%%%%%
%\par hacking
% We define our own version of Tex's \par and tell Latex it is the original
\global\let\origpar=\@@par% --- Latex's saved version of Tex's \par
\global\let\dopar=\origpar
\global\def\@@par{\dopar}
\@setpar{\dopar}
\def\setmypar{\global\let\dopar=\mypar
\global\prevgraf=0\myparshape}
\def\mypar{\origpar\global\advance\figboxlines by -\prevgraf%
\global\prevgraf=0\myparshape}
\def\myparshape{\relax%
\ifnum\figboxlines>1\theparshape \else
\global\hangindent=0pt\global\hangafter=1% these sometimes get restored...
\global\let\dopar=\origpar\fi}
\def\theparshape{%
\ifisleftbox\global\hangindent=-\figboxwid
\else\global\hangindent=\figboxwid \fi
\global\hangafter=-\figboxlines \global\advance\hangafter by 1%
}
%%%%%%%%%%%%%%%%
%
% Macros to make the list of figures and captions used by journals
%
% To use, execute \journalpicts{scalefactor} at the beginning of the
% file. All figbox figures and tables will be written to a .pic file
% and all captions to a .cap file. On \enddocument, these files are
% read in and the lists of figures and captions are produced.
%
% For non-english versions, or to add other types of floats, the
% macro \definefnum should be changed.
%
\def\definefnum#1{% --- ugly latex hacking to get the
% right figure/table name
\def\fnum@figure{Figure \ref{#1}}%
\def\fnum@table{Table \ref{#1}}%
\def\fnum@code{Algorithm \ref{#1}}%
}
\def\writepict#1{}
\def\writecapt#1{}
%%%%%%%%%%%%%%%%
% \journalpicts{globalscalefactor} writes all pictures and captions to
% files and modifies \enddocument to read these files and
% produce lists of enlarged figures and of captions
%
\def\journalpicts#1{% global scale factor
\newwrite\pictfile
\newwrite\captfile
%
\openout\pictfile\jobname.pic
\openout\captfile\jobname.cap
%
\gdef\writepict##1{\unexpandedwrite\pictfile{\doit##1}}%
\gdef\writecapt##1{\unexpandedwrite\captfile{\doit##1}}%
%
\global\let\ENDdocument=\enddocument
\gdef\enddocument{\DOjournalpicts{#1}\ENDdocument}
}
%
% generate enlarged figures 1 per page and list captions
%
\def\DOjournalpicts#1{{%
\def\writepict##1{}\closeout\pictfile
\def\writecapt##1{}\closeout\captfile
\@fileswfalse% --- turn off toc, lof, and so on.
%
\onecolumn
\def\globalscaling{#1}
\def\doit##1##2##3##4{% --- print a picture on blank page
% {box}{style}{label}{caption}
\figboxaux[t]{\hss##1\hss}{##2}{}{}%
\vspace*{1in}
\definefnum{##3}% --- use Figure \ref{label} or Table \ref{label}
\captionandlabel{##2}{}{##4}
\clearpage}%
%
\input\jobname.pic
%
\def\doit##1##2##3{% --- print caption
% {style}{label}{caption}
\definefnum{##2}% --- use Figure \ref{label} or Table \ref{label}
\captionandlabel{##1}{}{##3}}%
%
\raggedright\let\captionstyle=\raggedright
\def\@makecaption##1##2{##1: ##2\par}% --- Be sure captions aren't centered
%
\input\jobname.cap
}}% --- to be followed by \enddocument
%%%%%%%%%%%%%%%%
%
% Box macros
%
%%%%%%%%%%%%%%%%
%
% \boxeps[local commands] {file} {scale}
%
% options \DVIscaling \figuredirectory \scale \horizscale \vertscale
% \missingfigure \globalscaling
%
% \boxps[local commands] {file} {scale} {width} {height}
%
% \scale \horizscale \vertscale
%
% Various `boxers' --- which one is used depends on where the origin
% of the picture is. If you are using the bounding boxes of
% encapsulated postscript, just use the default \llboxer. Otherwise,
% you have to figure out where the origin is and set \boxer appropriately.
%
\def\llboxer#1{\vbox to \figboxht{\vfil\hbox to \figboxwid{#1\hfill}}}
\def\lcboxer#1{\vbox to \figboxht{\vfil\hbox to \figboxwid{\hfill#1\hfill}}}
\def\oldboxer#1{\vbox to \figboxht{\vfil
\hbox to \figboxwid{\hfill\llap{#1\hskip4.25in}\hfill}}}
\def\ulboxer#1{\vbox to \figboxht{\hbox to \figboxwid{#1\hfill}\vfil}}
\def\ccboxer#1{\vbox to \figboxht{\vfil
\hbox to \figboxwid{\hfill#1\hfill}\vfil}}
% a macro to strip the pt off a dimension
%
{\catcode`\p=12\catcode`\t=12
\gdef\removedimen#1pt{#1}}
\def\defscaled#1#2{#2=\DVIscaling#2%
\xdef#1{\expandafter\removedimen\the#2}}
\def\DVIspace{ }
\newdimen\hscalefactor
\newdimen\vscalefactor
\def\scale#1{\horizscale{#1}\vertscale{#1}}
\def\horizscale#1{\hscalefactor=#1\hscalefactor\figboxht=#1\figboxht}
\def\vertscale#1{\vscalefactor=#1\vscalefactor\figboxwid=#1\figboxwid}
%%%%%%%%%%%%%%%%
%
\def\boxps{%
\@ifnextchar[{\boxpsaux}{\boxpsaux[\relax]}}
\def\boxpsaux[#1]#2#3#4#5{%
{\figboxwid#4\figboxht#5\hscalefactor=1pt\vscalefactor=1pt%
\scale{#3}%
\scale{\globalscaling}%
#1%
\defscaled\DVIhscale\hscalefactor
\defscaled\DVIvscale\vscalefactor
\boxer{\special{psfile=\figuredirectory/#2.ps \DVIspace
hscale=\DVIhscale\DVIspace
vscale=\DVIvscale}}}%
}
%%%%%%%%%%%%%%%%
% The following is from tex/inputs/epsf.tex by Tom Rokicki
% with slight modifications by Jack Snoeyink.
%
% Written by Tomas Rokicki of Radical Eye Software, 29 Mar 1989.
%
% Include an ecapsulated postscript graphic.
% Works by finding the bounding box comment.
%
\newread\Epsffilein
\newif\ifEpsffileok
\newif\ifEpsfbbfound
\newdimen\pspoints
\pspoints=1in
\divide\pspoints by 72
\def\boxeps{%
\@ifnextchar[{\boxepsaux}{\boxepsaux[\relax]}}
\def\boxepsaux[#1]#2#3{%
%
% The first thing we need to do is to open the
% PostScript file, if possible.
%
\openin\Epsffilein=\figuredirectory/#2.ps
\ifeof\Epsffilein\message{I couldn't open \figuredirectory/#2.ps }%
\missingfigure{#2}
\else
%
% Okay, we got it. Now we scan lines until we find one
% that doesn't start with a percentage sign. We're
% looking for the bounding box comment.
%
{\Epsffileoktrue\Epsfbbfoundfalse
\catcode`\%=11 \catcode`\\=11
\catcode`\{=11 \catcode`\}=11
\catcode`\$=11 \catcode`\^=11
\catcode`\&=11 \catcode`\#=11
\catcode`\~=11 \catcode`\_=11
\loop
\read\Epsffilein to \Epsffileline
\ifeof\Epsffilein\Epsffileokfalse\else
%
% Now we check to make sure the first character is a % sign,
% and that the rest are `%BoundingBox:' if necessary.
%
\expandafter\Epsfaux\Epsffileline . .\\%
\fi
\ifEpsffileok\repeat
\ifEpsfbbfound
\figboxht=\Epsfury\pspoints
\advance\figboxht by-\Epsflly\pspoints
\figboxwid=\Epsfurx\pspoints
\advance\figboxwid by-\Epsfllx\pspoints
\else
\message{No bounding box comment in \figuredirectory/#2.ps }%
\figboxwid=2in\figboxht=1in%
\fi%
\immediate\closein\Epsffilein
\hscalefactor=1pt\vscalefactor=1pt%
\scale{#3}%
\scale{\globalscaling}%
#1%
%
\defscaled\DVIhscale\hscalefactor
\defscaled\DVIvscale\vscalefactor
%
\hscalefactor=-\Epsfllx\hscalefactor
\hscalefactor=1.00375\hscalefactor% --- 72.27/72 = TeXpoints/PSpoints
\defscaled\DVIhoffset\hscalefactor
%
\vscalefactor=-\Epsflly\vscalefactor
\vscalefactor=1.00375\vscalefactor% --- 72.27/72 = TeXpoints/PSpoints
\defscaled\DVIvoffset\vscalefactor
%
\llboxer{\special{psfile=\figuredirectory/#2.ps \DVIspace
hoffset=\DVIhoffset\DVIspace
voffset=\DVIvoffset\DVIspace
hscale=\DVIhscale\DVIspace
vscale=\DVIvscale}} }%
\fi
}%
%
% We need these for comparison purposes.
%
{\catcode`\%=11 \global\let\Epsfpar=\par
\global\let\Epsfpercent=%\global\def\Epsfbblit{%BoundingBox:}}%
%
% This is our function that checks for `%BoundingBox:' and grabs the
% values if they are found.
%
\long\def\Epsfaux#1#2 #3\\{\relax\ifx#1\Epsfpercent
\def\testit{#2}\ifx\testit\Epsfbblit
\Epsfsize #3 . . . .\\%
\global\Epsffileokfalse
\global\Epsfbbfoundtrue
\fi\else\ifx#1\Epsfpar\else\global\Epsffileokfalse\fi\fi}%
%
% Here we grab the values and stuff them in the appropriate definitions.
%
\def\Epsfsize#1 #2 #3 #4 #5\\{\global\def\Epsfllx{#1}\global\def\Epsflly{#2}%
\global\def\Epsfurx{#3}\global\def\Epsfury{#4}}%
\catcode`\@=12 % done with with internals
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For backward compatability
%
% \pic file; height in inches; width in inches; caption\par
% \picsc file; ht (in); wid (in); scale; caption\par
%
% \mpic file; height in inches; width in inches; caption\par
% \mpicsc file; ht (in); wid (in); scale; caption\par
%
%%%%%%%%%%%%%%%%
\def\pic#1;#2;#3;#4\par{\picsc#1;#2;#3;1;#4\par}% file; wd; ht; caption
\def\picsc#1;#2;#3;#4;#5\par{% file; wd; ht; scale; caption
\figbox[htb]{\boxeps{#1}{#4}%{#2in}{#3in}%
}{figure}{#1}{%
% (#1: #2x#3 sc #4) % Uncomment for debugging
#5}}
\def\mpic#1;#2;#3;#4\par{\mpicsc#1;#2;#3;1;#4\par}% file; wd; ht; caption
\def\mpicsc#1;#2;#3;#4;#5\par{% file; wd; ht; scale; caption
\figbox[l]{\boxeps{#1}{#4}%{#2in}{#3in}%
}{figure}{#1}{%
% (#1: #2x#3 sc #4) % Uncomment for debugging
#5}}
% !TEX root = ../Multi_Momentum_INFOCOM.tex
%\renewcommand{\thesection}{\Roman{section}}
%\renewcommand{\thesubsection}{\Roman{section}-\Alph{subsection}}
%\renewcommand{\thesubsubsection}{\Alph{subsection}.\arabic{subsubsection}}
\newcommand{\A}{\mathbf{A}}
\renewcommand{\a}{\mathbf{a}}
\newcommand{\B}{\mathbf{B}}
\newcommand{\Bt}{\mathbf{B}}
\renewcommand{\b}{\mathbf{b}}
\newcommand{\bt}{\mathbf{b}}
\renewcommand{\c}{\mathbf{c}}
\newcommand{\ct}{\widetilde{\mathbf{c}}}
\newcommand{\C}{\mathbf{C}}
\newcommand{\Ch}{\widehat{\mathbf{C}}}
\newcommand{\D}{\mathbf{D}}
\newcommand{\Dh}{\widehat{\mathbf{D}}}
\renewcommand{\d}{\mathbf{d}}
\newcommand{\E}{\mathbf{E}}
\newcommand{\e}{\mathbf{e}}
\newcommand{\et}{\widetilde{\mathbf{e}}}
\newcommand{\f}{\mathbf{f}}
\newcommand{\F}{\mathbf{F}}
\newcommand{\G}{\mathbf{G}}
\newcommand{\g}{\mathbf{g}}
\newcommand{\I}{\mathbf{I}}
\newcommand{\J}{\mathbf{J}}
\newcommand{\K}{\mathbf{K}}
\newcommand{\M}{\mathbf{M}}
\newcommand{\m}{\mathbf{m}}
\newcommand{\mh}{\widehat{\mathbf{m}}}
\newcommand{\Mh}{\widehat{\mathbf{M}}}
\newcommand{\Mt}{\widetilde{\mathbf{M}}}
\newcommand{\Mb}{\overline{\mathbf{M}}}
\newcommand{\N}{\mathbf{N}}
\newcommand{\n}{\mathbf{n}}
\newcommand{\p}{\mathbf{p}}
\renewcommand{\P}{\mathbf{P}}
\newcommand{\Q}{\mathbf{Q}}
\newcommand{\q}{\mathbf{q}}
\newcommand{\R}{\mathbf{R}}
\renewcommand{\r}{\mathbf{r}}
\renewcommand{\S}{\mathbf{S}}
\newcommand{\Sh}{\widehat{\mathbf{S}}}
\newcommand{\s}{\mathbf{s}}
\renewcommand{\t}{\mathbf{t}}
\newcommand{\T}{\mathbf{T}}
\newcommand{\Th}{\widehat{\mathbf{T}}}
\newcommand{\U}{\mathbf{U}}
\renewcommand{\u}{\mathbf{u}}
\newcommand{\V}{\mathbf{V}}
\renewcommand{\v}{\mathbf{v}}
\newcommand{\W}{\mathbf{W}}
\newcommand{\w}{\mathbf{w}}
\newcommand{\wt}{\widetilde{\mathbf{w}}}
\newcommand{\wts}{\widetilde{w}}
\newcommand{\X}{\mathbf{X}}
\newcommand{\Xh}{\widehat{\mathbf{X}}}
\newcommand{\x}{\mathbf{x}}
\newcommand{\xt}{\widetilde{\mathbf{x}}}
\newcommand{\xh}{\widehat{\mathbf{x}}}
\newcommand{\Y}{\mathbf{Y}}
\newcommand{\y}{\mathbf{y}}
\newcommand{\z}{\mathbf{z}}
\newcommand{\Z}{\mathbf{Z}}
\newcommand{\yt}{\widetilde{\mathbf{y}}}
\newcommand{\rb}{\mathbf{b}}
\newcommand{\rvec}{\mathbf{r}}
\newcommand{\0}{\mathbf{0}}
\newcommand{\1}{\mathbf{1}}
\newcommand{\Hc}{\mathbf{H}}
\newcommand{\tHc}{\widetilde{\mathbf{H}}}
\newcommand{\hHc}{\overline{\mathbf{H}}}
\newcommand{\DL}{\mathbf{DL}}
\newcommand{\mLambda}{\mathbf{\Lambda}}
\newcommand{\vlambda}{\boldsymbol{\lambda}}
\newcommand{\mOmega}{\mathbf{\Omega}}
\newcommand{\mOmegab}{\overline{\mathbf{\Omega}}}
\newcommand{\mGamma}{\mathbf{\Gamma}}
\newcommand{\mPi}{\mathbf{\Pi}}
\newcommand{\mPsi}{\mathbf{\Psi}}
\newcommand{\mPsib}{\overline{\mathbf{\Psi}}}
\newcommand{\valpha}{\mathbf{\alpha}}
\newcommand{\vpi}{\boldsymbol \pi}
\newcommand{\Sone}{\mathcal{S}}
\newcommand{\tr}{\mathrm{Tr}}
\newcommand{\new}{\mathrm{new}}
\newcommand{\net}{\mathrm{net}}
\newcommand{\link}{\mathrm{link-phy}}
\newcommand{\LL}{\mathbf{L}}
\newcommand{\Out}[1]{\mathcal{O}\left( #1 \right)}
\newcommand{\In}[1]{\mathcal{I}\left( #1 \right)}
\newcommand{\Int}[1]{\mathcal{F}\left( #1 \right)}
\newcommand{\sInt}[1]{\hat{\mathcal{F}}\left( #1 \right)}
\renewcommand{\Re}[1]{\mbox{$\mathfrak{Re} \left( #1 \right)$}}
\renewcommand{\Im}[1]{\mbox{$\mathfrak{Im} \left( #1 \right)$}}
\newcommand{\Resq}[1]{\mbox{$\mathfrak{Re}^{2} \left( #1 \right)$}}
\newcommand{\Imsq}[1]{\mbox{$\mathfrak{Im}^{2} \left( #1 \right)$}}
\newcommand{\LTM}{\langle L \rangle}
\newcommand{\con}[1]{\overline{#1}}
\newcommand{\abs}[1]{\mbox{$\lvert #1 \rvert$}}
\newcommand{\norm}[1]{\mbox{$\left\lVert #1 \right\rVert$}}
\newcommand{\dual}{$\mathbf{D}^{\mathrm{CRPBA}}$}
\newcommand{\dnet}{$\mathbf{D}^{\mathrm{CRPBA}}_{\net}$}
\newcommand{\dlink}{$\mathbf{D}^{\mathrm{CRPBA}}_{\link}$}
\newcommand{\Pwrcon}{\Omega_{+}(n)}
\newcommand{\src}[1]{\mathrm{Src}(#1)}
\newcommand{\dst}[1]{\mathrm{Dst}(#1)}
\newcommand{\rx}[1]{\mathrm{Rx}(#1)}
\newcommand{\tx}[1]{\mathrm{Tx}(#1)}
\newcommand{\Dy}{\Delta \y}
\newcommand{\Dyt}{\Delta \widetilde{\y}}
%\newcommand*\circled[1]{\tikz[baseline=(char.base)]{
% \node[shape=circle,draw,inner sep=2pt] (char) {#1};}}
%\newcommand{\seq}[1]{\mbox{$\{ #1 \} $}}
%\newcommand{\myref}[1]{(\ref{#1})}
%\newcommand{\sd}{\mbox{$\stackrel{\Delta}{=}$}}
%\newcommand{\sci}[2]{\mbox{$#1 \times 10^{- #2}$}}
%%\newtheorem{thm}{Theorem}
\newtheorem{thm}{Theorem}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{ex}{Example}[section]
%\theoremstyle{definition}
\newtheorem{defn}{Definition}
%\theoremstyle{remark}
\newtheorem{finalremark}[thm]{Final Remark}
\newtheorem{rem}{Remark}
\newtheorem{sol}{Solution}
\newtheorem{fact}{Fact}
%\numberwithin{equation}[section]
\newcommand{\po}{\mbox{$ \mbox{P}_{\rm out}$}}
\newcommand{\ga}{\bar{\gamma}}
\newcommand{\mtrx}[1]{\mbox{$\left[\begin{array} #1 \end{array}\right]$}}
\newcommand{\mtrxt}[1]{\mbox{$\begin{array} #1 \end{array}$}}
\newcommand{\diag}[1]{\mathrm{Diag}\left\{ #1 \right\}}
\newcommand{\diaginv}[1]{\mathrm{Diag}^{-1}\left\{ #1 \right\}}
\newcommand{\diagtran}[1]{\mathrm{Diag}^{T}\left\{ #1 \right\}}
\newcommand{\diagv}[1]{\mathrm{diag}\left\{ #1 \right\}}
\newcommand{\etal}{{\em et al. }}
\newcommand{\ind}{\mathbbm{1}}
\newcommand{\vbeta}{\boldsymbol \beta}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
\algtext*{EndWhile}% Remove "end while" text
\algtext*{EndIf}% Remove "end if" text
\algtext*{EndFor}% Remove "end while" text
\algtext*{EndFunction}% Remove "end if" text
\algtext*{EndProcedure}% Remove "end while" text
%\algtext*{EndIf}% Remove "end if" text
%\newcommand{\algorithmicinitialize}{\textbf{Initialization:}}
%\newcommand{\algorithmicmainiterate}{\textbf{Main Iteration:}}
%\newcommand{\algorithmicprimal}{\textbf{Primal Newton Direction:}}
%\newcommand{\algorithmicdual}{\textbf{Dual Variables Update:}}
% !TEX root = ../ML_Networking_TNSE20.tex
\begin{abstract}
%Thanks to the rise of machine learning (ML) and its wide range of applications,
Recent years have witnessed a rapid growth of distributed machine learning (ML) frameworks, which exploit the massive parallelism of computing clusters to expedite ML training.
However, the proliferation of distributed ML frameworks also introduces many unique technical challenges in computing system design and optimization.
In a networked computing cluster that supports a large number of training jobs, a key question is how to design efficient scheduling algorithms to allocate workers and parameter servers across different machines to minimize the overall training time.
Toward this end, in this paper, we develop an online scheduling algorithm that jointly optimizes resource allocation and locality decisions.
Our main contributions are three-fold: i) We develop a new analytical model that considers both resource allocation and locality;
ii) Based on an equivalent reformulation and observations on the worker-parameter server locality configurations, we transform the problem into a mixed packing and covering integer program, which enables approximation algorithm design;
iii) We propose a meticulously designed approximation algorithm based on randomized rounding and rigorously analyze its performance.
Collectively, our results contribute to the state of the art of distributed ML system optimization and algorithm design.
\end{abstract}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Proof of Lemma~\ref{lem_ILP}} \label{appdx:lem_ILP1}
\begin{proof}
Consider the probabilities of the following ``bad'' events: 1) $\c^{\top} \hat{\x} > \frac{3G_\delta}{\delta} \c^{\top}\bar{\x}$; 2) $\exists i$ such that $(\A\hat{\x})_{i} < a_{i}$; and 3) $\exists i$ such that $(\B\hat{\x})_{i} > b_{i}$.
Note that events 2) and 3) can be equivalently rewritten as: 2') $\exists i$ such that $\mathbb{E}\{(\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} < W_{a}\}$ and 3') $\exists i$ such that $\mathbb{E}\{(\B\hat{\x})_{i} \frac{W_{b}}{b_{i}} > W_{b}\}$.
Since $\mathbb{E}\{ \hat{\x} \} \!=\! \x' \!=\! G_\delta\bar{\x}$, by linearity of expectation, we have:
\begin{align}
\label{eqn_exp1}&\mathbb{E}\{\c^{\top} \hat{\x} \} = \c^{\top} \mathbb{E}\{ \hat{\x} \} = \c^{\top} G_\delta \bar{\x} = G_\delta \c^{\top}\bar{\x}, \\
%
\label{eqn_exp2}& \mathbb{E}\left\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \right\} = G_\delta \mathbb{E} \left\{ (\A\bar{\x})_{i}\frac{W_{a}}{a_{i}} \right\} \geq G_\delta W_{a}, \\
%
\label{eqn_exp3}& \mathbb{E}\left\{ (\B\hat{\x})_{i} \frac{W_{b}}{b_{i}} \right\} = G_\delta \mathbb{E} \left\{ (\B\bar{\x})_{i}\frac{W_{b}}{b_{i}} \right\} \leq G_\delta W_{b}.
\end{align}
Then, by the Markov inequality and ~(\ref{eqn_exp1}), we can obtain the probability $
\mathrm{Pr} \left\{ \c^{\top} \hat{\x} > \frac{3G_\delta}{\delta} \c^{\top} \bar{\x} \right\} \leq \frac{\delta}{3}$.
%
Next, we note that each $\hat{x}_{j}$ can be viewed as a sum of independent random variables in $[0,1]$ as follows:
The fixed part of $\lfloor x'_{j} \rfloor$ is a sum of $\lfloor x'_{j} \rfloor$ random variables with value 1 with probability 1.
Then, we have that $(\B\hat{\x})_{i} \frac{W_b}{b_{i}} = (\sum_{j}[\B]_{ij}\hat{x}_{j}) \frac{W_b}{b_{i}}$ is also a sum of independent random variables in $[0,1]$.
Using the Chernoff bound, we have
\begin{align*}
\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} \frac{W_{b}}{b_{i}} > (1+\epsilon) G_\delta W_{b} \Big\} \leq \exp \Big( -\epsilon^{2} \frac{G_\delta W_{b}}{3} \Big).
\end{align*}
%Using Chernoff bound, we have $\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq (1-\epsilon) GW_{a} \} \leq \exp(-\epsilon^{2}\frac{GW_{a}}{2} )$.
Setting $(1+\epsilon)G_\delta \!=\!1$, i.e., $\epsilon \!=\! \frac{1}{G_\delta}\!-\!1$, we have:
\begin{align} \label{eqn_chernoff_bnd1_1}
%\mathrm{Pr} \left\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq W_{a} \right\} \leq \exp\left(- \Big(1-\frac{1}{G}\Big)^{2} G \frac{W_{a}}{2} \right).
\mathrm{Pr} \left\{ (\B\hat{\x})_{i} \frac{W_b}{b_{i}} > W_b \right\} \leq \exp\left(- \Big(\frac{1}{G_\delta}-1\Big)^{2} G_\delta \frac{W_b}{3} \right). \!\!\!
\end{align}
Forcing $\exp\!\left(- (\frac{1}{G_\delta}\!-\!1)^{2} G_\delta \frac{W_b}{3} \right) \!=\! \frac{\delta}{3r}$ and solving $G_\delta$, we have:
\begin{align} \label{eqn_L_def_1}
G_\delta \triangleq 1 + \frac{3\ln(3r/\delta)}{2W_b}\! -\!\sqrt{ \bigg(\frac{3\ln(3r/\delta)}{2W_b} \bigg)^{2} + \frac{3\ln(3r/\delta)}{W_b} }. \!\!\!
\end{align}
Using (\ref{eqn_exp2}), the Chernoff bound, and following similar arguments, we have:
\begin{align*}
\mathrm{Pr} \Big\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq (1-\epsilon) G_\delta W_{a} \Big\} \leq \exp \Big(-\epsilon^{2}\frac{G_\delta W_{a}}{2} \Big).
\end{align*}
Forcing $\exp\left( -\epsilon^{2} \frac{G_\delta W_a}{2} \right)=\frac{\delta}{3m}$ and solving for $\epsilon$, we have $\epsilon = (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$.
It follows that
\begin{align*}
\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_a}{a_{i}} \leq ( 1 \!-\! (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}} )G_\delta W_a\} \leq \frac{\delta}{3m},
\end{align*}
which implies that:
\begin{align} \label{eqn_chernoff_bnd2_1}
\!\!\! \mathrm{Pr} \left\{ (\A\hat{\x})_{i} \!\leq\! a_i \left( 1 \!-\! \sqrt{\frac{2}{G_\delta W_a}\ln\Big(\frac{3m}{\delta}\Big)} \right)G_\delta, \exists i \right\} \leq \frac{\delta}{3m}. \!\!\!\!\!
\end{align}
By using union bound and (\ref{eqn_chernoff_bnd1_1}) and (\ref{eqn_chernoff_bnd2_1}), we have that events 1)--3) occur with probability less than $\frac{\delta}{3} + m\cdot\frac{\delta}{3m} + r\cdot\frac{\delta}{3r} = \delta$,
and the proof is complete.
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Proof of Lemma~\ref{lem_ILP2}} \label{appdx:lem_ILP2}
\begin{proof}
% {\em 2) When $G>1$: }
Similar to the case when $0<G_\delta\leq 1$, we can have the expectation equations for the bad cases as in Eqns.~(\ref{eqn_exp1})-(\ref{eqn_exp3}).
We also can view each $\hat{x}_j$ as a sum of independent random variables in $[0,1]$ in the same way.
Then, we have that $(\A\hat{\x})_{i} \frac{W_a}{a_{i}} = (\sum_{j}[\A]_{ij}\hat{x}_{j}) \frac{W_a}{a_{i}}$ is also a sum of independent random variables in $[0,1]$.
% Using Chernoff bound, we have $\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_a{a_i} \leq (1-\epsilon) GW_a \}
% \leq \exp( -\epsilon^{2} \frac{GW_a}{2} )$.
Using the Chernoff bound, we have that:
\begin{align*}
\mathrm{Pr} \{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq (1-\epsilon) G_\delta W_{a} \} \leq \exp(-\epsilon^{2}\frac{G_\delta W_{a}}{2} ).
\end{align*}
Setting $(1-\epsilon)G_\delta \!=\!1$, i.e., $\epsilon = 1-\frac{1}{G_\delta}$, we have:
\begin{align} \label{eqn_chernoff_bnd1}
\mathrm{Pr} \left\{ (\A\hat{\x})_{i} \frac{W_{a}}{a_{i}} \leq W_{a} \right\} \leq \exp\left(- \Big(1-\frac{1}{G_\delta}\Big)^{2} G_\delta \frac{W_{a}}{2} \right). \!\!\!
% \mathrm{Pr} \left\{ (\B\hat{\x})_{i} \frac{W_b}{b_{i}} > W_b \right\} \leq \exp\left(- \Big(\frac{1}{G}-1\Big)^{2} G \frac{W_b}{3} \right).
\end{align}
Forcing $\exp\!\left(- (1-\frac{1}{G_\delta})^{2} G_\delta \frac{W_a}{2} \right) \!=\! \frac{\delta}{3m}$ and solving $G_\delta$, we have:
\begin{align} \label{eqn_L_def}
G_\delta \triangleq 1 + \frac{\ln(3m/\delta)}{W_a}+\sqrt{ \bigg(\frac{\ln(3m/\delta)}{W_a} \bigg)^{2} + \frac{2\ln(3m/\delta)}{W_a} }. \!\!\!
\end{align}
Using (\ref{eqn_exp2}), the Chernoff bound, and following similar arguments, we have:
\begin{align*}
\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} \frac{W_b}{b_i} > (1+\epsilon) G_\delta W_b \Big\} \leq \exp \Big( -\epsilon^{2}\frac{G_\delta W_b}{3} \Big).
\end{align*}
Forcing $\exp\left( -\epsilon^{2} \frac{G_\delta W_b}{3} \right) = \frac{\delta}{3r}$ and solving for $\epsilon$, we have $\epsilon = (\frac{3}{G_\delta W_b}\ln(\frac{3r}{\delta}))^{\frac{1}{2}}$.
It follows that
\begin{align*}
\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} \frac{W_b}{b_i} > ( 1 + (\frac{3}{G_\delta W_b}\ln(\frac{3r}{\delta}))^{\frac{1}{2}} )G_\delta W_b \Big\} \leq \frac{\delta}{3r}, \end{align*}
which implies that:
\begin{align} \label{eqn_chernoff_bnd2}
\!\!\! \mathrm{Pr} \left\{ (\B\hat{\x})_{i} > b_i \left( 1+ \sqrt{\frac{3}{G_\delta W_b}\ln\Big(\frac{3r}{\delta}\Big)} \right)G_\delta, \exists i \right\} \leq \frac{\delta}{3r}. \!\!\!\!\!
\end{align}
By using union bound and (\ref{eqn_chernoff_bnd1}) and (\ref{eqn_chernoff_bnd2}), we have that events 1)--3) occur with probability less than $\frac{\delta}{3} + m\cdot\frac{\delta}{3m} + r\cdot\frac{\delta}{3r} = \delta$,
and the proof is complete.
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Proof of the Competitive Ratio} \label{appdx:thm_Competitive}
We use $OPT$ as the optimal objective value of Problem~R-DMLRS, which is also the optimum to Problem~D-R-DMLRS.
We let $\hat{\pi}_{i}$ denote the approximate schedule obtained by Algorithm~2, which inexactly solves Problem D-R-DMLRS.
Let $P_{i}$ and $D_{i}$ be the primal and dual objective values of Problems R-DMLRS and D-RMLRS after determining the schedule $\hat{\pi}_{i}$ in Algorithm~1.
Let $P_0$ and $D_0$ be the initial values of Problems R-DMLRS and D-RMLRS, respectively, where $P_0=0$ and $D_0=\sum_{t\in\mathcal{T}}\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}P_h^r[0]C_h^r$.
We also let $P_I$ and $D_i$ be the final primal and dual objective values returned by Algorithm~1.
We present our main result in Lemma~\ref{lem:result}.
\begin{lem}
\label{lem:result}
If there exists constants $\epsilon\geq 1$, $G_\delta> 0$ and $\delta\in(0,1]$ such that $P_i-P_{i-1}\geq \frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}), \forall i\in\mathcal{I}$, and if $P_0=0$ and $D_0\leq \frac{1}{2}OPT$, then Algorithm~1 is $\frac{6G_\delta\epsilon}{\delta}$-competitive.
\end{lem}
\begin{proof}[Proof of Lemma~\ref{lem:result}]
Since $P_I=\sum\limits_{i\in\mathcal{I}}(P_i-P_{i-1})$, and $D_I-D_0=\sum\limits_{i\in\mathcal{I}}(D_i-D_{i-1})$, we can have:
\begin{align*}
P_I&=\sum_{i\in\mathcal{I}}(P_i-P_{i-1})\geq \frac{\delta/3G_\delta}{\epsilon}\sum_{i\in\mathcal{I}}(D_i-D_{i-1})\\
&=\frac{\delta/3G_\delta}{\epsilon}(D_I-D_0).
\end{align*}
By weak duality theorem~\cite{Bertsekas99:Nonlinear}, we have
\begin{align*}
D_I\geq OPT\geq P_I.
\end{align*}
Thus, it follows that:
\begin{align*}
&D_I-D_0\geq \frac{1}{2}OPT,\\
&P_I\geq \frac{\delta/3G_\delta}{\epsilon}(D_I-D_0)\geq \frac{\delta/3G_\delta}{2\epsilon}OPT,
\end{align*}
and the proof is complete.
\end{proof}
Next, following similar arguments in~\cite{Buchbinder09,Bao18:ML_INFOCOM},
we introduce the relationship between the cost and resource consumption before and after processing one job.
Let $p_h^{r,i}[t]$ be the unit cost of type-$r$ resource on server $h$ at time $t$ after handling job $i$.
Let $\rho_h^{r,i}[t]$ be the amount of type-$r$ resource allocated to jobs on server $h$ at time $t$ after processing the job $i$.
For ease of our subsequent analysis, we now define the following allocation-cost relation that is implied by Algorithm~1:
\begin{defn} \label{defn:alloc_cost}
The allocation-cost relationship for Algorithm~1 with $\epsilon>1, G_\delta> 0$ and $\delta\in(0,1]$ is
\begin{align*}
&p_h^{r,i-1}[t](\rho_h^{r,i}[t]-\rho_h^{r,i-1}[t])\geq \frac{\delta/3G_\delta C_h^r}{\epsilon}(p_h^{r,i}[t]-p_h^{r,i-1}[t]),\\
&\hspace{2in}\forall i\in\mathcal{I},h\in\mathcal{H},r\in\mathcal{R}.
\end{align*}
\end{defn}
The allocation-cost relationship shows that the cost in each time slot for scheduling a new job is bounded by the increase of term $C_h^rp_h^r[t]$ in Problem~D-R-DMLRS, and the possible increment introduced by randomized rounding in Algorithm~4.
This is ensured by the update of the price function and the rounding scheme, respectively.
\begin{lem}
\label{lem:relation}If the allocation-cost relationship holds for $\epsilon\geq 1, G_\delta> 0$ and $\delta\in(0,1]$, then Algorithm~1 ensures $P_i-P_{i-1}\geq \frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}),\forall i\in\mathcal{I}$.
\end{lem}
\begin{proof}[Proof of Lemma~\ref{lem:relation}]
For any job $i\in\mathcal{I}$, if job $i$ is rejected, then we have $P_i-P_{i-1}=D_i-D_{i-1}=0$ according to Problems~R-DMLRS and D-R-DMLRS, the inequality must hold.
If job $i$ is accepted with schedule $\pi_i$, i.e., $x_{\pi_i}=1$, then the increment value of the primal objective value $P_i$ is
\begin{align*}
P_i-P_{i-1}=u_i(t_{\pi_i}-a_i).
\end{align*}
Since $x_{\pi_i}=1$, according to Algorithm~1, the constraint~(\ref{eqn_dualconstr}) is binding.
Then, we can have
\begin{align*}
&u_i(t_{\pi_i}-a_i)\\
&=\lambda_i+\sum_{t\in\mathcal{T}(\pi_i)}\sum_{h\in\mathcal{H}(\pi_i[t])}\sum_{r\in\mathcal{R}}(\alpha_i^rw_{ht}^{\pi_i}+\beta_i^rs_{ht}^{\pi_i})p_h^r[t]\\
&=\lambda_i+\sum_{t\in\mathcal{T}(\pi_i)}\sum_{h\in\mathcal{H}(\pi_i[t])}\sum_{r\in\mathcal{R}}p_h^r[t](\rho_h^{r,i}[t]-\rho_h^{r,i-1}[t]).
\end{align*}
Similarly, we can have the increment value of the dual objective value $D_i$ as follows:
\begin{align*}
D_i-D_{i-1}\!=\!\lambda_i+\sum_{t\in\mathcal{T}(\pi_i)}\sum_{h\in\mathcal{H}(\pi_i[t])}\sum_{r\in\mathcal{R}}(p_h^{r,i}[t]-p_h^{r,i-1}[t])C_h^r.
\end{align*}
Summing up the allocation-cost relationship over all $t\in\mathcal{T}(\pi_i),h\in\mathcal{H}(\pi_i[t]),r\in\mathcal{R}$, we have
\begin{align*}
P_i-P_{i-1}&\geq\frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}-\lambda_i)+\lambda_i\\
&=\frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1})+(1-\frac{\delta/3G_\delta}{\epsilon})\lambda_i.
\end{align*}
As $\lambda_i\geq0$, $\epsilon, G_\delta> 0$ and $\delta\in(0,1]$, we have
\begin{align*}
P_i-P_{i-1}\geq\frac{\delta/3G_\delta}{\epsilon}(D_i-D_{i-1}).
\end{align*}
This completes the proof of Lemma~\ref{lem:relation}.
\end{proof}
For specific $h\in\mathcal{H},r\in\mathcal{R}$, we define $\epsilon_h^r$ as the corresponding parameter in the allocation-cost relationship for any job $i\in\mathcal{I}$ and any time slot $t\in\mathcal{T}$.
Then, it holds that $\epsilon=\max_{h,r}\{\epsilon_h^r\}$.
Without loss of generality, we assume that the resource demand of each worker or parameter server is much smaller compared to the capacity of that resource on one server, i.e., $\alpha_i^r\ll C_h^r, \beta_i^r\ll C_h^r$.
This is common in real-world machine learning system as large percentage of resources in the whole server.
As $\rho_h^r[t]$ increases from 0 to $C_h^r$, then we can claim that $d\rho_h^r[t]=\rho_h^{r,i}-\rho_h^{r,i-1}$, and derive a differential version of the allocation-cost relationship, which is defined as follows:
\begin{defn} \label{defn:diff_alloc_cost}
The differential allocation-cost relationship for Algorithm~1 with $\epsilon_h^r\geq 1$ is
\begin{align*}
p_h^r[t]d\rho_h^r[t]\geq \frac{C_h^r}{\epsilon_h^r}dp_h^r[t],\forall t\in\mathcal{T},h\in\mathcal{H},r\in\mathcal{R}.
\end{align*}
\end{defn}
Next we show that a feasible $\epsilon_h^r$ satisfies the differential allocation-cost relationship with price function $p_h^r[t]$ defined in (\ref{eqn_cost_update_fn}).
\begin{lem}\label{lem:diff_alloc_cost}
$\epsilon_h^r=\ln\frac{U^r}{L}$, and the price function defined in (\ref{eqn_cost_update_fn}) satisfy the differential allocation-cost relationship.
\end{lem}
\begin{proof}[Proof of Lemma~\ref{lem:diff_alloc_cost}]
The derivation of the marginal cost function is
\begin{align*}
dp_h^r[t]=p_h^{r'}(\rho_h^r[t])d\rho_h^r[t]=L(\frac{U^r}{L})^{\frac{\rho_h^r[t]}{C_h^r}}\ln(\frac{U^r}{L})^{\frac{1}{C_h^r}}d\rho_h^r[t].
\end{align*}
The differential allocation-cost relationship is
\begin{align*}
L(\frac{U^r}{L})^{\frac{\rho_h^r[t]}{C_h^r}}d\rho_h^r[t]\geq \frac{C_h^r}{\epsilon_h^r}L(\frac{U^r}{L})^{\frac{\rho_h^r[t]}{C_h^r}}\ln(\frac{U^r}{L})^{\frac{1}{C_h^r}}d\rho_h^r[t],
\end{align*}
which holds for $\epsilon_h^r\geq \ln(\frac{U^r}{L})$.
Then, we can set $\epsilon\!=\!\max\limits_{r\in\mathcal{R}}(1,\ln(\frac{U^r}{L}))$, which satisfies the differential allocation-cost relationship.
This completes the proof.
\end{proof}
With the aforementioned lemmas, we are now in a position to prove Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}.
Note that the Theorems provide probabilistic guarantees.
We first analyze the performance ratio, which is followed by the probabilistic feasibility discussion for both cases.
\begin{proof}[Proof of Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}]
According to Lemma~\ref{lem:diff_alloc_cost}, the marginal cost function used in Algorithm~1 satisfies the differential allocation-cost relationship with $\epsilon=\max_r(1,\ln\frac{U^r}{L})$.
Since the resource demand in a job $i$ is much smaller than the capacity, we can derive
\begin{align*}
&d\rho_h^r[t]=\rho_h^{r,i}-\rho_h^{i-1}[t], \\
&dp_h^r[t]=p_h^{r'}(\rho_h^r[t])(\rho_h^{r,i}[t]-\rho_h^{r,i-1}[t])=p_h^{r,i}[t]-p_h^{r,i-1}[t].
\end{align*}
So, the differential allocation-cost relationship in Definition~\ref{defn:diff_alloc_cost} implies the allocation-cost relationship in Definition~\ref{defn:alloc_cost} holds for $\epsilon=\max_r(1,\ln\frac{U^r}{L})$.
According to Algorithm~1, we note that
\begin{align*}
\frac{1}{\mu}\leq \frac{\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})}{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}},\forall i\in\mathcal{I}.
\end{align*}
Then, the minimum amount of overall resource consumption of job $i$ can be computed as:
\begin{multline*}
\frac{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}}{\mu}\\
\leq \ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r}).
\end{multline*}
Then, it follows that:
\begin{align*}
&D_0=\sum_{t,h,r}LC_h^r\\
&=\sum_{t,h,r}\min_{i\in\mathcal{I},\pi_i\in\Pi_i}\frac{1/(2\mu) u_{i}(t_{\pi_i}-a_{i})C_h^r}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}\\
&=\frac{T\sum_{h,r}C_h^r}{2\mu}\min_{i,\pi_i}\frac{u_{i}(t_{\pi_i}-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}\\
&\leq \frac{1}{2}\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})\\
&\min_{i\in\mathcal{I},\pi_i\in\Pi_i}\frac{u_{i}(t_{\pi_i}-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}, \forall i\in\mathcal{I}.\\
&\overset{(a)}{\leq}\frac{1}{2}\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})\\
&\frac{u_{i}(t_{\pi_i}-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}\\
&\leq \frac{1}{2}u_i(t_{\pi_i}-a_i)\overset{(b)}{\leq} \frac{1}{2}OPT,
\end{align*}
where (a) follows by selecting $(i,\pi)=\arg\min_{i\in\mathcal{I},\pi_i\in\Pi_i}u_i(t_{\pi_i}-a_i)$, and (b) follows from the assumption that the offline optimal solution accepts at least one job, which is reasonable in real-world machine learning system.
Then we have $OPT\geq \min_{i,\pi}u_i(t_{\pi_i}-a_i)$.
According to Lemmas~\ref{lem:result} and \ref{lem:relation}, the competitive ratio is proved.
Recall that the randomized rounding algorithm is a key component in our algorithm.
Toward this end, we show the probability of obtaining a feasible solution with the proved competitive ratio.
Here, we consider the following two cases:
{\em 1) } When $0<G_\delta\leq 1$ (Theorem~\ref{thm_Competitive_1}): According to Theorem~\ref{thm_Alg4_1}, the probability of violating the cover constraint is no greater than $\delta/3$ at each randomized rounding iteration.
Recall that our Algorithm~1 runs a predetermined number of $S$ iterations to find a feasible integer solution.
Thus the probability that no feasible integer solution returned after $S$ iterations rounding is at most $(\delta/3)^S$.
It then follows that the probability of at least one feasible integer solution found is at least $1-(\delta/3)^S$.
Moreover, the number of states $(t,v)$ in the dynamic programming for each job $i$ is $O(TK_iE_i)$.
Therefore, with probability greater than $(1-(\delta/3)^S)^{T_iK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible integer solution with the proved competitive ratio.
{\em 2) } When $G_\delta>1$ (Theorem~\ref{thm_Competitive_2}): According to Theorem~\ref{thm_Alg4_2}, the probability of violating the packing constraint is no greater that $\delta/3(HR+1)$ at each randomized rounding iteration.
Following the similar arguments in {\em 1)}, we can show that with probability greater than $(1-(\delta/3(HR+1))^S)^{T_iK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible integer solution with the proved competitive ratio, and the proof is complete.
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
%\label{bio}
%\vspace{-.5in}
\begin{IEEEbiography}[{\includegraphics[width=1in, height=1.25in]{Figures/Menglu_Bio1.jpg}}]
{Menglu Yu} received her B.S. degree from the Department of Electrical and Information Engineering at Hunan University, China in 2014.
She is currently pursuing her Ph.D. degree in the Department of Computer Science at Iowa State University.
Her primary research interests include optimization for distributed machine learning systems and data centers, as well as network optimization.
\end{IEEEbiography}
%\vspace{-.5in}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/Liu_Bio.eps}}]
{Jia Liu} (S'03--M'10--SM'16) is an Assistant Professor in the Department of Electrical and Computer Engineering at The Ohio State University, where he joined in Aug. 2020.
He received his Ph.D. degree from the Department of Electrical and Computer Engineering at Virginia Tech in 2010.
From Aug. 2017 to Aug. 2020, he was an Assistant Professor in the Department of Computer Science at Iowa State University.
His research areas include theoretical machine learning, control and optimization for stochastic networks, and optimization for data analytics infrastructure and cyber-physical systems.
Dr. Liu is a senior member of IEEE and a member of ACM. He has received numerous awards at top venues, including IEEE INFOCOM'19 Best Paper Award, IEEE INFOCOM'16 Best Paper Award, IEEE INFOCOM'13 Best Paper Runner-up Award, IEEE INFOCOM'11 Best Paper Runner-up Award, and IEEE ICC'08 Best Paper Award.
Dr. Liu is a recipient of the NSF CAREER Award in 2020. He is a recipient of the Google Faculty Research Award in 2020.
He is also a winner of the LAS Award for Early Achievement in Research from the College of Liberal Arts and Sciences at Iowa State University in 2020, and the Bell Labs President Gold Award in 2001.
His research is supported by NSF, AFOSR, AFRL, and ONR.
\end{IEEEbiography}
%\vspace{-.5in}
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/wu.png}}]
{Chuan Wu} received her B.Engr. and M.Engr. degrees in 2000 and 2002 from the Department of Computer Science and Technology, Tsinghua University, China, and her Ph.D. degree in 2008 from the Department of Electrical and Computer Engineering, University of Toronto, Canada. Since September 2008, Chuan Wu has been with the Department of Computer Science at the University of Hong Kong, where she is currently a Professor. Her current research is in the areas of cloud computing, distributed machine learning systems and algorithms, and intelligent elderly care technologies. She is a senior member of IEEE, a member of ACM, and served as the Chair of the Interest Group on Multimedia services and applications over Emerging Networks (MEN) of the IEEE Multimedia Communication Technical Committee (MMTC) from 2012 to 2014. She is an associate editor of IEEE Transactions on Cloud Computing, IEEE Transactions on Multimedia, ACM Transactions on Modeling and Performance Evaluation of Computing Systems, and IEEE Transactions on Circuits and Systems for Video Technology. She was the co-recipient of the best paper awards of HotPOST 2012 and ACM e-Energy 2016.
\end{IEEEbiography}
%
%\vfill
\begin{IEEEbiography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/BoJi_Bio.jpg}}]{Bo Ji}(S'11-M'12-SM'18)
received his B.E. and M.E. degrees in Information Science and Electronic Engineering from Zhejiang University, Hangzhou, China, in 2004 and 2006, respectively, and his Ph.D. degree in Electrical and Computer Engineering from The Ohio State University, Columbus, OH, USA, in 2012. Dr. Ji is an Associate Professor in the Department of Computer Science at Virginia Tech, Blacksburg, VA, USA. Prior to joining Virginia Tech, he was an Associate Professor in the Department of Computer and Information Sciences and a faculty member of the Center for Networked Computing at Temple University, where he was an Assistant Professor from July 2014 to June 2020. He was also a Senior Member of the Technical Staff with AT\&T Labs, San Ramon, CA, from January 2013 to June 2014. His research interests are in the modeling, analysis, control, optimization, and learning of computer and network systems, such as wired and wireless networks, large-scale IoT systems, high performance computing systems and data centers, and cyber-physical systems. He currently serves on the editorial boards of the IEEE/ACM Transactions on Networking, IEEE Internet of Things Journal, and IEEE Open Journal of the Communications Society. Dr. Ji is a senior member of the IEEE and a member of the ACM. He is a National Science Foundation (NSF) CAREER awardee (2017) and an NSF CISE Research Initiation Initiative (CRII) awardee (2017). He is also a recipient of the IEEE INFOCOM 2019 Best Paper Award.
\end{IEEEbiography}
%
%
%\vspace{-5in}
\begin{IEEEbiography}
[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/Bentley_Bio.jpg}}]
{Elizabeth S. Bentley} has a B.S. degree in Electrical Engineering from Cornell University, a M.S. degree in Electrical Engineering from Lehigh University, and a Ph.D. degree in Electrical Engineering from University at Buffalo.
She was a National Research Council Post-Doctoral Research Associate at the Air Force Research Laboratory in Rome, NY.
Currently, she is employed by the Air Force Research Laboratory in Rome, NY, performing in-house research and development in the Networking Technology branch.
Her research interests are in cross-layer optimization, wireless multiple-access communications, wireless video transmission, modeling and simulation, and directional antennas/directional networking.
\end{IEEEbiography}
%\documentclass[9.5pt,journal]{IEEEtran}
%\documentclass[conference,compsoc]{IEEEtran}
\documentclass[10pt, journal,compsoc]{IEEEtran}
% *** CITATION PACKAGES ***
%
\ifCLASSOPTIONcompsoc
% IEEE Computer Society needs nocompress option
% requires cite.sty v4.0 or later (November 2003)
\usepackage[nocompress]{cite}
\else
% normal IEEE
\usepackage{cite}
\fi
% *** GRAPHICS RELATED PACKAGES ***
%
\ifCLASSINFOpdf
\else
\fi
\pdfoutput=1
\usepackage{ifpdf}
\usepackage{cite}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{mathrsfs}
\usepackage[dvips,dvipdf]{graphicx}
\usepackage{subfigure}
\usepackage{algorithm}
\usepackage{algorithmicx}
\usepackage{multirow}
\usepackage{bbm}
\usepackage{arydshln}
%\usepackage{flushend}
\usepackage{url}
\usepackage{pict2e,picture} % picture
\usepackage{tikz} % tikz
\usepackage{pifont} % ding
\usepackage{enumitem}
\usepackage{mathtools}
\usepackage{algpseudocode}
%\usepackage{tabularx}
\usepackage{comment}
\usepackage{xcolor}
%\usepackage{tabu}
%\usepackage{subfig}
\usepackage{caption}
\DeclarePairedDelimiter{\ceil}{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}
\bibliographystyle{IEEEtran}
%\include{Complete.bib}
\input{Supporting_Preambles/symbols_commands}
%set up for the captions
%\captionsetup[figure]{position=bottom,justification=centering,width=.85\textwidth,labelfont=bf,font=small}
%\captionsetup[figure]{position=bottom,calcwidth=.85\linewidth,font=scriptsize}
% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}
\begin{document}
\title{Toward Efficient Online Scheduling for Distributed Machine Learning Systems}
\author{Menglu~Yu,
Jia~Liu~\IEEEmembership{Senior Memeber,~IEEE},
Chuan~Wu~\IEEEmembership{Senior Memeber,~IEEE},
Bo~Ji~~\IEEEmembership{Senior Memeber,~IEEE}
Elizabeth~S.~Bentley~\IEEEmembership{Member,~IEEE}% <-this % stops a space
%\thanks{
%This work has been supported by NSF grants CNS-1527078, 1514260, 1446582, 1409336, 1012700, WiFiUS-1456806, ECCS-1444026, 1232118; ONR grant N00014-15-1-2166; ARO grant W911NF-14-1-0368; DTRA grants HDTRA 1-14-1-0058, 1-15-1-0003, AFRL VFRP'15 award; DARPA grant HROOll-15-C-0097and QNRF grant NPRP 7-923-2-344.
%DISTRIBUTION STATEMENT A: Approved for Public Release; distribution unlimited 88ABW-2015-5989 on Dec. 14, 2015.}
\thanks{
Menglu Yu is with the Department of Computer Science, Iowa State University, Ames, IA 50011, USA (e-mail: mengluy@iastate.edu).}
\thanks{
Jia Liu is with the Department of Electrical and Computer Engineering, The Ohio State University, Columbus, OH 43210, USA (e-mail: liu@ece.osu.edu).}
\thanks{
Bo Ji is with the Department of Computer Science, Virginia Tech, Blacksburg, VA 24061, USA (e-mail: boji@vt.edu).}
\thanks{
Chuan Wu is with the Department of Computer Science, The University of Hong Kong, Hong Kong (e-mail: cwu@cs.hku.hk).}
\thanks{
Elizabeth S. Bentley is with the Air Force Research Laboratory, Information Directorate, Rome, NY 13441, USA (e-mail: elizabeth.bentley.3@us.af.mil).}
\thanks{
Digital Object Identifier xx.xxxx/XXXX.xxxx.xxxxx.}%
}
%\author{Menglu Yu$^{\dag}$ \mbox{\hspace{0.4cm}} Jia Liu$^{\mathsection}$ \mbox{\hspace{0.4cm}} Chuan Wu$^{\mathparagraph}$ \mbox{\hspace{0.4cm}} Bo Ji$^{\ddag}$ \mbox{\hspace{0.4cm}} Elizabeth Bentley$^{*}$
%\IEEEcompsocitemizethanks{\IEEEcompsocthanksitem $^{\dag}$Department of Computer Science, Iowa State University
%\IEEEcompsocthanksitem $^{\mathsection}$Department of Electrical and Computer Engineering, The Ohio State University
%\IEEEcompsocthanksitem $^{\mathparagraph}$Department of Computer Science, University of Hong Kong
%\IEEEcompsocthanksitem $^{\ddag}$Department of Computer Science, Virginia Tech
%\IEEEcompsocthanksitem $^{*}$Air Force Research Laboratory, Information Directorate
%}}
%\author{Paper ID: 1570578306}
\IEEEtitleabstractindextext{%
%\begin{abstract}
\input{Abstract/Abstract}
%\end{abstract}
% Note that keywords are not normally used for peerreview papers.
\begin{IEEEkeywords}
Online resource scheduling, distributed machine learning, approximation algorithm%up to 5
\end{IEEEkeywords}}
\maketitle
\IEEEdisplaynontitleabstractindextext
\IEEEpeerreviewmaketitle
%\IEEEraisesectionheading{\section{Introduction}\label{sec:introduction}}
%\input{Abstract/Abstract}
\input{Sec1_Intro/Sec1_Intro}
\input{Sec2_Related/Sec2_Related}
\input{Sec3_Model/Sec3_Model}
\input{Sec4_Algorithm/Sec4_Algorithm}
\input{Sec5_Numerical/Sec5_Numerical}
\input{Sec6_Conclusion/Sec6_Conclusion}
%\appendices
%\input{Appdx_A/Appdx_A}
%\input{Appdx_B/Appdx_B}
%\input{Appdx_C/Appdx_C}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\bibliography{IEEEabrv,./BIB/Complete,./BIB/HeavyBall,./BIB/Kevin_All,./BIB/CSMA}
%\clearpage
\bibliography{IEEEabrv,./BIB/JobScheduling,./BIB/DMLF,./BIB/Kevin_All,./BIB/Complete,./BIB/CSMA}
\input{Bio/Bio}
%\begin{IEEEbiography}
%[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{Figures/Liu_Bio.eps}}]
%{Jia Liu} (S'03--M'10--SM'16) is an Assistant Professor in the Dept. of Electrical and Computer Engineering at The Ohio State University, where he joined in Aug. 2020.
%He received his Ph.D. degree from the Dept. of Electrical and Computer Engineering at Virginia Tech in 2010.
%From Aug. 2017 to Aug. 2020, he was an Assistant Professor in the Dept. of Computer Science at Iowa State University.
%
%Prior to that, he was a Postdoctoral Researcher and subsequently a Research Assistant Professor from Feb. 2010 to Jul. 2017, both in the ECE department at OSU.
%
%His research areas include theoretical machine learning, control and optimization for stochastic networks, and optimization for data analytics infrastructure and cyber-physical systems.
%
%Dr. Liu is a senior member of IEEE and a member of ACM. He has received numerous awards at top venues, including IEEE INFOCOM'19 Best Paper Award, IEEE INFOCOM'16 Best Paper Award, IEEE INFOCOM'13 Best Paper Runner-up Award, IEEE INFOCOM'11 Best Paper Runner-up Award, and IEEE ICC'08 Best Paper Award.
%
%Dr. Liu is a recipient of the NSF CAREER Award in 2020.
%
%He is a recipient of the Google Faculty Research Award in 2020 and the Bell Labs President Gold Award in 2001. His research is supported by NSF, AFOSR, AFRL, and ONR.
%\end{IEEEbiography}
\clearpage
\appendices
\input{Appdx_A/Appdx_A}
\input{Appdx_B/Appdx_B}
\input{Appdx_C/Appdx_C}
\end{document}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Introduction} \label{sec:intro}
Fueled by the rapid growth of data analytics and machine learning (ML) applications, recent years have witnessed an ever-increasing hunger for computing power.
However, with hardware capability no longer advancing at the pace of the Moore's law, it has been widely recognized that a viable solution to sustain such computing power needs is to exploit {\em parallelism} at both machine and chip scales.
Indeed, the recent success of deep neural networks (DNN)
%(a revival of artificial neural networks with a much larger number of layers in depth)
is enabled by the use of distributed ML frameworks, which exploit the massive parallelism over computing clusters with a large number of GPUs.
These distributed ML frameworks have significantly accelerated the training of DNN for many applications (e.g., image and voice recognition, natural language processing, etc.).
To date, prevailing distributed ML frameworks include TensorFlow~\cite{TensorFlow}, MXNet~\cite{MXNet}, PyTorch~\cite{Paszke19:Pytorch}, Caffe~\cite{Caffe}, to name just a few.
However, the proliferation of distributed ML frameworks also introduces many unique technical challenges on large-scale computing system design and network resource optimization.
%A {\em common} feature underlying distributed ML frameworks is
Particularly, due to the decentralized nature, at the heart of distributed learning system optimization lies the problem of scheduling ML jobs and resource provisioning across different machines to minimize the total training time.
Such scheduling problems involve dynamic and combinatorial %virtual-machine-based
worker and parameter server allocations, which are inherently NP-hard.
Also, the allocations of workers and parameter servers should take {\em locality} into careful consideration, since co-located workers and parameter servers can avoid costly network communication overhead.
However, locality optimization adds yet another layer of difficulty in scheduling algorithm design.
Exacerbating the problem is the fact that the future arrival times of training jobs at an ML computing cluster are hard to predict,
which necessitates {\em online} algorithm design without the knowledge of future job arrivals.
So far in the literature, there remains a lack of holistic theoretical studies that address all the aforementioned challenges.
Most of the existing scheduling schemes are based on simple heuristics without performance guarantee (see Section~\ref{sec:Related} for detailed discussions).
This motivates us to fill this gap and pursue efficient online scheduling designs for distributed ML resource optimization, which offer {\em provable} performance guarantee.
The main contribution of this paper is that we develop an online scheduling algorithmic framework that {\em jointly} yields resource scheduling and locality optimization decisions with strong competitive ratio performance.
Further, we reveal interesting insights on how distributed ML frameworks affect online resource scheduling optimization.
Our main technical results are summarized as follows:
\vspace{-.01in}
\begin{list}{\labelitemi}{\leftmargin=1em \itemindent=-0.0em \itemsep=.2em}
\item By abstracting the architectures of prevailing distributed ML frameworks, we formulate an online resource scheduling optimization problem that: i) models the training of ML jobs based on the parameter server (PS) architecture and stochastic gradient descent (SGD) method; and ii) explicitly takes {\em locality} optimization into consideration.
We show that, due to the heterogeneous internal (between virtual machines or containers) and external (between physical machines) communications, the locality-aware scheduling problem contains {\em non-deterministic} constraints and is far more complex compared to the existing works that are locality-oblivious (see, e.g., ~\cite{Chun16:Dolphin,Bao18:ML_INFOCOM}).
\item To solve the locality-aware scheduling problem, we
develop an equivalent problem reformulation to enable subsequent developments of online approximation algorithms.
Specifically, upon carefully examining the locality configurations of worker-server relationships, we are able to transform the original problem to a special-structured integer nonlinear program with mixed cover/packing-type constraints, and
% which further entails
the low-complexity approximation algorithm design with provable performance can be further entailed.
\item To tackle the integer nonlinear problem with mixed cover/packing-type constraints, we propose an approximation algorithm based on a meticulously designed randomized rounding scheme and then rigorously prove its performance.
We note that the results of our randomized rounding scheme are general and could be of independent theoretical interest.
Finally, by putting all algorithmic designs together, we construct a primal-dual online resource scheduling (PD-ORS) scheme, which has an overall competitive ratio that only {\em logarithmically} depends on ML job characteristics (e.g., required epochs, training samples).
\end{list}
Collectively, our results contribute to a comprehensive and fundamental understanding of distributed machine learning system optimization.
The remainder of this paper is organized as follows.
In Section~\ref{sec:Related}, we review the literature to put our work in comparative perspectives.
Section~\ref{sec:model_formulation} introduces the system model and problem formulation.
Section~\ref{sec:alg} presents our algorithms and their performance analysis.
Section~\ref{sec:numerical} presents numerical results and Section~\ref{sec:conclusion} concludes this paper.
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Related Work} \label{sec:Related}
As mentioned in Section~\ref{sec:intro}, due to the high computational workload of ML applications, many distributed ML frameworks (e.g., TensorFlow~\cite{TensorFlow}, MXNet~\cite{MXNet}, PyTorch~\cite{Paszke19:Pytorch}, Caffe~\cite{Caffe}) have been proposed to leverage modern large-scale computing clusters.
% to leverage modern computing platforms and offer massive parallelism.
% e.g., TensorFlow~\cite{TensorFlow}, MXNet~\cite{MXNet}, Cognitive ToolKit (CNTK)~\cite{CNTK}, Caffe~\cite{Caffe}, etc.
A common distributed training architecture implemented in these distributed ML frameworks is the PS architecture~\cite{Li14:ML_Static_OSDI,Chilim14:ML_Static_OSDI}, which employs multiple workers and PSs (implemented as virtual machines or containers) to collectively train a global ML model.
Coupled with the {\em iterative} ML training based on stochastic gradient descent (SGD), the interactions between machines in the distributed ML cluster are significantly different from those in traditional cloud computing platforms (e.g., MapReduce~\cite{Dean08:MapReduce} and Dryad~\cite{Isard07:Dryad} and references therein).
For example, a MapReduce job usually partitions the input data into independent chunks, which are then processed by the {\it map} step in a parallel fashion.
The output of the maps are then fed to the {\it reduce} step to be aggregated to yield the final result.
Clearly, the data flows in MapReduce are a ``one-way traffic", which is unlike those iterative data flows in ML training jobs whose completions highly depend on the ML job's convergence property.
%However, machine learning algorithms typically require a large number of iterations to optimize its parameters using optimization method such as SGD.}
%Although each iteration can be expressed as a MapReduce job, each job must reload data from the disk, which could lead to a significant performance degradation.
As a result, existing job scheduling algorithms for cloud systems are not suitable for distributed ML frameworks.
%~\cite{Curino14:Rayon_SoCC,Vavilapalli13:YARN_SoCC,Boutin14:Apollo_OSDI,Huang15:CORA_INFOCOM,Chen17:Maxmin_INFOCOM,Azar15:CloudSch_EC,Lucier13:CloudSch_SPAA,Zhou17:CloudSch_TON,Zhang17:CloudSch_TON,Xiao14:CloudSch_TC,Tumanov16:CloudSch_EuroSys}
Among distributed ML system studies, most of the early attempts (see, e.g., ~\cite{Li14:ML_Static_OSDI,Chilim14:ML_Static_OSDI} and references therein) only considered static allocation of workers and parameter servers.
To our knowledge, the first work on understanding the performance of distributed ML frameworks is~\cite{Yan15:MLFramework_KDD}, where Yan {\em et al.} developed analytical models to quantify the impacts of model-data partitioning and system provisioning for DNN.
Subsequently, Chun {\em et al.} ~\cite{Chun16:Dolphin} developed heuristic dynamic system reconfiguration algorithms to allocate workers and parameter servers to minimize the runtime, but without providing optimality guarantee.
The first dynamic distributed scheduling algorithm with optimality guarantee was reported in ~\cite{Sun17:ML_MILP}, where Sun {\em et al.} used standard mixed integer linear program (MILP) solver to dynamically compute the worker-parameter server partition solutions.
Due to NP-hardness of the MILP, the scalability of this approach is limited. %cannot be scaled up to handle large-size distributed ML systems in practice.
The most recent work~\cite{Zhang20:onlinescheduling} proposed an online scheduling algorithm to schedule synchronous training jobs in ML clusters with the goal to minimize the weighted completion time.
However, the consecutive time slots were allocated for each training job, and the numbers of workers and parameter servers could not be adjusted.
Another line of the research is to leverage the learning-based approach to do the resource scheduling.
There are a number of recent works using deep reinforcement learning (DRL) for resource allocation, device placement, and video streaming.
For example, Mao {\it et al.}~\cite{Mao16:DRL} and Chen {\it et al.}~\cite{Chen17:DRL} designed a multi-resource cluster scheduler using DRL with the goal to minimize average job slowdown.
The proposed scheduler picks one or more of the waiting jobs in the queue and allocate to machines at each time slot, and the resource demand of each job is unknown until after its arrival.
Later, Mao {\it et al.}~\cite{Mao18:graph-based,Mao19:learningscheduling} used DRL to heuristically train scheduling policies for graph-based parallel jobs by setting both parallelism level and execution order.
Meanwhile, Mirhoseini {\it et al.}~\cite{Mirhoseini18:hierarchicalmodel, Mirhoseini17:deviceplacement} utilized DRL to design a model for efficient placement of computational graphs onto hardware devices, aiming at minimize the running time of each individual TensorFlow job.
Although various performance gains have been empirically reported, these DRL-based studies do not offer optimality performance guarantee due to the lack of theoretical foundation of DRL as of today.
\begin{figure*}[t!]
\centering
\begin{minipage}[t]{0.3\textwidth}
\centering
\vspace{-1.26in}
\includegraphics[height=0.14\textheight]{Figures/fig_Distr_ML1.eps}
\vspace{-.03in}
\caption{Illustration of distributed training with the PS architecture.}
\label{fig:DistrML}
\end{minipage}
%
\hspace{0.01\columnwidth}%
%
\begin{minipage}[t]{0.3\textwidth}
\centering
\vspace{-1.3in}
\includegraphics[width=0.23\textheight]{Figures/fig_ML_Workflow1.eps}
\vspace{-.15in}
\caption{The workflow of iterative training.}
\label{fig:MLWorkflow}
\end{minipage}
%
\hspace{0.01\columnwidth}%
%
\begin{minipage}[t]{0.3\textwidth}
\centering
\includegraphics[width=0.16\textheight]{Figures/fig_Colocate1.eps}
\vspace{.06in}
\caption{Colocated parameter servers and workers on physical machines.}
\label{fig:Colocate}
\end{minipage}
\vspace{-.1in}
\end{figure*}
%\smallskip
The most relevant work to ours is ~\cite{Bao18:ML_INFOCOM}, where Bao {\em et al.} developed an online primal-dual approximation algorithm, OASiS, to solve the scheduling problem for distributed ML systems.
% by employing a similar primal-dual algorithmic framework.
Our work differs from~\cite{Bao18:ML_INFOCOM} in the following key aspects: 1) In \cite{Bao18:ML_INFOCOM}, the workers and parameter servers are allocated on two {\em strictly separated} sets of physical machines, i.e., {\em no} worker and parameter server can share the same physical machine, which significantly simplified the underlying optimization problem. In this work, we consider the cases that workers and parameter servers can be {\em co-located} on the same physical machine, which is the common practice in existing ML systems (see, e.g., ~\cite{MXNet1,MXNet2}). Such co-location can significantly reduce inter-server communication, expedite training, and improve resource utilization efficiency between workers and parameter servers.
However, as will be shown later, the co-location setting leads to an integer {\em non-convex} optimization problem with {\em non-deterministic} constraints, which is much harder and necessitates new algorithm design.
%To this end, we propose a low-complexity approximation algorithm to solve the problem and rigorously prove its performance.
%More importantly, we consider the setting with co-located workers and parameter servers because
2) Ref.~\cite{Bao18:ML_INFOCOM} advocates dynamic worker number adjustment, but does {\em not} guarantee the same global batch size across the training iterations.
According to recent literature~\cite{Goyal17:SGD}, maintaining a consistent global batch size is important for ensuring convergence of DNN training, when the worker number varies. We ensure a consistent global batch size in our model.
We note that the co-location setting was considered in~\cite{Peng18:ML_EuroSys}. However, the scheduling algorithm therein is a heuristic and does not provide performance guarantee. This motivates us to develop new algorithms with provable performance to fill this gap.
% !TEX root = ../ML_Networking_TNSE20.tex
\section{System Model and Problem Formulation} \label{sec:model_formulation}
In this section, we first provide a quick overview on the architecture of distributed ML frameworks to familiarize readers with the necessary background.
Then, we will introduce our analytical models for ML jobs and resource constraints, as well as the overall problem formulation.
% \begin{figure}
% \centering
% % \vspace{-1.26in}
% \includegraphics[width=0.3\textheight]{Figures/fig_Distr_ML1.eps}
% % \vspace{-.03in}
% \caption{Illustration of distributed training with the PS architecture.}
% \label{fig:DistrML}
% \end{figure}
\smallskip
{\bf 1) Distributed Machine Learning: A Primer.} %\label{subsec:distr_ML}
As illustrated in Fig.~\ref{fig:DistrML},
the key components of a PS-based distributed ML system include parameter servers, workers, and the training dataset, which are usually implemented over a connected computing cluster that contains multiple physical machines.
The training dataset of an ML job is stored in distributed data storage (e.g., HDFS~\cite{HDFS}) and usually divided into equal-sized data chunks.
Each data chunk further contains multiple equal-sized mini-batches.
%
%\begin{figure}[t!]
%\centering
%\begin{minipage}[t]{0.24\textwidth}
%\centering
%\vspace{-1.1in}
%\includegraphics[height=0.12\textheight]{Figures/fig_Distr_ML1.eps}
%\vspace{-.08in}
%\caption{An illustration of a distributed machine learning framework.}
%\label{fig:DistrML}
%\end{minipage}
%%
%\hspace{0.001\columnwidth}%
%%
%\begin{minipage}[t]{0.24\textwidth}
%\centering
%\includegraphics[width=0.14\textheight]{Figures/fig_Colocate1.eps}
%\vspace{-.06in}
%\caption{Colocated parameter servers and workers on physical machines.}
%\label{fig:Colocated}
%\end{minipage}
%\vspace{-.2in}
%\end{figure}
%
To date, one of the most widely adopted training algorithms in distributed ML frameworks is the {\em stochastic gradient descent method} (SGD)~\cite{Dean:2012:LSD}. %thanks to its low-complexity~\cite{????}.
With SGD, the interactions between workers and parameter servers are illustrated in Fig~\ref{fig:MLWorkflow}.
A worker is loaded with the DNN model (we focus on data parallel training) with current values of the model parameters (e.g., the weights of a DNN) and retrieves a new data chunk from the data storage.
In each training iteration, a worker processes one mini-batch from its data chunk to compute gradients (i.e., directions and magnitudes of parameter changes).\footnote{
As an example, in a DNN model, gradients can be computed by the well-known ``back-propagation'' approach.
}
Upon finishing a mini-batch, the worker sends the gradients to the parameter servers, receives updated global parameters, and then continues with the next training iteration to work on the next mini-batch.
%After finishing the current data chunk, the worker will repeat the same process on a new data chunk.
On the parameter server side, parameters are updated as: $\w[k] = \w[k-1] + \alpha_{k} \g[k]$, where $\w[k]$, $\alpha_{k}$, and $\g[k]$ denote the parameter values, step-size, and stochastic gradient in the $k$-th update, respectively.
%\begin{figure}[t!]
%\centering
%\includegraphics[width=0.76\columnwidth]{Figures/fig_ML_Workflow.eps}
%\vspace{-.05in}
%\caption{Workflow of the asynchronous stochastic gradient descent method (Async-SGD) in a distributed machine learning framework.}
%\label{fig:MLWorkflow}
%\vspace{-.3in}
%\end{figure}
%\begin{figure}[t!]
%\centering
%\begin{minipage}[t]{0.33\textwidth}
%\centering
%%\vspace{-1.1in}
%\includegraphics[height=0.102\textheight]{Figures/fig_Colocate1.eps}
%\vspace{-.2in}
%\caption{The workflow of Async-SGD on a distributed ML system.}
%\label{fig:MLWorkflow}
%\end{minipage}
%%
%\hspace{0.001\columnwidth}%
%%
%\begin{minipage}[t]{0.33\textwidth}
%\centering
%\vspace{-.9in}
%\includegraphics[width=0.18\textheight]{Figures/fig_locality1.eps}
%\vspace{-.07in}
%\caption{$b_{i}(h,p)$'s value is locality-dependent (internal or external).}
%\label{fig_bhp}
%\end{minipage}
%\vspace{-.2in}
%\end{figure}
%We can further see from Fig.~\ref{fig:MLWorkflow} that the training progress at different workers is {\em not} synchronized: Each parameter server updates its parameters without coordinating with other parameter servers, hence the name Async-SGD.
%It has been shown that Async-SGD achieves the {\em same} $O(1/\sqrt{k})$ convergence rate as its synchronous counterpart~\cite{lian2015asynchronous,huo2017asynchronous}, while avoiding technical complexities such as maintaining a common clock, slow machine(s) bottlenecks, periodic spikes of information exchanges and congestions, etc.
%Thanks to these salient features, Async-SGD is particularly suitable for distributed ML systems.
%For further details of Async-SGD, we refer interested readers to the excellent overview by XXX~\cite{???}.
%Upon understanding the basics of Async-SGD, we are now in a position to model jobs in distributed ML systems and facilitate scheduling algorithm design.
\smallskip
{\bf 2) Modeling of Learning Jobs:}
We consider a time-slotted system.
%, where time-slots are indexed by 0,1,2,$\dots$.
The scheduling time-horizon is denoted as $\mathcal{T}$ with $|\mathcal{T}|=T$.
We use $\mathcal{I}$ to represent the set of training jobs and let $a_{i}$ denote the arrival time-slot of job $i \in \mathcal{I}$.
As shown in Fig.~\ref{fig:Colocate}, parameter servers and workers %are usually implemented on virtual machines and
could spread over multiple physical machines.
We let $\mathcal{H}$ represent the set of physical machines.
For each job $i$, we use $w_{ih}[t], s_{ih}[t] \geq 0$ to represent the allocated numbers of workers and parameter servers on machine $h \in \mathcal{H}$ in each time-slot $t \geq a_{i}$, respectively.
Further, we let $ \mathcal{P}_{i}[t] \!\triangleq\! \{h \!\in\! \mathcal{H}|s_{ih}[t] \!>\! 0\}$ and $ \mathcal{W}_{i}[t] \!\triangleq\! \{h \!\in\! \mathcal{H}| w_{ih}[t] \!>\! 0\}$ denote the sets of physical machines that contain parameter servers and workers for job $i$ in time-slot $t$, respectively.
We use a binary variable $x_{i}\in \{0,1\}$ to indicate whether job $i$ is admitted ($x_{i} =1$) or not ($x_{i}=0$).
%We use $\tau_{i}$ to denote the training time for each sample of job $i$, computed by the time for training one mini-batch divided by the number of samples per mini-batch.
We use $\tau_{i}$ to denote the training for each sample of job $i$.
%Admission Control is common in cloud management systems and jobs get rejected can be queued or resubmitted later[citation].
We let $b_{i}(h,p)$ denote the data rate of the link between a worker for job $i$ (on machine $h$) and a parameter server (on machine $p$).
Each worker or parameter server is exclusively assigned to some job $i$, and $b_i(h,p)$ is reserved and decided by the user upon job submission, which is common to ensure the data transfer performance~\cite{Bao18:ML_INFOCOM}.
Note that the value of $b_{i}(h,p)$ is {\em locality-dependent} where the slowest worker will become the bottleneck since we focus on Bulk-Synchronous-Parallel (BSP) scheme ~\cite{Cheatham05:BSP}.
Specifically, we have:
\begin{align*}
b_{i}(h,p) =
\begin{cases}
b_{i}^{(i)}, & \text{if $h=p$}, \\
b_{i}^{(e)}, & \text{otherwise},
\end{cases}
\end{align*}
where $b_{i}^{(i)}$ and $b_{i}^{(e)}$ denote the internal and external communication link rates, respectively.
For example, as shown in Fig.~\ref{fig:Colocate}, since Job 1's worker $\mathrm{W}_{4}$ and parameter server $\mathrm{PS}_{2}$ are both on the same machine, they communicate at the internal link rate $b_{1}^{(i)}$.
On the other hand, since Job 1's worker $\mathrm{W}_{3}$ and parameter server $\mathrm{PS}_{2}$ are on different physical machines, they communicate at the external link rate $b_{1}^{(e)}$.
In practice, it usually holds that $b_{i}^{(e)} \ll b_{i}^{(i)}$.
%Usually in practice, parameters of the same job are evenly divided among its parameter servers (e.g., if three parameter servers are allocated for a job, then each will possess one third of the parameters).
%However, to reduce the implementation complexity of workers, the parameters divisions are made transparent to the workers.
Next, we calculate the amount of time for a worker on machine $h$ to process a sample.
We use $F_{i}$ to denote the global batch size of job $i$, which is a fixed constant across all time-slots.\footnote{We note that this fixed global batch size requirement is {\em compliant} with the standard SGD implementation~\cite{Bazaraa_Sherali_Shetty_93:NLP} and important for ensuring convergence~\cite{Goyal17:SGD}.
In contrast, the global batch size in some existing works on dynamic ML resource allocation (e.g.,~\cite{Bao18:ML_INFOCOM}) could be time-varying, which necessitates time-varying dynamic learning rate adjustments to offset correspondingly and further complicates the SGD implementation.
}
%\footnote{To address the problem of time-varying global batch size, one approach is to adjust learning rate in each time-slot.}
We assume $F_i$ is equally divided among workers, i.e., the local batch size at each worker is: $F_i/\sum_{h'\in\mathcal{H}}w_{ih'}[t]$.\footnote{ Most distributed ML frameworks (e.g., Tensorflow~\cite{Keras}) set the same local batch size to each worker for the distributed training.}
%(hence, number of samples in a mini-batch of a worker may vary with time $t$.
We assume symmetric link speed in both directions between a worker and a PS.
Let $g_{i}$ denote the size of gradients/parameters of job $i$.
% and download do not distinguish the sending and receiving gradients/parameters time.
%Then, in time-slot $t$, the roundtrip transmission time of job $i$ between a parameter server on machine $p$ and a worker on machine $p$ can be computed as:
%\begin{align}
%\frac{g_{i}(s_{ip}[t]/\sum_{h'\in\mathcal{H}} s_{ih'}[t])}{b_i(h,p)}
%\end{align}
Then, from a worker's perspective, to push gradients to and pull updated parameters from the PSs for job $i$,
the combined uplink/downlink communication time can be computed as:
$(2{g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t])/(\min_{p\in \mathcal{P}_i[t]}b_i(h,p)})$,
where the numerator term $g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t]$ follows from the assumption of even parameter distribution among the PSs, and the denominator is due to the fact that push/pull time is decided by the slowest link among all connections from the worker to all PSs (i.e., $\min_{p\in P_i[t]}b_i(h,p)$).
Hence, the average computation and communication time to process a sample on machine $h\in\mathcal{W}_i[t]$ for job $i$ in time slot $t$ can be computed as:
\begin{align*}
\underbrace{\tau_{i}}_{\substack{\mathrm{Training \,\, time} \\ {\mathrm{per \,\, sample}}}} \!\!\! + \underbrace{ \bigg(\frac{2g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t]}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)} \bigg) \bigg/ \bigg(\frac{F_i}{\sum_{h'\in\mathcal{H}}w_{ih'}[t]} \bigg) }_{\mathrm{Communication \,\, time \,\, per \,\, sample}}.
\end{align*}
%amount of time to train a sample on machine $h$ and then communicate the result with its parameter server(s).
%\textcolor{red}{The communication time is determined by the external (i.e, $b_i(h,p)=b_i^{(e)}$, if it exists) physical machine whose number of parameter servers related to that worker is largest. We note that $\frac{s_{ih}[t]}{\sum_{h\in\mathcal{H}}s_{ih}[t]}=1$ under internal communication.}
Recall that we focus on the BSP scheme, where all workers are synchronized before they proceed to the next iteration.
In other words, the total number of samples trained on machine $h\in\mathcal{W}_i[t]$ for job $i$ in time slot $t$ is determined by the slowest link among all connections from all workers to all PS (i.e., $\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)$).
It then follows that the {\em number of samples trained} on machine $h\in\mathcal{W}_i[t]$ for job $i$ in time-slot $t$ can be computed as:
\begin{align} \label{eqn_NumTrainedSamples}
\frac{w_{ih}[t]}{\tau_{i} + \Big( \frac{2g_i/\sum_{h'\in\mathcal{H}}s_{ih'}[t]}{\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)} \Big) \Big/ \Big( \frac{F_i}{\sum_{h'\in\mathcal{H}}w_{ih'}[t]}\Big)}.
\end{align}
Note that in practice, ML users usually specify a fixed ratio between worker number and PS number (e.g., often 1:1 \footnote{In practice, the ratio between numbers and parameter servers are specified by the user upon the job's submission (e.g., 1:1 in Kubernetes~\cite{Kubernetes}).}) when launching their training jobs to ensure appropriate coordinations between workers and PSs in terms of channel bandwidth, memory allocation, etc.
To model this practice, we define the ratio of worker number to PS number for each job $i$ as:
\begin{align}\label{eqn_WorkerPSRatio}
\gamma_i \triangleq \frac{\sum_{h'\in\mathcal{H}}w_{ih'}[t]}{\sum_{h'\in\mathcal{H}}s_{ih'}[t]}, \quad \forall i, t.
\end{align}
With $\gamma_{i}$, we can rewrite Eq.~(1) as:
\begin{align*}
\frac{w_{ih}[t]}{\tau_{i} +\frac{\gamma_i}{F_i} \frac{2g_i}{\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)}}.
\end{align*}
Suppose that, for job $i$, there are $K_{i}$ data samples in its training dataset.
%, which is estimated based on the open datasets from Pathmind ~\cite{pathmind}. \kliu{What is pathmind?}
In practice, $K_i \gg F_i$.
In ML systems, an epoch is defined as a round of training that exhausts all data samples.
We let $E_{i}$ denote the number of epochs needed by job $i$.
In this paper, we assume that
the epoch of each job is predetermined.
% in a certain range and can be known a prior.
This is because it is often difficult to estimate the required number of epochs for SGD-type methods' convergence.
Therefore, most SGD-type algorithms in practice stop after a fixed number of iterations (i.e., fixed number of epochs, see, e.g., ~\cite{Wang18:SpiderBoost} and references therein) to avoid excessive training delay.
% Moreover, the early stopping strategy is commonly used (e.g., scikit-learn). In such cases, we estimate the least number of iterations that can generalize well to unseen data based on the validation set.
%We let $F_i$ denote the global batch size where assumed to be equally processed by workers.
Then, the total number of samples to be processed for job $i$ over the entire training process is $E_iK_i$.
To make sure that there are sufficient workers allocated for job $i$ over the entire training horizon, we have:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} \label{eqn_min_workers_minibatch}
%&\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ih}[t]}{\tau_{i} + \frac{2g_i}{\min_{p\in \mathcal{P}_{i}[t]}b_{i}(h,p)}}\geq x_{i} E_{i}K_{i}M_{i},\forall i \in \mathcal{I}.
\!\!\!\!\!\! \sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ih}[t]}{\tau_{i} +\frac{\gamma_i}{F_i} \frac{2g_i}{\min_{p\in \mathcal{P}_i[t],h'\in\mathcal{W}_i[t]}b_i(h',p)}} \!\geq\! x_{i} E_{i}K_{i},\forall i \in \mathcal{I}. \!\!\!\!
\end{align}
We note that, with co-located workers and parameter servers on each machine, Eq.~(\ref{eqn_min_workers_minibatch}) is {\em non-deterministic} due to the existence of the $\min\{\cdot\}$ operator.
As will be shown later, this non-determistic constraint makes the scheduling design far more complicated than related works~\cite{Li14:ML_Static_OSDI,Chilim14:ML_Static_OSDI,Chun16:Dolphin,Bao18:ML_INFOCOM}.
To model the fact that the largest number of assigned concurrent workers is no more than the global batch size %the number of $F_i$ in each time slot
(otherwise, some workers will be idle), we have:
\begin{align} \label{eqn_max_workers_datachunk}
\sum_{h \in \mathcal{H}}w_{ih}[t] \leq x_{i} F_i, \quad \forall i \in \mathcal{I}, a_{i} \leq t \leq T.
\end{align}
%\smallskip
{\bf 3) Resource Constraint Modeling:}
We let $\mathcal{R}$ denote the set of resources (e.g., CPU/GPU, memory, storage, etc.).
Let $\alpha_{i}^{r}$ and $\beta_{i}^{r}$ be the amount of type-$r$ resource required by a worker and a parameter server for job $i$, respectively.
Let $C_{h}^{r}$ be the capacity of type-$r$ resource on machine $h$.
To ensure the resources do not exceed type-$r$'s limit, we have:
\begin{align}
\label{capacity}\sum_{i \in \mathcal{I}}(\alpha_{i}^{r}w_{ih}[t]+\beta_{i}^{r}s_{ih}[t])\leq C_{h}^{r}, \forall t \in \mathcal{T}, r \in \mathcal{R}, h \in \mathcal{H}.
\end{align}
%In a distributed ML system,
%%the total number of bandwidth of workers should not be greater than the total number of bandwidth of parameter servers.
%%In other words,
%the parameter servers should not be the bottleneck during gradient/parameter exchanges.
%To this end, we let $B_{i}$ denote the communication data rate of job $i$'s parameter server.
%Then, we have:
%\begin{align} \label{eqn_worker_ps_bandwidth}
%\sum_{h \in \mathcal{W}_{i}[t]} \sum_{p \in \mathcal{P}_{i}[t]} \!\!\! w_{ih}[t]b_{i}(h,p) \leq \!\!\!\sum_{h\in \mathcal{P}_{i}[t]} \!\!\!s_{ih}[t]B_{i},\forall i\in \mathcal{I},t\in \mathcal{T}. \!\!\!
%%\sum_{h \in \mathcal{H}} \sum_{p \in \mathcal{H}} \mathbb{E} [ w_{ih}[t] ]b_{i}(h,p) \leq \sum_{h\in \mathcal{H}} \mathbb{E} [ s_{ih}[t] ] B_{i}, \forall i\in \mathcal{I}, \forall t \in \mathcal{T}. \!\!\!
%\end{align}
%In practice, the number of parameter servers is upper bounded by the number of workers in each time slot for any job.
%This can be modeled as follows:
%\begin{align} \label{eqn_workers_ps_bound}
%\sum_{h\in \mathcal{P}_{i}[t]} s_{ih}[t] \leq \sum_{h\in \mathcal{W}_{i}[t]} w_{ih}[t], \quad \forall i\in \mathcal{I},t\in \mathcal{T}.
%\end{align}
Note that for job $i$, its completion time $\tilde{t}_{i}$ corresponds to the latest time-slot where there remain some active workers allocated for it.
Therefore, we have:
\begin{align} \label{eqn_completion_time}
\tilde{t}_{i}=\arg \max_{t \in \mathcal{T}}\bigg\{\sum_{h \in \mathcal{H}} w_{ih}[t]>0 \bigg\}, \quad \forall i \in \mathcal{I}.
\end{align}
To ensure that no workers and parameter servers are allocated before job $i$'s arrival, we have:
\begin{align}
\label{yz}&w_{ih}[t]=s_{ih}[t]=0, \quad \forall i \in \mathcal{I}, h \in \mathcal{H}, t<a_{i}.
\end{align}
{\bf 4) Objective Function and Problem Statement:}
Let $u_{i}(\tilde{t}_{i} - a_{i})$ be the utility function for job $i$, which is non-increasing with respect to the training time $\tilde{t}_{i} - a_{i}$.
The utility functions could play the role of various performance metrics based on job completion times (e.g., fairness).
%That is, the utility received by a job does not increase if its completion time is long.
In this paper, our goal is to maximize the overall utility for all jobs.
Putting all constraints and the objective function together, the offline (with knowledge of $a_{i}$, $\forall i$) distributed ML resource scheduling problem (DMLRS) can be formulated as:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} %\label{eqn_offline_formulation}
\textbf{DMLRS: } & \underset{\x,\w,\s}{\text{Maximize }} \sum_{i \in \mathcal{I}}x_{i} u_{i}(\tilde{t}_{i} - a_{i}) \nonumber\\
& \text{subject to } \text{Constraints (\ref{eqn_min_workers_minibatch}) -- (\ref{yz})}. \nonumber
\end{align}
Problem DMLRS is an integer nonlinear program, which is NP-hard in general~\cite{Hochbaum97}.
Also, Problem DMLRS involves two {\em non-deterministic} constraints in (\ref{eqn_min_workers_minibatch}) and (\ref{eqn_completion_time}), which are not amenable for conventional optimization techniques.
Moreover, the arrivals $\{a_{i}, \forall i\}$ are often unknown in practice, which necessitates {\em online optimization}.
Overcoming these challenges constitutes the rest of this paper.
To conclude this section, we summarize the key notation used in this paper in Table~\ref{table:notation} for easy reference.
\begin{table}[t!]
\centering
\caption{Notation.}
\label{table:notation}
\vspace{-.1in}
\begin{tabular}{| c | l |}
\hline
$\mathcal{I/T}$ & The set of jobs$/$System timespan \\ \hline
% $\mathcal{T}$ & System timespan \\ \hline
$\tilde{t}_{i}/a_{i}$ & Completion time of job $i$ $/$Arrival time of job $i$ \\ \hline
% $a_{i}$ & Arrival time of job $i$ \\ \hline
$u_{i}(\cdot)/K_{i}$ & Job $i$'$s$ utility function$/$Number of samples in $i$ \\ \hline
$\mathcal{R/H}$ & The set of resource types$/$The set of machines \\ \hline
% $K_{i}$ & \# of samples in $i$ \\ \hline
$E_{i}/F_{i}$ &\# of training epochs/Global batch size for job $i$ \\ \hline
% $M_{i}$ & Number of mini-batches in a data chunk for job $i$ \\ \hline
% $F_{i}$ & Global batch size for job $i$ \\ \hline
% $\mathcal{H}$ & Set of physical machines\\ \hline
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
$x_{i}$ & Admission decision variable to accept job $i$ or not \\ \hline
$C_{h}^{r}$ & Capacity of type-$r$ resource on server $h$ \\ \hline
$\alpha_{i}^{r}$ & Type-$r$ resource required by a worker in job $i$ \\ \hline
$\beta_{i}^{r}$ & \begin{tabular}[c]{@{}l@{}l@{}}Type-$r$ resource required by a PS in job $i$\end{tabular} \\ \hline
$w_{ih}[t]$ & Number of workers of job $i$ on server $h$ in $t$ \\ \hline
$s_{ih}[t]$ & Number of PSs of job $i$ on server $h$ in $t$ \\ \hline
%$B_{i}$ & Bandwidth offered by a parameter server of job $i$; \\ \hline
$b_{i}(h,p)$ &\begin{tabular}[c]{@{}l@{}l@{}}Bandwidth consumed by a worker of job $i$, where \\ $b_{i}(h,p) =b_{i}^{(e)}$, if $h\neq p$ or $b_{i}^{(i)}$, otherwise. \end{tabular} \\ \hline
$\tau_{i}$ & Time to train a sample for job $i$ \\ \hline
$g_{i}$ &\begin{tabular}[c]{@{}l@{}}Size of gradients and parameters for job $i$\end{tabular} \\ \hline
$\mathcal{W}_{i}[t]$ & \begin{tabular}[c]{@{}l@{}}Set of physical machines containing workers\\ for job $i$ in $t$\end{tabular} \\ \hline
$\mathcal{P}_{i}[t]$ & \begin{tabular}[c]{@{}l@{}}Machines containing parameter servers\\ for job $i$ in $t$\end{tabular} \\ \hline
$x_{\pi_i}$ & \begin{tabular}[c]{@{}l@{}} Binary decision variable to select schedule $\pi$ \\for job $i$ or not\end{tabular} \\ \hline
$\tilde t_{\pi_i}$ & The completion time slot of job $i$ with schedule $\pi$\\ \hline
$w_{ht}^{\pi_i}$ & \# of workers on server $h$ in $t$ for job $i$ in schedule $\pi$\\ \hline
$s_{ht}^{\pi_i}$ & \# of PSs on server $h$ for schedule $\pi$ in $t$ \\ \hline
$\Pi_{i}$ & Set of all feasible schedules for job $i$\\ \hline
$\gamma_i$ & The ratio of worker number to PS number for job $i$\\ \hline
$\rho_h^r[t]$ & Allocated type-$r$ resource on machine $h$ in time $t$ \\ \hline
$Q_h^r(\cdot)$ & Price function for type-$r$ resource on machine $h$\\ \hline
\end{tabular}
% \end{tabularx}
\vspace{-.1in}
\end{table}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Online Scheduling Algorithm Design} \label{sec:alg}
In this section, we structure the key components of our online scheduling algorithm design for solving Problem~DMLRS into three steps from Sections~\ref{subsec:reformulation} to \ref{subsec:best_schedule}.
We state our main theoretical performance results in Section~\ref{subsec:performance_analysis}.
\subsection{Reformulation for Non-Deterministic Constraint (\ref{eqn_completion_time})} \label{subsec:reformulation}
The first challenge in solving Problem~DMLRS stems from the non-deterministic ``argmax'' structure in constraint (\ref{eqn_completion_time}).
%In the literature, a commonly used technique to handle argmax-type constraints is via an equivalent reformulation that enumerates all possible schedules in the system (see, e.g., \cite{Bao18:ML_INFOCOM}).
To address this challenge, we let $\Pi_{i}$ be the set of all feasible schedules for job $i \in \mathcal{I}$ that satisfy constraints (\ref{eqn_min_workers_minibatch}), (\ref{eqn_max_workers_datachunk}).
%and (\ref{eqn_worker_ps_bandwidth}).
Each schedule $\pi_{i} \in \Pi_{i}$ is defined by the numbers of workers $w_{ht}^{\pi_i}$ and parameter servers $s_{ht}^{\pi_i}$ allocated for job $i$ on machine $h$ in each time-slot $t$, i.e., $\pi_i \!\triangleq\! \{ w_{ht}^{\pi_i}, s_{ht}^{\pi_i}, \forall t \!\in\! \mathcal{T}, h \!\in\! \mathcal{H} \}$.
Note that $w_{ht}^{\pi_i}$ and $s_{ht}^{\pi_i}$ are constants, not to be confused with decision variables $w_{ih}[t]$ and $s_{ih}[t]$.
We define a binary variable $x_{\pi_i} \!\in\! \{0,1\}$ that is equal to 1 if job $i$ is admitted and scheduled under schedule $\pi_{i}$, or 0, otherwise. %, $\forall i \in \mathcal{I}, \pi \in \Pi_{i}$.
Clearly, due to the combinatorial nature, $|\Pi_{i}|$ is exponential.
We let $\tilde{t}_{\pi_i}$ denote job $i$'s completion time under schedule $\pi_i$.
%{\color{red}We also let $w_{ih}^{\pi}[t]$ and $z_{ih}^{\pi}[t]$) denote the number of workers and parameter servers specified by schedule $\pi$ on machine $h \in \mathcal{H}$ in $t \in \mathcal{T}$ for job $i \in \mathcal{I}$, respectively.}
Then, one can equivalently reformulate Problem DMLRS as:
\begin{align} %\label{eqn_reformulation}
& \textbf{R-DMLRS:} \nonumber \\
& \underset{\x}{\text{Maximize }}\sum_{i \in \mathcal{I}} \sum_{\pi_{i} \in \Pi_{i}} x_{\pi_i} u_{i}(\tilde{t}_{\pi_i} - a_{i}) \nonumber \\
%&- \gamma\max \left\{0, x_{\pi_i} E_{i}K_{i}M_{i} -\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ht}^{\pi_i}}{\tau_i + \frac{2g_i}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)}} \right\} \nonumber\\
%& \nonumber \\
\label{eqn_rsc_limit}& \text{subject to } \sum_{i\in \mathcal{I}} \sum_{\pi_i \in \Gamma(t,h)} (\alpha_{i}^{r} w_{ht}^{\pi_i} + \beta_{i}^{r} s_{ht}^{\pi_i}) x_{\pi_i} \leq C_{h}^{r}, \nonumber \\
%
&\hspace{1in} \forall t \in \mathcal{T}, r\in \mathcal{R}, h \in \mathcal{H}, \\
\label{eqn_sch_select} &\hspace{.6in} \sum_{\pi_i \in \Pi_{i}} x_{\pi_i} \leq 1, \quad \forall i \in \mathcal{I}, \\
%%
&\hspace{.6in} x_{\pi_i} \in \{0,1\}, \quad \forall i \in \mathcal{I}, \pi_i \in \Pi_{i}, \nonumber
\end{align}
\noindent where we use $\Gamma(t,h)$ to represent the set of feasible schedules that use machine $h$ to deploy workers or parameter servers in time-slot $t$.
Constraint (\ref{eqn_rsc_limit}) guarantees that, in any time-slot $t$ and on any machine $h$, the total amount of consumed type-$r$ resources will not exceed the capacity limit $C_{h}^{r}$.
Constraint (\ref{eqn_sch_select}) ensures that, for each job $i$, at most one feasible schedule from $\Pi_{i}$ will be selected.
%
Note that Problem R-DMLRS is an integer linear program (ILP) and a feasible solution to Problem R-DMLRS has a corresponding feasible solution to the original Problem DMLRS, % with the same objective value,
and vice versa.
Notice that the non-deterministic constraint (\ref{eqn_completion_time}) {\em no longer exists} in Problem~R-DMLRS.
Further, if relaxing binary $x_{\pi_i}$-variables to real-valued, Problem R-DMLRS is a linear program (LP).
However, it remains difficult to solve Problem~R-DMLRS since it has an {\em exponential} number of $x_{\pi_i}$-variables due to the combinatorial nature of feasible schedules.
We will address this challenge in the next subsection.
%Therefore, directly solving Problem~R-DMLRS remains intractable.
%In what follows, we adopt a primal-dual online algorithmic framework, which is an effective approach to address this kind of challenge in the literature (see, e.g., ~\cite{Buchbinder09,Bao18:ML_INFOCOM}).
\subsection{An Online Primal-Dual Framework for R-DMLRS}
In what follows, we adopt a primal-dual online algorithmic framework to reduce the number of binary variables, which is an effective approach to address this kind of challenge in the literature (see, e.g., \cite{Buchbinder09,Bao18:ML_INFOCOM}).
Note that, in the dual of Problem~R-DMLRS, the number of dual variables is polynomial.
Meanwhile, although there are an exponential number of constraints in the dual problem, one only needs to be concerned with the set of active (binding) constraints, which are easier to deal with.
%
To see this, we associate dual variables $p_{h}^{r}[t] \geq 0$, $\forall t\in \mathcal{T}$, $h \in \mathcal{H}$, $r \in \mathcal{R}$ and $\lambda_{i} > 0$, $i \in \mathcal{I}$, with (\ref{eqn_rsc_limit}) and (\ref{eqn_sch_select}), respectively.
Then, following the standard procedure of dualization (relaxing the integrality constraints), we obtain the following dual problem:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} \label{eqn_reformulation_dual}
& \textbf{D-R-DMLRS:} \nonumber \\
& \underset{\vlambda,\p}{\text{Minimize }} && \hspace{-.3in} \sum_{i \in \mathcal{I}} \lambda_{i} + \sum_{t\in\mathcal{T}}\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}p_{h}^{r}[t] C_{h}^{r} & \\
& \text{subject to} && \hspace{-.3in} \lambda_{i} \geq u_{i}(\tilde{t}_{\pi_i} - a_{i}) - \sum_{t \in \mathcal{T}(\pi_i)} \sum_{h \in \mathcal{H}(\pi_i[t])} \sum_{r \in \mathcal{R}} (\alpha_{i}^{r} w_{ht}^{\pi_i} \nonumber\\
\label{eqn_dualconstr} & && \hspace{-.05in} + \beta_{i}^{r} s_{ht}^{\pi_i}) p_{h}^{r}[t], \quad \forall i \in \mathcal{I}, \pi_i \in \Pi_{i}, \\
& && \hspace{-.3in} p_{h}^{r}[t] \geq 0, \quad \forall t \in \mathcal{T}, h \in \mathcal{H}, r \in \mathcal{R}, \nonumber\\
& && \hspace{-.3in} \lambda_{i} \geq 0, \quad \forall i \in \mathcal{I}, \nonumber
\end{align}
\noindent where $\mathcal{T}(\pi_i)$ denotes the time-slots utilized by schedule $\pi_i$ and $\mathcal{H}(\pi_i[t])$ denotes the set of machines containing workers and/or parameter servers under $\pi_i$ in time-slot $t$.
Here, $p_{h}^{r}[t]$ can be viewed as the price for type-$r$ resource in time $t$.
Then, the right-hand-side (RHS) of (\ref{eqn_dualconstr}) can be interpreted as job utility minus overall resource cost of job $i$ using schedule $\pi_i$.
Thus $\lambda_{i}\geq 0$ can be viewed as the payoff of admitting job $i$ with $\pi_i$.
Next, we examine the properties of Problem D-R-DMLRS.
To minimize (\ref{eqn_reformulation_dual}), we tend to reduce $\lambda_{i}$ and $p_{h}^{r}[t]$ until they hit zero.
However, as $\lambda_{i}$ and $p_{h}^{r}[t]$ decrease, the left-hand-side (LHS) and RHS of (\ref{eqn_dualconstr}) decreases and increases, respectively (the term
%\begin{multline*}
$u_{i}(\tilde{t}_{\pi_i}-a_{i})$
%%
%\gamma\max \left\{0, x_{\pi_i} E_{i}K_{i}M_{i} -\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ht}^{\pi_i}}{\tau_i + \frac{2g_i}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)}} \right\}
%\end{multline*}
in the RHS of (\ref{eqn_dualconstr}) is a constant given $\pi_i$).
Therefore, $\lambda_{i}$ will drop to a value $\lambda_{i}^{*}$, which is equal to maximum of the RHS of (\ref{eqn_dualconstr}) achieved by some schedule $\pi_{i}^{*}$ and dual price $p_{h}^{r*}[t]$, i.e.,
\begin{align*}
&\lambda_{i}^{*} \!=\! u_{i}(\tilde{t}_{\pi_i^*}\!-\!a_{i})
%
%&-\! \gamma\max \left\{0, x_{\pi_i^*} E_{i}K_{i}M_{i}\!-\!\sum_{t \in \mathcal{T}}\sum_{h \in \mathcal{H}} \frac{w_{ht}^{\pi_i}}{\tau_i + \frac{2g_i}{\min_{p\in \mathcal{P}_i[t]}b_i(h,p)}} \right\}\nonumber\\
%
\!-\!\!\sum_{t \in \mathcal{T}(\pi_i^*)} \! \sum_{h \in \mathcal{H}(\pi_i^{*}[t])} \! \sum_{r \in \mathcal{R}} (\alpha_{i}^{r} w_{ht}^{\pi_i^{*}} \!+\! \beta_{i}^{r} s_{ht}^{\pi_i^{*}}) p_{h}^{r*}[t].
\end{align*}
This optimality structural insight implies that Problem~D-R-DMLRS is equivalent to finding an optimal schedule $\pi_i^{*}$ and dual price $p_{h}^{r*}[t]$ to maximize the RHS of (\ref{eqn_dualconstr}).
The above insights motivate the following primal-dual-based algorithm as shown in Algorithm~1.
% which motivates the following primal-dual~\cite{Buchbinder09} online resource scheduling (PD-ORS) framework in Algorithm~1:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~1:} {Primal-Dual Online Resource Scheduling (PD-ORS).}
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let $w_{ih}[t]=0$, $s_{ih}[t]=0$, $\forall i,t,h$. Let $\rho_{h}^{r}[t] = 0$, $\forall h,r,t$. Choose some appropriate initial values for $p_{h}^{r}[0]$.
\end{enumerate}
\noindent {\bf Main Loop:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Upon the arrival of job $i$, determine a schedule $\pi_{i}^{*}$ to maximize the RHS of (\ref{eqn_dualconstr}) and its corresponding payoff $\lambda_{i}$ using {\bf Algorithm~2} ({\em to be specified}).
%
\item[3.] If $\lambda_{i} > 0$, set $x_{i} = 1$.
Set $w_{ih}[t]$ and $s_{ih}[t]$ according to schedule $\pi_{i}^{*}$, $\forall t \in \mathcal{T}(\pi_{i}^{*})$, $h\in\mathcal{H}(\pi_{i}^{*}[t])$.
\begin{itemize}
\item Update $\rho_{h}^{r}[t] \leftarrow \rho_{h}^{r}[t] + \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r}s_{ih}[t]$, $\forall t \in \mathcal{T}(\pi_{i}^{*})$, $h\in\mathcal{H}(\pi_{i}^{*}[t])$, $r \in \mathcal{R}$.
\item Update $p_{h}^{r}[t] = Q_{h}^{r}(\rho_{h}^{r}[t])$, $\forall t \in \mathcal{T}(\pi_{i}^{*})$, $h\in\mathcal{H}(\pi_{i}^{*}[t])$, $r \in \mathcal{R}$.
Schedule job $i$ based on $\pi_{i}^{*}$ and go to Step 2.
\end{itemize}
%
\item[4.] If $\lambda_{i} \leq 0$, set $x_{i} = 0$ and reject job $i$ and go to Step 2.
\end{enumerate}
\smallskip
\hrule
\medskip
The intuition of Algorithm~1 is as follows:
By the complementary slackness condition of the Karush-Kuhn-Tucker (KKT) conditions~\cite{Bazaraa_Sherali_Shetty_93:NLP}, the primal constraint (\ref{eqn_sch_select}) must be tight when dual variable $\lambda_{i} > 0$, which implies that $x_{i} =1$ (Step 3) in Problem DMLRS.
Otherwise, if $\lambda_{i} = 0$, then the RHS of (\ref{eqn_dualconstr}) is non-positive, meaning the utility is low compared to the cost of resource consumption under schedule $\pi_{i}^{*}$.
Therefore, we should reject job $i$ ($x_{i} = 0$ in Step 4).
However, in order for the PD-ORS algorithm to work, two challenging components need to be specified: the schedule $\pi_{i}^{*}$ and how to update the cost function $Q_{h}^{r}(\cdot)$\footnote{
Here, we note that the ``cost function'' $Q_{h}^{r}(\cdot)$ is interpreted from servers' perspective rather than jobs' perspective.
Specifically, higher cost means servers allocated more resources to jobs, which implies higher utility for the jobs since jobs receive more resources from servers.}
In what follows, we will first focus on designing $Q_{h}^{r}(\cdot)$ and defer the challenging problem of finding $\pi_{i}^{*}$ to Section~\ref{subsec:best_schedule}.
For the design of $Q_{h}^{r}(\cdot)$,
%we adopt the existing approach in the literature ~\cite{Buchbinder09,Bao18:ML_INFOCOM} and
consider the following choice of $Q_{h}^{r}(\cdot)$:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align} \label{eqn_cost_update_fn}
Q_{h}^{r}(\rho_{h}^{r}[t]) = L(U^{r}/L)^{\frac{\rho_{h}^{r}[t]}{C_{h}^{r}}},
\end{align}
where constants $U^{r}$, $\forall r$, and $L$ are defined as follows:
%$\textcolor{red}{U^{r} \triangleq \gamma\max_{i\in\mathcal{I}}\frac{u_{i}(\ceil{E_{i}M_{i}\tau_{i}}-a_{i})}{\alpha_{i}^{r}+\beta_{i}^{r}}?}$
\begin{align}
\label{eqn_Ur} & \!\!\!\!\!\! U^{r} \!\triangleq\!\max_{i\in\mathcal{I}}\frac{u_{i}(\ceil{\frac{E_iK_i}{F_i}(\tau_{i}+2g_{i}\gamma_i/(b_i^{(i)}F_i))}-a_{i})}{\alpha_{i}^{r}+\beta_{i}^{r}},\forall r\in\mathcal{R},\!\!\!\!\!\! \\
%
\label{eqn_L} & \!\!\!\!\!\! L \triangleq \min_{i\in\mathcal{I}}\frac{1/(2\mu) u_{i}(T-a_{i})}{\sum_{r\in\mathcal{R}}\ceil{E_{i}K_i(\tau_{i}+2g_{i}\gamma_i/(b_i^{(e)}F_i)}(\alpha_{i}^{r}+\beta_{i}^{r})}. \!\!\!\!\!\!
\end{align}
The scaling factor $\mu$ in the definition of $L$ satisfies as follows:
\begin{align*}
\frac{1}{\mu}\leq \frac{\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})}{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}},\forall i\in\mathcal{I}.
\end{align*}
% $\frac{1}{\mu}\leq \frac{\ceil{E_{i}K_{i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}\sum_{r\in\mathcal{R}}(\alpha_{i}^{r}+\beta_{i}^{r})}{T\sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}}C_{h}^{r}},\forall i\in\mathcal{I}$.
Here, $U^{r}$ is the maximum unit-resource job utility to deploy workers and parameter servers with type-$r$ resource .
Here, $u_{i}(\ceil{\frac{E_iK_i}{F_i}(\tau_{i}+2g_{i}\gamma_i/(b_{i}^{(i)}F_i))}-a_{i})$ is the largest utility job $i$ can achieve by using the maximum number of {\em co-located} workers and parameter servers (hence communicating rate is $b_{i}^{(i)}$) at all times during all $E_{i}$ epochs, so that $\ceil{\frac{E_iK_i}{F_i}(\tau_{i}\!+\!2g_{i}\gamma_i/(b_{i}^{(i)}F_i))}\!-\!a_{i}$ is the earliest possible job completion time.
Similarly, $L$ represents the minimum unit-time unit-resource job utility among all jobs, with $u_{i}(T-a_{i})$ being the smallest utility for job $i$, and workers and parameter servers communicate at small external rate $b_{i}^{(e)}$.
We use $\rho_h^r[t]$ to denote the allocated amount of type-$r$ resource to machine $h$ for (future) time slot $t$.
% \textcolor{red}{Under external communication case, we have $\frac{s_{ih}[t]}{\sum_{h\in\mathcal{H}}s_{ih}[t]}<1$, and by relaxing it to 1, we can get the lower bound of $L$.
% We use $\rho_h^r[t]$ to denote the allocated amount of type-$r$ resource to machine $h$ at time $t$.}
The intuition behind the $Q_{h}^{r}(\cdot)$ function is as follows:
%(proofs follow similarly from ~\cite{Buchbinder09,Bao18:ML_INFOCOM} and are omitted due to space limitation):
i) At $t\!=\!0$, $\rho_{h}^{r}[0]\!=\!0, \forall h\in\mathcal{H},r\in\mathcal{R}$.
Hence, the price $p_{h}^{r}[0]=L$ is the lowest, $\forall h,r$, and any job can be admitted;
ii) As allocated resources increases, the price increases exponentially fast to reject early coming jobs with low utility and to reserve resources for later arrived jobs with higher utility;
iii) When some type-$r$ resource is exhausted, i.e., $\rho_{h}^{r}[t] = C_{h}^{r}, \exists r\in\mathcal{R}$, $Q_{h}^{r}[C_{h}^{r}]=U^{r}$ and no job that requires type-$r$ resources will be admitted since the $U^{r}$ is the highest price.
As will be shown later, this price function leads to a logarithmically scaling competitive ratio in online algorithm design.
Note that computing the price function in Algorithm~1 requires the information of constants $U^r$, $L$, which can usually be estimated empirically based historical data.
Here, we point out a few interesting insights on the design choices of the cost function in Eq.~\eqref{eqn_cost_update_fn}.
Note that $U^r$ and $L$ are defined in Eqns~(\ref{eqn_Ur}) and (\ref{eqn_L}), respectively.
Here, we intentionally choose $U^r$ to be dependent on $r$ and $L$ to be independent on $L$ due to the following reasons:
First, the rationality of choosing an upper bound $U^r$ that varies with different resource types is to ensure that when some type-$r$ resource is exhausted, no more jobs that require type-$r$ resource should be allocated.
In other words, when the allocated amount of type-$r$ resource reaches the capacity of physical machine $h$, i.e., $\rho_h^r[t]=C_n^r, \exists r\in\mathcal{R}$, the price $p_h^r=Q_h^r(C_h^r)$ should reach the upper bound $U^r$, indicating that any job that requires type-$r$ resource will not be allocated to $h$.
However, jobs that do not require type-$r$ resource should still be able to be scheduled on machine $h$.
For example, jobs that do not require GPU can still be placed on a machine with no available GPUs.
Second, the reason that we choose the lower bound $L$ to be independent of any resource type $r$ is to yield a larger ratio of $\frac{U^r}{L}$.
%to admit jobs incoming at $t=0$. Also, as the allocated resources increases, the price increases exponentially (see Eqn.~(\ref{eqn_cost_update_fn})).
The larger the ratio of $\frac{U^r}{L}$ is, the greater the price will be.
Intuitively, the ratio $\frac{U^r}{L}$ can be interpreted as the scheduling ``uncertainty,'' which increases as the ratio gets larger, implying the price function reacts to the accumulative resource consumption more aggressively.
Thus, choosing $L$ to be independent of resource type $r$ allows the price function $Q_h^r(\cdot)$ to react more aggressively to the accumulative allocated resource amount.
%Later, we show on Page~10, Theorems~5 and 6 that our choice have a good log performance.
We note that one can also choose the lower bound to be dependent on resource type $r$ by replacing $L$ with $L^r$.
By doing so, the log-scaling theoretical competitive ratio in Theorem~\ref{thm_Competitive_1} still holds and the proof in Appendix~\ref{appdx:thm_Competitive} only needs to be slightly updated with the new notation $L^r$.
However, the empirical performance of using $L^r$ as lower bound is worse since the price function reacts less aggressively to the accumulative allocated resources.
\begin{figure*}[t!]
\centering
\subfigure[{$|\mathcal{P}_{i}[t]|\ne 1$.}]{%
\label{fig:IntExt1}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_1.eps}}%
\hspace{6pt}%
\subfigure[{$|\mathcal{P}_{i}[t]|=1,|\mathcal{W}_{i}[t]|\ne 1$.}]{%
\label{fig:IntExt2}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_2.eps}}%
\hspace{6pt}%
\subfigure[{$|\mathcal{P}_{i}[t]|\!\!=\!\!|\mathcal{W}_{i}[t]|\!\!=\!\!1,\mathcal{P}_{i}[t]\!\!\ne\!\!\mathcal{W}_{i}[t]$.}]{%
\label{fig:IntExt3}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_3.eps}}%
\hspace{6pt}%
\subfigure[{$|\mathcal{P}_{i}[t]|\!\!=\!\!|\mathcal{W}_{i}[t]|\!\!=\!\!1,\mathcal{P}_{i}[t]\!\!=\!\!\mathcal{W}_{i}[t]$.}]{%
\label{fig:IntExt4}%
\includegraphics[width=0.24\textwidth]{Figures/fig_IntExt_4.eps}}%
\vspace{-.08in}
\caption{Values of $\frac{2g_{i}}{\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)}$ under various settings of $\mathcal{P}_{i}[t]$ and $\mathcal{W}_{i}[t]$.
% Clearly,
Here,
$\frac{2g_{i}}{\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)} = 2\frac{g_{i}}{b_{i}^{(i)}}$ if and only if in (d).}%
\label{fig:IntExt}%
\vspace{-.15in}
\end{figure*}
\vspace{-.1in}
\subsection{Determining Schedule $\pi_{i}^{*}$ in Step 2 of Algorithm 1} \label{subsec:best_schedule}
Now, consider the problem of finding a schedule $\pi_{i}^{*}$ in Step 2 of Algorithm~1 to maximize the RHS of (\ref{eqn_dualconstr}).
First, we note that {\em any} schedule for job $i$ has a unique completion time $\tilde{t}_{i}$, including the maximizer $\pi_{i}^{*}$ for (\ref{eqn_dualconstr}).
Hence, the problem of finding the maximum RHS of (\ref{eqn_dualconstr}) can be decomposed as:
%This key subproblem can be stated as follows:
\begin{align} \label{eqn_ProbFindSch}
\!\!\!\underset{{\tilde{t}_{i}}}{\text{Max}} \left\{
\begin{array}{l}
\underset{{ \w,\s }}{\text{Max}} \,\, u_{i}(\tilde{t}_{i} \!-\! a_{i}) \!-\! \sum\limits_{t\in\mathcal{T}}\sum\limits_{h\in \mathcal{H}}\sum\limits_{r\in\mathcal{R}} p_{h}^{r}[t] \\
%
\hspace{1.1in} \times (\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]) \\
%
\text{s.t. }\ \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t] \leq \hat{C}_{h}^{r}[t], \\
%
\hspace{1.1in} \forall t\in \mathcal{T},r\in\mathcal{R},h\in\mathcal{H}, \\
%
\hspace{.3in} \text{Constraints (\ref{eqn_min_workers_minibatch})(\ref{eqn_max_workers_datachunk})(\ref{yz}) for } x_{i}=1,
\end{array}
\right\}, \!\!\!\!\!
\end{align}
\noindent where $\hat{C}_{h}^{r}[t] \triangleq C_{h}^{r} - \rho_{h}^{r}[t]$.
Note that in the inner problem, $u_{i}(\tilde{t}_{i} \!-\! a_{i})$ is a constant for any given $\tilde{t}_{i}$.
Thus, the inner problem can be simplified as:
\begin{align}
\label{eqn_obj_given_t} \underset{\w,\s}{\text{Minimize }} & \sum_{t \in [a_{i}, \tilde{t}_{i}]} \sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}} p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]) \\
%
\label{eqn_workload_coupling} \text{subject to } & \sum_{t\in[a_{i},\tilde{t}_{i}]} \sum_{h\in\mathcal{H}} \frac{w_{ih}[t]}{\tau_{i}+\frac{\gamma_i}{F_i} \frac{2g_{i}}{\min_{p\in \mathcal{P}_{i}[t],h'\in\mathcal{W}_i[t]}b_{i}(h',p)}}\geq V_{i}, \!\!\! \\
%
\label{eqn_rsrc_limit} & \alpha_{i}^{r} w_{ih}[t] \!+\! \beta_{i}^{r} s_{ih}[t] \!\leq\! \hat{C}_{h}^{r}[t], \forall r,h, \forall t \!\in\! [a_{i},\tilde{t}_{i}], \!\!\!\\
&\text{Constraint (\ref{eqn_max_workers_datachunk}) for all } t\in[a_{i},\tilde{t}_{i}],\nonumber
\end{align}
where $V_{i} \triangleq E_{i} K_{i}$ represents the total training workload
(total count of samples trained, where a sample is counted $E_i$ times if trained for $E_i$ times).
Note that in Problem~(\ref{eqn_obj_given_t}), the only coupling constraint is (\ref{eqn_workload_coupling}).
This observation inspires a dynamic programming approach to solve Problem (\ref{eqn_obj_given_t}).
Consider the problem below if training workload at time $t$ is known (denoted as $V_{i}[t]$):
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{align}
\label{eqn_obj_given_t2} \underset{w_{ih}[t],s_{ih}[t],\forall h}{\text{Minimize }} & \hspace{.1in} \sum_{h\in\mathcal{H}}\sum_{r\in\mathcal{R}} p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]) \\
%
\label{eqn_nondeterministic} \text{subject to } & \hspace{.1in} \sum_{h\in\mathcal{H}} \frac{w_{ih}[t]}{\tau_{i}+\frac{\gamma_i}{F_i} \frac{2g_{i}}{\min_{p\in \mathcal{P}_{i}[t],h'\in\mathcal{W}_i[t]}b_{i}(h',p)}}\geq V_{i}[t], \\
%
& \hspace{.1in} \text{Constraints (\ref{eqn_max_workers_datachunk})(\ref{eqn_rsrc_limit}) for the given } t.\nonumber
\end{align}
Let $\Theta(\tilde{t}_{i},V_{i})$ and $\theta(t,V_{i}[t])$ denote the optimal values of Problems~(\ref{eqn_obj_given_t}) and (\ref{eqn_obj_given_t2}), respectively.
Then, Problem~(\ref{eqn_obj_given_t}) is equivalent to the following dynamic program:
\begin{align} \label{eqn_2d_dp}
\Theta(\tilde{t}_{i},V_{i}) = \min_{v\in[0,V_{i}]} \left\{ \theta(\tilde{t}_{i},v) + \Theta(\tilde{t}_{i}-1,V_{i}-v) \right\}.
\end{align}
We find the optimal workload $v$ to be completed in time slot $\tilde{t}_{i}$ by enumerating it from 0 to $E_iK_i$, and the remaining workload $E_iK_i-v$ will be carried out in time span $[a_i,\tilde{t}_{i}-1]$.
The optimal workload would be the schedule with minimum costs, i.e., the objective function of Problem~(\ref{eqn_obj_given_t2}) is minimum.
Then we proceed to the next time slot $\tilde{t}_{i}-1$ with the workload $E_iK_i-v$, which is the same as finding the optimal schedule and cost as the last time slot $\tilde{t}_{i}$ except in a smaller scale.
Then, by enumerating all $\tilde{t}_{i} \in [a_{i}, T]$ and solving the dynamic program $\Theta(\tilde{t}_{i},V_{i})$ in (\ref{eqn_2d_dp}) for every choice of $\tilde{t}_{i}$, we can solve Problem (\ref{eqn_ProbFindSch}) and determine the optimal schedule $\pi_{i}^{*}$.
We summarize this procedure in Algorithms~2 and 3:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~2:} Determine $\pi_{i}^{*}$ in Step 2 of Algorithm 1.
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let $\tilde{t}_{i}\!=\!a_{i}$. Let $\lambda_{i} \!=\! 0$, $\pi_{i}^{*} \!=\! \varnothing$, $w_{ih}[t] \!=\! s_{ih}[t] \!=\! 0$, $\forall i,t,h$.
\end{enumerate}
\noindent {\bf Main Loop:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Compute $\Theta(\tilde{t}_{i},V_{i})$ in (\ref{eqn_2d_dp}) using {\bf Algorithm 3}.
Denote the resulted schedule as $\pi_{i}$.
Let $\lambda'_{i} = u_{i}(\tilde{t}_{i}-a_{i}) - \Theta(\tilde{t}_{i},V_{i})$.
If $\lambda'_{i} > \lambda_{i}$, let $\lambda_{i} \leftarrow \lambda'_{i}$ and $\pi_{i}^{*} \leftarrow \pi_{i}$.
%
\item[3.] Let $\tilde{t}_{i} \leftarrow \tilde{t}_{i} + 1$. If $\tilde{t}_{i} > T$, stop; otherwise, go to Step 2.
\end{enumerate}
\smallskip
\hrule
\medskip
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~3:} Solving $\Theta(\tilde{t}_{i},V_{i})$ by Dynamic Programming.
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let {\em cost-min} = $\infty$, $\pi_{i} = \varnothing$, and $v = 0$.
\end{enumerate}
\noindent {\bf Main Loop:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Compute $\theta(\tilde{t}_{i}, v)$ using {\bf Algorithm~4} ({\em to be specified}).
Denote the resulted cost and schedule as {\em cost-v} and $\hat{\pi}_{i}$.
\item[3.] Compute $\Theta(\tilde{t}_{i}-1,V_{i}-v)$ by calling {\bf Algorithm~3} itself.
Denote the resulted cost and schedule as {\em cost-rest} and $\tilde{\pi}_{i}$.
%
\item[4.] If $\text{{\em cost-min}} > \text{{\em cost-v}} + \text{{\em cost-rest}}$ then $\text{{\em cost-min}} = \text{{\em cost-v}} + \text{{\em cost-rest}}$ and let $\pi_{i} \leftarrow \hat{\pi}_{i} \cup \tilde{\pi}_{i}$.
%
\item[5.] Let $v \leftarrow v+1$. If $v>V_{i}$ stop; otherwise go to Step 2.
\end{enumerate}
\smallskip
\hrule
\medskip
% \textcolor{red}{Note that the number of states $(t,v)$ in the dynamic programming for each job $i$ is $O(TK_iE_i)$, and the time complexity to execute dynamic programming function is $O(K_iE_i)$, thus we can have the $O(TK_i^2E_i^2)$ time complexity for dynamic programming since we save all the intermediate results to avoid re-computation.}
%{\em 2) Solving $\theta(t,v)$ (i.e., Problem (\ref{eqn_obj_given_t2}))}:
In Algorithm~3, however, how to compute $\theta(t,v)$ in Step 2 (i.e., Problem~(\ref{eqn_obj_given_t2})) is yet to be specified.
A challenge in solving (\ref{eqn_obj_given_t2}) is the {\em non-deterministic} constraint in (\ref{eqn_nondeterministic}), where $b_{i}(h,p)$ can be either $b_{i}^{(i)}$ or $b_{i}^{(e)}$.
%However, the workload constraint (\ref{eqn_nondeterministic}) is {\em non-deterministic} because $b_{i}(h,p)$ can take values either $b_{i}^{(i)}$ or $b_{i}^{(e)}$.
Therefore, we need to handle both cases.
To this end, we observe the following basic fact about $\frac{2g_{i}}{\min_{p \in \mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)}$ (also illustrated in Fig.~\ref{fig:IntExt}) as stated in Fact~\ref{fact1}.
We omit the proof of this fact due to its simplicity, which is illustrated in Fig.~\ref{fig:IntExt}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{fact} \label{fact1}
The function $(2g_{i}/\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)) = 2g_{i}/b_{i}^{(i)}$ if and only if $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]| =1$ and $\mathcal{P}_{i}[t] = \mathcal{W}_{i}[t]$; otherwise, $(2g_{i}/\min_{p\in\mathcal{P}_{i}[t],h\in\mathcal{W}_i[t]}b_{i}(h,p)) = 2g_{i}/b_{i}^{(e)}$.
\end{fact}
%\smallskip
With Fact~\ref{fact1}, we now consider the following two cases:
\smallskip
{\em Case 1): $b_{i}^{(i)}$ (Internal Communication):}
In Case 1), Fact~\ref{fact1} implies that Problem~(\ref{eqn_obj_given_t2}) reduces to a {\em single-machine} problem (i.e., discarding $\sum_{h\in\mathcal{H}}\{\cdot\}$ in (\ref{eqn_obj_given_t2}) and (\ref{eqn_nondeterministic})).
Note further that if we temporarily ignore the workload-coupling constraint (\ref{eqn_nondeterministic}) and use the worker-PS ratio in (\ref{eqn_WorkerPSRatio}), Problem~(\ref{eqn_obj_given_t2}) can be {\em decoupled} across resources and simplified as:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\begin{equation}\label{linear}
%\sum_{r\in\mathcal{R}}
%\begin{Bmatrix*}[l]
%&\hspace{-.1in} \text{Min} &\hspace{-.1in} p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t])\\
%%&\hspace{-.1in}&\hspace{-.1in}+ \gamma\max \{0, V_i[t]M_{i}\!-\!\frac{w_{ih}[t]}{\tau_i + \frac{2g_i}{b_i^{(i)}}}\}, \\
%&\hspace{-.1in} \text{s.t.} &\hspace{-.1in} \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t]\leq \hat{C}_{h}^{r}[t], \\
%&&\hspace{-.1in}w_{ih}[t] = \gamma_is_{ih}[t],\\
%&&\hspace{-.1in} \text{Constraint }(\ref{eqn_max_workers_datachunk}) \text{ for given } r,h
%\end{Bmatrix*}, \!\!\!\!
%\end{equation}
%and it can be further simplified as follows:
\begin{equation}\label{linear}
\sum_{r\in\mathcal{R}}
\begin{Bmatrix*}[l]
&\hspace{-.1in} \text{Min} &\hspace{-.1in}p_{h}^{r}[t]s_{ih}[t](\alpha_{i}^{r} \gamma_i + \beta_{i}^{r} )\\
%&\hspace{-.1in}&\hspace{-.1in}+ \gamma\max \{0, V_i[t]M_{i}\!-\!\frac{w_{ih}[t]}{\tau_i + \frac{2g_i}{b_i^{(i)}}}\}, \\
&\hspace{-.1in} \text{s.t.} &\hspace{-.1in} s_{ih}[t] (\alpha_{i}^{r} \gamma_i+ \beta_{i}^{r})\leq \hat{C}_{h}^{r}[t], \\
&&\hspace{-.1in} \text{Constraint }(\ref{eqn_max_workers_datachunk}) \text{ for given } r,h
\end{Bmatrix*}, \!\!\!\!
\end{equation}
where each summand in (\ref{linear}) is an integer linear program (ILP) having a trivial solution $w_{ih}[t]=s_{ih}[t]=0$, $\forall h \in \mathcal{H}$.
However, $w_{ih}[t]=0$, $\forall h\in\mathcal{H}$, clearly violates the workload constraint ($\ref{eqn_nondeterministic}$).
Thus, when (\ref{linear}) is optimal, there should be {\em exactly one} machine $h' \in\mathcal{H}$ with $w_{ih'}[t]\geq 1$ and {\em exactly one} machine $h'' \in\mathcal{H}$ with $s_{ih''}[t] \geq 1$.
This observation shows that the optimal solution of (\ref{eqn_obj_given_t2}) {\em tends to favor} $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]|=1$ if the workload constraint (\ref{eqn_nondeterministic}) is not binding.
%which matches the internal case condition in Fact~\ref{fact1}.
Notice that the workload constraint (\ref{eqn_nondeterministic}) in Case 1) becomes $\gamma_is_{ih}[t] \geq V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(i)}F_i})$.
This implies the following simple solution:
We can first sort each physical machine $h$ according to $\sum_{r\in\mathcal{R}}p_h^r[t](\alpha_i^r\gamma_i+\beta_i^r)$ and calculate the minimum number of $s_{ih}[t]=V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(i)}F_i})/\gamma_i$ from the workload constraint.
The last step is to check if the machine satisfy the resource capacity constraint (\ref{eqn_rsrc_limit}) and constraint (\ref{eqn_max_workers_datachunk}).
%{\color{red}This observation suggests that, for the ``$b_{i}^{(i)}$'' case, we should check the workload constraint (\ref{eqn_nondeterministic}) on each machine one by one (i.e., ensuring $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]|=1$ and $\mathcal{P}_{i}[t] = \mathcal{W}_{i}[t]$). }
%In this setting, the workload constraint (\ref{eqn_nondeterministic}) becomes $\gamma_is_{ih}[t] \geq V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(i)}F_i})$.
%After checking all machines, choose, if any, the machine $h$ that satisfies (\ref{eqn_nondeterministic}) and has the lowest cost.
If so, we return the schedule $(w_{ih}[t],s_{ih}[t])$ and the corresponding cost value.
\smallskip
{\em Case 2): $b_{i}^{(e)}$ (External Communication):}
For those settings that do not satisfy $|\mathcal{P}_{i}[t]| = |\mathcal{W}_{i}[t]| =1$ and $\mathcal{P}_{i}[t] = \mathcal{W}_{i}[t]$, Fact~\ref{fact1} indicates that parameter servers and workers are communicating at external rate $b_{i}^{(e)}$.
In this case, the workload constraint (\ref{eqn_nondeterministic}) simply becomes: $\sum_{h \in \mathcal{H}} w_{ih}[t] \geq V_{i}[t](\tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i})$.
%For convenience, we let $m_{w}[t] \triangleq \ceil{V_{i}[t]M_{i}(\tau_{i}+2g_{i}/b_{i}^{(e)})}$ and $m_{p}[t] \triangleq \ceil{m_{w}[t] b_{i}^{(e)}/B_{i}}$ represent the minimum required numbers of workers and parameter servers given workload $V_{i}[t]$, respectively.
Then, we can rewrite Problem~(\ref{eqn_obj_given_t2}) as:
\begin{align}
\label{eqn_obj_given_t3} \underset{w_{ih}[t],s_{ih}[t],\forall h}{\text{Minimize }} & \hspace{.1in} \sum_{h\in\mathcal{H}} p_{h}^{w}[t] w_{ih}[t] + p_{h}^{s}[t] s_{ih}[t] \\
%
\label{eqn_packing} \text{subject to } & \hspace{.1in} \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t] \leq \hat{C}_{h}^{r}[t], \,\, \forall h,r,\\
%
\label{eqn_packing2} &\hspace{.1in} \sum\nolimits_{h\in\mathcal{H}} w_{ih}[t] \leq F_i,\\
\label{eqn_cover1} &\hspace{.1in} \sum\nolimits_{h\in\mathcal{H}} w_{ih}[t] \geq V_{i}[t] \bigg( \tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i} \bigg),
%\label{eqn_cover2} &\hspace{.1in} \sum\nolimits_{h\in\mathcal{H}} s_{ih}[t] \geq m_{p}[t],
\end{align}
where $p_{h}^{w}[t] \triangleq \sum_{r\in\mathcal{R}} p_{h}^{r}[t] \alpha_{i}^{r}$ and $p_{h}^{s}[t] \triangleq \sum_{r\in\mathcal{R}} p_{h}^{r}[t] \beta_{i}^{r}$ denote the aggregated prices of all resources of allocating workers and PSs on machine $h$ in time $t$, respectively.
%Let denote $c_i\triangleq 1/(\tau_i\!+\! \frac{2g_i}{b_i^{(i)}})$, we can have the following equivalent problem under infeasible case:
%\begin{align}
%\label{eqn_obj_given_t4} \underset{w_{ih}[t],s_{ih}[t],\forall h}{\text{Minimize }} & \hspace{.1in} V_i[t]M_{i}\!+\!\sum_{h\in\mathcal{H}}( p_{h}^{w}[t]-c_i) w_{ih}[t]\!+\!p_{h}^{s}[t] s_{ih}[t]\\
%\label{feasible_packing} \text{subject to } & \hspace{.1in} \alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}[t] \leq \hat{C}_{h}^{r}[t], \,\, \forall h,r,
%\end{align}
Unfortunately, Problem~(\ref{eqn_obj_given_t3}) is a highly challenging integer programming problem with generalized packing and cover type constraints (i.e., integer variables rather than 0-1 variables) in (\ref{eqn_packing})--(\ref{eqn_cover1}), respectively,
which is clearly NP-Hard.
% Therefore, Problem~(\ref{eqn_obj_given_t3}) is a generalized mixed cover/packing problem, which is clearly NP-Hard.
Also, it is well-known that there are {\em no} polynomial time approximation schemes (PTAS) even for the basic set-cover and bin-packing problems unless P = NP~\cite{Hochbaum97}.
In what follows, we will pursue an instance-dependent constant ratio approximation scheme to solve Problem~(\ref{eqn_obj_given_t3}) in this paper.
%\textbf{this appears conflicting with the non-ptas claim above}.
To this end, we propose a randomized rounding scheme:
First, we solve the linear programming relaxation of Problem~(\ref{eqn_obj_given_t3}).
Let $\{ \bar{w}_{ih}[t], \bar{s}_{ih}[t], \forall h,t \}$ be the fractional optimal solution.
We let $\delta\in(0,1]$ be a parameter.
Let $G_\delta$ be a constant {(the notation $G_\delta$ signifies that $G_{\delta}$ is dependent on $\delta$) to be defined later, and let $w'_{ih}[t] \!=\! G_\delta\bar{w}_{ih}[t], s'_{ih}[t] \!=\! G_\delta\bar{s}_{ih}[t]$, $\forall h,t$.
Then, we randomly round $\{ w'_{ih}[t], s'_{ih}[t], \forall h,t \}$ to obtain an integer solution as follows:
\begin{align}
\label{eqn_rounding_w} w_{ih}[t] &= \begin{cases}
\lceil w'_{ih}[t] \rceil, & \hspace{-.1in} \text{with probability } w'_{ih}[t] - \lfloor w'_{ih}[t] \rfloor,\\
\lfloor w'_{ih}[t] \rfloor, & \hspace{-.1in} \text{with probability } \lceil w'_{ih}[t] \rceil - w'_{ih}[t],
\end{cases} \!\!\!\\
%
\label{eqn_rounding_s} s_{ih}[t] &= \begin{cases}
\lceil s'_{ih}[t] \rceil, & \hspace{-.1in} \text{with probability } s'_{ih}[t] - \lfloor s'_{ih}[t] \rfloor,\\
\lfloor s'_{ih}[t] \rfloor, & \hspace{-.1in} \text{with probability } \lceil s'_{ih}[t] \rceil - s'_{ih}[t].
\end{cases}
\end{align}
We will later prove in Theorem~\ref{thm_Alg4_1} (when $0<G_\delta\leq 1)$ and Theorem~\ref{thm_Alg4_2} (when $G_\delta>1$) that the approximation ratio of this randomized rounding scheme in (\ref{eqn_rounding_w})-(\ref{eqn_rounding_s}) enjoys a ratio that is independent on the problem size.
%\textbf{so the ratio is not constant?}.
\smallskip
Lastly, summarizing results in Cases 1) -- 2) yields the following approximation algorithm for solving Problem~(\ref{eqn_obj_given_t2}):
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\medskip
\hrule
\vspace{.03in}
\noindent {\bf Algorithm~4:} Solving $\theta(t,v)$ (i.e., Problem (\ref{eqn_obj_given_t2})).
\vspace{.03in}
\hrule
\vspace{0.1in}
\noindent {\bf Initialization:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[1.] Let $w_{ih}[t] \!=\! s_{ih}[t] \!=\! 0$, $\forall h$. Let $h\!=\!1$.
% Pick some $G$.
Pick some $\delta\in(0,1]$.
Let $G_\delta$ be defined as in Eq.~(\ref{eqn_Thm3_G}) or Eqn~(\ref{eqn_Thm4_G}).
Let $D \!=\! \lceil v(\tau_{i} + 2g_{i}\gamma_i/(b_{i}^{(i)}F_i)) \rceil$.
Let $h^* = \varnothing$. Let {\em cost-min}$=\infty$.
Choose some integer $S \geq 1$.
%Let $S = (1 - \delta/3)^{-1}$ if $G_\delta$ is defined as in Eq.~\eqref{eqn_Thm3_G} or $S = (1 - \delta/3m)^{-1}$ if $G_\delta$ is defined as in Eq.~\eqref{eqn_Thm4_G}.
Let $iter\leftarrow1$.
\end{enumerate}
\noindent {\bf Handling Internal Communication:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[2.] Sort machines in $\mathcal{H}$ according to $\sum_{r\in\mathcal{R}} p_h^r[t](\alpha_i^r\gamma_i+\beta_i^r)$ in non-decreasing order into $h_1,h_2,...,h_H$.
\item[3.] Calculate the minimum number of $s_{ih}[t]=V_i[t] \Big(\tau_i+\frac{2g_i\gamma_i}{b_i^{(i)}F_i} \Big)/\gamma_i$.
\item[4.] If Constraint (\ref{eqn_max_workers_datachunk}) is not satisfied, go to Step 7.
\item[5.] If Constraint (\ref{eqn_packing}) is not satisfied, go to Step 7.
\item[6.] Return {\em cost-min}$\sum_{r\in\mathcal{R}} p_h^r[t]s_{ih}[t](\alpha_i^r\gamma_i+\beta_i^r)$ and $h^* = h$.
\item[7.] Let $h\leftarrow h+1$. If $h>H$, stop; otherwise, go to Step 2.
%\item[2.] If constraint (\ref{eqn_packing}) is not satisfied, go to Step 7.
%
%\item[3.] If $w_{ih}[t] < D$, $w_{ih}[t] \leftarrow w_{ih}[t]+1$ and go to Step 2.
%
%\item[4.] If $w_{ih}[t] > K_{i}$, go to Step 7.
%
%%\item[5.] If $w_{ih}[t] b_{i}^{(i)} \!\!>\!\! s_{ih}[t] B_{i}$, $s_{ih}[t] \!\leftarrow\! s_{ih}[t]\!+\!1$ and go to Step 2.
%
%\item[6.] If {\em cost-min} $>p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}^{r}[t])$, then let {\em cost-min}$=p_{h}^{r}[t](\alpha_{i}^{r} w_{ih}[t] + \beta_{i}^{r} s_{ih}^{r}[t])$ and $h^* = h$.
%
%\item[7.] Let $h\leftarrow h+1$. If $h>H$, stop; otherwise, go to Step 1.
\end{enumerate}
\noindent {\bf Handling External Communication:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[8.] Solve the linear programming relaxation of Problem~(\ref{eqn_obj_given_t3}).
%with $m_{w}[t] \!\!=\!\! \ceil{v M_{i}(\tau_{i} \!+\! 2g_{i}\gamma_i/(b_{i}^{(e)}F_i))}$.
%, $m_{p}[t] \!=\! \ceil{m_{w}[t] b_{i}^{(e)}/B_{i}}$.
Let $\{ \bar{w}_{ih}[t], \bar{s}_{ih}[t], \forall h,t \}$ be the fractional optimal solution.
\item[9.] Let $w'_{ih}[t] = G_\delta\bar{w}_{ih}[t], s'_{ih}[t] = G_\delta\bar{s}_{ih}[t]$, $\forall h,t$.
\item[10.] Generate an integer solution $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$ following the randomized rounding scheme in (\ref{eqn_rounding_w})--(\ref{eqn_rounding_s}).
%\item[11.] If $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$ is infeasible, go to Step 10.
\item[11.] If $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$ is infeasible or $iter< S$, then $iter\leftarrow iter+1$,
go to Step 10.
\end{enumerate}
\noindent {\bf Final Step:}
\begin{enumerate} [topsep=1pt, itemsep=-.1ex, leftmargin=.2in]
\item[12.] Compare the solutions between internal and external cases. Pick the one with the lowest cost among them and return the cost and the corresponding schedule $\{ w_{ih}[t], s_{ih}[t], \forall h,t \}$.
\end{enumerate}
\smallskip
\hrule
\medskip
In the internal communication part of Algorithm~4, we first sort the machines and then check each machine one by one (Step 2).
We calculate the minimum number of $s_{ih}[t]$ needed to satisfy the learning workload demand $D$ (Step 3).
If Constraint (\ref{eqn_max_workers_datachunk}) is satisfied (Step 4), we further check the resource capacity constraint (\ref{eqn_rsrc_limit}) (Step 5).
%{\color{red}If the resource capacity constraint (\ref{eqn_packing}) is satisfied (Step 2), we increase workers to satisfy the learning workload demand $D$ (Step 3) and also increase parameter servers accordingly (Step 5).}
%{\color{red}If we detect a machine with lower cost, we update the cost and schedule accordingly (Step 6).}
If we detect a machine with all above constraints satisfied, we return the cost and schedule accordingly (Step 6).
After exploring one machine, we move on to the next one as long as it is not the last machine (Step 7).
The external communication part is based on LP relaxation (Step 8) and randomized rounding (Step 9-12).
Note that the randomized rounding will find at most $S$ integer feasible solutions (Step 12).
Finally, we choose the lowest cost among the solutions from the internal and external communication parts (Step 13).
\subsection{Theoretical Performance Analysis} \label{subsec:performance_analysis}
We now examine the competitive ratio of our PD-ORS algorithm.
Note that the key component in PD-ORS is our proposed randomized rounding scheme in (\ref{eqn_rounding_w})--(\ref{eqn_rounding_s}), which is in turn the foundation of Algorithm~1.
Thus, we first prove the following results regarding the randomized rounding algorithm.
Consider an integer program with generalized cover/packing constraints: $\min \{ \c^{\top} \x : \A\x \geq \a, \B\x \leq \b, \x \in \mathbb{Z}_{+}^{n} \}$, where $\A \in \mathbb{R}_{+}^{m\times n}$, $\B \in \mathbb{R}_{+}^{r\times n}$, $\a \in \mathbb{R}_{+}^{m}$, $\b \in \mathbb{R}_{+}^{r}$, and $\c \in \mathbb{R}_{+}^{n}$.
Let $\bar{\x}$ be a fractional optimal solution.
Consider the randomized rounding scheme: Let $\x' = G_\delta\bar{\x}$ for some $G_\delta$ (to be specified).
Randomly round $\x'$ to $\hat{\x} \in \mathbb{Z}_{+}^{n}$ as:
$\hat{x}_{j} = \lceil x'_{j} \rceil$ w.p. $x'_{j} - \lfloor x'_{j} \rfloor$ and $\hat{x}_{j} = \lfloor x'_{j} \rfloor$ o.w.
Note that in the rounding process, if $G_\delta > 1$ ($G_\delta \in (0,1]$), the packing (cover) constraint is prone to be violated and the cover (packing) constraint is easier to be satisfied.
Hence, depending on which constraint is more preferred to be feasible, we consider two cases with respect to $G_\delta$.
1) $0<G_\delta\leq1$: We have the following approximation result (see proof in Appendix~\ref{appdx:lem_ILP1}):
\begin{lem}[Rounding] \label{lem_ILP}
Let $W_{a} \!\!\triangleq\!\! \min \{ a_{i}/[\A]_{ij}\!: [\A]_{ij} \!\!>\!\! 0 \}$ and $W_{b} \!\!\triangleq\!\! \min \{ b_{i}/[\B]_{ij}\!: [\B]_{ij} \!\!>\!\! 0 \}$.
Let $\delta \in (0,1]$ be a given constant and define $G_\delta$ as:
\begin{align*}
G_\delta\triangleq1+ \frac{3\ln(3r/\delta)}{2W_b} - \sqrt{ \bigg(\frac{3\ln(3r/\delta)}{2W_b} \bigg)^{2} + \frac{3\ln(3r/\delta)}{W_b} }.
\end{align*}
Then, with probability greater than $1-\delta$, $\hat{\x}$ achieves a cost at most $\frac{3G_\delta}{\delta}$ times the cost of $\bar{\x}$.
Meanwhile, $\hat{\x}$ satisfies
% and $\hat{\x}$ satisfies
$\mathrm{Pr}\Big\{ (\A\hat{\x})_{i} \leq a_{i} (1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}})G_\delta, \exists i \Big\} \leq \frac{\delta}{3m}$.
\end{lem}
%\begin{figure}[t]
% \centering
% \includegraphics[width=0.9\columnwidth]{Figures/GdeltaStudy.eps}
% \caption{The feasibility study.}
% \label{fig:Gdelta}
%\end{figure}
\begin{rem}[Discussions on Feasibility]
{\em
An insightful remark of Lemma~\ref{lem_ILP} is in order.
Note that, theoretically, the expression $1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$ in Lemma~\ref{lem_ILP} could become negative.
In this case, the last probabilistic inequality in Lemma~\ref{lem_ILP} trivially holds and is not meaningful in characterizing the feasibility, even though the inequality remains valid.
In order for the probabilistic statement to be meaningful in characterizing the feasibility of the integer linear program, we solve for $\delta$ by enforcing $1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}} > 0$, which yields $\delta\geq3m /e^\frac{G_\delta W_a}{2}$.
On the other hand, we prefer $\delta$ to be small since it bounds the feasibility violation probability and approximation ratio achievability in Lemma~\ref{lem_ILP}.
Hence, the smaller the value of $3m /e^\frac{G_\delta W_a}{2}$, the less restrictive the condition $\delta\geq3m /e^\frac{G_\delta W_a}{2}$ is.
%($m$ becomes 1 and $W_a$ becomes $W_1$ in Theorem~\ref{thm_Alg4_1}).
% However, it will not impact the competitive ratio of Algorithm~1 since we conduct rounding until it finds the feasible solution.
%However, we note that the results in Lemma~\ref{lem_ILP} and Theorem~\ref{thm_Alg4_1} are still valid even though they are not useful in characterizing the feasibility when it is negative.
\begin{figure}[t]
\centering
\includegraphics[width=0.9\columnwidth]{Figures/GdeltaStudy.eps}
\caption{The feasibility study.}
\label{fig:Gdelta}
\end{figure}
To gain a deeper understanding on how restrictive the condition $\delta\geq 3m /e^\frac{G_\delta W_a}{2}$ is, we conduct a case study and the results are illustrated in Fig.~\ref{fig:Gdelta}.
Here, we let RHS $\triangleq3m /e^\frac{G_\delta W_a}{2}$ for convenience.
We vary $\delta$ from $0.02$ to $0.1$.
Clearly, the left-hand-side (LHS) of the condition is the $45^{\circ}$ straight line.
In order for the condition to hold, the curve of RHS should fall under this straight line.
Based on typical computing cluster parameters, we set $W_b$ to $15$, and set $r\triangleq RH+1$ to $401$ ($R=4, H=100$).
We can see from Fig.~\ref{fig:Gdelta} that as $W_a$ increases, the curve of RHS crosses the dashed line of LHS at a smaller $\delta$-value.
This means that, the larger the value of $W_a$, the easier for $1- (\frac{2}{G_\delta W_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$ to become positive.
Hence, the probabilistic feasibility characterization in Lemma~\ref{lem_ILP} is useful for typical system parameters in practice.
% Moreover,
% since $G\!\lessapprox\!1$, and
% $m$ becomes 1 and $W_a$ becomes $W_1$ in Theorem~\ref{thm_Alg4_1},
% which in practice cannot be small:
% due to the slow growth rate of $\ln(\frac{3}{\delta})$, the RHS becomes negative most likely when $W_a$ is small.
% However, $W_a$ (becomes $W_2$ in Theorem~2) in practice cannot be small:
% It can be seen in Theorem~2 that $W_2$ depends on the global batch size $F_i$ and several ratios between the system resource capacities and unit resource demands.
% which represents the number of workers should be allocated to the accepted job in each time slot.
% In practice, the value of $W_1$ is at least in 10s when typical system parameters are plugged in, implying that its positivity usually holds.
}
\end{rem}
2) $G_\delta>1$: We have the following approximation result (see proof in Appendix~\ref{appdx:lem_ILP2}):
\begin{lem}[Rounding] \label{lem_ILP2}
Let $W_{a}\triangleq \min \{ a_{i}/[\A]_{ij}: [\A]_{ij}> 0 \}$ and $W_{b}\triangleq \min \{ b_{i}/[\B]_{ij}: [\B]_{ij}> 0 \}$.
Let $\delta \in (0,1]$ be a given constant and define $G_\delta$ as:
\begin{align*}
G_\delta \triangleq 1 + \frac{\ln(3m/\delta)}{W_a} + \sqrt{ \bigg(\frac{\ln(3m/\delta)}{W_a} \bigg)^{2} + \frac{2\ln(3m/\delta)}{W_a} }.
\end{align*}
Then, with probability greater than $1-\delta$, $\hat{\x}$ achieves a cost at most $\frac{3G_\delta}{\delta}$ times the cost of $\bar{\x}$.
Meanwhile, $\hat{\x}$ satisfies
% and $\hat{\x}$ satisfies
$\mathrm{Pr} \Big\{ (\B\hat{\x})_{i} > b_{i} (1+ (\frac{3}{G_\delta W_b}\ln(\frac{3r}{\delta}))^{\frac{1}{2}})G_\delta, \exists i \Big\} \leq \frac{\delta}{3r}$.
\end{lem}
Several important remarks for Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} are summarized as follows:
\begin{list}{\labelitemi}{\leftmargin=1em \itemindent=-0.0em \itemsep=.2em}
\item[i)] Note that Alg.~4 is a randomized algorithm.
Therefore, its performance is also characterized probabilistically, and $\delta$ is used for such probabilistic characterization.
Here, it means that with probability $1 - \delta$, one achieves an approximation ratio at most $\frac{3G_\delta}{\delta}$ and a probabilistic feasibility guarantee as stated in Lemma~\ref{lem_ILP} when $0<G_\delta\leq 1$ and Lemma~\ref{lem_ILP2} when $G_\delta>1$.
% Recall that $G_\delta$ is defined as in Eqns~(\ref{eqn_L_def_1}) and (\ref{eqn_L_def}), based on which we can conclude that as $\delta$ increases, $G$ will also increase.
% However, the growth rate of $G_\delta$ is much smaller compared to that of $\delta$ due to the log functions.
% Thus $\delta$ is the dominance to control the performance ratio.
In other words, the statements mean that the probability of getting a better approximation ratio is smaller under randomized rounding (i.e., better approximation ratio $\Rightarrow$ smaller $\frac{3G_\delta}{\delta}$ $\Rightarrow$ larger $\delta$ $\Rightarrow$ smaller probability $1-\delta$).
%(the smaller the value of $\frac{3G}{\delta}$, the better the approximation ratio).
That is, the trade-off between the approximation ratio value and its achieving probability is quantified by $\delta$.
% The probability parameter $\delta$ controls the trade-off between approximation ratio and efficiency in finding a feasible rounding solution:
A larger $\delta$ implies a smaller approximation ratio, but the probability of obtaining a feasible solution of this ratio is also smaller (i.e., more rounds of rounding needed).
%This implies taking more rounds of rounding.
Interestingly, for $\delta=1$, Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} indicate that there is still {\em non-zero} probability to achieve an approximation ratio not exceeding $3G_\delta$.
\item[ii)] Note that if we pick $G_\delta\in(0,1]$, the approximation ratio $\frac{3G_\delta}{\delta}$ decreases (the smaller the approximation ratio, the better) as $\delta$ increases based on Eqn.~(\ref{eqn_L_def_1}) in Appendix~\ref{appdx:lem_ILP1}.
However, the growth rate of $G_\delta$ is slower compared to that of $\delta$ due to the $\log$ operator.
On the other hand, if we pick $G_\delta>1$, $\frac{3G_\delta}{\delta}$ decreases as $\delta$ increases according to Eqn.~(\ref{eqn_L_def}) in Appendix~\ref{appdx:lem_ILP2}.
Therefore, the approximation ratio is ultimately controlled by parameter $\delta$.
Also, the theoretical approximation ratio $\frac{3G_\delta}{\delta}$ is conservative.
%(due to the uses of Markov inequality and union bound in our proof).
Our numerical studies show that the approximation ratio performance in reality is much smaller than $\frac{3G_\delta}{\delta}$.
\item[iii)] The probabilistic guarantee of the cover constraint ($\A\x \geq \a$) when $0<G_\delta\leq 1$ and packing constraint ($\B\x \leq \b$) when $G_\delta>1$, is unavoidable and due to the fundamental hardness of satisfying both cover and packing constraints, which are of {\em conflicting} nature: Any strategy trying to better satisfy the packing constraints (multiplying a $G_\delta$-factor with $G_\delta\in (0,1]$) may increase the probability of violating the cover constraints, and the probability of violating the packing constraints may be increased otherwise.
However, the probabilistic bound here is for worst case and may be pessimistic.
\item[iv)] The results in Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} are in fact applicable for general ILP with mixed cover/packing constraints.
Hence, the results and their insights in Lemmas~\ref{lem_ILP} and \ref{lem_ILP2} could be of independent theoretical interest.
\end{list}
By specializing Lemma~\ref{lem_ILP} and Lemma~\ref{lem_ILP2} with parameters in Problem~(\ref{eqn_obj_given_t3}), we have the following approximation results for Algorithm 4.
The first result corresponds to the case where the feasibility of the resource constraint (packing) is more favored, i.e., $0<G_\delta\leq1$:
%
%\begin{figure}[t]
% \centering
% \includegraphics[width=.9\columnwidth]{Figures/jobFixedshaded}
% \caption{Total utility with increasing number of machines.}\label{fig:jobFixed}
% \vspace{-.1in}
%\end{figure}
%
%\begin{figure}[t]
% \centering
% \includegraphics[width=0.9\columnwidth]{Figures/machineFixedshaded}
% \caption{Total utility with increasing number of jobs.} \label{fig:machineFixed}
% \vspace{-.1in}
%\end{figure}
\begin{thm}[Approximation Performance of Alg.~4 When Resource Constraint Feasibility is Favored]\label{thm_Alg4_1}
Let $W_{1}\!\triangleq\! V_{i}[t] \big( \tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i}\big)$, $W_{2}\!\triangleq\!\min\{ F_i, \hat{C}_{h}^{r}[t]/\alpha_{i}^{r}, \hat{C}_{h}^{r}[t]/\beta_{i}^{r}, \forall r,h \}$, and
% $W_3\triangleq \ln(3(RH+1)/\delta)$.
$\delta \in (0,1]$. Define $G_\delta$ as:
\begin{align} \label{eqn_Thm3_G}
&\nonumber G_\delta\!\triangleq\! 1 \!+\! \frac{3\ln(3(RH\!+\!1)/\delta)}{2W_2}\\
&\!-\! \sqrt{ \bigg(\frac{3\ln(3(RH+1)/\delta)}{2W_2} \bigg)^{2} \!+\! \frac{3\ln(3(RH\!+\!1)/\delta)}{W_2} }.
\end{align}
Then, with probability greater than $1\!-\!\delta$, Algorithm~4 obtains a schedule $\{ w_{ih}[t], s_{ih}[t], \forall t,h \}$ that has an approximation ratio at most $\frac{3G_\delta}{\delta}$ with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_cover1})} \!\leq\! W_1(1- (\frac{2}{G_\delta W_1}\ln(\frac{3}{\delta}))^{\frac{1}{2}})G_\delta, \exists i\} \leq \frac{\delta}{3}$.
\end{thm}
% Note that $1- (\frac{2}{GW_a}\ln(\frac{3m}{\delta}))^{\frac{1}{2}}$ could be negative in Lemma~\ref{lem_ILP}.
% However, the results in Lemma~\ref{lem_ILP} and Theorem~\ref{thm_Alg4_1} are still valid even though they are not useful in characterizing the feasibility when it is negative.
% Moreover, since $G\!\lessapprox\!1$ and due to the slow growth rate of $\ln(\frac{3}{\delta})$, it becomes negative most likely when $W_a$ is small.
% However, $W_a$ (becomes $W_2$ in Theorem~\ref{thm_Alg4_1}) in practice cannot be small: It can be seen in Theorem~\ref{thm_Alg4_1} that $W_2$ depends on the global batch size $F_i$ and several ratios between the system resource capacities and unit resource demands.
% These values are in the range of 10s to 100s when typical system parameters are plugged in, implying that the positivity usually holds in practice.
The next result corresponds to the case where the feasibility of the workload constraint (cover) is more favored, i.e., $G_\delta > 1$:
\begin{thm}[Approximation Performance of Alg.~4 When Workload Constraint Feasibility is Favored]\label{thm_Alg4_2}
Let $W_{1}\triangleq V_{i}[t] \big( \tau_{i} + \frac{2g_{i}\gamma_i}{b_{i}^{(e)}F_i}\big)$, $W_{2} \triangleq \min\{ F_i, \hat{C}_{h}^{r}[t]/\alpha_{i}^{r}, \hat{C}_{h}^{r}[t]/\beta_{i}^{r}, \forall r,h \}$,
% $W_3\triangleq \ln(3(RH+1)/\delta)$.
and $\delta \in (0,1]$. Define $G_\delta$ as:
\begin{align} \label{eqn_Thm4_G}
&G_\delta \triangleq 1 + \frac{\ln(3/\delta)}{W_1} + \sqrt{ \bigg(\frac{\ln(3/\delta)}{W_1} \bigg)^{2} + \frac{2\ln(3/\delta)}{W_1} }.
\end{align}
Then, with probability greater than $1-\delta$, Algorithm~4 obtains a schedule $\{ w_{ih}[t], s_{ih}[t], \forall t,h \}$ that has an approximation ratio at most $\frac{3G_\delta}{\delta}$ with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_packing})} > \hat{C}_h^r[t] G_\delta(1+(\frac{3}{G_\delta W_2}\ln(\frac{3(HR+1)}{\delta}))^{\frac{1}{2}})\} \leq \frac{\delta}{3(HR+1)}$.
\end{thm}
Note that Eqn.~(\ref{eqn_packing2}) is guaranteed in practice since the number of samples is typically far more than the number of workers.
The competitive ratio of our online algorithm is the worst-case upper bound of the ratio of the overall utility of admitted jobs devided by the offline optimal solution of Problem~DMLRS to the total utility
of admitted jobs achieved by Algorithm~1 in the overall time horizon.
Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2} follow directly from Lemmas~\ref{lem_ILP} and \ref{lem_ILP2}, respectively, and we omit the proof here for brevity.
Based on these results, we can establish the overall competitive ratio for Algorithm~1 as follows.
\begin{thm}[Competitive Ratio of Alg~1 when $0\!<\!G_\delta\!\leq\! 1$]\label{thm_Competitive_1}
Let $\delta$, $G_\delta$ and $W_1$ be as defined in Theorem~\ref{thm_Alg4_1}.
Let $U^{r}$ and $L$ be as defined in (\ref{eqn_Ur}) and (\ref{eqn_L}), respectively.
Then, with probability greater than $(1-(\delta/3)^S)^{TK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible solution that is $\frac{6G_\delta}{\delta} \max_{r\in\mathcal{R}} (1,\ln\frac{U^{r}}{L})$--competitive.
% with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_cover1})} \!\leq\! W_1(1- (\frac{2}{G_\delta W_1}\ln(\frac{3}{\delta}))^{\frac{1}{2}})G_\delta, \exists i\} \leq \frac{\delta}{3}$.
\end{thm}
\begin{thm}[Competitive Ratio of Alg~1 when $G_\delta>1$]\label{thm_Competitive_2}
Let $\delta$, $G_\delta$ and $W_2$ be as defined in Theorem~\ref{thm_Alg4_2}.
Let $U^{r}$ and $L$ be as defined in (\ref{eqn_Ur}) and (\ref{eqn_L}), respectively.
Then, with probability greater than $(1-(\delta/3(HR+1))^S)^{TK_iE_i}$, PD-ORS in Algorithm~1 returns a feasible solution that is $\frac{6G_\delta}{\delta} \max_{r\in\mathcal{R}} (1,\ln\frac{U^{r}}{L})$--competitive.
% with $\mathrm{Pr}\{ \text{LHS(\ref{eqn_packing})} >\hat{C}_h^r[t]G_\delta(1+(\frac{3}{G_\delta W_2})^{\frac{1}{2}})\} \leq \frac{\delta}{3(HR+1)}$.
\end{thm}
It is worth pointing out that in Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}, the feasibility achieving probability values, i.e., $(1-(\delta/3)^S)^{TK_iE_i}$ and $(1-(\delta/3(HR+1))^S)^{TK_iE_i}$, can controlled by choosing appropriate values of $\delta$ and $S$ (i.e., rounds of rounding) to offset the impact of total number of DP iterations $TK_i E_i$.
The smaller $\delta$ and the larger $S$ are, the higher the feasibility achieving probability.
Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2} can be proved by weak duality and the approximation results in Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2}.
We provide a proof in Appendix~\ref{appdx:thm_Competitive}.
% due to space limitation.
\begin{thm}[Polynomial Running Time]\label{thm_Time}
% {\color{red}{Let $\delta$ be as defined in Theorems~\ref{thm_Alg4_1} when $0<G_\delta\leq1$ and \ref{thm_Alg4_2}} when $G_\delta>1$}.
%Let $\delta\in(0,1)$.
By combining Algorithms~1--4, % {\color{red}{with probability greater than $1-\delta$}},
the time complexity of PD-ORS is
% $O(\frac{1}{\delta}TK_{i}^{2}E_{i}^{2}H^{4})$,
$O(\sum_{i=1}^{|\mathcal{I}|} T K_i^2 E_{i}^{2} (H^{3} + S) )$,
which is polynomial.
% which is straightforward to prove by considering dynamic programming and randomized rounding.
\end{thm}
% \textcolor{red}{The proof can be found in Appendix~\ref{appdx:thm_Time}}
% The key components of Theorem~\ref{thm_Time} includes dynamic programming and randomized rounding.
\begin{proof}
When solving $\theta(t,v)$ using Algorithm~4, it takes $O(H\log H)$ iterations to sort machines in internal communication case under each time slot $t$ and looping all machines to calculate the minimum number $s_{ih}[t]$ takes $O(H)$.
Thus, it takes $(H\log H)$ time for the internal communication part in Algorithm~4.
For the external communication part in Algorithm~4, solving the LP relaxation of Problem~\eqref{eqn_obj_given_t3} can be approximately bounded $O(H^3)$ if we use a polynomial time LP solver (e.g., Vaidya's algorithm~\cite{Vaidya87:LP}).
According to Algorithm~4, the rounding time is proportional to $S$.
Hence, the running time for the external communications part is upper bounded by $O(|\mathcal{H}|^3 + S)$.
Combining the discussions above, the running time complexity of Algorithm~4 is $O(H\log H + H^3 + S) = O(H^3 + S)$.
%Hence it takes ${\color{red}O(H\log H+K_iE_iH)}$ to compute all $\theta(t,v),\forall v\in[0,K_iE_i]$.
Moreover, the number of states $(t,v)$ is $O(TK_iE_i)$ in the dynamic programming (DP) for each job $i$, and the time complexity of executing DP is $O(K_iE_i)$.
Thus, the time complexity is $O(TK_i^2E_i^2)$ in DP.
In Algorithm~1, the number of steps in the main loop is equal to the number of jobs.
Therefore, the overall running time complexity can be computed as $O(\sum_{i=1}^{|\mathcal{I}|} T K_i^2 E_{i}^{2} (H^{3} + S) )$.
%Therefore, it takes $O(TH\log H+TK_iE_iH+TK_i^2E_i^2+THV)=O(TK_iE_iH+TK_i^2E_i^2)$ for job $i$ to make a scheduling decision, and combined with randomized rounding, we establish the time complexity.
%{\color{red}{Note that $\delta$ quantifies the trade-off between the approximation ratio value of Alg.~4 and its achieving probability.
%That is, the better the approximation ratio, the smaller its achieving probability.
%Moreover, the approximation ratio value of Alg.~4 (a subroutine of PD-ORS) directly affects the overall running time of PD-ORS: The better the approximation ratio in each round of Step~2 of Alg.~1, the fewer number of iterations are needed on average.
%% in the main loop in Alg.~1 on average.
%Therefore, the time complexity of PD-ORS probabilistically depends on the subroutine Alg.~4, which can be quantified by $\delta$.}}
\end{proof}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Numerical Results}
\label{sec:numerical}
In this section, we conduct simulation studies to evaluate the efficacy of our proposed PD-ORS algorithm.
We test an ML system with jobs parameters generated uniformly at random from the following intervals: $E_{i} \in [50,200]$, $K_{i} \in [20000,500000]$,
% \textbf{the total number of samples is a bit too small},
$g_{i} \in [30,575]$ MB, $\tau_{i} \in [10^{-5},10^{-4}]$
%\textbf{it should be smaller since it is the time for computing one sample only}
time slots, $\gamma_i\in[1,10]$,
%\textbf{the max. ratio can be much smaller, e.g., 10},
$F_i\in[1,200]$. %all generated uniformly at random.
%The ML system running for $T=20-50$ time slots with $H=20$--$100$.
%$b_{i} \in [100\text{Mbps}, 4\text{Gbps}]$, and $B_{i}=[4\text{Gbps},20\text{Gbps}]$, all generated uniformly at random.
We consider four types of resources: GPU, CPU, memory, and storage.
For fair comparisons, following similar settings in~\cite{Li14:OSDI}~\cite{Chilimbi14:OSDI}~\cite{Sun17:Dorm}, we set resource demand of each worker as follows: 0--4 GPUs, 1--10 vCPUs, 2--32 GB memory, and 5--10GB storage.
We set resource demand of each parameter server as follows: 1--10 vCPUs, 2--32GB memory and 5-10GB storage.
%The resource capacities of each physical machine is set by following EC2 GPU instances P2 and G3 randomly~\cite{AmazonEC2Instances}.
The resource capacity of each physical machine is set roughly 18 times of the resource demands of a worker/PS following EC2 C5n instances~\cite{AmazonEC2Instances}. %that is, we set the capacity of each machine is relatively larger compared to each worker's/parameter server's demand of each job $i$.
We set the job arrival pattern according to the Google Cluster data~\cite{Reiss12:googlecluster}, but with normalized job arrival rates in alternating time-slots as follows: the arrival rates are $1/3$ and $2/3$ in odd and even time-slots, respectively.
% \textbf{clarify how the job rate is varied}.
% \textcolor{red}{For fair comparisons, we adopt the sigmoid utility function ~\cite{Huang15:CORA,Bao18:ML_INFOCOM}, as $f_i(t-a_i)=\frac{\theta_1}{1+e^{\theta_2(t-a_i-\theta_3)}}$, where $\theta_1\in[1,100]$, $\theta_2\in[0,6]$ and $\theta_3\in[1,15]$.}
For fair comparisons, we adopt the Sigmoid utility function ~\cite{Huang15:CORA,Bao18:ML_INFOCOM}: $u_i(t-a_i)=\frac{\theta_1}{1+e^{\theta_2(t-a_i-\theta_3)}}$, where $\theta_1\in[1,100]$ indicates the priority of job $i$, $\theta_2$ indicates how critical the time is for the job $i$, and $\theta_3\in[1,15]$ is the estimated target completion time.
We set $\theta_2=0$ for time-insensitive jobs (10\% of jobs), $\theta_2\in[0.01,1]$ for time-sensitive jobs (55\% of jobs) and $\theta_2\in[4,6]$ for time-critical jobs (35\% of jobs).
% For fair comparisons, we adopt the same sigmoid utility in ~\cite{Huang15:CORA,Bao18:ML_INFOCOM}.
%\textbf{reorder the figures according to their order of appearance in the main text}
%\begin{figure}[t!]
% \centering
% \includegraphics[width=.9\columnwidth]{Figures/jobFixedshaded}
% \caption{Total utility with increasing number of machines.}\label{fig:jobFixed}
% \vspace{-.1in}
%\end{figure}
%
%\begin{figure}[t!]
% \centering
% \includegraphics[width=0.9\columnwidth]{Figures/machineFixedshaded}
% \caption{Total utility with increasing number of jobs.} \label{fig:machineFixed}
% \vspace{-.1in}
%\end{figure}
We first compare our PD-ORS algorithm with three baseline job scheduling policies: (1) {\em FIFO} in Hadoop and Spark~\cite{Zaharia10:USENIX}, where the jobs are processed in the order of their arrival times. In our setting, the fixed number of workers (parameter servers) is between 1 to 30,
(2) {\em Dominant Resource Fairness Scheduling} (DRF) in Yarn~\cite{Vavilapalli13:Yarn} and Mesos~\cite{Hindman11:Mesos}, where the jobs are scheduled based on their dominant resource share in the cloud to achieve its max-min fairness.
The number of workers and parameter servers are computed and allocated dynamically,
and (3) {\em Dorm}~\cite{Sun17:Dorm}, where the numbers of workers (parameter servers) are computed and placed by an MILP resource utilization maximization problem with fairness and adjustment overhead constraints.
Workers and parameter servers are placed in a round-robin fashion on available machines in Baselines (1) and (2).
% {\color{red}{(1) FIFO: it is used in Hadoop and Spark [cite] and the jobs are processed in order of their arrival times.
% In our setting, the fixed number of workers (parameter servers) is between 1 to 30.
% (2) Dominant Resource Fairness Scheduling (DRF): it is used in Yarn[cite] and Mesos [~\cite] and jobs are scheduled based on their dominant resource share in the cloud to achieve its max-min fairness.
% The number of workers (parameter servers) are computed and allocated dynamically.
% (3) Dorm[cite]: The numbers of workers (parameter servers) are computed and placed by an MILP resource utilization maximization problem with fairness and adjustment overhead constraints.
% We employ a round-robin approach to place workers and parameter servers on available machines in Baselines (1) and (2).}}
The comparison results are shown in Figs.~\ref{fig:jobFixed} and~\ref{fig:machineFixed}.
In Fig.~\ref{fig:jobFixed}, we set $T=20$ and $I=50$, while in Fig.~\ref{fig:machineFixed}, we set $T=20$ and $H=100$.
% \textbf{give the following settings in producing the figures, including number of jobs in Fig.~\ref{fig:jobFixed}, number of machines in Fig ~\ref{fig:machineFixed}, time slots}.
We can see that PD-ORS significantly outperforms other policies and the gains in total utility becomes more pronounced as the numbers of jobs and machines increase.
%\textbf{but the figures show the total utility of PD-ORS is smallest. which shadowed bar is the PD-ORS case? possibly change the markers to the bars more distinghuishable}
%\begin{figure*}[t!]
%%\centering
%\begin{minipage}{0.45\textwidth}
%%\centering
%%\vspace{-1.26in}
%\includegraphics[width=1\textwidth]{Figures/jobFixedshaded}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed}
%\end{minipage}
%%
%\hspace{0.1\columnwidth}%
%%
%\begin{minipage}{0.45\textwidth}
%%\centering
%%\vspace{-1.3in}
%\includegraphics[width=1\textwidth]{Figures/machineFixedshaded}
%%\vspace{.15in}
%\caption{Total utility with increasing number of jobs.} \label{fig:machineFixed}
%\end{minipage}
%\end{figure*}
%
%
%\begin{figure*}[t!]
%\begin{minipage}{0.49\textwidth}
%\vspace{-.2in}
% \includegraphics[width=1\textwidth]{Figures/colocation.eps}
% \caption{Utility comparison between PD-ORS and OASiS.} \label{fig:UtilityComp}
%\end{minipage}
%%
%\hspace{0.02\columnwidth}%
%%
%\begin{minipage}{0.5\textwidth}
% \includegraphics[width=1\textwidth]{Figures/G2.eps}
%\vspace{-.25in}
% \caption{Impact of pre-rounding gain factor $G_\delta$ on competitive ratio.}\label{fig:Factor_ApprRatio}
%\end{minipage}
%\end{figure*}
\begin{figure*}[t!]
%\centering
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.26in}
\includegraphics[width=1\textwidth]{Figures/jobFixedshaded.png}
%\vspace{-.03in}
\caption{Total utility with increasing number of machines (synthetic data).}\label{fig:jobFixed}
\end{minipage}
%
\hspace{0.06\columnwidth}%
%
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.3in}
\includegraphics[width=1\textwidth]{Figures/machineFixedshaded.png}
%\vspace{.15in}
\caption{Total utility with increasing number of jobs (synthetic data).} \label{fig:machineFixed}
\end{minipage}
\hspace{0.06\columnwidth}%
%
\begin{minipage}{0.3\textwidth}
%\centering
\vspace{.05in}
\includegraphics[width=1.1\textwidth]{Figures/colocation.eps}
%\vspace{.15in}
\caption{Utility comparison between PD-ORS and OASiS with increasing number of jobs.} \label{fig:UtilityComp_1}
\end{minipage}
\end{figure*}
\begin{figure*}[t!]
\begin{minipage}{0.3\textwidth}
\vspace{-.2in}
\includegraphics[width=1.11\textwidth]{Figures/actual_time.eps}
%\vspace{-.25in}
\caption{Median of actual training time comparison.} \label{fig:actual_time}
\end{minipage}
\hspace{0.06\columnwidth}%
\begin{minipage}{0.3\textwidth}
\vspace{-.3in}
\includegraphics[width=1.1\textwidth]{Figures/ratio.eps}
\caption{Competitive ratio.} \label{fig:ratio}
\end{minipage}
\hspace{0.06\columnwidth}%
\begin{minipage}{0.3\textwidth}
\vspace{-.2in}
\includegraphics[width=1.1\textwidth]{Figures/G2.eps}
\caption{Impact of pre-rounding gain factor $G_\delta$ on competitive ratio.}\label{fig:Factor_ApprRatio}
\end{minipage}
\end{figure*}
%\begin{figure}[h]
%\centering
% \includegraphics[width=.4\textwidth]{Figures/G2.eps}
%%\vspace{-.25in}
% \caption{Impact of pre-rounding gain factor $G_\delta$ on competitive ratio.}\label{fig:Factor_ApprRatio}
%\end{figure}
\begin{figure*}[t!]
%\centering
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.26in}
\includegraphics[width=1\textwidth]{Figures/jobFixed_response.png}
%\vspace{-.03in}
\caption{Total utility with increasing number of machines (Google cluster data trace).}\label{fig:jobFixed_response}
\end{minipage}
%
\hspace{0.06\columnwidth}%
%
\begin{minipage}{0.3\textwidth}
%\centering
%\vspace{-1.3in}
\includegraphics[width=1\textwidth]{Figures/serverFixed_response.png}
%\vspace{.15in}
\caption{Total utility with increasing number of jobs (Google cluster data trace).} \label{fig:serverFixed_response}
\end{minipage}
\hspace{0.06\columnwidth}%
\begin{minipage}{0.3\textwidth}
%\vspace{-.2in}
\includegraphics[width=1.12\textwidth]{Figures/servervaried_1.eps}
%\vspace{-.2in}
\caption{Total utility with increasing number of machines [T=80, I=100, (10\%, 55\% , 35\%)].}\label{fig:jobFixed1}
\end{minipage}
\end{figure*}
\begin{figure*}[t!]
%\centering
%\begin{minipage}{0.22\textwidth}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%\end{minipage}
\begin{minipage}{0.3\textwidth}
\includegraphics[width=1.11\textwidth]{Figures/servervaried_2.eps}
%\vspace{.15in}
\caption{Total utility with increasing number of machines [T=80, I=100, (30\%, 69\% , 1\%)].} \label{fig:jobFixed2}
\end{minipage}
\hspace{0.03\columnwidth}%
\begin{minipage}{0.3\textwidth}
%\vspace{-.15in}
\includegraphics[width=1.11\textwidth]{Figures/jobvaried_1.eps}
\caption{Total utility with increasing number of jobs [T=80, H=30, (10\%, 55\% , 35\%)].} \label{fig:serverFixed1}
\end{minipage}
\hspace{0.03\columnwidth}%
\begin{minipage}{0.3\textwidth}
%\vspace{-.15in}
\includegraphics[width=1.25\textwidth]{Figures/jobvaried_2.eps}
%\vspace{-.2in}
\caption{Total utility with increasing number of jobs [T=80, H=30, (30\%, 69\% , 1\%)].}\label{fig:serverFixed2}
\end{minipage}
\end{figure*}
Next, we compare our PD-ORS algorithm with the OASiS algorithm in~\cite{Bao18:ML_INFOCOM}, which is also a dynamic scheduling scheme. %, which is the most related work to ours and the state-of-the-art of scheduling for distributed ML systems.
As mentioned earlier, the key difference in OASiS is that parameter servers and workers are located on two {\em strictly} separated sets of machines (i.e., no co-located workers and PSs).
Here, we let $H=100$ and $T=20$.
%{\color{blue} Let $T=80$ and $I=100$ in Fig.~\ref{fig:UtilityComp_2}.}
For OASiS, half of the machines host parameter servers and the other half host workers.
For fair comparisons, both algorithms adopt the same Sigmoid utility function.
The comparison results are shown in Fig.~\ref{fig:UtilityComp_1}.
We can see that PD-ORS outperforms OASiS by allowing co-located parameter servers and workers.
%\textbf{better discuss why the more jobs there are, the better the improvement is}
We can see from Fig.~\ref{fig:UtilityComp_1} that the performance gap between PD-ORS and OASiS widens as the number of jobs increases, which implies that PD-ORS is more scalable.
This is due to the advantage afforded by colocation of workers and parameter servers, which allows each physical machine to be fully utilized.
On the other hand, the strict separation of workers and parameter servers in OASiS may lead to the inability of placing workers on server-side machines, should there be available resources or vice verse.
%Next, we investigate the impact of $b_{i}^{(i)}/{b_{i}^{(e)}}$ on total utility value and the results are shown in Fig.~\ref{fig:IntExt_Ratio}.
%In this experiment, we let $H=20$, $T=50$, and vary $b_{i}^{(i)}/{b_{i}^{(e)}}$ from $1$ to $60$.
%We can see that the total utility rises rapidly as $b_{i}^{(i)}/{b_{i}^{(e)}}$ increases initially, which shows that PD-ORS reacts aggressively to a large $b_{i}^{(i)}/{b_{i}^{(e)}}$-value.
%On the other hand, the increase of total utility becomes more gradual when $b_{i}^{(i)}/{b_{i}^{(e)}}$ is greater than 30.
%This is because beyond this point, most jobs already have a large number of co-located parameter servers and workers.
%, which results in marginal performance improvements.
Next, we investigate the actual training time (completion time - arrival time) under different methods, where $T=80$, $H=30$ and $I=100$.
The median of the actual training time is shown in Fig.~\ref{fig:actual_time}.
Here, we simply set its training time to $T$ (i.e., 80) if the job cannot be finished within the scheduling time span $\mathcal{T}$.
%As we can see from the figure that only PD-ORS and OASiS are able to schedule all jobs within the time horizon.
As we can see from Fig.~\ref{fig:actual_time}, PD-ORS outperforms other scheduling policies, i.e., it has the smallest median time.
Also, due to the co-location advantage of PD-ORS, its median time is smaller compared to OASiS, where workers and parameter servers are placed in strictly separated sets of machines.
We expect that the difference between PD-ORS and OASiS will become more noticeable as the number of machines or the capacity of each machine increases since it will allow more co-location placements.
Next, we demonstrate the competitive ratio of our algorithm PD-ORS, which is the ratio between the total job utility of the offline optimal solution and the total job utility achieved by PD-ORS.
Recall that Problem~DMLRS is a non-convex problem with constraints (e.g., Eq.~(\ref{eqn_NumTrainedSamples})) that are not amenable to be directly solved by conventional optimization techniques. To obtain its offline optimum, all possible combinations of $w_{ih}[t], s_{ih}[t], \forall i, h, t$ need to be considered, which is time prohibitive.
%Although the optimum is hard to obtain and the actual performance ratio can not be derived, for online scheduling algorithms, competitive ratio is typically provided instead (e.g.,~[6]).''
Thus, we limit the number of jobs $I$ to 10 and time span $T$ to 10, and the result is shown in Fig.~\ref{fig:ratio}.
As we can see from the figure, the performance ratio is between 1.0 to 1.4, indicating that our proposed algorithm PD-ORS has a good competitive ratio performance.
Lastly, we examine the performance of the randomized rounding scheme in Algorithm~4, which is the key of PD-ORS.
We evaluate the rounding performance in terms of the ratio between the optimal total utility and the total utility obtained by our algorithm.
The optimal utility is computed using the Gurobi optimization solver.
We let $H=100$, $I=50$, $T=20$.
We vary the pre-rounding gain factor $G_{\delta}$ (Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2}) from $0.2$ to $1.2$.
The results are shown in Fig.~ \ref{fig:Factor_ApprRatio}. %, respectively.
The packing constraints are easier to satisfy with a smaller $G_\delta$, while the cover constraints are prone to be violated as $G_\delta$ gets smaller.
In our experiments, if the total rounds of randomized rounding before we find an integer feasible solution exceeds a preset threshold (e.g, $5000$), we will discard the corresponding job.
Theorem~\ref{thm_Alg4_1} suggests that there is a trade-off: if we set $G_\delta$ to be close to one to pursue a better total utility result, the rounding time could be large to obtain a feasible solution.
As $G_\delta$ increases, the probability of violating the packing constraints increases, meaning we need to have more rounding attempts to obtain an integer feasible solution.
However, according to our numerical experiences, if the machine's resource capacity is relatively large compared to the jobs' resource demands per worker/PS, the number of rounding attempts is small and not sensitive to the variation of $G_\delta$.
On the other hand, as $G_\delta$ decreases, the probability of violating the cover constraint increases.
However, in practice, the model usually converges with fewer number of iterations than the pre-defined training epochs since the required number of epochs is usually overestimated~\cite{Jeon18:multi-tenant}.
In other words, the violation of the cover constraint in one iteration may be acceptable.
As we can see from Fig.~\ref{fig:Factor_ApprRatio}, the best approximation ratio value is achieved when $G_\delta=1$.
This is because if $G_\delta$ approaches $0$, it implies that $\delta$ decreases at a larger rate (cf. Eq.~(\ref{eqn_Thm3_G})), resulting the increment of the performance ratio.
On the other hand, if $G_\delta$ goes to infinity, it implies $\delta$ decreases (cf. Eq.~\eqref{eqn_Thm4_G}), resulting in a much faster increment of the performance ratio.
%xx \textbf{explain why by pointing to earlier section} .
%This is because packing constraints are easier to satisfy with a smaller $G$.
%When $G>1.006$, we can see that times of rounding start to increase.
%This is because packing constraints are prone to be violated as $G$ gets large.
%We note that, in all cases, the times of rounding are less than $1000$, for which the runtime is short (especially given the size of the search space is $2^{50}$).
Also, we can see from Fig.~\ref{fig:Factor_ApprRatio} that the performance ratios for all choices of $G_\delta$ are much better than the theoretical bounds in Theorems~\ref{thm_Alg4_1} and \ref{thm_Alg4_2}, which shows that the approximation ratio is much tighter than the worse-case bound suggested in Theorems~\ref{thm_Competitive_1} and \ref{thm_Competitive_2}.
%%%%%%%%%%%%%%additional experiments results%%%%%%%%%%%%%%
%Thanks for your question.
%In the experiment, we set job arrival pattern according to the Google cluster data with varied arrival rates.
%Our goal was to spread jobs kinda evenly across the entire scheduling time horizon $\mathcal{T}$, which potentially would include all kinds of jobs, i.e., jobs that arrive early/later in the time span with small/large utility, to show our proposed algorithm performance.
%On the other hand, if we simulate job arrivals exactly based on timestamps recorded in the trace by scaling down the original job trace, traffic bustiness may occur and the experiments results may lack of representative due to its dependence on the selected ``snippets" from the trace.
%
%In addition, based on google cluster trace, the job arrival pattern is shown in Fig.~\ref{fig:pattern}, where x-axis is the time (seconds) and y-axis is the number of submitted jobs (see ref. [1] below).
%We can see from the figure that the distribution of job submissions has a repetitive and fractal pattern, which indicates a periodicity of 10, 100, 1000 and 10000.
%Note that every 10 minutes, there exists 1 minute absence of job submissions, and every 100 minutes, there exists 10 minutes absence of job submission, and so on so forth.
%However, such absence does not mean no job arrives in the data center but that no job is released by the global scheduler [1].
%Thus the job arrives somehow evenly spread over the entire time span, and our arrival rates design roughly follow the pattern.
%However, we agree that since we follow the google trace, we can simulate exactly follow its arrivals.
%We also include the comparison results in Figs.~\ref{fig:jobFixed_response} - \ref{fig:jobFixed_ratio}.
%Here, we set $T=80$, $I=100$, and we increase the number of machines from 10 to 50.
%And we set $T=80$, $H=30$, and we increase the number of jobs from 20 to 100.
%As we can see from the figures, our algorithm PD-ORS outperforms other scheduling policies.
%Similarly, due to the advantage of co-location in PD-ORS, it achieves more total job utility compared to OASiS.
%\begin{figure*}[h]
%\centering
%\begin{minipage}{0.3\textwidth}
%%\centering
%%\vspace{-1.26in}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_response.eps}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed_response}
%\end{minipage}
%%
%%\hspace{0.1\columnwidth}%
%%
%\begin{minipage}{0.3\textwidth}
%%\centering
%%\vspace{-1.3in}
%\includegraphics[width=1\textwidth]{response/Figures/jobvaried_response.eps}
%%\vspace{.15in}
%\caption{Total utility with increasing number of jobs.} \label{fig:serverFixed_response}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.2in}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%\vspace{-.03in}
%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%\end{minipage}
%\end{figure*}
%
%\begin{figure*}[h]
%\centering
%%\begin{minipage}{0.22\textwidth}
%%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%%\vspace{-.03in}
%%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_2.eps}
%%\vspace{.15in}
%\caption{Total utility with increasing number of machines.} \label{fig:jobFixed2}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_1.eps}
% \caption{Total utility with increasing number of jobs.} \label{fig:serverFixed1}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_2.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing number of jobs.}\label{fig:serverFixed2}
%\end{minipage}
%\end{figure*}
%\begin{figure*}[t!]
%\begin{minipage}{0.5\textwidth}
%%\vspace{-.2in}
% \includegraphics[width=1\textwidth]{Figures/jobvaried_ratio_response.eps}
% \caption{Total utility with increasing number of jobs.} \label{fig:serverFixed_ratio}
%\end{minipage}
%%
%%\hspace{0.02\columnwidth}%
%%
%\begin{minipage}{0.5\textwidth}
% \includegraphics[width=1\textwidth]{Figures/servervaried.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing number of machines.}\label{fig:jobFixed_ratio}
%\end{minipage}
%\end{figure*}
Next, we show further experimental results with real-world data traces.
%In the experiment, we set job arrival pattern according to the Google cluster data with varied arrival rates.
%Our goal was to spread jobs kinda evenly across the entire scheduling time horizon $\mathcal{T}$, which potentially would include all kinds of jobs, i.e., jobs that arrive early/later in the time span with small/large utility, to show our proposed algorithm performance.
We first compare our PD-ORS algorithm with baseline scheduling algorithms, where we follow job arrivals exactly based on timestamps recorded in the Google Cluster data~\cite{Reiss12:googlecluster} by scaling down the original job trace (i.e., a ``snippet'' of the trace).
Here, we set $T=80$, $I=100$ and $H=30$.
%We increase the number of machines from 10 to 50 with $I=100$.
%And we increase the number of jobs from 20 to 100 with $H=30$.
The comparison results are shown in Figures~\ref{fig:jobFixed_response} - \ref{fig:serverFixed_response}.
%Here, we set $T=80$, $I=100$, and we increase the number of machines from 10 to 50.
%And we set $T=80$, $H=30$, and we increase the number of jobs from 20 to 100.
Similarly, as we can see from the figures, our algorithm PD-ORS outperforms other scheduling policies.
In addition, due to the co-location advantage of PD-ORS, it achieves more total job utility compared to OASiS.
In the previous experiments, we have set the portions for time-insensitive jobs, time-sensitive jobs and time-critical jobs to 10\%, 55\% and 35\%, respectively, which follows the default setting in~\cite{Bao18:ML_INFOCOM} for fair comparison.
Theoretically, the larger the portion of time-sensitive and time-critical jobs is, the better the performance is in terms of job utility compared to other scheduling policies.
Based on the Google trace analysis~\cite{Minet18:google_trace}, there are four categories of scheduling class of a job to indicate the latency sensitivity of the job, where we label class 0 as time-insensitive, Classes 1 and 2 as time-sensitive, and Class 3 as time-critical.
%The portion of each class is depicted in Table~\ref{table:class}.
%We can observe from the results that most jobs are time critical.
%Here, we label class 0 as time-insensitive, class 1 and 2 as time-sensitive, and class 3 as time-critical.
In order to follow the practical setting in the trace, we roughly set the ratio to 30\%, 69\% and 1\%.
We set $T=80$.
The number of machines increases from $10$ to $50$ with the number of job fixed to $100$, and the number of jobs increases from $20$ to $100$ with the number of machines fixed at $30$.
We let Figs.~\ref{fig:jobFixed1} and \ref{fig:serverFixed1} follow the previous ratio setting (i.e., 10\%, 55\% and 35\%), and Figs.~\ref{fig:jobFixed2} and \ref{fig:serverFixed2} follow the revised ratio setting (i.e., 30\%, 69\% and 1\%).
We examine the utility gain compared to OASiS, where it is normalized.
We present our experimental results in Figs.~\ref{fig:jobFixed1} -- \ref{fig:serverFixed2}.
%In Figs.~\ref{fig:jobFixed1} and \ref{fig:jobFixed2}, we set $T=80, I=100$, and increases number of servers from 10 to 50.
%In Figs.~\ref{fig:serverFixed1} and \ref{fig:serverFixed2}, we set $T=80, H=30$, and increases number of jobs from 20 to 100.
%We can see that similar results are obtained.
We can see from the figures that as the portion of critical jobs decreases by 34\%, the utility gain becomes smaller.
That is, the advantage of our proposed algorithm PD-ORS becomes less prominent.
We note that although in theory we can compare our PD-ORS algorithm (a special case with utility function $u(x)=x$) with Optimus in~\cite{Peng18:ML_EuroSys} (which also takes co-location into consideration), it is not straightforward to do so in practice.
Optimus requires an offline stage to estimate the $\theta$-parameters of the speed function $f(p_j, w_j)$ (cf. $\theta_{0}$--$\theta_{3}$ in [\!\!\cite{Peng18:ML_EuroSys}, Eq.~(3)] for asynchronous training and $\theta_{0}$--$\theta_{4}$ for synchronous training in [\!\!\cite{Peng18:ML_EuroSys}, Eq.~(4)]).
Estimating these parameters requires specific hardware and software packages that are not available in our current experimental environment.
Due to the above computing resource limitations and time constraints, we are unable to conduct experiments to directly compare PD-ORS and Optimus in this work.
Also, our focus in this work is on scheduling algorithmic designs for deep learning training with main contributions being on the theoretical aspects (proving rigorous scheduling performance guarantees).
Implementing our PD-ORS algorithm in a similar testbed environment and having a comparison with Optimus is very interesting and will be our next step in future studies.
%\begin{figure*}[h]
%\centering
%%\begin{minipage}{0.22\textwidth}
%%\includegraphics[width=1\textwidth]{response/Figures/servervaried_1.eps}
%%%\vspace{-.03in}
%%\caption{Total utility with increasing number of machines.}\label{fig:jobFixed1}
%%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%\includegraphics[width=1\textwidth]{response/Figures/servervaried_2.eps}
%%\vspace{.15in}
%\caption{Total utility with increasing number of machines.} \label{fig:jobFixed2}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_1.eps}
% \caption{Total utility with increasing no. of jobs.} \label{fig:serverFixed1}
%\end{minipage}
%\begin{minipage}{0.3\textwidth}
%%\vspace{-.15in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_2.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing no. of jobs.}\label{fig:serverFixed2}
%\end{minipage}
%\end{figure*}
%\begin{figure*}[h]
%\centering
%\begin{minipage}{0.4\textwidth}
%%\vspace{-.2in}
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_1.eps}
% \caption{Total utility with increasing no. of jobs.} \label{fig:serverFixed1}
%\end{minipage}
%%
%%\hspace{0.02\columnwidth}%
%%
%\begin{minipage}{0.4\textwidth}
%%\centering
% \includegraphics[width=1\textwidth]{response/Figures/jobvaried_2.eps}
%%\vspace{-.2in}
% \caption{Total utility with increasing no. of jobs.}\label{fig:serverFixed2}
%\end{minipage}
%\end{figure*}
% !TEX root = ../ML_Networking_TNSE20.tex
\section{Conclusion}
\label{sec:conclusion}
In this paper, we investigated online resource scheduling for distributed machine learning jobs in a shared computing cluster.
We considered the most general setting that workers and parameter servers can be co-located on the same set of physical machines.
We showed that this problem can be formulated as a challenging integer nonlinear programming problem with {\em non-deterministic} constraints.
We developed an efficient online scheduling algorithm with competitive ratio guarantee.
Our main contributions are three-fold:
i) We developed a new analytical model that jointly considers resource locality and allocation;
ii) Through careful examinations of worker-server configuration relationships, we resolved the locality ambiguity in the model and reduce the problem to a mixed cover/packing integer program that entails low-complexity approximation algorithm design;
iii) We proposed a meticulously designed randomized rounding algorithm to solve the mixed cover/packing integer program and rigorously established its approximation ratio guarantee.
Collectively, our results expand the theoretical frontier of online optimization algorithm design for resource optimization in distributed machine learning systems.
%>From snoeyink%cs.ubc.ca@cs.Technion.AC.IL Wed Nov 3 08:18:27 1993
%Received: from csmail.cs.technion.ac.il by math.tau.ac.il (5.67/math.tau-930921)
%id AA17625; Wed, 3 Nov 93 08:18:23 +0200
%Received: from csc (csc.cs.technion.ac.il) by csmail.cs.Technion.AC.IL (4.1/3.14)
%id AA14244; Wed, 3 Nov 93 01:19:10 IST
%Received: from csa by csc (4.1/3.14)
%id AA24688; Wed, 3 Nov 93 01:19:48 IST
%Received: from grolsch.cs.ubc.ca by csa (4.1/3.14)
%id AA20525; Wed, 3 Nov 93 01:11:28 IST
%Return-Path: <snoeyink@cs.ubc.ca>
%Received: from cascade.cs.ubc.ca by grolsch.cs.ubc.ca with SMTP id AA13516
%(5.65c/IDA-1.3.5 for alon@cs.Technion.AC.IL); Tue, 2 Nov 1993 13:56:34 -0800
%Date: Tue, 2 Nov 1993 13:56:34 -0800
%From: Jack Snoeyink <snoeyink%cs.ubc.ca@cs.Technion.AC.IL>
%Message-Id: <199311022156.AA13516@grolsch.cs.ubc.ca>
%Received: by cascade.cs.ubc.ca (4.1/SMI-4.1)
%id AA21873; Tue, 2 Nov 93 13:56:33 PST
%To: alon@cs.Technion.AC.IL
%In-Reply-To: Alon Efrat's message of Sun, 31 Oct 93 23:38:48 IST <9310312138.AA19880@csc>
%Subject: LaTex Macros
%Status: RO
%Content-Length: 19065
%X-Lines: 638
%
%
%I'm in oberwolfach w/ micha, so I';ll just send you this and answer
%your questions on it when I get back to vancouver next week...
%
%o Jack
%_/\_.
%(')>-(`) snoeyink@cs.ubc.ca
%----figbox.tex----
% 12 Oct 90 LaTeX figure boxing macros by Jack Snoeyink
% w/ ideas from John Hershberger
% and code from Tom Rokicki
%%%%%%%%%%%%%%%%%
%
% This file contains two classes of macros for dealing with figures in
% Latex documents:
% \figbox (and its relatives) take a box containing a figure or table
% or whatever, caption it, and either place it on the page and wrap the
% text around it or let it float (as usual).
% \boxeps and \boxps take encasulated or plain postscript from a file
% and put it in a box for use by \figbox
%
%%%%%%%%%%%%%%%%
%
% The most important macro for dealing with boxes is:
%
% \figbox: caption a box as a figure or table and place it in the margin
% (a marginal box) or float it (a floating box).
% Wide marginal boxes are promoted to one column floats; wide one
% column floats are promoted to two column floats.
%
% Usage: \figbox [*htblr] {box} {style} {label} {caption}
%
% The optional argument default is [htb]. Other choices
% * two column (must come first),
% h here, t top, b bottom (as in latex),
% l left, r right (marginal boxes that do not float)
%
% NB: Left and right marginal boxes will not overlap in height.
%
% Two column figures default to htb unless some other
% options follow the *.
% They can also be specified by style {figure*} or {table*}.
%
% box is a box containing a figure or table.
% style is the caption style {figure}, {figure*}, {table}, or {table*}.
% label is the label assigned.
% if empty, the figure will not be labeled.
% caption is of the form {}, {Caption} or {[short caption] Long caption},
% where the short caption goes into the list of figures/tables.
% if empty, the figure will not be captioned or labeled.
%
% The lower level macro \captionbox, described much later, can be used
% to combine two or more captioned figures into a single marginal or
% floating box.
%
%%%%%%%%%%%%%%%%
%
% options defined by macros are: \captionstyle \boxcaptionstyle
% \maxfigfraction \figboxhang \figboxmargin
%
% \captionstyle is executed for all floating box captions
% \boxcaptionstyle is executed for all marginal box captions.
% Default \raggedright
% \maxfigfraction is the maximum fraction of a column that can be taken up
% by a marginal box before it gets converted to a float.
% Default .6
% dimen \figboxhang is the distance a marginal box hangs into the margin.
% Default 0pt
% dimen \figboxmargin is the width of space between the text and a
% marginal box.
% Default .15in
%
%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%
%
% The most important macros for turning postscript into boxes are:
%
% \boxeps: Form a box containing encapsulated postscript from a file at a
% given scale.
% \boxps: Form a box containing postscript from a file at a given scale,
% width, and height.
%
%
% Usage: \boxeps[local commands] {file} {scale}
% \boxps[local commands] {file} {scale} {width} {height}
%
% The optional local commands are executed after scaling. As described
% below, they are useful for setting origins, non-uniform scaling, etc?
% file is the filename in the \figuredirectory and without the .ps
% extension.
% scale is a real number: 1 to use the drawn size. 0.9 to use 90% of
% drawn size. Can be changed by local commands. (see below)
% width and height are dimensions of the box produced by \boxps.
%
%%%%%%%%%%%%%%%%
%
% options: \DVIscaling \figuredirectory \missingfigure \boxer
% local commands: \horizscale \vertscale
%
% \DVIscaling is the number the dvi to ps converter considers a unit
% scaling transformation. Default: 1 for dvi2ps --- 100 for
% some of Rokicki's versions of dvips.
%
% \globalscaling is the scaling applied to every picture --- useful
% for making double size figures for journals. Default: 1
%
% \figuredirectory is the pathname where figure postscript files are kept.
% Default: ./figure
%
% \missingfigure{file} is executed when the file is not found.
% Default: \def\missingfigure#1{\hbox{Missing figure #1.ps}}
%
% \boxer is the command used by \boxps to make a box of the desired size
% containing the figure. Possible commands depend on the origin of
% the figure, and include \llboxer \lcboxer \ulboxer and \ccboxer.
% (ll = lower left, ul = upper left, cc = dead center, etc.)
% \boxer can be defined in the local commands or globally.
%
% Local commands \horizscale, and \vertscale can scale the picture
% horizontally and vertically by different factors. \scale
% scales the picture uniformly. Scales accumulate.
%
%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%
%
% Default options
%
\def\captionstyle{}
\def\boxcaptionstyle{\raggedright}
\def\maxfigfraction{.6}% --- max frac of column for marg figure
\newdimen\figboxmargin% --- margin between text and picture
\figboxmargin = .15in
\newdimen\figboxhang% --- amount to hang picture into margin
\figboxhang = 0pt
\def\DVIscaling{1}% --- for dvi2ps
\def\globalscaling{1}% --- for scaling everything
\def\figuredirectory{./figures}%--- where to find the .ps files
\let\boxer=\llboxer% --- default origin in lower left for non-
% encapsulated postscript figures
\def\missingfigure#1{\hbox{Missing figure #1.ps}}
%%%%%%%%%%%%%%%%
%
% Use latex internals
%
\catcode `\@=11
\special{! /@scaleunit 1 def }
\newbox\figurebox
\def\figbox{% [place] box style label caption
\@ifnextchar[{\figboxaux}{\figboxaux[htb]}}
%
% Parse position parameters and try appropriate placement
%
\long\def\figboxaux[#1#2]#3#4#5#6{% --- want left margin figure?
\writepict{{#3}{#4}{#5}{#6}}% --- save figure if journalpicts
\setbox\figurebox\hbox{#3}%
\if l#1\tryleftbox{#4}{#5}{#6}%
\else% --- want right margin figure?
\if r#1\tryrightbox{#4}{#5}{#6}%
\else% --- want one or two column figure?
\if *#1\checktwocoloptions#2]{\box\figurebox}{#4*}{#5}{#6}%
\else\tryonecol[#1#2]{#4}{#5}{#6}%
\fi
\fi
\fi\ignorespaces}
%
% Try a placement; promote if too wide
%
\long\def\tryleftbox#1#2#3{% --- does left margin figure fit?
\ifdim\wd\figurebox>\maxfigfraction\columnwidth \tryonecol[htb]{#1}{#2}{#3}%
\else\leftbox{\captionbox{\box\figurebox}{#1}{#2}{#3}}\fi}
\long\def\tryrightbox#1#2#3{% --- does right margin figure fit?
\ifdim\wd\figurebox>\maxfigfraction\columnwidth \tryonecol[htb]{#1}{#2}{#3}%
\else\rightbox{\captionbox{\box\figurebox}{#1}{#2}{#3}}\fi}
\def\checktwocoloptions{% --- make sure twocol has htb or p
\@ifnextchar]{\floatbox[htb}{\floatbox[}}
\long\def\tryonecol[#1]#2#3#4{% --- does one column figure fit?
\ifdim\wd\figurebox>\columnwidth \floatbox[#1]{\box\figurebox}{#2*}{#3}{#4}%
\else\floatbox[#1]{\box\figurebox}{#2}{#3}{#4}\fi}
%
% Do the work for a floating box.
%
% \floatbox[place]{box} {style} {label} {caption}
%
\long\def\floatbox[#1]#2#3#4#5{%
\begin{#3}[#1]
\hbox to \hsize{\hfil#2\hfil}
\captionandlabel{#3}{#4}{#5}
\end{#3}
}
%
% Make a caption the width of the box with given style.
%
%\captionbox {box} {style} {label} {[optshortcap] caption}
%
% options: \boxcaptionstyle
%
%%%%%%%%%%%%%%%%
%
% This can be used with box commands (\hbox, \vbox, \mbox, and the like),
% along with \figbox with an empty caption, to place two or more captioned
% boxes in a single marginal or floating box.
%
% e.g. \figbox{\hbox{
% \captionbox{ \boxeps{fig1}1} {figure} {f1} {a figure}
% \hskip 1in
% \captionbox{ \boxeps{tab1}1} {table} {t1} {a table}
% }} {figure} {ignored-label} {}
%
\long\def\captionbox#1#2#3#4{% box style label {[optshortcap] caption}
\setbox\figurebox\hbox{#1}%
\parbox[t]{\wd\figurebox}{%
\bigskip\box\figurebox
\let\captionstyle=\boxcaptionstyle
\captionandlabel{#2}{#3}{#4}
\bigskip
}}
% --- caption and label if label is nonempty
\def\captionandlabel#1#2#3{% --- {style} {label} {caption}
\def\testit{#3}%
\ifx\testit\empty\else% --- if non empty caption
\writecapt{{#1}{#2}{#3}}% --- save for journal figs
\captypeunstarred#1*.% --- remove optional stars
\getcaption#3\endc@ption% --- parse the caption
\def\testit{#2}% --- check for empty label
\ifx\testit\empty\else\label{#2}\fi
\fi}
\def\captypeunstarred#1*#2.{% --- remove two-column star from style
\def\@captype{#1}}
%
% Parse the caption of the form {caption} or {[short] long caption}
%
\def\getcaption{\@ifnextchar[{\getcaptwo}{\getcapone}}
\long\def\getcapone#1\endc@ption{\caption[#1]%
{\def\baselinestretch{1}\Large\normalsize\captionstyle\ignorespaces #1}}
\long\def\getcaptwo[#1]#2\endc@ption{\caption[#1]%
{\def\baselinestretch{1}\Large\normalsize\captionstyle\ignorespaces #2}}
%
% Do the right thing with an already captioned marginal box
%
% \leftbox{box}
% \rightbox{box}
%
% options: \figboxmargin \figboxhang \figuredirectory
\newdimen\figboxht
\newcount\figboxn
\newcount\figboxlines% --- picture height in number of lines
\newdimen\figboxwid% --- picture width
\newif\ifisleftbox
\long\def\leftbox#1{%
\setbox\figurebox\hbox{#1}\global\isleftboxtrue
\startmarginbox
\vadjust{\smash{\rlap{\hskip\hsize\hskip\figboxhang
\llap{\raise.7\baselineskip\box\figurebox\hskip\rightskip}}}}%
\endmarginbox%
}
\long\def\rightbox#1{%
\setbox\figurebox\hbox{#1}\global\isleftboxfalse
\startmarginbox
\smash{\llap{\raise.7\baselineskip\box\figurebox\hskip\figboxmargin}}%
\endmarginbox%
}
\def\startmarginbox{%
\ifvmode\passpict\let\endmarginbox=\indent
\else\message{WARNING: marginbox in not in vmode}\hfilneg\ \passpict
\let\endmarginbox=\relax\fi
%
\figboxht=\dp\figurebox% actual height
\advance\figboxht by 1.3\baselineskip% round up
\vskip\figboxht\penalty-300\vskip-\figboxht% filbreak trick
\divide\figboxht by\baselineskip% how many lines?
\global\figboxlines=\figboxht
%
\global\figboxwid=\wd\figurebox
\global\advance\figboxwid by \figboxmargin
\global\advance\figboxwid by -\figboxhang
\setmypar\noindent}
%
%
% Useful macros
%
% for lengthening (and shortening) picture space
%
\def\addlines#1{\global\advance\figboxlines by #1\myparshape}
\def\zerolines{\origpar\global\figboxlines=0\myparshape}
%
% for passing any marginal figures
%
\def\passpict{\par\ifnum\figboxlines>1\vskip\figboxlines\baselineskip
\zerolines\fi}
\def\emptybox#1#2{\hbox to #1{\vbox to #2{\vss}\hss}}
%%%%%%%%%%%%%%%%%%%%%
%\par hacking
% We define our own version of Tex's \par and tell Latex it is the original
\global\let\origpar=\@@par% --- Latex's saved version of Tex's \par
\global\let\dopar=\origpar
\global\def\@@par{\dopar}
\@setpar{\dopar}
\def\setmypar{\global\let\dopar=\mypar
\global\prevgraf=0\myparshape}
\def\mypar{\origpar\global\advance\figboxlines by -\prevgraf%
\global\prevgraf=0\myparshape}
\def\myparshape{\relax%
\ifnum\figboxlines>1\theparshape \else
\global\hangindent=0pt\global\hangafter=1% these sometimes get restored...
\global\let\dopar=\origpar\fi}
\def\theparshape{%
\ifisleftbox\global\hangindent=-\figboxwid
\else\global\hangindent=\figboxwid \fi
\global\hangafter=-\figboxlines \global\advance\hangafter by 1%
}
%%%%%%%%%%%%%%%%
%
% Macros to make the list of figures and captions used by journals
%
% To use, execute \journalpicts{scalefactor} at the beginning of the
% file. All figbox figures and tables will be written to a .pic file
% and all captions to a .cap file. On \enddocument, these files are
% read in and the lists of figures and captions are produced.
%
% For non-english versions, or to add other types of floats, the
% macro \definefnum should be changed.
%
\def\definefnum#1{% --- ugly latex hacking to get the
% right figure/table name
\def\fnum@figure{Figure \ref{#1}}%
\def\fnum@table{Table \ref{#1}}%
\def\fnum@code{Algorithm \ref{#1}}%
}
\def\writepict#1{}
\def\writecapt#1{}
%%%%%%%%%%%%%%%%
% \journalpicts{globalscalefactor} writes all pictures and captions to
% files and modifies \enddocument to read these files and
% produce lists of enlarged figures and of captions
%
\def\journalpicts#1{% global scale factor
\newwrite\pictfile
\newwrite\captfile
%
\openout\pictfile\jobname.pic
\openout\captfile\jobname.cap
%
\gdef\writepict##1{\unexpandedwrite\pictfile{\doit##1}}%
\gdef\writecapt##1{\unexpandedwrite\captfile{\doit##1}}%
%
\global\let\ENDdocument=\enddocument
\gdef\enddocument{\DOjournalpicts{#1}\ENDdocument}
}
%
% generate enlarged figures 1 per page and list captions
%
\def\DOjournalpicts#1{{%
\def\writepict##1{}\closeout\pictfile
\def\writecapt##1{}\closeout\captfile
\@fileswfalse% --- turn off toc, lof, and so on.
%
\onecolumn
\def\globalscaling{#1}
\def\doit##1##2##3##4{% --- print a picture on blank page
% {box}{style}{label}{caption}
\figboxaux[t]{\hss##1\hss}{##2}{}{}%
\vspace*{1in}
\definefnum{##3}% --- use Figure \ref{label} or Table \ref{label}
\captionandlabel{##2}{}{##4}
\clearpage}%
%
\input\jobname.pic
%
\def\doit##1##2##3{% --- print caption
% {style}{label}{caption}
\definefnum{##2}% --- use Figure \ref{label} or Table \ref{label}
\captionandlabel{##1}{}{##3}}%
%
\raggedright\let\captionstyle=\raggedright
\def\@makecaption##1##2{##1: ##2\par}% --- Be sure captions aren't centered
%
\input\jobname.cap
}}% --- to be followed by \enddocument
%%%%%%%%%%%%%%%%
%
% Box macros
%
%%%%%%%%%%%%%%%%
%
% \boxeps[local commands] {file} {scale}
%
% options \DVIscaling \figuredirectory \scale \horizscale \vertscale
% \missingfigure \globalscaling
%
% \boxps[local commands] {file} {scale} {width} {height}
%
% \scale \horizscale \vertscale
%
% Various `boxers' --- which one is used depends on where the origin
% of the picture is. If you are using the bounding boxes of
% encapsulated postscript, just use the default \llboxer. Otherwise,
% you have to figure out where the origin is and set \boxer appropriately.
%
\def\llboxer#1{\vbox to \figboxht{\vfil\hbox to \figboxwid{#1\hfill}}}
\def\lcboxer#1{\vbox to \figboxht{\vfil\hbox to \figboxwid{\hfill#1\hfill}}}
\def\oldboxer#1{\vbox to \figboxht{\vfil
\hbox to \figboxwid{\hfill\llap{#1\hskip4.25in}\hfill}}}
\def\ulboxer#1{\vbox to \figboxht{\hbox to \figboxwid{#1\hfill}\vfil}}
\def\ccboxer#1{\vbox to \figboxht{\vfil
\hbox to \figboxwid{\hfill#1\hfill}\vfil}}
% a macro to strip the pt off a dimension
%
{\catcode`\p=12\catcode`\t=12
\gdef\removedimen#1pt{#1}}
\def\defscaled#1#2{#2=\DVIscaling#2%
\xdef#1{\expandafter\removedimen\the#2}}
\def\DVIspace{ }
\newdimen\hscalefactor
\newdimen\vscalefactor
\def\scale#1{\horizscale{#1}\vertscale{#1}}
\def\horizscale#1{\hscalefactor=#1\hscalefactor\figboxht=#1\figboxht}
\def\vertscale#1{\vscalefactor=#1\vscalefactor\figboxwid=#1\figboxwid}
%%%%%%%%%%%%%%%%
%
\def\boxps{%
\@ifnextchar[{\boxpsaux}{\boxpsaux[\relax]}}
\def\boxpsaux[#1]#2#3#4#5{%
{\figboxwid#4\figboxht#5\hscalefactor=1pt\vscalefactor=1pt%
\scale{#3}%
\scale{\globalscaling}%
#1%
\defscaled\DVIhscale\hscalefactor
\defscaled\DVIvscale\vscalefactor
\boxer{\special{psfile=\figuredirectory/#2.ps \DVIspace
hscale=\DVIhscale\DVIspace
vscale=\DVIvscale}}}%
}
%%%%%%%%%%%%%%%%
% The following is from tex/inputs/epsf.tex by Tom Rokicki
% with slight modifications by Jack Snoeyink.
%
% Written by Tomas Rokicki of Radical Eye Software, 29 Mar 1989.
%
% Include an ecapsulated postscript graphic.
% Works by finding the bounding box comment.
%
\newread\Epsffilein
\newif\ifEpsffileok
\newif\ifEpsfbbfound
\newdimen\pspoints
\pspoints=1in
\divide\pspoints by 72
\def\boxeps{%
\@ifnextchar[{\boxepsaux}{\boxepsaux[\relax]}}
\def\boxepsaux[#1]#2#3{%
%
% The first thing we need to do is to open the
% PostScript file, if possible.
%
\openin\Epsffilein=\figuredirectory/#2.ps
\ifeof\Epsffilein\message{I couldn't open \figuredirectory/#2.ps }%
\missingfigure{#2}
\else
%
% Okay, we got it. Now we scan lines until we find one
% that doesn't start with a percentage sign. We're
% looking for the bounding box comment.
%
{\Epsffileoktrue\Epsfbbfoundfalse
\catcode`\%=11 \catcode`\\=11
\catcode`\{=11 \catcode`\}=11
\catcode`\$=11 \catcode`\^=11
\catcode`\&=11 \catcode`\#=11
\catcode`\~=11 \catcode`\_=11
\loop
\read\Epsffilein to \Epsffileline
\ifeof\Epsffilein\Epsffileokfalse\else
%
% Now we check to make sure the first character is a % sign,
% and that the rest are `%BoundingBox:' if necessary.
%
\expandafter\Epsfaux\Epsffileline . .\\%
\fi
\ifEpsffileok\repeat
\ifEpsfbbfound
\figboxht=\Epsfury\pspoints
\advance\figboxht by-\Epsflly\pspoints
\figboxwid=\Epsfurx\pspoints
\advance\figboxwid by-\Epsfllx\pspoints
\else
\message{No bounding box comment in \figuredirectory/#2.ps }%
\figboxwid=2in\figboxht=1in%
\fi%
\immediate\closein\Epsffilein
\hscalefactor=1pt\vscalefactor=1pt%
\scale{#3}%
\scale{\globalscaling}%
#1%
%
\defscaled\DVIhscale\hscalefactor
\defscaled\DVIvscale\vscalefactor
%
\hscalefactor=-\Epsfllx\hscalefactor
\hscalefactor=1.00375\hscalefactor% --- 72.27/72 = TeXpoints/PSpoints
\defscaled\DVIhoffset\hscalefactor
%
\vscalefactor=-\Epsflly\vscalefactor
\vscalefactor=1.00375\vscalefactor% --- 72.27/72 = TeXpoints/PSpoints
\defscaled\DVIvoffset\vscalefactor
%
\llboxer{\special{psfile=\figuredirectory/#2.ps \DVIspace
hoffset=\DVIhoffset\DVIspace
voffset=\DVIvoffset\DVIspace
hscale=\DVIhscale\DVIspace
vscale=\DVIvscale}} }%
\fi
}%
%
% We need these for comparison purposes.
%
{\catcode`\%=11 \global\let\Epsfpar=\par
\global\let\Epsfpercent=%\global\def\Epsfbblit{%BoundingBox:}}%
%
% This is our function that checks for `%BoundingBox:' and grabs the
% values if they are found.
%
\long\def\Epsfaux#1#2 #3\\{\relax\ifx#1\Epsfpercent
\def\testit{#2}\ifx\testit\Epsfbblit
\Epsfsize #3 . . . .\\%
\global\Epsffileokfalse
\global\Epsfbbfoundtrue
\fi\else\ifx#1\Epsfpar\else\global\Epsffileokfalse\fi\fi}%
%
% Here we grab the values and stuff them in the appropriate definitions.
%
\def\Epsfsize#1 #2 #3 #4 #5\\{\global\def\Epsfllx{#1}\global\def\Epsflly{#2}%
\global\def\Epsfurx{#3}\global\def\Epsfury{#4}}%
\catcode`\@=12 % done with with internals
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For backward compatability
%
% \pic file; height in inches; width in inches; caption\par
% \picsc file; ht (in); wid (in); scale; caption\par
%
% \mpic file; height in inches; width in inches; caption\par
% \mpicsc file; ht (in); wid (in); scale; caption\par
%
%%%%%%%%%%%%%%%%
\def\pic#1;#2;#3;#4\par{\picsc#1;#2;#3;1;#4\par}% file; wd; ht; caption
\def\picsc#1;#2;#3;#4;#5\par{% file; wd; ht; scale; caption
\figbox[htb]{\boxeps{#1}{#4}%{#2in}{#3in}%
}{figure}{#1}{%
% (#1: #2x#3 sc #4) % Uncomment for debugging
#5}}
\def\mpic#1;#2;#3;#4\par{\mpicsc#1;#2;#3;1;#4\par}% file; wd; ht; caption
\def\mpicsc#1;#2;#3;#4;#5\par{% file; wd; ht; scale; caption
\figbox[l]{\boxeps{#1}{#4}%{#2in}{#3in}%
}{figure}{#1}{%
% (#1: #2x#3 sc #4) % Uncomment for debugging
#5}}
% !TEX root = ../Multi_Momentum_INFOCOM.tex
%\renewcommand{\thesection}{\Roman{section}}
%\renewcommand{\thesubsection}{\Roman{section}-\Alph{subsection}}
%\renewcommand{\thesubsubsection}{\Alph{subsection}.\arabic{subsubsection}}
\newcommand{\A}{\mathbf{A}}
\renewcommand{\a}{\mathbf{a}}
\newcommand{\B}{\mathbf{B}}
\newcommand{\Bt}{\mathbf{B}}
\renewcommand{\b}{\mathbf{b}}
\newcommand{\bt}{\mathbf{b}}
\renewcommand{\c}{\mathbf{c}}
\newcommand{\ct}{\widetilde{\mathbf{c}}}
\newcommand{\C}{\mathbf{C}}
\newcommand{\Ch}{\widehat{\mathbf{C}}}
\newcommand{\D}{\mathbf{D}}
\newcommand{\Dh}{\widehat{\mathbf{D}}}
\renewcommand{\d}{\mathbf{d}}
\newcommand{\E}{\mathbf{E}}
\newcommand{\e}{\mathbf{e}}
\newcommand{\et}{\widetilde{\mathbf{e}}}
\newcommand{\f}{\mathbf{f}}
\newcommand{\F}{\mathbf{F}}
\newcommand{\G}{\mathbf{G}}
\newcommand{\g}{\mathbf{g}}
\newcommand{\I}{\mathbf{I}}
\newcommand{\J}{\mathbf{J}}
\newcommand{\K}{\mathbf{K}}
\newcommand{\M}{\mathbf{M}}
\newcommand{\m}{\mathbf{m}}
\newcommand{\mh}{\widehat{\mathbf{m}}}
\newcommand{\Mh}{\widehat{\mathbf{M}}}
\newcommand{\Mt}{\widetilde{\mathbf{M}}}
\newcommand{\Mb}{\overline{\mathbf{M}}}
\newcommand{\N}{\mathbf{N}}
\newcommand{\n}{\mathbf{n}}
\newcommand{\p}{\mathbf{p}}
\renewcommand{\P}{\mathbf{P}}
\newcommand{\Q}{\mathbf{Q}}
\newcommand{\q}{\mathbf{q}}
\newcommand{\R}{\mathbf{R}}
\renewcommand{\r}{\mathbf{r}}
\renewcommand{\S}{\mathbf{S}}
\newcommand{\Sh}{\widehat{\mathbf{S}}}
\newcommand{\s}{\mathbf{s}}
\renewcommand{\t}{\mathbf{t}}
\newcommand{\T}{\mathbf{T}}
\newcommand{\Th}{\widehat{\mathbf{T}}}
\newcommand{\U}{\mathbf{U}}
\renewcommand{\u}{\mathbf{u}}
\newcommand{\V}{\mathbf{V}}
\renewcommand{\v}{\mathbf{v}}
\newcommand{\W}{\mathbf{W}}
\newcommand{\w}{\mathbf{w}}
\newcommand{\wt}{\widetilde{\mathbf{w}}}
\newcommand{\wts}{\widetilde{w}}
\newcommand{\X}{\mathbf{X}}
\newcommand{\Xh}{\widehat{\mathbf{X}}}
\newcommand{\x}{\mathbf{x}}
\newcommand{\xt}{\widetilde{\mathbf{x}}}
\newcommand{\xh}{\widehat{\mathbf{x}}}
\newcommand{\Y}{\mathbf{Y}}
\newcommand{\y}{\mathbf{y}}
\newcommand{\z}{\mathbf{z}}
\newcommand{\Z}{\mathbf{Z}}
\newcommand{\yt}{\widetilde{\mathbf{y}}}
\newcommand{\rb}{\mathbf{b}}
\newcommand{\rvec}{\mathbf{r}}
\newcommand{\0}{\mathbf{0}}
\newcommand{\1}{\mathbf{1}}
\newcommand{\Hc}{\mathbf{H}}
\newcommand{\tHc}{\widetilde{\mathbf{H}}}
\newcommand{\hHc}{\overline{\mathbf{H}}}
\newcommand{\DL}{\mathbf{DL}}
\newcommand{\mLambda}{\mathbf{\Lambda}}
\newcommand{\vlambda}{\boldsymbol{\lambda}}
\newcommand{\mOmega}{\mathbf{\Omega}}
\newcommand{\mOmegab}{\overline{\mathbf{\Omega}}}
\newcommand{\mGamma}{\mathbf{\Gamma}}
\newcommand{\mPi}{\mathbf{\Pi}}
\newcommand{\mPsi}{\mathbf{\Psi}}
\newcommand{\mPsib}{\overline{\mathbf{\Psi}}}
\newcommand{\valpha}{\mathbf{\alpha}}
\newcommand{\vpi}{\boldsymbol \pi}
\newcommand{\Sone}{\mathcal{S}}
\newcommand{\tr}{\mathrm{Tr}}
\newcommand{\new}{\mathrm{new}}
\newcommand{\net}{\mathrm{net}}
\newcommand{\link}{\mathrm{link-phy}}
\newcommand{\LL}{\mathbf{L}}
\newcommand{\Out}[1]{\mathcal{O}\left( #1 \right)}
\newcommand{\In}[1]{\mathcal{I}\left( #1 \right)}
\newcommand{\Int}[1]{\mathcal{F}\left( #1 \right)}
\newcommand{\sInt}[1]{\hat{\mathcal{F}}\left( #1 \right)}
\renewcommand{\Re}[1]{\mbox{$\mathfrak{Re} \left( #1 \right)$}}
\renewcommand{\Im}[1]{\mbox{$\mathfrak{Im} \left( #1 \right)$}}
\newcommand{\Resq}[1]{\mbox{$\mathfrak{Re}^{2} \left( #1 \right)$}}
\newcommand{\Imsq}[1]{\mbox{$\mathfrak{Im}^{2} \left( #1 \right)$}}
\newcommand{\LTM}{\langle L \rangle}
\newcommand{\con}[1]{\overline{#1}}
\newcommand{\abs}[1]{\mbox{$\lvert #1 \rvert$}}
\newcommand{\norm}[1]{\mbox{$\left\lVert #1 \right\rVert$}}
\newcommand{\dual}{$\mathbf{D}^{\mathrm{CRPBA}}$}
\newcommand{\dnet}{$\mathbf{D}^{\mathrm{CRPBA}}_{\net}$}
\newcommand{\dlink}{$\mathbf{D}^{\mathrm{CRPBA}}_{\link}$}
\newcommand{\Pwrcon}{\Omega_{+}(n)}
\newcommand{\src}[1]{\mathrm{Src}(#1)}
\newcommand{\dst}[1]{\mathrm{Dst}(#1)}
\newcommand{\rx}[1]{\mathrm{Rx}(#1)}
\newcommand{\tx}[1]{\mathrm{Tx}(#1)}
\newcommand{\Dy}{\Delta \y}
\newcommand{\Dyt}{\Delta \widetilde{\y}}
%\newcommand*\circled[1]{\tikz[baseline=(char.base)]{
% \node[shape=circle,draw,inner sep=2pt] (char) {#1};}}
%\newcommand{\seq}[1]{\mbox{$\{ #1 \} $}}
%\newcommand{\myref}[1]{(\ref{#1})}
%\newcommand{\sd}{\mbox{$\stackrel{\Delta}{=}$}}
%\newcommand{\sci}[2]{\mbox{$#1 \times 10^{- #2}$}}
%%\newtheorem{thm}{Theorem}
\newtheorem{thm}{Theorem}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{ex}{Example}[section]
%\theoremstyle{definition}
\newtheorem{defn}{Definition}
%\theoremstyle{remark}
\newtheorem{finalremark}[thm]{Final Remark}
\newtheorem{rem}{Remark}
\newtheorem{sol}{Solution}
\newtheorem{fact}{Fact}
%\numberwithin{equation}[section]
\newcommand{\po}{\mbox{$ \mbox{P}_{\rm out}$}}
\newcommand{\ga}{\bar{\gamma}}
\newcommand{\mtrx}[1]{\mbox{$\left[\begin{array} #1 \end{array}\right]$}}
\newcommand{\mtrxt}[1]{\mbox{$\begin{array} #1 \end{array}$}}
\newcommand{\diag}[1]{\mathrm{Diag}\left\{ #1 \right\}}
\newcommand{\diaginv}[1]{\mathrm{Diag}^{-1}\left\{ #1 \right\}}
\newcommand{\diagtran}[1]{\mathrm{Diag}^{T}\left\{ #1 \right\}}
\newcommand{\diagv}[1]{\mathrm{diag}\left\{ #1 \right\}}
\newcommand{\etal}{{\em et al. }}
\newcommand{\ind}{\mathbbm{1}}
\newcommand{\vbeta}{\boldsymbol \beta}
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Output:}}
\algtext*{EndWhile}% Remove "end while" text
\algtext*{EndIf}% Remove "end if" text
\algtext*{EndFor}% Remove "end while" text
\algtext*{EndFunction}% Remove "end if" text
\algtext*{EndProcedure}% Remove "end while" text
%\algtext*{EndIf}% Remove "end if" text
%\newcommand{\algorithmicinitialize}{\textbf{Initialization:}}
%\newcommand{\algorithmicmainiterate}{\textbf{Main Iteration:}}
%\newcommand{\algorithmicprimal}{\textbf{Primal Newton Direction:}}
%\newcommand{\algorithmicdual}{\textbf{Dual Variables Update:}}
% Generated by IEEEtran.bst, version: 1.14 (2015/08/26)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{TensorFlow}
M.~Abadi, P.~Barham \emph{et~al.}, ``{TensorFlow}: A system for large-scale
machine learning,'' in \emph{Proc. of USENIX OSDI}, 2016.
\bibitem{MXNet}
T.~Chen, M.~Li, Y.~Li, M.~Lin, N.~Wang, M.~Wang, T.~Xiao, B.~Xu, C.~Zhang, and
Z.~Zhang, ``{MXNet}: A flexible and efficient machine learning library for
heterogeneous distributed systems,'' in \emph{Proc. of NIPS Workshop on
Machine Learning Systems (LearningSys)}, 2016.
\bibitem{Paszke19:Pytorch}
A.~Paszke, S.~Gross, F.~Massa, A.~Lerer, J.~Bradbury, G.~Chanan, T.~Killeen,
Z.~Lin, N.~Gimelshein, L.~Antiga, A.~Desmaison \emph{et~al.}, ``Pytorch: An
imperative style, highperformance deep learning library,'' in \emph{In
Advances in Neural Information Processing Systems}, 2019, pp. 8024--8035.
\bibitem{Caffe}
Y.~Jia, E.~Shelhamer, J.~Donahue, S.~Karayev, J.~Long, R.~Girshick,
S.~Guadarrama, and T.~Darrell, ``Caffe: Convolutional architecture for fast
feature embedding,'' in \emph{Proc. of the 22nd ACM International Conference
on Multimedia}, 2014, pp. 675--678.
\bibitem{Chun16:Dolphin}
B.-G. Chun, B.~Cho, B.~Jeon \emph{et~al.}, ``Dolphin: Runtime optimization for
distributed machine learning,'' in \emph{Proc. of ICML ML Systems Workshop},
2016.
\bibitem{Bao18:ML_INFOCOM}
Y.~Bao, Y.~Peng, C.~Wu, and Z.~Li, ``Online job scheduling in distributed
machine learning clusters,'' in \emph{Proc. of IEEE INFOCOM}, 2018.
\bibitem{Li14:ML_Static_OSDI}
M.~Li, D.~G. Andersen \emph{et~al.}, ``Scaling distributed machine learning
with the parameter server,'' in \emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Chilim14:ML_Static_OSDI}
T.~M. Chilimbi, Y.~Suzue, J.~Apacible, and K.~Kalyanaraman, ``Project {Adam}:
Building an efficient and scalable deep learning training system,'' in
\emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Dean08:MapReduce}
J.~Dean and S.~Ghemawat, ``Mapreduce: Simplified data processing on large
clusters,'' in \emph{Commun. ACM}, vol.~1, no.~51, 2008, pp. 107--113.
\bibitem{Isard07:Dryad}
M.~Isard, M.~Budiu, Y.~Yu, A.~Birrell, and D.~Fetterly, ``Dryad: Distributed
data-parallel programs from sequential building blocks,'' in \emph{In EuroSys
2007}, 2007, pp. 59--72.
\bibitem{Yan15:MLFramework_KDD}
F.~Yan, O.~Ruwase, Y.~He, and T.~Chilimbi, ``Performance modeling and
scalability optimization of distributed deep learning systems,'' in
\emph{Proc. of ACM KDD}, 2015.
\bibitem{Sun17:ML_MILP}
P.~Sun, Y.~Wen, N.~B.~D. Ta, and S.~Yan, ``Towards distributed machine learning
in shared clusters: A dynamically-partitioned approach,'' in \emph{Proc. of
IEEE Smart Computing}, 2017.
\bibitem{Zhang20:onlinescheduling}
Q.~Zhang, R.~Zhou, C.~Wu, L.~Jiao, and Z.~Li, ``Online scheduling of
heterogeneous distributed machine learning jobs,'' in \emph{Proceedings of
the Twenty-First International Symposium on Theory, Algorithmic Foundations,
and Protocol Design for Mobile Networks and Mobile Computing}, 2020, pp.
111--120.
\bibitem{Mao16:DRL}
H.~Mao, M.~Alizadeh, I.~Menache, and S.~Kandula, ``Resource management with
deep reinforcement learning,'' in \emph{Proc. of ACM HotNets}, 2016.
\bibitem{Chen17:DRL}
W.~Chen, Y.~Xu, and X.~Wu, ``Deep reinforcement learning for multi-resource
multi-cluster job scheduling,'' in \emph{Proc. of IEEE ICNP}, 2017.
\bibitem{Mao18:graph-based}
H.~Mao, M.~Schwarzkopf, S.~Venkatakrishnan, and M.~Alizadeh, ``Learning
graph-based cluster scheduling algorithms,'' in \emph{Proc. of SysML}, 2018.
\bibitem{Mao19:learningscheduling}
H.~Mao, M.~Schwarzkopf, S.~B. Venkatakrishnan, Z.~Meng, and M.~Alizadeh,
``Learning scheduling algorithms for data processing clusters,'' in
\emph{Proc. of ACM SIGCOMM}, 2019.
\bibitem{Mirhoseini18:hierarchicalmodel}
A.~Mirhoseini, A.~Goldie, H.~Pham, B.~Steiner, Q.~V. Le, and J.~Dean, ``A
hierarchical model for device placement,'' in \emph{Proc. of ICLR}, 2018.
\bibitem{Mirhoseini17:deviceplacement}
A.~Mirhoseini, H.~Pham, Q.~V. Le, B.~Steiner, R.~Larsen, Y.~Zhou, N.~Kumar,
M.~Norouzi, S.~Bengio, and J.~Dean, ``Device placement optimization with
reinforcement learning,'' in \emph{Proc. of ICML}, 2017.
\bibitem{MXNet1}
\BIBentryALTinterwordspacing
V.~Kovalevskyi, ``{MXNet} distributed training explained in depth.'' [Online].
Available:
\url{https://blog.kovalevskyi.com/mxnet-distributed-training-explained-in-depth-part-1-b90c84bda725}
\BIBentrySTDinterwordspacing
\bibitem{MXNet2}
\BIBentryALTinterwordspacing
``Distributed training in {MXNet}.'' [Online]. Available:
\url{https://mxnet.incubator.apache.org/faq/distributed_training.html}
\BIBentrySTDinterwordspacing
\bibitem{Goyal17:SGD}
P.~Goyal, P.~Dolla, R.~Girshick, P.~Noordhuis, L.~Wesolowski, A.~Kyrola,
A.~Tulloch, Y.~Jia, and K.~He, ``Accurate, large minibatch {SGD}: Training
imagenet in 1 hour,'' \emph{arXiv preprint arXiv:1706.02677}, 2017.
\bibitem{Peng18:ML_EuroSys}
Y.~Peng, Y.~Bao, Y.~Chen, C.~Wu, and C.~Guo, ``Optimus: An efficient dynamic
resource scheduler for deep learning clusters,'' in \emph{Proc. of ACM
EuroSys}, 2018.
\bibitem{HDFS}
\BIBentryALTinterwordspacing
``{Apache Hadoop}.'' [Online]. Available: \url{http://hadoop.apache.org/.}
\BIBentrySTDinterwordspacing
\bibitem{Dean:2012:LSD}
J.~Dean, G.~S. Corrado, R.~Monga, K.~Chen, M.~Devin, Q.~V. Le, M.~Z. Mao,
M.~Ranzato, A.~Senior, P.~Tucker, K.~Yang, and A.~Y. Ng, ``Large scale
distributed deep networks,'' in \emph{Proceedings of the 25th International
Conference on Neural Information Processing Systems - Volume 1}, ser.
NIPS'12.\hskip 1em plus 0.5em minus 0.4em\relax USA: Curran Associates Inc.,
2012, pp. 1223--1231.
\bibitem{Cheatham05:BSP}
T.~Cheatham, A.~Fahmy, D.~Siefanescu, and L.~Valiani, ``Bulk synchronous
parallel computing-a paradigm for transportable software,'' in \emph{Hawaii
International Conference on Systems Sciences,}, 2005.
\bibitem{Bazaraa_Sherali_Shetty_93:NLP}
M.~S. Bazaraa, H.~D. Sherali, and C.~M. Shetty, \emph{Nonlinear Programming:
Theory and Algorithms}, 3rd~ed.\hskip 1em plus 0.5em minus 0.4em\relax New
York, NY: John Wiley \& Sons Inc., 2006.
\bibitem{Keras}
\BIBentryALTinterwordspacing
``Multi-worker training with keras.'' [Online]. Available:
\url{https://www.tensorflow.org/tutorials/distribute/multi_worker_with_keras}
\BIBentrySTDinterwordspacing
\bibitem{Kubernetes}
\BIBentryALTinterwordspacing
``Run deep learning with paddlepaddle on kubernetes,'' 2017. [Online].
Available:
\url{https://kubernetes.io/blog/2017/02/run-deep-learning-with-paddlepaddle-on-kubernetes/}
\BIBentrySTDinterwordspacing
\bibitem{Wang18:SpiderBoost}
Z.~Wang, K.~Ji, Y.~Zhou, Y.~Liang, and V.~Tarokh, ``Spiderboost: A class of
faster variance-reduced algorithms for nonconvex optimization,'' in
\emph{arXiv preprint arXiv:1810.10690}, 2018.
\bibitem{Hochbaum97}
D.~S. Hochbaum, \emph{Approximation Algorithms for NP-Hard Problems}, D.~S.
Hochbaum, Ed.\hskip 1em plus 0.5em minus 0.4em\relax PWS Publishing Company,
1997.
\bibitem{Buchbinder09}
N.~Buchbinder and J.~(Seffi)~Naor, \emph{The Design of Competitive Online
Algorithms via a Primal-Dual Approach}.\hskip 1em plus 0.5em minus
0.4em\relax Now Publishers Inc., feb 2009, vol.~3.
\bibitem{Vaidya87:LP}
P.~M. Vaidya, ``An algorithm for linear programming which requires
$o(((m+n)n^2+(m+n)^{1.5}n){L})$ algorithmic operations,'' in \emph{28th
Annual IEEE Syposium on Foundations of Computer Science}, 1987.
\bibitem{Li14:OSDI}
M.~Li, D.~G. Andersen \emph{et~al.}, ``Scaling distributed machine learning
with the parameter server,'' in \emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Chilimbi14:OSDI}
T.~M. Chilimbi, Y.~Suzue, J.~Apacible, and K.~Kalyanaraman, ``Project adam:
Building an efficient and scalable deep learning training system,'' in
\emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Sun17:Dorm}
P.~Sun, Y.~Wen, N.~B.~D. Ta, and S.~Yan, ``Towards distributed machine learning
in shared clusters: A dynamically-partitioned approach,'' in \emph{Proc. of
IEEE Smart Computing}, 2017.
\bibitem{AmazonEC2Instances}
\BIBentryALTinterwordspacing
``Amazon ec2 instances.'' [Online]. Available:
\url{https://aws.amazon.com/ec2/instance-types}
\BIBentrySTDinterwordspacing
\bibitem{Reiss12:googlecluster}
C.~Reiss, A.~Tumanov \emph{et~al.}, ``Heterogeneity and dynamicity of clouds at
scale: Google trace analysis,'' in \emph{Proc. of ACM SoCC}, 2012.
\bibitem{Huang15:CORA}
Z.~Huang, B.~Balasubramanian, M.~Wang, T.~Lan, M.~Chiang, and D.~H. Tsang,
``Need for speed: Cora scheduler for optimizing completiontimes in the
cloud,'' in \emph{Proc. of IEEE INFOCOM}, 2015.
\bibitem{Zaharia10:USENIX}
M.~Zaharia, M.~Chowdhury, M.~J. Franklin, and et~al, ``Spark: Cluster computing
with working sets,'' in \emph{n Proc. of USENIX HotCloud}, 2010.
\bibitem{Vavilapalli13:Yarn}
V.~K. Vavilapalli, A.~C. Murthy \emph{et~al.}, ``Apache hadoop yarn: Yet
another resource negotiator,'' in \emph{Proc. of ACM SoCC}, 2013.
\bibitem{Hindman11:Mesos}
B.~Hindman, A.~Konwinski \emph{et~al.}, ``Mesos: A platform for fine-grained
resource sharing in the data center,'' in \emph{Proc. of USENIX NSDI}, 2011.
\bibitem{Jeon18:multi-tenant}
M.~Jeon, S.~Venkataraman, A.~Phanishayee, J.~Qian, W.~Xiao, and F.~Yang,
``Multi-tenant gpu clusters for deep learning workloads: Analysis and
implications,'' in \emph{MSR-TR-2018- 13}, 2018.
\bibitem{Minet18:google_trace}
P.~Minet, \'{E}ric Renault, I.~Khoufi, and S.~Boumerdassi, ``Analyzing traces
from a google data center,'' in \emph{International Wireless Communications
and Mobile Computing Conference (IWCMC)}, 2018, pp. 1167--1172.
\bibitem{Bertsekas99:Nonlinear}
D.~Bertsekas, \emph{Nonlinear Programming}, 2nd~ed.\hskip 1em plus 0.5em minus
0.4em\relax Athena Scientific, 1999.
\end{thebibliography}
% Generated by IEEEtran.bst, version: 1.14 (2015/08/26)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{TensorFlow}
M.~Abadi, P.~Barham \emph{et~al.}, ``{TensorFlow}: A system for large-scale
machine learning,'' in \emph{Proc. of USENIX OSDI}, 2016.
\bibitem{MXNet}
T.~Chen, M.~Li, Y.~Li, M.~Lin, N.~Wang, M.~Wang, T.~Xiao, B.~Xu, C.~Zhang, and
Z.~Zhang, ``{MXNet}: A flexible and efficient machine learning library for
heterogeneous distributed systems,'' in \emph{Proc. of NIPS Workshop on
Machine Learning Systems (LearningSys)}, 2016.
\bibitem{Paszke19:Pytorch}
A.~Paszke, S.~Gross, F.~Massa, A.~Lerer, J.~Bradbury, G.~Chanan, T.~Killeen,
Z.~Lin, N.~Gimelshein, L.~Antiga, A.~Desmaison \emph{et~al.}, ``Pytorch: An
imperative style, highperformance deep learning library,'' in \emph{In
Advances in Neural Information Processing Systems}, 2019, pp. 8024--8035.
\bibitem{Caffe}
Y.~Jia, E.~Shelhamer, J.~Donahue, S.~Karayev, J.~Long, R.~Girshick,
S.~Guadarrama, and T.~Darrell, ``Caffe: Convolutional architecture for fast
feature embedding,'' in \emph{Proc. of the 22nd ACM International Conference
on Multimedia}, 2014, pp. 675--678.
\bibitem{Chun16:Dolphin}
B.-G. Chun, B.~Cho, B.~Jeon \emph{et~al.}, ``Dolphin: Runtime optimization for
distributed machine learning,'' in \emph{Proc. of ICML ML Systems Workshop},
2016.
\bibitem{Bao18:ML_INFOCOM}
Y.~Bao, Y.~Peng, C.~Wu, and Z.~Li, ``Online job scheduling in distributed
machine learning clusters,'' in \emph{Proc. of IEEE INFOCOM}, 2018.
\bibitem{Li14:ML_Static_OSDI}
M.~Li, D.~G. Andersen \emph{et~al.}, ``Scaling distributed machine learning
with the parameter server,'' in \emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Chilim14:ML_Static_OSDI}
T.~M. Chilimbi, Y.~Suzue, J.~Apacible, and K.~Kalyanaraman, ``Project {Adam}:
Building an efficient and scalable deep learning training system,'' in
\emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Dean08:MapReduce}
J.~Dean and S.~Ghemawat, ``Mapreduce: Simplified data processing on large
clusters,'' in \emph{Commun. ACM}, vol.~1, no.~51, 2008, pp. 107--113.
\bibitem{Isard07:Dryad}
M.~Isard, M.~Budiu, Y.~Yu, A.~Birrell, and D.~Fetterly, ``Dryad: Distributed
data-parallel programs from sequential building blocks,'' in \emph{In EuroSys
2007}, 2007, pp. 59--72.
\bibitem{Yan15:MLFramework_KDD}
F.~Yan, O.~Ruwase, Y.~He, and T.~Chilimbi, ``Performance modeling and
scalability optimization of distributed deep learning systems,'' in
\emph{Proc. of ACM KDD}, 2015.
\bibitem{Sun17:ML_MILP}
P.~Sun, Y.~Wen, N.~B.~D. Ta, and S.~Yan, ``Towards distributed machine learning
in shared clusters: A dynamically-partitioned approach,'' in \emph{Proc. of
IEEE Smart Computing}, 2017.
\bibitem{Zhang20:onlinescheduling}
Q.~Zhang, R.~Zhou, C.~Wu, L.~Jiao, and Z.~Li, ``Online scheduling of
heterogeneous distributed machine learning jobs,'' in \emph{Proceedings of
the Twenty-First International Symposium on Theory, Algorithmic Foundations,
and Protocol Design for Mobile Networks and Mobile Computing}, 2020, pp.
111--120.
\bibitem{Mao16:DRL}
H.~Mao, M.~Alizadeh, I.~Menache, and S.~Kandula, ``Resource management with
deep reinforcement learning,'' in \emph{Proc. of ACM HotNets}, 2016.
\bibitem{Chen17:DRL}
W.~Chen, Y.~Xu, and X.~Wu, ``Deep reinforcement learning for multi-resource
multi-cluster job scheduling,'' in \emph{Proc. of IEEE ICNP}, 2017.
\bibitem{Mao18:graph-based}
H.~Mao, M.~Schwarzkopf, S.~Venkatakrishnan, and M.~Alizadeh, ``Learning
graph-based cluster scheduling algorithms,'' in \emph{Proc. of SysML}, 2018.
\bibitem{Mao19:learningscheduling}
H.~Mao, M.~Schwarzkopf, S.~B. Venkatakrishnan, Z.~Meng, and M.~Alizadeh,
``Learning scheduling algorithms for data processing clusters,'' in
\emph{Proc. of ACM SIGCOMM}, 2019.
\bibitem{Mirhoseini18:hierarchicalmodel}
A.~Mirhoseini, A.~Goldie, H.~Pham, B.~Steiner, Q.~V. Le, and J.~Dean, ``A
hierarchical model for device placement,'' in \emph{Proc. of ICLR}, 2018.
\bibitem{Mirhoseini17:deviceplacement}
A.~Mirhoseini, H.~Pham, Q.~V. Le, B.~Steiner, R.~Larsen, Y.~Zhou, N.~Kumar,
M.~Norouzi, S.~Bengio, and J.~Dean, ``Device placement optimization with
reinforcement learning,'' in \emph{Proc. of ICML}, 2017.
\bibitem{MXNet1}
\BIBentryALTinterwordspacing
V.~Kovalevskyi, ``{MXNet} distributed training explained in depth.'' [Online].
Available:
\url{https://blog.kovalevskyi.com/mxnet-distributed-training-explained-in-depth-part-1-b90c84bda725}
\BIBentrySTDinterwordspacing
\bibitem{MXNet2}
\BIBentryALTinterwordspacing
``Distributed training in {MXNet}.'' [Online]. Available:
\url{https://mxnet.incubator.apache.org/faq/distributed_training.html}
\BIBentrySTDinterwordspacing
\bibitem{Goyal17:SGD}
P.~Goyal, P.~Dolla, R.~Girshick, P.~Noordhuis, L.~Wesolowski, A.~Kyrola,
A.~Tulloch, Y.~Jia, and K.~He, ``Accurate, large minibatch {SGD}: Training
imagenet in 1 hour,'' \emph{arXiv preprint arXiv:1706.02677}, 2017.
\bibitem{Peng18:ML_EuroSys}
Y.~Peng, Y.~Bao, Y.~Chen, C.~Wu, and C.~Guo, ``Optimus: An efficient dynamic
resource scheduler for deep learning clusters,'' in \emph{Proc. of ACM
EuroSys}, 2018.
\bibitem{HDFS}
\BIBentryALTinterwordspacing
``{Apache Hadoop}.'' [Online]. Available: \url{http://hadoop.apache.org/.}
\BIBentrySTDinterwordspacing
\bibitem{Dean:2012:LSD}
J.~Dean, G.~S. Corrado, R.~Monga, K.~Chen, M.~Devin, Q.~V. Le, M.~Z. Mao,
M.~Ranzato, A.~Senior, P.~Tucker, K.~Yang, and A.~Y. Ng, ``Large scale
distributed deep networks,'' in \emph{Proceedings of the 25th International
Conference on Neural Information Processing Systems - Volume 1}, ser.
NIPS'12.\hskip 1em plus 0.5em minus 0.4em\relax USA: Curran Associates Inc.,
2012, pp. 1223--1231.
\bibitem{Cheatham05:BSP}
T.~Cheatham, A.~Fahmy, D.~Siefanescu, and L.~Valiani, ``Bulk synchronous
parallel computing-a paradigm for transportable software,'' in \emph{Hawaii
International Conference on Systems Sciences,}, 2005.
\bibitem{Bazaraa_Sherali_Shetty_93:NLP}
M.~S. Bazaraa, H.~D. Sherali, and C.~M. Shetty, \emph{Nonlinear Programming:
Theory and Algorithms}, 3rd~ed.\hskip 1em plus 0.5em minus 0.4em\relax New
York, NY: John Wiley \& Sons Inc., 2006.
\bibitem{Keras}
\BIBentryALTinterwordspacing
``Multi-worker training with keras.'' [Online]. Available:
\url{https://www.tensorflow.org/tutorials/distribute/multi_worker_with_keras}
\BIBentrySTDinterwordspacing
\bibitem{Kubernetes}
\BIBentryALTinterwordspacing
``Run deep learning with paddlepaddle on kubernetes,'' 2017. [Online].
Available:
\url{https://kubernetes.io/blog/2017/02/run-deep-learning-with-paddlepaddle-on-kubernetes/}
\BIBentrySTDinterwordspacing
\bibitem{Wang18:SpiderBoost}
Z.~Wang, K.~Ji, Y.~Zhou, Y.~Liang, and V.~Tarokh, ``Spiderboost: A class of
faster variance-reduced algorithms for nonconvex optimization,'' in
\emph{arXiv preprint arXiv:1810.10690}, 2018.
\bibitem{Hochbaum97}
D.~S. Hochbaum, \emph{Approximation Algorithms for NP-Hard Problems}, D.~S.
Hochbaum, Ed.\hskip 1em plus 0.5em minus 0.4em\relax PWS Publishing Company,
1997.
\bibitem{Buchbinder09}
N.~Buchbinder and J.~(Seffi)~Naor, \emph{The Design of Competitive Online
Algorithms via a Primal-Dual Approach}.\hskip 1em plus 0.5em minus
0.4em\relax Now Publishers Inc., feb 2009, vol.~3.
\bibitem{Vaidya87:LP}
P.~M. Vaidya, ``An algorithm for linear programming which requires
$o(((m+n)n^2+(m+n)^{1.5}n){L})$ algorithmic operations,'' in \emph{28th
Annual IEEE Syposium on Foundations of Computer Science}, 1987.
\bibitem{Li14:OSDI}
M.~Li, D.~G. Andersen \emph{et~al.}, ``Scaling distributed machine learning
with the parameter server,'' in \emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Chilimbi14:OSDI}
T.~M. Chilimbi, Y.~Suzue, J.~Apacible, and K.~Kalyanaraman, ``Project adam:
Building an efficient and scalable deep learning training system,'' in
\emph{Proc. of USENIX OSDI}, 2014.
\bibitem{Sun17:Dorm}
P.~Sun, Y.~Wen, N.~B.~D. Ta, and S.~Yan, ``Towards distributed machine learning
in shared clusters: A dynamically-partitioned approach,'' in \emph{Proc. of
IEEE Smart Computing}, 2017.
\bibitem{AmazonEC2Instances}
\BIBentryALTinterwordspacing
``Amazon ec2 instances.'' [Online]. Available:
\url{https://aws.amazon.com/ec2/instance-types}
\BIBentrySTDinterwordspacing
\bibitem{Reiss12:googlecluster}
C.~Reiss, A.~Tumanov \emph{et~al.}, ``Heterogeneity and dynamicity of clouds at
scale: Google trace analysis,'' in \emph{Proc. of ACM SoCC}, 2012.
\bibitem{Huang15:CORA}
Z.~Huang, B.~Balasubramanian, M.~Wang, T.~Lan, M.~Chiang, and D.~H. Tsang,
``Need for speed: Cora scheduler for optimizing completiontimes in the
cloud,'' in \emph{Proc. of IEEE INFOCOM}, 2015.
\bibitem{Zaharia10:USENIX}
M.~Zaharia, M.~Chowdhury, M.~J. Franklin, and et~al, ``Spark: Cluster computing
with working sets,'' in \emph{n Proc. of USENIX HotCloud}, 2010.
\bibitem{Vavilapalli13:Yarn}
V.~K. Vavilapalli, A.~C. Murthy \emph{et~al.}, ``Apache hadoop yarn: Yet
another resource negotiator,'' in \emph{Proc. of ACM SoCC}, 2013.
\bibitem{Hindman11:Mesos}
B.~Hindman, A.~Konwinski \emph{et~al.}, ``Mesos: A platform for fine-grained
resource sharing in the data center,'' in \emph{Proc. of USENIX NSDI}, 2011.
\bibitem{Jeon18:multi-tenant}
M.~Jeon, S.~Venkataraman, A.~Phanishayee, J.~Qian, W.~Xiao, and F.~Yang,
``Multi-tenant gpu clusters for deep learning workloads: Analysis and
implications,'' in \emph{MSR-TR-2018- 13}, 2018.
\bibitem{Minet18:google_trace}
P.~Minet, \'{E}ric Renault, I.~Khoufi, and S.~Boumerdassi, ``Analyzing traces
from a google data center,'' in \emph{International Wireless Communications
and Mobile Computing Conference (IWCMC)}, 2018, pp. 1167--1172.
\bibitem{Bertsekas99:Nonlinear}
D.~Bertsekas, \emph{Nonlinear Programming}, 2nd~ed.\hskip 1em plus 0.5em minus
0.4em\relax Athena Scientific, 1999.
\end{thebibliography}