MetaML: Automating Customizable Cross-Stage Design-Flow for Deep Learning Acceleration
Extracted source text ยท arXiv 2306.08746
\begin{lstlisting}[float=tp,emph={with},emphstyle={\bfseries}, language=Python, caption=The implementation of the pruning strategy using our framework., label=code:strategy]
# The pruning strategy architecture - Fig. 2(a)
from MetaML import *
# design-flow architecture
with Dataflow() as df:
join = Join() << KerasModelGen()
branch = Branch('B') << (VivadoHLS() <<
(HLSML() << (Pruning() << join)))
branch >> [join, Stop()]
# design-flow configuration
cfg = {
'KerasModelGen::train_en': False,
'Pruning::tolerate_accuracy_loss': 0.02,
'Pruning::pruning_rate_threshold': 0.02,
'B@fn': lambda metamodel:...
'HLS4ML::default_precision': 'ap_fixed<18,8>',
'HLS4ML::IOType': 'io_parallel',
'HLS4ML::FPGA_part_number': 'xc7z020clg400-1',
'HLS4ML::clock_period' : 10,
'train_test_dataset': '/mypath/target_dataset',
'train_epochs': 15,
'Stop::fn': lambda metamodel: ...
}
# run design-flow
optimised_model = df.run(cfg)
\end{lstlisting}
%\vspace{-0.5cm}
\documentclass[conference]{IEEEtran}
% Some Computer Society conferences also require the compsoc mode option,
% but others use the standard conference format.
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[conference]{../sty/IEEEtran}
\makeatletter
\def\ps@IEEEtitlepagestyle{%
\def\@oddfoot{\mycopyrightnotice}%
\def\@oddhead{\hbox{}\@IEEEheaderstyle\leftmark\hfil\thepage}\relax
\def\@evenhead{\@IEEEheaderstyle\thepage\hfil\leftmark\hbox{}}\relax
\def\@evenfoot{}%
}
\def\mycopyrightnotice{%
\begin{minipage}{\textwidth}
%\centering
\scriptsize
\copyright~2023 IEEE. Personal use of this material is permitted. Permission from IEEE must be obtained for all other uses, in any current or future media, including reprinting/republishing this material for advertising or promotional purposes, creating new collective works, for resale or redistribution to servers or lists, or reuse of any copyrighted component of this work in other works.
This work has been accepted at The 33rd International Conference on Field-Programmable Logic and Applications (FPL), 2023.
\end{minipage}
}
\makeatother
%\settopmatter{printacmref=false} % Removes citation information below abstract
%\renewcommand\footnotetextcopyrightpermission[1]{} % removes footnote with conference information in first column
\pagestyle{plain} % removes running headers
\makeatletter % changes sub sub section to have newline
%\def\subsubsection{\@startsection{subsubsection}{3}{10pt}%
% {-.5\baselineskip \@plus -2\p@ \@minus -.2\p@}%
% {3.5\p@}{\subsubsectionfont}}
\makeatother
\setlength{\textfloatsep}{2pt} % makes gap after figures smaller
\setlength{\abovecaptionskip}{1pt} % makes gap above captions smaller
%\pagenumbering{gobble} % removes page numbering
\usepackage{booktabs} % For formal tables
\usepackage{enumerate}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{multirow}
\usepackage{footnote}
\usepackage{threeparttable}
\usepackage{amsmath}
\usepackage{textcomp}
%\usepackage{multicol}
\usepackage{balance}
\usepackage{hyperref}
\usepackage[table,xcdraw]{xcolor}
\newcommand{\figref}[1]{Fig.~\ref{#1}}
\newcommand{\eqnref}[1]{(\ref{#1})}
\newcommand{\secref}[1]{Section~\ref{#1}}
\newcommand{\tabref}[1]{Table~\ref{#1}}
\newcommand{\Hsection}[1]{\vspace{0.5\baselineskip}\par\noindent\textit{#1}~\textbf{---}~}
%\usepackage[square, comma, sort&compress, numbers]{natbib}
\usepackage[square, comma, numbers]{natbib}
\DeclareMathOperator{\argmax}{arg\,max}
\DeclareMathOperator{\argmin}{arg\,min}
\usepackage{listings}
\newcommand{\mycolor}{blue}
\usepackage[colorinlistoftodos]{todonotes}
%\ifCLASSOPTIONcompsoc
% \usepackage[caption=false, font=normalsize, labelfont=sf, textfont=sf]{subfig}
%\else
%\usepackage[caption=false, font=footnotesize]{subfig}
%\fi
%\usepackage{subcaption}
\usepackage[caption=false]{subfig}
%New colors defined below
\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.58,0,0.82}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\lstdefinestyle{mystyle}{
backgroundcolor=\color{backcolour},
commentstyle=\color{codegreen},
keywordstyle=\color{magenta},
numberstyle=\tiny\color{codegray},
stringstyle=\color{codepurple},
basicstyle=\ttfamily\scriptsize
breakatwhitespace=false,
language=Python,
breaklines=true,
captionpos=b,
keepspaces=true,
numbers=left,
numbersep=5pt,
showspaces=false,
showstringspaces=false,
showtabs=false,
tabsize=2,
float=tp
}
\lstset{style=mystyle}
\begin{document}
%
% paper title
% Titles are generally capitalized except for words such as a, an, and, as,
% at, but, by, for, in, nor, of, on, or, the, to and up, which are usually
% not capitalized unless they are the first or last word of the title.
% Linebreaks \\ can be used within to get better formatting as desired.
% Do not put math or special symbols in the title.
\title{MetaML: Automating Customizable Cross-Stage Design-Flow for Deep Learning Acceleration}
\author{
\IEEEauthorblockN{
Zhiqiang Que,
Shuo Liu,
Markus Rognlien,
Ce Guo,
Jose~G.~F.~Coutinho,
Wayne Luk
}
\IEEEauthorblockA{Department of Computing, Imperial College London, UK. \{z.que, c.guo, jgfc, w.luk\}@imperial.ac.uk }
%\IEEEauthorblockA{\IEEEauthorrefmark{1}Removed for a Blind Review}
%\IEEEauthorblockA{\IEEEauthorrefmark{2} Removed for a Blind Review}
}
% conference papers do not typically use \thanks and this command
% is locked out in conference mode. If really needed, such as for
% the acknowledgment of grants, issue a \IEEEoverridecommandlockouts
% after \documentclass
% for over three affiliations, or if they all won't fit within the width
% of the page, use this alternative format:
%
%\author{\IEEEauthorblockN{Michael Shell\IEEEauthorrefmark{1},
%Homer Simpson\IEEEauthorrefmark{2},
%James Kirk\IEEEauthorrefmark{3},
%Montgomery Scott\IEEEauthorrefmark{3} and
%Eldon Tyrell\IEEEauthorrefmark{4}}
%\IEEEauthorblockA{\IEEEauthorrefmark{1}School of Electrical and Computer Engineering\\
%Georgia Institute of Technology,
%Atlanta, Georgia 30332--0250\\ Email: see http://www.michaelshell.org/contact.html}
%\IEEEauthorblockA{\IEEEauthorrefmark{2}Twentieth Century Fox, Springfield, USA\\
%Email: homer@thesimpsons.com}
%\IEEEauthorblockA{\IEEEauthorrefmark{3}Starfleet Academy, San Francisco, California 96678-2391\\
%Telephone: (800) 555--1212, Fax: (888) 555--1212}
%\IEEEauthorblockA{\IEEEauthorrefmark{4}Tyrell Inc., 123 Replicant Street, Los Angeles, California 90210--4321}}
%Therefore, To tackle the problems, we propose a FPGA-based novel approach of anomaly detection using Autoencoder and LSTM to achieve the goal.
%LSTM-based RNNs have recently been shown to be very effective for anomaly detection in standard time series applications. %\cite{malhotra2015long}.
%However, its analysis speed will be significantly slower when the data dimension increases. In addition, LSTMs struggle to accurately predict $m$-dimensional outputs when $m$ is very large.
%\cite{hundman2018detecting}
%Thus,
%In our research,
% make the title area
\maketitle
\input{sections/abstract.tex}
\input{sections/intro.tex}
\input{sections/related_work.tex}
\input{sections/approach.tex}
%\input{sections/implementation.tex}
\input{sections/evaluation.tex}
\input{sections/conclusion.tex}
%\input{sections/backup.tex}
%\input{sections/old_sections.tex}
%\input{mainbody.tex}
\footnotesize
\bibliographystyle{IEEEtran}
\bibliography{main-bibliography}
%\newpage
% that's all folks
\end{document}
\begin{abstract}
%This paper proposes a novel framework for codifying and automating optimization strategies for Deep Neural Network (DNN) hardware accelerators. Our framework addresses two main challenges in developing efficient FPGA-based DNN accelerators. The first challenge is that optimizing DNN accelerators on FPGAs requires cross-stage co-optimizations that consider both the application (DNN) and hardware design aspects together, which can be difficult due to the different design spaces of neural networks and FPGAs, respectively. The second challenge is the complex process of selecting the most effective combination, order, and tuning of optimization techniques to achieve optimal results. To overcome these challenges, we propose a modular framework, which breaks down the optimization problem into smaller parts, allowing for a more detailed and precise solution. Moreover, we introduce DNN optimization blocks, involving hardware deployment and model fine-tuning, that can be combined and reused to improve accuracy, reduce both hardware resource consumption and the complexity of deep learning models, and improve performance and efficiency. Evaluation results show that our approach can reduce the DSP resource usage by up to 92\% and LUT usage by up to 89\% on two example networks, while maintaining accuracy and without human effort or domain expertise. Compared to designs optimized by the state-of-the-art approaches, our design achieves not only higher model accuracy but also 3 times lower DSP usage, demonstrating the benefits of the proposed framework.
This paper introduces a novel optimization framework for deep neural network (DNN) hardware accelerators, enabling the rapid development of customized and automated design flows. More specifically, our approach aims to automate the selection and configuration of low-level optimization techniques, encompassing DNN and FPGA low-level optimizations. We introduce novel optimization and transformation tasks for building design-flow architectures, which are highly customizable and flexible, thereby enhancing the performance and efficiency of DNN accelerators. Our results demonstrate considerable reductions of up to 92\% in DSP usage and 89\% in LUT usage for two networks, while maintaining accuracy and eliminating the need for human effort or domain expertise. In comparison to state-of-the-art approaches, our design achieves higher accuracy and utilizes three times fewer DSP resources, underscoring the advantages of our proposed framework.
\end{abstract}
%\newpage
\section{Our Approach} \label{sec:approach}
This paper introduces a novel framework that simplifies the development of customizable design flows for optimizing deep neural networks (DNNs) on FPGA platforms. A design flow consists of a series of tasks that aim to convert a high-level specification into a final hardware design. Typically, a design flow is implemented as a multi-stage pipeline, where each stage operates on a specific model abstraction. As the pipeline progresses, the model abstraction is gradually refined and optimized, taking into account the key features of the target device. For instance, to illustrate the process, let us consider a DNN model described in Tensorflow. The framework utilizes the HLS4ML tool to translate this model into a C++ HLS model in the initial stage. Subsequently, the resulting C++ HLS model can be further transformed into a Register Transfer Level (RTL) model using Vivado HLS.
Our approach enables the optimization strategies to be codified, automating the selection and configuration of DNN and hardware optimizations. It is designed to fulfill the following requirements:
\textbf{Customizable}: Users have the freedom to customize, extend, or modify the design-flow to meet specific needs and support experimentation. They can select a set of design-flow tasks, that implement specific optimizations or transformations, arrange them in a desired order, and fine-tune their parameters to create a specific optimization flow. Additionally, users can develop their own tasks and integrate them into the design-flow.
\textbf{Cross-stage}: This refers to the breadth of optimizations and the ability to target multiple levels of abstraction, typically associated with different stages of a design-flow. More specifically, optimizations performed at the neural network level operate at the graph-level, while optimizations at the HLS C++ level involve source-level transformations.
Using our framework, design flows are programmatically generated and consist of two types of components (Fig.~\ref{fig:pipeblocks}): the pipe task and the meta-model. The pipe task serves as the basic unit of the design flow, executing specific optimizations or transformations. By interconnecting these tasks, we construct a complete design flow. The architecture of the design-flow is depicted as a cyclic directed graph where nodes symbolize tasks and edges signify dependencies between tasks, denoting the need to complete one task before initiating another.
The meta-model, on the other hand, serves as a shared space for storing the states of the design flow. This model consists of three sections: configuration, log, and model space. The configuration section (CFG) acts as a key-value store, holding the parameters of all pipe tasks in the design flow. The log section (LOG) records the runtime execution trace, aiding in debugging. Furthermore, the model space can store the generated models obtained during the execution of the design flow. In the example presented in Fig.~\ref{fig:pipeblocks}, six models are stored, covering DNN, HLS C++, and RTL abstraction levels. Each model includes supporting files, tool reports, and computed metrics.
% https://docs.google.com/drawings/d/1bM5SfGFCLphPfhn4y5qVFLgVVid5uaNU2gvQ-pYel2g/edit?usp=share_link
%https://docs.google.com/drawings/d/1Q3OecTlaum0nm7jHnb-WjoVvjxdLRfSM1q2jYUton2g/edit
% a, b
\begin{figure}[tp]
\centering
\includegraphics[width=0.80\linewidth]{img/pipeblocks_b.pdf}
\caption{A connection between an $O$-task and a $\lambda$-task. Each connection defines a unidirectional flow between a source and a target task.}
\label{fig:pipeblocks}
\end{figure}
\input{tables/pipeblocks03.tex}
\section{Reusable Pipe Tasks} \label{sec:pipeblock}
%\subsection{Reusable Pipe Blocks}
Table~\ref{table:pipeblocks} provides a list of pipe tasks that are currently implemented, along with their roles, multiplicity, and parameters. The multiplicity indicates the number of input and output connections that a task can handle. We consider two types of tasks:
\begin{itemize}
\item \textbf{$O$-task}: These are self-contained optimization tasks that enhance a given model based on specific objectives and constraints. Our current pipe task repository includes PRUNING and SCALING, which are implemented using the Keras API (version 2.9.0), and QUANTIZATION, which employs C++ source-to-source transformations via the Artisan framework~\cite{fccm20_artisan};
\item \textbf{$\lambda$-task}: These tasks perform functional transformations on the model space, such as compilation and synthesis. Examples include HLS4ML (version 0.6.0), which translates a DNN model into an HLS C++ model, and Vivado HLS (version 20.1), which translates an HLS C++ model into an RTL model.
\end{itemize}
Our framework is customizable. Different parameters (see Table~\ref{table:pipeblocks}) can be used to customise a design flow, and new design-flows can be built from existing ones to tailor to specific needs.
% \subsection{Building a Strategy}
% \input{code/designflow01}
% The Python code in Listing.\ref{code:strategy}, which is depicted in Fig.\ref{fig:design_flow}(a), executes a pruning-based strategy in three steps. Firstly, lines 4--9 use $>>$ and $<<$ operators to create a cyclic-directed design-flow architecture graph from connected blocks, identified by their default or assigned names. Secondly, lines 11--23 establish a configuration object that stores parameters to customize blocks. Parameter naming conventions determine whether parameters apply globally, to all instances of a block type, or to a single instance. Lastly, line 25 initiates the design-flow using the configuration object. This process involves validating the graph, ensuring appropriate source blocks and multiplicity constraints, generating an empty meta-model, and running a source block. Blocks are executed via a thread pool, with the scheduler assigning or stalling jobs as required. The execution concludes when the STOP block confirms all jobs are completed. Altering the architecture and parameters enables new optimization strategies.
\section{Background}
%\textcolor{blue}{Transform, Control, Analysis}
\section{Plan to remove, but can be added back if necessary}
%\subsection{Design-flow control}
\subsection{$K$-block: Bottom-up Flows}
\label{sec:bottom_up}
%\textbf{$K$-block: bottom-up flows}.
Fig~\ref{fig:design_flow}(a) and (c) reveal two distinct flows in the optimization strategies: top-down, where information flows from the DNN to the hardware stage, and bottom-up, the reverse. We have automated both flows by customizing the $K$-block BRANCH with a user-defined predicate function to activate or stop the bottom-up flow if the resulting design overmaps. The BRANCH block also support a user-supplied action function that is triggered when the predicate condition is true. For our current strategies, the action function changes the \textbf{CFG} section of the meta-model, increasing the accuracy tolerance parameters $\alpha_{p}$, $\alpha_{s}$, and $\alpha_{q}$ (if applicable) for the next iteration. Note that our current strategies in Fig.~\ref{fig:design_flow}(a) and (c) have two DSE loops: an inner-loop that is codified within each $O$-block and an outer-loop supported by the bottom-up flow. Users can develop more complex strategies by customizing the outer loop, for instance, by updating the bottom-up condition and parameter tuning.
\subsection{$K$-block: Parallel Flows}
\label{sec:parallel_flow}
%\textbf{$K$-block: parallel flows}.
Our framework enables the execution of multiple optimization paths in parallel, allowing the selection of the best outcome among them. In \figref{fig:design_flow}(b), we show a design-flow with two parallel paths, where the execution order of two $O$-blocks is changed: scaling $\rightarrow$ pruning and pruning $\rightarrow$ scaling. To support parallel branches, we use the FORK block to connect multiple strategy paths. The results from each path are then evaluated based on predefined criteria, such as accuracy or resource utilization, using the REDUCE block. For this strategy, a Pareto analysis was performed on the designs resulting from both paths, as shown in \figref{fig:strategy_results} (j). By employing this design-flow, designers can explore various optimization combinations and sequences when the outcomes of these strategies are not clear.
\subsection{Optimization Strategy Search}\label{sec:opt_search}
%By combining different methods to optimize acceleration, we increase the solution space and the chances to find more designs that can offer better trade-offs between model accuracy and hardware efficiency. To identifying the most effective combination and order of the optimization modules, we conduct all the possible candidates of the optimization strategies based on the three existing $O$-blocks by updating the design-flow automatically. When there are more $O$-blocks, a random search or a machine-learning assistant method can be adapted for an more efficient search. We leave this for our future work.
% By combining different optimization methods for acceleration, the solution space becomes larger and more diverse, offering a wider range of potential designs to explore. This increased diversity enhances the probability of discovering designs that offer an optimal balance between model accuracy and hardware efficiency.
% To identify the most effective combination and order of optimization blocks, we conduct all the possible candidates of the optimization strategies based on the three existing $O$-blocks by updating the design-flow automatically.
% This process involves systematically evaluating various combinations and sequences of the optimization blocks to identify the most promising configurations.
% When dealing with a large number of $O$-blocks, the search for optimal combinations and sequences becomes more complex and computationally intensive. To address this challenge, more efficient search methods can be utilized, such as random search or machine-learning-assisted methods. This is left for our future work.
Combining optimization methods increases diversity and potential designs, improving optimal balance between accuracy and efficiency. We systematically evaluate all candidates of optimization strategies based on our current set of $O$-blocks to identify the most effective combination and order. In particular, \figref{fig:dnn_vgg7_all_resource}(a) shows all the results after each strategy. The final optimized model after scaling, pruning and quantization is depicted as ``S$\rightarrow$P$\rightarrow$Q" design, resulting in a reduction of the DSP usage by around 92\% and LUT usage by around 89\% compared with the original design (baseline~\cite{duarte2018fast}). In addition, the latency is reduced by 35\%.
The same search is performed on the VGG7 network with results shown in~\figref{fig:dnn_vgg7_all_resource}(b).
%It should be noted that searching for the optimal combinations and sequences with multiple $O$-blocks can be computationally demanding because the size of the search space increases exponentially with the number of $O$-blocks.
The search for optimal combinations and sequences with multiple $O$-blocks can be computationally demanding due to the exponential growth of the search space with the number of $O$-blocks. Advanced search methods can replace the current brute-force search.
%Although the current brute-force search might not be longer feasible, more advanced searching methods can still be effective.
Recent studies \cite{kurek2016knowledge, ferretti2022graph, wu2021ironman}
demonstrate surrogate models can expedite the search.
%have demonstrated that optimizers can use surrogate models to capture characteristics of the search space and expedite the optimization process.
Our framework facilitates the deployment of such techniques, enabling users to interface an search algorithm with the hardware design with minimal programming effort.
\subsection{figures}
% (a)
% https://docs.google.com/drawings/d/18bGih9gp5AbckjTamtKngySg2zKSJo-OyLSa6_S5YiM/edit?usp=share_link
% https://docs.google.com/drawings/d/1gWd4gub2PRIRKTihCJ_In_TeIJOwtp9BJAJf5DYUZGk/edit?usp=share_link
%
% https://docs.google.com/drawings/d/1gWd4gub2PRIRKTihCJ_In_TeIJOwtp9BJAJf5DYUZGk/edit
\begin{figure}
\centering
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.30\linewidth]{img/pruning_strategy05.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.32\linewidth]{img/pruning_scaling_strategy03.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.24\linewidth]{img/scaling_pruning_quant_strategy02.pdf}}
\hspace*{\fill}
% \vspace{0.2cm}
\caption{(a) Pruning strategy. (b) Combined strategy of scaling and pruning, exploring the order of $O$-blocks. (c) The combined strategy of scaling, pruning and quantization. }
% \vspace{-0.1cm}
\label{fig:design_flow_bak}
\end{figure}
\begin{figure*}
\centering
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/dnn_pruning02.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/dnn_pruning_resource02.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/resnet9_pruning_acc01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/ResNet9_pruning_resource01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.28\linewidth]{img/bit_width_visualization_vgg7s0.png}}
\\
\vspace{-0.42cm}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/dnn_pruning_scale_acc01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/dnn_pruning_scale_resource01.pdf}}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/dnn_scale_after_pruning_acc01.pdf}}
% \hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.18\linewidth]{img/dnn_scale_after_pruning_resource01.pdf}}
\subfloat[]{%
\includegraphics[width=0.30\linewidth]{img/scaling_pruning_pareto02.pdf}}
%\subfloat[]{%
% \includegraphics[width=0.20\linewidth]{img/bit_width_visualization_dnn64s0.png}}
%\hspace*{\fill}
%\\
%\subfloat[]{%
% \includegraphics[width=0.22\linewidth]{img/bit_width_visualization_vgg7s0.png}}
\caption{(a) Pruning rates \& accuracy of Jet-DNN.
%(b) Resource utilization of various design candidates of Jet-DNN on Xilinx Zynq 7020.
(b) Resource utilization of Jet-DNN design candidates with pruning.
%(c) Various search pruning rate and the corresponding model accuracy for ResNet9.
(c) Pruning rates \& accuracy of ResNet9.
%(d) Resource utilization of various design candidates of ResNet9 on Xilinx U250.
(d) Resource utilization of ResNet9 design candidates with pruning.
%(e) The quantized bitwidth in VGG7 model. Only the layers with weights are quantized.
(e) Quantized bitwidth of each layers in VGG7.
%(f) Model accuracy and pruning rate for the Jet-DNN with scaling followed by pruning.
(f) Jet-DNN pruning rates \& accuracy with scaling $\rightarrow$ pruning .
(g) Resource utilization in (e) designs
%(h) Model accuracy and layer size for the Jet-DNN with pruning followed by scaling.
(h) Jet-DNN pruning rates \& accuracy with pruning $\rightarrow$ scaling .
%(i) Resource utilization of each design in (g).
(i) Resource utilization in (g) designs
%(j) Designs on the Pareto Front in the meta-model after REDUCE block with designs from different path labeling in different colors.
(j) Pareto Front meta-model designs post REDUCE $K$-block, color-coded paths.
%(k) The quantized bitwidth of the weight, bias and output data of each layer in Jet-DNN model.
}
\vspace{-0.6cm}
\label{fig:strategy_results_bak}
\end{figure*}
\begin{figure}
\centering
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.9\linewidth]{img/dnn_all_resource05.pdf}}
\hspace*{\fill}
\\
\vspace{-0.42cm}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.9\linewidth]{img/vgg7_all_resource03.pdf}}
\hspace*{\fill}
\caption{(a) The hardware resources and latency of the Jet-DNN designs after various strategies. (b) The hardware resources and latency of the VGG7 designs after various strategies.}
% \vspace{-0.12cm}
\label{fig:dnn_vgg7_all_resource_bak}
\end{figure}
\section{Related Work}\label{sec:related_work}
The field of FPGA-based DNN acceleration has rapidly expanded, leading to the development of numerous optimization techniques and tools.
% techniques are not customizable
Several co-optimization techniques have been proposed that optimize both algorithm and hardware stages for DNNs on FPGAs, as discussed in various papers~\cite{yang2019synetgy, zhang2022algorithm, hao2018deep, hao2019fpga, hao2019nais, hao2020effective, jiang2020hardware, dong2021hao, hao2021enabling} and in hardware-aware neural architecture search studies like~\cite{ney2021half, abdelfattah2020best}. These optimization strategies are often coupled with design space exploration, but are typically hardcoded and cannot be easily changed or customized.
% tools are not designed to add new optimisation strategies
Other approaches offer end-to-end software frameworks, such as Xilinx's Vitis AI~\cite{kathail2020xilinx} and Intel's OpenVINO~\cite{OpenVINO2023}, which optimize DNNs with pre-built optimizations for deployment on specific target technologies. However, they are not designed for easy addition of new optimization strategies or to search for the most effective combined strategies.
% single stage (limited scope) e to describe strategies
Furthermore, some frameworks allow developers to describe and customize DNN optimization strategies, but their scope is limited. For instance, ScaleHLS~\cite{ye2022scalehls} and SOTA-OPT~\cite{agostini2022mlir} focus on HLS-based optimization strategies and hardware designs, but their optimization scope is restricted to MLIR. TVM~\cite{chen2018tvm} is a general-purpose DNN compilation framework that offers performance portability across different types of devices, but optimizations occur solely at the graph (IR) level.
% no automated cross-stage optimisations
Moreover, frameworks, such as FINN~\cite{umuroglu2017finn}, HLS4ML~\cite{duarte2018fast}, and fpgaConvNet~\cite{venieris2016fpgaconvnet}, provide optimized hardware building blocks for FPGA-based DNN accelerators, and allow optimization strategies to be codified using these blocks. However, they do not support automated bottom-up optimization flows, where hardware stage information guides the optimization of the application (DNN) stage.
To overcome the limitations of existing optimization techniques for DNNs and hardware, our framework derives fully automated high-level optimization flows backed by reusable target-specific and target-agnostic building blocks. Our approach enables automated top-down and bottom-up flows, allowing for the creation of customized cross-stage optimization strategies for various DNN designs. Moreover, our approach integrates new and existing optimization techniques targeting various levels of abstraction, such as neural networks through graph optimizations and HLS C++ through source-to-source optimizations~\cite{fccm20_artisan}.
\section{Conclusion}
\label{sec:conclusion}
This paper presents a novel co-optimization framework for FPGA-based DNN accelerators, enabling efficient development of customized cross-stage design flows.
Our approach reduces DSP resource usage by up to 92\% and LUT usage by up to 89\% while maintaining accuracy. Compared to existing research, our approach achieves higher model accuracy and uses 3 times fewer DSP resources, highlighting the benefits of our framework.
Future work includes exploring more efficient search techniques with larger numbers of $O$-tasks, introducing control tasks to cover bottom-up and parallel flows, supporting more types of neural networks such as graph~\cite{que2022ll} and transformer~\cite{wojcicki2022accelerating} networks,
and utilizing new FPGA resources such as AI Engines~\cite{xilinx_white} and AI Tensor Blocks~\cite{langhammer2021stratix}.
\section*{Acknowledgement}
The support of the United Kingdom EPSRC (grant numbers EP/V028251/1, EP/L016796/1, EP/N031768/1, EP/P010040/1, and EP/S030069/1), CERN, AMD and SRC is gratefully acknowledged.
%\newpage
\section{Evaluation}\label{sec:evaluation}
In this section, we demonstrate how optimization strategies can be built by revising design-flow architectures, combining and reusing pipe tasks, and modifying their configuration.
%(Sections~\ref{sec:single_opt} and~\ref{sec:multi_opt}). We compare evaluation results with other approaches in Section~\ref{sec:discussion}.
%We explain flow control in Sections~\ref{sec:bottom_up} and~\ref{sec:parallel_flow}. We discuss optimization search strategies and compare evaluation results with other approaches in Sections~\ref{sec:opt_search} and~\secref{sec:discussion}, respectively.
%In this section, we illustrate how different optimization strategies can be realized by revising design-flow architectures, for instance, combining and reusing existing pipe tasks, as well as modifying their configuration (Sections~\ref{sec:single_opt} and ~\ref{sec:multi_opt}). We then explain how we control the flows in Sections~\ref{sec:bottom_up} and ~\ref{sec:parallel_flow}. This is followed by \secref{sec:all_three} which presents a software-hardware co-optimization strategy that combines scaling, pruning and quantization. We conclude this section with a discussion in~\secref{sec:discussion}.
\subsection{Experimental Setup}
Experiments were conducted in Python 3.9.15 with benchmark workloads from typical DNN applications, including jet identification~\cite{duarte2018fast, moreno2020jedi} (Jet-DNN), image classification using VGG7~\cite{simonyan2014very} and ResNet9~\cite{he2016deep} networks. The datasets used are: Jet-HLF~\cite{duarte2018fast}, MNIST~\cite{lecun1998gradient} and SVHN~\cite{netzer2011reading}, respectively. The jet identification task targeted FPGA-based CERN Large Hadron Collider (LHC) triggers with a 40 MHz input rate and a response latency of less than 1 microsecond. Default frequencies were 100MHz for Zynq 7020 and 200MHz for Alveo U250 and VU9P, and the HLS4ML task used 18-bit fixed-point precision with 8 integer bits.
% The experiments were conducted using Python version 3.9.15 and benchmark workloads were selected from typical DNN applications, as presented in Table~\ref{table:benchmark}. These applications include jet identification using MLP networks~\cite{duarte2018fast, moreno2020jedi}, as well as image classification using VGG~\cite{simonyan2014very} and ResNet~\cite{he2016deep} networks. The datasets used in the experiments consist of MNIST~\cite{lecun1998gradient}, SVHN~\cite{netzer2011reading}, and jet HLFs~\cite{duarte2018fast, moreno2020jedi}. The jet identification task was designed for FPGA-based CERN Large Hadron Collider (LHC) triggers and was able to handle a 40 MHz input rate with a response latency of less than 1 microsecond. The default frequencies used in the experiments were 100MHz for Zynq 7020 and 200MHz for Alveo U250 and VU9P. The HLS4ML task employed 18-bit fixed-point precision with 8 integer bits.
%\subsection{$O$-task: Single Optimization Task}\label{sec:single_opt}
\subsection{Optimization Strategies}\label{sec:single_opt}
In this subsection, we initially focus on three strategies, each supported by a single $O$-task. Then, we shift our focus towards combined strategies that utilize multiple $O$-tasks.
%We describe the effects of each $O$-task separately, while in the next subsection, we discuss combining them.
%\input{tables/benchmarks.tex}
\textbf{Pruning strategy}.
Pruning is a technique that improves the performance and efficiency of neural networks by removing insignificant weights.
Our framework includes an implementation of this technique through the PRUNING $O$-task. A design-flow which employs the PRUNING $O$-task is illustrated in Fig.~\ref{fig:design_flow}(a). This optimization task gradually zeroes out weights during training to create a more compact and efficient network while maintaining accuracy. In addition, it supports auto-pruning, which automatically determines the highest pruning rate while maintaining a given level of accuracy loss. Formally, the objective of this $O$-task is defined as:
\begin{equation}
\begin{footnotesize}
\begin{aligned}
& \underset{}{\text{maximum}}
& & Pruning\_rate \\
& \text{subject to}
& & Accuracy\_loss(Pruning\_rate) \leq \alpha_p
\end{aligned}
\end{footnotesize}
\end{equation}
Starting at 0\% pruning rate, the auto-pruning algorithm obtains initial accuracy
$Acc_{p0}$ at step~1 (s1).
It then uses a binary search approach, increasing or decreasing the pruning rate based on whether the accuracy loss is within a user-defined tolerance ($\leq \alpha_{p}$). The algorithm terminates when the rate difference is below a threshold ($\beta_{p}$). The number of steps is determined by $1+log_{2}(1/\beta_{p})$. Algorithm search steps and direction are shown in Fig.~\ref{fig:auto_pruning}.
Both the tolerance ($\leq \alpha_{p}$) and threshold ($\beta_{p}$) values are initially set to 2\% in this work. %The algorithm is illustrated in Fig.~\ref{fig:auto_pruning}, and
The effectiveness of the auto-pruning algorithm is demonstrated in Fig.~\ref{fig:strategy_results}. Fig. 4(a) and (b) depict the pruning rate and accuracy for Jet-DNN and ResNet9 in each step, while Fig. 4(c) and (d) show the resources utilization. As the pruning rate increases, hardware resource requirements, particularly DSPs and LUTs, decrease, leading to improved FPGA performance. The design candidate with the highest pruning rate within the allowed tolerance is selected.
% (a)
% https://docs.google.com/drawings/d/1RMC905EmaWLEAHFuIfKadZEiPiA0VoNRyJ4e1b6sYwM/edit?usp=sharing
% https://docs.google.com/drawings/d/1XTSx8BV6mIDOjejiVJzlDF99m9jcvT4c-jPO0_4Bydw/edit?usp=sharing
% https://docs.google.com/drawings/d/1rJNBwltv0pmNqnSzu5EIFHH3uRPQiNpFQjQNekRP7dc/edit?usp=sharing
\begin{figure}
% \vspace{-0.3cm}
\centering
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.44\linewidth]{img/pruning_strategy_a02.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.28\linewidth]{img/scaling_pruning_quant_strategy_a01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.28\linewidth]{img/scaling_pruning_quant_strategy_b01.pdf}}
\hspace*{\fill}
% \vspace{0.2cm}
\caption{(a) Pruning strategy.
%(b) Combined strategy of scaling and pruning, exploring the order of $O$-tasks.
(b) The combined strategy of scaling, pruning and quantization.
(c) The combined strategy with a different order of $O$-tasks.
}
%\vspace{-1.5cm}
\label{fig:design_flow}
\vspace{0.2cm}
\end{figure}
% a/b/c/d
% https://docs.google.com/drawings/d/1LITQ6iUaKzns4Ov-5o1hatKcYCzojA7SZ2455dSsOEo/edit?usp=share_link
% https://docs.google.com/drawings/d/1LITQ6iUaKzns4Ov-5o1hatKcYCzojA7SZ2455dSsOEo/edit?usp=share_link
% https://docs.google.com/drawings/d/1kcL-_9dCiM6kxITpDKcIwppiBt4_O1d2IdWRsQNmSCA/edit?usp=share_link
% https://docs.google.com/drawings/d/1zL_MBbEh3A08n6iYpvFzOMaSWnGH13OwgLMCnyLb1jo/edit?usp=share_link
% \begin{figure}
% \vspace{-0.2cm}
% \centering
% \subfloat[Jet-DNN\label{fig:lhc_dnn_pruning}]{%
% \includegraphics[width=0.80\linewidth]{img/lhc_dnn_pruning05.pdf}}
% \hspace*{\fill}
% \subfloat[Jet-CNN\label{fig:lhc_cnn_pruning}]{%
% \includegraphics[width=0.48\linewidth]{img/lhc_cnn_pruning01.pdf}}
% \\
% \vspace{-0.36cm}
% \subfloat[VGG7\label{fig:vgg7_pruning}]{%
% \includegraphics[width=0.48\linewidth]{img/lhc_dnn_pruning01.pdf}}
%\hspace*{\fill}
% \subfloat[ResNet9\label{fig:resnet8_pruning}]{%
% \includegraphics[width=0.80\linewidth]{img/resnet8_pruning01.pdf}}
% \vspace{0.2cm}
% \caption{ The auto-pruning algorithm applied to models with binary search direction shown. Omitting step s1 for visibility. The blue arrow indicates an accuracy loss $>$ user threshold; red denotes the optimal pruning rate. }
% \label{fig:auto_pruning}
% \end{figure}
% https://docs.google.com/drawings/d/1DH2dKiBfmLvpV3Fo2YWsfBpW6WmL6bR66Osrk03leCM/edit?usp=sharing
\begin{figure}
\begin{center}
\includegraphics[width=0.95\linewidth]{img/LHC_DNN_ResNet9_FPL01.pdf}
%\includegraphics[width=0.9\linewidth]{img/jedi_net03.pdf}
\vspace{-0.3cm}
\end{center}
\caption{The auto-pruning algorithm applied to models, (a) Jet-DNN and (b) ResNet9, with binary search direction shown. Omitting step s1 for visibility. The blue arrow indicates an accuracy loss $>$ user threshold; red denotes the optimal pruning rate. }
\label{fig:auto_pruning}
\end{figure}
% https://docs.google.com/drawings/d/1hnTQ89omvv73Tl-kG-n3LZX-SMFa8eNfOvDuuJvS3w4/edit?usp=sharing
\begin{figure}
\begin{center}
\includegraphics[width=1.0\linewidth]{img/pruning_results_fpl.pdf}
\end{center}
\vspace{-0.1cm}
\caption{(a) Pruning rates and accuracy of Jet-DNN.
(b) Resource utilization of Jet-DNN design candidates with pruning on Xilinx Zynq 7020.
(c) Pruning rates and accuracy of ResNet9.
(d) Resource utilization of ResNet9 design candidates with pruning on Xilinx U250.}
\label{fig:strategy_results}
\vspace{0.1cm}
\end{figure}
% f, g, h, i,j
% https://docs.google.com/drawings/d/1pvUmqb5l-Ufo6vRvLBpOA-Amg9LlYXwP6Qh-oR7evnk/edit?usp=share_link
% https://docs.google.com/drawings/d/1JCIkY_xKqzHJ0FBOBZxsG8owN2pe1gZKNODX4c9CNXI/edit?usp=share_link
% https://docs.google.com/drawings/d/1UgHHzk3CtQuo7fYh7wT4YCW6tTsQs1bVKZZdwdeAxac/edit?usp=share_link
% https://docs.google.com/drawings/d/1m71oPluaW2QvfEW9p86NCQYeUGtqOruQvr3MKIKz8KU/edit?usp=share_link
% https://docs.google.com/drawings/d/1HnzOJKvK9zPeJKcAXblqph1YOIYqEfRjD5Pa835HAR8/edit?usp=sharing
%\begin{figure}
% \centering
% \subfloat[]{%
% \includegraphics[width=0.33\linewidth]{img/dnn_pruning_scale_acc01.pdf}}
%\hspace*{\fill}
% \subfloat[]{%
% \includegraphics[width=0.33\linewidth]{img/dnn_pruning_scale_resource01.pdf}}
% \subfloat[]{%
% \includegraphics[width=0.33\linewidth]{img/dnn_scale_after_pruning_acc01.pdf}}
% \hspace*{\fill}
% \subfloat[]{%
% \includegraphics[width=0.18\linewidth]{img/dnn_scale_after_pruning_resource01.pdf}}
% \subfloat[]{%
% \includegraphics[width=0.30\linewidth]{img/scaling_pruning_pareto02.pdf}}
%\subfloat[]{%
% \includegraphics[width=0.20\linewidth]{img/bit_width_visualization_dnn64s0.png}}
%\hspace*{\fill}
%\\
%\subfloat[]{%
% \includegraphics[width=0.22\linewidth]{img/bit_width_visualization_vgg7s0.png}}
% \caption{
% (a) Jet-DNN pruning rates \& accuracy with scaling $\rightarrow$ pruning .
% (b) Resource utilization in (e) designs
% (c) Jet-DNN pruning rates \& accuracy with pruning $\rightarrow$ scaling .
% }
% \vspace{-0.6cm}
% \label{fig:combined_strategy_results}
%\end{figure}
\textbf{Scaling strategy.} To accommodate a large DNN design on an FPGA, our framework supports the SCALING $O$-task which automatically reduces the layer size while tracking the accuracy loss $\alpha_s$. The search stops when the loss exceeds $\alpha_s$.
%or when the design can fit within the target FPGA while maintaining the desired accuracy.
This parameter can be adjusted to achieve further size reduction with minimal impact on accuracy. This work sets $\alpha_s$ to 0.05\%, which allows for model size reduction with negligible accuracy loss.
\textbf{Quantization strategy.} Our framework supports the QUANTIZATION $O$-task to automate mixed-precision quantization for networks. It operates at the HLS C++ level, providing more direct control over hardware optimizations and reducing unintended side effects when translating DNN models to HLS C++ using tools such as HLS4ML.
%Although FPGA customizability creates a vast search space of quantization configurations, dependencies between layers help limit design options. For example, if a convolutional layer has an 8-bit result type, a directly subsequent max-pooling layer does not need 16 bits and vice versa. This $O$-task gradually decreases the bit-widths of weights, biases, and layer outputs, given an acceptable accuracy loss $\alpha_q$.
The resulting precision configuration is directly instrumented into the C++ kernel, and a co-design simulation evaluates the accuracy of the quantized model.
If the accuracy loss is within tolerance ($< \alpha_q$), this process is repeated. This work sets $\alpha_q$ to 1\%.
%The precision of VGG7 model's layers after quantization with $\alpha_q$ set to 1\% is shown in Fig.~\ref{fig:strategy_results}(e).
% https://docs.google.com/drawings/d/1Daa8IhAh8mF7NldtMb97p30nr7pGgQ6eUUIZLJgUL_s/edit?usp=sharing
\begin{figure}
\begin{center}
\includegraphics[width=1.0\linewidth]{img/dnn_scale_pruning_results_fpl.pdf}
\end{center}
\vspace{-0.1cm}
\caption{
(a) Jet-DNN accuracy and pruning rates with scaling then pruning.
(b) Jet-DNN accuracy and layer size with pruning then scaling.}
\label{fig:combined_strategy_results}
\vspace{0.1cm}
\end{figure}
\textbf{Combining $O$-tasks.}
%\subsection{$O$-task: Combining Optimization Tasks}\label{sec:multi_opt}
%Combining Optimization tasks}\label{sec:multi_opt}
With our framework, new strategies can be derived by building and revising the design-flow architecture. For instance, by inserting the SCALING $O$-task before the PRUNING $O$-task in Fig.~\ref{fig:design_flow}(a), a custom combined strategy is created with results shown in Fig.~\ref{fig:combined_strategy_results}(a). The new optimal pruning rate is 84.4\%, lower than the previous 93.8\%, due to reduced redundancy from the preceding scaling task. By switching the order of the $O$-tasks, a different optimization is achieved, resulting in a 0.7\% accuracy drop after one scaling step, as seen in Fig.~\ref{fig:combined_strategy_results}(b). Moreover, the three optimization $O$-tasks, pruning, scaling, and quantization, can be integrated into a single automated cross-stage strategy to enhance both performance and hardware efficiency, as illustrated in Fig.~\ref{fig:design_flow}(b) and (c).
%We discuss the effects of different design-flow architectures in Section~\ref{sec:opt_search}.
% With our framework, it is easy to cascade different optimization tasks by simply modifying the design-flow architecture. For example, inserting a scaling $O$-task before the pruning $O$-task in \figref{fig:design_flow}(a) creates a custom combined strategy with results shown in \figref{fig:strategy_results}(f)-(g).
% The new optimal pruning rate is 84.4\% (\figref{fig:design_flow}(f)), lower than the previous 93.8\% (\figref{fig:design_flow}(a)) due to reduced redundancy from the preceding scaling task. By switching the $O$-tasks order, a different optimization is achieved, resulting in a 0.7\% accuracy drop after one scaling step, as seen in \figref{fig:strategy_results}(h). Moreover, the three optimization $O$-tasks, pruning, scaling, and quantization, can be integrated into a single automated cross-stage strategy to improve both performance and hardware efficiency, as illustrated in \figref{fig:design_flow}(c).
% https://docs.google.com/drawings/d/1aZOOKmluX_QTShldzXHURcg9EzyHuzBmutCnHVTKeyY/edit?usp=share_link
%\begin{figure}%[tp]
% \centering
% \includegraphics[width=0.9\linewidth]{img/dnn_all_resource05.pdf}
% \caption{ The hardware resources and the latency of the Jet-DNN designs after different strategies on a Zynq 7020 FPGA. The clock period is 10 ns. }
% \label{fig:dnn_all_resource}
%\end{figure}
% https://docs.google.com/drawings/d/1vFn_MRg9-6zVi1nuA0fjESslmnl2LJndPLGPp02FfZE/edit?usp=share_link
%\begin{figure}%[tp]
% \centering
% \includegraphics[width=0.9\linewidth]{img/vgg7_all_resource03.pdf}
% \caption{ The hardware resources and the latency of the VGG7 designs after different strategies using a U250 FPGA. The clock period is 5 ns. }
% \label{fig:vgg7_all_resource}
%\end{figure}
\iffalse
\fi
%\subsection{Discussion and Comparison}\label{sec:discussion}
%Our evaluation results indicate that the combined strategy typically gets a better result than employing a single optimization technique. In addition, the order in which these optimization techniques are applied plays a crucial role. Applying the same optimization techniques in a different order produces varying final results, as shown in~\figref{fig:dnn_vgg7_all_resource}.
%\subsection{Discussion and Comparison}\label{sec:discussion}
\textbf{Discussion and Comparison.}
Our evaluation results indicate that our combined $O$-task optimization strategy typically outperforms single $O$-task techniques. Furthermore, the order in which these optimization techniques are applied plays a crucial role, as different orders produce varying final results.
%as depicted in~\figref{fig:dnn_all_resource}.
%To demonstrate the advantages of our framework, we compare the design flow results to those of original Jet-DNN~\cite{duarte2018fast} using HLS4ML, LogicNets~\cite{umuroglu2020logicnets}, QKeras-based Q6~\cite{coelho2021automatic} and AutoQKeras-based QE and QB~\cite{coelho2021automatic} in~\tabref{table:cmp_fpga}. These studies target low-latency, low-resource, fully unfolded FPGA implementations. All designs use the same architecture, except the JSC-L which uses a larger one.
%For JSC-M and JSC-L, accuracy is 70.6\% and 71.8\%, over 3.8\% lower than our "S+P+Q" design with $\alpha_q$ as 0.01. Compared to Q6, our design has 0.8\% higher accuracy, 2.5 times fewer DSP tasks, and 5.7 times fewer LUTs. Our design also outperforms QE and QB, with over 3.3\% higher accuracy and lower resource usage. Furthermore, our design has lower latency than all Q6, QE and QB designs, showcasing our framework's benefits.
To highlight the advantages of our framework, we compared our design flow results to those of other studies targeting low-latency, low-resource, fully unfolded FPGA implementations, including original Jet-DNN~\cite{duarte2018fast} using HLS4ML, LogicNets~\cite{umuroglu2020logicnets}, QKeras-based Q6~\cite{coelho2021automatic}, and AutoQKeras-based QE and QB~\cite{coelho2021automatic}. All designs use the same architecture, except for JSC-L, which employs a larger architecture. Our "S+P+Q" design with $\alpha_q$ set to 0.01 achieves an accuracy of 74.1\% for JSC-S, outperforming JSC-M and JSC-L, which have accuracies of 70.6\% and 71.8\%, respectively. Compared to Q6, our design has 0.8\% higher accuracy, uses 2.5 times fewer DSP blocks, and 5.7 times fewer LUTs. Furthermore, our design outperforms both QE and QB, achieving over 3.3\% higher accuracy and lower resource usage. Our design also boasts lower latency than all Q6, QE, and QB designs, demonstrating the benefits of our framework. Inspired by the AutoQKeras design QB, which minimizes model bit consumption, we further optimize our design by tuning the parameters, such as $\alpha_q$. Increasing the quantization $O$-task's tolerant accuracy loss ($\alpha_q$) from 1\% to 4\% results in a smaller model with DSP usage 3 times lower than AutoQkeras' most efficient model, QE (see~\tabref{table:cmp_fpga}).
%The DSP usage drops to 23, which is three times lower than AutoQKeras' most resource-efficient model, QE, as shown in~\tabref{table:cmp_fpga}.
Although the model accuracy decreases to 72.8\%, it remains higher than the optimized AutoQKeras designs QB and QE.
\input{tables/cmp_fpga03.tex}
% \section{Quant in HLS}
% I believe the main reason was that I found that the bit-width values in the HLS C++ project did not always completely correspond to what they had been specified as at the python level. If I remember correctly, the bias-precision, for instance, was not changed at the HLS-level regardless of what value it was assigned at the python-level.
% I also had a general idea that it is better to make modifications at a lower level, where I knew exactly what modifications were being made, than at a higher level where there could be unknown side-effects as a result of the changes made. Let me know if this does not make sense to you, and say hi to Gabriel for me :)
\section{Introduction}
%% quantisation: float => custom fixed int
%% quantization-aware training: better accuracy, quantisation is performed during training
%% post-training quantization: no training data, only weights (activations?) are quantised
The field of deep learning has witnessed unprecedented growth in recent years, driven by the increasing demand for efficient and high-performance applications~\cite{lecun2015deep}. Consequently, FPGA-based deep neural network (DNN) accelerator design and optimization have gained significant attention~\cite{zhang2015optimizing, que2021recurrent, zhang2020dnnexplorer}. The development of an efficient FPGA-based DNN design requires a diverse skill set that combines expertise in machine learning with low-level knowledge of the target hardware architecture~\cite{sze2017efficient}. Optimizing these DNN designs is a complex process, as it involves balancing competing objectives. On one hand, high accuracy during inference is crucial from an application perspective. On the other hand, the design must be optimized for the underlying hardware architecture, meeting power, latency and throughput requirements while fitting into the FPGA device~\cite{coelho2021automatic, que2021accelerating}. Achieving an optimal balance between these conflicting objectives requires careful consideration and effective optimization strategies~\cite{wang2019deep}.
In this paper, we focus on the key challenge of codifying optimization strategies that automate the selection and configuration of low-level optimization techniques covering multiple abstraction levels from DNN optimizations to FPGA optimizations. Achieving optimal results for a given problem is complex~\cite{deng2020model}. It is challenging to identify the most effective combination, order and tuning of optimization techniques to ensure optimal outcomes~\cite{zhang2020dnnexplorer, xu2020autodnnchip}. In order to address this challenge, we propose the following contributions:
\begin{enumerate}
\item A co-optimization framework for FPGA-based DNN accelerators, which includes novel building tasks that enable rapid development of customized design flows, automating the entire design iteration process (Section~\ref{sec:approach}).
\item A library of reusable optimization and transformation tasks designed to be customizable and flexible, and that can be easily integrated into our co-optimization framework. Some of the tasks in our library are specific to certain applications and/or target technologies, while others are agnostic, providing versatility and adaptability to the framework (Section~\ref{sec:pipeblock}).
\item The evaluation of the proposed framework using multiple benchmarks and different optimization strategies. This evaluation provides insights into the effectiveness of the framework and its optimization modules under different scenarios (Section~\ref{sec:evaluation}).
\end{enumerate}
We present related work in~\secref{sec:related_work} and conclude this paper and present future work in~\secref{sec:conclusion}.
% . With this approach, each optimization module can be self-contained and independent of other modules, simplifying individual testing and verification. This can help minimize design errors, enhance overall system reliability, and facilitate the introduction of new optimization techniques;
% I think we should also highlight that we aim to automate this selection and permutation of these blocks to achieve efficient designs.
% % There are little work about the automation of modularized optimization techniques to optimize FPGA-based DNN accelerators.
% For the selection of the O-blocks, we could say that our work uses greedy algorithm to try all the possible permutation of the existing optimization blocks. But currently there are only two Python-space O-blocks that are scaling (S) and pruning (P). And one HLS-space O-block which is quantization (Q). Since we have to perform HLS-space block after hls4ml, so Q should be after S and P. Thus, the total number of the permutation cases is 2. They are S->P->Q and P->S->Q. I guess this might be too small and I am still thinking if I can add one more HLS-space O-block, such as folding.
\newpage
\section{Related Work}\label{sec:related_work}
TVM (Tensor Virtual Machine) framework provides a modularized optimization pipeline including a range of optimizations, such as quantization and kernel fusion. However it focus more on the software-based optimizations but does not considering various hardware design spaces, such as various HLS or RTL design options. This work also provides hardware level optimization strategy blocks, such as mixed-precision quantization and folding.
This work also proposes a set of hardware-level optimization strategies, such as mixed-precision quantization and folding, which are automatically applied to an HLS-based DNN model. These strategies are implemented using Artisan, a source-to-source transformations tool that manipulates the hardware designs in HLS C++. They can be combined with algorithm level optimization strategies to create a customized optimization pipeline, enabling efficient algorithm and hardware co-optimizations and resulting in a highly optimized designs on FPGAs.
Xilinx's Vitis AI and Intel's OpenVINO provide a set of pre-built optimization techniques that can be applied to DNN models, but they are not designed in a modular fashion, leading to difficult add new optimization strategies or demonstrate the combination of various optimization techniques.
FINN and HLS4ML provide a library of optimized building blocks that can be easily reused in different designs, allowing users to build FPGA-based DNN accelerators with high performance and energy efficiency. This work provide a framework with a library of optimization strategy blocks which can be reused and combined, targeting various DNN designs to achieve high performance and energy efficiency. FINN and HLS4ML can be an useful buliding block of our framework.
\cite{hao2018deep} introduces the co-design of DNN and hardware accelerator, which co-optimizes the software and hardware simultaneously. Some subsequent studies~\cite{hao2019fpga, hao2019nais, yang2019synetgy, hao2020effective, jiang2020hardware, dong2021hao, hao2021enabling, zhang2022algorithm} target co-design for various applications. However, these optimizations are fused with the design space exploration and many of them target a specific application domain. It could be difficult to reuse these optimization blocks for especially future no-existing machine learning designs.
DNNExplorer~\cite{zhang2020dnnexplorer} is an automation tool to enable fast accelerator architecture exploration based on both dedicated pipeline structure and generic reusable structure to provide improved performance.
HAO~\cite{dong2021hao} proposes to jointly optimize the neural architecture, quantization and hardware design to minimize latency with integer programming formulation.
\cite{chitty2022neural} presents a comprehensive discussion about hardware-aware neural architecture search (HW-NAS) to automate the co-search of neural algorithm and hardware specification.
%for the search algorithm to prune the design space.
\cite{roorda2022fpga} explores FPGA architectures for DNN acceleration.
LogicNets~\cite{umuroglu2020logicnets} maps DNNs to fine-grained FPGA building blocks to obtain efficient implementations for extreme-throughput applications. \cite{coelho2021automatic} presents QKeras to support a quantization-aware training technique to achieve high hardware efficiency. This work achieves higher performance than using LogicNets~\cite{umuroglu2020logicnets}, QKeras~~\cite{umuroglu2020logicnets} and AutoQkeras~\cite{umuroglu2020logicnets}.
%There are also many HLS-base optimization approachs, such as supporting task-parallelization~\cite{chi2021extending}, dynamic scheduing~\cite{josipovic2018dynamically} and
TVM~\cite{chen2018tvm} is a general-purpose compilation framework for deep learning that aims to decouple the algorithm definition from its schedule. HeteroCL~\cite{lai2019heterocl} decouples the algorithm from a temporal compute schedule while HeteroFlow~\cite{xiang2022heteroflow} decouples data placement and co-optimization with other hardware customizations such as tiling and data quantization. All of these frameworks target overlay-base accelerators. Other frameworks, such as HLS4ML~\cite{duarte2018fast}, FINN~\cite{umuroglu2017finn} and fpgaConvNet~\cite{venieris2016fpgaconvnet} map high level Python-based neural networks to FPGAs in dedicated circuit designs.
The studies mentioned in this section are orthogonal to our proposed approach, which focuses on combining and customizing device-independent and device-specific cross-stage optimizations, incorporating new and existing approaches.
% \begin{figure}%[ht]
% \begin{center}
% \includegraphics[width=0.9\linewidth]{img/design_flow.jpg}
% \end{center}
% \caption{Optimizations Flow (TODO, redraw to increase the image quality)}
% \label{fig:design_flow}
% \end{figure}
%\subsection{Related Work}
\section{Our Approach} \label{sec:approach}
\subsection{Design-Flow}
We define a design-flow as a sequence of processes that transform a high-level model into a low-level model which is capable of running, potentially optimized, on a specific hardware platform. A \textit{model}, on the other hand, is a design refinement associated to a design-flow stage, which operates on a specific level of abstraction and model of computation. Examples of models associated to different stages include: a DL model which captures the neural network architecture, a C++ HLS model which describes hardware behavior using software description, and an RTL model which captures a synchronous digital circuit by representing connections between registers and logical operations. Each stage has its own concerns and optimization techniques.
Current design-flows are built as pipelines that cover multiple stages. As we reach the lower stages of the pipeline, we gradually lower the model abstraction, specializing towards the target machine until we reach the final executable. For instance, using the HLS4ML framework, we can build a design-flow that translates a DL model into C++ HLS code. In turn, with Vivado HLS we synthesize C++ HLS code to an RTL model. In addition, we can use feedback cycles to allow information accrued at the lower stages of a pipeline to feed into its higher stages. This information allows optimizations to refine their strategies by employing more accurate metrics that were not previously available.
We contend that current design-flow mechanisms do not scale well with the extremely large design-space that stems from mapping DL architectures to reconfigurable architectures. In particular:
\begin{itemize}
\item[] \textbf{(C1) Unstructured compilation artifacts.} During the execution of a design-flow, considerable amounts of data are processed and generated, which we call \textit{design-flow artifacts}. These artifacts may include models from different stages, support files and tool reports. All these artifacts are typically fragmented and not structured in a coherent way.
\item[] \textbf{(C2) Limited scope.} Typically, each design-flow stage has a restricted view and control over the artifacts it can operate on, focusing on a single stage (e.g., either at DL level or at C++ but not both). Thus, each stage remains oblivious about all other artifacts generated up to the point of their execution. For instance, AutoKeras focuses only on DL models, while Vivado HLS optimization only focuses on C++ HLS.
%HLS4ML provides limited support for both domains by supporting accuracy analysis using custom fixed-point integers, but not additional hardware metrics.
\item[] \textbf{(C3) Limited modularity.} Due to sheer size of the design space, there is no single recipe to optimize a DL model to an FPGA design. Thus multiple optimization strategies can be realized by combining different techniques and parameters to meet desired requirements, such as accuracy, resource area and performance, but can also produce an optimized design in a reasonable amount of time. Each framework, such as QKeras and HLS4ML, has its own interface and parameters, making it difficult to combine components without ad hoc code adapters to translate between interfaces.
\item[] \textbf{(C4) Limited traceability.} As a result of a complex design-space exploration strategy that can take hours to complete and the large volume of design-flow artifacts produced during this time, it is often difficult to explain how the end-result was produced, specially in cases where there is evidence of bugs: which parts of the design-flow misbehaved? Key issues when debugging design-flows include the lack of traceability of artifacts produced, as well as the inability to replay a design-flow from a specific point (checkpointing).
\end{itemize}
In this paper, we address these technical challenges by proposing an approach that hinges on two mechanisms: the \textbf{meta-model} and the \textbf{pipe block}. We detail both mechanisms in the following subsections.
%https://docs.google.com/drawings/d/1otwTGTE-WFJpWIBUUN6kqYy2-vy7La48QeGxbWY0XuQ/edit?usp=sharing
\begin{figure}[tp]
\centering
\includegraphics[width=1\columnwidth]{img/meta-model.pdf}
\caption{An example of a meta-model structure. The meta-model is a static snapshot of the design-flow runtime execution at a specific point in time, which captures the history of each stored artifact. This allows easier debugging as well as checkpointing. The meta-model contains 3 parts: (I) \textbf{configuration} which contains the design-flow parameters supplied by the user, (II) \textbf{log} which contains historic data and (III) \textbf{model space} which contains a repository of all committed models during design-flow execution.}
\label{fig:meta-model}
\end{figure}
\subsection{Meta-Model}
In our approach, we extend the \textit{model} concept and replace it with the \textit{meta-model}. Similar to its counterpart, the \textit{meta-model} is the basic data unit that streams into every part of the design-flow and is produced as the result. However, regardless of stage, the input and output of every component of our design-flow is the same: a meta-model.
The meta-model is designed to capture a static and complete snapshot of a design-flow runtime since the beginning of its execution. It captures all the artifacts and historic data in a structured manner, allowing any stored data to be inspected at any time. As illustrated in Fig.~\ref{fig:meta-model}, the meta-model contains 3 parts: \textbf{(I)}~configuration, \textbf{(II)}~log and \textbf{(III)}~model space.
The \textit{model space} inside a meta-model references all models generated during the execution of a design-flow at a specific point in time. Each stored model is versioned, and models derived by different stages can co-exist in the same model space. In our figure example, we have stored three types of models: DL, C++ HLS and RTL.
An important feature of the meta-model is that it keeps track of the history of all the models in its \textit{log} section (\textbf{C4}). This is analogous to the git repository log which contains the history of all commits. As can be seen in the example of Fig.~\ref{fig:meta-model}, each model has tracing information about where it is derived from and which design-flow part produced it. A model can be a \textit{variation} of an existing model, both belonging to the same type. Or it can be a \textit{specialization} in which we lower the abstraction closer to the target platform. For instance, model \textit{v2} is a variation of \textit{v1} by applying a neural architecture transformation (e.g. pruning), model \textit{v1.2} is a variation of model \textit{v1.1} using a source-to-source C++ transformation, while model \textit{v2.1} is a specialization of model \textit{v2} using HLS4ML.
Each model in the model space references data in a physical storage space. This storage space encapsulates all the model supporting files, as well as tool reports and computed metrics. Every model and its history can be accessed and manipulated through programmatic access via an API (\textbf{C1}). We provide more details about the implementation in Section~\ref{sec:implementation}.
%fused figure: https://docs.google.com/drawings/d/1Q3OecTlaum0nm7jHnb-WjoVvjxdLRfSM1q2jYUton2g/edit
% a, b
% https://docs.google.com/drawings/d/1bM5SfGFCLphPfhn4y5qVFLgVVid5uaNU2gvQ-pYel2g/edit?usp=share_link
% https://docs.google.com/drawings/d/1bM5SfGFCLphPfhn4y5qVFLgVVid5uaNU2gvQ-pYel2g/edit?usp=share_link
\begin{figure}[tp]
\centering
\subfloat[\label{fig:pruning}]{%
\includegraphics[width=0.5\linewidth]{img/pipeblocks_a01.pdf}}
\\
\subfloat[\label{fig:pruning}]{%
\includegraphics[width=0.8\linewidth]{img/pipeblocks_b01.pdf}}
\vspace{0.2cm}
\caption{(a) A pipe block has a uniform interface allowing any two pipe blocks to be connected (although there may be constraints about how many connections a block can handle); (b) A connection between a $\lambda$-block and a $\kappa$-block. A $\lambda$-block typically updates the model space by adding new model variations or specializations. A $\kappa$-block on the other hand, manages the control flow. Each connection defines a unidirectional stream between a source block and a target block.}
\label{fig:pipeblocks}
\end{figure}
\subsection{Pipe Block}
The pipe block, as illustrated in Fig.~\ref{fig:pipeblocks}(a), is the basic component used to build a design-flow architecture. A block can be connected to multiple blocks, and multiple blocks can be connected to a single block (\textbf{C3}). Each connection defines a unidirectional stream channel between a source block and a target block, where a meta-model is the only data transmitted between them. The design-flow architecture can be represented by a directed graph with feedback cycles, and follows a dataflow execution model. A block is triggered for execution when it receives a meta-model on one of its input channels, and must produce one or more meta-models as a result to trigger the next pipe block in the flow. It then becomes idle until receiving the next input.
Each pipe block in a design-flow architecture is an instance of a specific type. Instances of the same type share the same behavior. A list of pipe block types currently implemented can be found in Table.~\ref{table:pipeblocks}. A block type is characterized as follows:
\begin{itemize}
\item \textbf{Role.} We consider two roles: lambda ($\lambda$) and control ($\kappa$). The \textbf{$\lambda-$block} typically updates the model-space by adding, for instance, a new model variation (say pruning a DL model) or a specialization. The \textbf{$\kappa$-block}, on the other hand, handles the control flow, for instance, branching according to a given condition, or forking to enable concurrent flows.
\item \textbf{Multiplicity.} Specifies the number of inputs and output channels that a block can handle. For instance, some blocks types can support a one-to-one flow (one input and one output) such as the PRUNING and the HLS4ML blocks, while others may support one-to-many (e.g., branch block) and many-to-one (join block). Therefore, multiplicity restricts how blocks can be combined in a design-flow architecture.
\item \textbf{Parameters.} The parameters allow users to modify and control block behavior. Some of the parameters may be required, such as the training dataset location, while others are optional and based on default values.
\end{itemize}
Fig.~\ref{fig:pipeblocks}(b) illustrates the connection between two blocks: a $\lambda$-block and a $\kappa$-block. When a $\lambda$-block receives meta-model \textit{A}, it checks the configuration (CFG) section for arguments (parameter values) for execution. Note that the configuration section in the meta-model contains arguments for all the blocks in the design-flow, however as we explain in the next section, we can restrict arguments for specific block types and also for specific instances. The $\lambda$-block may commit a new model in the model-space: every action, including block execution, is added into the log. The resulting meta-model (A') is then flushed to the output channel. The next block in our example is a fork block, which clones the incoming meta-model A' to derive meta-models A'' and A''', respectively, which are forwarded to the corresponding output channels.
Every pipe block is strictly stateless. Thus, by storing the configuration and all design-flow artifacts inside a meta-model, as well as tracing every step in the log section, we can naturally follow its evolution during the life-cycle of a design-flow (\textbf{C2}). Moreover, we are able to use a meta-model as a checkpoint, rolling back to a specific version, or replaying from a version that was the result of hours of execution.
\section{Implementation}\label{sec:implementation}
\input{tables/pipeblocks03.tex}
Table~\ref{table:pipeblocks} provides a list of pipe blocks that have been implemented using our approach, along with their corresponding role, multiplicity and parameters:
\begin{itemize}
\item \textbf{$\kappa$-Blocks}. The basic control blocks include BRANCH which creates two streams that are selected at runtime according to a boolean condition, and JOIN which funnels two or more streams into one. In addition, the FORK block creates multiple streams allowing multiple strategy paths to be executed concurrently, and a REDUCE block which waits for a meta-model from every incoming branch, and selects a meta-model based on a specific condition. The STOP block, as the name implies, terminates the design-flow.
\item \textbf{$\lambda$-Blocks}. We have implemented three well-known techniques to improve Deep Learning models, namely pruning and scaling which are implemented using the Keras API (2.9.0), and quantization which is performed using C++ source-to-source transformations using the Artisan framework~\cite{fccm20_artisan}. We also wrapped two existing tools as $\lambda$-blocks: HLS4ML (v0.6.0) which translates a DL model into HLS C++ model, and Vivado HLS (v20.1) which translates an HLS C++ model into an RTL model.
\end{itemize}
Listing.~\ref{code:strategy} presents the Python code which implements the pruning strategy illustrated in Fig.~\ref{fig:pruning_scaling_strategy}(a) and detailed in Section~\ref{sec:pruning}. It has three sections: (1)~ lines~5--10 build the design-flow architecture graph by connecting the corresponding blocks. We begin by creating a context using the $with$ operator which creates a \texttt{Dataflow} object that holds all the pipe blocks instances and their connections. The $>>$ and $<<$ operators specify the connections between block instances, allowing individual blocks or a list of blocks to be used at either side of the operators. Each block instance has a default name as an identifier, however it can be overridden by passing it as an argument (line~7). The end-result is a cyclic-directed graph; (2)~lines 13--28, define the configuration object which contains all the parameters that we use to configure all blocks. Parameters referenced as \texttt{BlockType::parameter} are passed to every block of that block type (line~14), while parameter names with format \texttt{InstanceName@parameter} are passed to a specific instance (line~19). All remaining parameters are global (lines~24--26) and thus can be accessed by any block; finally line~31 executes the design-flow by passing the configuration object (\texttt{cfg}).
% put the list and table on differnt pages will save some space
\input{code/designflow01.tex}
The design-flow execution starts with graph validation: it must contain at least one source block (a block without inputs) and it must not violate any multiplicity constraints. Once validation is completed, we proceed to execute each block from the source until reaching a Stop block. Block execution relies on an internal scheduler which implements a thread (worker) pool. When a job is submitted to execute a block, the scheduler assigns it to an available worker or stalls until a new worker is free.
Once the architecture graph is validated, we proceed to generate an initial meta-model composed by the supplied configuration (line~31) along with an empty log and empty model space. We then instruct the scheduler to run the source block with the initial meta-model. When a block finishes its execution, it submits a job to run the next set of blocks defined by its output streams and control function. The Stop block will instruct the scheduler to terminate execution once all submitted jobs are completed, and invoke the user-supplied function (line~27) to return arbitrary data back as the output of the \texttt{run} method (line~31).
It can be seen that new optimization strategies can be attained by revising the design-flow architecture (lines~5--10) as well updating its configuration parameters (lines~13--28). In the next section, we outline different design-flow architectures and evaluate their effectiveness.
\section{Evaluation}\label{sec:evaluation}
This section presents the evaluation results of various strategies using our framework. We wish to illustrate how different optimization strategies can be realized by revising design-flow architectures, for instance combining and reusing existing pipe blocks, as well as modifying their configuration. In this context, we introduce two separate strategies, based on the PRUNING and SCALING $\lambda$-blocks in~\secref{sec:pruning} and~\ref{sec:scaling}, respectively. These two $\lambda$-blocks, as previously mentioned, are implemented using the Keras API. We then develop a strategy that combines both optimizations as part of a single strategy in~\secref{sec:scaling_pruning}. Next, we introduce a strategy that employs the QUANTIZATION $\lambda$-block, which operates at HLS C++ level and is presented in~\secref{sec:quant}. This is followed by \secref{sec:all_three} which presents a software-hardware co-optimization strategy that combines scaling, pruning and quantization. We conclude this section with a discussion in~\secref{sec:discussion}.
\subsection{Experimental setup}
We conduct our experiments using Python $3.9.15$. For our study, we choose our benchmark workloads from a few typical DNN applications which are listed in~\tabref{table:benchmark}.
%including jet identification in particle physics, and Image classification.
Some common datasets for image classification, such as MNIST~\cite{lecun1998gradient}, SVHN~\cite{netzer2011reading}, are used. We also include a dataset of jet high-level features (HLFs)~\cite{duarte2018fast, moreno2020jedi} for particle physics. This task focuses on the FPGA-based triggers of the CERN LHC (Large Hadron Collider) experiments, which handle a input data rate of 40 MHz and has a response latency constraint of less than 1$\mu$s.
%Some typical models are used, such as VGG7
The default frequency of the designs on Zynq 7020 is 100MHz and it is 200MHz for the designs using XCU250 (Alveo U250) and VU9P. The HLS4ML block is configured with 18-bit fixed-point precision as its default, including 8 integer bits.
%The MNIST dataset is the most common dataset across various work.
\input{tables/benchmarks.tex}
%\subsection{One strategy to multiple applications}
%\subsection{The strategy of pruning}
\subsection{The pruning strategy}\label{sec:pruning}
Pruning is a technique used to reduce the number of parameters in a neural network by removing weights that do not significantly affect the performance of the model. This can improve the performance and efficiency of neural networks deployed on FPGAs by reducing compute requirements. The proposed strategy gradually sets parameters close to zero during the training process until a certain percentage of parameters are zero. This is done by identifying and removing the least important weights in a network, while preserving its accuracy. The end result is typically a smaller, faster, and more efficient neural network.
\begin{equation}
\begin{aligned}
& \underset{x}{\text{maximum}}
& & Pruning\_rate \\
& \text{subject to}
& & Acc\_loss(x) \leq \alpha_p
\end{aligned}
\end{equation}
%pruning DSE, draw a figure to show the model accuracy with different pruning rate from 50\% to 99\%. 5\% each point
%\newline
The design-flow of the pruning strategy is shown in~\figref{fig:pruning_scaling_strategy}(a).
The proposed pruning strategy block implements a binary search algorithm which can automatically find the optimal pruning rate within a given threshold.
%This optimal rate along with the corresponding trained model are returned by this block.
For this experiment, the maximum tolerant accuracy loss, $\alpha_{p}$, is set to 2\%. The search starts at the pruning rate of 0\% and gets the initial model accuracy as $Acc_{p0}$. This is at step 1 (s1). Then the search will increase the pruning rate if the model accuracy using the pruning rate at current step is $\geq (Acc_{p0} - \alpha_{p})$, otherwise the search will decrease the rate. This is based on the assumption that model accuracy decreases as the pruning rate increases.
The search ends when the difference between the next pruning rate and the current rate is less than a user defined pruning rate threshold, defined as $\beta_{p}$. The $\beta_{p}$ is set to 2\% in the examples.
The number of steps is determined by the $\beta_{p}$ and it is $1+log_{2}(1/\beta_{p})$, resulting in 6 steps when $\beta_{p}$ is 2\%.
The developer can choose to store only the optimal model, or all the searched design candidates in the resulting meta-model, marking the optimal one.
The search steps and direction of the algorithm on the four benchmarks are illustrated in~\figref{fig:auto_pruning}.
% (a)
% https://docs.google.com/drawings/d/18bGih9gp5AbckjTamtKngySg2zKSJo-OyLSa6_S5YiM/edit?usp=share_link
% (b)
% https://docs.google.com/drawings/d/1vQvr6OXt8jQkADP7_Tdq2seFxSiXkN0nRG5wN9Zbgo0/edit?usp=share_link
\begin{figure}
\centering
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.33\linewidth]{img/pruning_strategy03.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.33\linewidth]{img/scaling_strategy02.pdf}}
\hspace*{\fill}
\vspace{0.2cm}
\caption{(a) The pruning strategy. (b) The scaling strategy. }
\label{fig:pruning_scaling_strategy}
\end{figure}
% a/b/c/d
% https://docs.google.com/drawings/d/1LITQ6iUaKzns4Ov-5o1hatKcYCzojA7SZ2455dSsOEo/edit?usp=share_link
% https://docs.google.com/drawings/d/1LITQ6iUaKzns4Ov-5o1hatKcYCzojA7SZ2455dSsOEo/edit?usp=share_link
% https://docs.google.com/drawings/d/1kcL-_9dCiM6kxITpDKcIwppiBt4_O1d2IdWRsQNmSCA/edit?usp=share_link
% https://docs.google.com/drawings/d/1zL_MBbEh3A08n6iYpvFzOMaSWnGH13OwgLMCnyLb1jo/edit?usp=share_link
\begin{figure}
\centering
\subfloat[Jet-DNN\label{fig:lhc_dnn_pruning}]{%
\includegraphics[width=0.6\linewidth]{img/lhc_dnn_pruning05.pdf}}
\\
% \hspace*{\fill}
\subfloat[Jet-CNN\label{fig:lhc_cnn_pruning}]{%
\includegraphics[width=0.6\linewidth]{img/lhc_cnn_pruning01.pdf}}
\\
\subfloat[VGG7\label{fig:vgg7_pruning}]{%
\includegraphics[width=0.6\linewidth]{img/lhc_dnn_pruning01.pdf}}
\\
\subfloat[ResNet9\label{fig:resnet8_pruning}]{%
\includegraphics[width=0.6\linewidth]{img/resnet8_pruning01.pdf}}
\vspace{0.2cm}
\caption{ Applying the auto pruning strategy on various models. The arrows show the direction of the binary search algorithm. The first step of the search (s1) is omitting in the figures to make the interested parts more visible. The blue arrow points to a step that has an accuracy loss larger than the user defined threshold. }
\label{fig:auto_pruning}
\end{figure}
To demonstrate the effectiveness of our auto pruning strategy, we show two examples with detailed results for each step in~\figref{fig:auto_pruning_results}. \figref{fig:auto_pruning_results}(a) and (c) show the pruning rate and the corresponding model accuracy in each step for Jet-DNN on Zynq 7020 and ResNet9 on U250, respectively.
In addition, \figref{fig:auto_pruning_results}(b) and (d) show the corresponding hardware resource utilization. As the pruning rate increases, the hardware resource requirements drop dramatically, particularly the DSPs and the LUTs which are the most important resources on FPGAs. In particular, as shown in~\figref{fig:auto_pruning_results}(b), the best DSP utilization is 41.4\% at step 6. However, the model accuracy drops significantly at this step with a pruning rate of 96.9\%. Therefore, the design candidate at step 5 is selected, which has a DSP utilization of 100.5\%. In our design-flow architecture, the hardware information is fed back to the top to fine tune the strategy parameters, more specifically the $\alpha_p$ and $\beta_p$, as indicated by the up arrow in~\figref{fig:pruning_scaling_strategy}(a).
% With our framework, it is easily to build the design-flow supporting other pruning strategies with a different control, such as an exhaustive search. This work introduce an auto pruning strategy based on a binary search which can quickly achieve the optimal pruning rate with a given threshold.
%Generally, the users may be more interested in the optimal pruning rate which may be inefficient to be found out by using DSE (another word here?).
% Alternatively, to achieve better results, more strategies can be applied together. Details are described below.
% a, b, c, d
% https://docs.google.com/drawings/d/1rkwCyGlR5qiEnzfB8YlFk6hKANpXDrKeTCRJsk3e-yg/edit?usp=share_link
% https://docs.google.com/drawings/d/1rkwCyGlR5qiEnzfB8YlFk6hKANpXDrKeTCRJsk3e-yg/edit?usp=share_link
% https://docs.google.com/drawings/d/1hbHz_j2x6jLrtz5ECg86enztNUXHWzMLYJ6lw2wJAmo/edit?usp=share_link
% https://docs.google.com/drawings/d/1H3blmXPUWjUpdM-qG30GzMylUt5R-durNwAkh70nDZM/edit?usp=share_link
\begin{figure}
\centering
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_pruning02.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_pruning_resource02.pdf}}
\\
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/resnet9_pruning_acc01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/ResNet9_pruning_resource01.pdf}}
\caption{(a) Various search pruning rates and the corresponding model accuracy for Jet-DNN. (b) Resource utilization of various design candidates of Jet-DNN on Xilinx Zynq 7020. (c) Various search pruning rate and the corresponding model accuracy for ResNet9. (d) Resource utilization of various design candidates of ResNet9 on Xilinx U250. }
\label{fig:auto_pruning_results}
\end{figure}
\subsection{The scaling strategy}\label{sec:scaling}
When the design is too large to fit a given FPGA, a common approach is to reduce the size of the DL model, which is the purpose of the scaling strategy as shown in~\figref{fig:pruning_scaling_strategy}(b).
The SCALING $\lambda$-block reduces the layer size by a factor in each step, and checks if the loss of model accuracy exceeds a threshold defined as $\alpha_s$. If it does, the search stops and the meta-model, including the final design candidate, is returned.
The $\alpha_s$ could be set to a very small value to allow for the reduction of model size with little or no loss of model accuracy. If the optimized model is still too large to fit on the targeted FPGA after the RTL synthesis, the $\alpha_s$ can be increased to enable a further reduction of model size via the feedback on the left side of ~\figref{fig:pruning_scaling_strategy}(b).
%The second mode of this strategy is to set a maximum size for each layer and then search for the optimal model with smaller layer sizes using AutoKeras and Bayesian optimization. The further reduction is done by reducing the maximum size of each layer.
% a, b
% https://docs.google.com/drawings/d/1hjuimYuCn87CxMLBQBk673muBAwLyuJwNtv6hRQMnrM/edit?usp=share_link
% https://docs.google.com/drawings/d/17vaa8QMQou1dsrr3BlqGdGDLVWdP6r1nrRhtpHCpfD4/edit?usp=share_link
\begin{figure}
\centering
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_scale_acc01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_scale_resource01.pdf}}
\caption{(a) Various search steps and the corresponding model accuracy for Jet-DNN using only the scaling strategy. (b) Resource utilization of various design candidates of Jet-DNN on Xilinx Zynq 7020 using only the scaling strategy. }
\label{fig:scaling_results}
\end{figure}
We apply our scaling strategy on the Jet-DNN model used in the jet tagging in~\cite{duarte2018fast, coelho2021automatic}. We also show the details of each search step in~\figref{fig:scaling_results} for a better illustration. \figref{fig:scaling_results}(a) shows how the accuracy of the Jet-DNN model changes as the search progresses through different steps. \figref{fig:scaling_results}(b) shows how the utilization of hardware resources changes as different design candidates are considered, with the goal of fitting the final design onto the targeted Xilinx Zynq 7020 FPGA.
In this design-flow, the search will stop depending on the value of $\alpha_s$ or when the optimized design can fit on the targeted FPGA as determined by the feedback from hardware synthesis results, as shown in~\figref{fig:pruning_scaling_strategy}(b). \figref{fig:scaling_results}(b) shows that on step 8, the utilization of all four types of hardware resources on FPGAs goes below 100\%, and thus selected as the candidate design.
% a , b
% https://docs.google.com/drawings/d/18IvNg_m8g26jrUlP4rV0Kf49qkzGqeE2-J6obaFKoZg/edit?usp=share_link
% https://docs.google.com/drawings/d/1OzVdpudJp_QE9wQcqzU3FywTzzfq10d6M1p6yon7SsU/edit?usp=sharing
\begin{figure}
\centering
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.30\linewidth]{img/scale_pruning_strategy03.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.40\linewidth]{img/pruning_scaling_strategy03.pdf}}
\hspace*{\fill}
\vspace{0.2cm}
\caption{(a) The combined strategy of scaling and pruning. (b) Another combined strategy of scaling and pruning, exploring the order of $\lambda-$blocks }
\label{fig:scaling_pruning_strategy}
\end{figure}
\subsection{Combining scaling and pruning}\label{sec:scaling_pruning}
% The design flow framework outline in~\secref{sec:approach} facilitates the combination of multiple optimization algorithms into a single automated process. As both scaling and pruning strategies presented above primarily aim to reduce the hardware resources required to deploy a neural network on an FPGA, combining them could lead to even greater resource savings than either strategy alone.
Can we combine the effects of scaling and pruning?
With the proposed framework, it is easy to cascade different optimization blocks by simply revising the graph architecture. \figref{fig:scaling_pruning_strategy}(a) shows the extended strategy in which we connect the scaling block with the pruning block along with the remaining parts of the architecture.
In this case, our new combined strategy performs an accuracy lossless scaling step first, followed by auto pruning. The value of $\alpha_s$ is set to 0.005, so that the model accuracy can be maintained while the model size is reduced as much as possible.
% a, b
% https://docs.google.com/drawings/d/1pvUmqb5l-Ufo6vRvLBpOA-Amg9LlYXwP6Qh-oR7evnk/edit?usp=share_link
% https://docs.google.com/drawings/d/1JCIkY_xKqzHJ0FBOBZxsG8owN2pe1gZKNODX4c9CNXI/edit?usp=share_link
% c, d
% https://docs.google.com/drawings/d/1UgHHzk3CtQuo7fYh7wT4YCW6tTsQs1bVKZZdwdeAxac/edit?usp=share_link
% https://docs.google.com/drawings/d/1m71oPluaW2QvfEW9p86NCQYeUGtqOruQvr3MKIKz8KU/edit?usp=share_link
% https://docs.google.com/drawings/d/1HnzOJKvK9zPeJKcAXblqph1YOIYqEfRjD5Pa835HAR8/edit?usp=sharing
\begin{figure}
\centering
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_pruning_scale_acc01.pdf}}
\hspace*{\fill}
%\\
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_pruning_scale_resource01.pdf}}
\\
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_scale_after_pruning_acc01.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.48\linewidth]{img/dnn_scale_after_pruning_resource01.pdf}}
\\
\subfloat[]{%
\includegraphics[width=0.80\linewidth]{img/scaling_pruning_pareto02.pdf}}
\caption{(a) Model accuracy and pruning rate for the Jet-DNN with scaling followed by pruning. (b) Resource utilization of each design in (a). (c) Model accuracy and layer size for the Jet-DNN with pruning followed by scaling. (d) Resource utilization of each design in (c). (e) Designs on the Pareto Front in the meta-model after REDUCE block with designs from different path labeling in different colors. }
\label{fig:scaling_pruning_results}
\end{figure}
The new optimal pruning rate becomes 84.4\%, as shown in~\figref{fig:scaling_pruning_results}(a), which is smaller than the previous optimal pruning rate of 93.8\% as shown in~\figref{fig:auto_pruning_results}(a). It is due to the reduction of model redundancy by the preceding pipe block of scaling strategy. However, the scaling strategy alone is not able to remove all model redundancy and the following pruning strategy is able to further remove redundancy.
%is shown in \figref{fig:scaling_pruning_results}(b)
%pruning is performed prior to scaling resulting in a different optimization.
A variant of our strategy can perform pruning prior to scaling by flipping the order of the corresponding $\lambda$-blocks, resulting in a different optimization.
%This involves flipping the order of the $\lambda-$ blocks.
In this case, the accuracy drops by 0.7\% just after one step in the scaling, as shown in~\figref{fig:scaling_pruning_results} (c).
We can also use the FORK block to support an optimization pattern where different strategy paths (SCALING $\rightarrow$ PRUNING and PRUNING $\rightarrow$ SCALING) are executed in parallel, as shown in~\figref{fig:scaling_pruning_strategy}(b), and the best design (or designs) are chosen in a REDUCE block. Moreover, we can do a Pareto analysis on the designs that resulted from both strategies, and select those on the Pareto Front which provide the best trade-off in terms of accuracy and resource utilization, as shown in~ \figref{fig:scaling_pruning_results} (e).
%The returned meta-model can contain all the designs on the Pareto Front from both path, as shown in~ \figref{fig:scaling_pruning_results} (e).
% To achieve a better design, a single strategy is often not sufficient and multiple optimization strategies should be used together. One of the advantages of the proposed framework in this work is that each strategy is modularized, allowing for easy combination with other strategies.
% And design-flows with different orders of these blocks create different strategies that may achieve good results.
%Since the two strategies block are modularized, the order is
%And in the new search of pruning strategy, a pruning rate of 87.5\% leads to around 15\% accuracy loss.
%The hardware resource utilization also reduces a lot, as shown in~\figref{fig:scaling_pruning_results}(b). Now the optimal design consumes only 60\% of DSP blocks on Zynq 7020, while the optimal design with only pruning strategy consumes 100.5\% DSPs.
\subsection{The quantization strategy}\label{sec:quant}
% a, b
% https://drive.google.com/file/d/1QsbyI5l4efaF4utM_DDtantkJXLsEaKQ/view?usp=share_link
% https://drive.google.com/file/d/1kQchKeoSxlyghMjO4mzsXBJr-h9BXoiC/view?usp=share_link
\begin{figure}
\centering
\subfloat[Jet-DNN]{%
\includegraphics[width=0.60\linewidth]{img/bit_width_visualization_dnn64s0.png}}
%\hspace*{\fill}
\\
\subfloat[VGG7]{%
\includegraphics[width=0.9\linewidth]{img/bit_width_visualization_vgg7s0.png}}
\vspace{0.2cm}
\caption{The quantized bitwidth of the weight, bias and output data of each layer in (a) Jet-DNN model and (b) VGG7 model. Only the layers with weights are quantized.}
\label{fig:quantization_results}
\end{figure}
\input{tables/quant_results02.tex}
As previously mentioned, the QUANTIZATION $\lambda$-block operates at the HLS C++ level, changing the fixed-point integer numerical representations directly at source-level. At this level, there is more direct control over hardware optimizations, resulting in fewer unintended side-effects than making changes at a higher level.
% In addition, low level pipe blocks developed by hardware optimization experts could be combined with high level pipe blocks created by machine learning experts to achieve better results through software-hardware co-optimization using our framework.
%As GPUs (or other AI-targeted accelerators) dominate the space of AI-computation, and these devices typically require uniform precision across most computations in order to efficiently utilize their hardware, most state-of-the-art quantization schemes aim to reduce the bit-widths of all parameters and operations to some uniform precision. The vast customizability of FPGAs, however, allow them to ignore these restrictions on uniformity of precision.
The QUANTIZATION $\lambda$-block performs mixed-precision quantization for neural networks, allowing separate precisions for the weights, biases and results of each layer of a model, taking the advantages of the customizability of FPGAs. While this customizability creates a massive search space of quantization configurations, dependencies within the network can be used to reduce the degrees of freedom of the design space. If the result type of a convolutional layer has 8 bits, for instance, there is no point in a directly subsequent max-pooling layer having 16 bits. Conversely, if the max-pooling layer were to have a bit-width of 4, an equivalently accurate and more efficient design would set the bit-width of the results of the preceding convolutional layer to be 4 as well. This strategy pipe block automatically compacts a method into a series of virtual layers using these dependencies. These virtual layers are constructed by grouping weights-layers, such as convolutional or linear layers, together with batch-normalization, pooling, and activation functions on their outputs.
% a, b, c
% https://docs.google.com/drawings/d/17TgOMyzpR0I5i1GC0pCZ14gFzK_GkJr5QO5vevGGQNs/edit?usp=share_link
% https://docs.google.com/drawings/d/1gWd4gub2PRIRKTihCJ_In_TeIJOwtp9BJAJf5DYUZGk/edit?usp=share_link
% https://docs.google.com/drawings/d/14dyqPtTo8ou2syLEYOgjxXZWjFmFmEc3rgD8vbjBsWg/edit?usp=share_link
\begin{figure}
\centering
\hspace*{\fill}
\subfloat[\label{fig:quant_strategy}]{%
\includegraphics[width=0.30\linewidth]{img/quant_strategy02.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.30\linewidth]{img/scaling_pruning_quant_strategy02.pdf}}
\hspace*{\fill}
\subfloat[]{%
\includegraphics[width=0.30\linewidth]{img/scaling_pruning_quant_strategy03.pdf}}
\hspace*{\fill}
\caption{ (a) The quantization strategy. (b) The combined strategy of scaling, pruning, quantization. (c) Another combined strategy of scaling, pruning, quantization. }
\label{fig:combined_strategy}
\end{figure}
% https://docs.google.com/drawings/d/1aZOOKmluX_QTShldzXHURcg9EzyHuzBmutCnHVTKeyY/edit?usp=share_link
\begin{figure}%[tp]
\centering
\includegraphics[width=0.9\linewidth]{img/dnn_all_resource03.pdf}
\caption{ The hardware resources and the latency of the Jet-DNN designs after different strategies on a Zynq 7020 FPGA. The clock period is 10 ns. }
\label{fig:dnn_all_resource}
\end{figure}
% https://docs.google.com/drawings/d/1vFn_MRg9-6zVi1nuA0fjESslmnl2LJndPLGPp02FfZE/edit?usp=share_link
\begin{figure}%[tp]
\centering
\includegraphics[width=0.9\linewidth]{img/vgg7_all_resource02.pdf}
\caption{ The hardware resources and the latency of the VGG7 designs after different strategies using a U250 FPGA. The clock period is 5 ns. }
\label{fig:vgg7_all_resource}
\end{figure}
\figref{fig:combined_strategy}(a) shows the design flow of the quantization strategy. After the quantization block accepts the meta-model with a maximum tolerant accuracy loss defined as $\alpha_q$, it works to gradually reduce the bit-widths of weights, bias and layer output. The resulting configuration of precision is applied to the C++ kernel and a runtime simulation is used to evaluate the accuracy of the quantized model over the validation dataset. If the accuracy loss is within the user-specified tolerance ($< \alpha_q$), this stage is repeated. Details can be found in our github.
\figref{fig:quantization_results}(a) and (b) show the precision of the weights, biases and output of each virtual layer of the Jet-DNN and VGG7 models respectively after being modified by the quantization strategy with the "max accuracy loss" parameter, $\alpha_q$, set to 1\%. \tabref{table:results_only_quant} shows how the quantization affected key evaluation metrics relating to the performance and resource usage of the FPGA design. The strategy reduces the DSP usage of a VGG7 model by a factor of 4.9. In addition, the strategy reduces the DSP usage of a Jet-DNN model by a factor of 8.5 whilst decreasing the LUT usage by a factor of 1.2, showing the effectiveness of the strategy.
\subsection{Combining quantization, scaling and pruning}\label{sec:all_three}
We now combine our three optimization techniques, pruning, scaling and quantization as part of a single automated cross-stage strategy to enhance performance and hardware efficiency, as illustrated in \figref{fig:combined_strategy}(b). All the three techniques aim to increase hardware efficiency of the neural networks deployed on FPGAs while minimizing any loss in model accuracy. By combining different methods for model compression, we may encounter improved hardware efficiency than using any single strategy alone.
The final optimized model after scaling, pruning and quantization is depicted as "S+P+Q" design in~\figref{fig:dnn_all_resource}, showing a reduction of the DSP usage by around 92\% and LUT usage by around 89\% compared with the original design (baseline~\cite{duarte2018fast}). In addition, the latency is also reduced by 35\%.
%It is also worth noting that the DSP usage of the "S+P+Q" design is not the smallest among all the designs. Both "S+P" and "P+Q" designs have smaller DSP usage but their LUT usage is much higher than the "S+P+Q". These changes suggest that as the precision of operands in certain layers were reduced below some threshold, the Vivado HLS tool moves the implementation of the multiplications in those layers from DSP blocks to LUTs.
The same optimization strategy is also applied to the VGG7 network. \figref{fig:vgg7_all_resource} shows that when compared to the baseline, the optimal design after the combined strategy of scaling, pruning and quantization results in a decrease of DSP usage by around 96\% and LUT usage by around 78\%, demonstrating the effectiveness of the strategy. It also shows that with our proposed framework, these optimization blocks can be reused for a variety of DL models without any modifications, resulting in good reusability.
\figref{fig:combined_strategy}(c) shows another combination of these blocks with the $\kappa$-block ``join" just before the quantization block. When the design is too large to fit on an FPGA, the design flow can feed back to low level HLS domain strategies to decrease the time requirred to generate a solution, or the design flow can feed back to TensorFlow/Keras domain space for a more fine-grained optimization at the cost of more optimization time.
By combining different methods to optimize acceleration, we increase the solution space and the chances to find more designs that can offer better trade-offs between model accuracy and hardware efficiency. This can be specially beneficial when deploying deep learning models on resource-constrained devices such as FPGAs.
%\input{tables/results_all_strategy.tex}
\input{tables/cmp_fpga02.tex}
\subsection{Discussion}\label{sec:discussion}
% This section evaluates three $\lambda-$blocks, namely pruning, scaling and quantization, and their combination to optimize different concerns. Although the proposed blocks are about reducing computation complexity and hardware resource utilisation while maintaining model accuracy and increasing hardware efficiency, other strategies are possible, such as computation folding that conducts the trade-off between hardware performance (latency) and hardware resources. With the proposed framework, it is easy to reuse the existing blocks and combine them with new strategy blocks to explore complex scope optimization.
%Combining multiple optimization strategies can potentially lead to better hardware efficiency and reduced redundancy of model deployment on FPGAs. However, it is important to note that the effectiveness of a combination of strategies will depend on the specific optimization architecture being used. It is the motivation of our framework that help to explore various optimization techniques for neural network acceleration on FPGAs.
To demonstrate the advantages of our proposed framework, we compare the results obtained using our design flow framework to those of original DNN~\cite{duarte2018fast} using HLS4ML, LogicNets~\cite{umuroglu2020logicnets} , QKeras-based design Q6~\cite{coelho2021automatic} and AutoQKeras-based designs QE and QB~\cite{coelho2021automatic} in~\tabref{table:cmp_fpga}. All of these studies focus on extreme low-latency, low-resource, fully unfolded (initiation interval = 1) FPGA implementations. We show the model accuracy, latency and hardware resource utilization in the table. All these designs use the same architecture as the Jet-DNN in this work, except the JSC-L which uses a larger architecture. For JSC-M and JSC-L, the accuracy is 70.6\% and 71.8\%, over 3.8\% lower than our design ``S+P+Q" with $\alpha_q$ as 0.01. Furthermore, our design uses 1.9 times fewer LUTs than JSC-L. No DSP are used in JSC-M and JSC-L while 50 DSPs are used in our design.
When compared to the Q6 designs optimized by Qkeras, the accuracy of the design optimized by the proposed combined strategy is 0.8\% higher. In addition, our design consumes 2.5 times fewer DSP blocks and 5.7 times fewer LUTs than Q6. Compared to other two designs QE and QB, our design has over 3.3\% higher accuracy. In addition, our design consumes less resources of DSP, LUT and FF than both QE and QB.
With a clock frequency of 200MHz (5ns), the latency of our design with $\alpha_q$ as 0.01 is 45ns (9 cycles), reduced to 20ns (4 cycles) without the final softmax layer like the LogicNets designs~\cite{umuroglu2020logicnets}. When the same clock frequency of 384MHz is used, the latency of this design without softmax is 10.4ns which is lower than the LogicNets JSC-L~\cite{umuroglu2020logicnets}. When compared with the designs QB, QE and QB~\cite{coelho2021automatic} optimized by the Qkeras and AutoQkeras, our design has lower latency than all of them, showing the benefits of our framework.
Inspired by the QKeras design QB which minimizes the model bit consumption, our design can be further reduced by tuning the strategy parameters, such as $\alpha_q$. When the maximum tolerant accuracy loss ($\alpha_q$) of the quantization strategy increases from 0.01 to 0.04, the strategy can further reduce the model size. The last row of~\tabref{table:cmp_fpga} shows the results with $\alpha_q$ as 0.04. The DSP usage drops to 23 that is 3 times lower than the most resource-efficient model, QE, using AutoQKeras. It also has a reduction in latency from 45ns to 40ns. The model accuracy drops to 72.8\% but it is still higher than the optimized AutoQKeras design QB and QE. For the power consumption, compared with the baseline design, the final optimized design achieves 12.6$\sim$15.1 times lower power consumption.
Please note that the quantization-aware training (QAT) technique used in the QKeras design Q6, QE, QB can be complementary to our framework, such as working as a QAT optimization $\lambda$-block and combining with other existing optimization blocks, to explore new optimization opportunities.
\section{Related Work}\label{sec:related_work}
The field of FPGA-based DNN acceleration has rapidly expanded, leading to the development of numerous optimization techniques and tools.
% techniques are not customizable
Several co-optimization techniques have been proposed that optimize both algorithm and hardware stages for DNNs on FPGAs, as discussed in various papers~\cite{yang2019synetgy, hao2019fpga, hao2020effective, jiang2020hardware, dong2021hao, fan2022algorithm, zhang2022algorithm} and in hardware-aware neural architecture search studies like~\cite{ney2021half, abdelfattah2020best}. These optimization strategies are often coupled with design space exploration, but are typically hardcoded and cannot be easily changed or customized.
% tools are not designed to add new optimisation strategies
Other approaches offer end-to-end software frameworks, such as Xilinx's Vitis AI~\cite{kathail2020xilinx} and Intel's OpenVINO~\cite{OpenVINO2023}, which optimize DNNs with pre-built optimizations for deployment on specific target technologies. %However, they are not designed for easy addition of new optimization strategies.
%or to search for the most effective combined strategies.
% single stage (limited scope) e to describe strategies
%Furthermore, some frameworks allow developers to describe and customize DNN optimization strategies, but their scope is limited. For instance, ScaleHLS~\cite{ye2022scalehls} and SOTA-OPT~\cite{agostini2022mlir} focus on HLS-based optimization strategies and hardware designs, but their optimization scope is restricted to MLIR. TVM~\cite{chen2018tvm} is a general-purpose DNN compilation framework that offers performance portability across different types of devices, but optimizations occur solely at the graph (IR) level.
% no automated cross-stage optimisations
Moreover, frameworks, such as FINN~\cite{umuroglu2017finn}, HLS4ML~\cite{duarte2018fast}, and fpgaConvNet~\cite{venieris2016fpgaconvnet}, provide optimized hardware building blocks for FPGA-based DNN accelerators.
%and allow optimization strategies to be codified using these blocks.
However, they do not support automated cross-stage optimization strategies.
%where hardware stage information guides the optimization of the application (DNN) stage.
Our framework addresses the limitations of current optimization techniques for deep neural networks (DNNs) and hardware by supporting the design of fully automated optimization flows. These flows can be described by reusable tasks that are specific to the target platform as well as ones that can be used across different platforms.
% By leveraging our approach, users can create personalized optimization strategies that span multiple stages for different DNN designs.
Additionally, our approach seamlessly integrates both new and existing optimization techniques, catering to different levels of abstraction. For instance, it includes graph optimizations for neural networks and source-to-source optimizations for HLS C++ as described in~\cite{fccm20_artisan}.
%Our approach enables automated top-down and bottom-up flows, allowing for the creation of customized cross-stage optimization strategies for various DNN designs. Moreover, our approach integrates new and existing optimization techniques targeting various levels of abstraction, such as neural networks through graph optimizations and HLS C++ through source-to-source optimizations~\cite{fccm20_artisan}.
\subsection{evaluation-objectives}
scale:
(1) want a small model with a more or less accuracy
(2) want the best accuracy no matter how large of the model
(3) a usr defined design space of various sizes for differnt layers and get the best accuracy.
(1) forks, one design targets two objectives.
(2) branchs
(3) multi-objectives,
multi-scope, co-optimization,
\begin{table}[tp]
\centering
\caption{Benchmarks used in this work}
\label{table:benchmark}
%\renewcommand{\arraystretch}{1.2}
\scalebox{.70}{
\begin{threeparttable}
\centering
\begin{tabular}{c|c|c}
\toprule
Model Name & Dataset & Application Domain \\ \midrule
Jet-DNN~\cite{duarte2018fast, coelho2021automatic} / Jet-CNN & Jet-HLF~\cite{duarte2018fast} &
\begin{tabular}[c]{@{}c@{}}Jet Identification \\ (Particle Physics)\end{tabular}
\\ \midrule
VGG-7~\cite{tridgell2019unrolling, simonyan2014very} & MNIST~\cite{lecun1998gradient} & Image Classification \\ \midrule
ResNet-9~\cite{hamanaka2023exploration, he2016deep} & SVHN~\cite{netzer2011reading} & Image Classification \\ \midrule
%VGG-11~\cite{simonyan2014very} / ResNet-18~\cite{he2016deep} & CIFAR10~\cite{krizhevsky2009learning} & Image Classification \\ \midrule
\end{tabular}
\end{threeparttable}
}
%\vspace{-0.85cm}
\end{table}
\begin{table*}[t]
\centering
\caption{Performance comparison with other FPGA implementations of the Jet-DNN network on a Xilinx VU9P FPGA, showing model accuracy, latency, resource utilization. }
\label{table:cmp_fpga}
\begin{threeparttable}
\centering
\begin{tabular}{c| c | c |c|c | c |c }
\toprule
Model & Accuracy(\%) & Latency(ns) & Latency(cycles) & DSP(\%) & LUT(\%) & FF(\%) \\
%\midrule
%Jet-DNN~\cite{duarte2018fast} & 75\% & & NA & 0 (0) & 1.2 (14,428) & 0.02 (440) \\
\midrule
LogicNets JSC-M~\cite{umuroglu2020logicnets} & 70.6 & NA\tnote{a} & NA & 0 (0) & 1.2 (14,428) & 0.02 (440) \\
\midrule
LogicNets JSC-L~\cite{umuroglu2020logicnets} & 71.8 & 13\tnote{b} & 5 & 0 (0) & 3.2 (37,931) & 0.03 (810) \\
\midrule
Qkeras Q6~\cite{coelho2021automatic} & 74.8 & 55 & 11 & 1.8 (124) & 3.4 (39,782) & 0.3 (8,128) \\
\midrule
Qkeras QE~\cite{coelho2021automatic} & 72.3 & 55 & 11 & 1.0 (66) & 0.8 (9,149) & 0.1 (1,781) \\
\midrule
Qkeras QB~\cite{coelho2021automatic} & 71.9 & 70 & 14 & 1.0 (69) & 0.9 (11,193) & 0.1 (1,771) \\
\midrule
\midrule
This work\tnote{c} & \textbf{75.6} & 45 & 9 & 0.7 (50) & 0.6 (6,698) & 0.05 (1,180) \\
\bottomrule
\end{tabular}
\begin{tablenotes}
\footnotesize
\item[a] Not provided.
\item[b] A clock frequency of 384 MHz is used and the final softmax layer is removed from this estimate.
\item[c] Using the combination of the proposed scale, pruning and quantization strategies.
\normalsize
\end{tablenotes}
\end{threeparttable}
\end{table*}
\begin{table*}[t]
\centering
\caption{Performance comparison with other FPGA implementations of the Jet-DNN network on a Xilinx VU9P FPGA
%, showing model accuracy, latency, resource utilization.
}
\label{table:cmp_fpga}
\scalebox{.78}{
\begin{threeparttable}
\centering
\begin{tabular}{c| c | c | c |c |c | c |c |c |c }
\toprule
Model & FPGA & Accuracy(\%) & Latency(ns) & Latency(cycles) & DSP(\%) & LUT(\%) & FF(\%) & Power (W) & $\alpha_q$ \\
\midrule
HLS4ML DNN~\cite{duarte2018fast} & KU115 & 75 & 75 & 15 & 17.3 (954) & - & - & - & - \\
\midrule
LogicNets JSC-M~\cite{umuroglu2020logicnets} & VU9P & 70.6 & NA & NA & 0 (0) & 1.2 (14,428) & 0.02 (440) & - & -\\
\midrule
LogicNets JSC-L~\cite{umuroglu2020logicnets} & VU9P & 71.8 & 13\tnote{a} & 5 & 0 (0) & 3.2 (37,931) & 0.03 (810) & - & -\\
\midrule
Qkeras Q6~\cite{coelho2021automatic} & VU9P & 74.8 & 55 & 11 & 1.8 (124) & 3.4 (39,782) & 0.3 (8,128) & - & -\\
\midrule
AutoQkeras QE~\cite{coelho2021automatic} & VU9P & 72.3 & 55 & 11 & 1.0 (66) & 0.8 (9,149) & 0.1 (1,781) & - & -\\
\midrule
AutoQkeras QB~\cite{coelho2021automatic} & VU9P & 71.9 & 70 & 14 & 1.0 (69) & 0.9 (11,193) & 0.1 (1,771) & - & -\\
\midrule
\midrule
This work Baseline & VU9P & \textbf{76.1} & 70 & 14 & 9.3 (638) & 5.9 (69,751) & 0.5 (10,933)& 2.51~\tnote{b} & 0.01 \\
\midrule
This work S+P+Q & VU9P & \textbf{75.6} &45 & 9 & 0.7 (50) & 0.6 (6,698) & 0.05 (1,180)& 0.199~\tnote{b} & 0.01 \\
\midrule
This work S+P+Q & VU9P & \textbf{72.8} & 40 & 8 & 0.2 (23) & 0.6 (7,224) & 0.04 (942) & 0.166~\tnote{b} & 0.04 \\
\bottomrule
\end{tabular}
\begin{tablenotes}
\footnotesize
%\item[a] Not provided.
\item[a] A clock frequency of 384 MHz is used and the final softmax layer is removed from this estimate.
\item[b] Dynamic power reported by Vivado place and route. The static power of all the three designs are about 2.5W.
\normalsize
\end{tablenotes}
\end{threeparttable}}
\end{table*}
\begin{table}[t]
\centering
\caption{Performance comparison with the FPGA designs of Jet-DNN network using other approaches on Xilinx FPGAs
%, showing model accuracy, latency, resource utilization.
}
\label{table:cmp_fpga}
\scalebox{.80}{
\begin{threeparttable}
\centering
\begin{tabular}{c| c | c | c |c |c | c |c |c }
\toprule
Model
& $\alpha_q$
& FPGA
%& Acc.(\%)
& \begin{tabular}[c]{@{}c@{}}Acc. \\(\%) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Lat. \\(ns) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Lat. \\(cycles) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}DSP \\(\%) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}LUT \\(\%) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}FF \\(\%) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Power \\(W) \end{tabular}
\\
\midrule
%HLS4ML DNN~\cite{duarte2018fast}
\begin{tabular}[c]{@{}c@{}}HLS4ML \\Jet-DNN~\cite{duarte2018fast} \end{tabular}
& - & KU115 & 75 & 75 & 15
%& 17.3 (954)
& \begin{tabular}[c]{@{}c@{}}954 \\(17.3) \end{tabular}
& -
%& -
& - \\
\midrule
%LogicNets JSC-M~\cite{umuroglu2020logicnets}
\begin{tabular}[c]{@{}c@{}}LogicNets \\JSC-M~\cite{umuroglu2020logicnets} \end{tabular}
& - & VU9P & 70.6 & NA & NA & 0 (0)
& \begin{tabular}[c]{@{}c@{}}14,428 \\(1.2) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}440 \\(0.02) \end{tabular}
& - \\
\midrule
%LogicNets JSC-L~\cite{umuroglu2020logicnets}
\begin{tabular}[c]{@{}c@{}}LogicNets \\JSC-L~\cite{umuroglu2020logicnets} \end{tabular}
& - & VU9P & 71.8 & 13\tnote{a} & 5 & 0 (0)
%& 3.2 (37,931)
%& 0.03 (810)
& \begin{tabular}[c]{@{}c@{}}37,931 \\(3.2) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}810 \\(0.03) \end{tabular}
& - \\
\midrule
\begin{tabular}[c]{@{}c@{}}Qkeras \\Q6~\cite{coelho2021automatic} \end{tabular}
& - & VU9P & 74.8 & 55 & 11
%& 1.8 (124)
%& 3.4 (39,782)
%& 0.3 (8,128)
& \begin{tabular}[c]{@{}c@{}}124 \\(1.8) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}39,782 \\(3.4) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}8,128 \\(0.3) \end{tabular}
& - \\
\midrule
\begin{tabular}[c]{@{}c@{}}AutoQkeras \\QE~\cite{coelho2021automatic} \end{tabular}
& - & VU9P & 72.3 & 55 & 11
%& 1.0 (66)
%& 0.8 (9,149)
%& 0.1 (1,781)
& \begin{tabular}[c]{@{}c@{}}66 \\(1.0) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}9,149 \\(0.8) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}1,781 \\(0.1) \end{tabular}
& - \\
\midrule
\begin{tabular}[c]{@{}c@{}}AutoQkeras \\QB~\cite{coelho2021automatic} \end{tabular}
& - & VU9P & 71.9 & 70 & 14
%& 1.0 (69)
%& 0.9 (11,193)
%& 0.1 (1,771)
& \begin{tabular}[c]{@{}c@{}}69 \\(1.0) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}11,193 \\(0.9) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}1,771 \\(0.1) \end{tabular}
& - \\
\midrule
\midrule
%This work Baseline
\begin{tabular}[c]{@{}c@{}}This work \\(same to~\cite{duarte2018fast}) \end{tabular}
& 1\% & VU9P & \textbf{76.1} & 70 & 14
%& 9.3 (638)
%& 5.9 (69,751)
%& 0.5 (10,933)
& \begin{tabular}[c]{@{}c@{}}638 \\(9.3) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}69,751 \\(5.9) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}10,933 \\(0.5) \end{tabular}
& 2.51~\tnote{b} \\
\midrule
%This work S+P+Q
\begin{tabular}[c]{@{}c@{}}This work \\S$\rightarrow$P$\rightarrow$Q \end{tabular}
& 1\% & VU9P & \textbf{75.6} &45 & 9
%& 0.7 (50)
%& 0.6 (6,698)
%& 0.05 (1,180)
& \begin{tabular}[c]{@{}c@{}}50 \\(0.7) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}6,698 \\(0.6) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}1,180 \\(0.05) \end{tabular}
& 0.199~\tnote{b} \\
\midrule
\begin{tabular}[c]{@{}c@{}}This work \\S$\rightarrow$P$\rightarrow$Q \end{tabular}
& 4\% & VU9P & \textbf{72.8} & 40 & 8
%& 0.2 (23)
%& 0.6 (7,224)
%& 0.04 (942)
& \begin{tabular}[c]{@{}c@{}}23 \\(0.2) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}7,224 \\(0.6) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}942 \\(0.04) \end{tabular}
& 0.166~\tnote{b} \\
\bottomrule
\end{tabular}
\begin{tablenotes}
\footnotesize
%\item[a] Not provided.
\item[a] A clock frequency of 384 MHz is used and the final softmax layer is removed.
\item[b] Dynamic power reported by Vivado. The static power is about 2.5W for all these designs.
\normalsize
\end{tablenotes}
\end{threeparttable}}
\end{table}
\begin{table}[t]
\centering
\caption{Performance comparison with the FPGA designs of Jet-DNN network using other approaches on Xilinx FPGAs
%, showing model accuracy, latency, resource utilization.
}
\label{table:cmp_fpga}
\scalebox{.80}{
\begin{threeparttable}
\centering
\begin{tabular}{c| c | c | c |c |c | c |c |c }
\toprule
Model
& $\alpha_q$
& FPGA
%& Acc.(\%)
& \begin{tabular}[c]{@{}c@{}}Acc. \\(\%) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Lat. \\(ns) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Lat. \\(cycles) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}DSP \\(\%) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}LUT \\(\%) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}FF \\(\%) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Power \\(W) \end{tabular}
\\
\midrule
%HLS4ML DNN~\cite{duarte2018fast}
\begin{tabular}[c]{@{}c@{}}HLS4ML \\Jet-DNN~\cite{duarte2018fast} \end{tabular}
& - & KU115 & 75 & 75 & 15
%& 17.3 (954)
& \begin{tabular}[c]{@{}c@{}}954 \\(17.3) \end{tabular}
& -
%& -
& - \\
\midrule
%LogicNets JSC-M~\cite{umuroglu2020logicnets}
\begin{tabular}[c]{@{}c@{}}LogicNets \\JSC-M~\cite{umuroglu2020logicnets} \end{tabular}
& - & VU9P & 70.6 & NA & NA & 0 (0)
& \begin{tabular}[c]{@{}c@{}}14,428 \\(1.2) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}440 \\(0.02) \end{tabular}
& - \\
\midrule
%LogicNets JSC-L~\cite{umuroglu2020logicnets}
\begin{tabular}[c]{@{}c@{}}LogicNets \\JSC-L~\cite{umuroglu2020logicnets} \end{tabular}
& - & VU9P & 71.8 & 13\tnote{a} & 5 & 0 (0)
%& 3.2 (37,931)
%& 0.03 (810)
& \begin{tabular}[c]{@{}c@{}}37,931 \\(3.2) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}810 \\(0.03) \end{tabular}
& - \\
\midrule
\begin{tabular}[c]{@{}c@{}}Qkeras \\Q6~\cite{coelho2021automatic} \end{tabular}
& - & VU9P & 74.8 & 55 & 11
%& 1.8 (124)
%& 3.4 (39,782)
%& 0.3 (8,128)
& \begin{tabular}[c]{@{}c@{}}124 \\(1.8) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}39,782 \\(3.4) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}8,128 \\(0.3) \end{tabular}
& - \\
\midrule
\begin{tabular}[c]{@{}c@{}}AutoQkeras \\QE~\cite{coelho2021automatic} \end{tabular}
& - & VU9P & 72.3 & 55 & 11
%& 1.0 (66)
%& 0.8 (9,149)
%& 0.1 (1,781)
& \begin{tabular}[c]{@{}c@{}}66 \\(1.0) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}9,149 \\(0.8) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}1,781 \\(0.1) \end{tabular}
& - \\
\midrule
\begin{tabular}[c]{@{}c@{}}AutoQkeras \\QB~\cite{coelho2021automatic} \end{tabular}
& - & VU9P & 71.9 & 70 & 14
%& 1.0 (69)
%& 0.9 (11,193)
%& 0.1 (1,771)
& \begin{tabular}[c]{@{}c@{}}69 \\(1.0) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}11,193 \\(0.9) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}1,771 \\(0.1) \end{tabular}
& - \\
\midrule
\midrule
%This work Baseline
\begin{tabular}[c]{@{}c@{}}This work \\(Same to~\cite{duarte2018fast}) \end{tabular}
& 1\% & VU9P & \textbf{76.1} & 70 & 14
%& 9.3 (638)
%& 5.9 (69,751)
%& 0.5 (10,933)
& \begin{tabular}[c]{@{}c@{}}638 \\(9.3) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}69,751 \\(5.9) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}10,933 \\(0.5) \end{tabular}
& 2.51~\tnote{b} \\
\midrule
%This work S+P+Q
\begin{tabular}[c]{@{}c@{}}This work \\S$\rightarrow$P$\rightarrow$Q \end{tabular}
& 1\% & VU9P & \textbf{75.6} &45 & 9
%& 0.7 (50)
%& 0.6 (6,698)
%& 0.05 (1,180)
& \begin{tabular}[c]{@{}c@{}}50 \\(0.7) \end{tabular}
& \begin{tabular}[c]{@{}c@{}}6,698 \\(0.6) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}1,180 \\(0.05) \end{tabular}
& 0.199~\tnote{b} \\
%\midrule
%\begin{tabular}[c]{@{}c@{}}This work \\S$\rightarrow$P$\rightarrow$Q \end{tabular}
%& 4\% & VU9P & \textbf{72.8} & 40 & 8
%& \begin{tabular}[c]{@{}c@{}}23 \\(0.2) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}7,224 \\(0.6) \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}942 \\(0.04) \end{tabular}
%& 0.166~\tnote{b} \\
\bottomrule
\end{tabular}
\begin{tablenotes}
\footnotesize
%\item[a] Not provided.
\item[a] A clock frequency of 384 MHz is used and the final softmax layer is removed.
\item[b] Dynamic power reported by Vivado. The static power is about 2.5W for all these designs.
\normalsize
\end{tablenotes}
\end{threeparttable}}
\end{table}
\begin{table*}[]
\centering
\caption{}
\label{table:pipeblocks}
\begin{tabular}{l|c|c|l|c}
\hline
\rowcolor[HTML]{C0C0C0}
\textbf{Type} &
\textbf{Role} &
\textbf{Multiplicity} &
\textbf{Parameters} &
\textbf{Description} \\ \hline
JOIN & $\kappa$ & many-to-1 & & \\ \hline
BRANCH & $\kappa$ & 1-to-2 & fn: meta-model $\rightarrow$ bool & \\ \hline
FORK & $\kappa$ & 1-to-many & & \\ \hline
REDUCE & $\kappa$ & many-to-1 & fn: [meta-model] $\rightarrow$ meta-model & \\ \hline
PRUNING &
$\lambda$ &
1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
default\_pruning\_rate \\
initial\_max\_accuracy\_loss ($\alpha_p$) \\
pruning\_rate\_threshold ($\beta_b$)\\
pruning\_auto \\
train\_dataset\\
test\_dataset\\
train\_epochs \\
train\_batch\_size \\
train\_learning\_rate \\
train\_callbacks \\
train\_LOSS \\
train\_optimizer \\
train\_metrics
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
When pruning\_auto is high, this block \\
searches the optimal pruning rate by using \\
a binary search algorithm. \\
When pruning\_auto is low, this block \\
returns a trained model pruned using the \\
default pruning rate statically.
\end{tabular}}
\\ \hline
SCALING
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
default\_scale\_factor \\
initial\_max\_accuracy\_loss ($\alpha_s$)\\
scale\_auto \\
max\_trials\_num \\
train\_dataset\\
test\_dataset\\
train\_epochs \\
train\_batch\_size \\
train\_learning\_rate \\
train\_callbacks \\
train\_LOSS \\
train\_optimizer \\
train\_metrics
\end{tabular}} & \\ \hline
HLS4ML
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
param1: default\_precision \\
param2: IOType \\
param3: FPGA\_part\_number \\
param4: clock\_period \\
test\_dataset
\end{tabular}} & \\ \hline
QUANTIZATION
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
param1: initial\_max\_accuracy\_loss ($\alpha_q$)\\
test\_dataset
\end{tabular}} & \\ \hline
VIVADO-HLS
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
none
\end{tabular}} & \\ \hline
KERAS-MODEL-GEN
& $\lambda$
& 0-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
param1: train\_en \\
train\_dataset\\
test\_dataset\\
train\_epochs \\
train\_batch\_size \\
train\_learning\_rate \\
train\_callbacks \\
train\_LOSS \\
train\_optimizer \\
train\_metrics
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
Train\_en should be set when there\\
is no Keras-based strategy block before \\
HLS4ML block.
\end{tabular}}
\\ \hline
STOP &
$\lambda$ &
1-to-0 &
fn: meta-model $\rightarrow$ output &
\textbf{} \\ \hline
\end{tabular}
\end{table*}
\begin{table*}[]
\centering
\caption{}
\label{table:pipeblocks}
\begin{tabular}{l|c|c|l l}
\hline
\rowcolor[HTML]{C0C0C0}
\textbf{Type} &
\textbf{Role} &
\textbf{Multiplicity} &
\multicolumn{2}{c|}{\textbf{Parameters}}
\\ \hline
JOIN & $\kappa$ & many-to-1 & & \\ \hline
BRANCH & $\kappa$ & 1-to-2 & fn: meta-model $\rightarrow$ bool & \\ \hline
FORK & $\kappa$ & 1-to-many & & \\ \hline
REDUCE & $\kappa$ & many-to-1 & fn: [meta-model] $\rightarrow$ meta-model & \\ \hline
PRUNING &
$\lambda$ &
1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
default\_pruning\_rate \\
initial\_max\_accuracy\_loss ($\alpha_p$) \\
pruning\_rate\_threshold ($\beta_b$)\\
pruning\_auto \\
train\_dataset\\
test\_dataset \\
train\_epochs \\
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
train\_batch\_size \\
train\_learning\_rate \\
train\_callbacks \\
train\_LOSS \\
train\_optimizer \\
train\_metrics
\end{tabular}}
\\ \hline
SCALING
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
default\_scale\_factor \\
initial\_max\_accuracy\_loss ($\alpha_s$)\\
scale\_auto \\
max\_trials\_num \\
train\_dataset \\
test\_dataset \\
train\_epochs \\
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
train\_batch\_size \\
train\_learning\_rate \\
train\_callbacks \\
train\_LOSS \\
train\_optimizer \\
train\_metrics
\end{tabular}}
\\ \hline
HLS4ML
& $\lambda$
& 1-to-1
& \textbf{\begin{tabular}[c]{@{}l@{}}
default\_precision \\
IOType \\
FPGA\_part\_number
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
clock\_period \\
test\_dataset \\
\end{tabular}}
\\ \hline
QUANTIZATION
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
initial\_max\_accuracy\_loss ($\alpha_q$)
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
test\_dataset
\end{tabular}}
\\ \hline
VIVADO-HLS
& $\lambda$
& 1-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
none
\end{tabular}} & \\ \hline
KERAS-MODEL-GEN
& $\lambda$
& 0-to-1 &
\textbf{\begin{tabular}[c]{@{}l@{}}
train\_en \\
train\_dataset\\
test\_dataset\\
train\_epochs \\
train\_batch\_size \\
\end{tabular}}
& \textbf{\begin{tabular}[c]{@{}l@{}}
train\_learning\_rate \\
train\_callbacks \\
train\_LOSS \\
train\_optimizer \\
train\_metrics
\end{tabular}}
\\ \hline
STOP &
$\lambda$ &
1-to-0 &
fn: meta-model $\rightarrow$ output &
\textbf{} \\ \hline
\end{tabular}
\end{table*}
\begin{table}[pb]
\centering
\caption{ A list of implemented pipe tasks.}
\label{table:pipeblocks}
\scalebox{.90}{
\begin{tabular}{l|c|c|l}
\hline
\rowcolor[HTML]{C0C0C0}
\textbf{Type} &
\textbf{Role} &
\textbf{Multiplicity} &
\textbf{Parameters}\\ \hline
% JOIN & $K$ & many-to-1 & - \\ \hline
% FORK & $K$ & 1-to-many & - \\ \hline
% REDUCE
% & $K$
% & many-to-1
% & \begin{tabular}[c]{@{}l@{}} fn: [meta-model] \\ $\rightarrow$ meta-model \end{tabular}
% \\ \hline
% STOP &
% $K$ &
% 1-to-0 &
% fn: meta-model $\rightarrow$ output
% \textbf{} \\ \hline
HLS4ML
& $\lambda$
& 1-to-1 &
\begin{tabular}[c]{@{}l@{}}
default\_precision \\
IOType \\
FPGA\_part\_number \\
clock\_period \\
test\_dataset
\end{tabular}\\ \hline
VIVADO-HLS
& $\lambda$
& 1-to-1 &
project\_dir \\ \hline
\begin{tabular}[c]{@{}l@{}}
KERAS-MODEL\\-GEN
\end{tabular}
& $\lambda$
& 0-to-1 &
\begin{tabular}[c]{@{}l@{}}
train\_en \\
train\_test\_dataset\\
train\_epochs
\end{tabular}
\\ \hline
PRUNING &
$O$ &
1-to-1 &
\begin{tabular}[c]{@{}l@{}}
tolerate\_acc\_loss ($\alpha_p$) \\
pruning\_rate\_thresh ($\beta_p$)\\
train\_test\_dataset\\
train\_epochs
\end{tabular}
\\ \hline
SCALING
& $O$
& 1-to-1 &
\begin{tabular}[c]{@{}l@{}}
default\_scale\_factor \\
tolerate\_acc\_loss ($\alpha_s$)\\
scale\_auto \\
max\_trials\_num \\
train\_test\_dataset\\
train\_epochs
\end{tabular} \\ \hline
QUANTIZATION
& $O$
& 1-to-1 &
\begin{tabular}[c]{@{}l@{}}
tolerate\_acc\_loss ($\alpha_q$)\\
train\_test\_dataset
\end{tabular} \\ \hline
\end{tabular}
}
\end{table}
\begin{table}[t]
\centering
\caption{Results of designs with the quantization strategy. }
\label{table:results_only_quant}
\begin{threeparttable}
\centering
\begin{tabular}{c | c |c|c|c | c |c |c }
\toprule
Model & FPGA & Lat. & II & DSP & LUT & FF & Acc \\
\midrule
\begin{tabular}[c]{@{}c@{}}Jet-DNN \\ Baseline \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Zynq \\ 7020 \end{tabular}
& \begin{tabular}[c]{@{}c@{}}26 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}1 \\ cycle \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{638} \\ \textbf{290\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}66k \\ 124\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}30k \\ 28\% \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}4 \\ 1.4\% \end{tabular}
& 76.1\%
%& \begin{tabular}[c]{@{}c@{}}Zynq\\ 7Z045 \end{tabular}
\\ \midrule
\begin{tabular}[c]{@{}c@{}}Jet-DNN \\ Quant. \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Zynq \\ 7020 \end{tabular}
& \begin{tabular}[c]{@{}c@{}}23 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}1 \\ cycle \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{75} \\ \textbf{34.1\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}57k \\ 107\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}17k \\ 16\% \end{tabular}
& 75.5\%
\\
\midrule
\midrule
\begin{tabular}[c]{@{}c@{}}VGG7 \\ Baseline \end{tabular}
& \begin{tabular}[c]{@{}c@{}}U250 \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 1189 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 905 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{4568} \\ \textbf{37.2\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}345k \\ 20.0\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}65k \\ 1.9\% \end{tabular}
& 98.2\%
\\
\midrule
\begin{tabular}[c]{@{}c@{}}VGG7 \\ Quant. \end{tabular}
& \begin{tabular}[c]{@{}c@{}}U250 \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 1189 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 905 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{934} \\ \textbf{7.6\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}313k \\ 18.1\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}51k \\ 1.5\% \end{tabular}
& 97.5\%
\\
\bottomrule
\end{tabular}
\vspace{0.1cm}
\end{threeparttable}
\end{table}
\begin{table}[t]
\centering
\caption{Results of designs with the quantization strategy. }
\label{table:results_only_quant}
\begin{threeparttable}
\centering
\begin{tabular}{c | c |c|c | c |c |c }
\toprule
Model & FPGA & Latency / II & DSP & LUT & FF & Acc \\
\midrule
\begin{tabular}[c]{@{}c@{}}Jet-DNN \\ Baseline \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Zynq \\ 7020 \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 26 / 1 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{638} \\ \textbf{290\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}66k \\ 124\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}30k \\ 28\% \end{tabular}
%& \begin{tabular}[c]{@{}c@{}}4 \\ 1.4\% \end{tabular}
& 76.1\%
%& \begin{tabular}[c]{@{}c@{}}Zynq\\ 7Z045 \end{tabular}
\\ \midrule
\begin{tabular}[c]{@{}c@{}}Jet-DNN \\ Quant. \end{tabular}
& \begin{tabular}[c]{@{}c@{}}Zynq \\ 7020 \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 23 / 1 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{75} \\ \textbf{34.1\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}57k \\ 107\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}17k \\ 16\% \end{tabular}
& 75.5\%
\\
\midrule
\midrule
\begin{tabular}[c]{@{}c@{}}VGG7 \\ Baseline \end{tabular}
& \begin{tabular}[c]{@{}c@{}}U250 \end{tabular}
%& \begin{tabular}[c]{@{}c@{}} 1189 \\ cycles \end{tabular}
%& \begin{tabular}[c]{@{}c@{}} 905 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 1189 / 905 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{4568} \\ \textbf{37.2\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}345k \\ 20.0\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}65k \\ 1.9\% \end{tabular}
& 98.2\%
\\
\midrule
\begin{tabular}[c]{@{}c@{}}VGG7 \\ Quant. \end{tabular}
& \begin{tabular}[c]{@{}c@{}}U250 \end{tabular}
%& \begin{tabular}[c]{@{}c@{}} 1189 \\ cycles \end{tabular}
%& \begin{tabular}[c]{@{}c@{}} 905 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}} 1189 / 905 \\ cycles \end{tabular}
& \begin{tabular}[c]{@{}c@{}}\textbf{934} \\ \textbf{7.6\%} \end{tabular}
& \begin{tabular}[c]{@{}c@{}}313k \\ 18.1\% \end{tabular}
& \begin{tabular}[c]{@{}c@{}}51k \\ 1.5\% \end{tabular}
& 97.5\%
\\
\bottomrule
\end{tabular}
\vspace{0.1cm}
\end{threeparttable}
\end{table}
% Generated by IEEEtran.bst, version: 1.14 (2015/08/26)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{lecun2015deep}
Y.~LeCun, Y.~Bengio, and G.~Hinton, ``{Deep learning},'' \emph{nature}, vol.
521, no. 7553, pp. 436--444, 2015.
\bibitem{zhang2015optimizing}
C.~Zhang, P.~Li, G.~Sun, Y.~Guan, B.~Xiao, and J.~Cong, ``{Optimizing
FPGA-based accelerator design for deep convolutional neural networks},'' in
\emph{Proceedings of the 2015 ACM/SIGDA international symposium on
field-programmable gate arrays}, 2015, pp. 161--170.
\bibitem{que2021recurrent}
Z.~Que, H.~Nakahara, E.~Nurvitadhi, A.~Boutros, H.~Fan, C.~Zeng, J.~Meng, K.~H.
Tsoi, X.~Niu, and W.~Luk, ``{Recurrent Neural Networks With Column-Wise
Matrix--Vector Multiplication on FPGAs},'' \emph{IEEE Transactions on Very
Large Scale Integration (VLSI) Systems}, vol.~30, no.~2, pp. 227--237, 2021.
\bibitem{zhang2020dnnexplorer}
X.~Zhang, H.~Ye, J.~Wang, Y.~Lin, J.~Xiong, W.-m. Hwu, and D.~Chen,
``{DNNExplorer: a framework for modeling and exploring a novel paradigm of
FPGA-based DNN accelerator},'' in \emph{Proceedings of the 39th International
Conference on Computer-Aided Design}, 2020, pp. 1--9.
\bibitem{sze2017efficient}
V.~Sze, Y.-H. Chen, T.-J. Yang, and J.~S. Emer, ``{Efficient processing of deep
neural networks: A tutorial and survey},'' \emph{Proceedings of the IEEE},
vol. 105, no.~12, pp. 2295--2329, 2017.
\bibitem{coelho2021automatic}
C.~N. Coelho, A.~Kuusela, S.~Li, H.~Zhuang, J.~Ngadiuba, T.~K. Aarrestad,
V.~Loncar, M.~Pierini, A.~A. Pol, and S.~Summers, ``{Automatic heterogeneous
quantization of deep neural networks for low-latency inference on the edge
for particle detectors},'' \emph{Nature Machine Intelligence}, vol.~3, no.~8,
pp. 675--686, 2021.
\bibitem{que2021accelerating}
Z.~Que, E.~Wang, U.~Marikar, E.~Moreno, J.~Ngadiuba, H.~Javed,
B.~Borzyszkowski, T.~Aarrestad, V.~Loncar, S.~Summers \emph{et~al.},
``{Accelerating recurrent neural networks for gravitational wave
experiments},'' in \emph{2021 IEEE 32nd International Conference on
Application-specific Systems, Architectures and Processors (ASAP)}.\hskip 1em
plus 0.5em minus 0.4em\relax IEEE, 2021, pp. 117--124.
\bibitem{wang2019deep}
E.~Wang, J.~J. Davis, R.~Zhao, H.-C. Ng, X.~Niu, W.~Luk, P.~Y. Cheung, and
G.~A. Constantinides, ``{Deep neural network approximation for custom
hardware: Where we've been, where we're going},'' \emph{ACM Computing Surveys
(CSUR)}, vol.~52, no.~2, pp. 1--39, 2019.
\bibitem{deng2020model}
L.~Deng, G.~Li, S.~Han, L.~Shi, and Y.~Xie, ``{Model compression and hardware
acceleration for neural networks: A comprehensive survey},''
\emph{Proceedings of the IEEE}, vol. 108, no.~4, pp. 485--532, 2020.
\bibitem{xu2020autodnnchip}
P.~Xu, X.~Zhang, C.~Hao, Y.~Zhao, Y.~Zhang, Y.~Wang, C.~Li, Z.~Guan, D.~Chen,
and Y.~Lin, ``{AutoDNNchip: An automated DNN chip predictor and builder for
both FPGAs and ASICs},'' in \emph{Proceedings of the 2020 ACM/SIGDA
International Symposium on Field-Programmable Gate Arrays}, 2020, pp. 40--50.
\bibitem{yang2019synetgy}
Y.~Yang, Q.~Huang, B.~Wu, T.~Zhang, L.~Ma, G.~Gambardella, M.~Blott,
L.~Lavagno, K.~Vissers, J.~Wawrzynek \emph{et~al.}, ``Synetgy:
Algorithm-hardware co-design for convnet accelerators on embedded fpgas,'' in
\emph{Proceedings of the 2019 ACM/SIGDA international symposium on
field-programmable gate arrays}, 2019, pp. 23--32.
\bibitem{hao2019fpga}
C.~Hao, X.~Zhang, Y.~Li, S.~Huang, J.~Xiong, K.~Rupnow, W.-m. Hwu, and D.~Chen,
``{FPGA/DNN co-design: An efficient design methodology for 1ot intelligence
on the edge},'' in \emph{2019 56th ACM/IEEE Design Automation Conference
(DAC)}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2019, pp. 1--6.
\bibitem{hao2020effective}
C.~Hao, Y.~Chen, X.~Zhang, Y.~Li, J.~Xiong, W.-m. Hwu, and D.~Chen,
``{Effective algorithm-accelerator co-design for ai solutions on edge
devices},'' in \emph{Proceedings of the 2020 on Great Lakes Symposium on
VLSI}, 2020, pp. 283--290.
\bibitem{jiang2020hardware}
W.~Jiang, L.~Yang, E.~H.-M. Sha, Q.~Zhuge, S.~Gu, S.~Dasgupta, Y.~Shi, and
J.~Hu, ``{Hardware/software co-exploration of neural architectures},''
\emph{IEEE Transactions on Computer-Aided Design of Integrated Circuits and
Systems}, vol.~39, no.~12, pp. 4805--4815, 2020.
\bibitem{dong2021hao}
Z.~Dong, Y.~Gao, Q.~Huang, J.~Wawrzynek, H.~K. So, and K.~Keutzer, ``{Hao:
Hardware-aware neural architecture optimization for efficient inference},''
in \emph{2021 IEEE 29th Annual International Symposium on Field-Programmable
Custom Computing Machines (FCCM)}.\hskip 1em plus 0.5em minus 0.4em\relax
IEEE, 2021, pp. 50--59.
\bibitem{fan2022algorithm}
H.~Fan, M.~Ferianc, Z.~Que, H.~Li, S.~Liu, X.~Niu, and W.~Luk, ``{Algorithm and
hardware co-design for reconfigurable cnn accelerator},'' in \emph{2022 27th
Asia and South Pacific Design Automation Conference (ASP-DAC)}.\hskip 1em
plus 0.5em minus 0.4em\relax IEEE, 2022, pp. 250--255.
\bibitem{zhang2022algorithm}
X.~Zhang, Y.~Li, J.~Pan, and D.~Chen, ``{Algorithm/Accelerator Co-Design and
Co-Search for Edge AI},'' \emph{IEEE Transactions on Circuits and Systems II:
Express Briefs}, vol.~69, no.~7, pp. 3064--3070, 2022.
\bibitem{ney2021half}
J.~Ney, D.~Loroch, V.~Rybalkin, N.~Weber, J.~Kr{\"u}ger, and N.~Wehn, ``{HALF:
Holistic auto machine learning for FPGAs},'' in \emph{2021 31st International
Conference on Field-Programmable Logic and Applications (FPL)}.\hskip 1em
plus 0.5em minus 0.4em\relax IEEE, 2021, pp. 363--368.
\bibitem{abdelfattah2020best}
M.~S. Abdelfattah, {\L}.~Dudziak, T.~Chau, R.~Lee, H.~Kim, and N.~D. Lane,
``{Best of both worlds: Automl codesign of a cnn and its hardware
accelerator},'' in \emph{2020 57th ACM/IEEE Design Automation Conference
(DAC)}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2020, pp. 1--6.
\bibitem{kathail2020xilinx}
V.~Kathail, ``{Xilinx vitis unified software platform},'' in \emph{Proceedings
of the 2020 ACM/SIGDA International Symposium on Field-Programmable Gate
Arrays}, 2020, pp. 173--174.
\bibitem{OpenVINO2023}
``{OpenVINO toolkit},'' \url{https://github.com/openvinotoolkit/openvino},
2023.
\bibitem{umuroglu2017finn}
Y.~Umuroglu, N.~J. Fraser, G.~Gambardella, M.~Blott, P.~Leong, M.~Jahre, and
K.~Vissers, ``{FINN: A framework for fast, scalable binarized neural network
inference},'' in \emph{Proceedings of the 2017 ACM/SIGDA international
symposium on field-programmable gate arrays}, 2017, pp. 65--74.
\bibitem{duarte2018fast}
J.~Duarte, S.~Han, P.~Harris, S.~Jindariani, E.~Kreinar, B.~Kreis, J.~Ngadiuba,
M.~Pierini, R.~Rivera, N.~Tran \emph{et~al.}, ``{Fast inference of deep
neural networks in FPGAs for particle physics},'' \emph{Journal of
Instrumentation}, vol.~13, no.~07, p. P07027, 2018.
\bibitem{venieris2016fpgaconvnet}
S.~I. Venieris and C.-S. Bouganis, ``{fpgaConvNet: A framework for mapping
convolutional neural networks on FPGAs},'' in \emph{2016 IEEE 24th Annual
International Symposium on Field-Programmable Custom Computing Machines
(FCCM)}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2016, pp. 40--47.
\bibitem{fccm20_artisan}
J.~Vandebon, J.~G.~F. Coutinho, W.~Luk, E.~Nurvitadhi, and T.~Todman,
``{Artisan: a Meta-Programming Approach For Codifying Optimisation
Strategies},'' in \emph{2020 IEEE 28th Annual International Symposium on
Field-Programmable Custom Computing Machines (FCCM)}, 2020.
\bibitem{moreno2020jedi}
E.~A. Moreno, O.~Cerri, J.~M. Duarte, H.~B. Newman, T.~Q. Nguyen, A.~Periwal,
M.~Pierini, A.~Serikova, M.~Spiropulu, and J.-R. Vlimant, ``{JEDI-net: a jet
identification algorithm based on interaction networks},'' \emph{The European
Physical Journal C}, vol.~80, no.~1, pp. 1--15, 2020.
\bibitem{simonyan2014very}
K.~Simonyan and A.~Zisserman, ``{Very deep convolutional networks for
large-scale image recognition},'' \emph{arXiv preprint arXiv:1409.1556},
2014.
\bibitem{he2016deep}
K.~He, X.~Zhang, S.~Ren, and J.~Sun, ``{Deep residual learning for image
recognition},'' in \emph{Proceedings of the IEEE conference on computer
vision and pattern recognition}, 2016, pp. 770--778.
\bibitem{lecun1998gradient}
Y.~LeCun, L.~Bottou, Y.~Bengio, and P.~Haffner, ``{Gradient-based learning
applied to document recognition},'' \emph{Proceedings of the IEEE}, vol.~86,
no.~11, pp. 2278--2324, 1998.
\bibitem{netzer2011reading}
Y.~Netzer, T.~Wang, A.~Coates, A.~Bissacco, B.~Wu, and A.~Y. Ng, ``{Reading
digits in natural images with unsupervised feature learning},'' 2011.
\bibitem{umuroglu2020logicnets}
Y.~Umuroglu, Y.~Akhauri, N.~J. Fraser, and M.~Blott, ``{LogicNets: co-designed
neural networks and circuits for extreme-throughput applications},'' in
\emph{2020 30th International Conference on Field-Programmable Logic and
Applications (FPL)}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2020, pp.
291--297.
\bibitem{que2022ll}
Z.~Que, M.~Loo, H.~Fan, M.~Blott, M.~Pierini, A.~D. Tapper, and W.~Luk,
``{LL-GNN: Low Latency Graph Neural Networks on FPGAs for Particle
Detectors},'' \emph{arXiv preprint arXiv:2209.14065}, 2022.
\bibitem{wojcicki2022accelerating}
F.~Wojcicki, Z.~Que, A.~D. Tapper, and W.~Luk, ``{Accelerating Transformer
Neural Networks on FPGAs for High Energy Physics Experiments},'' in
\emph{2022 International Conference on Field-Programmable Technology
(ICFPT)}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2022, pp. 1--8.
\bibitem{xilinx_white}
``{Xilinx AI Engines and Their Applications},'' in \emph{WP506(v1.1)}, July 10,
2020.
\bibitem{langhammer2021stratix}
M.~Langhammer, E.~Nurvitadhi, B.~Pasca, and S.~Gribok, ``{Stratix 10 NX
Architecture and Applications},'' in \emph{The 2021 ACM/SIGDA International
Symposium on Field-Programmable Gate Arrays}, 2021, pp. 57--67.
\end{thebibliography}