CoFHEE: A Co-processor for Fully Homomorphic Encryption Execution (Extended Version)
Extracted source text · arXiv 2204.08742
\documentclass[lettersize,journal]{IEEEtran}
\input{util/commands}
\input{util/packages}
\usepackage{amsmath,amsfonts}
% \usepackage{algorithmic}
% \usepackage{algorithm}
% \usepackage{array}
% \usepackage[caption=false,font=normalsize,labelfont=sf,textfont=sf]{subfig}
% \usepackage{textcomp}
% \usepackage{stfloats}
% \usepackage{url}
% \usepackage{verbatim}
% \usepackage{graphicx}
% \usepackage{cite}
% \hyphenation{op-tical net-works semi-conduc-tor IEEE-Xplore}
% updated with editorial comments 8/9/2021
\begin{document}
\title{Silicon-proven ASIC design for the Polynomial Operations of Fully Homomorphic Encryption}
\author{
Mohammed Nabeel,~\IEEEmembership{}
Deepraj Soni,~\IEEEmembership{Graduate Student Member,~IEEE,}
Mohammed Ashraf,~\IEEEmembership{}
Mizan Abraha Gebremichael,~\IEEEmembership{}
Homer Gamil,~\IEEEmembership{Graduate Student Member,~IEEE,}
Eduardo Chielle,~\IEEEmembership{}
Ramesh Karri,~\IEEEmembership{Fellow,~IEEE,}
Mihai Sanduleanu,~\IEEEmembership{Member,~IEEE,}
Michail Maniatakos,~\IEEEmembership{Senior Member,~IEEE,}
}
% <-this % stops a space
% \thanks{This paper was produced by the IEEE Publication Technology Group. They are in Piscataway, NJ.}% <-this % stops a space
% \thanks{Manuscript received April 19, 2021; revised August 16, 2021.}}
% \author{
% Mohammed Nabeel\inst{1} \and
% Deepraj Soni\inst{2} \and
% Mohammed Ashraf\inst{1} \and
% Mizan Abraha Gebremichael\inst{3} \and
% Homer Gamil\inst{2} \and
% Eduardo Chielle\inst{1} \and
% Ramesh Karri\inst{2} \and
% Mihai Sanduleanu\inst{3} \and
% Michail Maniatakos\inst{1}
% }
% \institute{
% New York University Abu Dhabi, Abu Dhabi, UAE
% \and
% New York University, New York, USA
% \and
% Khalifa University, Abu Dhabi, UAE
% % \and
% % Institute B, City, Country, \email{john@institute}
% }
% The paper headers
\markboth{IEEE TRANSACTIONS ON COMPUTER-AIDED DESIGN OF INTEGRATED CIRCUITS AND SYSTEMS}
{Shell \MakeLowercase{\textit{et al.}}: A Sample Article Using IEEEtran.cls for IEEE Journals}
% \markboth{Journal of \LaTeX\ Class Files,~Vol.~14, No.~8, August~2021}%
% {Shell \MakeLowercase{\textit{et al.}}: A Sample Article Using IEEEtran.cls for IEEE Journals}
% \IEEEpubid{0000--0000/00\$00.00~\copyright~2021 IEEE}
% Remember, if you use this you must call \IEEEpubidadjcol in the second
% column for its text to clear the IEEEpubid mark.
\maketitle
\input{0_abstract}
\input{1_introduction}
\input{2_preliminaries}
\input{3_method}
% \input{3.1_operation}
\input{3.2_front_end}
\input{3.3_back_end}
\input{4_evaluation}
\input{5_relatedWork}
\input{6_discussion}
\input{7_conclusion}
% \clearpage
\bibliographystyle{IEEEtran}
\bibliography{references}
% \newpage
% \section{Biography Section}
% If you have an EPS/PDF photo (graphicx package needed), extra braces are
% needed around the contents of the optional argument to biography to prevent
% the LaTeX parser from getting confused when it sees the complicated
% $\backslash${\tt{includegraphics}} command within an optional argument. (You can create
% your own custom macro containing the $\backslash${\tt{includegraphics}} command to make things
% simpler here.)
% \vspace{11pt}
% \bf{If you include a photo:}\vspace{-33pt}
\vspace{-20pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/nabeel-modified.png}}]{Mohammed Nabeel}
received his Bachelor’s degree in
electrical and electronics engineering from the National Institute of Technology – Calicut, India. He is currently working as a Research Engineer at the Center for Cyber Security at New York University Abu Dhabi (CCS-NYUAD). Apart from working on research in the field of hardware security, he also focuses on implementing and prototyping the research ideas in Chip. He is also working towards a Ph.D. degree at the Department of Computer Science and Engineering, Tandon School of Engineering at New York University, Brooklyn, NY. Before joining CCS-NYUAD, he worked at Texas Instruments, where he worked on chips targeted for IoT and Automotive. Prior to that, he was with Qualcomm, where he worked on chips targeted for mobile phones and data
cards. He has more than 20 conference and journal papers and 2 issued US patents.
\end{IEEEbiography}
\vspace{-40pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/deepraj-modified.png}}]{Deepraj Soni}
is a Ph.D. student at the NYU Tandon School of Engineering. Deepraj works on hardware implementation, evaluation, and security of post-quantum cryptographic algorithms using HLS. Based on this work, he has published a book “Hardware Architectures for Post-Quantum Digital Signature Schemes”. He also works on the hardware implementation of Fully Homomorphic Encryption (FHE) primitives and architecture. He received his M.Tech from the Department of Electrical Engineering, Indian Institute of Technology Bombay (IIT-B). After graduation, Deepraj worked as a design engineer in the semiconductor division of Samsung and SanDisk.
\end{IEEEbiography}
\vspace{-40pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/ashraf-modified.png}}]{Mohammed Ashraf}
received the bachelor’s degree in electronics and telecommunication engineering from the College of Engineering Trivandrum, Thiruvananthapuram, India, in 2005.
He is a Senior Physical Design Engineer from India. He carries an experience of ten years in the VLSI industry. He has worked with various multinational companies like NVIDIA Graphics, Santa Clara, CA, USA, Advanced Micro Devices, Santa Clara, and Wipro Technologies, Bengaluru, India. He worked also with Dubai Circuit Design,
UAE. He is currently a Research Engineer with the Center for Cyber Security, New York University Abu Dhabi, UAE.
\end{IEEEbiography}
\vspace{-60pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/Diagonals-of-a-square.png}}]{Mizan Abraha Gebremichael}
received his B.Sc. from the Department of Electrical and Electronics
Engineering, Mekelle Institute of Technology, Ethiopia, in 2007; MTech from the Department of
Electrical Engineering, Indian Institute of Technology, Bombay, India, in 2010, Ph.D. in
interdisciplinary engineering from Khalifa University, Abu-Dhabi, UAE. Mizan is currently a postdoctoral
fellow at Khalifa University. His research interests include areas of high-speed Clock and Data Recovery
Circuits, PLLs, Digital and Mixed-signal IC design.
\end{IEEEbiography}
\vspace{-20pt}
\begin{IEEEbiography}[
{\vspace{-1cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/Homer_Gamil-modified.jpg}}]{Homer Gamil}
is Ph.D. candidate at New York University. He holds a bachelor degree in Computer Engineering from the New York University Abu Dhabi. His research interests are in cybersecurity and applied cryptography, with a special focus in hardware security and privacy outsourcing.
\end{IEEEbiography}
\vspace{-60pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/Eduardo-modified.jpg}}]{Eduardo Chielle}
is a Research Associate at New York University Abu Dhabi.
He holds a jointly-supervised Ph.D. in Microelectronics from the Universidade Federal do Rio Grande do Sul (UFRGS) and Universidad de Alicante, a M.Sc. in Computer Science from UFRGS, and a five-year bachelor in Computer Engineering from the Universidade Federal do Rio Grande (FURG). Eduardo placed first in the TTTC’s McCluskey Best 2016 Latin American Ph.D. Thesis Contest for his work on microprocessor soft error mitigation. His interests include privacy-preserving computation and machine learning.
\end{IEEEbiography}
\vspace{-30pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/Ramesh_Karri_IMG_4837-modified.jpeg}}]{Ramesh Karri}
received the B.E. degree from the Department of Electronics and Communication Engineering, Andhra University,
and the Ph.D. degree in computer science and engineering from the University of California at San Diego, San Diego, CA, USA. He is currently
a Professor of electrical and computer engineering with New York University, where he co-directs the Center for Cyber Security. His research interests include hardware cybersecurity, trustworthy integrated circuits, processors
and cyber–physical systems, security-aware computer-aided design, test,
verification, validation, and reliability, nano meets security, hardware security competitions, benchmarks and metrics, biochip security, and additive
manufacturing security.
% received the B.E. degree from the Department of Electronics and Communication Engineering, Andhra University,
% and the Ph.D. degree in computer science and engineering from the University of California at San Diego, San Diego, CA, USA. He is currently
% a Professor of electrical and computer engineering with New York University, where he co-directs the Center for Cyber Security. He co-founded
% the Trust-Hub and organizes the Embedded Systems Challenge and the
% Annual Red Team Blue Team Event. He has published over 275 articles in leading journals and conference proceedings. His research interests
% include hardware cybersecurity, trustworthy integrated circuits, processors
% and cyber–physical systems, security-aware computer-aided design, test,
% verification, validation, and reliability, nano meets security, hardware security competitions, benchmarks and metrics, biochip security, and additive
% manufacturing security. He is a fellow for his contributions and leadership
% in Trustworthy Hardware. His work in trustworthy hardware received the
% Best Paper Award nominations (ICCD 2015 and DFTS 2015), and the
% ACM TODAES 2017 Award, the ITC 2014 Award, the CCS 2013 Award,
% the DFTS 2013 Award, the VLSI Design 2012 Award, the ACM Student
% Research Competition from the DAC 2012, the ICCAD 2013, the DAC 2014,
% the ACM Grand Finals 2013, the Kaspersky Challenge, and the Embedded
% Security Challenge Award. He also received the Humboldt Fellowship and
% the National Science Foundation CAREER Award. He is the Editor-in-Chief
% of the ACM Journal of Emerging Technologies in Computing. He served/s as
% an Associate Editor for the IEEE TRANSACTIONS ON INFORMATION FORENSICS AND
% SECURITY, the IEEE TRANSACTIONS ON COMPUTER-AIDED DESIGN OF INTEGRATED
% CIRCUITS AND SYSTEMS, the ACM Journal of Emerging Computing Technologies, the ACM Transactions on Design Automation of Electronic Systems
% in 2014, IEEE ACCESS, the IEEE TRANSACTIONS ON EMERGING TECHNOLOGIES
% IN COMPUTING, IEEE Design and Test in 2015, and IEEE EMBEDDED SYSTEMS
% LETTERS in 2016. He served as an IEEE Computer Society Distinguished
% Visitor, from 2013 to 2015. He served on the Executive Committee for
% the IEEE/ACM Design Automation Conference leading the Security@DAC
% initiative, from 2014 to 2017. He has given keynotes, talks, and tutorials on
% Hardware Security and Trust.
\end{IEEEbiography}
\vspace{-50pt}
\begin{IEEEbiography}[
{\vspace{-0cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/mihai-modified.jpg}}]{Mihai Sanduleanu}
received the M.Sc. degree from
the Technical University of Ia¸si, Ia¸si, Romania,
in 1990, the M.E.E. degree from the Eindhoven University of Technology, Eindhoven, The Netherlands,
in 1993, and the Ph.D. degree from the University
of Twente, Enschede, The Netherlands, in 1999. His areas of expertise include wireless transceiver
design for RF/millimeter-wave (mmWave)/THz communication, THz electronics, high-speed communication circuits for serial input and output, highspeed analog-to-digital converters, phased-array systems, imaging, fiber-optic
interface IC, ultra-low-power radios, data and clock recovery and phaselocked loop (PLL), high-accuracy/low-power analog-/mixed-signal circuits
and systems, system-on-chip (SoC) design, on-chip sensors and actuators
for autonomous transceivers (wired/wireless), high-speed digital circuits and
systems, electromagnetics, and antenna design.
% From 1999 to 2000, he was with Philips Semiconductors, Nijmegen, The Netherlands, where he
% worked on fiber optic communication circuits. He
% then moved to Philips Research, Eindhoven, where
% he was involved in fiber-optic interface circuits,
% radio frequency (RF) interconnected chip (IC) design, mmWaves’ transceiver
% design, and ultra-low-power radios from 2000 to 2007. While at Philips
% Research, he was an Adjunct Professor with the Microelectronics Group,
% Eindhoven University of Technology. In 2008, he spent six months at Interuniversity Microelectronics Center (IMEC), Leuven, Belgium, conducting
% research on 60-GHz transceiver design and THz electronics. From 2008 to
% 2013, he conducted research at the IBM Thomas J. Watson Research Center,
% Yorktown Heights, NY, USA, in mmWave transceivers for communication,
% imaging and radar, high-speed analog-to-digital converters for wired/wireless
% communication systems, and ultra-low-power radios. He is currently an
% Associate Professor with the Masdar Institute of Science and Technology,
% Abu Dhabi, United Arab Emirates. He has authored/coauthored four books
% and more than 80 articles in international conferences and journals. He holds
% 51 granted U.S. patents.
% His areas of expertise include wireless transceiver
% design for RF/millimeter-wave (mmWave)/THz communication, THz electronics, high-speed communication circuits for serial input and output, highspeed analog-to-digital converters, phased-array systems, imaging, fiber-optic
% interface IC, ultra-low-power radios, data and clock recovery and phaselocked loop (PLL), high-accuracy/low-power analog-/mixed-signal circuits
% and systems, system-on-chip (SoC) design, on-chip sensors and actuators
% for autonomous transceivers (wired/wireless), high-speed digital circuits and
% systems, electromagnetics, and antenna design.
% Dr. Sanduleanu received the Philips Foundation M.Sc. and Ph.D. Fellowships in 1993 and 1994, respectively. He has served as an Associate Editor for
% the IEEE TRANSACTIONS ON CIRCUITS AND SYSTEMS. He serves on the
% Editorial Board of Analog Integrated Circuits and Signal Processing journal.
% received the M.Sc. degree from
% the Technical University of Ia¸si, Ia¸si, Romania,
% in 1990, the M.E.E. degree from the Eindhoven University of Technology, Eindhoven, The Netherlands,
% in 1993, and the Ph.D. degree from the University
% of Twente, Enschede, The Netherlands, in 1999.
% From 1999 to 2000, he was with Philips Semiconductors, Nijmegen, The Netherlands, where he
% worked on fiber optic communication circuits. He
% then moved to Philips Research, Eindhoven, where
% he was involved in fiber-optic interface circuits,
% radio frequency (RF) interconnected chip (IC) design, mmWaves’ transceiver
% design, and ultra-low-power radios from 2000 to 2007. While at Philips
% Research, he was an Adjunct Professor with the Microelectronics Group,
% Eindhoven University of Technology. In 2008, he spent six months at Interuniversity Microelectronics Center (IMEC), Leuven, Belgium, conducting
% research on 60-GHz transceiver design and THz electronics. From 2008 to
% 2013, he conducted research at the IBM Thomas J. Watson Research Center,
% Yorktown Heights, NY, USA, in mmWave transceivers for communication,
% imaging and radar, high-speed analog-to-digital converters for wired/wireless
% communication systems, and ultra-low-power radios. He is currently an
% Associate Professor with the Masdar Institute of Science and Technology,
% Abu Dhabi, United Arab Emirates. He has authored/coauthored four books
% and more than 80 articles in international conferences and journals. He holds
% 51 granted U.S. patents. His areas of expertise include wireless transceiver
% design for RF/millimeter-wave (mmWave)/THz communication, THz electronics, high-speed communication circuits for serial input and output, highspeed analog-to-digital converters, phased-array systems, imaging, fiber-optic
% interface IC, ultra-low-power radios, data and clock recovery and phaselocked loop (PLL), high-accuracy/low-power analog-/mixed-signal circuits
% and systems, system-on-chip (SoC) design, on-chip sensors and actuators
% for autonomous transceivers (wired/wireless), high-speed digital circuits and
% systems, electromagnetics, and antenna design.
% Dr. Sanduleanu received the Philips Foundation M.Sc. and Ph.D. Fellowships in 1993 and 1994, respectively. He has served as an Associate Editor for
% the IEEE TRANSACTIONS ON CIRCUITS AND SYSTEMS. He serves on the
% Editorial Board of Analog Integrated Circuits and Signal Processing journal.
\end{IEEEbiography}
\vspace{-100pt}
\begin{IEEEbiography}[
{\vspace{-0.6cm}\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{authors/mihalis-modified.jpg}}]{Michail Maniatakos} is an Associate Professor of Electrical and Computer Engineering at NYU Abu Dhabi. He received his Ph.D., M.Phil. and M.Sc. from the Electrical Engineering Department at Yale University in 2012, 2010, and 2009 respectively. Prior joining Yale, he received his B.Sc. in Computer Science and M.Sc. in Embedded Systems from the University of Piraeus in 2006 and 2007 respectively. His research interests include hardware acceleration of homomorphic encryption and industrial control systems security. He is a senior member of the IEEE.
\end{IEEEbiography}
% \vspace{11pt}
% \bf{If you will not include a photo:}\vspace{-33pt}
% \begin{IEEEbiographynophoto}{John Doe}
% Use $\backslash${\tt{begin\{IEEEbiographynophoto\}}} and the author name as the argument followed by the biography text.
% \end{IEEEbiographynophoto}
\vfill
\end{document}
\newcommand{\fix}[1]{{\color{red}#1}}
\usepackage{mathtools}
% \usepackage{etextools}
\newcommand{\cofhee}{CoFHEE}
% Math
\newcommand{\Bigrceil}{{\Bigr\rceil}}
\newcommand{\Biglfloor}{{\Bigl\lfloor}}
\DeclarePairedDelimiter{\ceil}{\lceil}{\rceil}
\DeclarePairedDelimiter{\floor}{\lfloor}{\rfloor}
\DeclarePairedDelimiter{\round}{\lfloor}{\rceil}
\DeclarePairedDelimiter{\Biground}{\Biglfloor}{\Bigrceil}
\newcommand{\um}{\mu m} % to be used in math mode $$
\newcommand{\uW}{\mu W} % to be used in math mode $$
\DeclareRobustCommand{\hlcyan}[1]{{\sethlcolor{cyan}\hl{#1}}}
\DeclareRobustCommand{\hlorange}[1]{{\sethlcolor{orange}\hl{#1}}}
\DeclareRobustCommand{\hlpink}[1]{{\sethlcolor{pink}\hl{#1}}}
% \DeclareRobustCommand{\hl}[1]{{\sethlcolor{yellow}\hl{#1}}}
% \usepackage{lipsum}
% % \usepackage{algpseudocode}
% % % \usepackage{algorithm}
% % \usepackage{mathtools}
% \usepackage{amssymb}
% \usepackage{amsmath}
% \usepackage{multirow}
% % \usepackage{pifont}
% % \usepackage{subcaption} % subfigure
% % \usepackage{arydshln} % \hdashline
% % \usepackage{float}
% % \usepackage{eso-pic,graphicx}
% ---------------------------------------
%%%% 2. PACKAGES %%%%
% %\usepackage{lipsum} % Example package -- can be removed
% % \usepackage{todonotes}
% \usepackage{lipsum}
% % \usepackage{algpseudocode}
% % \usepackage{algorithm}
% \usepackage{mathtools}
% \usepackage{amssymb}
% \usepackage{amsmath}
% \usepackage{multirow}
% \usepackage{pifont}
% % \usepackage{subcaption} % subfigure
% % \usepackage{arydshln} % \hdashline
% \usepackage{float}
% % \usepackage{multirow}
% % \usepackage{algorithm} % Added by Edu
% % \usepackage{algcompatible} % Added by Edu
% % \usepackage{algpseudocode} % Added by Edu
%\usepackage[showframe=true]{geometry}
%\usepackage
%\usepackage{lipsum} % Example package -- can be removed
% \usepackage{todonotes}
\usepackage{lipsum}
\usepackage{algpseudocode}
\usepackage{algorithm}
\usepackage{mathtools}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{multirow}
\usepackage{pifont}
\usepackage{cuted}
% \usepackage{subcaption} % subfigure
% \usepackage{arydshln} % \hdashline
\usepackage{float}
% \usepackage{multirow}
% \usepackage{algorithm} % Added by Edu
% \usepackage{algcompatible} % Added by Edu
% \usepackage{algpseudocode} % Added by Edu
%\usepackage[showframe=true]{geometry}
%\usepackage
\usepackage{soul}
\usepackage{color}
\usepackage{graphics}
\usepackage{subfig}
% \usepackage{caption}
\usepackage{tablefootnote}
\usepackage{threeparttable}
\usepackage[symbol]{footmisc}
% FIXME arxiv trickery
\usepackage{hyperref}
\hypersetup{
% bookmarks=true, % show bookmarks bar?
% unicode=false, % non-Latin characters in Acrobat’s bookmarks
% pdftoolbar=true, % show Acrobat’s toolbar?
% pdfmenubar=true, % show Acrobat’s menu?
% pdffitwindow=false, % window fit to page when opened
% % pdfstartview={FitW}, % fits the width of the page to the window
% pdftitle={Certificate}, % title
% pdfauthor={Dr. Harish Kumar}, % author
% pdfsubject={TEQIP certificates}, % subject of the document
pdfcreator={}, % creator of the document
pdfproducer={}, % producer of the document
% pdfkeywords={Certificates,} {TEQIP} {Participation}, % list of keywords
% pdfnewwindow=true, % links in new window
% colorlinks=false, % false: boxed links; true: colored links
% linkcolor=red, % color of internal links
% citecolor=green, % color of links to bibliography
% filecolor=magenta, % color of file links
% urlcolor=cyan % color of external links
}
\begin{abstract}
The proliferation of cloud computing has led to heightened concerns regarding the security and privacy of sensitive data, as their need to be decrypted before processing, renders them susceptible to potential breaches. Fully Homomorphic Encryption (FHE) serves as a countermeasure to this issue by enabling computation to be executed directly on encrypted data. Nevertheless, the execution of FHE is orders of magnitude slower compared to unencrypted computation, thereby impeding its practicality and adoption. Therefore, enhancing the performance of FHE is crucial for its implementation in real-world scenarios. In this study, we elaborate on our endeavors to design, implement, fabricate, and post-silicon validate a co-processor for FHE, named \cofhee{}. With a compact design area of $12mm^2$, \cofhee{} features ASIC implementations of fundamental polynomial operations, including polynomial addition and subtraction, Hadamard product, and Number Theoretic Transform, which underlie all higher-level FHE primitives. \cofhee{} is capable of natively supporting polynomial degrees of up to $n = 2^{14}$ with a coefficient size of 128 bits, and has been fabricated and silicon-verified using 55nm CMOS technology. To evaluate it, we conduct performance and power experiments on our chip, and compare it to state-of-the-art software implementations and other ASIC designs.
% The migration of computation to the cloud has raised privacy concerns as sensitive data becomes vulnerable to attacks since they need to be decrypted for processing.
% Fully Homomorphic Encryption (FHE) mitigates this issue as it enables meaningful computations to be performed directly on encrypted data.
% Nevertheless, FHE is orders of magnitude slower than unencrypted computation, which hinders its practicality and adoption.
% Therefore, improving FHE performance is essential for its real world deployment.
% \hl{In this paper, we present, in detail, our efforts to design, implement, fabricate, and post-silicon validate a co-processor for Fully Homomorphic Encryption dubbed \cofhee{}.
% With a small design area of $12mm^2$, \cofhee{} incorporates ASIC implementations of fundamental polynomial operations, such as polynomial addition and subtraction, Hadamard product, and Number Theoretic Transform, which are underneath all higher-level FHE primitives. \cofhee{} has native support of polynomial degrees of up to $n = 2^{14}$ with a coefficient size of 128 bits, and is fabricated and silicon-proven in 55nm CMOS technology. We evaluate our chip with performance and power experiments and compare it against state-of-the-art software implementations and other ASIC designs.}
\end{abstract}
\begin{IEEEkeywords}
Data privacy, Encrypted computation, Fully Homomorphic Encryption, Co-processor, ASIC
\end{IEEEkeywords}
\section{Introduction}
The proliferation of cloud services has intensified the user dependency on outsourced computation.
While standard encryption schemes like RSA and AES protect data in-transit and data at-rest, they cannot protect data in-use, since they require data decryption before processing; thus, sensitive data is exposed during computation.
High-profile attacks in cloud services \cite{barron2013cloud} have shown that access control is not sufficient.
The advent of Fully Homomorphic Encryption (FHE) in 2009 \cite{gentry2009thesis} came as a solution for the problem of data in-use as it enables computation to be performed directly in the encrypted domain without the need for decryption.
Although promising from a security and privacy standpoint, FHE is multiple orders of magnitude slower than unencrypted computation, a characteristic that hinders its adoption in the industry.
Much progress has been done in the last decade through improvements in FHE schemes and their software implementation.
However, software-only performance improvement of FHE computation is still not sufficient for the majority of applications.
In recent years, hardware acceleration of encrypted computation is becoming a means to improve its practicality. % of homomorphic encryption.
Several works have been proposed using GPUs \cite{dai2014accelerating} and FPGAs \cite{cousins2012update} for FHE acceleration.
Furthermore, ASIC accelerator protypes demonstrated significant speedups compared to software solutions \cite{gentry2011implementing} \cite{doroz2014accelerating}. To the best of our knowledge, however, there is no fabricated and silicon proven ASIC design supporting FHE computation.
% \hl{
% Many existing works on hardware acceleration have been focusing on the improvement of certain operations that are often times not utilized in real-life applications. One of those operations is bootstrapping, which is a technique that aims to refresh the noise of ciphertexts, preventing them from becoming irrecoverable during their decryption. Most real-life applications however do not utilize this operation. For this reason, we focus on a more realistic context, and use Leveled Fully Homomorphic Encryption, as it does not consider the bootstrapping operation.
% }
% Primitive FHE operations can be broken down to low and high-level.
% Low-level operations are defined as operations that area heavily focused on polynomial computation.
% On the other hand, high-level operations use the former as their constituents. By doing so, the high-level often times become complex operations that require careful scheduling, which creates a bottleneck in the process of data movement.
% On the contrary, low-level operations are not bottle-necked by data movement processes as they heavily depend on the architecture of the given design, and therefore hardware optimization is the key to acquiring considerable performance improvements for low-level operations. Towards that end, we focus on improving the performance of low-level operations. Consequently, we investigate the context of leveled Fully Homomorphic Encryption, and the acceleration of high-level operations such as bootstrapping and key-switching are out of the scope of this study.
FHE computation operations can be broadly characterized in two levels: High-level ones, such as key switching and bootstrapping, and low-level, such as polynomial addition and multiplication. All high-level FHE primitives are realized using low-level ones. Recent work~\cite{feldmann2021f1} has shown that a prototype of an ASIC FHE accelerator which includes also high level operations \emph{exceeds} $150mm^2$ (GF12/14) in size, being a shuttle run which can eventually cost many hundreds of thousands of dollars. Therefore, in order to produce a real chip, we need to constraint the design to focus on acceleration of the low-level primitives only. Such a design will eventually serve as a small component in a much bigger design, where the larger design will mostly focus on data movement.
% the acceleration of polynomial computations, which are the constituents of high-level operations.
% In this work we focus on
% we opt for a design with a design area of $12 mm^2$. Towards that end,
% As an example, one noteworthy high-level operation in FHE cryptosystems is bootstrapping, a technique that aims to refresh the noise of ciphertexts, preventing them from becoming irrecoverable during their decryption. Bootstrapping is an operation that is considerable orders of magnitude slower than other FHE operations, and therefore has been a target for improvement. In practical scenarios, FHE is often used without the need of bootstrapping. Such a concept is possible as long as the multiplicative depth of the FHE application is known a priori, which is often the case. The context in which the bootstrapping operation is not considered is in other words named leveled Fully Homomorphic Encryption, and is the main focus of this work.
% Due to that reason, the main bottleneck of high-level operations has become problem for high level operations is data movement,
% while low-level operations depend on the architecture of a design, and therefore an optimized hardware can provide considerable performance improvements.
% As an example, one noteworthy high-level operation in FHE cryptosystems is bootstrapping, a technique that aims to refresh the noise of ciphertexts, preventing them from becoming irrecoverable during their decryption. Bootstrapping is an operation that is considerable orders of magnitude slower than other FHE operations, and therefore has been a target for improvement. In practical scenarios, FHE is often used without the need of bootstrapping. Such a concept is possible as long as the multiplicative depth of the FHE application is known a priori, which is often the case. The context in which the bootstrapping operation is not considered is in other words named leveled Fully Homomorphic Encryption, and is the main focus of this work.
% One noteworthy operation in FHE cryptosystems is bootstrapping, a technique that aims to refresh the noise of ciphertexts, preventing them from becoming irrecoverable during their decryption. Bootstrapping is an operation that is considerable orders of magnitude slower than other FHE operations, and therefore has been a target for improvement. In practical scenarios, FHE is often used without the need of bootstrapping. Such a concept is possible as long as the multiplicative depth of the FHE application is known a priori, which is often the case. The context in which the bootstrapping operation is not considered is in other words named leveled Fully Homomorphic Encryption, and is the main focus of this work.
% \hl{Many FHE functions have received a lot of attention for acceleration. One is Bootstrapping. Bootstrapping however is not used often in realistic scenarios. For this reason we attempt to perform research on a more realistic scenario. We work in the context of leveled fully homomorphic encryption. Expand/Elaborate/etc.}
\noindent \textbf{Contributions:} Given a limited design area of $12 mm^2$ available to us given budget constraints, we design an architecture that accelerates the underlying polynomial computation, and it is the base of all high-level FHE operations. This study describes our process for designing, implementing, fabricating, and validating \cofhee{}, the first silicon-proven ASIC co-processor for FHE.
More specifically, our contributions are:
\begin{itemize}
\item We present the design and implementation of \cofhee{}, a specialized hardware architecture that supports low-level operations of Fully Homomorphic Encryption (FHE). The CoFHEE architecture comprises of specialized units, engineered to execute a variety of arithmetic operations, as well as an Advanced High-Performance Bus (AHB) lite interconnect that provides high-speed data transfer between the units.
% We present \cofhee{}, a hardware design supporting low-level FHE operations. \cofhee{} consists of special units capable of performing several arithmetic operations, and an Advanced High-Performance Bus lite interconnect. %More specifically the components of the design are carefully selected to comprise an architecture capable of supporting ciphertext multiplication fully on chip
% \cofhee{} consists of special units such as a Processing Element (PE) capable of performing several arithmetic operations, and an Advanced High-Performance Bus lite interconnect, which addresses major data movement issues
\item We fabricate \cofhee{} using 55nm CMOS technology from GlobalFoundries, utilizing a compact design area of $12mm^2$. The CoFHEE architecture is capable of supporting polynomial degrees of up to $n = 2^{14}$, with a maximum native coefficient size of 128 bits.
% We fabricate \cofhee{} using 55nm CMOS Globalfoundries in a design area of $12mm^2$. \cofhee{} supports polynomial degrees of up to $n = 2^{14}$ with a maximum native coefficient size of 128 bits.%, and runs on a frequency of $250$ MHz
% \item We evaluate the realized chip with performance and power experiments, and compare it against the state-of-the-art
\item In addition to the fabrication and implementation of \cofhee{}, we intend to open-source the Register Transfer Level (RTL) code of CoFHEE and its functional units, which can serve as building blocks for future research and development efforts in the design of Fully Homomorphic Encryption (FHE) co-processors
% We will open source the RTL of \cofhee{} as well as its functional units, which can serve as components for future work towards the design of FHE co-processors.
\end{itemize}
% The rest of the paper is organized as follows. Section \ref{sec:prelim} provides the necessary background behind our work. Section \ref{sec:method} provides a design flow overview of \cofhee{}. Section \ref{sec:fun_units} discusses the functional units that were utilized, while section \ref{sec:backend} describes the physical design aspects of \cofhee{} and the procedure that followed for its fabrication. Furthermore, an evaluation of the realized design is detailed in section \ref{sec:eval}, and a comparison with other hardware architectures is provided in section \ref{sec:related_work}. Lastly, the scalability of \cofhee{} is discussed in section \ref{sec:discussion}, and a conclusion of the work is provided in \ref{sec:conc}
\clearpage
% develop an architecture to accelerate polynomial multiplications, since it is the main constituent of ciphertext multiplications.
% \cofhee{} has special units for Number Theoretic Transform (NTT) and a Processing Element (PE) capable of performing several arithmetic operations.
% The \cofhee{} chip has been fabricated using 55nm CMOS Globalfoundries and has been successfully brought up. \hl{This paper presents a comprehensive report from the initial specification to post-silicon validation, presenting implementation challenges and lessons learned. \cofhee{} will be open sourced, and its architecture as well as its functional units can serve as a components for future work towards the design of FHE co-processors and accelerators.}
% - Hardware architecture for the acceleration of the NTT
% - Accelerate primitive operations
% - Fabrication using CMOS
% - Open source our work.
% ..
% \noindent \textbf{Contributions:} \hl{In this work, we describe our process for designing, implementing, fabricating, and validating \cofhee{}, the first silicon-proven ASIC accelerator for FHE}.
% \cofhee{} targets ciphertext multiplications, the slowest arithmetic operation in FHE.
% Towards that end, we develop an architecture to accelerate polynomial multiplications, since it is the main constituent of ciphertext multiplications.
% \cofhee{} has special units for Number Theoretic Transform (NTT) and a Processing Element (PE) capable of performing several arithmetic operations.
% The \cofhee{} chip has been fabricated using 55nm CMOS Globalfoundries and has been successfully brought up. \hl{This paper presents a comprehensive report from the initial specification to post-silicon validation, presenting implementation challenges and lessons learned. \cofhee{} will be open sourced, and its architecture as well as its functional units can serve as a components for future work towards the design of FHE co-processors and accelerators.}
\section{Preliminaries}
\label{sec:prelim}
\subsection{Fully Homomorphic Encryption}
Homomorphic encryption is a special type of encryption that enables performing meaningful computations directly on encrypted data.
It can be understood as the functional equivalent of Eq. \ref{eq:he}, where $f(\cdot)$ is a function over plaintexts and $F(\cdot)$ is its equivalent over ciphertexts, $D_k(\cdot)$ and $E_k(\cdot)$ are respectively the decryption and encryption functions for a given key $k$, and $R(\cdot)$ is a function that provides randomness for $E_k(\cdot)$ from ciphertexts $c_a$ and $c_b$.
Although functionally equivalent to Eq. \ref{eq:he}, homomorphic encryption enables equivalent computation on the encrypted domain without the need for any decryption function during computation.
\begin{equation}\label{eq:he}
F(c_a, c_b) = E_k( f( D_k(c_a), D_k(c_b) ), R(c_a, c_b) )
\end{equation}
While there are several types of homomorphic encryption, Fully Homomorphic Encryption (FHE) possesses at least two orthogonal homomorphic operations allowing any number of arbitrary computations.
FHE was introduced in 2009 \cite{gentry2009thesis} and it is deemed as the "holy grail" of cryptography.
It has seen much progress since its inception with the development of new encryption schemes such as BFV \cite{bfv}, BGV \cite{bgv}, CGGI \cite{cggi}, and CKKS \cite{ckks}.
\subsection{BFV Encryption Scheme}
The Brakerski/Fan-Vercauteren (BFV) scheme is an FHE scheme based on the Ring-Learning With Errors (RLWE) problem \cite{rlwe}.
It works over two polynomial rings, one for the plaintext space and another for the ciphertext space.
The plaintext space is defined over the polynomial ring $\mathcal{P} = \mathbb{Z}_t[x]/(x^n+1)$, while the ciphertext space in defined over $\mathcal{C} = \mathbb{Z}_q[x]/(x^n+1)$, where $n$ is the polynomial degree, and $t$, $q$, and $x^n+1$ are the plaintext, ciphertext, and polynomial moduli, respectively.
% It is important to note that
BFV is supported by several libraries and frameworks, such as ALCHEMY \cite{alchemy}, Cingulata \cite{cingulata}, E3 \cite{e3}, nGraph-HE2 \cite{ngraph}, Palisade \cite{palisade}, Ramparts \cite{ramparts}, and SEAL \cite{seal}.
% ALCHEMY \cite{alchemy}, Cingulata \cite{cingulata}, E3 \cite{e3}, nGraph-HE2 \cite{ngraph}, Palisade \cite{palisade}, Ramparts \cite{ramparts}, and SEAL \cite{seal}.
% \subsection{Core \& auxiliary operations}
\subsection{Ciphertext Multiplication}
\label{sss:ctmul}
% The Homomorphic Encryption Security Standard defines the basic primitives of BFV \cite{hestd}.
% Out of those primitives, the homomorphic multiplication of ciphertexts (\texttt{EvalMult}) is the slowest operation, and therefore, the main candidate for hardware acceleration.
% In order to understand ciphertext multiplication, we first need to understand encryption.
% Let $m$ be a plaintext in the plaintext space ($m \in \mathcal{P}$), $k_p = (k_{p1}, k_{p2})$ be an encryption key in the ciphertext space $\mathcal{C}$, and $c = (c_1, c_2) \in \mathcal{C}$ be an encryption of $m$ with key $k_p$;
% the encryption function $E(k_p, m) \rightarrow c$ defines a map from $\mathcal{P}$ to $\mathcal{C}$.
% The ciphertext $c$ is composed of two polynomials $c_1$ and $c_2$, which are computed according to Eqs. \ref{eq:encc1} and \ref{eq:encc2}, where $u$ is a random polynomial from the set $\{-1, 0, 1\}$, $e_1$ and $e_2$ are small random polynomials from a discrete Gaussian distribution, and $\Delta = \floor{q/t}$ is a scaling factor.
% \begin{align}
% c_1 = k_{p1} \cdot u + e_1 + \Delta m \mod q\label{eq:encc1}\\
% c_2 = k_{p2} \cdot u + e_2 \mod q\label{eq:encc2}
% \end{align}
% \begin{strip}
% \begin{equation}\label{eq:ctmul}
% (c_{c1}, c_{c2}, c_{c3}) = \Big( \Biground{ \frac{t(c_{a1} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a1} \cdot c_{b2} + c_{a2} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a2} \cdot c_{b2})}{q} }_q \Big)
% \end{equation}
% \end{strip}
The Homomorphic Encryption Security Standard defines the basic primitives of BFV \cite{hestd}.
Out of those primitives, the homomorphic multiplication of ciphertexts (\texttt{EvalMult}) is the slowest operation, and therefore, the main candidate for hardware acceleration.
In order to understand ciphertext multiplication, we first need to understand encryption.
Let $m$ be a plaintext in the plaintext space ($m \in \mathcal{P}$), $k_p = (k_{p1}, k_{p2})$ be an encryption key in the ciphertext space $\mathcal{C}$, and $c = (c_1, c_2) \in \mathcal{C}$ be an encryption of $m$ with key $k_p$;
the encryption function $E(k_p, m) \rightarrow c$ defines a map from $\mathcal{P}$ to $\mathcal{C}$.
The ciphertext $c$ is composed of two polynomials $c_1$ and $c_2$, which are computed according to Eqs. \ref{eq:encc1} and \ref{eq:encc2}, where $u$ is a random polynomial from the set $\{-1, 0, 1\}$, $e_1$ and $e_2$ are small random polynomials from a discrete Gaussian distribution, and $\Delta = \floor{q/t}$ is a scaling factor.
\begin{align}
c_1 = k_{p1} \cdot u + e_1 + \Delta m \mod q\label{eq:encc1}\\
c_2 = k_{p2} \cdot u + e_2 \mod q\label{eq:encc2}
\end{align}
The ciphertext multiplication $c_c = c_a \cdot c_b \Leftrightarrow (c_{c1}, c_{c2}, c_{c3}) = (c_{a1}, c_{a2}) \cdot (c_{b1}, c_{b2})$ is calculated by evaluating the tensor in Eq. \ref{eq:ctmul}.
The term $t/q$ is a scaling factor, while the remaining operations are polynomial multiplications and additions over rings.
The polynomial addition is a simple operation with linear time complexity.
However, a naive implementation of polynomial multiplication has quadratic time complexity.
In addition, there is need for polynomial reduction after the polynomial multiplication.
More efficient algorithms using the Number Theoretic Transform (NTT) with nega-cyclic convolutions have been proposed \cite{cooley-tukey, gentleman-sande}, reducing the time complexity to $\mathcal{O}(n\log n)$ and avoiding the need for polynomial reduction over the polynomial modulus $x^n+1$.
% \begin{equation}\label{eq:ctmul}
% (c_{c1}, c_{c2}, c_{c3}) = \Big( \Biground{ \frac{t(c_{a1} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a1} \cdot c_{b2} + c_{a2} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a2} \cdot c_{b2})}{q} }_q \Big)
% \end{equation}
\begin{equation}\label{eq:ctmul}
\begin{split}
(c_{c1}, c_{c2}, c_{c3}) = \Big( \Biground{ \frac{t(c_{a1} \cdot c_{b1})}{q} }_q, \\
\Biground{ \frac{t(c_{a1} \cdot c_{b2} + c_{a2} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a2} \cdot c_{b2})}{q} }_q \Big)
\end{split}
\end{equation}
% \subsection{Number Theory Transform (NTT)}
% The NTT is the generalized Discrete Fourier Transform (DFT).
% It is an integral part of FHE algorithms for accelerating polynomial multiplications over finite fields.
% Without NTT, polynomial multiplications have quadratic complexity $\mathcal{O}(n^2)$.
% NTT converts polynomials to a domain that reduces the complexity of polynomial multiplications to linear $\mathcal{O}(n)$.
% A polynomial multiplication in the NTT domain is simply a coefficient-wise multiplication of two polynomials.
% Nevertheless, NTT has a time complexity of $\mathcal{O}(n\log n)$, therefore, the complete polynomial multiplication including NTT has this complexity.
% There are several algorithms for NTT \cite{cooley-tukey, gentleman-sande}. \cofhee{} implements the Cooley-Tukey algorithm \cite{cooley-tukey}.
% % as shown in Algorithm \ref{alg:ntt}.
% Each of the $\log n$ stages of NTT computes $n/2$ butterfly operations.
% % The butterfly operation is discussed in \hlorange{Section \ref{ss:pe}}.
% % \begin{equation}\label{eq:ctmul2}
% % (c_{c1}, c_{c2}, c_{c3}) = \Big( \Biground{ \frac{t(c_{a1} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a1} \cdot c_{b2} + c_{a2} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a2} \cdot c_{b2})}{q} }_q \Big)
% % \end{equation}
% The parameters $n$, $t$, and $q$ are application-specific.
% For efficient computations, $n$ is defined as a power of two.
% Typical values for $n$ are $2^{12}$, $2^{13}$, and $2^{14}$ \cite{kim2015private, dowlin2016cryptonets, 10.1145/3133956.3134061, pmt, gursoy}.
% The parameter $q$ is defined based on the selected $n$ and desired security level.
% Table \ref{tab:parameters} presents the recommended encryption parameters against classical and quantum computers.
% \input{tables/parameters}
% \subsection{Residue Number System}
% \hl{As one can see, the coefficient size $\log q$ is usually larger than 64 bits, making computations on 64-bit processors inefficient.}
% To cope with that, $q$ is usually broken into smaller $q_i$ using the Residue Number System (RNS).
% In RNS, a large number is represented by its value modulo several coprime moduli following the Chinese Remainder Theorem.
% This effectively breaks each polynomial into several polynomials with smaller coefficients.
% In the case of ciphertexts, each pair of polynomials $(c_1, c_2)$ is broken into several pair of polynomials $(c_{1_1}, c_{2_1}), (c_{1_2}, c_{2_2}), (c_{1_3}, c_{2_3}), ...$, called towers.
% During ciphertext multiplication, each tower operates independently.
% , and Eq. \ref{eq:ctmul} must be applied to all towers individually.
% \subsection{Polynomial Multiplication}
% % Algorithm \ref{alg:polynomial_mult} describes the polynomial multiplication operation supported by \cofhee{}.
% Polynomial multiplication over rings is the main operation in ciphertext multiplication for RLWE-based FHE.
% This operation can be divided into two parts: (1) Multiplication and (2) Reduction.
% 1) For the first part, we use NTT to reduce the complexity of the operation from $\mathcal{O}(n^2)$ to $\mathcal{O}(n\log n)$.
% % , as discussed in \hlorange{Section \ref{ss:ntt}}.
% 2) The second part reduces the polynomial over the polynomial modulus.
% This operation is avoided using Negative Wrapped Convolution (NWC).
% NWC requires the $2n^{th}$ primitive root of unity and reduces the polynomial degree to $n-1$ over the polynomial $x^n+1$.
% \subsection{Number Theory Transform (NTT)}
% The NTT is the generalized Discrete Fourier Transform (DFT).
% It is an integral part of FHE algorithms for accelerating polynomial multiplications over finite fields.
% Without NTT, polynomial multiplications have quadratic complexity $\mathcal{O}(n^2)$.
% NTT converts polynomials to a domain that reduces the complexity of polynomial multiplications to linear $\mathcal{O}(n)$.
% A polynomial multiplication in the NTT domain is simply a coefficient-wise multiplication of two polynomials.
% Nevertheless, NTT has a time complexity of $\mathcal{O}(n\log n)$, therefore, the complete polynomial multiplication including NTT has this complexity.
% There are several algorithms for NTT \cite{cooley-tukey, gentleman-sande}. \cofhee{} implements the Cooley-Tukey algorithm \cite{cooley-tukey}.
% % as shown in Algorithm \ref{alg:ntt}.
% Each of the $\log n$ stages of NTT computes $n/2$ butterfly operations.
% % The butterfly operation is discussed in \hlorange{Section \ref{ss:pe}}.
\subsection{Residue Number System}
\label{sss:rns}
As one can see, the coefficient size $\log q$ is usually larger than 64 bits, making computations on 64-bit processors inefficient.
To cope with that, $q$ is usually broken into smaller $q_i$ using the Residue Number System (RNS).
In RNS, a large number is represented by its value modulo several coprime moduli following the Chinese Remainder Theorem.
This effectively breaks each polynomial into several polynomials with smaller coefficients.
In the case of ciphertexts, each pair of polynomials $(c_1, c_2)$ is broken into several pair of polynomials $(c_{1_1}, c_{2_1}), (c_{1_2}, c_{2_2}), (c_{1_3}, c_{2_3}), ...$, called towers.
During ciphertext multiplication, each tower operates independently, and Eq. \ref{eq:ctmul} must be applied to all towers individually.
\section{\cofhee{} Design Flow Overview}
\label{sec:method}
% The architecture consists of .... . A detailed description of each main component is provided in the following sections.
\subsection{Ciphertext Multiplication}
\label{sss:ctmul}
% The Homomorphic Encryption Security Standard defines the basic primitives of BFV \cite{hestd}.
% Out of those primitives, the homomorphic multiplication of ciphertexts (\texttt{EvalMult}) is the slowest operation, and therefore, the main candidate for hardware acceleration.
% In order to understand ciphertext multiplication, we first need to understand encryption.
% Let $m$ be a plaintext in the plaintext space ($m \in \mathcal{P}$), $k_p = (k_{p1}, k_{p2})$ be an encryption key in the ciphertext space $\mathcal{C}$, and $c = (c_1, c_2) \in \mathcal{C}$ be an encryption of $m$ with key $k_p$;
% the encryption function $E(k_p, m) \rightarrow c$ defines a map from $\mathcal{P}$ to $\mathcal{C}$.
% The ciphertext $c$ is composed of two polynomials $c_1$ and $c_2$, which are computed according to Eqs. \ref{eq:encc1} and \ref{eq:encc2}, where $u$ is a random polynomial from the set $\{-1, 0, 1\}$, $e_1$ and $e_2$ are small random polynomials from a discrete Gaussian distribution, and $\Delta = \floor{q/t}$ is a scaling factor.
% \begin{align}
% c_1 = k_{p1} \cdot u + e_1 + \Delta m \mod q\label{eq:encc1}\\
% c_2 = k_{p2} \cdot u + e_2 \mod q\label{eq:encc2}
% \end{align}
% \begin{strip}
% \begin{equation}\label{eq:ctmul}
% (c_{c1}, c_{c2}, c_{c3}) = \Big( \Biground{ \frac{t(c_{a1} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a1} \cdot c_{b2} + c_{a2} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a2} \cdot c_{b2})}{q} }_q \Big)
% \end{equation}
% \end{strip}
% The most computationally intense low-level FHE operation is the ciphertext multiplication, $c_c = c_a \cdot c_b \Leftrightarrow (c_{c1}, c_{c2}, c_{c3}) = (c_{a1}, c_{a2}) \cdot (c_{b1}, c_{b2})$. It is calculated by evaluating the tensor in Eq. \ref{eq:ctmul}.
% The term $t/q$ is a scaling factor, while the remaining operations are polynomial multiplications and additions over rings.
% The polynomial addition is a simple operation with linear time complexity.
% However, a naive implementation of polynomial multiplication has quadratic time complexity.
% %In addition, there is need for polynomial reduction after the polynomial multiplication.
% More efficient algorithms using the Number Theoretic Transform (NTT) with nega-cyclic convolutions have been proposed \cite{cooley-tukey, gentleman-sande}, reducing the time complexity to $\mathcal{O}(n\log n)$ and avoiding the need for polynomial reduction over the polynomial modulus $x^n+1$.
% \begin{equation}\label{eq:ctmul}
% \begin{split}
% (c_{c1}, c_{c2}, c_{c3}) = \Big( \Biground{ \frac{t(c_{a1} \cdot c_{b1})}{q} }_q, \\
% \Biground{ \frac{t(c_{a1} \cdot c_{b2} + c_{a2} \cdot c_{b1})}{q} }_q, \Biground{ \frac{t(c_{a2} \cdot c_{b2})}{q} }_q \Big)
% \end{split}
% \end{equation}
The most computationally intense low-level FHE operation is the ciphertext multiplication. Consequently, the main objective of the \cofhee{} co-processor is native support for modular operations, which is essential for accelerating FHE cryptosystems. \cofhee{} is designed to support polynomial degrees of up to $2^{14}$ in powers of two and modulus size up to 128 bits, while it is optimized to work with a polynomial degree $n = 2^{13}$.
These values offer a trade-off between maximizing performance of FHE applications \cite{kim2015private, pmt, gursoy} and minimizing \cofhee{}'s area.
% Consequently, the main objective of the \cofhee{} co-processor is native support for modular operations for degrees of up to $2^{14}$, which is essential for accelerating FHE cryptosystems such as BFV.
% \cofhee{} is designed to support polynomial degrees of up to $2^{14}$ in powers of two and modulus size up to 128 bits, while it is optimized to work with a polynomial degree $n = 2^{13}$.
% These values offer a trade-off between maximizing performance of FHE applications \cite{kim2015private, pmt, gursoy} and minimizing \cofhee{}'s area.
\begin{figure}[t]
\centering
% \includegraphics[width=\linewidth]{figs/toparch.png}
\includegraphics[width=\linewidth]{figs/toparch_DATE.png}
\caption{\cofhee{}~Top Level Architecture}
% Remake Diagram
\label{fig:TopArch}
% \vspace{-1cm}
\end{figure}
\cofhee{}'s design is constrained by a limited chip area of $12mm^2$ and the use of $55nm$ CMOS technology from GlobalFoundries. Given the aforementioned limitations and the common encryption parameters used in FHE applications (Section \ref{sss:ctmul}), we have determined that the maximum polynomial degree that can be supported on the chip is $n=2^{13}$, with coefficient sizes of 128 bits. This is the largest coefficient size that can be accommodated within the design area constraints. In the case of $n=2^{14}$, extra communication is required. To achieve this goal, we have developed a system architecture that includes 1 Processing Element (PE), 3 dual-port and 5 single-port SRAMs. This configuration of units allows for full on-chip ciphertext multiplication with an Initiation Interval ($II$) of 1, as the dual-port SRAMs enable the simultaneous fetching and storing of 2 different operands in the same cycle. The majority of the available chip area is occupied by the SRAMs, with the remaining space allocated for the PEs, Multiplier Data Mover and Controller (MDMC), Direct Memory Access controller (DMA), General Purpose Configuration registers (GPCFG), and an ARM Cortex-M0 processor with its own memory. Additionally, the chip operates at a target frequency of $250$ MHz, which is limited by the memory latency, and has two voltage supplies, $3.3$ V for the IO pads and $1.2$ V for the logic core. Table \ref{tab:gpcfg} shows a representative subset of the 35 Configuration Registers in \cofhee{}. The configuration registers in CoFHEE are mapped to the memory range of 0x4002 0000 – 0x4002 FFFF, and the chip's memory base address follows the ARM Cortex M series memory map convention for memory and peripheral addresses.
% The chip area available for \cofhee{}'s design is $12mm^2$ and the available technology node is GF $55nm$. Considering this limitation in area, and common encryption parameters used in FHE applications (Section \ref{sss:ctmul}), we can achieve our goal of performing ciphertext multiplication on chip for a maximum polynomial degree $n=2^{13}$ with 128-bit coefficient sizes (in case of $n=2^{14}$, extra communication is needed).
% A 128-bit coefficient size is the largest coefficient we can fit in \cofhee{} given design area limitations. The aforementioned restrictions allowed us to develop a system architecture with 1 Processing Element (PE), 3 dual-port and 5 single-port SRAMs.
% This selection of units enables support of ciphertext multiplication fully on chip, with an $II$ (Initiation Interval) $=1$, as the dual-port SRAMs provide the ability to fetch and store 2 different operands in the same cycle.
% The biggest portion of the provided area is occupied by the dual and single-port SRAMs. In the remaining space, we fit the Processing Elements (PE), Multiplier Data Mover and Controller (MDMC), Direct Memory Access controller (DMA), General Purpose Configuration registers (GPCFG), and an ARM Cortex-M0 with its own memory. Table \ref{tab:gpcfg} shows a representative subset of the 35 Configuration Registers in \cofhee{}.
% Our configuration registers map to the 0x4002\_0000 -- 0x4002\_FFFF memory range.
% In our design, the memory base address follows the ARM Cortex M series memory map convention for memory and peripheral addresses. Lastly, the chip operates at a target frequency of $250$ MHz (bounded by the memory latency), and has two voltage supplies, namely $3.3$ V (IO pads) and $1.2$ V (logic core).
% \begin{figure}[ht!]
% % \hspace{0.2cm}
% \begin{minipage}[t][][t]{0.32\textwidth}
% \centering
% \input{tables/gpcfg}
% \end{minipage}
% \end{figure}
% % Ciphertext multiplication requires four polynomials and twiddle factors as input.
% % Hence, it requires five memories to store the input data.
% It is important to note that dual-port memories play a crucial role in our design. Dual-port memories enable computation with an Initiation Interval $II=1$ clock cycle.
% $II$ is defined as the number of clock cycles needed to feed new operands to the computation units.
% If the polynomial coefficients are stored in a single-port SRAM, we cannot fetch two operands and store another two operands in the same cycle.
% Hence, we choose to use dual-port SRAMs as it is possible to fetch/write two operands per memory in the same cycle at a cost in area.
% Dual-port SRAMs takes almost 2 times the area compared to single-port SRAMs.
% Therefore, we limit the number of dual-port SRAMs to three, as two are required for computation to enable loading of operands and storing of the results in the same cycle, and one serves as a buffer for loading the next polynomial to go under NTT.
% DMA is used to move the next polynomial from a single-port SRAM to the third dual-port memory while NTT is happening, adding no extra latency.
% The chip area available for \cofhee{}'s design is $12mm^2$, which is quite limited.
% As the FHE application performs operation on big polynomial with large coefficient size, \cofhee{} needs a large area to perform operation on big numbers and store big polynomials.
% It is not possible to completely fit all the required operands in this chip to perform any real-world application.
% Hence, \cofhee{} only stores subset of data to perform computationally intensive operations.
% We select to perform ciphertext multiplication on \cofhee{} without any interaction from an external interface, as ciphertext multiplication is the slowest FHE operation.
% Ciphertext multiplication requires four polynomials and twiddle factors as input.
% Hence, it requires five memories to store the input data.
% Ciphertext multiplication also requires two memories to store internal calculations.
% The size of each of this memory is shown in Table .
% Total memory area is the sum of dual-port memories, single-port memories for storing the data, and for loading ARM instructions.
% This totals to around $9.8 mm^2$.
% In the remaining area, we fit Processing Elements (PE), Multiplier Data Mover and Controller (MDMC), Direct Memory Access controller (DMA), General Purpose Configuration registers (GPCFG), and an ARM Cortex-M0 with its own memory.
% Table shows a representative subset of the 35 Configuration Registers in \cofhee{}.
% Our configuration registers map to the 0x4002\_0000 -- 0x4002\_FFFF memory range.
% In our design, the memory base address follows the ARM Cortex M series memory map convention for memory and peripheral addresses.}
% \hl{Fig. \ref{fig:TopArch} shows \cofhee{}'s top architecture.}
% \hl{The PE is connected to the MDMC, which in turn has a direct connection to the command FIFO and communicates with the memories using the AHB lite interconnect.
% The AHB lite interconnect connects all IPs and ensures reliable data movement with variable data sizes.
% The MDMC performs memory reads and writes for computation; DMA transfer data from one memory to another; and the external interfaces communicate with the host.}
% \subsection{Clock frequency}
% \cofhee{}'s frequency is defined by the critical path of the design as well as the technology node targeted.
% The critical path can be either in the computation unit due to a complex data path or it can be in the memory as a consequence of memory read latency.
% While the former can be reduced with pipelining, memory read latency is fixed for the technology node and memory size used.
% As expected, \cofhee{}'s critical path is in the memory read for loading data from the memory to the computation unit.
% For the technology node targeted, i.e., Globalfoundries (GF) 55nm low power enhanced (LPE) process, the memory read path is around $4ns$, which translates to a clock frequency of 250 Mhz.
% %A 22nm TSMC memory compiler \cite{noauthor_synopsys_memcomp} reports the memory read latency to be around 1.4 ns, meaning that \cofhee{} can easily reach 600-700 Mhz or higher frequencies if implemented in this technology node.
\subsection{Execution \& Operations}
% \input{tables/operations}
% \cofhee{} is designed to support polynomial degrees of up to $2^{14}$ in powers of two and modulus size up to 128 bits, while it is optimized to work with a polynomial degree $n = 2^{13}$.
% These values offer a trade-off between maximizing performance of FHE applications \cite{kim2015private, pmt, gursoy} and minimizing \cofhee{}'s area.
% dowlin2016cryptonets, 10.1145/3133956.3134061
%As is a co-processor for the acceleration of polynomial multiplication, it has to support all the fundamental operations.
% \cofhee{} supported operations can be divided into two types: Compute and memory operations.
% Compute operations consist of NTT, inverse NTT, and a set of pointwise operations, namely normal and modular multiplication, modular squaring, modular multiplication by a constant, and modular addition and subtraction.
% \cofhee{} provides an ISA to execute any of these operations when relevant polynomials are loaded into the memory.
% Meanwhile, memory operations replicate or transfer data from one memory to another.
% Although compute operations run sequentially similar to memory operations, both types can run simultaneously.
\cofhee{} supports a range of operations that can be broadly classified into two categories: compute and memory operations. Compute operations include the Number Theoretic Transform (NTT), Inverse Number Theoretic Transform (INTT), and a set of pointwise operations such as normal and modular multiplication, modular squaring, modular multiplication by a constant, and modular addition and subtraction. \cofhee{} provides an Instruction Set Architecture (ISA) to execute any of these operations when the relevant polynomials are loaded into the memory. On the other hand, memory operations include the replication or transfer of data from one memory to another. While compute operations run in a sequential manner, memory operations can be run simultaneously, which allows for concurrent execution of both types of operations.
The operations supported by \cofhee{} are summarized in Table \ref{tab:basic_ops}.
\input{tables/gpcfg}
% \subsubsection{Modular Multiplication}
% Modular multiplication involves normal multiplication followed by a modular reduction of the product. There are few prominent types of modular multiplication; interleaved, Barrett, and Montgomery \cite{cryptohandbook}.
% Barrett is selected for our implementation as there is no need to transform the arguments, as required for Montgomery \cite{modmulalgocomp}. It has an $II = 1$ and
% the target frequency can be achieved by pipelining a single Barrett design whereas for the interleaved multiplier, to achieve $II$ of $1$, one needs to increase the number of said multipliers linearly, thus increasing the design area as well as the number of nets in the design.
% \input{tables/operations}
% \subsubsection{Algorithms}
\textit{Algorithms}:
% There are few prominent types of modular multiplication; interleaved, Barrett, and Montgomery \cite{cryptohandbook}.
% Barrett is selected for our implementation as there is no need to transform the arguments, as required for Montgomery \cite{modmulalgocomp}.
Regarding polynomial multiplication, \cofhee{} implements and operates using the Cooley-Tukey NTT algorithm \cite{cooley-tukey}.
\cofhee{} also handles a ciphertext multiplication algorithm.
% Algorithm \ref{alg:ctmul} describes the ciphertext multiplication operation as handled by \cofhee{}.
Two ciphertexts (polynomial pairs) are passed as input.
The ciphertext polynomials are multiplied following Eq. \ref{eq:ctmul}.
Each polynomial is used twice in polynomial multiplications.
Nevertheless, we only need one NTT operation per polynomial as they can stay in the NTT domain during the polynomial multiplication and addition.
At the end, we can convert back from the NTT domain using the iNTT operation.
The entire ciphertext multiplication in \cofhee{} utilizes 4 NTTs, 4 Hadamard products, 1 pointwise addition, and 3 iNTTs.
% \input{algo/ctmul}
% \begin{figure}[ht!]
% % \begin{minipage}[t][][t]{0.46\textwidth}
% % \centering
% % \input{algo/ntt}
% % \vspace{-0.5cm}
% % \input{algo/polymul}
% % \end{minipage}
% % \hspace{0.2cm}
% \begin{minipage}[t][][t]{0.46\textwidth}
% \centering
% \input{algo/ctmul}
% \end{minipage}
% \end{figure}
\subsection{\cofhee{} API}
\cofhee{} exposes assembly-like instructions for the operations listed in Table \ref{tab:basic_ops}.
Composed operations such as polynomial multiplication or ciphertext multiplication build on top of the atomic ones.
\cofhee{}'s design is optimized for $n=2^{13}$.
Nevertheless, it supports any $n \leq 2^{14}$, assuming that $n$ is a power of two.
It can perform NTT, polynomial multiplications, and ciphertext multiplications on chip with $II=1$, without requiring back-and-forth communication to the host during the computation for any $n \leq 2^{13}$.
In the case of $n=2^{14}$, \cofhee{} can perform NTT on chip with $II=2$, as single-port memories need to be used for computation.
%, while polynomial and ciphertext multiplications require back-and-forth communication.
The maximum coefficient size supported by \cofhee{} is 128 bits.
Additions and subtractions take 1 cycle and multiplication takes 4 cycles, independently of the coefficient size.
Coefficients larger than 128 bits must be broken using RNS, similarly to how it is done in software, to fit in \cofhee{}.
\input{tables/operations}
\subsection{Clock frequency}
\cofhee{}'s frequency is defined by the critical path of the design as well as the technology node targeted.
The critical path can be either in the computation unit due to a complex data path or it can be in the memory as a consequence of memory read latency.
While the former can be reduced with pipelining, memory read latency is fixed for the technology node and memory size used.
As expected, \cofhee{}'s critical path is in the memory read for loading data from the memory to the computation unit.
For the technology node targeted, i.e., Globalfoundries (GF) 55nm low power enhanced (LPE) process, the memory read path is around $4ns$, which translates to a clock frequency of 250 Mhz.
%A 22nm TSMC memory compiler \cite{noauthor_synopsys_memcomp} reports the memory read latency to be around 1.4 ns, meaning that \cofhee{} can easily reach 600-700 Mhz or higher frequencies if implemented in this technology node.
\vspace{-0.1cm}
\subsection{Processing Element}
% \subsubsection{Processing Elements (PEs)}
% A processing element is a basic processing unit.
\cofhee{}'s PE supports modular multiplication, modular addition, and modular subtraction.
Each operation has an $II=1$.
Modular addition and subtraction have a latency of one clock cycle, while modular multiplication completes in five clock cycles.
In contrast to the state-of-the-art that uses Montgomery multipliers, \cofhee{} employs a Barrett multiplier to implement multiplication and modular reduction due to its pipelining capabilities \cite{modmulalgocomp}.
The pipeline depth is chosen such that the critical path matches the memory read latency.
%In addition, a Barrett multiplier enables normal multiplication as it is possible to bypass the modular reduction at the end.
Our PE operates in four distinct modes defined by the mode selection input.
The operations modes are: (1) Modular Multiplication, (2) Modular Addition, (3) Modular Subtraction, and (4) Butterfly operation.
The butterfly operation is an atomic computational unit of NTT.
It is executed by performing modular multiplication followed by modular addition and modular subtraction.
We implement the butterfly operation using radix 2, and acquire a PE that occupies $6\%$ of the design area.
% Using the four modes, \cofhee{}'s PE executes all compute operations mentioned in Table \ref{tab:basic_ops} while occupying 6\% of the design area.
% Normal and modular multiplication, as well as modular squaring work on mode (1).
% Different bits in the operation command set flags to, for example, bypass the modular reduction, enabling a variety of similar operations to use the same hardware.
\vspace{-0.4cm}
\subsection{NTT considerations}
For a $2\times$ performance improvement of NTT, the input and output polynomials are stored in dual-port memories.
In addition, there is a third dual-port memory to store the next polynomial to go under NTT.
If the next polynomial is in a single-port memory, DMA is used to transfer polynomial coefficients to the available dual-port memory while NTT operates on the current polynomial.
This is possible since the bus architecture allows the MDMC, DMA, and ARM CM0 to access memories in parallel.
Once NTT completes, the dual-port memories switch tasks; the one containing the next polynomial participates in the following NTT, while the memory containing the output uses DMA to offload the result to a single-port memory and load the next polynomial for NTT.
This process happens transparently in the background without performance degradation due to data movement.
\vspace{-0.4cm}
\subsection{Internal Data Flow}
\subsubsection{Interconnect}
We implement a parameterized Advanced High-Performance Bus (AHB) lite interconnect \cite{noauthor_amba_2001} as it supports the required bus transfer operations and has low area utilization. \cofhee{} requires single memory transfers as well as burst memory operations, with the data sizes ranging from 32 to 128 bits.
% \hlorange{Table} shows a representative subset of the 35 Configuration Registers in \cofhee{}.
Our configuration registers map to the 0x4002\_0000 -- 0x4002\_FFFF memory range.
In our design, the memory base address follows the ARM Cortex M series memory map convention for memory and peripheral addresses.
% The interconnect should provide this flexibility.
% \hl{+++}
% Explain that this creates no significant bottlenecks
\subsubsection{Multiplier Data Mover and Controller (MDMC)}
The sequence of commands to be executed is stored in the command FIFO, which decodes the command and triggers the MDMC for the requested operation. The FIFO also provides the memory base addresses of input operands and output result.
When running computation-based operations, the MDMC fetches data from memory and forwards to the PE on every clock cycle until the operation is completed.
Once the data is processed, the MDMC stores the data back to the output memory.
% \hl{For NTT and iNTT, the MDMC needs to forward three input data and store two output data per cycle.}
In order to achieve an $II=1$ for each butterfly operation, we store the input and output polynomials in dual-port memories.
Thus, we can fetch two polynomial coefficients from one memory (dual-port) and a twiddle factor from another (single-port) in a single clock cycle.
After computing the butterfly operation, the MDMC stores the two outputs in the output memory (dual-port).
Once an NTT stage completes, the output memory acts as input memory and vice-versa, until the NTT/iNTT is finalized.
%Due to the complexity of the operations ran by the MDMC, we employ a finite state machine (FSM) which maintains internal information regarding ongoing operations.
% The MDMC performs complex operations which run for thousands of clock cycles.
% For that reason, it employs a finite state machine (FSM) to maintain internal information about the ongoing operation.
\vspace{-0.2cm}
\subsection{External Interfaces}
% \subsubsection{IO}
\cofhee{} provides SPI and UART interfaces for external host communication.
These interfaces are used for loading polynomials, triggering the required operation and reading back the result.
SPI and UART are chosen mainly for their simplicity in implementation as well as the ability to communicate from an external PC.
One can always replace these interfaces with faster ones such as PCI-express or HSIC (High-Speed Inter-Chip).
Other IOs in \cofhee{} are clock, reset, voltage supplies, bias voltages, bias currents, PLL controls, and debugging IOs.
% \input{algo/ctmul}
% \subsection{Design Blocks}
%\subsection{Pre-silicon verification}
%We verified the functionality of our RTL design using both simulation and FPGA-based validation.
%The simulation was performed using Synopsys VCS at the top-level.
%A Python script is used to calculate the modulus following the equation $q = 2 k \cdot n + 1$, where $k \ge 1$ is an arbitrary constant.
%In addition, the script finds twiddle factors, generate random input polynomial coefficients, and calculate the expected results.
%We use random coefficient values modulo $q$ for our test polynomials since the 128-bit operand range cannot be exhaustively tested.
%These values are then ported to the Verilog testbench, where they are loaded and the result from the design is compared against the expected result.
% Once the polynomial degree $n$ is fixed, the python script finds different prime numbers in the form of $q = k*n+1$, where k is some constant, to be used as modulus. Then corresponding twiddle factors are calculated.
% Moreover, for our FPGA design, we implemented a scaled-down version of \cofhee{}, as $n=2^{13}$ is incompatible with the available resources of our FPGAs.
% Specifically, the maximum polynomial degree that could be supported on a Digilent Nexys 4 is $n=2^{12}$ running at 10 MHz.
\subsection{Execution modes}
There are three ways to execute the basic operations supported by \cofhee{} (Table \ref{tab:basic_ops}). The simplest option is for the external host to directly trigger the MDMC to perform the commands through a configuration register write.
This mode is slow as there are delays imposed by the communication interface when writing to the configuration register.
A second possibility is to use the command FIFO, where the external host preloads the sequence of commands to be executed and waits for an interrupt issued by \cofhee{} signaling that the queue is empty.
As soon as the first command is written to the queue, the command FIFO sends it to the MDMC.
The MDMC starts the operation and once it completes, it sends a signal to the command FIFO.
The process repeats until all commands have been executed.
The command FIFO guarantees the execution of a single command at a time in a predefined order.
Thus, it requires less control logic and avoids complicated out-of-order executions.
In addition, the command FIFO gives the flexibility to execute different complex operations as combinations of basic instructions.
%As an example, one can load a sequence of commands to perform a ciphertext multiplication.
We define the length of the queue to be 32 commands, as it is more than sufficient for our target applications.
%Whenever the command FIFO becomes empty, the external host has to load another set of commands.
%This can be time consuming depending on the external host interface speed.
The host can continuously load more commands while the queue is not full. Lastly, for a faster and flexible sequencing and execution of commands, we introduce a third mode, which utilizes a 32-bit ARM Cortex M0 (CM0) along with a dedicated instruction memory.
In this mode, the processor is used instead of the command FIFO.
One can write complex subroutines and sequence of operations in embedded C, then compile and preload it in CM0's instruction memory for later execution.
\subsection{Pre-silicon verification}
We verified the functionality of our RTL design using both simulation and FPGA-based validation.
The simulation was performed using Synopsys VCS at the top-level.
A python script is used to calculate the modulus following the equation $q = 2 k \cdot n + 1$, where $k \ge 1$ is an arbitrary constant.
In addition, the script finds twiddle factors, generate random input polynomial coefficients, and calculate expected results.
We use random coefficient values modulo $q$ for our test polynomials since the 128-bit operand range cannot be exhaustively tested.
These values are then ported to the verilog testbench, where they are loaded and the result from the design is compared against the expected result.
% Once the polynomial degree $n$ is fixed, the python script finds different prime numbers in the form of $q = k*n+1$, where k is some constant, to be used as modulus. Then corresponding twiddle factors are calculated.
Moreover, for our FPGA design, we implemented a scaled-down version of \cofhee{}, as $n=2^{13}$ is incompatible with the available resources of our FPGAs.
Specifically, the maximum polynomial degree that could be supported on a Digilent Nexys 4 is $n=2^{12}$ running at 10 MHz.
\subsection{Synthesis}
% \input{tables/synthesis} % merged with table tables/area.tex
\cofhee{}'s RTL code is synthesized using a $55nm$ standard cell library from GlobalFoundries and a clock constraint of 250MHz.
SPI IO timing is constrained to meet 50Mhz of interface speed.
There is no specific IO timing constraint set for UART as it is an asynchronous interface.
Following standard practices, the standard cell library used for synthesis was the one characterized for the worst voltage (1.08V), temperature (125C), resistance, and capacitance.
Synthesizing with such a library ensures that we can achieve the target frequency in various operating conditions.
For synthesis, we use the Synopsys Design Compiler (DC).
In Table \ref{tab:area}, we present the post synthesis area and timing of the major \cofhee{} blocks.
Other than memory, the largest design is the PE, followed by the AHB and configuration registers.
Post-synthesis, we observe many blocks with critical timing path much higher than the target clock period of $4ns$. This is because synthesis setup uses restricted standard cell library with no access to faster library cells such as the ones with lower threshold voltage $V_t$.
As these paths are long combinational paths, they easily meet timing in the backend, where all the standard cell libraries are used.
In this condition, paths starting from memory become the critical path due to memory read latency (around $3.1ns$).
We verified the functionality of our RTL design using both simulation and FPGA-based validation.
% \begin{figure}[t]
% \hspace{0.2cm}
% % \begin{minipage}[t][][t]{0.20\textwidth}
% % \input{tables/io_ports}
% % \end{minipage}
% \begin{minipage}[t][][t]{0.50\textwidth}
% \input{tables/area}
% \end{minipage}
% \end{figure}
% \cofhee{} includes an additional UART interface that sends the start address of the result polynomial to the external host once the triggered operation is done.
% In addition, \cofhee{} provides an interrupt to the external host as a way of signaling the completion of the triggered operation.
% Other IOs in \cofhee{} are clock, reset, voltage supplies, bias voltages, bias currents, PLL controls, and debugging IOs.
% The clock input provides reference clock for the PLL, reset is active low asynchronous, voltage supplies for digital are 3.3V for the IO pads and 1.2V for the core standard cells and memories.
\begin{table}[h]
\centering
\resizebox{\columnwidth}{!}{%
\begin{tabular}{||l|l|c||}
\hline
\multirow{2}{*}{\parbox{2.7cm}{{\bf Register Name}}} &
\multirow{2}{*}{\parbox{4.1cm}{{\bf Description}}} &
\multirow{2}{*}{\parbox{0.7cm}{\centering {\bf Bit Size}}} \\
{} & {} & {} \\
\hline \hline
%\textbf{IO Pin Name} & \textbf{Direction} & \textbf{Description}\\ \hline \hline
UARTMTX\_PAD\_CTL & IO pad control for primary UART TX & 32 \\ \hline
UARTMRX\_PAD\_CTL & IO pad control for primary UART RX & 32 \\ \hline
UARTSTX\_PAD\_CTL & IO pad control for secondary UART TX & 32 \\ \hline
SPIMOSI\_PAD\_CTL & SPI data in pad control & 32 \\ \hline
SPIMISO\_PAD\_CTL & SPI data out pad control & 32 \\ \hline
SPICLK\_PAD\_CTL & SPI clock pad control & 32 \\ \hline
SPICSN\_PAD\_CTL & SPI chip select pad control & 32 \\ \hline
HOSTIRQ\_PAD\_CTL & IO pad control for Host Interrupt & 32 \\ \hline
% GPIO0\_PAD\_CTL & IO pad control for GPIO & 32 \\ \hline
% PAD11\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD12\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD13\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD14\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD15\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD16\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD17\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD18\_PAD\_CTL & Reserved & 32 \\ \hline
% PAD19\_PAD\_CTL & Reserved & 32 \\ \hline
UARTM\_BAUD\_CTL & Baud control for primary UART & 32 \\ \hline
UARTS\_BAUD\_CTL & Baud control for secondary UART & 32 \\ \hline
UARTM\_CTL & Primary UART control & 32 \\ \hline
UARTS\_CTL & Secondary UART control & 32 \\ \hline
% SPI\_CTL & Reserved & 32 \\ \hline
% SPI\_CTL2 & Reserved & 32 \\ \hline
% UARTS\_TXDATA & Debug Uart txdata & 32 \\ \hline
% UARTS\_RXDATA & Debug Uart rxdata & 32 \\ \hline
% UARTS\_TXSEND & Trigger bit to send debug uart TX data & 32 \\ \hline
%SPARE0 & Reserved register & 32 \\ \hline
%SPARE1 & Reserved register & 32 \\ \hline
%SPARE2 & Reserved register & 32 \\ \hline
SIGNATURE & Stores Chip ID & 32 \\ \hline
Q & Modulus $q$ & 128 \\ \hline
N & Polynomial degree $n$ & 128 \\ \hline
INV\_POLYDEG & $n^{-1} \ \mod q$ & 128 \\ \hline
BARRETTCTL1 & $barrettk = 2 \cdot \log n$ & 32 \\ \hline
BARRETTCTL2 & barrett constant = $2^{barrettk}/q$ & 160 \\ \hline
FHECTL1 & Command FIFO select and $n$ & 32 \\ \hline
FHECTL2 & Trigger bits for different commands & 32 \\ \hline
FHECTL3 & Select or bypass PLL clock & 32 \\ \hline
PLLCTL & Control bits required for the PLL & 32 \\ \hline
COMMANDFIFO & Trigger bits for different commands & 32 \\ \hline
DBG\_REG & Debug register & 32 \\ \hline
% DPMEM0 & Argument A for modular blocks & 2048 \\ \hline
% DPMEM1 & Argument A for modular blocks & 2048 \\ \hline
% DPMEM2 & Argument A for modular blocks & 2048 \\ \hline
\end{tabular}
}
\caption{Subset of \cofhee{} Configuration Registers.} \label{tab:gpcfg}
\end{table}
% % Please add the following required packages to your document preamble:
% % \usepackage{multirow}
% \begin{table}[]
% \begin{tabular}{|p{3cm}|p{5cm}|p{7cm}|}
% \hline
% \multicolumn{1}{|c|}{\textbf{Command}} & \multicolumn{1}{|c|}{\textbf{Inputs}} & \multicolumn{1}{c|}{\textbf{Comments}} \\ \hline
% NTT & \begin{tabular}[c]{@{}l@{}}Polynomial Degree\\ Polynomial coefficients \\ Twiddle Factors\\ Modulus\\ Twiddle factors and Polynomial \\ coefficients memory base address \\ Memory base address to store \\ intermediate result\end{tabular} & NTT of polynomial coefficients. \\ \hline
% iNTT & \begin{tabular}[c]{@{}l@{}}Polynomial Degree\\ Polynomial coefficients \\ Twiddle Factors\\ Modulus\\ Inverse of Polynomial Degree\\ Twiddle factors and Polynomial \\ coefficients memory base address \\ Memory base address to store \\ intermediate result\end{tabular} &inverse NTT of polynomial coefficients. \\ \hline
% PWISEMODMUL & \begin{tabular}[c]{@{}l@{}}Polynomial Degree, coefficients, Modulus\\ Polynomial coefficients memory \\ base address \\ Memory base address to store result\end{tabular} & point-wise modular multiplication of two vectors. \\ \cline{1-1} \cline{3-3}
% PWISEMODADD & &point-wise modular addition of two vectors. \\ \cline{1-1} \cline{3-3}
% PWISEMODSUB & & point-wise modular subtraction of two vectors. \\ \cline{1-1} \cline{3-3}
% PWISEMODSQR & &modular squaring of polynomial coefficients. \\ \hline
% PWISEMUL & \begin{tabular}[c]{@{}l@{}}Polynomial Degree\\ Polynomial coefficients \end{tabular} & point-wise multiplication of two vectors. \\ \hline \hline
% MEMCPY & \begin{tabular}[c]{@{}l@{}}Number of words to move\\ Source memory start address \\ Destination memory start address\\ \end{tabular} & Transfers data from one memory to another. \\ \hline
% MEMCPY\_BR & \begin{tabular}[c]{@{}l@{}}Number of words to move\\ Source memory start address \\ Destination memory start address\\ \end{tabular} & Transfers data from one memory to another in bit-reverse addressing. \\ \hline
% \end{tabular}
% \caption{Basic operations on the chip.}
% \label{tab:basic_ops}
% \end{table}
\begin{table}[h!]
\centering
\resizebox{\columnwidth}{!}{%
\begin{tabular}{||l|cccccccccc|l||}
\hline
\multirow{2}*{\textbf{Command}} & \multicolumn{10}{c|}{\textbf{Inputs}} & \multirow{2}*{\textbf{Operations}} \\
% {} & \multicolumn{1}{c}{$n$} & \multicolumn{1}{c}{$\vec{x}$} & \multicolumn{1}{c}{$\vec{\psi}$}
% & \multicolumn{1}{c}{$q$} & \multicolumn{1}{c}{$n^{-1}$} & \multicolumn{1}{c}{$\delta$}
% & \multicolumn{1}{c}{$\Rsh$} & \multicolumn{1}{c|}{$\rightarrowtail$} & {} \\ \hline
{} & $n$ & $[\vec{x}]$ & $[\vec{y}]$ & $[\vec{\omega}]$ & $q$ & $n^{-1}$ & $[\vec{t}]$ & $\delta$ & $\Rsh$ & $\rightarrowtail$ & {} \\ \hline \hline
NTT & $\bullet$ & $\bullet$ & {} & $\bullet$ & $\bullet$ & {} & $\bullet$ & {} & {} & {} & Performs NTT on $\vec{x}$. \\ \hline
\multirow{2}*{iNTT} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & {} & \multirow{2}*{$\bullet$}
& \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & {} & {} & {} & Performs inverse NTT \\
{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} & on $\vec{x}$. \\ \hline
\multirow{2}*{PMODADD} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$}
& {} & \multirow{2}*{$\bullet$} & {} & {} & {} & {} & \multirow{2}*{$\bullet$} & Pointwise modular \\
{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} & addition of $\vec{x}$ and $\vec{y}$. \\ \hline
\multirow{2}*{PMODMUL} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$}
& {} & \multirow{2}*{$\bullet$} & {} & {} & {} & {} & \multirow{2}*{$\bullet$} & Pointwise mod. multi- \\
{} & {} & {} & & {} {} & {} & {} & {} & {} & {} & {} & plication of $\vec{x}$ and $\vec{y}$. \\ \hline
\multirow{2}*{PMODSQR} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & {}
& {} & \multirow{2}*{$\bullet$} & {} & {} & {} & {} & \multirow{2}*{$\bullet$} & Pointwise modular \\
{} & {} & {} & & {} {} & {} & {} & {} & {} & {} & {} & squaring of $\vec{x}$. \\ \hline
\multirow{2}*{PMODSUB} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$}
& {} & \multirow{2}*{$\bullet$} & {} & {} & {} & {} & \multirow{2}*{$\bullet$} & Pointwise mod. sub- \\
{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} & traction of $\vec{x}$ and $\vec{y}$. \\ \hline
\multirow{2}*{CMODMUL} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & {}
& {} & \multirow{2}*{$\bullet$} & {} & {} & {} & {} & \multirow{2}*{$\bullet$} & Mod. multiplication \\
{} & {} & {} & & {} {} & {} & {} & {} & {} & {} & {} & of $\vec{x}$ by a constant. \\ \hline
\multirow{2}*{PMUL} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$}
& {} & {} & {} & {} & {} & {} & \multirow{2}*{$\bullet$} & Pointwise multiplica- \\
{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} & tion of $\vec{x}$ and $\vec{y}$. \\ \hline
\multirow{2}*{MEMCPY} & {} & {} & {} & {} & {} & {} & {} & \multirow{2}*{$\bullet$}
& \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & Memory-to-memory \\
{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} & data transfer. \\ \hline
\multirow{2}*{MEMCPYR} & {} & {} & {} & {} & {} & {} & {} & \multirow{2}*{$\bullet$}
& \multirow{2}*{$\bullet$} & \multirow{2}*{$\bullet$} & Memory data transfer \\
{} & {} & {} & {} & {} & {} & {} & {} & {} & {} & {} & in bit-reverse. \\ \hline
\end{tabular}
}
\vspace{1em}
\caption{\cofhee's operations. $[\cdot]$: memory address function, $n$: polynomial degree, $\vec{x}$ and $\vec{y}$: polynomials, $\vec{\omega}$: twiddle factors, $q$: modulus, $n^{-1}$: inverse of $n$, $\vec{t}$: temporary values, $\delta$: length (in words), $\Rsh$: source address, $\rightarrowtail$: output/destination address}
\vspace{-1em}
\label{tab:basic_ops}
\end{table}
\section{Functional Units}
\label{sec:fun_units}
% \hl{Add intro paragraph}
% In this section we provide a description of the utilized units at an algorithmic level.
\subsection{Modular Multiplier Unit}
Modular multiplication involves normal multiplication followed by a modular reduction of the product. There are few popular ways to implement modular multipliers; interleaved multiplier, Barrett multiplier, and Montgomery multiplier \cite{cryptohandbook}.
Barrett is selected for our implementation as there is no need to transform the arguments, as required for Montgomery \cite{modmulalgocomp}.
% \hl{It has an $II = 1$ and
% the target frequency can be achieved by pipelining a single Barrett design whereas for the interleaved multiplier, to achieve $II$ of one, one needs to increase the number of such multipliers linearly, thus increasing the design area and the number of nets in the design.}
\subsection{NTT Unit}
\label{ss:ntt}
The Number Theory Transform (NTT) is the generalized Discrete Fourier Transform (DFT).
It is an integral part of FHE algorithms for accelerating polynomial multiplications over finite fields.
Without NTT, polynomial multiplications have quadratic complexity $\mathcal{O}(n^2)$.
NTT converts polynomials to a domain that reduces the complexity of polynomial multiplications to linear $\mathcal{O}(n)$.
% A polynomial multiplication in the NTT domain is simply a coefficient-wise multiplication of two polynomials.
% Nevertheless, NTT has a time complexity of $\mathcal{O}(n\log n)$, therefore, the complete polynomial multiplication including NTT has this complexity.
There are several algorithms for NTT \cite{cooley-tukey, gentleman-sande}; \cofhee{} implements the Cooley-Tukey algorithm \cite{cooley-tukey} as shown in Algorithm \ref{alg:ntt}.
Each of the $\log n$ stages of NTT computes $n/2$ butterfly operations.
As discussed earleir, the butterfly operation is an atomic computational unit of the NTT. The operation is comprised of the primitive operations of modular multiplication, modular addition, and finally, modular subtraction.
\subsection{Polynomial Multiplication Unit}
Polynomial multiplication over rings is the main operation in ciphertext multiplication for RLWE-based FHE.
This operation can be divided into two parts: (1) Multiplication and (2) Reduction.
1) For the first part, we use NTT to reduce the complexity of the operation from $\mathcal{O}(n^2)$ to $\mathcal{O}(n\log n)$, as discussed in Section \ref{ss:ntt}.
2) The second part reduces the polynomial over the polynomial modulus.
This operation is avoided using Negative Wrapped Convolution (NWC).
NWC requires the $2n^{th}$ primitive root of unity and reduces the polynomial degree to $n-1$ over the polynomial $x^n+1$.
Algorithm \ref{alg:polynomial_mult} describes the polynomial multiplication operation supported by \cofhee{}.
\subsection{Ciphertext Multiplication}
The ciphertext multiplication is the slowest operation in FHE computation.
We discuss it in detail in Section \ref{sss:ctmul}.
Algorithm \ref{alg:ctmul} describes this operation as handled by \cofhee{}.
Two ciphertexts (polynomial pairs) are passed as input.
The ciphertext polynomials are multiplied following Eq. \ref{eq:ctmul}.
Each polynomial is used twice in polynomial multiplications.
Nevertheless, we only need one NTT operation per polynomial as they can stay in the NTT domain during the polynomial multiplication and addition.
At the end, we can convert back from the NTT domain using the iNTT operation.
The entire ciphertext multiplication in \cofhee{} utilizes 4 NTT operations, 4 Hadamard products, 1 pointwise addition, and 3 iNTT operations.
% \clearpage
\begin{figure}[ht!]
\begin{minipage}[t][][t]{0.46\textwidth}
\centering
\input{algo/ntt}
\vspace{-0.5cm}
\input{algo/polymul}
\end{minipage}
\hspace{0.2cm}
\begin{minipage}[t][][t]{0.46\textwidth}
\centering
\input{algo/ctmul}
\end{minipage}
\end{figure}
\begin{algorithm}[H]
\small
\caption{Number Theory Transform}
{\bf Input:} polynomial ($\vec{x}$)\\
{\bf Input:} roots of unity ($\vec{\omega}$)\\
{\bf Input:} modulus ($q$)\\
{\bf Output:} polynomial ($\vec{x}$)
\vspace{0.3em}
\begin{algorithmic}[1]
\State{idx = 0}
\For{$i\gets n/2$; $i >= 2$;$i = i >> 1 $}
\For{$j\gets 0$; $j < n/2$;$j = j + i $}
\State{twiddle = $\vec{\omega}$[idx++]}
\For{$k\gets j$; $k < j + i$; $k = k + 1 $}
\State{m = twiddle\ * $\vec{x}$[k+i]}
\State{m = reduce$_q$(m)}
\State{$\vec{x}$[k+i] = reduce$_q$($\vec{x}$[k] - m)}
\State{$\vec{x}$[k] = reduce$_q$($\vec{x}$[k] + m)}
\EndFor
\EndFor
\EndFor
\end{algorithmic}
\label{alg:ntt}
\end{algorithm}
\begin{algorithm}[H]
\small
\caption{Polynomial Multiplication}
\bf{Input:} $A(x)$, $B(x) \in \mathbb{Z}_q[x]/x^n+1$\\
\bf{Input:} $n^{th} \ \text{roots of unity} \ \vec{\omega} \in \mathbb{Z}_q$ \\
\bf{Input:} $2n^{th} \ \text{roots of unity} \ \vec{\psi} \in \mathbb{Z}_q$ \\
\bf{Output:} $Y(x) \in \mathbb{Z}_q[x]/x^n+1$ \\
$\centerdot \quad Y(x) = A(x) \times B(x)$
\vspace{0.3em}
\begin{algorithmic}[1]
\State{ $A'(x) = \text{NTT}((A(x) \ \bullet \ \vec{\psi}),\ \vec{\omega})$ }
\State{ $B'(x) = \text{NTT}((B(x) \ \bullet \ \vec{\psi}),\ \vec{\omega})$ }
\State{ $Y'(x) = A'(x) \ \bullet \ B'(x)$ }
\State{ $Y(x) = \text{iNTT}((Y'(x) \ \bullet \ {\vec{\psi}}^{-1}),\ \vec{\omega}$) } \label{lst:line:intt}
\State {\bf return} $Y(x)$
\end{algorithmic}
\label{alg:polynomial_mult}
\end{algorithm}
% \begin{algorithm}[H]
% \small
% \caption{Polynomial Multiplication}
% \bf{Input:} $A(x)$, $B(x) \in \mathbb{Z}_q[x]/x^n+1$\\
% \bf{Input:} $n^{th} \ \text{roots of unity} \ \vec{\omega} \in \mathbb{Z}_q$ \\
% \bf{Input:} $2n^{th} \ \text{roots of unity} \ \vec{\psi} \in \mathbb{Z}_q$ \\
% \bf{Output:} $Y(x) = A(x) \times B(x); Y(x) \in \mathbb{Z}_q[x]/x^n+1$
% \vspace{0.3em}
% \begin{algorithmic}[1]
% \State{ $A'(x) = \text{NTT}((A(x) \ \bullet \ \psi^0,\ \psi^1,\ \psi^2,...,\ \psi^{n-1}),\ \vec{\omega})$ }
% \State{ $B'(x) = \text{NTT}((B(x) \ \bullet \ \psi^0,\ \psi^1,\ \psi^2,...,\ \psi^{n-1}),\ \vec{\omega})$ }
% \State{ $Y'(x) = A'(x) \ \bullet \ B'(x)$ }
% \State{ $Y(x) = \text{iNTT}((Y'(x) \ \bullet \ (\psi^0,\ \psi^{-1},\ \psi^{-2},...,\ \psi^{-(n-1)})),\ \vec{\omega}$) } \label{lst:line:intt}
% \State {\bf return} $Y(x)$
% \end{algorithmic}
% \label{alg:polynomial_mult}
% \end{algorithm}
% \begin{algorithm}[t]
% \small
% \caption{Ciphertext Multiplication}\label{alg6}
% \bf{Input:} $C(x) = \{ C0(x), C1(x)\}$, $D(x) = \{ D0(x), D1(x)\}$; $C0(x)$, $C1(x)$, $D0(x)$, and $D1(x) \in \mathbb{Z}_q[x]/x^n+1$, $q$ = modulus\\
% \bf{Output:} $Y(x) = \{ Y0(x), Y1(x), Y2(x)\}$ %= \{ C0(x) \bullet D0(x)$, $C0(x) \bullet D1(x)$ + $C1(x) \bullet D0(x)$, $C1(x) \bullet D1(x)\}$
% \vspace{0.3em}
% \begin{algorithmic}[1]
% \State{ $Y0(x)$ = $C0(x) \bullet D0(x)$}
% \State{ $Y1(x)$ = $C0(x) \bullet D1(x)$ + $C1(x) \bullet D0(x)$}
% \State{ $Y2(x)$ = $C1(x) \bullet D1(x)$}
% \State {\bf return} $Y(x)$
% \end{algorithmic}
% \label{alg:ctmul}
% \end{algorithm}
\begin{algorithm}[H]
\small
\caption{Ciphertext Multiplication}
\bf{Input:} $(A_0(x), A_1(x)) \in \mathbb{Z}_q[x]/x^n+1$ \\
\bf{Input:} $(B_0(x), B_1(x)) \in \mathbb{Z}_q[x]/x^n+1$ \\
\bf{Input:} $n^{th} \ \text{roots of unity} \ \vec{\omega} \in \mathbb{Z}_q$ \\
\bf{Input:} $2n^{th} \ \text{roots of unity} \ \vec{\psi} \in \mathbb{Z}_q$ \\
\bf{Output:} $Y_i(x) \in \mathbb{Z}_q[x]/x^n+1 \ \forall i \in [0, 2]$ \\
$\centerdot \quad Y_0(x) = A_0(x) \times B_0(x)$ \\
$\centerdot \quad Y_1(x) = A_0(x) \times B_1(x) + A_1(x) \times B_0(x)$ \\
$\centerdot \quad Y_2(x) = A_1(x) \times B_1(x)$
\vspace{0.3em}
\begin{algorithmic}[1]
\State{ $B_0'(x) = \text{NTT}((B_0(x) \ \bullet \ \vec{\psi}),\ \vec{\omega})$ }
\State{ $A_0'(x) = \text{NTT}((A_0(x) \ \bullet \ \vec{\psi}),\ \vec{\omega})$ }
\State{ $Y_0'(x) = A_0'(x) \ \bullet \ B_0'(x)$ }
\State{ $Y_0(x) = \text{iNTT}((Y_0'(x) \ \bullet \ {\vec{\psi}}^{-1}),\ \vec{\omega}$) }
\State{ $B_1'(x) = \text{NTT}((B_1(x) \ \bullet \ \vec{\psi}),\ \vec{\omega})$ }
\State{ $Y_{01}'(x) = A_0'(x) \ \bullet \ B_1'(x)$ }
\State{ $A_1'(x) = \text{NTT}((A_1(x) \ \bullet \ \vec{\psi}),\ \vec{\omega})$ }
\State{ $Y_2'(x) = A_1'(x) \ \bullet \ B_1'(x)$ }
\State{ $Y_2(x) = \text{iNTT}((Y_2'(x) \ \bullet \ {\vec{\psi}}^{-1}),\ \vec{\omega}$) }
\State{ $Y_{10}'(x) = A_1'(x) \ \bullet \ B_0'(x)$ }
\State{ $Y_1'(x) = Y_{01}'(x) + Y_{10}'(x)$ }
\State{ $Y_1(x) = \text{iNTT}((Y_1'(x) \ \bullet \ {\vec{\psi}}^{-1}),\ \vec{\omega}$) }
\State {\bf return} $(Y_0(x),Y_1(x),Y_2(x))$
\end{algorithmic}
\label{alg:ctmul}
\end{algorithm}
% # memories used per line (twiddle factor not included)
% 2->1
% 3->2
% 3
% 4->3
% 5->4
% 5->4
% 6->5
% 6->5
% 6->5
% 6->4
% 5->3
% 4->3
\vspace{-0.2cm}
\section{Physical Design}
\label{sec:backend}
%This section covers \cofhee{}'s physical design aspects.
\input{tables/pnr}
\input{figs/backend}
\begin{figure*}[t]
\begin{minipage}[t][][t]{0.33\textwidth}
\input{tables/performance}
\end{minipage}
\hspace{0.5cm}
\begin{minipage}[t][][t]{0.33\textwidth}
% \input{tables/layout}
\input{tables/eda_tools}
\end{minipage}
\hspace{0cm}
\begin{minipage}[t][][t]{0.33\textwidth}
\input{tables/redundant_vias}
\end{minipage}
\vspace{-0.5cm}
\end{figure*}
This section covers \cofhee{}'s physical design aspects.
Table \ref{tab:stages_tools} summarizes the stages and EDA tools utilized. \cofhee{}'s total die area, including the seal ring, is $15 mm^2$. We also note that we have implemented the use of inline pads placed on all four sides of the chip.
% \cofhee{}'s total die area including the seal ring is $15 mm^2$.
% We use inline pads placed on all four sides of the chip.
Table \ref{tab:design_features} provides information about the signal and power/ground pads, obtained from Synopsys Design Compiler. Table \ref{tab:layout} shows the physical parameters after place and route multiple iterations.
\begin{figure}[t]
% \hspace{0.2cm}
\begin{minipage}[t][][t]{0.5\textwidth}
% \input{tables/layout}
\vspace{-0.3cm}
\input{tables/layout}
\end{minipage}
% \begin{minipage}[t][][t]{0.48\textwidth}
% \input{tables/design_features}
% \vspace{-0.3cm}
% \input{tables/redundant_vias}
% \end{minipage}
\end{figure}
\subsection{Floor Planning}
The die dimensions are $3660 \um \times 3842 \um$.
Fig. \ref{fig:floorplan} shows the design's floorplan, which focused on utilizing the maximum available silicon area. The overlapping regions within the corner pads are empty, thus they do not cause any DRC/LVS concerns. At the same time, they maximize the available area for logic gates or memories placement. The Phase-Locked Loop (PLL) is placed at the upper right corner of the chip. IO pads dedicated for the PLL are also placed in the same corner.
%This includes AVDD/AVSS pads which are the PLL power and ground pads.
%There are four bias voltage and four bias current pads that are used to bias the PLL to operate in the designed operation range.
Overall, there are 68 memory instances, out of which 48 (16x2096) are dual-port, and 16 (32x8192) plus 4 (32x4096) are single-port.
There are 47 digital IO pads including power pads.
%We use two special pads to generate Power On Control (POC), an important signal to stabilize IOs.
%In addition
There are two pads for VDD (core power supply), VSS (core ground), and IO power/ground (DVDD/DVSS).
Furthermore, in this stage we place edge cells at the boundaries, and well tap cells throughout the layout in a staggered fashion (according to the foundry specifications) to avoid latchup violations.
\subsection{Power Planning}
In this section we discuss our efforts regarding power planning. Our power network consists of four pairs of VDD/VSS rings around the core region.
Out of the eight metal layers, the top two (BA and BB)
are used for rings.
% , which are captured by \hlorange{Fig.} \ref{fig:powerring}.
Furthermore, all the power and ground pads are connected to the aforementioned rings through multiple connections, as shown in Fig. \ref{fig:powerpad}.
Multiple power and ground straps in layers BA, BB, M5, and M4 run over the entire core region at different pitches of $30 \um$ (BA/BB), and $50 \um$ (M4/M5).
M1 rails are connected directly to M4 straps through stacked vias.
Fig. \ref{fig:powernetwork} provides a high-level illustration of our power/ground network, which includes power rings, straps, standard cell power rails, and connections to pad and memory pins.
% \hlorange{Fig.} \ref{fig:powerstraps} shows how the straps
The power straps are connected to each other in the power network until M4, which delivers power to the standard cell rails through stacked vias.
The connection, in Fig. \ref{fig:powerconnection}, is done, so that power straps in M2 are avoided, else there can be standard cell pin access issues.
%A smaller pitch makes the power structure more robust but reduces signal routing resources, while a wider pitch weakens the power structure but provides more resources for signal routing.
Optimal pitches were selected after multiple iterations of analyzing the routing congestion, IR drop and effective resistance.
As it is a memory dominant design, delivering power in all the channels between the memories was another challenge we faced.
%One of the challenges in power network synthesis (PNS) is to ensure delivering power in all channels between the memories.
The flow was modified to ensure that every such channel is delivered power and ground sufficiently, and that the memory power ground pins run on top of them, on layer M4.
The connections to the memories power/ground pins are made by dropping vias from M5 straps which run over them.
\subsection{Place and Route}
The design is then taken through place and route.
Placement optimization is done in one slow and one typical corner.
%Special variables are enabled in order to prevent cell clustering and congestion hot spots.
No-buffering blockages are added in the narrow channels between memories to
prevent the placement of logic cells in the channels, as placing them in the wrong location can result in either timing, and congestion issues, or affect the clock tree synthesis.
%A few iterations are performed between placement, memory, and IO pad placement to acquire the final floorplan.
Fig. \ref{fig:placement} shows the module placement.
Post placement, the clock tree synthesis is done using standard rules like double width and double spacing for the clock nets. In addition, it is verified that clock buffers and inverters used are of optimal drive strength, so that they don't contribute much to IR drop.
%Clock tree synthesis follows placement and optimization.
%All clock trunk nets are routed with double width and spacing.
%We have restricted the usage of clock buffers and inverters to avoid very low and high driving strength buffers.
%Very low driving strength buffers are more susceptible to small changes in load and very high driving strengths are power hungry and may affect dynamic IR drop.
Fig. \ref{fig:clocknetwork} shows clock network routing, followed by signal routing and post-route optimization.
Redundant via insertion is performed in order to improve yield, and decap and standard cell fillers are inserted before streaming out GDS from the IC compiler.
% \input{tables/clock_tree_synthesis} % Merged with table tables/design_features.tex
Table \ref{tab:design_features} provides the details of clock tree QOR for the main clock. The clock tree was built in the slow process corner for around $18k$ sinks and achieved skew of $240 ps$ with $2ns$ latency.
In Table \ref{tab:redundant_vias}, we present the percentage of redundant vias for various via layers.
We were able to achieve more than 98\% conversion of single to multi-cut vias for the lower via layers V1, V2, V3, V4, yet a lower percentage was achieved for higher layers.
In Table \ref{tab:pnr}, we present design statistics over various stages in the place \& route.
We remark that the standard cell count increases as the design moves from initial to final routing stages, primarily due to buffers/inverters inserted in the design to fix design rule violations, clock tree synthesis, and timing issues.
Our design started with 100\% HVT cells and ended up with 13.4\%, as HVT cells were swapped with RVT and LVT cells to address timing and DRV fixes (Table \ref{tab:pnr}).
% \hlcyan{The PLL needs four bias voltages and four bias currents in addition to the PG pads AVDD and AVSS.
% Connecting the PLL to these pads is also performed at this stage.
% An abstract view of the PLL is generated from the PLL layout and annotated in the top-level chip for this purpose.}
\vspace{-0.2cm}
\subsection{Sign-Off Analysis}
The design GDS obtained from the IC compiler is merged with the standard cell, memory, and IO pad GDS.
The seal ring is added on top, followed by the metal fill insertion procedure, %which is done in Mentor-Calibre,
and the corresponding GDS is generated, which is then merged with the design GDS to obtain the final GDS.
The sign-off DRC/LVS is then performed on this GDS using Cadence PVS.
The design parasitic extraction is done using STARRC-XT and the resulting SPEF along with the design netlist is fed to Synopsys Primetime-SI for static timing analysis.
The final DRC/LVS and timing clean GDS is then sent for fabrication.
%to MOSIS for further evaluation before releasing it to GlobalFoundries$
\subsection{PLL design}
In this section we discuss our efforts regarding the design of the PLL. Silicon area, power consumption, range of operation, and timing uncertainty are among the metrics that dictate the choices of a PLL architecture as well as implementation strategies.
For a given jitter performance, analog implementations require a loop filter with a big capacitor making the overall PLL size large.
On the other hand, a digital PLL provides an alternative with significantly smaller silicon area.
A digital implementation results in a low power implementation.
In addition, a wide range of operation is essential to run the chip at different frequencies.
This enables reusing the PLL in different designs, avoiding PLL redesign.
In this work, a compact, low power, and wide tuning range All Digital PLL (ADPLL) has been implemented.
The block diagram of the proposed ADPLL is shown in Fig. \ref{fig:ADPLL_diagram}, while the layout is presented in Fig. \ref{fig:ADPLL_layout}.
It is a dual-loop architecture with dedicated frequency and phase-locking loops.
The Frequency-Locking Loop (FLL) is a feedback loop which forces the frequency difference between the input signal and the oscillator output down to the capture range of the phase-locking loop.
Similarly, the Phase-Locking Loop (PLL) forces the phase error between input signal and the oscillator output to zero.
\begin{figure}[t!]
\subfloat[Block diagram \label{fig:ADPLL_diagram}]{%
\includegraphics[width=0.25\textwidth, height = 0.2\textwidth ]{figs/PLL_CoFHEE.jpg}}
\hspace{0.5mm}
\subfloat[Layout in GF 55nm \label{fig:ADPLL_layout}]{%
\includegraphics[width=0.23\textwidth, height = 0.2\textwidth ]{figs/PLL_layout.PNG}}\\
\\
\subfloat[Power Straps \label{fig:powerstraps}]{%
\includegraphics[width=0.25\textwidth, height = 0.2\textwidth ]{figs/PNS_straps_example_zoomed_in.PNG}}
\hspace{0.5mm}
\subfloat[Power Ring\label{fig:powerring}]{%
\includegraphics[width=0.23\textwidth, height = 0.2\textwidth ]{figs/Zoomed_top_left_corner_with_pns_ring.PNG}}\\
\caption{Proposed ADPLL design and additional Backend Results}
\vspace{-0.2cm}
\vspace{-0.10in}
\end{figure}
% \begin{figure}
% \centering
% % \begin{subfigure}[t]{}
% \raisebox{-\height}{\includegraphics[width=0.4\textwidth]{figs/PLL_CoFHEE.jpg}}
% \caption{Block diagram.}
% \label{fig:ADPLL_diagram}
% % \end{subfigure}
% % \hspace{0.2cm}
% % \begin{subfigure}[t]{0.43\textwidth}
% \raisebox{-\height}{\includegraphics[width=0.4\textwidth]{figs/PLL_layout.PNG}}
% \caption{Layout implemented in GF 55nm.}
% \label{fig:ADPLL_layout}
% % \end{subfigure}
% \caption{Proposed ADPLL design.}
% \end{figure}
% \begin{figure}[t!]
% \vspace{-0.8cm}
% \subfloat[Time for all towers \label{fig:times}]{%
% \includegraphics[width=0.24\textwidth, height = 0.24\textwidth ]{figs/cofhee_times.png}}
% \hspace{0.5mm}
% \subfloat[Power \label{fig:power}]{%
% \includegraphics[width=0.24\textwidth, height = 0.24\textwidth ]{figs/cofhee_power.png}}
% \caption{Comparison to CPU execution}
% \vspace{-0.3cm}
% \label{fig:results}
% \vspace{-0.3cm}
% \end{figure}
The capture range of the phase detector (PD) is a few percent of the reference clock frequency which is much smaller than the required tuning range of the ADPLL.
Consequently, a frequency-locking mechanism is required to pull the frequency of the local oscillator close to the operating frequency such that the PD with its narrow pull-in range can assume responsibility.
The choice of the frequency detector architecture used is based on the requirement that it should be easily interfaced with digital loop filters and allow a wide range of operation.
A digitized Phase and Frequency Detector (PFD) with a Successive Approximation Register (SAR) algorithm \cite{Rossi1996NonredundantSA}\cite{7452270}\cite{1084494} is employed to generate the appropriate digital control word to switch the required current value.
The phase-locking loop consists of a modified Alexander (Bang-Bang) phase detector \cite{Alexander1975ClockRF}\cite{Lee2004AnalysisAM}, and an all-digital loop filter in order to detect the phase error and generate an appropriate control signal.
The main requirements on the choice of the phase detector architecture originate from the need to use an all-digital loop filter, as it is compact, can easily be integrated with ASIC designs, and ported between technology nodes.
For these reasons, the phase detector should generate outputs which enable a digital implementation.
In this design, a modified version of the widely used Bang-Bang phase detector (BBPD) is proposed.
BBPD, also known as the early-late detection method, utilizes three data samples taken by three consecutive clock edges to determine whether a data transition is present or not, and if the clock leads or lags the data.
In the absence of data transitions, all three samples are equal, and no action is taken.
The loop filters in the FLL and PLL produce respectively digital control values from the instantaneous phase and frequency error signals generated by the frequency and phase detectors.
The outputs of the loop filters are used to allow the appropriate amount of supply current to the oscillator in order to correct for possible frequency or phase errors.
Since the oscillator frequency is controlled by current switching, segmented decoding is employed to avoid potential discontinuities and glitches.
This is achieved by implementing a combination of binary and unary weighted current sources.
To avoid any conflict between the frequency and phase correcting loops, a digital lock detector is used.
We implement the ADPLL in GF 55nm CMOS technology.
It occupies an active area of $0.05mm^2$ and consumes $350 \uW$ from a supply of $1.1V$.
\subsection{Post-Silicon Validation}
\cofhee{} is packaged in a 48-pin QFN, and connects to a breadboard via QFN to the DIP adapter for chip bring-up and testing.
For interfacing with a host computer, we use a UMFT230XA development board that features an FTDI chip for USB-to-UART conversion.
The UMFT230XA board provides a 3.3V supply for \cofhee{}'s IO pad, as well as a clock output which is used as reference clock.
% \hlcyan{for the PLL and also can be used as the clock source of the chip when PLL is bypassed.}
Moreover, the required 1.2V supply was generated using a DC-DC adjustable step-down module that converts the 5V source of the UMFT230XA board.
In addition, a USB-to-UART breakout is used to receive the computation complete signal from \cofhee{}. For the power measurement, we use a Tektronix MSO 5204B oscilloscope with current probe. Our post-silicon validation setup is shown in Fig. \ref{fig:setup}, and confirms that the fabricated chip is fully functional.
\begin{figure}[!t]
\centering
\includegraphics[scale=0.09, trim={0 1in 0 0.6in},clip, width=0.4\textwidth,]{figs/setup.jpg}
\caption{\cofhee{}'s validation and experimental setup}
% \vspace{-0.5cm}
\label{fig:setup}
\end{figure}
\begin{table}[t]
\centering
\vspace{0.45cm}
\begin{tabular}{||l|r|r|r|r||}
\hline
{\bf Parameter} & {\bf Initial} & {\bf Place} & {\bf CTS} & {\bf Route} \\ \hline \hline
Standard cells & 225,797 & 376,853 & 378,957 & 379,921 \\ \hline
Sequential cells & 18,686 & 18,686 & 18,686 & 18,686 \\ \hline
Buffer/Inverter cells & 22,561 & 89,072 & 91,372 & 92,379 \\ \hline
Std. Cell Utilization & 45\% & 54\% & 56.5\% & 59\% \\ \hline
Signal nets & 257,856 & 398,340 & 401,407 & 401,510 \\ \hline
HVT cells & 100\% & 13.75\% & 13.5\% & 13.4\% \\ \hline
RVT cells & 0\% & 17\% & 12.1\% & 12\% \\ \hline
LVT cells & 0\% & 69.25\% & 74.4\% & 74.6\% \\ \hline
\end{tabular}
\caption{Design statistics through PnR}\label{tab:pnr}
\vspace{-0.5cm}
\end{table}
% \begin{table}[t]
% \centering
% \caption{Design statistics through PnR.}\label{tab:pnr}
% \begin{tabular}{||l|c|c|c|c||}
% \hline
% \multirow{2}{*}{\parbox{1cm}{{\bf Parameter}}} &
% \multirow{2}{*}{\parbox{0.8cm}{\centering{\bf Initial}}} &
% \multirow{2}{*}{\parbox{0.6cm}{\centering{\bf Place}}} &
% \multirow{2}{*}{\parbox{0.7cm}{\centering{\bf CTS}}} &
% \multirow{2}{*}{\parbox{0.8cm}{\centering {\bf Route}}} \\
% {} & {} & {} & {} & {} \\
% %\bf{Parameter} & \bf{Initial} & \bf{Place} & \bf{CTS} & \bf{Route} \\
% \hline \hline
% \# of Standard cells & 225797 & 376853 & 378957 & 379921 \\
% \hline
% \# of Sequential cells & 18686 & 18686 &18686 & 18686 \\
% \hline
% \# of Buffer/Inverter cells & 22,561 & 89,072 & 91,372 & 92,379 \\
% \hline
% Standard Cell Utilization & 45 \% & 54 \% & 56.5 \% & 59 \% \\
% \hline
% \# of Signal nets & 257856 & 398340 & 401407 & 401510 \\
% \hline
% HVT cells & 100 \% &13.75 \% & 13.5 \% & 13.4 \% \\
% \hline
% RVT cells & 0 \% & 17 \% & 12.1 \% & 12 \%\\
% \hline
% LVT cells & 0\% & 69.25 \% & 74.4 \% & 74.6 \%\\
% \hline
% \end{tabular}
% \end{table}
% 3 x 3
\begin{figure}[t!]
\subfloat[Floorplan \label{fig:floorplan}]{%
\includegraphics[width=0.15\textwidth, height = 0.15\textwidth ]{figs/Chip_floorplan.PNG}}
\hspace{0.5mm}
\subfloat[Power Network \label{fig:powernetwork}]{%
\includegraphics[width=0.15\textwidth, height = 0.15\textwidth ]{figs/Chip_power_network.PNG}}
\hspace{0.5mm}
\subfloat[Clock Network\label{fig:clocknetwork}]{%
\includegraphics[width=0.15\textwidth, height = 0.15\textwidth ]{figs/clock_network.PNG}}\\
\\
\subfloat[Power Pad \label{fig:powerpad}]{%
\includegraphics[width=0.15\textwidth, height = 0.15\textwidth ]{figs/Power_pads_connection_to_chip_power_ring.PNG}}
\hspace{0.5mm}
\subfloat[Power Connection\label{fig:powerconnection}]{%
\includegraphics[width=0.15\textwidth, height = 0.15\textwidth ]{figs/M4_strap_connection_to_M1_rail.PNG}}
\hspace{0.5mm}
\subfloat[ Module placement\label{fig:placement}]{%
\includegraphics[width=0.15\textwidth, height = 0.15\textwidth ]{figs/Module_placement.PNG}}\\
\caption{Backend Results}
\vspace{-0.2cm}
\vspace{-0.10in}
\end{figure}
% \begin{figure}[t!]
% \centering
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Chip floorplan.PNG}}
% \caption{Floorplan.}
% \label{fig:floorplan}
% \end{subfigure}
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Chip power network.PNG}}
% \caption{Power network.}
% \label{fig:powernetwork}
% \end{subfigure}
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/clock network.PNG}}
% \caption{Clock network.}
% \label{fig:clocknetwork}
% \end{subfigure}
% \centering
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Power pads connection to chip power ring.PNG}}
% \caption{Power pad connections.}
% \label{fig:powerpad}
% \end{subfigure}
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/PNS straps example zoomed in.PNG}}
% \caption{Power straps (zoomed in).}
% \label{fig:powerstraps}
% \end{subfigure}
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Zoomed top left corner with pns ring.PNG}}
% \caption{Power ring (zoomed in).}
% \label{fig:powerring}
% \end{subfigure}
% \centering
% \begin{subfigure}[t]{0.32\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/M4 strap connection to M1 rail.PNG}}
% \caption{Power connection to standard cell rails from M4.}
% \label{fig:powerconnection}
% \end{subfigure}
% \begin{subfigure}[t]{0.45\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Module placement.PNG}}
% \caption{Module placement. AHB (peach), Cortex M0 (green), DMA (yellow), MDMC (purple), Multiplier pool (red), and SPIM (blue).}
% \label{fig:placement}
% \end{subfigure}
% \caption{Backend results.}
% \end{figure}
% 8 x 2
% \begin{figure}
% \centering
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Chip floorplan.PNG}}
% \caption{Floorplan.}
% \label{fig:floorplan}
% \end{subfigure}
% \hspace{1.3cm}
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Chip power network.PNG}}
% \caption{Power network.}
% \label{fig:powernetwork}
% \end{subfigure}
% \centering
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Power pads connection to chip power ring.PNG}}
% \caption{Power pad connections.}
% \label{fig:powerpad}
% \end{subfigure}
% \hspace{1.3cm}
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/PNS straps example zoomed in.PNG}}
% \caption{Power straps (zoomed in).}
% \label{fig:powerstraps}
% \end{subfigure}
% \centering
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/M4 strap connection to M1 rail.PNG}}
% \caption{Power connection to standard cell rails from M4.}
% \label{fig:powerconnection}
% \end{subfigure}
% \hspace{1.3cm}
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/clock network.PNG}}
% \caption{Clock network.}
% \label{fig:clocknetwork}
% \end{subfigure}
% \centering
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Zoomed top left corner with pns ring.PNG}}
% \caption{Power ring (zoomed in).}
% \label{fig:powerring}
% \end{subfigure}
% \hspace{1.3cm}
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/Module placement.PNG}}
% \caption{Module placement. AHB (peach), Cortex M0 (green), DMA (yellow), MDMC (purple), Multiplier pool (red), and SPIM (blue).}
% \label{fig:placement}
% \end{subfigure}
% \caption{Backend results.}
% \end{figure}
\begin{table}[H]
\centering
\begin{tabular}{||c|r|r|c|c||}
\hline
\multirow{2}{*}{Algorithm} & \multicolumn{2}{c|}{Latency} & \multicolumn{2}{c||}{Power {\footnotesize ($mW$)}} \\ %\cline{4-5}
{} & \multicolumn{1}{c}{(cc)} & \multicolumn{1}{c|}{($\mu$s)} & \multicolumn{1}{c}{avg.} & \multicolumn{1}{c||}{peak} \\ \hline \hline
\multicolumn{5}{||c||}{$n = 2^{12}$} \\ \hline
PolyMul & 83,777 & 335.1 & 22.9 & 30.4 \\ %\hline
NTT & 24,841 & 99.4 & 24.5 & 30.4 \\ %\hline
iNTT & 29,468 & 117.9 & 19.9 & 27.2 \\ \hline \hline
\multicolumn{5}{||c||}{$n = 2^{13}$} \\ \hline
PolyMul & 179,045 & 716.2 & 21.2 & 29.7 \\ %\hline
NTT & 53,535 & 214.1 & 24.4 & 29.7 \\ %\hline
iNTT & 62,770 & 251.1 & 18.3 & 23.9 \\ \hline
\end{tabular}
\caption{\cofhee{} performance for $n=\{2^{12},2^{13}\}$}
\label{tab:sys_latency}
\end{table}
% \begin{table}[H]
% \centering
% \begin{tabular}{||c|c|r|r|r|r||}
% \hline
% \multirow{2}{*}{Algorithm} & \multirow{2}{*}{n} & \multicolumn{2}{c|}{Latency} & \multicolumn{2}{c|}{Power $(mW)$} \\ %\cline{4-5}
% {} & {} & \multicolumn{1}{c}{(cc)} & \multicolumn{1}{c|}{($\mu$s)} & \multicolumn{1}{c}{Avg.} & \multicolumn{1}{c|}{Peak} \\ \hline \hline
% PolyMul & $2^{13}$ &179045 &716.2 &21.2 &29.7 \\ %\hline
% PMODMUL & $2^{13}$ &9235 &36.9 &16.4 &23.7 \\ %\hline
% NTT & $2^{13}$ &53535 &214.1 &24.4 &29.7 \\ %\hline
% iNTT & $2^{13}$ &62770 &215.1 &18.3 &23.9 \\ \hline
% PolyMul & $2^{12}$ &83777 &335.1 &22.9 &30.4 \\ %\hline
% PMODMUL & $2^{12}$ &4627 &18.5 &17.4 &24.1 \\ %\hline
% NTT & $2^{12}$ &24841 &99.4 &24.5 &30.4 \\ %\hline
% iNTT & $2^{12}$ &29468 &117.9 &19.9 &27.2 \\ \hline
% \end{tabular}
% \caption{System-level performance for $n=\{2^{12}, 2^{13}\}$. Frequency: 250 MHz.}
% \label{tab:sys_latency}
% \end{table}
% \begin{table}[H]
% \centering
% \begin{tabular}{||c|c|c|r|r|r|r||}
% \hline
% \multirow{2}{*}{Algorithm} & \multirow{2}{*}{n} & Freq. & \multicolumn{2}{c|}{Latency} & \multicolumn{2}{c|}{Power $(mW)$} \\ %\cline{4-5}
% {} & {} & (MHz) & \multicolumn{1}{c}{(cc)} & \multicolumn{1}{c|}{($\mu$s)} & \multicolumn{1}{c}{Avg.} & \multicolumn{1}{c|}{Peak} \\ \hline \hline
% PolyMul & $2^{13}$ & 250 &179045 &716.18 &21.2 &29.7 \\ %\hline
% PMODMUL & $2^{13}$ & 250 &9235 &36.9 &16.4 &23.7 \\ %\hline
% NTT & $2^{13}$ & 250 &53535 &214.1 &24.4 &29.7 \\ %\hline
% iNTT & $2^{13}$ & 250 &62770 &215.1 &18.3 &23.9 \\ \hline
% PolyMul & $2^{12}$ & 250 &83777 &335.1 &22.9 &30.4 \\ %\hline
% PMODMUL & $2^{12}$ & 250 &4627 &18.5 &17.4 &24.1 \\ %\hline
% NTT & $2^{12}$ & 250 &24841 &99.4 &24.5 &30.4 \\ %\hline
% iNTT & $2^{12}$ & 250 &29468 &117.9 &19.9 &27.2 \\ \hline
% \end{tabular}
% \caption{System-level performance for $n=\{2^{12}, 2^{13}\}$. Frequency: 250 MHz.}
% \label{tab:sys_latency}
% \end{table}
\begin{table}[H]
\centering
\begin{tabular}{||l|c||}
\hline
\multicolumn{1}{||c|}{\textbf{Stage}} & \multicolumn{1}{c||}{\textbf{Tool}} \\ \hline \hline
\multirow{2}*{Place and Route} & Synopsys \\
{} & IC compiler \\ \hline
Interconnect & Synopsys \\
parasitic extraction & STAR-RCXT \\ \hline
Static timing & Synopsys \\
analysis & Prime-Time-SI \\ \hline
GDS merging and & Cadence \\
layout modification & Virtuoso \\ \hline
Physical & Cadence PVS \\
verification & System \\ \hline
\end{tabular}
\caption{Stages and EDA tools.}
\label{tab:stages_tools}
\end{table}
% \begin{table}[t!]
% \centering
% \begin{tabular}{||l|l||}
% \hline
% \multicolumn{1}{||c|}{\textbf{Stage}} & \multicolumn{1}{c||}{\textbf{Tool}} \\ \hline \hline
% Place and Route & Synopsys IC compiler \\ \hline
% Interconnect parasitic extraction & Synopsys STAR-RCXT \\ \hline
% Static timing analysis & Synopsys PrimeTime-SI \\ \hline
% GDS merging and layout modification & Cadence Virtuoso \\ \hline
% Physical Verification & Cadence Physical Verification System (PVS) \\ \hline
% \end{tabular}
% \caption{Stages and EDA tools.}
% \label{tab:stages_tools}
% \end{table}
\begin{table}[H]
\centering
\begin{tabular}{||c|c|c|c||}
\hline
\multirow{2}*{\bf Layer} & {\bf multi-} & {\bf total} & {\bf multi-} \\
{} & {\bf cut {\footnotesize(\#)}} & {\footnotesize(\#)} & {\bf cut {\footnotesize(\%)}} \\ \hline \hline
V1 & 21,659 & 21,945 & 98.70 \\ \hline
V2 & 21,732 & 21,844 & 99.49 \\ \hline
V3 & 21,991 & 22,035 & 99.80 \\ \hline
V4 & 26,391 & 26,455 & 99.76 \\ \hline
WT & 2,438 & 2,450 & 99.51 \\ \hline
WA & 1,390 & 1,393 & 99.78 \\ \hline
\end{tabular}
\caption{Redundant via statistics}\label{tab:redundant_vias}
\end{table}
% \begin{table}[t]
% \centering
% \caption{Redundant via statistics.}\label{tab:redundant_vias}
% \begin{tabular}{||c|c|c|c||}
% \hline
% \multirow{2}{*}{\parbox{1.2cm}{\centering{\bf Layer}}} &
% \multirow{2}{*}{\parbox{2.6cm}{\centering{\bf \# of multi-cut vias}}} &
% \multirow{2}{*}{\parbox{2.4cm}{\centering{\bf \# of total vias}}} &
% \multirow{2}{*}{\parbox{2.6cm}{\centering {\bf \% of multi-cut vias}}} \\
% {} & {} & {} & {} \\
% %\bf{Via layer} & \bf{\# of multi-cut vias} & \bf{\# of total vias} & \bf{\% of multi-cut vias} \\
% \hline \hline
% V1 & 21659 & 21945 & 98.70 \\
% \hline
% V2 & 21732 & 21844 & 99.49 \\
% \hline
% V3 & 21991 & 22035 & 99.8 \\
% \hline
% V4 & 26391 & 26455 & 99.76 \\
% \hline
% WT & 2438 & 2450 & 99.51 \\
% \hline
% WA & 1390 & 1393 & 99.78 \\
% \hline
% \end{tabular}
% \end{table}
\begin{table}[H]
\centering
\begin{tabular}{||l|c||}
\hline
{\bf Parameter} & {\bf Value} \\ \hline \hline
IU (Initial Utilization) & 45 \% \\ \hline
FU (Final Utilization) & 59 \% \\ \hline
MA (Macro Area) & 8,941,959 ${\um}^2$ \\ \hline
HIO (IO PAD Height) & 120 $\um$ \\ \hline
CIO (Core to IO spacing) & 10 $\um$ \\ \hline
A (Aspect ratio) & 1.05 \\ \hline
CA (std cell Area) & 1,963,585 ${\um}^2$ \\ \hline
CW (Core Width) & 3400 $\um$ \\ \hline
CH (Core Height) & 3582 $\um$ \\ \hline
DW (Die Width) & 3660 $\um$ \\ \hline
DH (Die Height) & 3842 $\um$ \\ \hline
\end{tabular}
\caption{Layout physical parameters}\label{tab:layout}
\end{table}
\section{Experimental Evaluation}
\label{sec:eval}
%In this section we investigate the performance of \cofhee{} by executing a power and latency evaluation, followed by a comparison against a software implementation.
\subsection{Power and latency}
We have measured the power consumption and latency for several operations supported by \cofhee{} using polynomial degrees of $n=\{2^{12}, 2^{13}\}$. The results of these measurements are summarized in Table \ref{tab:sys_latency}. Our measurements show that the NTT operation results in the highest peak power consumption, while the Hadamard and Inverse NTT (iNTT) operations consume less power. The iNTT operation includes a multiplication with a constant ($n^{-1}$) and a decimation in frequency operation, which results in a lower average power consumption compared to NTT, due to the lower power consumption of the constant multiplication. However, iNTT takes more cycles to execute than NTT. To conclude, \cofhee{} requires a power supply with a peak power rating of around $30mA$ and an average power of around $25mA$ to perform polynomial multiplication in a fraction of a millisecond.
% We measure the power consumption and latency for several operations supported by \cofhee{} using $n=\{2^{12}, 2^{13}\}$.
% Table \ref{tab:sys_latency} summarizes our findings.
% Peak power is observed during NTT, while Hadamard and iNTT consume less power.
% The iNTT operation involves multiplication with a constant ($n^{-1}$), and a decimation in frequency operation.
% Its average power is lower as the constant multiplication consumes less power, and therefore reduces the average.
% For the same reason, iNTT takes more cycles than NTT to execute.
% To summarize, \cofhee{} needs a power supply with peak power rating of around $30mA$ and an average power of around $25mA$ to run polynomial multiplication in a fraction of a millisecond.
\subsection{Comparison to CPU}
We have performed a comparative analysis of \cofhee{} against a software implementation in terms of both execution time and power consumption by executing a ciphertext multiplication without relinearization using one instance of \cofhee{}. For the software implementation, we have employed the Microsoft SEAL 3.7 library \cite{seal}, executing it on an AMD Ryzen 7 5800h (TSMC 7nm FinFET) processor operating at 3.8Ghz, paired with 16GB of RAM on Ubuntu 20.04 LTS, and we have gathered power measurements using \texttt{powertop}. We have established the polynomial degree as $n = \{2^{12}, 2^{13}\}$ and $\log q = \{109, 218\}$ bits, which provide a security level of 128 bits against classical computers. The results of this comparison are illustrated in Fig. \ref{fig:results}.
% We compare \cofhee{} against a software implementation in terms of execution time and power consumption by executing a ciphertext multiplication without relinearization.
% We use one instance of \cofhee{}.
% For the software implementation, we use the Microsoft SEAL 3.7 library \cite{seal} running on an AMD Ryzen 7 5800h (TSMC 7nm FinFET) at 3.8Ghz with 16GB of RAM on Ubuntu 20.04 LTS, and we collect the power measures using \texttt{powertop}.
% We set $n = \{2^{12}, 2^{13}\}$ and $\log q = \{109, 218\}$ bits, which provide a security level of 128 bits against classical computers.
% Fig. \ref{fig:results} presents the results.
For $(n, \log q) = (2^{12}, 109)$, SEAL breaks the 109-bit modulus into three smaller moduli of 36, 36, and 37 bits using RNS for faster computation in the native 64-bit architecture.
Each of these three towers must perform the ciphertext multiplication according to Eq. \ref{eq:ctmul}.
For the same operation, \cofhee{} natively supports 128 bits, therefore, it requires only one tower.
As shown in Fig. \ref{fig:times}, the software implementation takes $2.1ms$ to operate on the three towers, while \cofhee{} needs $0.84ms$ to finalize the ciphertext multiplication.
Regarding power (Fig. \ref{fig:power}), \cofhee{} is two orders of magnitude more efficient as it requires $22mW$ of power, while the software implementation uses $1.47W$.
\begin{figure}[t]
\vspace{-0.5cm}
\begin{minipage}[t][][t]{0.20\textwidth}
% \vspace{-0.7cm}
\input{tables/area}
\end{minipage}
\hspace{-0.08\textwidth}
\begin{minipage}[t][][t]{0.5\textwidth}
% \vspace{-0.7cm}
\input{tables/design_features}
\end{minipage}
\vspace{-0.5cm}
\end{figure}
% \input{tables/performance}
\begin{figure}[t!]
\vspace{-0.2cm}
\subfloat[Time for all towers \label{fig:times}]{%
\includegraphics[width=0.24\textwidth, height = 0.24\textwidth ]{figs/cofhee_times.png}}
\hspace{0.5mm}
\subfloat[Power \label{fig:power}]{%
\includegraphics[width=0.24\textwidth, height = 0.24\textwidth ]{figs/cofhee_power.png}}
\caption{Comparison to CPU execution}
\vspace{-0.3cm}
\label{fig:results}
\vspace{-0.3cm}
\end{figure}
% \begin{figure}[t]
% \begin{minipage}[t][][t]{0.42\textwidth}
% % \vspace{-0.7cm}
% \begin{figure}[H]
% \centering
% \includegraphics[scale=0.066]{figs/setup.jpg}
% \caption{\cofhee{}'s validation and experimental setup.}
% \label{fig:setup}
% \end{figure}
% \end{minipage}
% \hspace{0.02\textwidth}
% \begin{minipage}[t][][t]{0.52\textwidth}
% % \vspace{-0.7cm}
% \input{tables/performance}
% \end{minipage}
% \end{figure}
% \begin{figure}[h]
% \centering
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/cofhee_times.png}}
% \caption{Time for all towers.}
% \label{fig:times}
% \end{subfigure}
% \hspace{0.5cm}
% \begin{subfigure}[t]{0.4\textwidth}
% \raisebox{-\height}{\includegraphics[width=\textwidth]{figs/cofhee_power.png}}
% \caption{Power.}
% \label{fig:power}
% \end{subfigure}
% \caption{Comparison between \cofhee{} and a software implementation performing a ciphertext multiplication for $n=\{2^{12}, 2^{13}\}$ without relinearization.}
% \label{fig:results}
% \end{figure}
When $(n, \log q) = (2^{13}, 218)$, the software implementation uses five towers of around 44 bits ($43 + 43 + 44 + 44 + 44 = 218$) and \cofhee{} requires two towers ($109 + 109 = 218$).
For the five $<64$-bit towers, SEAL spends $8.5 ms$, while \cofhee{} takes $3.58ms$ to operate on the two $<128$-bit towers.
Power readings are $21.2mW$ for \cofhee{} and $2.32W$ for the software implementation.
While at its current instantiation \cofhee{} provides a modest speed-up compared to a modern CPU, it is much smaller in size and uses a much bigger technology node. Therefore, when fabricated in the same node and normalized for the area used, \cofhee{} will offer orders of magnitude improvement over CPU execution of FHE.
%\begin{table}[H]
% \centering
% \begin{tabular}{||l|r||}
% \hline
% \multirow{2}*{\bf Module} & \multicolumn{1}{c|}{\bf Area} \\
% {} & \multicolumn{1}{c|}{($mm^2$)} \\ \hline \hline
% 3 DP SRAMs & 5.3506 \\ \hline
% 4 SP SRAMs & 3.2036 \\ \hline
% PE & 0.6394 \\ \hline
% CM0 SRAM & 0.4062 \\ \hline
% AHB & 0.0747 \\ \hline
% GPCFG & 0.0534 \\ \hline
% ARM CM0 & 0.0354 \\ \hline
% MDMC & 0.0273 \\ \hline
% SPI & 0.0202 \\ \hline
% DMA & 0.0075 \\ \hline
% UART & 0.0065 \\ \hline
% GPIO & 0.0035 \\ \hline
% Others & 0.0063 \\ \hline \hline
% {\bf Total} & {\bf 9.8345} \\ \hline
% \end{tabular}
% %\vspace{1em}
% \captionsetup{justification=centering}
%
% \caption{Area estimations}
% \label{tab:area}
%\end{table}
\begin{table}[H]
\centering
\begin{tabular}{||l|r|r||}
\hline
\multirow{2}*{\bf Module} & \multicolumn{1}{c|}{\bf Area} & \multicolumn{1}{c||}{\bf Delay} \\
{} & \multicolumn{1}{c|}{($mm^2$)} & \multicolumn{1}{c||}{\bf $(ns)$} \\ \hline \hline
3 DP SRAMs & 5.3506 & 4.22 \\ \hline
4 SP SRAMs & 3.2036 & 4.19 \\ \hline
PE & 0.6394 & 5.65 \\ \hline
CM0 SRAM & 0.4062 & 6.13 \\ \hline
AHB & 0.0747 & 5.76 \\ \hline
GPCFG & 0.0534 & 7.03 \\ \hline
ARM CM0 & 0.0354 & 5.24 \\ \hline
MDMC & 0.0273 & 4.16 \\ \hline
SPI & 0.0202 & 7.74 \\ \hline
DMA & 0.0075 & 7.17 \\ \hline
UART & 0.0065 & 5.66 \\ \hline
GPIO & 0.0035 & 6.73 \\ \hline
Others & 0.0063 & - \\ \hline \hline
{\bf Total} & {\bf 9.8345} & - \\ \hline
\end{tabular}
\vspace{-1em}
\captionsetup{justification=centering}
\captionsetup{width=1.3\linewidth}
% \captionsetup{margin={-0.5cm,0cm}}
% \hspace{-10em}
\caption{Part estimations}
\label{tab:area}
\end{table}
% \begin{table}[H]
% \centering
% \begin{tabular}{||l|r||}
% \hline
% \multirow{2}*{\bf Module} & \multicolumn{1}{c||}{\bf Area} \\
% {} & \multicolumn{1}{c||}{($mm^2$)} \\ \hline \hline
% 3 Dual-port SRAMs & 5.3506 \\ \hline
% 4 Single-port SRAMs & 3.2036 \\ \hline
% Processing Element & 0.6394 \\ \hline
% ARM Cortex-M0 SRAM & 0.4062 \\ \hline
% AHB Interconnect & 0.0747 \\ \hline
% GPCFG & 0.0534 \\ \hline
% ARM Cortex-M0 & 0.0354 \\ \hline
% MDMC & 0.0273 \\ \hline
% SPI & 0.0202 \\ \hline
% DMA & 0.0075 \\ \hline
% UART & 0.0065 \\ \hline
% GPIO & 0.0035 \\ \hline
% Others & 0.0063 \\ \hline \hline
% {\bf Total} & {\bf 9.8345} \\ \hline
% \end{tabular}
% \caption{\cofhee{}'s chip area usage.}
% \label{tab:area}
% \end{table}
\begin{table}[H]
\centering
\begin{tabular}{||l|r||}
\hline
\textbf{Parameter} & \textbf{Value} \\ \hline \hline
Width & 3660 $\um$ \\ \hline
Height & 3842 $\um$ \\ \hline
Signal pads & 26 \\ \hline
PG pads & 11 \\ \hline
PLL bias pads & 8 \\ \hline
Memories & 68 \\ \hline
clock name & HCLK \\ \hline
CTS synth. corner & slow \\ \hline
Levels & 26 \\ \hline
Sinks & 18413 \\ \hline
Clock tree buffers & 464 \\ \hline
Global Skew & 240 $ps$ \\ \hline
Longest Ins. delay & 2.079 $ns$ \\ \hline
Shortest Ins. delay & 1.838 $ns$ \\ \hline
\end{tabular}
% \vspace{1em}
\caption{Design statistics}
% \caption{Design features and statistics.}
\label{tab:design_features}
\end{table}
% \vspace{-0.2cm}
\section{Related Work}
\label{sec:related_work}
\input{tables/f1_heax_comp}
% FHE acceleration has become a major research interest in the recent years.
% So far, many different approaches have been proposed, consequently leading to a state-of-the-art that is comprised of FHE acceleration solutions that are based on software, GPU, FPGA, and ASIC \cite{jung2020heaan}, \cite{HEAX}, \cite{DesignMert20}, \cite{hsu2020vlsi}, \cite{reagen2021cheetah}, \cite{cilardo2016securing}, \cite{cousins2016designing}, \cite{bradbury2021ntt}.
% Each method has demonstrated considerable advantages, however due to hardware differences, establishing an accurate comparison point between each technology is not feasible, therefore we focus on ASIC designs in our discussion. Table \ref{tab:related_designs} summarizes existing works on ASIC.
% In addition, there are also works that have focused on the acceleration of polynomial multiplications using ASIC designs, however they do not focus in FHE applications.
% While some may focus on different variants of homomorphic encryption such as PHE and SHE, many works focus purely on developing an acceleration method without taking into account the application the method can potentially be applied to \cite{asif18} \cite{boateng18} \cite{ding2017modular} \cite{asif16} \cite{kuang16} \cite{wang12} \cite{sakiyama11} \cite{knezevic10}.
%Since we propose an ASIC-based implementation that focuses on the acceleration of polynomial multiplications in FHE applications, we only consider papers of similar nature in our comparison.
% Table \ref{tab:related_designs} summarizes existing works on ASIC. One of the most recent works produced by Tan et al. presents a scalable and parallel design for NTT-based polynomial multiplication \cite{tan2021ultra}.
% The implementation is designed with a variable number of reconfigurable PEs, which allows its optimization according to the constraint parameters of the targeted hardware platform.
% The authors also manage to minimize the memory access conflicts and achieve a very high utilization rate by proposing addressing and scheduling schemes.
% In order to evaluate their proposal, they implement the design using $32$ $nm$ technology, and select a 32-bit prime $q$, and polynomial degrees of $n = 2^{10}$ and $n = 2^{12}$.
% Tan et al. also present another scalable architecture for polynomial multiplication \cite{tan2019efficient}.
% This work manages to achieve full utilization of the processing elements using a conflict-free memory management scheme, however the implementation is focused towards improving the performance of the bootstrapping algorithm.
% The proposed design is implemented using $32$ $nm$ technology.
FHE acceleration has become a major research interest in the recent years.
So far, many different approaches have been proposed, consequently leading to a state-of-the-art that is comprised of FHE acceleration solutions that are based on software, GPU, FPGA, and ASIC \cite{jung2020heaan}, \cite{HEAX}, \cite{DesignMert20}, \cite{hsu2020vlsi}, \cite{reagen2021cheetah}, \cite{cilardo2016securing}, \cite{cousins2016designing}, \cite{bradbury2021ntt}.
Each method has demonstrated considerable advantages, however due to hardware differences, establishing an accurate comparison point between each technology is not feasible, therefore we focus on ASIC designs in our discussion. Table \ref{tab:F1_HEAX_comp} summarizes existing works on ASIC and FPGA.
In addition, there are also works that have focused on the acceleration of polynomial multiplications using ASIC designs, however they do not focus in FHE applications.
While some may focus on different variants of homomorphic encryption such as PHE and SHE, many works focus purely on developing an acceleration method without taking into account the application the method can potentially be applied to \cite{asif18} \cite{boateng18} \cite{ding2017modular} \cite{asif16} \cite{kuang16} \cite{wang12} \cite{sakiyama11} \cite{knezevic10}.
The closest work to \cofhee{} is F1, a work proposed by Feldmann et al. that focuses on developing an accelerator for the execution of FHE programs \cite{feldmann2021f1}.
The solution can be described as a wide-vector processor with functional units specialized to FHE primitives like modular arithmetic, NTT, and structured permutations.
However, there is limited information about the backend process of the design, and whether the proposed architecture can be placed on real silicon without routing congestion, as well as power and clock issues. The API is not clear either, since the focus is more on architecture exploration and less on getting a realized chip.
When comparing \cofhee{} with F1, we need to consider the differences between the area, frequency, technology nodes, as well as the input size (bits) of each design. Consequently, to conduct a fair and accurate evaluation, we have normalized the performance in terms of the area and the scaling factor between the technology nodes. To obtain the scaling factor, we synthesized the Barrett modular multiplier using the GF14/12nm technology library, which is the same as that used for F1. The results indicate that the scaling factor reduces the area by $9.8\times$ and the critical path by $8\times$.
% to establish a fair evaluation, we normalize the performance in terms of the area and scaling factor between the technology nodes. In order to derive the scaling factor, the Barrett modular multiplier is synthesized using the GF14/12nm technology library (same as F1). Results indicate a scaling factor that reduces the area by $9.8\times$ and the critical path by $8\times$.
We compare the NTT operation of CoFHEE and F1 in terms of time per area with a polynomial degree of $16$k and a modulus size of $128$ bits.
% Ideally, to compare, one should consider the PE area along with other components like interconnects and memory. But comparing memory is a bit complicated as we are not sure about the memory budgeting only for NTT or cipher text multiplication in F1. The same applies to the interconnect as well; the area specific to the interconnect used interface between memory and register file is not mentioned in F1.
We compare the PE and RF area in F1 against the PE and MDMC area of CoFHEE. We exclude memory as F1 contains extra on-chip memory to support higher-level operations, such as key switching. Also, we consider that F1 has to do RNS to split 128-bit coefficients into 32-bit towers.
Upon normalization of the performance metrics, it was found that F1 executed $1.73\cdot{10}^{-4}$ NTT operations per $ns$ per $mm^2$, while \cofhee{} achieved $2.6 \cdot {10}^{-4}$, resulting in a speedup of $1.5\times$. This enhancement in performance is mainly attributed to the use of a pipelined Barrett multiplier, as opposed to an iterative Montgomery multiplier.
% After normalizing the performance figures, F1 performs $1.73\cdot{10}^{-4}$ NTT operations per $ns$ per $m m^2$, while \cofhee{} achieves $2.6 \cdot {10}^{-4}$, which implies a speedup of $1.5\times$. The speed-up is mainly attributed to the use of a pipelined Barrett multiplier, instead of an iterative Montgomery multiplier.
% A lot of FPGA-based works
% achieve better clock cycles
% but utilize tremendous amount of resources
% which translates to enormous areas
% we cannot establish a fair comparison as they are based in LUTs DSPs and FFs so a conversion to mm2 is hard
Besides ASIC, there is a number of works that propose FPGA-based solutions for FHE acceleration. HEAX \cite{riazi_heax} is a parallelizable architecture focusing on number-theoretic transform (NTT), and Roy~\cite{roy2019fpga} presents an FPGA-based co-processor for FHE computation using the BFV scheme. A comparison of the NTT performance against the related work is presented in Table \ref{tab:F1_HEAX_comp}, in terms of clock cycles. The performance per $m m^2$ efficiency metric cannot be accurately calculated, however, as we cannot map FPGA resources to silicon area. In terms of raw numbers, F1 reports a $1,733\times$ improvement over HEAX, so we expect \cofhee{} to also be significantly more efficient than HEAX.
% Other similar work is HEAX \cite{riazi_heax}, an FPGA-based FHE acceleration design. As it is FPGA based design and its area is not publicly available and there is no straight forward way to estimate the area from the FPGA resources used, we cannot do a fair comparison with \cofhee{}.
%We see that if we just consider the performance metric of 128-bit 4k NTT per $ns$, HEAX is ahead by x$3.6$ after considering the tecnology node frequency scaling factor of 4 as the CoFHEE frequency improves by more than x$4$ in 22 nm, So we use the same scaling factor for 20nm as well.
% \cofhee{} uses a 128-bit multiplier while Feldman's work uses 32-bit multipliers. Furthermore, the presented solution has two multipliers, two adders, and one NTT unit which provides faster performance compared to \cofhee{}.
% Our chip can break down to less towers of RNS, which leads to a more compact relinearization in terms of memory usage.
% It is also important to note that a considerable difference between the two works is their node technology. While our work is fabricated and silicon proven in $55$ $nm$, Feldman's design targets $14/12$ $nm$ technology which makes comparison across nodes difficult to accurately perform.
% Nevertheless, to the best of our knowledge, \cofhee{} is the only FHE co-processor to be proven in silicon.
% \input{tables/related_work}
% \input{tables/f1_heax_comp}
% \begin{figure*}[t]
% \begin{minipage}[t][][t]{0.80\textwidth}
% \input{tables/f1_heax_comp}
% \end{minipage}
% \end{figure*}
\begin{table*}[t]
\centering
\resizebox{2\columnwidth}{!}{%
\begin{tabular}{||c|c|c|c|c|c|c|c|c|c||}
\hline
\multirow{2}*{\bf Design} & {\bf Technology} & {\bf n} & {\bf log q*} & {\bf Area} & {\bf Power } & {\bf Freq.} & {\bf Clock } & {\bf Efficiency} & {\bf Silicon} \\
{} & & & ($bits$) & ($mm^2$ | LUT/FF/BRAM/DSP) & {\bf ($W$)} & ($Mhz$) & {\bf Cycles} & {\bf (Performance per $mm^2$)} & {\bf Proven} \\ \hline \hline
\cofhee{} & ASIC - GF 55$nm$ &$2^{14}$ & 128 & 12 & $2.3 \cdot 10^{-2}$ & 250 & 24576 & $2.60$ $\cdot$ $10^{-4}$ & \ding{108} \\ \hline
F1 \cite{feldmann2021f1} & ASIC - GF 14/12$nm$ & $2^{14}$ & 32 & 151.4 & $1.8 \cdot 10^2$ & 1000 & 16 & $1.73$ $\cdot$ $10^{-4}$ & \ding{109} \\ \hline
HEAX \cite{riazi_heax} & FPGA - Intel Arria10 GX 1150 & $2^{14}$ & 27 & $582148$ / $1554005$ / $3986$ / $2018$ & - & 300 & 1536 & $\dagger$ & N/A \\ \hline
Roy \cite{roy2019fpga} & Xilinx Zynq UltraScale+ ZCU102 & $2^{12}$ & 30 & $63522$ / $25622$ / $400$ / $200$ & - & 200 & 16425 & $\dagger$ & N/A \\ \hline
% Liu \cite{liu2019optimized} & 20 & $2^{13}$ & 27 & - & - & 333 & 33966 & - & - & \ding{109} \\ \hline
\end{tabular}
}
% \vspace{1em}
\caption{Comparative table and performance of the NTT operation against related work}
% \caption{Performance comparison of the NTT operation against related work}
\footnotesize{$\dagger$ No information is available to accurately map FPGA resources to silicon area }\\
%\vspace{-0.4cm}
\label{tab:F1_HEAX_comp}
% \vspace{-0.4cm}
\end{table*}
% $^1$ Efficiency is measured in NTT operations per $ns$ per $mm^2$ $\cdot$ $10^{-4}$,
\section{Discussion}
\label{sec:discussion}
\subsection{\cofhee{} scalability}
To support operations on higher polynomial degrees $n \ge 2^{14}$ with $II = 1$, \cofhee{} needs more area for memories, which increase linearly to the polynomial degree.
As the memory size increases, memory read latency increases, which leads to a minor reduction in clock frequency.
However, if the goal is to improve performance for a particular $n$, there are a few options:
First, we could perform more than one one butterfly operations per cycle.
For that, one has to duplicate the multiplier pool and add more memories to increase parallel accesses of operands.
For NTT with $II=1$, two dual-port memories and one multiplier pool are required.
Doubling this improves throughput by close to $2 \times$ as one can split the polynomial of degree $n$ into two smaller polynomials of degree $n/2$, perform NTT on these smaller polynomials in parallel, recombine them and perform the last stage of NTT of the original polynomial.
This effectively means that $\log (n) - 1$ stages operate with $II=0.5$, while the last stage has an $II$ of $1$. Furthermore, a second approach is to increase the number of memories and processing units available.
The former increases the number of polynomials on chip, enabling more complex scheduling methods which reduce communication costs with the host.
With more memories and processing units, parallel operations on non-dependent data is also possible.
Combined better scheduling and parallel processing, doubling \cofhee{}'s resources would more than double the throughput.
\cofhee{}'s AHB interface connects to scalable and flexible components.
It supports up to 16 controllers, memories, and peripherals and, with ARM Cortex M0, it has 28 bits for addressing internal memory.
Hence, it is possible to increase the total memory size from 1 MB (currently used) to 256 MB.
% Hence, We could increase the total memory size from current 1MB to 3GB.
For faster communication, UART or SPI can be replaced with protocols such as Peripheral Component Interconnect Express.
% The maximum performance of CoFHEE is II = 1 for any operation.
% This sets upper limit for NTT, element-wise multiplication, and other workloads.
% However, we can parallely perform these workloads on different operands if we increase number of MDMCs, number of PEs, and memories.
\vspace{-0.1in}
\subsection{Lessons Learned}
\vspace{-0.05in}
% Using parallel modular reduction algorithms like Barrett gives a way for pipelined architecture when compared to serial implementation like Montgomery or interleaved modular multipliers.
% In the latter case, one needs to duplicate the multiplier as many times as needed to achieve II of one.
Using the AHB bus to fetch operands and load them to the computation unit avoids complex MUX structures \cite{riazi_heax_2020} that come from non trivial addressing of operands and twiddle factors in every NTT stage.
During NTT, \cofhee{}'s address generation unit generates the relevant addresses for operands and twiddle factors at every cycle and issues a bus transaction.
Although the addition of dual-port memories improves the throughput significantly, their area is $2 \times$ the area of single-port memories of the same size.
Therefore, it is important to minimize the number of dual-port memories without affecting throughput.
In \cofhee{}, the number of dual-port memories is three.
Thus, two of them can be used for computation, while the third is used along with DMA to buffer the next polynomial for execution.
While separate twiddle factors have been used for NTT and iNTT in previous works \cite{riazi_heax_2020}, \cofhee{} uses the same twiddle factors for both operations by combining MDMC and DMA operations.
Thus, it reduces the required chip area for FHE acceleration.
% It is easier and efficient to perform NTT on two separate polynomial parallelly than incriminating 2$\times$ performance of a single NTT.
\vspace{-0.2cm}
\section{conclusion}
\label{sec:conc}
In this work, we presented a year-long effort to design, implement, and validate \cofhee{}, a co-processor for Fully Homomorphic Encryption targeting processing of low-level polynomial operations.
\cofhee{} is a $12mm^2$ design fabricated in $55nm$ technology node and it natively supports polynomial degrees of up to $2^{14}$, as well as coefficient sizes up to 128 bits.
% Our fabricated chip reaches a frequency of 250 Mhz \hlcyan{with our custom-built low-power digital PLL design.}
The architecture design is highly scalable and can be expanded to support higher polynomial degrees, more polynomials on chip, and more processing units given a larger design area. This paper presents all required steps for a fully functional silicon, from the RTL design to fabrication and validation. The RTL code and functional units of \cofhee{} will be open-sourced, to serve as components for future FHE acceleration work.
% Generated by IEEEtran.bst, version: 1.14 (2015/08/26)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus
\fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{barron2013cloud}
C.~Barron, H.~Yu, and J.~Zhan, ``Cloud computing security case studies and
research,'' in \emph{World Congress on Engineering}, 2013.
\bibitem{gentry2009thesis}
C.~Gentry, \emph{A fully homomorphic encryption scheme}.\hskip 1em plus 0.5em
minus 0.4em\relax Stanford University, 2009.
\bibitem{dai2014accelerating}
W.~Dai, Y.~Dor{\"o}z, and B.~Sunar, ``Accelerating \uppercase{NTRU} based
homomorphic encryption using \uppercase{GPU}s,'' in \emph{HPEC}, 2014.
\bibitem{cousins2012update}
D.~B. Cousins, K.~Rohloff, C.~Peikert, and R.~Schantz, ``An update on
\uppercase{SIPHER} (scalable implementation of primitives for homomorphic
encryption)—\uppercase{FPGA} implementation using simulink,'' in \emph{2012
IEEE Conference on High Performance Extreme Computing}, 2012.
\bibitem{gentry2011implementing}
C.~Gentry and S.~Halevi, ``Implementing gentry’s fully-homomorphic encryption
scheme,'' in \emph{Annual international conference on the theory and
applications of cryptographic techniques}, 2011.
\bibitem{doroz2014accelerating}
Y.~Dor{\"o}z, E.~{\"O}zt{\"u}rk, and B.~Sunar, ``Accelerating fully homomorphic
encryption in hardware,'' \emph{IEEE Transactions on Computers}, vol.~64,
no.~6, pp. 1509--1521, 2014.
\bibitem{feldmann2021f1}
A.~Feldmann, N.~Samardzic, A.~Krastev, S.~Devadas, R.~Dreslinski, K.~Eldefrawy,
N.~Genise, C.~Peikert, and D.~Sanchez, ``F1: A fast and programmable
accelerator for fully homomorphic encryption (extended version),''
\emph{arXiv preprint arXiv:2109.05371}, 2021.
\bibitem{bfv}
J.~Fan and F.~Vercauteren, ``Somewhat practical fully homomorphic encryption,''
\emph{IACR Cryptology ePrint Archive}, vol. 2012, p. 144, 2012.
\bibitem{bgv}
Z.~Brakerski, C.~Gentry, and V.~Vaikuntanathan, ``(leveled) fully homomorphic
encryption without bootstrapping,'' \emph{ACM Transactions on Computation
Theory (TOCT)}, vol.~6, no.~3, pp. 1--36, 2014.
\bibitem{cggi}
I.~Chillotti, N.~Gama, M.~Georgieva, and M.~Izabach{\`e}ne, ``Faster fully
homomorphic encryption: Bootstrapping in less than 0.1 seconds,'' in
\emph{Advances in Cryptology -- ASIACRYPT 2016}, 2016.
\bibitem{ckks}
J.~H. Cheon, A.~Kim, M.~Kim, and Y.~Song, ``Homomorphic encryption for
arithmetic of approximate numbers,'' in \emph{ASIACRYPT}, 2017.
\bibitem{rlwe}
V.~Lyubashevsky, C.~Peikert, and O.~Regev, ``On ideal lattices and learning
with errors over rings,'' \emph{J. ACM}, vol.~60, no.~6, 2013.
\bibitem{alchemy}
\BIBentryALTinterwordspacing
E.~Crockett, C.~Peikert, and C.~Sharp, ``Alchemy: A language and compiler for
homomorphic encryption made easy,'' in \emph{Proceedings of the 2018 ACM
SIGSAC Conference on Computer and Communications Security}, ser. CCS
'18.\hskip 1em plus 0.5em minus 0.4em\relax New York, NY, USA: Association
for Computing Machinery, 2018, p. 1020–1037. [Online]. Available:
\url{https://doi.org/10.1145/3243734.3243828}
\BIBentrySTDinterwordspacing
\bibitem{cingulata}
\BIBentryALTinterwordspacing
S.~Carpov, P.~Dubrulle, and R.~Sirdey, ``Armadillo: A compilation chain for
privacy preserving applications,'' in \emph{Proceedings of the 3rd
International Workshop on Security in Cloud Computing}, ser. SCC '15.\hskip
1em plus 0.5em minus 0.4em\relax New York, NY, USA: Association for Computing
Machinery, 2015, p. 13–19. [Online]. Available:
\url{https://doi.org/10.1145/2732516.2732520}
\BIBentrySTDinterwordspacing
\bibitem{e3}
E.~Chielle, O.~Mazonka, N.~G. Tsoutsos, and M.~Maniatakos, ``E$^3$: A framework
for compiling c++ programs with encrypted operands,'' Cryptology ePrint
Archive, Report 2018/1013, 2018, online:
\url{https://eprint.iacr.org/2018/1013}, GitHub repository:
\url{https://github.com/momalab/e3}.
\bibitem{ngraph}
\BIBentryALTinterwordspacing
F.~Boemer, A.~Costache, R.~Cammarota, and C.~Wierzynski, ``Ngraph-he2: A
high-throughput framework for neural network inference on encrypted data,''
in \emph{Proceedings of the 7th ACM Workshop on Encrypted Computing \&
Applied Homomorphic Cryptography}, ser. WAHC'19.\hskip 1em plus 0.5em minus
0.4em\relax New York, NY, USA: Association for Computing Machinery, 2019, p.
45–56. [Online]. Available: \url{https://doi.org/10.1145/3338469.3358944}
\BIBentrySTDinterwordspacing
\bibitem{palisade}
Y.~Polyakov, K.~Rohloff, G.~W. Ryan, and D.~Cousins, ``{PALISADE} lattice
cryptography library user manual (v.1.10.6),'' 2020,
\url{https://palisade-crypto.org/documentation}.
\bibitem{ramparts}
\BIBentryALTinterwordspacing
D.~W. Archer, J.~M. Calder\'{o}n~Trilla, J.~Dagit, A.~Malozemoff, Y.~Polyakov,
K.~Rohloff, and G.~Ryan, ``Ramparts: A programmer-friendly system for
building homomorphic encryption applications,'' in \emph{Proceedings of the
7th ACM Workshop on Encrypted Computing \& Applied Homomorphic Cryptography},
ser. WAHC'19.\hskip 1em plus 0.5em minus 0.4em\relax New York, NY, USA:
Association for Computing Machinery, 2019, p. 57–68. [Online]. Available:
\url{https://doi.org/10.1145/3338469.3358945}
\BIBentrySTDinterwordspacing
\bibitem{seal}
``{M}icrosoft {SEAL} (release 3.7),'' \url{https://github.com/Microsoft/SEAL},
Sep. 2021, microsoft Research, Redmond, WA.
\bibitem{hestd}
M.~Albrecht, M.~Chase, H.~Chen, J.~Ding, S.~Goldwasser, S.~Gorbunov, S.~Halevi,
J.~Hoffstein, K.~Laine, K.~Lauter, S.~Lokam, D.~Micciancio, D.~Moody,
T.~Morrison, A.~Sahai, and V.~Vaikuntanathan, ``Homomorphic encryption
security standard,'' HomomorphicEncryption.org, Toronto, Canada, Tech. Rep.,
November 2018.
\bibitem{cooley-tukey}
J.~W. Cooley and J.~W. Tukey, ``An algorithm for the machine calculation of
complex fourier series,'' \emph{Mathematics of Computation}, vol.~19, no.~90,
pp. 297--301, 1965.
\bibitem{gentleman-sande}
W.~M. Gentleman and G.~Sande, ``Fast fourier transforms: For fun and profit,''
in \emph{Proceedings of the November 7-10, 1966, Fall Joint Computer
Conference}, 1966, p. 563–578.
\bibitem{kim2015private}
M.~Kim and K.~Lauter, ``Private genome analysis through homomorphic
encryption,'' \emph{BMC medical informatics and decision making}, 2015.
\bibitem{pmt}
E.~Chielle, H.~Gamil, and M.~Maniatakos, ``Real-time private membership test
using homomorphic encryption,'' in \emph{2021 Design, Automation Test in
Europe Conference Exhibition (DATE)}, 2021, pp. 1282--1287.
\bibitem{gursoy}
G.~Gürsoy, E.~Chielle, C.~M. Brannon, M.~Maniatakos, and M.~Gerstein,
``Privacy-preserving genotype imputation with fully homomorphic encryption,''
\emph{Cell Systems}, vol.~13, no.~2, pp. 173--182.e3, 2022.
\bibitem{modmulalgocomp}
A.~Bosselaers, R.~Govaerts, and J.~Vandewalle, ``Comparison of three modular
reduction functions,'' in \emph{Annual International Cryptology Conference},
1993, pp. 175--186.
\bibitem{noauthor_amba_2001}
ARM, ``{AMBA} 3 {AHB}-{Lite} {Protocol} {Specification},'' p.~72, 2001.
\bibitem{cryptohandbook}
A.~J. Menezes, P.~C. Van~Oorschot, and S.~A. Vanstone, \emph{Handbook of
applied cryptography}.\hskip 1em plus 0.5em minus 0.4em\relax CRC press,
2018.
\bibitem{Rossi1996NonredundantSA}
A.~Rossi and G.~Fucili, ``Nonredundant successive approximation register for
a/d converters,'' \emph{Electronics Letters}, vol.~32, pp. 1055--1057, 1996.
\bibitem{7452270}
B.~Murmann, ``The successive approximation register adc: a versatile building
block for ultra-low- power to ultra-high-speed applications,'' \emph{IEEE
Communications Magazine}, vol.~54, no.~4, pp. 78--83, 2016.
\bibitem{1084494}
H.~Russell, ``An improved successive-approximation register design for use in
a/d converters,'' \emph{IEEE Transactions on Circuits and Systems}, vol.~25,
no.~7, pp. 550--554, 1978.
\bibitem{Alexander1975ClockRF}
J.~D. Alexander, ``Clock recovery from random binary signals,''
\emph{Electronics Letters}, vol.~11, pp. 541--542, 1975.
\bibitem{Lee2004AnalysisAM}
J.~Lee, K.~S. Kundert, and B.~Razavi, ``Analysis and modeling of bang-bang
clock and data recovery circuits,'' \emph{IEEE Journal of Solid-State
Circuits}, vol.~39, pp. 1571--1580, 2004.
\bibitem{riazi_heax}
M.~S. Riazi, K.~Laine, B.~Pelton, and W.~Dai, ``\uppercase{HEAX}: An
architecture for computing on encrypted data,'' in \emph{ASPLOS '20}, 2020,
p. 1295–1309.
\bibitem{roy2019fpga}
S.~S. Roy, F.~Turan, K.~Jarvinen, F.~Vercauteren, and I.~Verbauwhede,
``\uppercase{FPGA}-based high-performance parallel architecture for
homomorphic computing on encrypted data,'' in \emph{HPCA}, 2019.
\bibitem{jung2020heaan}
W.~Jung, E.~Lee, S.~Kim, K.~Lee, N.~Kim, C.~Min, J.~H. Cheon, and J.~H. Ahn,
``Heaan demystified: Accelerating fully homomorphic encryption through
architecture-centric analysis and optimization,'' \emph{arXiv preprint
arXiv:2003.04510}, 2020.
\bibitem{HEAX}
\BIBentryALTinterwordspacing
M.~S. Riazi, K.~Laine, B.~Pelton, and W.~Dai, ``Heax: An architecture for
computing on encrypted data,'' in \emph{Proceedings of the Twenty-Fifth
International Conference on Architectural Support for Programming Languages
and Operating Systems}, ser. ASPLOS '20.\hskip 1em plus 0.5em minus
0.4em\relax New York, NY, USA: Association for Computing Machinery, 2020, p.
1295–1309. [Online]. Available:
\url{https://doi.org/10.1145/3373376.3378523}
\BIBentrySTDinterwordspacing
\bibitem{DesignMert20}
A.~C. Mert, E.~{\"O}zt{\"u}rk, and E.~Sava{\c{s}}, ``Design and implementation
of encryption/decryption architectures for bfv homomorphic encryption
scheme,'' \emph{IEEE Transactions on Very Large Scale Integration (VLSI)
Systems}, vol.~28, no.~2, pp. 353--362, 2019.
\bibitem{hsu2020vlsi}
H.-J. Hsu and M.-D. Shieh, ``Vlsi architecture of polynomial multiplication for
bgv fully homomorphic encryption,'' in \emph{2020 IEEE International
Symposium on Circuits and Systems (ISCAS)}.\hskip 1em plus 0.5em minus
0.4em\relax IEEE, 2020, pp. 1--4.
\bibitem{reagen2021cheetah}
B.~Reagen, W.-S. Choi, Y.~Ko, V.~T. Lee, H.-H.~S. Lee, G.-Y. Wei, and
D.~Brooks, ``Cheetah: Optimizing and accelerating homomorphic encryption for
private inference,'' in \emph{2021 IEEE International Symposium on
High-Performance Computer Architecture (HPCA)}.\hskip 1em plus 0.5em minus
0.4em\relax IEEE, 2021, pp. 26--39.
\bibitem{cilardo2016securing}
A.~Cilardo and D.~Argenziano, ``Securing the cloud with reconfigurable
computing: An fpga accelerator for homomorphic encryption,'' in \emph{2016
Design, Automation \& Test in Europe Conference \& Exhibition (DATE)}.\hskip
1em plus 0.5em minus 0.4em\relax IEEE, 2016, pp. 1622--1627.
\bibitem{cousins2016designing}
D.~B. Cousins, K.~Rohloff, and D.~Sumorok, ``Designing an fpga-accelerated
homomorphic encryption co-processor,'' \emph{IEEE Transactions on Emerging
Topics in Computing}, vol.~5, no.~2, pp. 193--206, 2016.
\bibitem{bradbury2021ntt}
J.~Bradbury, N.~Drucker, and M.~Hillenbrand, ``Ntt software optimization using
an extended harvey butterfly,'' \emph{Cryptology ePrint Archive}, 2021.
\bibitem{asif18}
S.~{Asif}, O.~{Andersson}, J.~{Rodrigues}, and Y.~{Kong}, ``65-nm cmos
low-energy rns modular multiplier for elliptic-curve cryptography,''
\emph{IET Computers Digital Techniques}, vol.~12, no.~2, pp. 62--67, 2018.
\bibitem{boateng18}
R.~Boateng~Nti and K.~Ryoo, ``Asic design of low area rsa crypto-core based on
montgomery multiplier,'' \emph{International Journal of Engineering and
Technology}, vol.~7, pp. 278--283, 08 2018.
\bibitem{ding2017modular}
J.~Ding and S.~Li, ``A modular multiplier implemented with truncated
multiplication,'' \emph{IEEE Transactions on Circuits and Systems II: Express
Briefs}, vol.~65, no.~11, pp. 1713--1717, 2017.
\bibitem{asif16}
S.~Asif and Y.~Kong, ``Highly parallel modular multiplier for elliptic curve
cryptography in residue number system,'' \emph{Circuits, Systems, and Signal
Processing}, vol.~36, 05 2016.
\bibitem{kuang16}
S.~{Kuang}, K.~{Wu}, and R.~{Lu}, ``Low-cost high-performance vlsi architecture
for montgomery modular multiplication,'' \emph{IEEE Transactions on Very
Large Scale Integration (VLSI) Systems}, vol.~24, no.~2, pp. 434--443, 2016.
\bibitem{wang12}
S.~{Wang}, W.~{Lin}, J.~{Ye}, and M.~{Shieh}, ``Fast scalable radix-4
montgomery modular multiplier,'' in \emph{2012 IEEE International Symposium
on Circuits and Systems (ISCAS)}, 2012, pp. 3049--3052.
\bibitem{sakiyama11}
K.~Sakiyama, M.~Knežević, J.~Fan, B.~Preneel, and I.~Verbauwhede,
``Tripartite modular multiplication,'' \emph{Integration}, vol.~44, pp.
259--269, 09 2011.
\bibitem{knezevic10}
M.~{Knezevic}, F.~{Vercauteren}, and I.~{Verbauwhede}, ``Faster interleaved
modular multiplication based on barrett and montgomery reduction methods,''
\emph{IEEE Transactions on Computers}, vol.~59, no.~12, pp. 1715--1721, 2010.
\bibitem{riazi_heax_2020}
\BIBentryALTinterwordspacing
M.~S. Riazi, K.~Laine, B.~Pelton, and W.~Dai, ``{HEAX}: {An} {Architecture} for
{Computing} on {Encrypted} {Data},'' \emph{arXiv:1909.09731 [cs]}, Jan. 2020,
arXiv: 1909.09731. [Online]. Available: \url{http://arxiv.org/abs/1909.09731}
\BIBentrySTDinterwordspacing
\end{thebibliography}