TD-Interpreter: Enhancing the Understanding of Timing Diagrams with Visual-Language Learning
Extracted source text · arXiv 2507.16844
\documentclass[9pt,conference]{IEEEtran}
\IEEEoverridecommandlockouts
\usepackage{algorithm}
\usepackage{enumitem}
\setlist[itemize]{align=parleft,left=1pt..1em}
\usepackage{tikz}
\usetikzlibrary{arrows,automata,positioning,calc,arrows,arrows.meta}
\usepackage{wrapfig}
\PassOptionsToPackage{hyphens}{url}
\usepackage{hyperref}
%% These commands are for a PROCEEDINGS abstract or paper.
% \acmConference[Woodstock '18]{Woodstock '18: ACM Symposium on Neural
% Gaze Detection}{June 03--05, 2018}{Woodstock, NY}
% \acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
% June 03--05, 2018, Woodstock, NY}
% \acmPrice{15.00}
% \acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
%\documentclass{article}
\usepackage{algorithm}
\usepackage[noend]{algpseudocode}
\usepackage{cite}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{hyperref} % hyperlinks
\usepackage{url} % simple URL typesetting
\usepackage{booktabs} % professional-quality tables
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypograph
\usepackage{xcolor} % colors
\usepackage{listings} % listings
% \usepackage{indentfirst}
\usepackage{amsmath}
\newtheorem{definition}{Definition}
%\usepackage{amssymb}
% \usepackage{algorithmic}
% \usepackage{wrapfig, blindtext}
% \usepackage{graphicx}
\usepackage{multirow}
% % \usepackage[upright]{fourier}
% \usepackage{tkz-kiviat,numprint,fullpage}
\usepackage{tkz-kiviat}
% %\usepackage[usenames,dvipsnames]{xcolor}
\usetikzlibrary{arrows}
\graphicspath{ {./figures/} }
\usepackage{pgfplots}
\pgfplotsset{compat=1.17}
\usepackage{caption}
\captionsetup[table]{skip=-10pt}
\usepackage{subcaption}
\usepackage{indentfirst}
\usepackage{bm}
\input macros
\newcommand{\tocite}[1]{\textcolor{red}{[TO CITE]}}
\newcommand{\todo}[1]{\textcolor{red}{[TODO: #1]}}
\def\BibTeX{{\rm B\kern-.05em{\sc i\kern-.025em b}\kern-.08em
T\kern-.1667em\lower.7ex\hbox{E}\kern-.125emX}}
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
%\title{From Pictures of Timing Diagrams to Formal Specifications}
\title{TD-Interpreter: Enhancing the Understanding of Timing Diagrams with Visual-Language Learning}
% \author{\large
% Meixun Qu$^{\ast, \dag}$, Jie He$^{\ast, \dag}$, Zlatan Tucaković$^{\dag}$, Ezio Bartocci$^{\dag}$, Dejan Ničković$^{\ddag}$, Haris Isaković$^{\dag}$, Radu Grosu$^{\dag}$}
% \author{
% Jie He$^{\dag}$, Vincent Theo Willem Kenbeek$^{\dag}$, Zhantao Yang$^{\S}$, Meixun Qu$^{\dag}$, Ezio Bartocci$^{\dag}$, Dejan Ničković$^{\ddag}$, Radu Grosu$^{\dag}$ \\
% \small $^{\dag}$Technische Universität Wien, Austria \\
% \small $^{\S}$Shanghai Jiao Tong University, China \\
% \small $^{\ddag}$AIT Austrian Institute of Technology, Austria % \\
% % \small Email: \{jie.he, vincent.kenbeek, meixun.qu, ezio.bartocci, radu.grosu\}@tuwien.ac.at, \\
% % \small ztyang196@gmail.com, dejan.nickovic@ait.ac.at
% }
\author{
Jie He$^{\dag}$, Vincent Theo Willem Kenbeek$^{\dag}$, Zhantao Yang$^{\S}$, Meixun Qu$^{\dag}$, Ezio Bartocci$^{\dag}$, Dejan Ničković$^{\ddag}$, Radu Grosu$^{\dag}$ \\
\small $^{\dag}$Technische Universität Wien, Austria \\
\small $^{\S}$Shanghai Jiao Tong University, China \\
\small $^{\ddag}$AIT Austrian Institute of Technology, Austria % \\
% \small Email: \{jie.he, vincent.kenbeek, meixun.qu, ezio.bartocci, radu.grosu\}@tuwien.ac.at, \\
% \small ztyang196@gmail.com, dejan.nickovic@ait.ac.at
}
% \author{Jie He}
% \email{jie.he@tuwien.ac.at}
% \affiliation{%
% \institution{Technische Universität Wien}
% \city{Vienna}
% \country{Austria}
% }
% \author{Ezio Bartocci}
% \email{ezio.bartocci@tuwien.ac.at}
% \affiliation{%
% \institution{Technische Universität Wien}
% \city{Vienna}
% \country{Austria}
% }
% \author{Dejan Ničković}
% \email{Dejan.Nickovic@ait.ac.at}
% \affiliation{%
% \institution{AIT Austrian Institute of Technology}
% \city{Vienna}
% \country{Austria}
% }
% \author{Haris Isakovic}
% \email{haris@vmars.tuwien.ac.at}
% \affiliation{%
% \institution{Technische Universität Wien}
% \city{Vienna}
% \country{Austria}
% }
% \author{Radu Grosu}
% \email{radu.grosu@tuwien.ac.at}
% \affiliation{%
% \institution{Technische Universität Wien}
% \city{Vienna}
% \country{Austria}
% }
\maketitle
\input abstract
%\makeatletter
\newcommand\footnoteref[1]{\protected@xdef\@thefnmark{\ref{#1}}\@footnotemark}
%\makeatother
\begin{IEEEkeywords}
Digital Design, Timing Diagram, Multi-Modal LLM, Visual-Language Learning
\end{IEEEkeywords}
\input{1_introduction}
\input{2_related_work}
\input{questionnaire}
\input{3_dataset_construction}
% \input{4_dataset_construction}
% \input{5_object_detection}
\input{5_method}
\input{6_evaluation}
\input{7_conclusion}
% \input{main.bbl}
% \section*{Acknowledgments}
% This project has received funding from the Austrian FFG and the European Key Digital Technologies Joint Undertaking (KDT JU) under grant agreement No 101097300. It is also partially supported by the Chips Joint Undertaking and the Austrian Research Promotion Agency (FFG) through the AIMS5.0 project (EU grant agreement number 101112089).
\section*{Acknowledgments}
This project has received funding from the Austrian FFG and the European Key Digital Technologies Joint Undertaking (KDT JU) under grant agreement No 101097300. It is also partially supported by the Chips Joint Undertaking and the Austrian Research Promotion Agency (FFG) through the AIMS5.0 project (EU grant agreement number 101112089).
\input{main.bbl}
% % \bibliographystyle{unsrt}
% \bibliographystyle{IEEEtran}
% % % \bibliography{reference}
% \bibliography{main}
\end{document}
\endinput
%%
%% End of file `sample-sigconf.tex'.
\newcommand{\until}{\textbf{U}}
\newcommand{\since}{\textbf{S}}
\newcommand{\F}{\textbf{F}}
\newcommand{\GG}{\textbf{G}}
\newcommand{\opP}{\textbf{O}}
\newcommand{\opH}{\textbf{H}}
\newcommand{\true}{\textbf{true}}
\newcommand{\false}{\textbf{false}}
\newcommand{\rise}{\textbf{rise}}
\newcommand{\fall}{\textbf{fall}}
\newcommand{\SP}{\text{SP}}
\newcommand{\TP}{\text{TP}}
\newcommand{\NTP}{\text{NTP}}
\newcommand{\PP}{\text{P}}
\newcommand{\CP}{\text{CP}}
\newcommand{\TO}{\textbf{TO}}
\newcommand{\UTO}{\textbf{UTO}}
\newcommand{\BTO}{\textbf{BTO}}
\begin{abstract}
% We introduce TD-Magic, the first neuro-symbolic approach for translating an image of a timing-diagram (TD) to a formal specification. We overcome the lack of labelled data for supervised learning, by first developing a synthetic data generator of labelled TDs. We then use object detection techniques to identify rising and failing edges, OCR to recognise the text, and image processing algorithms to capture synchronisation patterns. Finally, we use semantic interpretation to analyse the extracted features and generate the associated formal specification. Our experiments on industrial TDs show high translation accuracy opening the way to more sophisticated requirements-extraction algorithms from pictures.
% We introduce TD-Interpreter, a specialized ML tool that assists engineers in understanding complex timing diagrams (TDs) originating from a third party, during their own design and verification process. TD-Interpreter is a visual question-answer environment, which allows engineers to input a set of TDs and ask design and verification queries regarding these TDs. We implemented TD-Interpreter with multi-modal learning. To address limited training data availability, we developed a synthetic data generation workflow that aligns visual information with its textual interpretation. We fine-tuned a lightweight 7B Multimodal Large Language Model (MLLM) which significantly outperformed large commercial LLMs like GPT-4.0 for such queries. This advancement enables digitizing tacit industrial knowledge and accelerating development process with low cost.
% We introduce TD-Interpreter, a specialized ML tool that assists engineers in understanding complex timing diagrams (TDs), originating from a third party, during their design and verification process. TD-Interpreter is a visual question-answer environment which allows engineers to input a set of TDs and ask design and verification queries regarding these TDs. We implemented TD-Interpreter with Multimodal learning by fine-tuning LLaVA, a lightweight 7B Multimodal Large Language Model (MLLM). To address limited training data availability, we developed a synthetic data generation workflow that aligns visual information with its textual interpretation. Our experimental evaluation demonstrates the usefulness of TD-Interpreter which outperformed untuned GPT-4.0 by a large margin on the evaluated questions.
% \end{abstract}
We introduce TD-Interpreter, a specialized ML tool that assists engineers in understanding complex timing diagrams (TDs), originating from a third party, during their design and verification process. TD-Interpreter is a visual question-answer environment which allows engineers to input a set of TDs and ask design and verification queries regarding these TDs. We implemented TD-Interpreter with multimodal learning by fine-tuning LLaVA, a lightweight 7B Multimodal Large Language Model (MLLM). To address limited training data availability, we developed a synthetic data generation workflow that aligns visual information with its textual interpretation. Our experimental evaluation demonstrates the usefulness of TD-Interpreter which outperformed untuned GPT-4o by a large margin on the evaluated benchmarks.
\end{abstract}
\section{Introduction}
\label{sec:intro}
Digital design is a complex process involving the transformation of requirements into digital circuits by using hardware description languages. While High-Level Synthesis methods and advanced IP cores reduce the time and complexity of the design process, familiarity of engineers with the function of the various hardware components remains essential. Timing diagrams (TDs), as a visual requirements specification of the functionality and interaction between different signals, play a central role in the design and verification processes. As a consequence, a solid understanding of TDs is crucial.
Different TDs may vary in their level of abstraction. Documentations and datasheets often contain general high-level TDs describing basic timing constraints or the operational logic of specific communication protocols. TDs generated from existing code, often provide detailed insights from senior engineers. TDs drawn by engineers in daily work, are in between of these two levels. In order to help engineers better understand TDs, this paper proposes the \textit{TD-Interpreter}, an AI tool that can interpret TDs in various scenarios, as illustrated in Figure~\ref{fig:chatbot}.
% \begin{figure}[htbp]
% \centering
% \includegraphics[width=1\columnwidth]{figures/adc.pdf}
% \caption{Timing diagram of the ADC128S002 chip\cite{adc}}
% \label{fig:adc}
% \end{figure}
\begin{figure}[htbp]
\centering
\includegraphics[width=0.97\columnwidth]{figures/chat_robot.png}
%\vspace*{-0.5ex}
\caption{Conversations with TD-Interpreter}
\vspace*{-3ex}
\label{fig:chatbot}
\end{figure}
Our first motivating example is understanding a set of \textit{abstract} TDs during the initial design phase. Figure~\ref{fig:chatbot}(A) shows how \textit{TD-Interpreter} can assist the engineer in designing the interface between an FPGA and an analog-to-digital converter (ADC). As shown in Figure~\ref{fig:chatbot}(A), the engineer uploads a TD picture from the datasheet\cite{adc}, and asks a question about its specification, such as, important timing events. \textit{TD-Interpreter} can then automatically analyse and interpret the TD.
% \begin{enumerate}
% \item What are the meanings of the four signals in the TD?
% \item What are the Input/Output (I/O) relations between these signals?
% \item What are the important timing events that should be noticed?
% \end{enumerate}
% For \textbf{Q1}, ``CS'' controls the procedure for a single cycle of data transmission, which always starts with its falling edge ends with its rising edge. ``SCLK'' is the generated clock signal that handles data transmission. ``DIN'' means the address of a specific channel, and ``DOUT'' is the data bit coming from the chosen channel. Consequently, the answer for \textbf{Q2} should be: The FPGA sends ``CS'',``SCLK'' and ``DIN'' signals to the ADC, and reads ``DOUT'' from it.
% \begin{figure}[htbp]
% \centering
% \includegraphics[width=0.9\columnwidth]{figures/chatbot.pdf}
% \caption{TD-interpreter}
% \label{fig:chatbot}
% \end{figure}
% To answer \textbf{Q3}, we draw 35 dashed lines (0-34) to extract important timing events. For example: (1) After signal``CS'' goes down ($t=0$), the FPGA should send the first falling edge of "SCLK" to the ADC ($t=2$); (2) The FPGA should send address bit ``DIN'' to the ADC at 3rd, 4th, 5th falling edges of ``SCLK'' ($t=6, 8, 10$), which enables the ADC to stably read these bits at the 3rd, 4th, 5th rising edges of ``SCLK'' ($t=7, 9, 11$); (3) The ADC sends effective ``DOUT'' bit to the FPGA at 5th-16th falling edges of ADC ($t=10, 12,...,32)$, which enables the FPGA to stably read each ``DOUT'' bit at 5th-16th rising edges of ``SCLK'' ($t=11, 13,...,33)$.
% With the help of clarifying these questions, the engineer is then able to implement the interface module with minimum possible errors.
% \begin{figure}[htbp]
% \centering
% \includegraphics[width=1\columnwidth]{figures/afifo.pdf}
% \caption{Timing diagram of an asynchronous FIFO}
% \label{fig:afifo}
% \end{figure}
Our second motivating example is understanding \textit{concrete} TDs in later design stages. Figure~\ref{fig:chatbot}(B) illustrates a use case where the engineer has to customize a third-party asynchronous first-in-first-out (FIFO) module for specialized hardware. The engineer can familiarize herself/himself with the existing module, by generating its TD pictures, and uploading them with associated questions to \textit{TD-Interpreter}.
% demonstrates a TD generated from the \textit{Verilog} code of an asynchronous FIFO (First In First Out) module. The engineer's job is to customize this FIFO for a specialized hardware. Before modifying the codes, several questions may also arise:
% \begin{enumerate}
% \item What data transaction procedure does this TD show?
% \item How do signals correlate each other in different clock domains?
% \item How to count the number of data inside the FIFO, and how to signify the FIFO is full?
% \end{enumerate}
% These are only some questions that are critical in understanding the working mechanism of this FIFO. Although there were many works analyzing real traces from execution, this use case focuses more on helping engineers understand waveforms from existing modules for modification or redesign purposes. This is a common task in digital design but is beyond the capability of currently available tools.
\textit{TD-Interpreter} was developed by leveraging the advancements in multi-modal large language models (MLLMs), which have enabled diverse applications such as text-to-image generation, image captioning, and visual question-answering. However, the direct use of commercial MLLMs such as GPT-4o for TD interpretation, is still problematic due to IP-privacy concerns, fine-tuning limitations, and insufficient training data. To address these problems, we explored cost-effective solutions for data construction, training, and local deployment, specially tailored for companies with limited resources and IP concerns. In particular:
\begin{itemize}
\item\textit{We conduct a human-centered empirical study}, by interviewing a group 31 experts in embedded systems and digital design. We collected statistical information about their professional experience with TDs, the challenges and difficulties they encountered when working with TDs, their responses towards practical TDs in the wild, and their expectation regarding the functions of \textit{TD-Interpreter}.
% \vspace*{2mm}
\item\textit{We solve the lack-of-dataset problem}, by developing an automatic synthetic data generator based on the empirical study, transforming the TD-interpretation problem into a visual question-answering task in multimodal learning. We generate two types of data. The first type focuses on generating captions for given TDs, enhancing the MLLM's ability to describe detailed information within a TD. The second type targets analytical tasks, enabling the MLLM to reason about diverse timing relations.
% \vspace*{-2mm}
\item\textit{We solve the IP and fine-tuning problems}, by successfully fine-tuning the open-source MLLM LLaVA~\cite{liu2023visualinstructiontuning}, with a caption-enhanced visual-reasoning approach. By combining captioning and reasoning tasks during training, LLaVA achieves very promising capabilities in reasoning about detailed information within TDs. With only 7B parameters, our model is also well-suited for local deployment.
\end{itemize}
% As shown in Figure~\ref{fig:arch}, this model accepts one TD image and one textual question as inputs, and sends them to the \textit{Vision Backbone} and \textit{Textual Transformer} respectively. The \textit{Vision Backbone} will transform the image into \textit{visual embeddings}. Then the \textit{Textual Transformer} will take both the \textit{visual embeddings} and the textual questions as inputs, and finally outputs the textual answer. For the \textit{Vision Backbone}, we choose vanilla \textit{Vision Transformer} mainly for its simplicity and wide usage in multi-modal LLMs. The \textit{Textual Transformer} adopts LLaMA~\cite{touvron2023llama}, an open-source textual LLM as the main model.
\noindent{}In a nutshell, \textit{TD-Interpreter} combines the expertise in digital design and advances in multi-modal learning. It promotes the automatic analysis of TDs, and particularly, it digitizes the tacit knowledge in the industry, thereby accelerating the overall development process.
% In a nutshell, the contributions of this paper can be summarized as:
% \begin{itemize}
% \item To the best of our knowledge, we are the \textit{first} to consider developing an AI tool that can assist engineer interpret complex TDs and allow them to ask questions related to design and verification tasks.
% \item We transformed the investigated problem into a standard \textit{Visual Question Answering} (VQA) task, and tamed the lack of training data problem by creating a specialized dataset.
% \item We proposed a method to finetune a lightweight 7B multi-modal LLM, which outperforms huge commercial LLMs like GPT4.0.
% \end{itemize}
% Our work combines the expertise in digital design and multi-modal learning.
% It reduces the amount of time spent on analyzing TDs, and particularly, it digitizes the tacit knowledge in the industry, thereby accelerating the overall development process.
The organization of this paper is as follows. Section~\ref{sec:realated} provides the related work. Section~\ref{sec:questionnaire} shows details of our empirical study. Section~\ref{sec:dataset} illustrates how we generated the training data, and Section~\ref{sec:method} explores how we fine-tune the multi-modal LLM. Section~\ref{sec:evaluate} introduces and analyzes our experimental results.
% \begin{figure}[htbp]
% \centering
% \includegraphics[width=0.4\columnwidth]{figures/image_3.pdf}
% \caption{Model Architecture}
% \label{fig:arch}
% \end{figure}
% In a nutshell, this paper focuses on developing an AI tool for engineers, allowing them to ask questions about TDs related to design and verification tasks. This capability will reduce the amount of time spent on analyzing these diagrams and accelerate the
% overall development process.
\section{Related Work}
\label{sec:realated}
\noindent Since LLM's advent, multiple studies explored their use in facilitating hardware design. For example,~\cite{liu2023verilogeval, liu2023chipnemo, thakur2023benchmarking, thakur2024verigen, iccad} studied the translation of textual design requirements into Verilog code. These works either used prompt engineering in commercial LLMs, or fine-tuned open-source LLMs such as Llama~\cite{touvron2023llama} and CodeGen~\cite{nijkamp2022codegen}. Some works also extended the use of LLMs in other domains, such as accelerator design~\cite{fu2023gpt4aigchip}. However, all these works tried to convert the given problems into a natural-language-processing (NLP) code-generation task. Hence, the LLMs used by most of these works, only process textual data. However, the VQA task proposed in this paper, explores the potential of MLLM through visual-language learning, which enables the \textit{TD-Interpreter to process data in both visual and textual modalities}. Recently,~\cite{chang2024natural} also considered MLLM for hardware design, but this work did not focus on TDs, and the purpose was still code generation.
\paragraph{Timing diagrams}
% One pioneer work that is closest to this paper is ~\cite{selyunin2017runtime}, where the authors proposed \textit{TD-Magic} that aims to extract Strict Partial Order (SPO) relation as a formal specification from pictures of TDs. However, only a small portion of TDs have explicit SPO relations, hence \textit{TD-Magic} only limits its usages in creating library files for circuit components provided from foundries, or in creating scripts for timing verification in tools like \textit{PrimeTime (PT)}. Another drawback of \textit{TD-Magic} is its multi-module design and rule-based semantic interpretation mechanism have limited its capability to process TDs in large scale. In contrast, by leveraging MLLM, \textit{TD-Interpreter} is designed for a broader group of engineers in design and verification, helping them understand a wider variety of TDs with more complexity and more implicit semantics.
In a recent work~\cite{he2023td}, the authors introduced \textit{TD-Magic} that can extract Strict Partial Order (SPO) relations as formal specifications from TD images. However, since only a tiny fraction of TDs contain explicit SPO relations, \textit{TD-Magic's} application is mainly restricted to creating library files for circuit components from foundries or creating timing verification scripts for tools like \textit{PrimeTime}~\cite{Bhatnagar2002}. Additionally, the multi-module, rule-based design for \textit{TD-Magic} limits its capabilities to process large-scale TD pictures. In contrast, \textit{TD-Interpreter} leverages MLLM to assist a broader range of engineers in design and verification, enabling them to interpret more complex TDs through a visual question-answer environment. However, what \textit{TD-Magic} and \textit{TD-Interpreter} have in common is that they both simulate the process of an engineer reading a TD image in digital circuits, which distinguishes them from another line of studies that directly analyze data points from possibly mixed-signal waveforms for testing or monitoring purposes~\cite{maler2004monitoring}. These works adopted a model-to-model approach to transform the data points into formal languages such as automata~\cite{fisler1999timing} and temporal logic~\cite{amla1999efficient, bartocci2022survey}.
% \noindent \textbf{MLLM.} Language models are statistical models that describe the probability distribution of natural language. Given part of a sentence, such a model can predict the contents of the other parts of the sentence. This forms the core task of natural language processing (NLP) \cite{wu2023brief}. Although such language models have existed since the 1980s, they have gained massive popularity in recent years after the introduction of the transformer architecture in 2017 which is specifically designed for sequence-to-sequence tasks previously handled by recurrent neural networks (RNNs) \cite{vaswani2023attentionneed}. RNNs process data sequentially which makes it difficult to parallelize computation, increasing training time on large corpora of text. To improve the computational efficiency, the attention mechanism was introduced with which each data element attends to every other element simultaneously \cite{bahdanau2016neuralmachinetranslationjointly}. Additionally, such a mechanism allows for long-range dependencies in sequences in contrast to RNNs where each data point highly dependent on the previous. However, this attention mechanism was generally used in conjunction with recurrent networks, so the underlying inefficiency of sequential computations remained. The transformer architecture shows that the attention mechanism alone is sufficient for NLP, removing the sequential dependencies and allowing for highly parallelized computation.
% The transformer forms the backbone of large language models (LLMs) such as OpenAI's ChatGPT and Google's BERT which have made headlines in recent years for their performance in NLP related tasks \cite{brown2020languagemodelsfewshotlearners} \cite{devlin2019bertpretrainingdeepbidirectional}. These LLMs are trained on a vast library of text, which is made possible by the parallelism offered by the transformer.
% The aim of the thesis is to use LLMs such as these to explain timing diagrams in clear natural language.\newline
% \paragraph{Multi-modal architectures} Multi-modal architectures can process information from several data modalities such as text, images, or audio, and providing outputs in others.
% % This concept is familiar; classification models which take an image and output a label have existed for several decades, for example. However, the transformer architecture has significantly boosted the performance of multi-modal architectures \cite{DosovitskiyB0WZ21}.
% For example, the Vision Transformer (ViT)~\cite{DosovitskiyB0WZ21}, splits an image into a sequence of patches, after which they can be treated the same way sequential data is treated in NLP tasks. When trained on small data sets, ViT does not perform as well as classical convolutional neural networks in classification tasks, but the use of large dataset for training improves significantly its performance. This initial success on large datasets has inspired many more complex multi-modal architectures \cite{wadekar2024evolutionmultimodalmodelarchitectures}. One of the latest such architectures is LLaVA \cite{liu2023visualinstructiontuning}. This architecture leverages visual-instruction tuning, a new method for fine-tuning multi-modal models based on instruction-tuning, previously used in language models. Here models are trained using pairs of input-output instructions.
% % Visual-instruction tuning extends this method by generating sequences of input-output instructions as a multi-turn conversation consisting of questions and answers for images.
% % At its core, LLaVa can use any pre-existing LLM to handle language and any pre-existing visual model for visual encoding. A simple linear layer connects the image features to the word embedding space used by the LLM.
% Through visual-instruction tuning, this relatively simple architecture can achieve over 90\% accuracy on the ScienceQA dataset~\cite{lu2022learnexplainmultimodalreasoning}. Our work shows that LLaVa can also be tuned to interpret complex TDs.
% MLLM
\paragraph{Multi-modal LLM} MLLMs can process information from several data modalities such as text, images, or audio, and providing outputs in others~\cite{chen2024internvl}.
As one of the most popular open-source MLLMs, LLaVA \cite{liu2023visualinstructiontuning} employs a Vision Transformer (ViT)-based vision encoder \cite{DosovitskiyB0WZ21}, a LLaMA-based language model~\cite{touvron2023llama}, and a simple linear layer as an adapter to reduce training parameters. Recently, MLLMs have been enhanced through the use of larger and more powerful text encoders or adapters~\cite{chen2024internvl}, as well as improved alignment methods~\cite{team2023gemini}. MLLMs have been demonstrated to be powerful tools for performing multimodal tasks such as image captioning, visual question answering, and visual reasoning. However, applying them to domain-specific images, such as technical diagrams, remains a challenge. In this paper, we address this issue and demonstrate that lightweight MLLMs like LLaVA can function as robust interpreters for complex tasks.
% {\color{purple}
\section{Human-Centered Empirical Analysis}
\label{sec:questionnaire}
To ensure our system design reflects real-world challenges encountered by engineers, we conducted a human-centered empirical analysis on 31 professional participants regarding the use of AI tools for interpreting timing diagrams. We provide the results as follows.
\subsection{Background of Participants}
Most of the participants come from universities, research institutes, or hardware companies. Among all participants, 3 (9.7\%) have a bachelor's degree, 11 (35.5\%) have a master's degree. 17 (54.8\%) people hold a PhD's degree or during PhD study, among which there are three professors in computer engineering.
These participants cover a wide range of professional expertise in hardware design, such as digital circuit design, FPGA development, ASIC development, hardware verification, and embedded systems.
\subsection{Experience of Timing Diagrams}
We asked the participants in what contexts they typically use timing diagrams (TDs), and interestingly we found a diverse of of use cases. For example, more than half of the participants (18, 58.1\%) mentioned they used TDs to understand communication protocols (e.g., SPI, I2C, UART). Ten people (32.2\%) emphasized that they relied on TDs in debugging new hardware designs, or understanding and even reverse engineering existing designs from their colleagues or a third party. Four people (12.9\%) working in hardware verification used TDs particularly in writing/verifying Verilog/VHDL testbenches. Eight (25.8\%) highlighted the importance of using TDs in communicating with other people, such as presenting designs, group discussions, and team work. Two thirds of the participants told us they used TDs in their junior career period, such as coursework/homework assignment, or industrial internship projects. These findings are consistent with our motivation for interpreting TDs in Section~\ref{sec:intro}.
We further asked the participants \textit{Where do they usually encounter TDs}, and their responses can be divided into two groups, similar to the motivating examples in Section~\ref{sec:intro}. From their answers, half of the resources are \textit{abstract TDs} from datasheets, internal design documents, or textbooks; while the rest half are \textit{concrete TDs} waveforms from Verilog/VHDL codes, or MATLAB/Simulink simulations.
As it is important to decide which format of TDs we will support, we specially designed a question to ask the participants \textit{What tools do they use to create or analyze timing TDs}. Interestingly, we still received two groups of evenly distributed answers. The first group of people mentioned ``they mainly manually drew TDs''. The second group of resources are waveforms from simulation tools such as GTKWave and ModelSim. Despite that the participants mentioned different TD formats, in industry people tend to use standard formats for the convenience of communication, and among them \textit{wavedrom}~\cite{wavedrom-cli} is commonly used. As will be introduced in Section~\ref{sec:dataset}, \textit{wavedrom} supports the use of scripts to draw standard TDs in any level of complexity. It also provides interfaces to directly transform simulation waveforms to its standard format. Given these advantages, we chose to interpret TDs represented in the \textit{wavedrom} format.
\subsection{Challenges and Difficulties}
In this group of questions, participants were asked to use a scale from 1 (very easy) to 5 (very difficult) to rate the common challenges in understanding TDs. In the following, we show the challenges and the associated average scores of difficulty.
\begin{description}
\item[C1:] Extracting meaningful information from datasheets. (2.97/5.0)
\item[C2:] Understanding the overall transaction procedure of a communication protocol. \textbf{(3.16/5.0)}
\item[C3:] Understanding the Input/Output relations between signals. (2.84/5.0)
\item[C4:] Extracting key events from signal behavior, e.g., what a certain rising/falling edge means. (2.77/5.0)
\item[C5:] Understanding relations between multiple signals over multiple clock cycles. \textbf{(3.68/5.0)}
\item[C6:] Recognizing timing relations/constraints in multi-clock systems. \textbf{(3.77/5.0)}
\item[C7:] Analyzing clock-domain crossing behavior. \textbf{(3.84/5.0)}
\item[C8:] Measuring timing parameters (e.g., delay and latency) and identifying setup and hold time violations. \textbf{(3.26/5.0)}
\item[C9:] Detecting metastability related problems. \textbf{(4.0/5.0)}
\item[C10:] Matching timing diagrams with real-world hardware behavior. \textbf{(3.23/5.0)}
\end{description}
The challenges with average scores above 3 are considered relatively more difficult for engineers, which aligns with common experiences in digital design. In particular, understanding timing relationships across multiple signals and clock cycles, as well as identifying metastability-related issues in clock-domain crossing (CDC) scenarios, are widely recognized as complex and often intractable tasks.
Except the difficulties listed above, several participants also added that it is challenging to extract finite state machines from waveform traces, which is a common task in analyzing communication protocols. These findings will guide our procedure in generating training data, which will be detailed in Section~\ref{sec:dataset}.
\subsection{Use of AI}
In this subsection, we investigated participants’ attitudes toward the development of \textit{TD-Interpreter} as a potential tool to assist in interpreting timing diagrams in their work. The majority of participants (30/31) indicated that they would feel comfortable using such a tool if it existed, and 24 participants expressed clear interest in its application. Over 90\% of the participants were expecting to see \textit{reasoning} procedure when \textit{TD-Interpreter} is answering questions.
Furthermore, when asked to envision potential use cases for \textit{TD-Interpreter}, participants suggested several helpful features, which are summarized below:
\begin{itemize}
\item Automatic interpretation of timing diagrams (21, 67.7\%);
\item Explaining step-by-step transitions in the diagram (22, 71\%);
\item Comparing multiple timing diagrams (17, 54.8\%);
\item Explaining ambiguous notations (16, 51.6\%);
\item Checking for timing violations (18, 58.1\%);
\item Generating design and verification suggestions (8, 25.8\%);
\item Providing Verilog code recommendations based on timing diagrams (9, 29\%).
\end{itemize}
Finally, we asked participants about their expectations for the first demo version of \textit{TD-Interpreter}. The top three anticipated applications are: (1) Helping with solving problems in the HDLBits website~\cite{hdlbits}, primarily because it is a well-known platform for practicing hardware design skills; (2) Answering questions related to TDs of common peripheral protocols such as UART, I2C, SPI, mainly because these protocols are widely used by junior hardware designer, and are common in embedded systems development; (3) Answering questions related to TDs of bus protocols such as AHB or AXI, as these protocols are typically encountered by more experienced engineers and involve a rich set of timing relationships.
Moreover, two senior participants noted that \textit{TD-Interpreter} would be more helpful if it could synthesize state machines from waveforms and identify potential issues related to clock-domain crossings (CDC).
% \textbf{Q15}:Do you have any other other difficulties or challenges related to timing diagrams you would like to share with us?
% \textbf{Responses:} 1. Extracting state diagrams; 2. In a synchronous design when a signal visually changes right at the clock edge (ex. drawn like in GTKWave), often I mistakenly think that this changed data is already available at that clock edge, when in reality the change is not noticable until the next clock edge. When a data change is drawn with slanted lines or a slight delay from the clock edge, it is much clearer for me; 3. Not too much related experiences of handling timing diagrams.
% \textbf{Timing Diagram Examples}
% \textbf{Q16}:Please select one timing diagram example to answer some additional questions for: ~\ref{fig:Q16}
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.5\textwidth]{Questionnare/16.png}
% \caption{Please select one timing diagram example to answer some additional questions for:}
% \label{fig:Q16}
% \end{figure}
% \textbf{Use of AI}
% \textbf{Q17}:On a scale from 1 (Not comfortable at all) to 5 (Very comfortable), how comfortable would you be if an AI tool can help you interpret timing diagrams when you do not understand certain parts?
% ~\ref{fig:Q17}
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.5\textwidth]{Questionnare/17.png}
% \caption{On a scale from 1 (Not comfortable at all) to 5 (Very comfortable), how comfortable would you be if an AI tool can help you interpret timing diagrams when you do not understand certain parts?}
% \label{fig:Q17}
% \end{figure}
% \textbf{Q18}:Would you feel more comfortable if the AI was able to explain its reasoning when answering questions related to timing diagrams?
% ~\ref{fig:Q18}
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.5\textwidth]{Questionnare/18.png}
% \caption{Would you feel more comfortable if the AI was able to explain its reasoning when answering questions related to timing diagrams?}
% \label{fig:Q18}
% \end{figure}
% \textbf{Q19}:If you answered yes in the previous question, how comfortable would you feel about relying on such an AI tool if it had the ability to explain its reasoning?
% ~\ref{fig:Q19}
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.5\textwidth]{Questionnare/19.png}
% \caption{If you answered yes in the previous question, how comfortable would you feel about relying on such an AI tool if it had the ability to explain its reasoning?}
% \label{fig:Q19}
% \end{figure}
% \textbf{Q20}:Which of the following AI-based features would you find most helpful when working with timing diagrams?
% ~\ref{fig:Q20}
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.5\textwidth]{Questionnare/20.png}
% \caption{Which of the following AI-based features would you find most helpful when working with timing diagrams?}
% \label{fig:Q20}
% \end{figure}
% \textbf{Q21}:What are your expectations for the first demo of this AI tool? For example, in which applications do you expect it to be useful?
% ~\ref{fig:Q21}
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.5\textwidth]{Questionnare/21.png}
% \caption{What are your expectations for the first demo of this AI tool? For example, in which applications do you expect it to be useful?}
% \label{fig:Q21}
% \end{figure}
\section{Dataset Construction}
\label{sec:dataset}
\begin{figure}[h]
\centering
% \vspace*{-3ex}
\includegraphics[width=0.5\textwidth]{figures/generate_workflow.pdf}
\caption{Workflow of Generating TD Pictures}
\label{fig:gen_workflow}
\end{figure}
% Fine-tuning an MLLM for interpreting TDs, requires a dataset of annotated TDs. However, no such a dataset currently exists. Thus we developed a strategy for generating a large dataset of TDs paired with question-answer pairs for visual instruction tuning. Our automatic-generation process is shown in Figure~\ref{fig:td-gen-flow}.
% %
% Each Verilog module is parsed to extract basic information, such as the module's name, input and output ports, and clock and reset ports. This information is then passed to the Verilog's Testbench Generator EDA-Utils, to generate a testbench providing random input values to the module's ports. The generated testbench is then compiled and simulated using iverilog, resulting in a \textrm{.vcd} file containing the simulation results. Finally, the wavedrom-cli tool takes this \textrm{.vcd} file and creates a timing diagram illustrating the transition of the module's signals over time.
% This strategy allowed us to generate TDs from Verilog modules, without any need for manual input. Applying its associated process on publicly available datasets, such as the one created by \cite{verilogdataset}, supports the rapid generation of TDs. What about the question-answer pairs?
% %
% \begin{wrapfigure}{l}{0.5\columnwidth}
% % \begin{figure}[htbp]
% \centering
% \vspace*{-3.5ex}
% \includegraphics[width=0.4\columnwidth]{figures/td-gen-flow.pdf}
% \vspace*{-2ex}
% \caption{Timing Diagram Generation}
% \label{fig:td-gen-flow}
% \vspace*{-1.5ex}
% % \end{figure}
% \end{wrapfigure}
% %
% The simulation results of the \textrm{.vcd} file can also be used to automatically create relatively simple question-answer pairs, which are suitable for training an MLLM to understand the basics of timing diagrams. For instance, the \textrm{.vcd} file contains for each signal $s$, its value at clock-cycle $n$. Parsing this data allows the automatic creation of the question-answer pair "\textit{What is the value of signal $s$ at clock cycle $n$?}". Additional question-answer pairs that can be automatically generated are, for example, "\textit{What is the sequence of values for the signal \textit{s}?}", "\textit{How many transitions does the signal \textit{s} have?}", and "\textit{How many rising edges does the signal \textit{s} have?}". These questions emphasize a basic understanding of signals' temporal characteristics, providing a solid foundation for more complex question-answering pairs.
Fine-tuning an MLLM for interpreting TDs, requires a dataset of annotated TDs. However, no such a dataset currently exists. Thus we developed a strategy for generating a large dataset of TDs paired with question-answer (QA) pairs for visual instruction tuning.
Figure~\ref{fig:gen_workflow} shows our workflow to generate TD pictures. The first path is to obtain a picture of a \textit{concrete TD}. This path starts with ``input 1'' in Figure~\ref{fig:gen_workflow}, where a Verilog module is parsed to extract basic information, such as its input and output ports. This information is then passed to the Testbench Generator \textit{Vlogtbgen} from EDAUtils~\cite{vlogtbgen}, to generate a testbench providing random input values to the module's ports. The generated testbench is then compiled and simulated using \textit{iverilog}~\cite{iverilog}, resulting in a \textrm{.vcd} file containing the simulation results.
We use \textit{vcd2json}~\cite{vcd2json}, a tool to parse the \textrm{.vcd} file and obtain a \textrm{.json} dictionary, which includes all the necessary graphical information within the TD. Finally, the \textit{wavedrom-cli}~\cite{wavedrom-cli} tool takes this \textrm{.json} file and creates a TD picture in a \textit{wavedrom} format. This diagrammatic format is widely used in digital design. For generating a picture of an \textit{abstract TD}, we directly start from ``input 2'' by manually providing the \textrm{.json} contents from TDs in datasheets.
Our approach for data generation incorporates two phases. In the first phase, we focus on generating caption-based data. These data are centered on providing a description for a given TD picture, aiming to help MLLM to capture the details inside the TD. The second phase focuses on generating reasoning-based data. These data are based on practical use cases, aiming to enhance MLLM's ability in reasoning about diverse timing relations inside the TD.
\subsection{Caption-based Data Generation}
\subsubsection{Approach}
The strategy shown in Figure~\ref{fig:gen_workflow} allowed us to generate TDs from Verilog modules without manual input. Applying its associated process on publicly available datasets such as~\cite{verilogdataset}, supports the rapid generation of TDs. What about the QA pairs?
The simulation results of the \textrm{.vcd} file can be used to automatically create relatively simple QA pairs, which are suitable for training an MLLM to understand the basics of TDs. For instance, the \textrm{.vcd} file contains for each signal $s$, its value at clock-cycle $n$. Parsing this data allows the automatic creation of the QA pair "\textit{What is the value of signal $s$ at clock cycle $n$?}". Additional QA pairs that can be generated are, for example, "\textit{What is the sequence of values for the signal \textit{s}?}", "\textit{How many transitions does the signal \textit{s} have?}", and "\textit{How many rising edges does the signal \textit{s} have?}". These questions emphasize a basic understanding of signals' temporal characteristics, offering a solid foundation for more complex QA pairs.
More advanced QA pairs are generally difficult to programmatically generate. However, for certain Verilog modules, we successfully generated more abstract QA pairs such as "\textit{Provide a description of what you see in this TD}". This was possible because some of the modules have highly descriptive names. By leveraging these descriptive names and the available information on the module's ports, we utilized DeepSeek-Coder-V2~\cite{deepseekai2024deepseekcoderv2breakingbarrierclosedsource}, an LLM specifically designed for code-related tasks with support for Verilog code.
Using the generated-knowledge-prompting technique~\cite{0010LLWWBCH22}, the DeepSeek-Coder-V2 builds a context around a provided Verilog module. This is achieved through chain-of-thought prompting~\cite{3600270.3602070}, where the LLM is guided by a set of sub-tasks with which it can analyse the provided module in small steps. The following prompt was used to structure this chain-of-thought-prompting process:
"\textit{You are tasked with providing brief descriptions of various digital components based on limited data. For each one, provide a description of the component. To do this perform the following steps: 1. Assume it is a Verilog module. 2. Split the module's name by the '\_' character. 3. List and analyse each individual part of the module name. 4. List and count the input and output ports. 5. Analyse the names of the input and output ports. 6.~Determine the possible functionality of each of the ports. 7.~Determine the functionality of the module based on the individual parts of the name, and the I/O ports.}".
Following this prompt, the LLM was able to provide clear and detailed responses, to prompts asking it to provide a description, a summary, and a list of use cases, for the provided Verilog module.
\begin{figure}[h]
\centering
% \vspace*{-3ex}
\includegraphics[width=0.48\textwidth]{figures/2-1-negative-mux.pdf}
\caption{TD of a 2-to-1 negative multiplexer}
\label{fig:2-1-negative-mux}
\vspace*{-2ex}
\end{figure}
An example of our training strategy, is the outcome obtained for the TD shown in Figure \ref{fig:2-1-negative-mux}. This represents a Verilog module named "\textit{maxv\_nmux21}". Through the above-described prompting process, the DeepSeek-Coder-V2 LLM successfully identified the module as a 2-to-1 negative multiplexer. It thus provided a detailed description, where it explains its purpose and its input and output ports, a fitting caption for the TD, a brief summary of the TD, and a list of use cases of the module. To highlight the need for such training data, we provided this TD to GPT-4o with a prompt asking for explanation. It only responded with a basic enumeration of the signals, and a generic explanation of how they switch between high and low. It also misidentified the A and B input signals as clock signals, and it misinterpreted the diagram as potentially representing some sort of clock signal relationship or a state machine.
\begin{figure}[h]
\centering
\vspace*{-2ex}
\includegraphics[width=0.48\textwidth]{figures/qa-pairs-chart_1.pdf}
\caption{Question Types Distribution}
\label{fig:qa-pairs-chart}
\vspace*{-1ex}
\end{figure}
\subsubsection{Statistics}
Through the outlined caption-based data-generation method, a total of 221,983 TDs were generated. For these TDs, we were able to generate 9,915,694 QA pairs, of which 4,306,008 are of the form "\textit{What is the value of signal s at clock cycle n?}", which requires an MLLM to detect concrete signal values at specific time instances. The distribution of the other questions are shown in Figure \ref{fig:qa-pairs-chart}. In addition, for 28,271 of the TDs, the corresponding Verilog module's name was sufficiently descriptive enough to generate additional captions using DeepSeek-Coder-V2. For these TDs we were also able to generate the following questions: "\textit{Provide a description/caption/summary/use case of the TD}".
% "\textit{Provide a description of what you see in this timing diagram}", "\textit{Provide a fitting caption for this timing diagram}", "\textit{List some use cases for the component shown in this timing diagram}", "\textit{Provide a fitting summary for this timing diagram}".
\subsection{Reasoning-based Data Generation}
\label{subsec:reasoning_data_generate}
Unlike caption-based data, which can be partially automated with EDA tools, generating reasoning-based data relies heavily on manual inspection of our empirical analysis in Section~\ref{sec:questionnaire}, including the selection of use cases and the reflection of typical challenges engineers face when working with TDs.
% We explored Medium/Hard problems from HDLBits~\cite{hdlbits} and classic designs like synchronous/asynchronous FIFO and classic protocols such as SPI and AXI; and for each task we conducted a case-by-case investigation, due to its unique context.
We identified a set of commonly used components in the system designs. They include memory, communication protocols, and timers. We also considered design problems from the HDLBits website~\cite{hdlbits}, which are widely recognized benchmarks to evaluate the coding generation capability of LLMs.
For each component, we manually went through every design step, and particularly, we paid a special attention on some critical timing points where designers can easily make subtle mistakes. We also intentionally crafted very specific and even unconventional questions, which require solutions depending highly on experience and the understanding of the concrete context.
To demonstrate how we generated QA pairs from a \textit{concrete TD} picture, we first use the design of a serial receiver protocol~\cite{serialprotocol}. This is an illustrative example, due to its complexity. The same methods can be extended to other use cases. We then briefly discussed the approach in generating data for \textit{abstract TDs}, which is simpler. We also provide a statistical analysis for the generated data.
\begin{figure}[h]
\centering
% \vspace*{-3ex}
\includegraphics[width=0.48\textwidth]{figures/serial_1.pdf}
\caption{Desired TDs for the Serial Receiver Protocol}
\label{fig:serial-td}
\end{figure}
\begin{figure}[h]
\centering
\vspace*{-4ex}
\includegraphics[width=0.34\textwidth]{figures/fsm.pdf}
\caption{Moore FSM for the Serial Receiver}
\label{fig:serial-fsm}
\vspace*{-2ex}
\end{figure}
\subsubsection{Generate QA pairs}
The \textit{Serial Receiver} example describes a communication protocol that receives a stream of bytes with the extra 9th bit and 10th bit for parity and stop checking respectively. The requirements for correctly receiving a byte are: \textbf{(R1)} The first 9 bits should have odd number of 1; \textbf{(R2)} The 10th bit is 1. Besides, the engineer will often see the desired TDs as shown in Figure~\ref{fig:serial-td}.
Accompanied with textual requirements, a typical design workflow requires the engineer to analyze and reason about the desired TDs. Our methods of constructing QA pairs also follow this procedure.
\paragraph{FSM design} Figure~\ref{fig:serial-fsm} shows a Finite State Machine (FSM) that represents the transaction logic of the aforementioned protocol. Figure~\ref{fig:serial-td} (A) shows the success scenario where the states transit between $S_0, S_1, S_2$, while Figure~\ref{fig:serial-td} (B) shows the failure situation where the states transit between $S_0, S_1, S_3$. As given by the textual requirements and the desired-TDs, building a partial/complete FSM is a classic problem in digital design, which we consider in particular.
\paragraph{Datapath design} As shown in Figure~\ref{fig:serial-fsm}, $data\_cnt=9$ is a critical condition for state transitions, hence the design of counter signals like $data\_cnt$ is very important. Given the desired-TD, we may ask: \textit{In which scenarios will ``data\_cnt'' become 0, keep unchanged, and add 1?} Additionally, as shown in Figure~\ref{fig:serial-td}, signal $out\_byte$ is closely related to the sequence in signal $in$. We can also ask: \textit{How does ``out\_byte'' store the bits from ``in'' sequentially}?
\paragraph{Critical timing analysis} Since engineers are often confused about specific behaviors of the waveform, this type of questions are specially designed for this scenario. They can also be used to enhance MLLM's sensitivity in combinatorial logic and sequential logic. For example, given the proposed FSM in Figure~\ref{fig:serial-fsm}, we can ask questions about the event when the protocol starts to receive the first data stream: \textit{What happens at the 3rd clk rising edge and the following one clk cycle?} The answer would provide an analysis from the FSM: \textit{At the 3rd rising edge, state=$S_0$. However, ``in=0'' in the subsequent clk cycle, which triggers ``next\_state'' to become $S_1$ through combinatorial logic. Then in the 4th clk rising edge, ``state'' will get sampled as $S_1$ through sequential logic.} Similarly, we can ask what happens when the first data stream of date finishes transmission: \textit{Please give an explanation of state transition when data\_cnt=9.}
Apart from the type of questions, we also considered their formats. For example, we changed some statement questions into True/False form, or a multiple choice form to enrich the diversity of questions.
\subsubsection{Generate Concrete TDs}
As the state transition of the FSM relies heavily on the input signal $in$, a purely random approach to generate TDs will result in unpredictable outputs, for which it is difficult to assign QA pairs. To address this challenge, we developed a \textit{top-down-conditioned-random-generation} approach, balancing between the variety of TDs, and the convenience of designing QA pairs.
\paragraph{Scenario division} This procedure divides the transaction procedure into different scenarios. As shown in Figure~\ref{fig:serial-td}, we group the traces of the FSM into two groups: success (A) and failure (B).
\paragraph{Clamp insertion} Before randomly assigning inputs, we add fixed clamps in the input sequence, as shown by Bands $a, c, d, f, g$ in both Figure~\ref{fig:serial-td} (A) and (B). Band $a$ represents a ``110'' sequence to ensure that $state$ will become $S_1$ at the 4th clk rising edge to start bit transmission. We assign 1 in Bands $c, f$ for parity checking.
\paragraph{Random generation} Bands $b, e, h$ in Figure~\ref{fig:serial-td} (A) and (B) are where we assign conditioned random inputs. For example, for Band $b$ in Figure~\ref{fig:serial-td} (A) and (B), the sequence must have even number of 1s, which makes the number of 1s in Bands $b+c$ become odd. Conversely, we require the sequence in Band $e$ to have odd number of 1s, which makes the number of 1s in Bands $e+f$ become even.
\paragraph{Motif reproduction} With the clamps and the conditioned random inputs, we can reproduce fixed motifs in the TDs. For example, as the number of 1s in Bands $b+c$ is odd, we can observe a high pulse of $odd$ in Band $k$ from both Figure~\ref{fig:serial-td} (A) and (B). In contrast, the value of $odd$ in Band $l$ is low due to the even number of 1s detected. Moreover, in Figure~\ref{fig:serial-td} (A) and (B), the chain reactions of of Bands $d, k$ and Bands $g, l$ will respectively cause fixed results in Band $i$ and Band $j$. The stable motif shown in Bands $i, j, k, l$ will thus facilitate the assignment of QA pairs.
\subsubsection{Generate Data for Abstract TDs}
% For abstract TDs, we looked at the commonly used AMBA AXI~\cite{axi}, AHB~\cite{ahb}, and APB~\cite{apb} protocols. The manuals of these protocols contain several TDs illustrating key scenarios, such as read or write transfers in APB and burst transfers in AHB. From these resources, we selected several TDs and manually crafted QA pairs for them. These QA pairs were specifically designed to address one of TD-Interpreter's primary use cases: enhancing accessibility and comprehensibility of technical documentation. For example, in the TD from Figure \ref{fig:ahb-wrapping-burst-transfer}, an engineer might wonder why address 0x3C is followed by address 0x30 rather than 0x40. To address this concern, the question \textit{"Why is address 0x3C followed by address 0x30?"} is paired with the answer \textit{"The address wraps at 16-byte boundaries because the burst is a four-beat burst of word transfers (as indicated by the HBURST and HSIZE signals)"}.
For \textit{abstract TDs}, we focused on the commonly used AMBA AXI~\cite{axi}, AHB~\cite{ahb}, and APB~\cite{apb} protocols, and also the interface design of ADCs~\cite{adc}, which are widely used in embedded systems. The manuals of these protocols contain several TDs illustrating key scenarios, such as read or write transfers in APB and burst transfers in AHB. From these resources, we selected several TDs and manually crafted QA pairs for them. These QA pairs were specifically designed to address common engineering challenges. Given the small scale of these \textit{abstract TDs}, we adopted a lightweight method for randomizing their presentation, including altering signal order, varying clock cycles, and changing letter case.
% of TD-Interpreter's primary use cases: enhancing accessibility and comprehensibility of technical documentation. For example, in the TD from Figure \ref{fig:ahb-wrapping-burst-transfer}, an engineer might wonder why address 0x3C is followed by address 0x30 rather than 0x40. To address this concern, the question \textit{"Why is address 0x3C followed by address 0x30?"} is paired with the answer \textit{"The address wraps at 16-byte boundaries because the burst is a four-beat burst of word transfers (as indicated by the HBURST and HSIZE signals)"}.
% \begin{figure}[h]
% \centering
% % \vspace*{-3ex}
% \includegraphics[width=0.48\textwidth]{figures/ahb-wrapping-burst-transfer.pdf}
% \caption{AHB Wrapping Burst Write Transfer}
% \label{fig:ahb-wrapping-burst-transfer}
% \end{figure}
\subsubsection{Statistics}
\begin{table}[h]
\caption{\# Types of Questions}
\vspace{4ex}
\centering
\small
\resizebox{0.48\textwidth}{!}{
\begin{tabular}{l|c|l|c}
\hline
\textbf{Task} & \textbf{\# Type} & \textbf{Task} & \textbf{\# Type} \\ \hline
serial\_datapath\_stop~\cite{serialdatapath} & 24 & motor\_timer\_failure~\cite{motor_timer} & 13 \\ \hline
serial\_datapath\_wait~\cite{serialdatapath} & 24 & complete\_fsm~\cite{complete_fsm} & 13 \\ \hline
serial\_parity\_stop~\cite{serialprotocol} & 22 & fancy\_timer~\cite{fancy_timer} & 16 \\ \hline
serial\_parity\_wait~\cite{serialprotocol} & 22 & sync\_fifo & 15 \\ \hline
HDLC\_correct~\cite{hdlc} & 13 & async\_fifo & 14 \\ \hline
HDLC\_discard~\cite{hdlc} & 12 & spi\_no\_clk & 25 \\ \hline
HDLC\_error~\cite{hdlc} & 13 & spi\_with\_clk & 33 \\ \hline
w\_counter~\cite{wcounter} & 10 & ADC~\cite{adc} & 12 \\ \hline
motor\_timer\_success~\cite{motor_timer} & 13 & AXI/AHB/APB~\cite{axi,ahb,apb} & 16 \\ \hline
\end{tabular}
}
\label{tab:questions}
\end{table}
\begin{figure}[h]
\centering
% \vspace*{-3ex}
\includegraphics[width=0.32\textwidth]{figures/question.pdf}
\caption{Question Distribution}
\label{fig:question_distribute}
\end{figure}
Table~\ref{tab:questions} showcases all the design tasks we considered. Overall, we designed 310 different types of questions. Figure~\ref{fig:question_distribute} further shows their distribution. Some tasks have a specific type of questions. For the ``async\_fifo'' case, the mostly asked questions are related to clock-domain crossing (CDC), as shown from the second example in Figure~\ref{fig:chatbot}. We also find that most questions come from the groups of simple-timing analysis (STA) and complex-timing analysis (CTA). STA questions require an explanation of events for a single signal, or for a single clock cycle, while CTA questions require an answer about the relations across different signals. For design cases in practice, such as ``spi\_no\_clk'' (the slave has no clock) and ``spi\_with\_clk'' (the slave has an independent clock), the complexity escalates significantly, so most of the questions fall in the CTA group. In Section~\ref{subsec:exp_results} we provided concrete examples for several types of questions.
\section{The TD-Interpreter}\label{sec:method}
\subsection{Architecture}\label{subsec:model_architecture}
\begin{figure}[htbp]
\centering
\includegraphics[width=0.48\textwidth]{figures/model.png}
\caption{Model Architecture of TD-Interpreter}
\label{fig:arch}
\end{figure}
As shown in Figure~\ref{fig:arch}, \textit{TD-Interpreter} is a fine-tuned LLaVA model~\cite{liu2023visualinstructiontuning} developed using our dataset. It incorporates a
vision encoder $g(\cdot)$ based on the vision transformer (ViT)~\cite{dosovitskiy2020image}, which enables the model to comprehend image information, alongside a language model $f(\cdot)$ that generates textual responses based on both the visual input and the textual request. \textit{TD-Interpreter} takes an image of a TD $\mathbf{X}_v\in \mathbb{R}^{H\times W\times C}$ and a textual question $\mathbf{X}_q$ as the inputs, where $H \times W$ represents the resolution of $\mathbf{X}_v$, and $C=3$ represents the number of RGB channels in the TD picture. The vision encoder $g(\cdot)$ first extracts the visual features from the input image as $\mathbf{h}_v = g(\mathbf{X}_v)$.
% Subsequently, the language model predicts the next token of the target textual response $\mathbf{X}_a$ by producing the logits $\mathbf{h}(t)$ based on the visual embedding $\mathbf{h}_v$ and the textual input sequence of tokens $\mathbf{x}_t$ converted from $\mathbf{X}_q$, represented as $\mathbf{h}_t = f(\mathbf{x}_t, \mathbf{h}_v)$.
Subsequently, the language model predicts the next token of the target textual response $\mathbf{X}_a$ from the input embedding $\mathbf{e}_t$ based on the visual embedding $\mathbf{h}_v$ and the textual input sequence of tokens $\mathbf{x}_t$ converted from $\mathbf{X}_q$, represented as $\mathbf{e}_t = f(\mathbf{h}_v, \mathbf{x}_t)$.
\subsubsection{Vision Encoder}\label{subsec:vision_encoder}
The ViT-based~\cite{dosovitskiy2020image} vision encoder $g(\cdot)$ consists of a sequential stack of $n_v$ identical transformer blocks $g_i(\cdot), 1 \le i\le n_v$, along with an input embedding layer $e_v(\cdot)$, and an output linear layer. Each block includes a multi-head self-attention (MSA) layer, a multi-layer-perceptron (MLP) layer, and two Layernorm (LN)~\cite{ba2016layer} layers. The detailed encoding of the input TD image $\mathbf{X}_v$ can be represented as follows.
First, the image $\mathbf{X}_v$ is patchified into a sequence of flattened 2D patches represented as
$\mathbf{x}_v \in\mathbb{R}^{N\times (P^2 \cdot C)}$, where $N=(H\times W)/P^2$ is the number of patches, and $P \times P$ indicates the resolution of each patch. Then, the sequence of patches is encoded by the input layer $e_v$ as follows: $\mathbf{z}_p^0 = e_v(\mathbf{x}_v)$.
Subsequently, the sequence $\mathbf{z}_p^0$ is processed by the transformer blocks $g_i(\cdot), 1 \le i\le n_v$, where the encoding procedure of the $i$-th block $\mathbf{z}_p^{i} = g_i(\mathbf{z}_p^{i-1})$ can be denoted as follows: $\hat{\mathbf{z}}_p^{i-1} = \text{MSA}(\text{LN}(\mathbf{z}_p^{i-1})) + \mathbf{z}_p^{i-1}$, and $\mathbf{z}_p^{i} = \text{MLP}(\text{LN}(\hat{\mathbf{z}}_p^{i-1})) + \hat{\mathbf{z}}_p^{i-1}$.
Finally, we get the visual feature $\mathbf{h}_v = \text{Linear}(\mathbf{z}_p^{n_v})$ by transforming $\mathbf{z}_p^{n_v}$ using the last linear layer of the vision encoder.
\subsubsection{Language Model}\label{subsec:language_model}
The language model $f(\cdot)$ is a pre-trained large-language model (LLM) based on the LLaMA architecture~\cite{touvron2023llama}, parameterized by a Vicuna~\cite{chiang2023vicuna} model as described in~\cite{liu2023visualinstructiontuning}. Specifically, $f(\cdot)$ consists of an input embedding layer $e_t(\cdot)$, $n_t$ identical transformer blocks $f_i(\cdot), 1\le i \le n_t$, and an output block comprising an RMSNorm (RN)~\cite{zhang2019root} layer and a linear layer. Given an textual input sequence of tokens $\mathbf{x}_t$ converted from the request $\mathbf{X}_q$, it is first encoded by the input layer $e_t(\cdot)$ as follows: $\mathbf{z}_t^0 = e_t(\text{Concatenate}(\mathbf{x}_t, \mathbf{h}_v))$.
Then, $\mathbf{z}_t^0$ is encoded by $n_t$ transformer blocks, where the encoding procedure for the $i$-th block $\mathbf{z}_t^{i} = f_i(\mathbf{z}_t^{i-1})$ can be represented by $\hat{\mathbf{z}}_t^{i-1} = \text{MSA}(\text{RN}(\mathbf{z}_t^{i-1}))+\mathbf{z}_t^{i-1}$ and $\mathbf{z}_t^i = \text{MLP}(\text{RN}(\hat{\mathbf{z}}_t^{i-1})) + \hat{\mathbf{z}}_t^{i-1}$.
Finally, the predicted logits $\mathbf{h}_t$ can be expressed by $\mathbf{h}_t = \text{Linear}(\text{RN}(\mathbf{z}_t^{n_t}))$. By employing the \textit{softmax} on $\mathbf{h}_t$ and extracting the most likely token in each decoding step, we get the predicted next token of target textual response $\mathbf{X}_a$.
\subsection{Training}\label{subsec:training}
As detailed in Section~\ref{sec:dataset}, for each TD image $\mathbf{X}_v$, we construct a single-turn conversation data $(\mathbf{X}_q, \mathbf{X}_a)$. Then, for a target answer sequence $\mathbf{X}_a$ of length $l$, we compute the probability of it by: $p(\mathbf{X}_a|\mathbf{X}_v, \mathbf{X}_q) = \prod_{i=1}^l p_\theta(x_i|\mathbf{X}_v, \mathbf{X}_{q}, \mathbf{X}_{a,<i})$.
Here, we fix the parameters of the vision encoder $g(\cdot)$ while leaving the parameters of the language model $f(\cdot)$ trainable, denoted by $\theta$. Additionally, $\mathbf{X}_{q,<i}$ and $\mathbf{X}_{a,<i}$ represent the tokens of the question and the generated answer before the the current $i$-th token.
To enhance the speed and efficiency of training, we employ the low-rank-adaptation (LoRA) technique~\cite{hu2021lora}, on all layers of our TD-interpreter, and conduct fine-tuning. Intuitively, LoRA introduces a trainable adapter that contains significantly fewer parameters than the original weights. By training only the lightweight adapter instead of the full weight matrix, LoRA substantially alleviates the training burden.
Specifically, given a pre-trained weight matrix $W_0 \in \mathbb{R}^{d\times k}$, where $d\times k$ is the shape of the matrix, LoRA modifies the weight matrix from $W_0$ to $W_0 + \Delta W = W_0+B \cdot A$, where $A\in \mathbb{R}^{r\times k}$ and $B\in \mathbb{R}^{d\times r}$ are two trainable low-rank matrices with $r \ll \min(d, k)$. This technique effectively reduces the number of trainable parameters for each layer from $d \times k$ to $r \times (d + k)$.
\section{Experimental Evaluation}
\label{sec:evaluate}
\subsection{Training details.}
We first prepared an evaluation dataset as a benchmark for testing. This dataset covered almost all use cases in Table~\ref{tab:questions}. It also contained all types of questions in Figure~\ref{fig:question_distribute} that represented various cause-effect relations between signals. For each question-answer pair, due to random generation, all TD pictures looked different and were never seen during training. The most complex picture included up to 51 signals and 80 clock cycles. As illustrated in Section~\ref{subsec:reasoning_data_generate}, these data were first created by humans then automated for random generation, so evaluation on these data is able to manifest the model’s performance across different real use cases and question types.
For the training data set, although the use of our data generator described in Section~\ref{sec:dataset} is capable of generating millions of synthetic data, we found that it was not necessary to train the total amount of data, and this was not economical for institutes or companies with limited computation resources. We empirically found it would yield desirable evaluation results if the overall amount of training data reached around 10k. Consequently, for the caption-based data, we randomly sampled 4942 TD pictures and associated each picture with one QA pair; For the reasoning-based data, we randomly sampled 5292 TD pictures, with each picture a single QA pair. We then were able to train our model on 8 NVIDIA A800 GPUs with a batch size of 32. The training process takes 17 hours with 100 epochs. We utilized a learning rate of $10^{-4}$ and set the LoRA rank to 8. The LoRA adapter consists of 20M parameters.
\subsection{Experimental results.}
\label{subsec:exp_results}
As the outputs of the Visual QA task are texts, and the ground truth is also given in form of texts, we can use commonly used metrics in text comparison, such as Bleu-x and Rouge-x to evaluate the accuracy of model outputs. ``-x'' represents different approaches in calculating the specific metrics. The range of Bleu-x and Rouge-x are between 0 and 100, while 100 means the model outputs completely overlap the labelled answers, and 0 means the worst case. In this paper we only focus on the absolute values of Bleu-x and Rouge-x.
We first evaluated the untuned LLaVA and got the following results: Bleu-4=13.9, Rouge-1/2/l=23.2/6.6/16.3. In comparison, after finetuning, LLaVA can promisingly achieve: Bleu-4=95.9, Rouge-1/2/l=96.7/95.9/96.5. The significance of these comparison results is two-fold: (1) Before finetuning, LLaVA has very limited capability in understanding TDs, and after finetuning, there was a huge improvement in its performance; (2) We observe that after finetuning, LLaVA can achieve over 95 scores across all metrics of Bleu-x and Rouge-x. This means that for all the questions in the evaluation dataset, LLaVA can answer them correctly, except only some small errors.
In the following, we provide four illustrative examples showing the capabilities of \textit{TD-Interpreter}. After each index of question, we added its question type that mapped to Figure~\ref{fig:question_distribute}. We selected relatively more complex questions, such as questions in groups of FSM, CDC and CTA, to showcase the reasoning capability of \textit{TD-Interpreter} in complicated timing relations. Some questions require unconventional answers as they came from specific design issues. We also compared our results with the ones of GPT-4o. Texts in blue show where GPT-4o made mistakes. Moreover, we provided human feedback to reflect the typical problems that designers may face.
\subsubsection{AHB} The TD in Figure \ref{fig:ahb-burst-transfer} is a burst-write-transfer example using the advanced-high-performance-bus (AHB) specification.
\begin{figure}[htbp]
\centering
\includegraphics[width=1\columnwidth]{figures/ahb-burst-transfer.pdf}
\caption{AHB Burst Write Transfer}
\label{fig:ahb-burst-transfer}
\end{figure}
\begin{itemize}
\item \textbf{Q1 (CTA):} Why takes writing to address 0x3C longer than other transfers?
\item \textbf{A1 (TD-Interpreter):} <<{\color{blue}The write to address 0x3C does not take longer. Instead it's the write to address 0x38 which takes longer.} One of the subordinates adds a wait state in this transfer, as indicated by the HREADY signal being LOW. This extends the address phase of the transfer after it. In this case this is the write transfer to address 0x3C. {\color{blue}In summary, the data phase of the transfer to address 0x38 is including a wait state, which extends the address phase of the transfer to address 0x3C.}>>
\item \textbf{A1 (GPT-4o):} GPT-4o notices that signal HREADY is low during the write transfer. However, it fails to see that this occurs due to the wait states inserted during the previous transfer. It then falsely identifies the write to address 0x3C as the first transfer in the burst.
\item \textbf{Human feedbcak:} In our empirical study, five participants who claimed to be familiar with AHB protocol tried to analyze this TD. However, only 1 person correctly analyzed the timing behavior as asked in \textbf{Q1}, while most participants simply said ``don't know''. The average confidence level is also low, which is only 2.2/5 (1 means not confident, 5 means very confident).
\end{itemize}
\begin{figure}[htbp]
\centering
\includegraphics[width=1\columnwidth]{figures/count_fsm_new.pdf}
\caption{Fancy Timer}
\label{fancyTimer}
\end{figure}
\subsubsection{Fancy Timer} The TD in Figure~\ref{fancyTimer} shows a module of a fancy timer, satisfying requirements: \textit{(R1)} Get started when a particular pattern (1101) is detected, \textit{(R2)} Shift in 4 more bits to determine the duration to delay, \textit{(R3)} Wait for the counters to finish counting, \textit{(R4)}~Notify the user and wait for the user to acknowledge the timer.
\begin{itemize}
\item \textbf{Q1 (FSM):} Can you synthesize a Moore FSM?
% \item \textbf{A1}: \textbf{(R1)} S0-x=1->S1, S0-x=0->S0, S1-x=1->S2, S1-x=0->S0,S2-x=0->S3, S2-x=1->S2, S3-x=1->S4, S3-x=0->S0; \textbf{(R2)} S4-rst=0->S5, S4-rst=1->S0, S5-rst=0->S6, S5-rst=1->S0,S6-rst=0->S7, S6-rst=1->S0, S7-rst=0->S8, S7-rst=1->S0; \textbf{(R3)} S8-done=1->S9, S8-done=0->S8; \textbf{(R4)} S9-ack=1->S0, S9-ack=0->S9; \textbf{(Out)} S4,S5,S6->shift\_ena=1, S8->counting=1, S9->done=1.
\item\textbf{A1 (TD-Interpreter):} <<{\color{blue}\textit{(R1)} $s_0$-\textit{x}->$s_1$, $s_0$-$\sim$\textit{x}->$s_0$, $s_1$-\textit{x}->$s_2$, $s_1$-$\sim$\textit{x}->$s_0$, $s_2$-\textit{x}->$s_2$, $s_2$-$\sim$\textit{x}->$s_3$, $s_3$-\textit{x}->$s_4$, $s_3$-$\sim$\textit{x}->$s_0$; \textit{(R2)} $s_4$-$\sim$\textit{rst}->$s_5$, $s_4$-\textit{rst}->$s_0$, $s_5$-$\sim$\textit{rst}->$s_6$, $s_5$-\textit{rst}->$s_0$, $s_6$-$\sim$\textit{rst}->$s_7$, $s_6$-\textit{rst}->$s_0$, $s_7$-$\sim$\textit{rst}->$s_8$, $s_7$-\textit{rst}->$s_0$; \textit{(R3)} $s_8$-\textit{finish}->$s_9$, $s_8$-$\sim$\textit{finish}->$s_8$; \textit{(R4)} $s_9$-\textit{ack}->$s_0$, $s_9$-$\sim$\textit{ack}->$s_9$; \textit{(Out)} $s_4 \sim s_7$->\textit{shift\_ena}=1, $s_8$->\textit{counting}=1, $s_9$->\textit{done}=1.}>>
% \item \textbf{Q2}: Suppose at each clk rising edge, ``count'' keeps checking if shift\_ena=1. If yes, it will execute: count <= \{count[2:0], data\}. Why ``count'' changes from 0 to 1 at the 13th clk rising edge?
% \item \textbf{A2}: At the 13th clk rising edge, count[2:0]=3b'000, data=1. As shift\_ena is 1, according to ``count <= \{count[2:0], data\}'', the new value of ``count'' will be 4'b0001, namely 1.
\item \textbf{A1 (GPT-4o):} A very high-level design of the FSM.
\vspace*{1mm}\item \textbf{Q2 (Counter):} In which clk cycle do both ``cnt'' and ``count'' equal to 0, and why in the next clk cycle the timer immediately stops working?
\item \textbf{A2 (TD-Interpreter)}: <<{\color{blue} Look at the 20th clk cycle. During this cycle, both ``cnt'' and ``count'' equal to 0.} This could be used as a triggering signal to set ``finish'' as 1.
Since current state is $s_8$, and according to the FSM, if $finish=1$ in this state, ``next\_state'' will be $s_9$.
Then at the 21st clk rising edge, ``state'' gets sampled as $s_9$. This will disable signal ``count'', and enable signal ``done''>>.
\item \textbf{A2 (GPT-4o):} Fails to locate the correct clk cycle, where both signals ``cnt'' and ``count'' have a value equal to 0.
\item \textbf{Human feedback:} We did not ask participants to analyze this TD as it takes time even for human experts to synthesize a FSM from this TD. However, as mentioned in Section~\ref{sec:questionnaire}, it is a practical need to generate FSMs out of waveforms, so it would save considerable time for engineers when \textit{TD-Interpreter} provides this function.
\end{itemize}
% \begin{figure}[htbp]
% \centering
% \includegraphics[width=1\columnwidth]{figures/afifo.pdf}
% \caption{Asynchronous FIFO}
% \end{figure}
\subsubsection{Asynchronous FIFO} The TD in Figure~\ref{fig:chatbot}(B) shows an asynchronous FIFO module with $depth=4$.
\begin{itemize}
% \item \textbf{Q1}: Observe the following five registers: \textit{wr\_exp}, \textit{wr\_exp\_r}, \textit{wr\_exp\_cross}, \textit{wr\_exp\_cross\_r}, \textit{wr\_exp\_cross\_trans}. Describe how the values of these five registers are propagated with gray encoding/decoding in different clock domains.
% \item \textbf{A1:} grayencode(\textit{wr\_exp}) -> \textit{wr\_clk} -> \textit{wr\_exp\_r} -> \textit{rd\_clk} -> \textit{wr\_exp\_cross} -> \textit{rd\_clk} -> \textit{wr\_exp\_cross\_r} -> \textit{wr\_exp\_cross\_trans} = graydecode(\textit{wr\_exp\_cross\_r}).
\item \textbf{Q1 (CDC):} Is the following relation correct?
grayencode(\textit{wr\_exp}) -> \textit{rd\_clk} -> \textit{wr\_exp\_r} -> \textit{wr\_clk} -> \textit{wr\_exp\_cross} -> \textit{wr\_clk} -> \textit{wr\_exp\_cross\_r} -> \textit{wr\_exp\_cross\_trans} = graydecode(\textit{wr\_exp\_cross\_r}).
\item \textbf{A1 (TD-Interpreter):} <<{\color{blue} No, the correct relation should exchange \textit{rd\_clk} and \textit{wr\_clk}.}>>
\item \textbf{A1 (GPT-4o):} Sometimes GPT-4o thought the provided relation is correct, and sometimes GPT-4o thought it is wrong, but it tended to provide an incorrect solution.
\vspace*{1mm}\item \textbf{Q2 (CDC):} Observe the first reading operation, and answer: \textit{(1)} Does ``wr\_num'' represent the real number of data inside the FIFO? \textit{(2)} Does ``rd\_num'' represent the real number of data inside the FIFO?
\item \textbf{A2 (TD-Interpreter):} <<\textit{(1)} No. ``wr\_num'' works in the ``wr\_clk'' domain, so in scenarios where consecutive reading operation happens, ``wr\_num'' cannot reflect the real number of data inside FIFO.
\textit{(2)} {\color{blue} Yes. From the TD, $rd\_num=wr\_exp\_cross\_trans-rd\_exp$. }``rd\_exp'' has no information lag in the ``rd\_clk'' domain, but ``wr\_exp\_cross\_trans'' has. However, before the reading operation, ``wr\_exp\_cross\_trans'' has already reached the correct value 4, and remains unchanged during reading. {\color{blue} Hence, ``wr\_exp\_cross\_trans'' is correct in this case.}>>
\item \textbf{A2 (GPT-4o):} In most cases ``rd\_num'' cannot reflect the number of data inside the FIFO, but the particular case shown in \textit{Asynchronous FIFO} is an exception. GPT-4o simply provided a negative answer with textbook-style explanations, and the way it proposed to compute ``rd\_num'' was wrong.
\item \textbf{Human feedback:} Six participants in our empirical study tried to analyze this TD. First, they all agreed that they may ask the questions in Figure~\ref{fig:chatbot}(B) to gain a basic understanding of this asynchronous FIFO module. However, when it came to questions about identifying clock domain crossing (CDC) issues, these participants made mistakes, despite that FIFO was known to be basic for digital designers. For example, in \textbf{Q1}, only two people answered correctly, while the rest four people even had no clues. In \textbf{Q2}, four out of six people provided the correct answer for ``rd\_num''; For ``wr\_num'', the situation was worse, only one person answered correctly.
\end{itemize}
\begin{figure}[htbp]
\centering
\includegraphics[width=1\columnwidth]
{figures/spi_no_clk_new.pdf}
\caption{Complete SPI protocol (slave has no internal clock)}
\label{spiProtocol}
\end{figure}
\subsubsection{SPI Protocol} The TD in Figure~\ref{spiProtocol} shows a complete SPI protocol, with a master module, a slave module, and a testbench.
\begin{itemize}
\item \textbf{Q1 (CTA):} Please describe the transaction procedure of the protocol.
\item \textbf{A1 (TD-Interpreter):} <<\textit{(1) Master Write}: {\color{blue}The testbench first reads data $D$ from address $X$ in the master's memory, then writes data $D$ to address $X$ in the slave's memory.} \textit{(2) Slave Feedback}: {\color{blue} The testbench takes data from address $X$ in slave's memory,} and sends back to the master to check if the written operation is correct.>>
\item \textbf{A1 (GPT-4o):} Only describes the general transaction procedure of the SPI protocol, while overlooking the particular role of the testbench in transferring data between the master and the slave.
\vspace*{1mm}\item \textbf{Q2 (CTA):} How does ``sending\_tmp'' in the master get address $X$ and $D$ from the testbench? How does ``MOSI'' transmit bits sequentially?
% \item \textbf{A2:} When ``m\_cnt'' changes from 0 to 1, ``sending\_tmp'' will read address $X$ from ``master\_addr'' . When ``m\_cnt'' changes from 16 to 17, the high level of ``wdat\_req\_r'' is used as a triggering condition for the master to latch data from ``wdat'' to ``sending\_tmp''.``MOSI'' transmits the ``sending\_tmp'' bit at each falling edge of ``SLCK''.
\item \textbf{A2 (TD-Interpreter):} <<When ``m\_cnt'' changes from 0 to 1, ``sending\_tmp'' will read address $X$ from ``master\_addr''. {\color{blue} When ``m\_cnt'' changes from 16 to 17, by capturing the falling edge of ``wdat\_req\_r'', the master can then latch data from ``wdat'' to ``sending\_tmp''.} ``MOSI'' transmits the ``sending\_tmp'' bit at each falling edge of ``SLCK''.>>
\item \textbf{A2 (GPT-4o):} Wrongly captures the falling edge of ``wdat\_req'' rather than ``wdat\_req\_r'' when ``m\_cnt'' changes from 16 to 17.
\vspace*{1mm}\item \textbf{Q3 (CTA):} How does the ``MISO'' signal inside the slave take the data in ``slave\_out\_dat'' from the testbench?
\item \textbf{A3 (TD-Interpreter):} <<This occurs during \textit{Slave Feedback}. At the falling edge of ``dat\_rcv\_clk'' while ``s\_cnt'' equals to 9, ``slave\_out\_dat'' begins to retrieve data $D$ from address $X$ in the slave’s memory. {\color{blue} Then ``MISO'' can exploit the second half duration of s\_cnt=9, to sample the highest bit of ``slave\_out\_dat'' via combinatorial logic.} Next, ``MISO'' can use the entire duration of s\_cnt=10 to sample the secondly highest bit, until the first half duration of s\_cnt=17 where "MISO" samples the lowest bit of ``slave\_out\_dat''. Since the slave has no internal clock, {\color{blue} ``MISO'' has to use combinatorial logic.}>>
\item \textbf{A3 (GPT-4o):} The signal ``MISO'' is normally driven by sequential logic, but the SPI protocol here shows that using combinatorial logic to drive the signal ``MISO'' is more reasonable. However, GPT-4o is still insisting on using sequential logic.
\item \textbf{Human feedback:} Four participants in our empirical study tried to analyze this TD. Although SPI is a basic peripheral protocol in embedded systems, when the participants were asked to rate the complexity of this TD, the average score reached 3.5/5 (1 means very simple, and 5 means very complex), indicating that they were confused with many details. We thus assumed most of them only used SPI in a high level, as when they had to deal with its exact transaction procedure as asked in \textbf{Q1}, they rated 3.5/5 (1 means not helpful, 5 means very helpful) for the need of interpretation. We further asked if they can identify the concrete value of $X$ and $D$ as mentioned in (\textbf{A1}), only one person answered correctly. Interestingly, only half of the participants found out the lack of internal clock in the slave module might cause problem, which means the slave module has to be driven by combinatorial logic.
\end{itemize}
\subsection{Discussion and Limitation.} From the experiments, we observed that GPT-4o faced challenges in analyzing detailed information within the TDs. For certain specific design scenarios, GPT-4o often provided general, textbook-style answers without conducting case-by-case analysis. This proves the usefulness and effectiveness of \textit{TD-Interpreter}.
One limitation of this paper is that we adopted the \textit{wavedrom} plotting style of TDs, due to its simplicity and widespread use. Our main focus was on \textit{enabling} MLLMs to interpret TDs. Thus, developing front-end tools supporting a more diverse set of TD representations is necessary. Additionally, due to resource constraints, we were unable to construct a dataset with a larger scale. However, the most significant contribution of this work is that we provided the industry, particularly its AI-enabling departments, with a very expressive, IP-protecting, low-cost approach. In addition, using MLLMs to assist engineers in understanding TDs, may definitely appeal to many R\&D teams. Furthermore, integrating backend verification in dataset construction, and improving reasoning capability of MLLMs with reinforcement learning, are both promising directions for future research.
\section{Conclusion}
\label{sec:conclusion}
We have introduced \textit{TD-Interpreter}, a multi-modal large language model, obtained by fine-tuning LLaVA on our generated dataset of TDs and query-answer pairs. \textit{TD-Interpreter} can answer natural language questions about design and verification in timing diagrams provided as pictures. We show that \textit{TD-Interpreter} outperforms GPT-4o, and consequently the untuned LLaVA model, too, on a rich set of tasks.
% Generated by IEEEtran.bst, version: 1.14 (2015/08/26)
\begin{thebibliography}{10}
\providecommand{\url}[1]{#1}
\csname url@samestyle\endcsname
\providecommand{\newblock}{\relax}
\providecommand{\bibinfo}[2]{#2}
\providecommand{\BIBentrySTDinterwordspacing}{\spaceskip=0pt\relax}
\providecommand{\BIBentryALTinterwordstretchfactor}{4}
\providecommand{\BIBentryALTinterwordspacing}{\spaceskip=\fontdimen2\font plus
\BIBentryALTinterwordstretchfactor\fontdimen3\font minus \fontdimen4\font\relax}
\providecommand{\BIBforeignlanguage}[2]{{%
\expandafter\ifx\csname l@#1\endcsname\relax
\typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
\typeout{** loaded for the language `#1'. Using the pattern for}%
\typeout{** the default language instead.}%
\else
\language=\csname l@#1\endcsname
\fi
#2}}
\providecommand{\BIBdecl}{\relax}
\BIBdecl
\bibitem{adc}
``Adc129s022,'' \url{https://www.ti.com/lit/ds/symlink/adc128s022.pdf}.
\bibitem{liu2023visualinstructiontuning}
\BIBentryALTinterwordspacing
H.~Liu, C.~Li, Q.~Wu, and Y.~J. Lee, ``Visual instruction tuning,'' 2023. [Online]. Available: \url{https://arxiv.org/abs/2304.08485}
\BIBentrySTDinterwordspacing
\bibitem{liu2023verilogeval}
M.~Liu, N.~Pinckney, B.~Khailany, and H.~Ren, ``Verilogeval: Evaluating large language models for verilog code generation,'' in \emph{Proc. of ICCAD 2023: IEEE/ACM International Conference on Computer Aided Design}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2023, pp. 1--8.
\bibitem{liu2023chipnemo}
M.~Liu, T.-D. Ene, R.~Kirby, C.~Cheng, N.~Pinckney, R.~Liang, J.~Alben, H.~Anand, S.~Banerjee, I.~Bayraktaroglu \emph{et~al.}, ``Chipnemo: Domain-adapted llms for chip design,'' \emph{arXiv preprint arXiv:2311.00176}, 2023.
\bibitem{thakur2023benchmarking}
S.~Thakur, B.~Ahmad, Z.~Fan, H.~Pearce, B.~Tan, R.~Karri, B.~Dolan-Gavitt, and S.~Garg, ``Benchmarking large language models for automated verilog rtl code generation,'' in \emph{Proc. of DATE 2023: Design, Automation \& Test in Europe Conference \& Exhibition}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2023, pp. 1--6.
\bibitem{thakur2024verigen}
S.~Thakur, B.~Ahmad, H.~Pearce, B.~Tan, B.~Dolan-Gavitt, R.~Karri, and S.~Garg, ``Verigen: A large language model for verilog code generation,'' \emph{ACM Transactions on Design Automation of Electronic Systems}, vol.~29, no.~3, pp. 1--31, 2024.
\bibitem{iccad}
``Llm4hwdesign,'' \url{https://nvlabs.github.io/LLM4HWDesign/}.
\bibitem{touvron2023llama}
H.~Touvron, L.~Martin, K.~Stone, P.~Albert, A.~Almahairi, Y.~Babaei, N.~Bashlykov, S.~Batra, P.~Bhargava, S.~Bhosale \emph{et~al.}, ``Llama 2: Open foundation and fine-tuned chat models,'' \emph{arXiv preprint arXiv:2307.09288}, 2023.
\bibitem{nijkamp2022codegen}
E.~Nijkamp, B.~Pang, H.~Hayashi, L.~Tu, H.~Wang, Y.~Zhou, S.~Savarese, and C.~Xiong, ``Codegen: An open large language model for code with multi-turn program synthesis,'' \emph{arXiv preprint arXiv:2203.13474}, 2022.
\bibitem{fu2023gpt4aigchip}
Y.~Fu, Y.~Zhang, Z.~Yu, S.~Li, Z.~Ye, C.~Li, C.~Wan, and Y.~C. Lin, ``Gpt4aigchip: Towards next-generation ai accelerator design automation via large language models,'' in \emph{Proc. of ICCAD 2023: IEEE/ACM International Conference on Computer Aided Design}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2023, pp. 1--9.
\bibitem{chang2024natural}
K.~Chang, Z.~Chen, Y.~Zhou, W.~Zhu, H.~Xu, C.~Li, M.~Wang, S.~Liang, H.~Li, Y.~Han \emph{et~al.}, ``Natural language is not enough: Benchmarking multi-modal generative ai for verilog generation,'' \emph{arXiv preprint arXiv:2407.08473}, 2024.
\bibitem{he2023td}
J.~He, D.~Ni{\v{c}}kovi{\'c}, E.~Bartocci, and R.~Grosu, ``Td-magic: From pictures of timing diagrams to formal specifications,'' in \emph{Proc. of DAC 2023: 60th ACM/IEEE Design Automation Conference}.\hskip 1em plus 0.5em minus 0.4em\relax IEEE, 2023, pp. 1--6.
\bibitem{Bhatnagar2002}
\BIBentryALTinterwordspacing
\emph{Advanced ASIC Chip Synthesis Using Synopsys$^{\textregistered}$ Design Compiler{\texttrademark} Physical Compiler{\texttrademark} and PrimeTime$^{\textregistered}$}.\hskip 1em plus 0.5em minus 0.4em\relax Boston, MA: Springer US, 2002, ch. Static Timing Analysis, pp. 261--303. [Online]. Available: \url{https://doi.org/10.1007/0-306-47507-3_13}
\BIBentrySTDinterwordspacing
\bibitem{maler2004monitoring}
O.~Maler and D.~Nickovic, ``Monitoring temporal properties of continuous signals,'' in \emph{International symposium on formal techniques in real-time and fault-tolerant systems}.\hskip 1em plus 0.5em minus 0.4em\relax Springer, 2004, pp. 152--166.
\bibitem{fisler1999timing}
F.~Kathi, ``Timing diagrams: Formalization and algorithmic verification,'' \emph{J. Log. Lang. Inf.}, vol.~8, no.~3, pp. 323--361, 1999.
\bibitem{amla1999efficient}
N.~Amla, E.~A. Emerson, and K.~S. Namjoshi, ``Efficient decompositional model checking for regular timing diagrams,'' in \emph{Proc. of CHARME}, ser. LNCS, vol. 1703.\hskip 1em plus 0.5em minus 0.4em\relax Springer, 1999, pp. 67--81.
\bibitem{bartocci2022survey}
E.~Bartocci, C.~Mateis, E.~Nesterini, and D.~Nickovic, ``Survey on mining signal temporal logic specifications,'' \emph{Information and Computation}, vol. 289, p. 104957, 2022.
\bibitem{chen2024internvl}
Z.~Chen, J.~Wu, W.~Wang, W.~Su, G.~Chen, S.~Xing, M.~Zhong, Q.~Zhang, X.~Zhu, L.~Lu \emph{et~al.}, ``Internvl: Scaling up vision foundation models and aligning for generic visual-linguistic tasks,'' in \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 2024, pp. 24\,185--24\,198.
\bibitem{DosovitskiyB0WZ21}
\BIBentryALTinterwordspacing
A.~Dosovitskiy, L.~Beyer, A.~Kolesnikov, D.~Weissenborn, X.~Zhai, T.~Unterthiner, M.~Dehghani, M.~Minderer, G.~Heigold, S.~Gelly, J.~Uszkoreit, and N.~Houlsby, ``An image is worth 16x16 words: Transformers for image recognition at scale,'' in \emph{9th International Conference on Learning Representations, {ICLR} 2021, Virtual Event, Austria, May 3-7, 2021}.\hskip 1em plus 0.5em minus 0.4em\relax OpenReview.net, 2021. [Online]. Available: \url{https://openreview.net/forum?id=YicbFdNTTy}
\BIBentrySTDinterwordspacing
\bibitem{team2023gemini}
G.~Team, R.~Anil, S.~Borgeaud, J.-B. Alayrac, J.~Yu, R.~Soricut, J.~Schalkwyk, A.~M. Dai, A.~Hauth, K.~Millican \emph{et~al.}, ``Gemini: a family of highly capable multimodal models,'' \emph{arXiv preprint arXiv:2312.11805}, 2023.
\bibitem{vlogtbgen}
\BIBentryALTinterwordspacing
{EDAUtils}, ``Verilog testbench generator.'' [Online]. Available: \url{https://www.edautils.com/VlogTBGen.html}
\BIBentrySTDinterwordspacing
\bibitem{iverilog}
\BIBentryALTinterwordspacing
{Stephen Williams}, ``Icarus verilog compilation system.'' [Online]. Available: \url{https://github.com/steveicarus/iverilog}
\BIBentrySTDinterwordspacing
\bibitem{vcd2json}
\BIBentryALTinterwordspacing
{Github user: Namake Nanamaru}, ``vcd2json.'' [Online]. Available: \url{https://github.com/nanamake/vcd2json}
\BIBentrySTDinterwordspacing
\bibitem{wavedrom-cli}
\BIBentryALTinterwordspacing
{WaveDrom}, ``wavedrom-cli.'' [Online]. Available: \url{https://github.com/wavedrom/cli}
\BIBentrySTDinterwordspacing
\bibitem{verilogdataset}
\BIBentryALTinterwordspacing
S.~Thakur, B.~Ahmad, Z.~Fan, H.~Pearce, B.~Tan, R.~Karri, B.~Dolan-Gavitt, and S.~Garg, ``Benchmarking large language models for automated verilog rtl code generation,'' 2022. [Online]. Available: \url{https://arxiv.org/abs/2212.11140}
\BIBentrySTDinterwordspacing
\bibitem{deepseekai2024deepseekcoderv2breakingbarrierclosedsource}
\BIBentryALTinterwordspacing
DeepSeek-AI, Q.~Zhu, D.~Guo, Z.~Shao, D.~Yang, P.~Wang, R.~Xu, Y.~Wu, Y.~Li, H.~Gao, S.~Ma, W.~Zeng, X.~Bi, Z.~Gu, H.~Xu, D.~Dai, K.~Dong, L.~Zhang, Y.~Piao, Z.~Gou, Z.~Xie, Z.~Hao, B.~Wang, J.~Song, D.~Chen, X.~Xie, K.~Guan, Y.~You, A.~Liu, Q.~Du, W.~Gao, X.~Lu, Q.~Chen, Y.~Wang, C.~Deng, J.~Li, C.~Zhao, C.~Ruan, F.~Luo, and W.~Liang, ``Deepseek-coder-v2: Breaking the barrier of closed-source models in code intelligence,'' 2024. [Online]. Available: \url{https://arxiv.org/abs/2406.11931}
\BIBentrySTDinterwordspacing
\bibitem{0010LLWWBCH22}
\BIBentryALTinterwordspacing
J.~Liu, A.~Liu, X.~Lu, S.~Welleck, P.~West, R.~L. Bras, Y.~Choi, and H.~Hajishirzi, ``Generated knowledge prompting for commonsense reasoning,'' in \emph{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2022, Dublin, Ireland, May 22-27, 2022}, S.~Muresan, P.~Nakov, and A.~Villavicencio, Eds.\hskip 1em plus 0.5em minus 0.4em\relax Association for Computational Linguistics, 2022, pp. 3154--3169. [Online]. Available: \url{https://doi.org/10.18653/v1/2022.acl-long.225}
\BIBentrySTDinterwordspacing
\bibitem{3600270.3602070}
J.~Wei, X.~Wang, D.~Schuurmans, M.~Bosma, B.~Ichter, F.~Xia, E.~H. Chi, Q.~V. Le, and D.~Zhou, ``Chain-of-thought prompting elicits reasoning in large language models,'' in \emph{Proceedings of the 36th International Conference on Neural Information Processing Systems}, ser. NIPS '22.\hskip 1em plus 0.5em minus 0.4em\relax Red Hook, NY, USA: Curran Associates Inc., 2024.
\bibitem{hdlbits}
``Hdlbits,'' \url{https://hdlbits.01xz.net/wiki/Problem_sets}.
\bibitem{serialprotocol}
``serialdp,'' \url{https://hdlbits.01xz.net/wiki/Fsm_serialdp}.
\bibitem{axi}
\emph{AMBA AXI and ACE Protocol Specification}, Arm, October 2011.
\bibitem{ahb}
\emph{AMBA AHB Protocol Specification}, Arm, September 2021.
\bibitem{apb}
\emph{AMBA APB Protocol Specification}, Arm, April 2021.
\bibitem{serialdatapath}
``serialdata,'' \url{https://hdlbits.01xz.net/wiki/Fsm_serialdata}.
\bibitem{motor_timer}
``motor timer,'' \url{https://hdlbits.01xz.net/wiki/Exams/2013_q2bfsm}.
\bibitem{complete_fsm}
``complete fsm,'' \url{https://hdlbits.01xz.net/wiki/Exams/review2015_fsm}.
\bibitem{fancy_timer}
``fancy timer,'' \url{https://hdlbits.01xz.net/wiki/Exams/review2015_fancytimer}.
\bibitem{hdlc}
``hdlc,'' \url{https://hdlbits.01xz.net/wiki/Fsm_hdlc}.
\bibitem{wcounter}
``wcounter,'' \url{https://hdlbits.01xz.net/wiki/Exams/2014_q3fsm}.
\bibitem{dosovitskiy2020image}
A.~Dosovitskiy, ``An image is worth 16x16 words: Transformers for image recognition at scale,'' \emph{arXiv preprint arXiv:2010.11929}, 2020.
\bibitem{ba2016layer}
J.~L. Ba, ``Layer normalization,'' \emph{arXiv preprint arXiv:1607.06450}, 2016.
\bibitem{chiang2023vicuna}
W.-L. Chiang, Z.~Li, Z.~Lin, Y.~Sheng, Z.~Wu, H.~Zhang, L.~Zheng, S.~Zhuang, Y.~Zhuang, J.~E. Gonzalez \emph{et~al.}, ``Vicuna: An open-source chatbot impressing gpt-4 with 90\%* chatgpt quality,'' \emph{See https://vicuna. lmsys. org (accessed 14 April 2023)}, vol.~2, no.~3, p.~6, 2023.
\bibitem{zhang2019root}
B.~Zhang and R.~Sennrich, ``Root mean square layer normalization,'' \emph{Advances in Neural Information Processing Systems}, vol.~32, 2019.
\bibitem{hu2021lora}
E.~J. Hu, Y.~Shen, P.~Wallis, Z.~Allen-Zhu, Y.~Li, S.~Wang, L.~Wang, and W.~Chen, ``Lora: Low-rank adaptation of large language models,'' \emph{arXiv preprint arXiv:2106.09685}, 2021.
\end{thebibliography}