Tree-Based versus Hybrid Graphical-Textual Model Editors: An Empirical Study of Testing Specifications
Extracted source text · arXiv 2404.05846
%%
%% This is file `sample-sigplan.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigplan')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigplan.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% Commands for TeXCount
%TC:macro \cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf,review]{acmart}
\usepackage{enumitem}
\usepackage{lipsum}
\usepackage{subfig}
\usepackage{makecell}
\usepackage[usestackEOL]{stackengine}
\usepackage{multirow}
\newcommand{\ip}[1]{{\color{purple} [IP: #1]}}
\newcommand{\agd}[1]{{\color{blue} [AGD: #1]}}
\newcommand{\dk}[1]{{\color{teal} [DK: #1]}}
\newcommand{\sg}[1]{{\color{magenta} [SG: #1]}}
\newcommand{\jrh}[1]{{\color{green} [JRH: #1]}}
\newcommand*{\metaclass}[1]{\emph{#1}}
\newcommand*{\attr}[1]{\emph{#1}}
\newcommand*{\val}[1]{\emph{#1}}
\newcommand{\quotes}[1]{``#1''}
\newcommand{\significant}[1]{\textcolor{blue}{#1}}
%% NOTE that a single column version is required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen,review]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmlicensed}
\copyrightyear{2024}
\acmYear{2024}
\acmDOI{XXXXXXX.XXXXXXX}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[MODELS '24]{ACM/IEEE 27th International Conference on Model Driven Engineering Languages and Systems}{September 22--27,
2024}{Linz, Austria}
%
% Uncomment \acmBooktitle if th title of the proceedings is different
% from ``Proceedings of ...''!
%
%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
% June 03--05, 2018, Woodstock, NY}
\acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title[Tree-Based versus Hybrid Graphical-Textual Model Editors: An Empirical Study]
% {An Empirical Study of Hybrid Graphical-Textual\\ Model Editors against Tree-Based Model Editors}
% {An Empirical Study of Testing Specification Model Editors: Tree-Based versus Hybrid Graphical-Textual}
% {An Empirical Study of Testing Specifications: Tree-Based Model Editors versus Hybrid Graphical-Textual Model Editors}
% {An Empirical Study of Testing Specifications:
% \\Tree-Based versus Hybrid Graphical-Textual Model Editors}
{Tree-Based versus Hybrid Graphical-Textual Model Editors:
\\An Empirical Study of Testing Specifications}
%% NEW TITLE NEEDED
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
% \author{Ionut Predoaia}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{ionut.predoaia@york.ac.uk}
% \author{James Harbin}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{james.harbin@york.ac.uk}
% \author{Simos Gerasimou}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{simos.gerasimou@york.ac.uk}
% \author{Christina Vasiliou}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{christina.vasiliou@york.ac.uk}
% \author{Dimitris Kolovos}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{dimitris.kolovos@york.ac.uk}
% \author{Antonio García-Domínguez}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{a.garcia-dominguez@york.ac.uk}
\author{Ionut Predoaia, James Harbin, Simos Gerasimou,
\\Christina Vasiliou, Dimitris Kolovos, Antonio García-Domínguez}
\affiliation{%
\institution{University of York, United Kingdom}
% \city{York}
\country{}
}
\email{ionut.predoaia, james.harbin, simos.gerasimou, christina.vasiliou, dimitris.kolovos, a.garcia-dominguez@york.ac.uk}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Predoaia et al.}
\newcommand{\squishlist}{
\begin{list}{$ullet$}
{ \setlength{\itemsep}{0pt}
\setlength{\parsep}{3pt}
\setlength{\topsep}{3pt}
\setlength{\partopsep}{0pt}
\setlength{\leftmargin}{1.5em}
\setlength{\labelwidth}{1em}
\setlength{\labelsep}{0.5em} } }
\newcommand{\squishlisttwo}{
\begin{list}{$ullet$}
{ \setlength{\itemsep}{0pt}
\setlength{\parsep}{0pt}
\setlength{\opsep}{0pt}
\setlength{\partopsep}{0pt}
\setlength{\leftmargin}{2em}
\setlength{\labelwidth}{1.5em}
\setlength{\labelsep}{0.5em} } }
\newcommand{\squishend}{
\end{list} }
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
%\agd{define acronyms on their first use}
\begin{abstract}
Tree-based model editors and hybrid graphical-textual model editors have advantages and limitations when editing domain models. Data is displayed hierarchically in tree-based model editors, whereas hybrid graphical-textual model editors capture high-level domain concepts graphically and low-level domain details textually. We conducted an empirical user study with 22 participants, to evaluate the implicit assumption of system modellers that hybrid notations are superior, and to investigate the tradeoffs between tree-based and hybrid model editors. The results of the user study indicate that users largely prefer hybrid editors and are more confident with hybrid notations for understanding the meaning of conditions. Furthermore, we found that tree editors provide superior performance for analysing ordered lists of model elements, whereas activities requiring the comprehension of complex conditions are carried out faster through a hybrid editor.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10011007.10011006.10011050.10011017</concept_id>
<concept_desc>Software and its engineering~Domain specific languages</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011050.10011058</concept_id>
<concept_desc>Software and its engineering~Visual languages</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011074.10011099.10011693</concept_id>
<concept_desc>Software and its engineering~Empirical software validation</concept_desc>
<concept_significance>500</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Software and its engineering~Domain specific languages}
\ccsdesc[500]{Software and its engineering~Visual languages}
\ccsdesc[500]{Software and its engineering~Empirical software validation}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{Hybrid Notations, Model Editors, Fuzz Testing, Empirical Study, Domain-Specific Languages, Model-Driven Engineering, Robotics}
%\jrh{Set CCS Concepts and Keywords}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
\label{SEC-INTRO}
Domain-specific languages (DSLs) can have graphical, textual, or hybrid syntaxes. Hybrid graphical-textual DSLs are languages that have a part-graphical and part-textual syntax, i.e., a hybrid graphical-textual syntax. They are particularly appropriate for cases in which one would like to use graphical representations for capturing high-level domain concepts, and textual representations for defining low-level domain details, such as behaviour and complex expressions \cite{cooper2019engineering}. Furthermore, they are effectively used via hybrid graphical-textual model editors \cite{predoaia2023streamlining,zolotas2018towards,wei2020automatic}, i.e., editors that enable the definition of domain models via graphical and textual representations.
Throughout this paper, we use the term \textit{model editor} rather than the term \textit{workbench}, as a workbench covers a broader scope, incorporating a model editor in addition to other components such as menus, toolbars and perspectives \cite{burnette2005eclipse}. For brevity, the term \textit{hybrid} will be used instead of the term \textit{hybrid graphical-textual}, and additionally, the term \textit{editor} will refer to a \textit{model editor}.
Choosing between a tree-based and a hybrid editor for editing domain models is not a straightforward decision, as both have advantages and drawbacks. Tree editors display domain data hierarchically, in a structured manner, by exposing parent-child relationships. Diagrammatic editors contained within hybrid editors do not provide explicit structure, and they are not appropriate for displaying parent-child relationships of multiple levels. For instance, parent elements can be displayed as containers that incorporate their child elements. However, if the children have corresponding child elements, which in turn have child elements as well, displaying all these elements in a diagram can hinder comprehension, as they would effectively be displayed as a container that contains other containers, that in turn contain other containers incorporating a set of leaf child elements. Thus, in such cases, it may be preferred to use multiple diagrams, e.g., by instantiating new diagrams containing the child elements of a parent element. Consequently, the domain data is split across multiple diagrams, and users may face difficulties in being able to relate them. A common drawback of hybrid editors is the initial cost of learning the associated part-graphical and part-textual syntaxes. However, once learned, models may potentially be edited faster, considering also that typically fewer clicks are required due to the definition of textual expressions \cite{addazi2017towards}.
Acknowledging the importance of selecting the most suitable editor to support effective and efficient modelling, in this paper we investigate the tradeoffs between tree-based and hybrid editors. System modellers often take for granted the assumption that hybrid notations are superior, however, this claim is not yet supported by empirical evidence. To evaluate this implicit assumption, we conducted an empirical user study to analyse the editor type preferred by system modellers. We recruited 22 subjects, that are either students or professionals working in industry or academia, with various levels of knowledge and expertise in computer programming, modelling, and model-driven engineering (MDE). In the user study, the participants used a DSL that enables the definition of testing specifications of robotic systems in simulation. Each participant used the DSL via both a tree editor and a hybrid editor, to execute a set of tasks that are inspired from an industrial case study of a large European Commission research project named Secure and Safe Multi-Robot Systems (SESAME \cite{sesame_website}). The user study aims to evaluate the performance and preferences of users, and provide insights on the usage of hybrid editors and tree editors.
Our empirical study revealed that users largely prefer hybrid editors and are more confident with hybrid editors for understanding the meaning of conditions. From a performance perspective, the results pinpoint that certain types of activities can be carried out faster via a hybrid editor. Furthermore, the results evidence that hybrid editors require more keystrokes and fewer clicks for modelling conditions.
Paper structure. Section~\ref{SEC-BACKGROUND} introduces the necessary background, Section~\ref{SEC-CASE-STUDY} describes our industrial case study, and Section~\ref{SEC-DSL} presents the fuzzing-based testing DSL. Sections~\ref{SEC-METHODOLOGY} and~\ref{SEC-RESULTS} detail the methodology and results of the user study, respectively.
Section~\ref{SEC-THREATS} identifies threats to validity. Section~\ref{SEC-RELATED-WORK} presents related work and, finally, Section~\ref{SEC-CONCLUSIONS} concludes the paper, providing future work directions.
\section{Background}
\label{SEC-BACKGROUND}
\noindent
\textbf{Modelling Editors.}
Hybrid editors are commonly composed of graphical diagram(s), embedded textual editor(s) with syntax-aware editing features, such as syntax highlighting, auto-completion, and refactoring, a view in which one can edit the properties of a selected model element, a view with various symbols that can be dragged and dropped on top of diagrams to create various types of model elements, and a view that consistently reports errors from the model \cite{cooper2019engineering}. Textual expressions are able to reference model elements that are defined graphically in diagrams, and consistency between the two is automatically enforced as the model evolves.
Hybrid DSLs and their supporting editors can be developed using Graphite \cite{predoaia2023streamlining, predoaia2023towards}, a tool that facilitates the systematic engineering of hybrid editors via automatic code generation.
Graphite (\url{https://github.com/epsilonlabs/graphite}) uses Sirius \cite{sirius_website} for the specification of the graphical part of the syntax, and Xtext \cite{xtext_website} for defining the textual part of the syntax. Alternatively, projectional editors, such as JetBrains MPS \cite{jetbrains_website}, can be employed for developing such hybrid editors. The research from~\cite{scheidgen2008textual, paper_obeo_typefox, cooper2018msc, solution_altran}, which are based on hand-written code, introduce techniques for engineering such kind of hybrid editors.
However, they do not address consistency enforcement between textual expressions and graphical model elements. Furthermore, except~\cite{cooper2018msc}, no techniques have been proposed for uniformly reporting errors. Other streams of work involve using blended modelling~\cite{ciccozzi2019blended} and Capella~\cite{capella_extension}; however, they do not address hybrid DSLs, as justified in~\cite{predoaia2023streamlining}.
\vspace{1mm}\noindent
\textbf{Testing Specification. }
Multi-robot systems (MRS) are distributed and interconnected robotic
teams capable of collectively performing tasks that are beyond the
competency of a single
robot~\cite{parker2012reliability}. Simulation-based testing (SBT) enables
investigating the capacity of an MRS to operate dependably using a
virtual environment that closely resembles the real deployment
environment. Modern robotic frameworks enable realistic simulation of
robots, with strong conformance to physical hardware making the software
deployment to real robots easier~\cite{timperley-icst18}.
Compared to field testing,
simulation-based testing supports searching large design spaces of
components and system configurations, using limited physical hardware
resources, thus reducing significantly the overheads for detecting
potential violations of safety
requirements~\cite{harbin-models-2021,gerasimou2019towards}.
Fuzz testing is an automated software testing technique that involves providing unexpected, invalid or random data as input to a computer program, with the purpose of finding exceptions, crashes and potential memory leaks \cite{godefroid2020fuzzing}.
Fuzz testing is considered among the most efficient approaches for finding security vulnerabilities in software systems~\cite{liang2018fuzz}. In the context of our work, fuzz testing is applied as a simulation-based testing technique, by injecting intentional errors or faulty messages into a robotic simulation to monitor its responses and measure its robustness.
% TO BE ADDED IN OTHER SECTION
%In this paper we consider a general architecture for specifying generic fuzz testing campaigns to raise the level of abstraction and make fuzz testing more comprehensible to robotics engineers, as explored in the SESAME project \cite{sesame-d6.6}. Our focus here is specifically upon user comprehensibility; exploring the accessibility of the fuzz testing configuration and results representation within a domain specific language (DSL). We quantify with a user study how easy it is for new users to understand the MRS system and testing campaigns represented, and to interact with the testing configuration and results contained within.
\section{Case Study}
\label{SEC-CASE-STUDY}
%\agd{testing does not guarantee anything: it'd be better to say that you're aiming to obtain evidence that supports an argument that the system is safe --- check wording with Simos}
%KUKA~\cite{kuka-deliverable}.
%KUKA
The use case is an industrial case study in gearbox manufacturing
provided by our SESAME industrial partner, who manufactures industrial robots and factory automation systems. This use case incorporates mobile robots to assemble gearboxes, together with human workers surrounding the assembly cell who assist in the process.
During testing, we aim to obtain evidence to validate the safety requirement that robots will not generate hazards to human operators at operation.
The primary safety consideration is to ensure that robots do not enter regions that could cause a collision hazard to human operators placed around the perimeter of the assembly cell frame.
Under normal circumstances, motion trajectories are pre-planned to ensure conformity to this safety requirement. However, faults in communication, sensing, or motor interlock actuation may lead to robots malfunctioning and
entering these forbidden areas.
The simulation scenario was supplied for testing by end users, and implemented using a proprietary simulation platform provided by a SESAME industrial partner to be used as a digital twin~\cite{tts-dtwin-deliverable}.
\begin{figure}
\centering
\includegraphics[trim={0cm 5cm 0cm 6.5cm},clip,width=0.86\linewidth]{figures/ttsScreenshot.png}
\vspace{-3mm}
\caption{Case study screenshot from MRS simulation}
\vspace{-4mm}
\label{fig:kuka-case-study}
\end{figure}
% Individual figures version
\begin{figure}[!tb]
\centering
\includegraphics[width=\linewidth]{figures/testing-metamodel-uml}
\vspace{-8mm}
\caption{Fuzzing-based testing DSL metamodel (excerpt)}
\label{FIG-TESTSPACE-DSL}
\vspace{-6mm}
\end{figure}
\begin{figure}[!tb]
\centering
\includegraphics[width=0.9\linewidth]{figures/dsl-conditions-uml}
\vspace{-5mm}
\caption{Meta-classes of condition-based fuzzing (excerpt)}
\label{FIG-CONDITIONS-DSL}
\vspace{-3mm}
\end{figure}
% JRH: subfig version of metamodel figures
\if 0
\begin{figure*}[!t]
\centering
\subfloat[][Metamodel Excerpt of the Fuzzing-Based Testing DSL]{\includegraphics[width=0.5\linewidth]{figures/testing-metamodel-uml}
\vspace{-4mm}
\label{FIG-TESTSPACE-DSL}}
\subfloat[][Meta-classes of condition-based fuzzing (excerpt)]{\includegraphics[width=0.5\linewidth]{figures/dsl-conditions-uml}
\vspace{-4mm}
\label{FIG-CONDITIONS-DSL}}
\vspace{-5mm}
\end{figure*}
\fi
%\caption{Metamodel Excerpt of the Fuzzing-Based Testing DSL}
%\agd{if you show an association, name it and hide the field, and add %cardinalities to associations}\agd{abstract classes should use italics}%%\agd{What are GrammarSpec and GrammarConstraints, and do we need to show it for our discussion?}\agd{How are Metrics defined?}\agd{What is the difference between ExecutionEndTrigger and ExecutionEndCondition?}\agd{Why are the results of a CampaignResultSet just references to tests? Where are their results (e.g. pass/fail)?}
Figure~\ref{fig:kuka-case-study} shows the layout of the assembly cell.
The primary safety requirement is that the robot grippers are forbidden to enter static safety zones surrounding the cell, in order to prevent hazards to human workers.
The safety zones are illustrated as blue cuboid regions (Z1, Z2, Z3) on the right-hand side of the external cell frame in Figure~\ref{fig:kuka-case-study}.
If the safety zone surrounding the gripper of the right-hand robot (green cylinder labeled ZR3200) collides with these regions, it will trigger violations of the static safety requirement.
An output metric is used to quantify violations of the principal safety requirement, by tracking the occurrence of collisions of the gripper safety zone with any of Z1, Z2 or Z3.
Following a safety analysis, we have identified the communication from the controller to the robot arm joints as vulnerable to message disruptions.
Therefore, the primary safety requirement will be tested by applying fuzzing, altering the target motion value for robot arm joints with targeted uniform random jitter around the intended point.
We employ a DSL will to raise the level of abstraction and make testing specification %and fuzz testing
more comprehensible to robotics engineers.
Modelling testing specifications via a DSL tailored to this specific domain is highly desirable, enabling non-expert users to easily test robotics systems~\cite{casalaro2021model,bruyninckx2013brics,harbin2023model}.
Participants of the study will apply the DSL in the context of this case study, via a tree editor and a hybrid editor, to investigate the following research questions:
\begin{enumerate}[font={\bfseries},label={RQ\arabic*}]
\setcounter{enumi}{0}
\item \hypertarget{link_rq1}{\textbf{(Performance)}}
How do users perform when executing certain tasks with a hybrid editor compared to a tree editor?
\end{enumerate}
\begin{enumerate}[font={\bfseries},label={RQ\arabic*}]
\setcounter{enumi}{1}
\item \hypertarget{link_rq2}{(\textbf{Confidence})}
How confident are users regarding the correctness and completeness of their solutions to given tasks?
\end{enumerate}
\begin{enumerate}[font={\bfseries},label={RQ\arabic*}]
\setcounter{enumi}{2}
\item \hypertarget{link_rq3}{(\textbf{Preference})}
Do users prefer hybrid editors or tree editors?
\end{enumerate}
%TO BE REVIEWED: MAY HAVE TO MOVE THIS INTO A FOLLOWING SECTION
%A part of the fuzzing model instantiated for the KUKA industrial use case is presented in Figure~\ref{FIG-DSL-TTS-EXAMPLE}. The operations $attackJointR3200ProductMove\_Link1$ and $attackJointR3200ProductMove\_Link0$ of the model illustrate this fuzzing offset value change, using the \metaclass{RandomValueFromSet} fuzzing operation to apply relative changes to the joint value time series.
%\begin{figure}[t]
% \centering
% \includegraphics[width=\linewidth]{figures/tts-model-example-partial}
% \caption{A fragment of the DSL example for the TTS use case
% showing selected metrics and fuzzing operations}
% \label{FIG-DSL-TTS-EXAMPLE}
%\end{figure}
\section{Fuzzing-Based Testing DSL}
\label{SEC-DSL}
This section presents the fuzzing-based testing DSL developed in the
context of the SESAME project, and applied to the case study
from Section~\ref{SEC-CASE-STUDY}. Through the DSL, users can
define testing specifications for an MRS simulation, which control how
fuzzing is applied in MRS testing. We present next the abstract
syntax and the concrete syntaxes of the DSL and the SBT methodology
enabled by the DSL.
\subsection{Abstract Syntax}
\label{SEC-DSL-ABSTRACT}
The abstract syntax (i.e., the metamodel of the DSL) has been defined
with the Eclipse Modelling Framework (EMF) through the Ecore
metamodelling language.
Figure~\ref{FIG-TESTSPACE-DSL} shows an excerpt from the metamodel of the DSL as a UML diagram.
The core meta-classes are \metaclass{TestingSpace}, \metaclass{TestCampaign}
and \metaclass{Test}. A \metaclass{TestingSpace} (highlighted)
represents the root element of the domain and comprises a set of
permissible fuzzing operations (\attr{possibleOperations}) that bound
the potential fuzzing space. The testing space also includes
performance \attr{metrics}, which are used to quantify the robotic
system performance for safety violations.
% JRH: removed as this is mentioned in "Sec. 4.3: simulation-based testing"
%, and allow multi-objective optimisation of test performance
%(optimising for most impactful violations).
A \metaclass{TestCampaign} specifies and constrains an individual
experiment, referencing from the \metaclass{TestingSpace} a subset of
\attr{metrics} to use in campaign evaluation, together with a
subset of chosen fuzzing operations in \attr{includedOperations}. The
\attr{performedTests} attribute in \metaclass{TestCampaign} is
automatically populated during the experiment with dynamically
generated and executed \metaclass{Test}s.
%The \metaclass{TestGenerationApproach}
%allows specifying the parameters for an experiment by selecting one of
%its subclasses. For example, \metaclass{NSGAEvolutionaryAlgorithm}
%allows an evolutionary experiment with the NSGA-II
%algorithm~\cite{nsga2}.
The \metaclass{Test} class represents a test configuration,
corresponding to a specific selection of fuzzing \attr{operations}.
During \metaclass{Test} execution, instances of
\metaclass{MetricInstance} are recorded, to quantify the values of
particular \metaclass{Metric}s under the applied fuzzing.
The \metaclass{FuzzingOperation} class is a specific fuzzing
operation, i.e., an entity that represents a specific strategy for
making runtime modifications to the
MRS. \attr{variableToAffect} selects the MRS variable to which fuzzing
operations are applied. These variables will be automatically
manipulated by the testing platform when fuzzing is active.
% Not shown here;
Subclasses of \metaclass{FuzzingOperation} represent distinct
fuzzing semantics; e.g. \metaclass{RandomValueFromSetOperation}
enables randomised changes to structured parameters within a message.
\if 0
\begin{figure}[t]
\centering
\includegraphics[width=0.9\linewidth]{figures/dsl-conditions-uml}
\vspace{-4mm}
\caption{Meta-classes of condition-based fuzzing (excerpt)
\sg{see comments for polishing}
}
%\agd{we need to be consistent with capitalisation: choose one approach and stick with it}\agd{remove fields which are shown as associations, and add cardinalities to associations}\agd{shouldn't there be two compositions from CompositeCondition to ConditionElement?}}
\label{FIG-CONDITIONS-DSL}
\vspace{-3mm}
\end{figure}
\fi
%\agd{what is the Pareto front measured in?} \ip{ideally define what a Pareto front is, what it contains and in what it is measured?}.
The \metaclass{CampaignResultSet} class represents either final or
partial intermediate testing results (selected by the
\metaclass{ResultSetStatus} enumeration) and its \attr{name}. The
results are shown as references to notable tests under
\attr{results}. In an evolutionary experiment, the set of
non-dominated tests in the final campaign result set constitute the
Pareto front of optimal metrics obtained during the experiment. The
inclusion of a Test in the non-dominated result sets shows that no other tests dominate it with respect to the set of metrics used.
% Explain the metric direction and how it is used in optimisation
Condition-based fuzzing provides the ability to activate and
deactivate fuzzing based on custom scenario-specific events,
incorporating complex mission logic and spatial or geometric
properties. A condition is a logical expression built hierarchically
from comparison and logical operations, incorporating state variables
from the robotic system. The values of condition variables change
dynamically during test execution, with every variable value supplied
from a \metaclass{Metric} of corresponding name. Activation of a
condition-based fuzzing operation is controlled by its start and end
conditions. When the start condition of a fuzzing operation evaluates
to true, that fuzzing operation will be activated, and will manipulate
simulation events. Once activated, the fuzzing operation remains
active until its end condition evaluates to true.
Condition-based fuzzing experiments aims at discovering these
conditions automatically, identifying high-level MRS events that lead
to failures or safety violations.
The primary role of a system tester will be to inspect, parse and understand these auto-generated conditions.
However, during debugging and exploratory testing, users may modify existing conditions or create new from scratch, to verify the impact of these changes upon the
behaviour of a test.
%\agd{so these are not written by users? does
% this mean that the role of the tester is to read, rather than write
% these condition expressions?}.
\begin{figure}
\centering
\frame{\includegraphics[width=\linewidth]{figures/tree_editor.png}}
\vspace{-6mm}
\caption{Tree Editor — A Testing Space}
\label{FIG-TREE-EDITOR}
\vspace{-1mm}
\end{figure}
\begin{figure}[t]
\centering
\frame{\includegraphics[width=0.96\linewidth]{figures/tree_editor_conditions.png}}
\vspace{-2mm}
\caption{Tree Editor — Activation Condition}
\label{FIG-TREE-EDITOR-CONDITIONS}
\vspace{-1mm}
\end{figure}
\begin{figure}[t]
\centering
\frame{\includegraphics[width=\linewidth]{figures/tree_editor_properties.png}}
\vspace{-6mm}
\caption{Tree Editor — Properties View}
\label{FIG-TREE-EDITOR-PROPERTIES}
\vspace{-1mm}
\end{figure}
\begin{figure}[t]
\centering
\frame{\includegraphics[width=\linewidth]{figures/hybrid_editor_testingspace.png}}
\vspace{-6mm}
\caption{Hybrid Editor — Diagram of a Testing Space\vspace{3.6mm}}
\label{FIG-HYBRID-EDITOR-TESTINGSPACE}
\vspace{-6mm}
\end{figure}
The metamodel fragment in Figure~\ref{FIG-CONDITIONS-DSL} shows
the meta-classes for condition-based fuzzing. The
\metaclass{Condition} class comprises an abstract
\metaclass{ConditionElement}, which is a
\metaclass{BasicCondition} or \metaclass{CompositeCondition}. A
\metaclass{BasicCondition} represents comparisons of a
\metaclass{ConditionVariable} with a \metaclass{ConditionExpression},
which is either a \metaclass{ConditionVariable} or
\metaclass{ConditionInteger}. The
\metaclass{BinaryComparisonOperation} is either \val{LESS\_THAN},
\val{GREATER\_THAN}, or \val{EQUALS}. \metaclass{ConditionVariables}
contain a reference to an associated \attr{metric} that supplies its
value, and \metaclass{ConditionIntegers} representing primitive
integers. A \metaclass{CompositeCondition} contains two other
\metaclass{ConditionElements} in the left and right branches, together
with a \metaclass{BinaryLogicalOperation}. These logical operations
permit \val{AND} or \val{OR} operators. Since condition
expressions are dynamically generated during experiments,
\metaclass{CompositeCondition} permits complex multi-level boolean
expressions of arbitrary depth.
%TO BE REVIEWED: MAY BE MOVED TO A FOLLOWING SECTION
%In this paper, a major aspect is to investigate how different DSL syntaxes permit the comprehension of deeply nested condition expressions, allowing the user to interpret their meanings. We will also investigate how the syntaxes allow the users to manually modify these syntaxes.
\subsection{Concrete Syntaxes}
The fuzzing-based testing DSL can be used through two concrete syntaxes: a graphical tree-based syntax and a hybrid syntax. The graphical syntax is provided by the built-in EMF tree-based editor, whereas the hybrid syntax has been developed using Graphite~\cite{predoaia2023streamlining}.
\vspace{1mm}\noindent
%The testing space contains two metrics (a \metaclass{FuzzingOperationTimesMetric} and a \metaclass{StreamMetric}), a \metaclass{FuzzingOperation} of type \metaclass{RandomValueFromSetOperation} named \val{attackJointPosR3200ProductMove\_Link0}, a \metaclass{TestCampaign}, and the definition of an MRS.
\textbf{Tree editor.} A subset of a testing space used in the case study is shown in Figure~\ref{FIG-TREE-EDITOR}. The \metaclass{TestCampaign} is selected in the editor, and its attributes (e.g., \attr{includedOperations} and \attr{metrics}) are displayed in the Properties View from Figure~\ref{FIG-TREE-EDITOR-PROPERTIES}. The test campaign contains three executed tests and two campaign result sets. Recall that a \metaclass{CampaignResultSet} represents a logical grouping of executed tests. The first test contains the \metaclass{RandomValueFromSetOperation} under \attr{includedOperations} in the \metaclass{TestCampaign}, and also two metric instances: one for \val{fuzzingOperationTimes} and one for \val{collisionOccurrence}, as specified by the \attr{metrics} attribute displayed in the Properties View from Figure~\ref{FIG-TREE-EDITOR-PROPERTIES}. Figure~\ref{FIG-TREE-EDITOR-CONDITIONS} expands the \metaclass{RandomValueFromSetOperation} named \val{attackJointPosR3200ProductMove\_Link0} and displays its child elements. This operation is activated when its start condition evaluates to true. The \metaclass{Condition} element represents the root of the activation condition and it contains a \metaclass{CompositeCondition}, in which the left side of the condition is a \metaclass{BasicCondition} and the right side is another \metaclass{CompositeCondition}. %Note that the \metaclass{Condition} element and its children conform to the metamodel fragment from Figure~\ref{FIG-CONDITIONS-DSL}.
\begin{figure*}
\centering
\frame{\includegraphics[width=0.78\linewidth]{figures/hybrid_editor_test.png}}
\vspace{-2mm}
\caption{Hybrid Editor — Test with Condition Textual Expressions\vspace{3.6mm}}
\label{FIG-HYBRID-EDITOR-TEST}
\vspace{-4mm}
\end{figure*}
\begin{figure*}
\centering
\frame{\includegraphics[width=0.9\linewidth]{figures/hybrid_editor_resultsets.png}}
\vspace{-2mm}
\caption{Hybrid Editor — Diagram of Campaign Result Sets}
\label{FIG-HYBRID-EDITOR-RESULTSETS}
\vspace{-1mm}
\end{figure*}
\vspace{1mm}\noindent
\textbf{Hybrid editor.} The Graphite-based concrete syntax is shown in Figures~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} - %, \ref{FIG-HYBRID-EDITOR-TEST}, and~
\ref{FIG-HYBRID-EDITOR-RESULTSETS}.
A graphically defined testing space is illustrated in the diagram in Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}. The elements of the testing space are displayed as rectangles with various colours and symbols depending on each element type. Arrows highlight relationships between elements. To create new elements, users can drag and drop various symbols from the Palette on top of the diagram. The diagram displays the high-level elements of the testing space; however, users can view lower-level details by double-clicking on specific elements.
By double-clicking on a \metaclass{Test} contained in a \metaclass{TestCampaign}, a new diagram is instantiated with the child elements of the \metaclass{Test}. For instance, Figure~\ref{FIG-HYBRID-EDITOR-TEST} shows a hybrid editor that contains the child elements of the first test from the test campaign. The activation condition is specified with a textual expression in an embedded textual editor, with developer assistance features such as syntax highlighting and auto-completion.
Also, users can double-click the \metaclass{TestCampaign} from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} to view its campaign result sets in a newly instantiated diagram. Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} illustrates the campaign result sets of the test campaign, ordered from first to last via arrows. By double-clicking a \metaclass{Test} from the campaign result sets, a new diagram similar to the one from Figure~\ref{FIG-HYBRID-EDITOR-TEST} is instantiated, illustrating the details of the \metaclass{Test}.
We should note that the capabilities and behaviour of the hybrid editor, e.g., the instantiation of new diagrams via double-clicking, are shown to the participants of the user study via a video tutorial.
\subsection{Simulation-Based Testing}
The SESAME tools for simulation-based testing \cite{sesame-deliverable-d6.2,sesame-deliverable-d6.6} provide a model-driven
code generation engine. Input models conforming to the DSL
in Section \ref{SEC-DSL-ABSTRACT} configure an evolutionary optimisation
loop for a given \metaclass{TestCampaign}. This loop optimises a
population of fuzzing tests with the intent of maximising safety
requirement violations. For every \metaclass{Test} generated and added
to the model, the SESAME code generation engine generates a Java test
runner, including an MRS logical interface enabling two-way runtime
communication. The test runner launches the simulation,
monitors and manipulates fuzzing variables to perform the operations
defined for that test. When a fuzzing operation is active, all
messages upon the target variable will be modified, delayed or deleted
as specified by the fuzzing operation. Performance metrics are
computed from monitored variables, and their final values logged under
\metaclass{Test} as a \metaclass{MetricInstance}.
\begin{table}[t]
\begin{center}
\caption{\label{tab:demographics}Breakdown of expertise in participants}
\vspace{-3mm}
\begin{tabular}{ |c|c|c|c|c| }
\hline
Expertise & Modelling & DSLs & MDE & MRS\\
\hline
None/Low & 6 & 7 & 8 & 17\\
Medium
& 6 & 8 & 4 & 3\\
High
& 10 & 7 & 10 & 2\\
\hline
\end{tabular}
\end{center}
\vspace{-6mm}
\end{table}
\section{Study Methodology}
\label{SEC-METHODOLOGY}
% To answer the questions, w
% We conducted an empirical study, recruiting individuals to test the DSL concrete syntaxes for a diverse set of scenarios and tasks.
\subsection{Recruitment}
As the study was part of a wider research project with a broad range of industry and academic partners in modelling platforms, we emailed an invitation letter to target individuals with relevant experience, with a request to indicate their availability for the 2-hour-long study. We shared the same email invite within the Computer Science department of the University of York, for undergraduate and postgraduate students and academics with experience in computer programming, modelling or MDE. Furthermore, the invitation has been shared with various research groups in academia and professionals working in the industry, with modelling expertise.
\subsection{Participants}
A total of 22 participants have participated in the study. The participants currently work either in the industry or academia: 5 participants work in the industry (3 companies across the UK, Germany and Romania), and 17 participants come from academia across 8 universities in the UK, Spain, Austria and Germany. Two of the academic participants were computer science students. Participants had a range of years of programming experience, with 10 participants with more than 10 years of experience, 5 participants with 7-10 years of experience, 5 participants with 4-6 years of experience, and 2 participants with less than 1 year. Table \ref{tab:demographics} provides a more detailed view of the knowledge and expertise of participants.
\subsection{Procedures}
The study was pilot-tested with two users (who did not participate in the final study). During the pilot, we identified practical issues and revised the study accordingly, e.g., to reduce the time needed for the study and improve the wording of task descriptions.
Once a participant agreed to join the study, the first author would contact them to arrange a time for the user study.
A facilitator, i.e., the first author, guided and observed participants during the sessions. The faculty research committee ethically approved the study.
Each participant joined a virtual meeting with video conferencing together with the facilitator via Zoom.%\footnote{a video conferencing tool that allows users to meet online through audio and video}.
At the beginning of the session, the facilitator described the context of the user study and shared two documents with the participant: an information sheet containing a description of the study and a consent form. The participant read these documents, and then signed the consent form if they agreed to continue with the user study session.
Next, the facilitator shared a 20-minute video tutorial, presenting the abstract and concrete syntaxes of the fuzzing-based testing DSL and practical examples of its usage in the Eclipse Integrated Development Environment (IDE) via the hybrid and tree editors. After watching the video, the participants had the opportunity to ask any questions regarding the presented information.
Each participant completed a Google Form %\footnote{web-based software for surveys and questionnaires that collects and stores responses}
survey, which contained the tasks %- for participants to complete -
and follow-up questions (e.g. regarding the correctness and confidence in their answers/solutions). Throughout the user study, the facilitator clarified any uncertainties related to the task descriptions. After completing all tasks, participants were asked to report their preferences, including features they liked and disliked about the editors. Lastly, the participants completed a demographic survey, including details of their years of experience and level of expertise in areas such as computer programming, modelling, MDE, DSLs and robotics software.
The participant connected remotely to the computer of the facilitator via TeamViewer. %\footnote{a tool that allows users to remotely access and control computers}.
The computer was running a Windows Operating System (OS). On the computer, two Eclipse IDE instances were running for defining and editing testing specifications: one containing the tree editor and another containing the hybrid editor. Participants executed given tasks with both syntaxes, i.e., a task had to be executed via both the tree and hybrid editors. Both editors used over the same semantic model, thus they were informationally equivalent.
To reduce memorising a task solution,
% prevent them from memorising
participants were not told that the underlying semantic model is the same.
To reduce bias and raise the generalisability of the results, the order in which the two types of editors were used for a specific task was interchanged between participants. Half of the participants first carried out a given task with the hybrid editor, and then performed the same task once more with the tree editor. For the other half, the order was reversed, thus participants first executed the task with the tree editor. By alternating the order we can reduce its effects on performance, confidence, or preference, as a task is typically performed more efficiently and faster the second time than the first.
% it is the first time.
To meticulously analyse how users performed, all sessions were audio- and video-recorded.
The participants were encouraged to think aloud while completing the tasks, to record and later analyse any relevant comments.
Also, a tool for visually highlighting mouse clicks was used to display a red-coloured circle around the mouse pointer when a mouse click occurred. This allows for counting only relevant mouse clicks when analysing the video recordings. For example, mouse clicks outside the editors are not relevant.
\subsection{Tasks}
Participants were given the tasks from Table~\ref{tab:tasks}, categorised into two types.
Tasks \hyperlink{U1}{U1}—\hyperlink{U5}{U5} focus on understanding a testing space, used to evaluate which type of model editor is better for comprehension.
Correspondingly, tasks \hyperlink{M1}{M1}—\hyperlink{M5}{M5} focus on modelling a testing space, used for evaluating which model editor provides better performance when modelling.
The table shows a summary of the tasks; more verbose and explicit descriptions have been given to the participants of the study.
%The tasks for understanding a testing space are used to evaluate which type of model editor is better for comprehension.
% Correspondingly, the tasks for modelling a testing space are used for evaluating which model editor provides superior performance when modelling.
All given tasks were driven by the industrial case study, aiming to reflect realistic modelling scenarios performed in practice by domain experts while covering a wide range of modelling activities such as finding specific elements, finding relationships and differences between elements, changing references, and creating and copying elements. Participants operate over the semantic model presented in Figure~\ref{FIG-TREE-EDITOR}, which contains a subset of the relevant model elements from the actual model used in the case study.
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.75}
\caption{\label{tab:tasks}Tasks for Understanding (U) and Modelling (M)}
\vspace{-3mm}
\begin{tabular}{|>{\centering}m{0.5cm}|>{\raggedright\arraybackslash}m{7.2cm}|}
\hline
\textbf{{ ID}} & \vspace{1.6mm} \textbf{{ Description}} \\[1mm]
\hline
\vspace{0.5mm} \hypertarget{U1}{U1} & \vspace{0.5mm} In the first test, find the \metaclass{Result Value} of the \metaclass{Metric Result} associated with \metaclass{FuzzingOperationTimesMetric}. \\
\hline
\vspace{0.5mm} \hypertarget{U2}{U2} & \vspace{0.5mm} In the first test, in the \metaclass{RandomValueFromSetOperation}, describe the activation starting condition. \\
\hline
\vspace{0.5mm} \hypertarget{U3}{U3} & \vspace{0.5mm} In the campaign result sets of the \metaclass{TestCampaign}, find the differences among the last 3 \val{NONDOM} sets. \\
\hline
\vspace{0.5mm} \hypertarget{U4}{U4} & \vspace{0.5mm} In the testing space, find the variable(s) related to the \metaclass{StreamMetric} named \val{collisionOccurrence}. \\
\hline
\vspace{0.5mm} \hypertarget{U5}{U5} & \vspace{0.5mm} In the testing space, find the fuzzing operation with the largest \metaclass{DoubleRange} interval. \\
\hline
\vspace{0.5mm} \hypertarget{M1}{M1} & \vspace{0.5mm} In the testing space, create a new \metaclass{TestCampaign} and set its name as \val{NewTestCampaign}. \\[0.25cm]
\hline
\vspace{0.5mm} \hypertarget{M2}{M2} & \vspace{0.5mm} Copy the first test from the prior existing \metaclass{TestCampaign} into the \val{NewTestCampaign}. \\
\hline
\vspace{0.5mm} \hypertarget{M3}{M3} & \vspace{0.5mm} In the \metaclass{RandomValueFromSetOperation}, modify the activation starting condition by replacing the operators. \\
\hline
\vspace{0.5mm} \hypertarget{M4}{M4} & \vspace{0.5mm} In the \metaclass{RandomValueFromSetOperation}, change the first referenced variable of the activation starting condition. \\
\hline
\vspace{0.5mm} \hypertarget{M5}{M5} & \vspace{0.5mm} Define from scratch the activation ending condition for the \metaclass{RandomValueFromSetOperation}. \\
\hline
\end{tabular}
\endgroup
\end{center}
\vspace{-5mm}
\end{table}
\vspace{1mm}\noindent
\textbf{Understanding Tasks (U1-U5).}
To perform the tasks using the tree editor, participants have to carefully inspect all model elements, and accordingly expand them to view their child elements and scan their attributes displayed in the Properties View. We describe next how a user should execute the tasks via the hybrid editor.
Task U1 requires participants to locate various model elements with specific values of interest. More specifically, the task asks participants to locate the first test contained in the test campaign from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} and view its details through a newly instantiated diagram by double-clicking the test. In the instantiated diagram, that looks like that in Figure~\ref{FIG-HYBRID-EDITOR-TEST}, participants must locate the \metaclass{Metric Result} that contains a \metaclass{FuzzingOperationTimesMetric}, and then identify its \metaclass{Result Value}. For the diagram from Figure~\ref{FIG-HYBRID-EDITOR-TEST}, the \metaclass{Result Value} is 0.01.
Task U2 asks participants to describe conditions represented through a tree representation and a textual representation. The task entails viewing the details of the first test as presented above, locating the \metaclass{RandomValueFromSetOperation}, finding the activation starting condition, and describing it.
In the tree editor, the activation starting condition looks as in Figure~\ref{FIG-TREE-EDITOR-CONDITIONS}, whereas in the hybrid editor, it looks as the textual expression displayed in the embedded textual editor from Figure~\ref{FIG-HYBRID-EDITOR-TEST}. When using the tree editor, the participant was asked to construct an equivalent textual expression as pseudocode. Correspondingly, when using the hybrid editor, the participant was asked to find the textual expression, describe its meaning, and categorise conditions as either \metaclass{BasicCondition} or \metaclass{CompositeCondition}, and identify the left and right branches of \metaclass{CompositeConditions}.
Task U3 asks participants to report differences or similarities among an ordered list of model elements.
Participants should locate the \metaclass{TestCampaign} from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} and analyse its campaign result sets using a newly instantiated diagram by double-clicking the \metaclass{TestCampaign}. In the instantiated diagram, like that in Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS}, participants must find the last 3 campaign result sets whose names are prefixed with \val{NONDOM}, and then notice their differences and similarities.
Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} contains a subset of campaign result sets from those used in the user study, which included 15 result sets.
Task U4 asks participants to find relationships (references) between elements.
Participants should find the \val{collisionOccurrence} \metaclass{StreamMetric} from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}, and then identify the related variables. For instance, as indicated by the arrows in Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}, the \metaclass{StreamMetric} is related to the variables \val{safetyzones/Z1} and \val{safetyzones/Z2}.
Task U5 asks participants to find specific types of model elements and perform minor mental computations. Participants should locate the fuzzing operation with the largest \metaclass{DoubleRange} interval. Note that the largest interval is that with the highest difference between the upper bound and the lower bound. Although multiple fuzzing operations existed in the testing space of the user study, in Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} there is only one fuzzing operation: the \metaclass{RandomValueFromSetOperation}, which has a \metaclass{DoubleRange} interval of [-0.3; 0.4].
\vspace{1mm}\noindent
\textbf{Modelling Tasks.}
In the tasks for modelling, participants edit existing model elements, create new ones, and change relevant values of their properties, via both editors. Note that the tasks for modelling are carried out after having completed the tasks for understanding; therefore, the participants have a considerable understanding of the testing space before engaging in these tasks.
Task M1 requests participants to create a new model element and set one of its properties. For instance, in the hybrid editor from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}, the participant would have to identify in the Palette the symbol related to \metaclass{TestCampaign}, then drag and drop it on the diagram, and lastly set its name using the Properties View.
Task M2 requires participants to execute operations within the model editors, specifically, the copy and paste operations. The participant would have to either use the context menu or a keyboard shortcut to trigger the copy operation of the first test, and then execute the paste operation into the new test campaign.
Task M3 asks participants to modify an existing condition, through its tree representation as in Figure~\ref{FIG-TREE-EDITOR-CONDITIONS} and its textual representation as in Figure~\ref{FIG-HYBRID-EDITOR-TEST}. More specifically, the task requires the participant to replace all \val{AND} operators with \val{OR} operators, and replace the comparison operators \val{LESS\_THAN} and \val{EQUALS} with \val{GREATER\_THAN}.
Task M4 requests participants to change a reference within a condition.
In particular, the task involves changing the first referenced metric from the activation starting condition with another metric. For instance, the first referenced metric in the condition from Figure~\ref{FIG-HYBRID-EDITOR-TEST} is \val{collisionOccurrence}, and participants had to use \val{fuzzingOperationTimes} instead.
This change would be realised in the tree editor by changing the reference via a selection menu, whereas in the hybrid editor, this would be done by either typing and deleting text manually, or by using the auto-completion menu that lists all metrics that can be referenced.
Task M5 requires participants to define a condition from scratch that meets certain requirements.
Participants should define the activation ending condition such that it contains one root \metaclass{CompositeCondition} containing two \metaclass{BasicConditions}. When defining conditions as in Figure~\ref{FIG-TREE-EDITOR-CONDITIONS}, multiple clicks are required to create new elements, whereas via the hybrid editor users can leverage the auto-completion menu to define the conditions textually.
\subsection{Data Collection and Analysis}
We analysed the video recordings and the surveys completed by the participants to derive the following metrics:
% Various metrics have been collected from the video recordings and from the surveys completed by the participants, as follows:
\vspace{1mm}\noindent\textbf{Duration}. The duration quantifies the amount of effort required to do a task, measuring the time-on-task, in seconds, within the editors.
The duration does not include time spent reading task descriptions. or
% It does not include time spent reading task descriptions.
time wasted when the remote access and control session via TeamViewer was lost. %Note that
Participants alternated %switched back and forth
between the Google form survey and the Eclipse IDE instances containing the editors, thus we were able to monitor the time spent on each.
\vspace{1mm}\noindent
\textbf{Correctness}. After performing all user study sessions, examined if participants completed the tasks correctly or made minor/major mistakes.
% we noticed that almost all participants completed the tasks correctly and completely. Otherwise, the participants made minor mistakes.
Thus, we marked solutions as either 1 (correct and complete) and 0 (minor mistakes). When a task was not solved correctly, we recorded the specific mistake(s) made.
\vspace{1mm}\noindent
\textbf{Errors Corrected}. We recorded the mistakes made during modelling, which were spotted and corrected at a later point by participants. In this case, a mistake represents a performed action that is either incorrect, redundant or certainly unrelated to the solution of a modelling task. This metric was collected for task M5, in which conditions are modelled from scratch.
\vspace{1mm}\noindent
\textbf{Clicks and Keystrokes}. Mouse clicks and keystrokes are metrics that reflect modelling effort \cite{hill2011measuring}. The number of clicks and keystrokes was recorded for the tasks related to the modelling of conditions, i.e., tasks M3 and M5.
\vspace{1mm}\noindent
\textbf{Confidence}. After performing each task, participants self-reported their confidence level on their solution, ranging from 1 (very low confidence) to 5 (very high confidence).
% The scale for the self-reported confidence is between 1 (very low confidence) to 5 (very high confidence).
% \begin{itemize}[leftmargin=2mm]
% \item \textbf{Duration}. The duration quantifies the amount of effort required to do a task, measuring the time-on-task, in seconds, within the editors.
% The duration does not include time spent reading task descriptions. or
% % It does not include time spent reading task descriptions.
% time wasted when the remote access and control session via TeamViewer was lost. %Note that
% Participants alternated %switched back and forth
% between the Google form survey and the Eclipse IDE instances containing the editors, thus we were able to monitor the time spent on each.
% \vskip 0.8mm
% \item \textbf{Correctness}. After performing all user study sessions, examined if participants completed the tasks correctly or made minor/major mistakes.
% % we noticed that almost all participants completed the tasks correctly and completely. Otherwise, the participants made minor mistakes.
% Thus, we marked solutions as either 1 (correct and complete) and 0 (minor mistakes). When a task was not solved correctly, we recorded the specific mistake(s) made.
% \vskip 0.8mm
% \item \textbf{Errors Corrected}. We recorded the mistakes made during modelling, which were spotted and corrected at a later point by participants. In this case, a mistake represents a performed action that is either incorrect, redundant or certainly unrelated to the solution of a modelling task. This metric was collected for task M5, in which conditions are modelled from scratch.
% \vskip 0.8mm
% \item \textbf{Clicks and Keystrokes}. Mouse clicks and keystrokes are metrics that reflect modelling effort \cite{hill2011measuring}. The number of clicks and keystrokes was recorded for the tasks related to the modelling of conditions, i.e., tasks M3 and M5.
% \vskip 0.8mm
% \item \textbf{Confidence}. After performing each task, participants self-reported their confidence level on their solution, ranging from 1 (very low confidence) to 5 (very high confidence).
% % The scale for the self-reported confidence is between 1 (very low confidence) to 5 (very high confidence).
% \end{itemize}
\section{Results and Discussion}
\label{SEC-RESULTS}
% This section discusses the results of the user study, from the perspective of performance, confidence and preference. Additionally, we will provide aggregated insights and preferences of participants, on the usage of the model editors.
We discuss the results of the user study, considering performance, confidence and preference.
Also, we provide aggregated insights and preferences of participants on the usage of the model editors.
\subsection{Performance (RQ1)}
We evaluate the performance when carrying out various tasks using the tree-based and hybrid editors by considering the metrics: duration, correctness, errors corrected and clicks and keystrokes.
% Research question RQ1 aims to evaluate the performance of
\vspace{1mm}\noindent
\textbf{Duration.}
Table~\ref{tab:durations_statistics} shows the absolute differences between the durations for completing the tasks via the tree (T) and hybrid (H) editors. The table is split into tasks for understanding (U) and modelling (M), and the five numeric columns indicate tasks U1—U5 and M1—M5.
The delta subtracts the duration when using the hybrid editor from the duration taken with the tree editor, i.e., $Delta = Duration(Tree) - Duration(Hybrid)$. For each task, the delta of the mean and median durations are displayed. A positive value signifies a higher duration with the tree editor compared to the hybrid editor; the opposite holds for a negative value.
\begin{table}[t]
\begin{center}
\begingroup
\caption{\label{tab:durations_statistics}Deltas of Mean and Median Durations (seconds)}
\vspace{-4mm}
\renewcommand*{\arraystretch}{0.7}
\begin{tabular}{|>{\centering}m{0.9cm}|>{\centering}m{0.9cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|>{\centering\arraybackslash}m{0.77cm}|}
\hline
\vspace{0.8mm}
\textbf{{Type}} & \vspace{0.8mm} \textbf{{Delta $(T - H)$}} & \vspace{0.8mm} \textbf{{ \#1}} & \vspace{0.8mm} \textbf{{\#2}} & \vspace{0.8mm} \textbf{{\#3}} & \vspace{0.8mm} \textbf{{\#4}} & \vspace{0.8mm} \textbf{{\#5}} \\
\hline
\vspace{2mm} \multirow{2}{*}{\textbf{U}} & \vspace{1mm} Mean & \vspace{1mm} 1.55 & \vspace{1mm} \significant{315.95} & \vspace{1mm} \significant{-15.32} & \vspace{1mm} 10 & \vspace{1mm} \significant{36.45} \\
\cline{2-7}
& \vspace{1mm} Median & \vspace{1mm} 1.50 & \vspace{1mm} \significant{301} & \vspace{1mm} \significant{-23} & \vspace{1mm} 0.50 & \vspace{1mm} \significant{32.50} \\
\hline
\vspace{2mm} \multirow{2}{*}{\textbf{M}} & \vspace{1mm} Mean & \vspace{1mm} 0.59 & \vspace{1mm} -9.50 & \vspace{1mm} \significant{27.09} & \vspace{1mm} 8.94 & \vspace{1mm} 9.51 \\
\cline{2-7}
& \vspace{1mm} Median & \vspace{1mm} -0.50 & \vspace{1mm} 2 & \vspace{1mm} \significant{34} & \vspace{1mm} 3.50 & \vspace{1mm} 19.50 \\
\hline
\end{tabular}
\endgroup
\end{center}
% \vspace{2mm}
\vspace{-4mm}
\end{table}
Completing task U1 required on average (median) 1.55 (1.50) seconds more with the tree editor than with the hybrid editor.
% Correspondingly, completing task U1 required a median duration that is higher by 1.50 seconds with the tree editor compared to the hybrid editor.
Thus, for solving task U1, which requires locating model elements with specific values of interest, the hybrid editor had slightly better performance; however, no significant difference between the two exists.
For task U2, the hybrid editor was faster on average by 315.95 seconds, indicating that using textual representations for expressing conditions improves comprehension significantly.
In task U3, the tree editor was faster by 15.32 seconds than the hybrid editor; the median value outlines a higher difference of 23 seconds. Concretely, the relative change reported to the mean and median durations of the hybrid editor was 15\% and 27\%, respectively. This shows that tree editors are more appropriate to explore and analyse ordered lists of model elements, as navigating long ordered element lists in diagrams often incurs overheads.
For task U4, the hybrid editor was faster by 10 seconds on average, however, this difference is negated by the delta median value, showing the existence of some outliers for this task.
We conclude that no significant difference exists in performance between the editors when executing a task that requires finding references between elements.
The hybrid editor yielded faster results in task U5, showing that finding elements that meet certain criteria is quicker in diagrams, as users have to meticulously expand and explore all elements in a tree editor.
Considering the modelling tasks, as indicated by the mean and median delta values of tasks M1 and M2, we cannot conclude that there is a difference in performance between the editors for these tasks.
For task M3, the hybrid editor resulted in improved performance, indicating that modifying existing conditions via a textual editor with syntax-aware editing features is quicker than through a tree editor.
In task M4, the hybrid editor was faster on average by 8.94 seconds, however the delta values of the median suggest a negligible difference between the performance of the editors.
Thus, for solving a task that requires changing a reference, the editors have similar performance, although the hybrid editor may be slightly faster. For task M5, the hybrid editor was faster on average (median) by 9.51 (19.50) seconds, however, the relative change reported to the mean and median durations of the tree editor was 8\% and 16\%, respectively, thus the difference in performance is minor.
\vspace{1mm}\noindent
\textbf{Correctness.}
All participants solved task U1 correctly through the hybrid editor, whereas three participants solved the task incorrectly.
From those, two participants reported an incorrect value as a solution, whereas one participant did not find a solution and abandoned the task. For task U2, two participants reported incorrect solutions with the hybrid editor, not being able to accurately state the left and right branches of the conditions represented textually. Correspondingly, seven participants solved the task incorrectly with the tree editor, as they constructed incorrect textual representations of the conditions by either grouping sub-conditions incorrectly, missing operators or using unsuitable operators.
All participants solved correctly tasks U3—U5 using both editors.
The results evidence that the hybrid editor surpasses the tree editor in correctly locating specific model elements and in comprehending conditions.
In modelling tasks, one participant incorrectly performed task M1 via the hybrid syntax. The participant tried to set the name of a \metaclass{TestCampaign} by double-clicking its name on the diagram, and then entering the name in the pop-up menu displayed after double-clicking. The participant forgot a detail shown in the video tutorial: double-clicking specific elements in the hybrid editor instantiates new diagrams. As such, the participant instantiated a new diagram with the name entered in the pop-up menu, without noticing that the name of the \metaclass{TestCampaign} had not been set. Correspondingly, all participants solved task M1 correctly through the tree editor.
Task M2 was solved correctly by all participants through both editors. One and six participants solved task M3 incorrectly, with the hybrid editor and tree editor, respectively. The most encountered mistake made through the tree editor was that participants did not expand all condition elements to view their child elements, to be able to locate the operators that needed to be replaced. All participants solved task M4 correctly with the hybrid editor, while only one participant failed using the tree editor.
For task M5, two participants incompletely defined the required condition from scratch with the tree editor, while one participant made a mistake when textually defining the required condition in task M5; specifically, they used an \val{AND} operator instead of an \val{OR} operator.
Consequently, we conclude that there is no significant difference between the editors for solving the tasks, with the exception of task M3, for which the hybrid editor yielded better results. %clearly superior.
Fine-grained changes to conditions require a high level of attention to detail, and the results indicate that it is more appropriate to modify existing conditions through a textual representation rather than a tree representation.
\vspace{1mm}\noindent
\textbf{Errors Corrected. }
The errors corrected have been recorded for task M5, in which participants defined conditions from scratch.
During the condition definition, 14 and 8 participants made incorrect actions with the hybrid editor and tree editor, respectively; the mistakes were corrected later in both cases.
% , which they later corrected. Correspondingly, 8 participants made mistakes when using the tree editor, that they later corrected.
The results highlight that more than half of the participants faced challenges during modelling with the hybrid editor.
The reason for the high number of mistakes is that the embedded textual editor from the hybrid editor allows any invalid input, such as typos.
For all participants, in total, 8 errors have been corrected with the tree editor and 17 errors have been corrected with the hybrid editor. The mistakes made with the hybrid editor were minor, e.g, participants typed \val{LESS\_THAN} instead of \val{GREATER\_THAN}, typed lowercase \val{OR} instead of uppercase, and used curly braces instead of parentheses for defining \metaclass{BasicConditions}. However, the errors that have been corrected in the tree editor were of a higher magnitude. For example, participants created a \metaclass{BasicCondition} as a root element, then continued modelling the conditions, and later realised that the root should have been a \metaclass{CompositeCondition}. Next, participants deleted the \metaclass{BasicCondition}, and defined once again the conditions from scratch, as the tree editor does not allow replacing a model element.
\vspace{1mm}\noindent
\textbf{Clicks and Keystrokes.}
Table~\ref{tab:clicks_statistics} shows statistics for clicks and keystrokes, %. The number of clicks and keystrokes have been
recorded for task M3, which required modifying existing conditions, and task M5, which involved modelling conditions from scratch. The delta subtracts the mean and median number of clicks or keystrokes when using the hybrid editor from the corresponding values %mean and median number of clicks or keystrokes
when using the tree editor. The results show that more clicks (24.32 on average) and fewer keystrokes (32.55 on average) are needed for solving task M3 with the tree editor. Similar to task M3, the results reveal that more clicks and fewer keystrokes are needed to solve task M5 with the tree editor compared to the hybrid editor.
The results are not surprising, %reflect what is typically expected,
as hybrid editors require more typing to define textual expressions, which are typically defined via clicks through a tree editor. %However, the way in which the editors are used will dictate the results. For instance, there has been a participant who completed task M3 using the tree editor with 21 clicks and 0 keystrokes, whereas there has been another participant who completed the same task via the hybrid editor with 21 clicks and 11 keystrokes, as many unnecessary clicks have performed.
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.7}
\caption{\label{tab:clicks_statistics}Deltas of Mean and Median No. Clicks, Keystrokes}
\vspace{-4mm}
\begin{tabular}{|>{\centering}m{1cm}|>{\centering}m{1cm}|
>{\centering}m{1.75cm}|
>{\centering}m{1cm}|>{\centering\arraybackslash}m{1.75cm}|}
\hline
\vspace{0.8mm}
\textbf{{Delta $(T - H)$}} & \vspace{0.8mm} \textbf{{Clicks M3}} & \vspace{0.8mm} \textbf{{Keystrokes M3}} & \vspace{0.8mm} \textbf{{Clicks M5}} & \vspace{0.8mm} \textbf{{Keystrokes M5}} \\
\hline
\vspace{1mm} Mean & \vspace{1mm} 24.32 & \vspace{1mm} -32.55 & \vspace{1mm} 29.70 & \vspace{1mm} -53.49 \\
\hline
\vspace{1mm} Median & \vspace{1mm} 25 & \vspace{1mm} -14.50 & \vspace{1mm} 30 & \vspace{1mm} -48 \\
\hline
\end{tabular}
\endgroup
\end{center}
% \vspace{2mm}
\vspace{-3mm}
\end{table}
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.7}
\caption{\label{tab:confidence_statistics}Deltas of Mean and Median Confidence Levels}
\vspace{-4mm}
\begin{tabular}{|>{\centering}m{0.9cm}|>{\centering}m{0.9cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|>{\centering\arraybackslash}m{0.77cm}|}
\hline
\vspace{0.8mm}
\textbf{{ Type}} & \vspace{0.8mm} \textbf{{ Delta $(T - H)$}} & \vspace{0.8mm} \textbf{{ \#1}} & \vspace{0.8mm} \textbf{{ \#2}} & \vspace{0.8mm} \textbf{{ \#3}} & \vspace{0.8mm} \textbf{{ \#4}} & \vspace{0.8mm} \textbf{{ \#5}} \\
\hline
\vspace{2mm} \multirow{2}{*}{\textbf{U}} & \vspace{1mm} Mean & \vspace{1mm} -0.68 & \vspace{1mm} \significant{-1.18} & \vspace{1mm} -0.14 & \vspace{1mm} 0.03 & \vspace{1mm} -0.05 \\
\cline{2-7}
& \vspace{1mm} Median & \vspace{1mm} -0.50 & \vspace{1mm} \significant{-1} & \vspace{1mm} 0 & \vspace{1mm} 0 & \vspace{1mm} 0 \\
\hline
\vspace{2mm} \multirow{2}{*}{\textbf{M}} & \vspace{1mm} Mean & \vspace{1mm} -0.09 & \vspace{1mm} 0.09 & \vspace{1mm} -0.18 & \vspace{1mm} -0.34 & \vspace{1mm} -0.32 \\
\cline{2-7}
& \vspace{1mm} Median & \vspace{1mm} 0 & \vspace{1mm} 0 & \vspace{1mm} 0 & \vspace{1mm} 0 & \vspace{1mm} 0 \\
\hline
\end{tabular}
\endgroup
\end{center}
% \vspace{2mm}
\vspace{-4mm}
\end{table}
\subsection{Confidence (RQ2)}
We answer research question RQ2 concerning the confidence of users when using both editors for carrying out the specified tasks by analysing the self-reported confidence level metric, with range between 1 (very low confidence) to 5 (very high confidence).
% As a reminder, research question RQ2 aims to evaluate the confidence of users when using both editors for carrying out various tasks. Recall that the confidence is a self-reported metric on a scale between 1 (very low confidence) to 5 (very high confidence).
%
Table~\ref{tab:confidence_statistics} presents statistics of the confidence level of participants concerning their solutions to the tasks.
As before, the table is divided into tasks for understanding (U) and modelling (M), and the five numeric columns are associated with tasks U1—U5 and M1—M5. The delta subtracts the mean and median confidence when using the hybrid editor from the mean and median confidence when using the tree editor. A positive value indicates a higher confidence with the tree editor compared to the hybrid editor, vice versa.
For task U1, participants were on average slightly more confident with their solution when using the hybrid editor. This suggests that users are slightly more confident with their solutions when using a hybrid editor, for tasks that require understanding a domain and locating various model elements with specific values of interest. Participants were on average more confident by 1.18 points, with their solution to task U2 when using the hybrid editor, signifying that users are more confident with their solutions when using a hybrid editor for understanding the meaning of conditions. For the remaining tasks, the results indicate no significant differences in the confidence of the solutions realised through the two editors.
We also analyse the correlation between incorrect solutions and confidence level, aiming to determine how confident participants are with their incorrect solutions. A mean confidence of 3.13 and a median confidence of 3 has been reported for the incorrect solutions to tasks U1—U5 when using the tree editor.
This shows that participants are neutral about the correctness of their incorrect solutions. Considering that their solutions have been partially correct, this result is expected.
Further, a mean and median confidence of 4.5 has been reported for the incorrect solutions to tasks M1—M5 when using the tree editor. Similarly, a mean confidence of 4.67 and a median confidence of 5 has been reported for the incorrect solutions to tasks M1—M5 when using the hybrid editor. These results show that participants were confident with their partially incorrect solutions, as they had not noticed any faults. This finding suggests that the confidence level is related to the ability of users to spot their mistakes or inadequacies.
Participants realised that they could not solve tasks that verify their understanding, but they did not observe some of their mistakes in tasks that required modelling.
\subsection{Preference (RQ3)}
We answer research question RQ3 concerning the preferences of users with respect to the two editor types.
Before engaging in the study, participants were asked for their preferred editor type for modelling. As reported, 3 participants preferred a tree editor, 11 participants a hybrid editor, and 8 participants stated no preference. After performing the given tasks, the participants were asked again about their preferences. As reported, 2 participants preferred a tree editor, 16 participants (73\%) a hybrid editor, and 4 participants had no preference. The results show that one participant changed their preference from a tree editor to a hybrid editor, and half of the participants with no initial preference now prefer a hybrid editor.
Concerning understanding a testing space that does not contain condition expressions, 12 participants reported their preference on a diagrammatic editor, 7 participants for a tree editor, and 3 participants have no preference.
For exploring and navigating a testing space that does not contain condition expressions, 13 participants preferred a diagrammatic editor, 7 participants a tree editor, and 2 participants have no preference.
For finding relationships between the elements of a testing space that does not contain condition expressions, 19 participants preferred a diagrammatic editor, 0 participants a tree editor, and 2 participants have no preference.
For understanding the meaning of condition expressions, 17 participants stated they prefer a textual representation, 3 participants a tree representation, whereas 2 participants have no preference.
For exploring to and within condition expressions, 17 participants preferred a hybrid editor, 4 participants a tree editor, and 1 participant has no preference.
For modelling conditions from scratch, 16 participants stated they prefer a textual representation, 3 participants a tree representation, and 3 participants have no preference.
For modifying existing conditions, 17 participants preferred a textual representation, 4 participants a tree representation, and 1 participant has no preference. As reported, 20 participants consider that adding a textual syntax for modelling conditions inside the tree editor would improve usability and modelling efficiency, whereas 2 participants did not share this opinion.
The feature of the tree editor that participants particularly liked was its conciseness and hierarchical representation that evidences the order and containment of elements. Participants also noted that the tree editor provided a better sense of control over the model, enabling faster exploration, especially when using the keyboard. However, several participants reported difficulties to understand and define complex conditions with the tree editor. Many participants would have liked the tree editor to use graphical symbols and colour-coding, to be able to differentiate between element types.
Concerning the hybrid editor, participants particularly liked the flexibility in defining complex conditions and finding relationships between elements. Several participants liked the visual aspect of the hybrid editor, as they could relate colours and symbols to specific element types.
However, some participants did not like the need to switch between multiple diagrams and considered that diagrams would not be readable for large models. Also, the textual syntax of conditions was not favoured by all participants, as it contained multiple kinds of brackets for grouping sub-conditions rather than a single kind, and it did not provide any formatting (pretty printing).
\subsection{Threats to Validity}
\label{SEC-THREATS}
% This study is subject to a number of threats to validity that are discussed in the following.
\vspace{0.1mm}\noindent
\textbf{Construct Validity. }
A construct validity threat is that participants may misinterpret some task descriptions, although they would be able to eventually understand the descriptions with the help of the session facilitator.
If participants begin carrying out a task without completely understanding its description, more time would be needed to solve the task, and more mistakes would be made that may later be corrected by them.
Thus, the performance metrics for the duration, correctness and errors corrected are by design influenced by the clarity and understandability of task descriptions.
\vspace{0.5mm}\noindent
\textbf{Internal Validity. }
The main internal validity threat is that the study has been carried out by accessing a remote computer via TeamViewer.
For tasks M3 — M5, participants used keyboard shortcuts, e.g., control+space to display an auto-completion menu in the textual editor from the hybrid editor.
Some participants faced challenges, either because TeamViewer randomly did not trigger the actions associated with the keyboard shortcuts, or because they did not know how to correctly trigger keyboard shortcuts, due to their limited knowledge of the Windows OS. In such instances, the duration reported with the hybrid editor were higher than expected.
\vspace{0.5mm}\noindent
\textbf{External Validity. }
An external validity threat is that the tree and hybrid editors have been evaluated with a single concrete syntax.
Using a more appealing hybrid notation, with a different graphical representation and with a more intuitive textual syntax, may yield a better performance. Although the tasks covered a broad range of modelling activities, they were uncomplicated, as almost all participants could solve all tasks correctly.
Further experiments would be useful to strengthen the generalizability of our findings.
% Therefore, another threat to external validity is the elementary nature of the tasks.
\section{Related Work}
\label{SEC-RELATED-WORK}
\input{dslpapers}
%\textbf{Hybrid DSLs}
%\lipsum[3]
% ACM's consolidated article template, introduced in 2017, provides a
% consistent \LaTeX\ style for use across ACM publications, and
% incorporates accessibility and metadata-extraction functionality
% necessary for future Digital Library endeavors. Numerous ACM and
% SIG-specific \LaTeX\ templates have been examined, and their unique
% features incorporated into this single new template.
\vspace{-3mm}
\section{Conclusions and Future Work}
% \vspace{-1mm}
\label{SEC-CONCLUSIONS}
% We explored the tradeoffs between tree-based and hybrid graphical-textual model editors.
% We executed an empirical user study in the context of an industrial use case, investigating the performance, confidence and preferences of users for both model editor types.
Towards understanding the tradeoffs between tree-based and hybrid graphical-textual model editors, we conducted an empirical study using an industrial use case, analysing the performance, confidence and preferences of users for both model editor types.
Concerning performance, there was a tradeoff between the speed of executing tasks using the editors.
% certain activities can be done faster via a hybrid editor, whereas other activities with a tree editor.
In particular, the hybrid editor was better for comprehension of complex conditions, while the tree editor was better for analysing ordered lists of model elements.
When modelling conditions, more corrected errors were recorded with the hybrid editor, however, the mistakes made were minor.
Also, more keystrokes and fewer clicks were required for modelling conditions with the hybrid editor.
Users were more confident with their solutions when using the hybrid editor for understanding the meaning of conditions.
Finally, before the study, 50\% of participants preferred a hybrid editor; at the end, the percentage increased to 73\%, showing an increased preference towards the hybrid editor.
We plan to validate the generality of our findings by using multiple graphical tree-based and hybrid syntaxes. Also, tree-based and hybrid editors should be evaluated with other domains and case studies, by carrying out a wider range of modelling activities.
\vspace{2mm}
% \begin{acks}
\noindent
\textbf{Acknowledgements.}
We thank our SESAME partners and the study participants.
This work was supported by
the SCHEME InnovateUK project (\#10065634),
the EU Horizon 2020 project SESAME (\#101017258), and
the Horizon Europe project SOPRANO (\#101120990).%. This work was also partially supported by
%(Safety Critical Harsh Environment Micro-processing Evolution)
% \end{acks}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
\end{document}
\endinput
%%
%% End of file `sample-sigplan.tex'.
\subsection{KUKA Case Study}
The KUKA use case is an industrial case study used by KUKA AG in
gearbox manufacture \cite{kuka-deliverable}. This use case
incorporates mobile robots to assemble the gear boxes, together with
human workers surrounding the assembly cell who assist the process. In
testing we aim to guarantee the safety requirement that robots must
not generate hazards to the human operators during operation. The
primary safety consideration is to ensure that robots do not enter
regions which could cause a collision hazard to human operators placed
around the outside of the assembly cell frame. Under normal
circumstances, motion trajectories are pre-planned to ensure
conformity to this safety requirement. However, faults in
communication, sensing or motor/interlock actuation may lead to robots
malfunctioning and rotating in order to enter these forbidden areas.
The simulation scenario was supplied for testing by end users,
implemented using the TTS simulation platform \cite{tts-simulator}.
\begin{figure}
\centering
\includegraphics[width=\linewidth]{figures/ttsScreenshot.png}
\caption{KUKA assembly case primary view in TTS simulator}
% TODO: new diagram here of the complete simulation
\label{fig:kuka-case-study}
\end{figure}
The requirements for the case study are illustrated in Table
\ref{table:caseStudyTTSReqs}. The layout of the assembly cell is
illustrated in Figure \ref{fig:kuka-case-study}. The primary safety
requirement R1 is that the safety zones which the robots are forbidden
to enter are as illustrated as blue cuboid regions (Z1, Z2, Z3) on the
right-hand side of the external cell frame in Figure
\ref{fig:kuka-case-study}. If the region surrounding the gripper of
the right-hand robot (green cylinder labelled ZR3200) collides with
these regions, it will trigger violations which will be tracked and
counted as towards the static safety violation metric. The output
metric $staticCollision$ is used to quantify R1 by tracking the number
of intervals in which a collisions of the gripper safety zone with any
safety zone occured. The interval length is set to 0.2 seconds, which
provides independence of the event rate of the simulation.
Following a safety analysis, we have identified the communication from
the controller to the robot arm joints as vulnerable to message
disruptions. Therefore, requirement R1 will be tested by applying
fuzzing, altering the target motion value for robot arm joints with
jitter around the intended point. A part of the fuzzing model
instantiated for the KUKA industrial use case is presented in Figure
\ref{FIG-DSL-TTS-EXAMPLE}. The operations
$attackJointR3200ProductMove\_Link1$ and
$attackJointR3200ProductMove\_Link0$ of the model illustrate this
fuzzing offset value change, using the \emph{RandomValueFromSet}
fuzzing operation to apply relative changes to the joint value time
series.
\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{figures/tts-model-example-partial}
\caption{A fragment of the DSL example for the TTS use case
showing selected metrics and fuzzing operations}
\label{FIG-DSL-TTS-EXAMPLE}
\end{figure}
\subsection{Fuzzing Domain-Specific Language}
This section defines the domain-specific language (DSL) metamodel used
in this study, that specifies the fuzzing space, fuzzing operations,
and associated results. Employing DSLs tailored to a specific domain
is a highly desirable technique for extending the applicability of
modern software and systems engineering, enabling non-expert users to
easily test systems using our framework
~\cite{casalaro2021model,bruyninckx2013brics,gerasimou2019towards}.
In this paper, we explore two syntaxes for this metamodel, a graphical
syntax using the Exceed tree-based editor, and a hybrid syntax
developed using Sirius.
\begin{figure*}[!tb]
\centering
\includegraphics[width=0.93\linewidth]{figures/dsl-metamodel.jpg}
\caption{UML classes for the core elements of the testing space DSL}
\label{FIG-TESTSPACE-DSL}
\end{figure*}
A UML diagram showing a fragment of the DSL with three of the main
classes included is presented in Figure \ref{FIG-TESTSPACE-DSL}. The
figure presents the TestingSpace, TestCampaign and Test
classes. TestingSpace is the DSL's root element, consists of a set of
permissible fuzzing operations (possibleOperations) that collectively
specify the boundaries of the potential fuzzing space that can be
explored. The testing space also includes performance metrics which
are used to quantify the robotic system performance in regards to
safety violations, which allow multi-objective optimisation of system
performance to be performed.
The TestCampaign class specifies an experiment that can be performed
and sets parameters for a specific fuzzing experiment. A TestCampaign
references a choice of particular metrics to use in evaluating that
campaign. The includedOperations reference list permits selecting a
subset of fuzzing operations in order to constrain an experiment. The
TestGenerationApproach allows the user to specify the parameters for
an experiment by selecting one of these subclasses. For example,
NSGAEvolutionaryAlgorithm allows an evolutionary experiment with the
NSGA-II algorithm \cite{nsga2}. The performedTests attribute is
populated during the execution of experiments, with Tests generated
and executed for that campaign. The resultSets attribute is also
populated as the experiments proceed.
The Test class represents one test configuration that can be applied
to the MRS, corresponding to a particular selection of fuzzing
operations. During execution of the Test, metric instances will also
be recorded and stored. The latter are represented by the containment
of MetricInstances, which record performance metric results for the
evaluation of that particular Test. This provides a record of the
impact of the fuzzing performed.
The FuzzingOperation class represents one specific fuzzing operation -
an entity which represents a specific strategy for making runtime
modifications or disruptions to the MRS. Subclasses of
FuzzingOperation represent specific semantics of this fuzzing
operation; for example, the PacketLossNetworkOperation class
represents probabilistic loss of a proportion of packets. Currently,
\emph{variableToAffect} is the main attribute used when setting up
variable subscriptions and defining the variables to which fuzzing
operations are applied. Other FuzzingOperation subclasses such as
RandomValueFromSetOperation permits the replacement of structured
parameters within a message with newly generated random values.
The CampaignResultSet class represents either the final outcome of a
particular campaign, or partial intermediate results obtained during
its execution. An enumeration, ResultSetStatus, which is set to either
FINAL or INTERMEDIATE, determines this status of a result set. Result
sets contain references to particular Tests that comprise the
results. For example, in an evolutionary experiment, the set of Tests
for a final result set would contain the Pareto front obtained during
an experiment. Intermediate result sets would allow the user to
monitor the progression of the experiments and examine the changes in
the front at particular generations as the experiment progresses.
\subsection{Condition-Based Fuzzing}
Condition-based fuzzing provides the ability to activate and
deactivate fuzzing based upon custom scenario-specific events,
potentially incorporating complex mission logic, spatial and geometric
properties. A condition is a logical expression built hierarchically
from comparison and logical operations, incorporating state variables
from the robotic system during mission execution. The values of
condition variables change dynamically during the execution of the
mission, with every variable value supplied from an equally named
performance metric. Conditions can restrict the search space, making
the evolution of testing campaigns more tractable.
Activation of a condition-based fuzzing record is controlled by start
and end conditions. For each fuzzing record in the inactive state, the
start conditions will be evaluated at runtime on every simulation
event in order to determine a boolean status. If the condition
evaluates to true, that fuzzing record will be set to active. Then,
for every applicable event, the fuzzing operation will be invoked to
transform the incoming events for that fuzzing record. Once activated,
the fuzzing record remains active until the end condition evaluates to
true. Once the end condition is satisfied, fuzzing is deactivated for
this record. We assume that conditions can only be activated once,
such that a record which is activated is no longer available to be
reactivated later.
A primary intent of our paper is to assess the comprehensibility of
various syntaxes for understanding the meaning of particular
expressions. The DSL fragment for condition expressions is contained
in a seperate package, and its presented in Figure
\ref{FIG-CONDITIONS-DSL}. The Condition class consists of an abstract
ConditionElement, which is either a BasicCondition or
CompositeCondition. A BasicCondition represents comparisons of a
ConditionVariable with a ConditionExpression, which is either a
ConditionVariable or ConditionInteger. The BinaryCompisonOperation is
either LESS\_THAN, GREATER\_THAN, or EQUALS. ConditionVariables
contain a reference to an associated Metric that supplies its value,
and ConditionIntegers merely the primitive integer. A
CompositeCondition contains two other ConditionElements in the left
and right branches, together with a BinaryLogicalOperation. These
logical operations permit either AND or OR operations, and allow the
nesting of condition expressions to be arbitrarily deep.
\begin{figure}[!tb]
\centering
\includegraphics[width=\linewidth]{figures/dsl-metamodel-conditions.jpg}
\caption{UML classes for the core elements of the testing space DSL}
\label{FIG-CONDITIONS-DSL}
\end{figure}
Since condition expressions are dynamically generated during fuzzing
experiments, they may produce complex multi-level expressions. In this
paper, a major aspect is to investigate how different DSL syntaxes
permit the comprehension of deeply nested condition expressions,
allowing the user to interpret their meanings. We will also
investigate how the syntaxes allow the users to manually modify these
syntaxes.
%\subsection{MDE for MRS and Testing}
% TODO: sort the papers into categories
% TODO: cut detail on long papers
\vspace{1.3mm}\noindent
\textbf{DSLs for Robotics.}
Model driven notations and toolchains have been used in
RobotML~\cite{dhouib2012robotml}, BRICS~\cite{bruyninckx2013brics},
SmartSoft~\cite{schlegel2009robotic},
RoboChart~\cite{miyazawa2019robochart}, and in engineering robotic
perception and control
~\cite{ciccozzi2017engineering,schlegel2010design}. Developing
model-driven solutions for the robotics domain is an established area,
with a common mechanism for MDE engineering using finite state
machines and statecharts
\cite{gascuena2012model,paraschos2012model,skubch2011modelling}.
% Removing this to save space
%However, multi-robot interactions are under-studied, with
%\cite{cattivera2015model} surveying studies and finding only a
%minority dealing with MRS rather than single robots.
MRS interactions rather than single robots are studied
in \cite{cattivera2015model}. The FLYAQ family of graphical DSLs
~\cite{ciccozzi2016adopting} model the structure and behaviour of
multi-robot aerial systems, considering their spatial and
situational surroundings and tasks assigned to robots and their
behaviour. FLYAQ is extended with a specification language in~\cite{dragule2017generated}, enabling the definition of
domain-specific constraints in a declarative manner
and~\cite{pinciroli2016buzz} proposes a textual DSL for specifying the
behaviour of robot swarms.
\vspace{1.3mm}\noindent
\textbf{Testing DSL Tools.}
FuzzFactory ~\cite{padhye2019fuzzfactory} is a DSL for developing
domain-specific fuzzing applications. It generates code necessary for
fuzzing applications to communicate with a fuzzer, and obtain dynamic
domain-specific feedback during testing.
VerifAI \cite{verifai-cav19} proposes a complementary
approach for tool-supported testing of cyber-physical systems,
typically integrated with AI and machine learning approaches.
%It uses a runtime monitor to assess the performance of the MRS, together with
%a search process to automatically find counterexamples.
VerifAI uses Scenic \cite{scenic-2019}, which presents a scenario
testing DSL, capable of specifying probabilistic features that can be
combined to generate test cases of interest.
\vspace{1.3mm}\noindent
\textbf{DSL Evaluation.}
FQAD~\cite{fqad-dsls-2013} introduces a methodology for DSL
evaluation, considering quality characteristics like expressiveness,
usability, and productivity to assess DSL suitability.
%Using these characteristics and
%evaluator priorities, the process emits an assessment of DSL
%suitability. The technique was used in multiple case studies and
%found to be useful by stakeholders.
Despite its usefulness in multiple case studies, FQAD relies on
potentially subjective user perspectives, and also lacks
empirical validation in textual languages. Similarly, our empirical
study relies on subjective data such as user preferences; however, it
is also based on objective information, e.g., correctness, number of
clicks, errors corrected.
DSL evaluation for multi-agent systems is used in~\cite{mas-dsl-2016},
analysing language (elements and transformations), execution and
quality. The results show that 80\% of the final product code could be
auto-generated. However, it did not consider mistakes from language
users and their impact on learnability by new users. The textual DSL
Athos~\cite{athos-2022} for specifying transport and traffic routing
simulations was evaluated to determine its effectiveness and
efficiency in model creation and understanding. Athos was found
useful in modelling time and test scores when used by users
inexperienced in software development, and to a lesser degree, in more
experienced users.
User comprehensibility of a business use case between textual and
graphical flowchart (BPMN) representations is explored
in~\cite{ottensooser-compare-2010}. All user groups could
understand the textual representation (especially strong readers) and
experienced domain analysts obtained increased comprehension from
BPMN.
%Users with
%proxies for domain experience (business analysts) obtained increased
%comprehension from the graphical BPMN notation.
This suggests comprehension can be optimised by showing both
representations; first textual and then graphical. This may
not apply directly to our study where both representations are
significantly graphical, with the textual elements restricted to a
subset of the syntax (condition expressions).
In \cite{familiar-jaksic-2014}, tests were performed to determine
whether users could better create and manage feature models (FMs)
using a diagrammatic representation, as compared to a textual
representation.
The results are aligned with our insights and showed statistically that users authored FMs of a higher quality and managed FMs with better efficiency and effectiveness with the diagrammatic representation.
Finally, in \cite{whispers-jolak-2020}, comparisons were performed between a graphical UML language and textual descriptions of the associated system, a familiar MVC fitness centre architecture. It found that the graphical representation is generally better than a textual description of the associated system, which broadly corroborates the findings in our investigation.
%
% \camera{In \cite{progeducation-saito-2017}, the effects of text-based and visual-based input on first-time programming learning were investigated, finding that the visual input method was more appropriate for an introduction to programming. However, the results are not comparable to those from our user study, due to different participant backgrounds (users in this study were unfamiliar with programming and formal grammar, whereas the participants from our user study were trained computer scientists and industry professionals).}
%%
%% This is file `sample-sigplan.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `sigplan')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigplan.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%% Commands for TeXCount
%TC:macro \cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%%
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[sigconf]{acmart}
\usepackage{enumitem}
\usepackage{lipsum}
% \usepackage{subfig}
\usepackage{subcaption}
\usepackage{makecell}
\usepackage[usestackEOL]{stackengine}
\usepackage{multirow}
\usepackage{float}
\newcommand{\ip}[1]{{\color{purple} [IP: #1]}}
\newcommand{\agd}[1]{{\color{blue} [AGD: #1]}}
\newcommand{\dk}[1]{{\color{teal} [DK: #1]}}
\newcommand{\sg}[1]{{\color{magenta} [SG: #1]}}
\newcommand{\jrh}[1]{{\color{green} [JRH: #1]}}
\newcommand{\jrhchanged}[1]{{\color{blue} [JRHCHANGED: #1]}}
\newcommand*{\metaclass}[1]{\emph{#1}}
\newcommand*{\attr}[1]{\emph{#1}}
\newcommand*{\val}[1]{\emph{#1}}
\newcommand{\quotes}[1]{``#1''}
\newcommand{\camera}[1]{{\color{magenta}#1}}
%% NOTE that a single column version is required for
%% submission and peer review. This can be done by changing
%% the \doucmentclass[...]{acmart} in this template to
%% \documentclass[manuscript,screen,review]{acmart}
%%
%% To ensure 100% compatibility, please check the white list of
%% approved LaTeX packages to be used with the Master Article Template at
%% https://www.acm.org/publications/taps/whitelist-of-latex-packages
%% before creating your document. The white list page provides
%% information on how to submit additional LaTeX packages for
%% review and adoption.
%% Fonts used in the template cannot be substituted; margin
%% adjustments are not allowed.
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
%% These commands are for a PROCEEDINGS abstract or paper.
\copyrightyear{2024}
\acmYear{2024}
\setcopyright{acmlicensed}\acmConference[MODELS '24]{ACM/IEEE 27th International Conference on Model Driven Engineering Languages and Systems}{September 22--27, 2024}{Linz, Austria}
\acmBooktitle{ACM/IEEE 27th International Conference on Model Driven Engineering Languages and Systems (MODELS '24), September 22--27, 2024, Linz, Austria}
\acmDOI{10.1145/3640310.3674102}
\acmISBN{979-8-4007-0504-5/24/09}
%
% Uncomment \acmBooktitle if th title of the proceedings is different
% from ``Proceedings of ...''!
%
%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
% June 03--05, 2018, Woodstock, NY}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title[Tree-Based versus Hybrid Graphical-Textual Model Editors: An Empirical Study]
% {An Empirical Study of Hybrid Graphical-Textual\\ Model Editors against Tree-Based Model Editors}
% {An Empirical Study of Testing Specification Model Editors: Tree-Based versus Hybrid Graphical-Textual}
% {An Empirical Study of Testing Specifications: Tree-Based Model Editors versus Hybrid Graphical-Textual Model Editors}
% {An Empirical Study of Testing Specifications:
% \\Tree-Based versus Hybrid Graphical-Textual Model Editors}
{Tree-Based versus Hybrid Graphical-Textual Model Editors:
\\An Empirical Study of Testing Specifications}
%% NEW TITLE NEEDED
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
% \author{Ionut Predoaia}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{ionut.predoaia@york.ac.uk}
% \author{James Harbin}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{james.harbin@york.ac.uk}
% \author{Simos Gerasimou}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{simos.gerasimou@york.ac.uk}
% \author{Christina Vasiliou}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{christina.vasiliou@york.ac.uk}
% \author{Dimitris Kolovos}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{dimitris.kolovos@york.ac.uk}
% \author{Antonio García-Domínguez}
% \orcid{1234-5678-9012}
% \affiliation{%
% \institution{University of York}
% \city{York}
% \country{United Kingdom}
% }
% \email{a.garcia-dominguez@york.ac.uk}
% \newcommand{\authorsN}[1]{#1}
\author{Ionut Predoaia, James Harbin, Simos Gerasimou}
\author{Christina Vasiliou, Dimitris Kolovos, Antonio García-Domínguez}
% \authorsN{Ionut Predoaia, James Harbin, Simos Gerasimou, Christina Vasiliou, Dimitris Kolovos, Antonio García-Domínguez}
\affiliation{%
\institution{University of York, United Kingdom}
% \city{York}
\country{}
}
\email{ionut.predoaia, james.harbin, simos.gerasimou, christina.vasiliou, dimitris.kolovos, a.garcia-dominguez@york.ac.uk}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Predoaia et al.}
\newcommand{\squishlist}{
\begin{list}{$ullet$}
{ \setlength{\itemsep}{0pt}
\setlength{\parsep}{3pt}
\setlength{\topsep}{3pt}
\setlength{\partopsep}{0pt}
\setlength{\leftmargin}{1.5em}
\setlength{\labelwidth}{1em}
\setlength{\labelsep}{0.5em} } }
\newcommand{\squishlisttwo}{
\begin{list}{$ullet$}
{ \setlength{\itemsep}{0pt}
\setlength{\parsep}{0pt}
\setlength{\opsep}{0pt}
\setlength{\partopsep}{0pt}
\setlength{\leftmargin}{2em}
\setlength{\labelwidth}{1.5em}
\setlength{\labelsep}{0.5em} } }
\newcommand{\squishend}{
\end{list} }
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
%\agd{define acronyms on their first use}
\begin{abstract}
Tree-based model editors and hybrid graphical-textual model editors have advantages and limitations when editing domain models. Data is displayed hierarchically in tree-based model editors, whereas hybrid graphical-textual model editors capture high-level domain concepts graphically and low-level domain details textually. We conducted an empirical user study with 22 participants to evaluate the implicit assumption of system modellers that hybrid notations are superior, and to investigate the tradeoffs between the default EMF-based tree model editor and a Sirius/Xtext-based hybrid model editor. The results of the user study indicate that users largely prefer the hybrid editor and are more confident with hybrid notations for understanding the meaning of conditions. Furthermore, we found that the tree editor provided superior performance for analysing ordered lists of model elements, whereas activities requiring the comprehension or modelling of complex conditions were carried out faster through the hybrid editor.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10011007.10011006.10011050.10011017</concept_id>
<concept_desc>Software and its engineering~Domain specific languages</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011006.10011050.10011058</concept_id>
<concept_desc>Software and its engineering~Visual languages</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10011007.10011074.10011099.10011693</concept_id>
<concept_desc>Software and its engineering~Empirical software validation</concept_desc>
<concept_significance>500</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Software and its engineering~Domain specific languages}
\ccsdesc[500]{Software and its engineering~Visual languages}
\ccsdesc[500]{Software and its engineering~Empirical software validation}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{Hybrid Notations, Model Editors, Fuzz Testing, Empirical Study}
%\jrh{Set CCS Concepts and Keywords}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
\label{SEC-INTRO}
Domain-specific languages (DSLs) can have graphical, textual, or hybrid syntaxes. Hybrid graphical-textual DSLs are languages that have a part-graphical and part-textual syntax, i.e., a hybrid graphical-textual syntax. They are particularly appropriate for cases in which one would like to use graphical representations for capturing high-level domain concepts, and textual representations for defining low-level domain details, such as behaviour and complex expressions~\cite{cooper2019engineering}. Furthermore, they are effectively used via hybrid graphical-textual model editors \cite{predoaia2023streamlining,zolotas2018towards,wei2020automatic}, i.e., editors that enable the definition of domain models via graphical and textual representations.
Throughout this paper, we use the term \textit{model editor} instead of the term \textit{workbench}; a workbench covers a broader scope, including a model editor and other components like menus, toolbars and perspectives~\cite{burnette2005eclipse}. For brevity, we use the term \textit{hybrid} instead of \textit{hybrid graphical-textual}, and the term \textit{editor} refers to a \textit{model editor}.
Choosing between a tree-based and a hybrid editor for editing domain models is not a straightforward decision, as both have advantages and drawbacks. Tree editors display domain data hierarchically and in a structured manner by exposing parent-child relationships. Diagrammatic editors contained within hybrid editors do not provide explicit structure, and they are not appropriate for displaying parent-child relationships of multiple levels. For instance, parent elements can be displayed as containers that incorporate their child elements. However, if the children have corresponding child elements, which in turn have child elements as well, displaying all these elements in a diagram can hinder comprehension, as they would effectively be displayed as a container that contains other containers, that in turn contain other containers incorporating a set of leaf child elements. Thus, in such cases, it may be preferred to use multiple diagrams, e.g., by instantiating new diagrams containing the child elements of a parent element. Consequently, the domain data is split across multiple diagrams, and users may face difficulties in being able to relate them. A common drawback of hybrid editors is the initial cost of learning the associated part-graphical and part-textual syntaxes. However, once learned, models may potentially be edited faster, considering also that typically fewer clicks are required due to the definition of textual expressions~\cite{addazi2017towards}.
Acknowledging the importance of selecting the most suitable editor to support effective and efficient modelling, in this paper we investigate the tradeoffs between tree-based and hybrid editors. System modellers often take for granted the assumption that hybrid notations are superior, however, this claim is not yet supported by empirical evidence. To evaluate this implicit assumption, we conducted an empirical user study to analyse the editor type preferred by system modellers. We recruited 22 subjects, that are either students or professionals working in industry or academia, with various levels of knowledge and expertise in computer programming, modelling, and model-driven engineering (MDE). In the user study, the participants used a DSL that enables the definition of testing specifications of robotic systems in simulation. Each participant used the DSL via both a tree editor and a hybrid editor to execute a set of tasks that were inspired by an industrial case study of the large European Commission research project SESAME (Secure and Safe Multi-Robot Systems)~\cite{sesame_website}. The user study aims to evaluate the performance and preferences of users, and provide insights on the usage of hybrid editors and tree editors.
Our empirical study revealed that users largely prefer hybrid editors and are more confident with hybrid editors for understanding the meaning of conditions. From a performance perspective, the results pinpoint that hybrid editors are superior for activities involving the comprehension or modelling of complex conditions, whereas tree editors are more efficient for tasks concerning the exploration and analysis of ordered lists of model elements.
%Furthermore, the results evidence that hybrid editors require more keystrokes and fewer clicks for modelling conditions.
\textbf{Paper structure.} Section~\ref{SEC-BACKGROUND} introduces the necessary background, Section~\ref{SEC-CASE-STUDY} describes our industrial case study, and Section~\ref{SEC-DSL} presents the fuzzing-based testing DSL. Sections~\ref{SEC-METHODOLOGY} and~\ref{SEC-RESULTS} detail the methodology and results of the user study, respectively.
Section~\ref{SEC-THREATS} identifies threats to validity. Section~\ref{SEC-RELATED-WORK} presents related work and, finally, Section~\ref{SEC-CONCLUSIONS} concludes the paper, providing future work directions.
\section{Background}
\label{SEC-BACKGROUND}
\vspace{1mm}\noindent
\textbf{Modelling Editors.}
Hybrid editors are commonly composed of graphical diagram(s), embedded textual editor(s) with syntax-aware editing features, such as syntax highlighting, auto-completion, and refactoring, a view in which one can edit the properties of a selected model element, a view with various symbols that can be dragged and dropped on top of diagrams to create various types of model elements, and a view that consistently reports errors from the model \cite{cooper2019engineering}. Textual expressions are able to reference model elements that are defined graphically in diagrams, and consistency between the two is automatically enforced as the model evolves.
Hybrid DSLs and their supporting editors can be developed using Graphite \cite{predoaia2023streamlining, predoaia2023towards}, a tool that facilitates the systematic engineering of hybrid editors via automatic code generation.
Graphite (\url{https://github.com/epsilonlabs/graphite}) uses Sirius \cite{sirius_website} for the specification of the graphical part of the syntax, and Xtext~\cite{xtext_website} for defining the textual part of the syntax. Alternatively, projectional editors, such as JetBrains MPS~\cite{jetbrains_website}, can be employed for developing such hybrid editors. The research works from~\cite{scheidgen2008textual, paper_obeo_typefox, cooper2018msc, solution_altran}, which are based on hand-written code, introduce techniques for engineering such kind of hybrid editors.
However, they do not address consistency enforcement between textual expressions and graphical model elements. Furthermore, except~\cite{cooper2018msc}, no techniques have been proposed for uniformly reporting errors. Other streams of work involve using blended modelling~\cite{ciccozzi2019blended} and Capella~\cite{capella_extension}; however, they do not address hybrid DSLs, as justified in~\cite{predoaia2023streamlining}.
\vspace{1.2mm}\noindent
\textbf{Testing Specification. }
Multi-robot systems (MRS) are distributed and interconnected robotic
teams capable of collectively performing tasks that are beyond the
competency of a single
robot~\cite{parker2012reliability}. Simulation-based testing (SBT) enables
investigating the capacity of an MRS to operate dependably using a
virtual environment that closely resembles the real deployment
environment. Modern robotic frameworks enable realistic simulation of
robots, with strong conformance to physical hardware, making the software
deployment to real robots easier~\cite{timperley-icst18}.
Compared to field testing,
simulation-based testing supports searching large design spaces of
components and system configurations, using limited physical hardware
resources, thus reducing significantly the overheads of detecting
potential violations of safety
requirements~\cite{harbin-models-2021,gerasimou2019towards}.
Fuzz testing is an automated software testing technique that involves providing unexpected, invalid or random data as input to a computer program, aiming to find exceptions, crashes and potential memory leaks \cite{godefroid2020fuzzing}.
Fuzz testing is considered among the most efficient approaches for finding security vulnerabilities~\cite{liang2018fuzz}.
In this work, fuzz testing is applied as a simulation-based testing technique to measure the robustness of a robotic mission.
% , by injecting intentional errors or faulty messages into a robotic simulation to monitor its responses and measure its robustness.
% TO BE ADDED IN OTHER SECTION
%In this paper we consider a general architecture for specifying generic fuzz testing campaigns to raise the level of abstraction and make fuzz testing more comprehensible to robotics engineers, as explored in the SESAME project \cite{sesame-d6.6}. Our focus here is specifically upon user comprehensibility; exploring the accessibility of the fuzz testing configuration and results representation within a domain specific language (DSL). We quantify with a user study how easy it is for new users to understand the MRS system and testing campaigns represented, and to interact with the testing configuration and results contained within.
\section{Case Study}
\label{SEC-CASE-STUDY}
%\agd{testing does not guarantee anything: it'd be better to say that you're aiming to obtain evidence that supports an argument that the system is safe --- check wording with Simos}
%KUKA~\cite{kuka-deliverable}.
%KUKA
The use case is an industrial case study in gearbox manufacturing
provided by our SESAME industrial partner, KUKA \cite{kuka_website}, a company that manufactures industrial robots and factory automation systems. This use case incorporates mobile robots to assemble gearboxes, together with human operators surrounding the assembly cell who assist in the process.
During testing, we aim to obtain evidence to validate the safety requirement that robots will not generate hazards to human workers during operation.
The primary safety consideration is to ensure that robots do not enter regions that could cause a collision hazard to human operators placed around the perimeter of the assembly cell frame.
Under normal circumstances, motion trajectories are pre-planned to ensure conformity to this safety requirement. However, faults in communication, sensing, or motor interlock actuation may lead to robots malfunctioning and
entering these forbidden areas.
The simulation scenario was supplied for testing by end users, and implemented using a proprietary simulation platform provided by our industrial partner and used as a digital twin~\cite{tts-dtwin-deliverable}.
\begin{figure}
\centering
\includegraphics[trim={0cm 5cm 0cm 6.5cm},clip,width=0.86\linewidth]{figures/ttsScreenshot.png}
\vspace{-3mm}
\caption{Case study screenshot from MRS simulation}
\vspace{-4mm}
\label{fig:kuka-case-study}
\end{figure}
% Individual figures version
\begin{figure}[!tb]
\centering
\includegraphics[width=\linewidth]{figures/testing-metamodel-uml}
\vspace{-8mm}
\caption{Fuzzing-based testing DSL metamodel (excerpt)}
\label{FIG-TESTSPACE-DSL}
\vspace{-3mm}
\end{figure}
\begin{figure}[!tb]
\centering
\includegraphics[width=\linewidth]{figures/dsl-conditions-uml}
\vspace{-9mm}
\caption{Meta-classes of condition-based fuzzing (excerpt)}
\label{FIG-CONDITIONS-DSL}
\vspace{-3mm}
\end{figure}
% JRH: subfig version of metamodel figures
\if 0
\begin{figure*}[!t]
\centering
\subfloat[][Metamodel Excerpt of the Fuzzing-Based Testing DSL]{\includegraphics[width=0.5\linewidth]{figures/testing-metamodel-uml}
\vspace{-4mm}
\label{FIG-TESTSPACE-DSL}}
\subfloat[][Meta-classes of condition-based fuzzing (excerpt)]{\includegraphics[width=0.5\linewidth]{figures/dsl-conditions-uml}
\vspace{-4mm}
\label{FIG-CONDITIONS-DSL}}
\vspace{-5mm}
\end{figure*}
\fi
%\caption{Metamodel Excerpt of the Fuzzing-Based Testing DSL}
%\agd{if you show an association, name it and hide the field, and add %cardinalities to associations}\agd{abstract classes should use italics}%%\agd{What are GrammarSpec and GrammarConstraints, and do we need to show it for our discussion?}\agd{How are Metrics defined?}\agd{What is the difference between ExecutionEndTrigger and ExecutionEndCondition?}\agd{Why are the results of a CampaignResultSet just references to tests? Where are their results (e.g. pass/fail)?}
Figure~\ref{fig:kuka-case-study} shows the layout of the assembly cell.
The primary safety requirement is that the robot grippers are forbidden from entering static safety zones surrounding the cell in order to prevent hazards to human workers.
The safety zones are illustrated as blue cuboid regions (Z1, Z2, Z3) on the right-hand side of the external cell frame in Figure~\ref{fig:kuka-case-study}.
If the safety zone surrounding the gripper of the right-hand robot (green cylinder labelled ZR3200) collides with these regions, it will trigger violations of the static safety requirement.
An output metric is used to quantify violations of the principal safety requirement, by tracking the occurrence of collisions of the gripper safety zone with any of Z1, Z2 or Z3.
Following a safety analysis, we have identified the communication from the controller to the robot arm joints as vulnerable to message disruptions.
Therefore, the primary safety requirement will be tested by applying fuzzing, altering the target motion value for robot arm joints with targeted uniform random jitter around the intended point.
We employ a DSL to raise the level of abstraction and make testing specification %and fuzz testing
more comprehensible to robotics engineers.
Modelling testing specifications via a DSL tailored to this specific domain is highly desirable, enabling non-expert users to easily test robotics systems~\cite{casalaro2021model,bruyninckx2013brics,harbin2023model,wood2020supporting}.
Participants of the user study will apply the DSL in the context of this case study, via a tree editor and a hybrid editor, to investigate the following research questions:
\begin{enumerate}[font={\bfseries},label={RQ\arabic*}]
\setcounter{enumi}{0}
\item \hypertarget{link_rq1}{\textbf{(Performance)}}
How do users perform when executing certain tasks with a hybrid editor compared to a tree editor?
\end{enumerate}
\begin{enumerate}[font={\bfseries},label={RQ\arabic*}]
\setcounter{enumi}{1}
\item \hypertarget{link_rq2}{(\textbf{Confidence})}
How confident are users regarding the correctness and completeness of their solutions to given tasks?
\end{enumerate}
\begin{enumerate}[font={\bfseries},label={RQ\arabic*}]
\setcounter{enumi}{2}
\item \hypertarget{link_rq3}{(\textbf{Preference})}
Do users prefer hybrid editors or tree editors?
\end{enumerate}
%TO BE REVIEWED: MAY HAVE TO MOVE THIS INTO A FOLLOWING SECTION
%A part of the fuzzing model instantiated for the KUKA industrial use case is presented in Figure~\ref{FIG-DSL-TTS-EXAMPLE}. The operations $attackJointR3200ProductMove\_Link1$ and $attackJointR3200ProductMove\_Link0$ of the model illustrate this fuzzing offset value change, using the \metaclass{RandomValueFromSet} fuzzing operation to apply relative changes to the joint value time series.
%\begin{figure}[t]
% \centering
% \includegraphics[width=\linewidth]{figures/tts-model-example-partial}
% \caption{A fragment of the DSL example for the TTS use case
% showing selected metrics and fuzzing operations}
% \label{FIG-DSL-TTS-EXAMPLE}
%\end{figure}
\section{Fuzzing-Based Testing DSL}
\label{SEC-DSL}
This section presents the fuzzing-based testing DSL developed in the context of the SESAME project, and applied to the case study from Section~\ref{SEC-CASE-STUDY}.
Through the DSL, users can define testing specifications for an MRS simulation, which controls how fuzzing is applied during testing.
Next, we present the abstract syntax and the concrete syntaxes of the DSL and the testing process.
\subsection{Abstract Syntax}
\label{SEC-DSL-ABSTRACT}
The abstract syntax (i.e., the metamodel of the DSL) has been defined with the Eclipse Modelling Framework (EMF) through the Ecore metamodelling language.
Figure~\ref{FIG-TESTSPACE-DSL} shows an excerpt from the metamodel of the DSL as a class diagram.
The core meta-classes are \metaclass{TestingSpace}, \metaclass{TestCampaign}
and \metaclass{Test}. A \metaclass{TestingSpace} (highlighted)
represents the root element of the domain and comprises permissible fuzzing operations (\attr{possibleFuzzingOperations}) that bind
the potential fuzzing space. The testing space also includes
performance \attr{metrics}, which are used to quantify the robotic
system performance for safety violations.
% JRH: removed as this is mentioned in "Sec. 4.3: simulation-based testing"
%, and allow multi-objective optimisation of test performance
%(optimising for most impactful violations).
A \metaclass{TestCampaign} specifies and constrains an individual experiment, referencing from the \metaclass{TestingSpace} a subset of
\attr{metrics} to use in campaign evaluation, together with a subset of chosen fuzzing operations in \attr{includedOperations}.
The \attr{performedTests} attribute in \metaclass{TestCampaign} is automatically populated during the experiment with dynamically generated and executed \metaclass{Test}s.
%The \metaclass{TestGenerationApproach}
%allows specifying the parameters for an experiment by selecting one of
%its subclasses. For example, \metaclass{NSGAEvolutionaryAlgorithm}
%allows an evolutionary experiment with the NSGA-II
%algorithm~\cite{nsga2}.
The \metaclass{Test} class represents a test configuration, corresponding to a specific selection of fuzzing \attr{operations}.
During \metaclass{Test} execution, instances of
\metaclass{MetricInstance} are recorded, to quantify the values of particular \metaclass{Metric}s under the applied fuzzing.
The \metaclass{FuzzingOperation} class is a specific fuzzing operation, i.e., an entity that represents a specific strategy for making runtime modifications to the
MRS. The property \attr{variableToAffect} selects the MRS variable to which fuzzing operations are applied.
These variables will be automatically manipulated by the testing platform when fuzzing is active.
% Not shown here;
Subclasses of \metaclass{FuzzingOperation} represent distinct fuzzing semantics; e.g., \metaclass{RandomValueFromSetOperation}
enables randomised changes to structured parameters within a message.
\if 0
\begin{figure}[t]
\centering
\includegraphics[width=0.9\linewidth]{figures/dsl-conditions-uml}
\vspace{-4mm}
\caption{Meta-classes of condition-based fuzzing (excerpt)
\sg{see comments for polishing}
}
%\agd{we need to be consistent with capitalisation: choose one approach and stick with it}\agd{remove fields which are shown as associations, and add cardinalities to associations}\agd{shouldn't there be two compositions from CompositeCondition to ConditionElement?}}
\label{FIG-CONDITIONS-DSL}
\vspace{-3mm}
\end{figure}
\fi
%\agd{what is the Pareto front measured in?} \ip{ideally define what a Pareto front is, what it contains and in what it is measured?}.
A \metaclass{CampaignResultSet} has a \attr{name} and represents either final or partial intermediate testing \attr{results}, as specified by the \metaclass{ResultSetStatus} enumeration. The
results are shown as references to notable tests under
\attr{results}. In an evolutionary experiment, the set of
non-dominated tests in the final campaign result set constitutes the Pareto front of optimal metrics obtained during the experiment.
The inclusion of a \metaclass{Test} in the non-dominated result sets shows that no other tests dominate it with respect to the set of \attr{metrics} used.
% Explain the metric direction and how it is used in optimisation
Condition-based fuzzing allows users to activate and deactivate fuzzing based on custom scenario-specific events, incorporating complex logic and spatial or geometric properties.
A condition is a logical expression built hierarchically
from comparison and logical operators, incorporating state variables from the robotic system.
The values of condition variables change dynamically during test execution, with every variable value supplied
from a \metaclass{Metric} of the corresponding name. Activation of a condition-based fuzzing operation is controlled by its start and end conditions.
When the start condition of a fuzzing operation holds, that fuzzing operation will be activated, and will manipulate simulation events.
Once activated, the fuzzing operation remains active until its end condition holds.
Condition-based fuzzing experiments aim to discover these
conditions automatically, identifying high-level MRS events that lead to failures or safety violations.
The primary role of a system tester will be to inspect, parse and understand these auto-generated conditions.
However, during debugging and exploratory testing, users may modify existing conditions or create new ones from scratch, to verify the impact of these changes on the
behaviour of a test.
\begin{figure}
\centering
\frame{\includegraphics[width=.935\linewidth]{figures/tree_editor.png}}
\vspace{-2.9mm}
\caption{Tree Editor --- Testing Space}
\label{FIG-TREE-EDITOR}
\vspace{-1.8mm}
\end{figure}
\begin{figure}[t]
\centering
\frame{\includegraphics[width=0.935\linewidth]{figures/tree_editor_conditions.png}}
\vspace{-2.9mm}
\caption{Tree Editor --- Activation Condition}
\label{FIG-TREE-EDITOR-CONDITIONS}
\vspace{-1.8mm}
\end{figure}
\begin{figure}[t]
\centering
\frame{\includegraphics[width=\linewidth]{figures/tree_editor_properties.png}}
\vspace{-6.8mm}
\caption{Tree Editor --- Properties View}
\label{FIG-TREE-EDITOR-PROPERTIES}
\vspace{-1.8mm}
\end{figure}
\begin{figure}[t]
\centering
\frame{\includegraphics[width=\linewidth]{figures/hybrid_editor_testingspace.png}}
\vspace{-6.8mm}
\caption{Hybrid Editor --- Diagram of a Testing Space}
\label{FIG-HYBRID-EDITOR-TESTINGSPACE}
\vspace{-2.2mm}
\end{figure}
The metamodel fragment in Figure~\ref{FIG-CONDITIONS-DSL} shows the meta-classes for condition-based fuzzing. The
\metaclass{Condition} class comprises an abstract
\metaclass{ConditionElement}, which is a
\metaclass{BasicCondition} or \metaclass{CompositeCondition}. A
\metaclass{BasicCondition} represents comparisons of a
\metaclass{ConditionVariable} with a \metaclass{ConditionExpr},
which is either a \metaclass{ConditionVariable} or
\metaclass{ConditionInteger}. The
\metaclass{BinaryComparisonOperation} is either \val{LESS\_THAN},
\val{GREATER\_THAN}, or \val{EQUALS}. \metaclass{ConditionVariables}
contain a reference to an associated \attr{metric} that supplies its
value, and \metaclass{ConditionIntegers} represent primitive
integers. A \metaclass{CompositeCondition} contains two other
\metaclass{ConditionElements} in the left and right branches, together
with a \metaclass{BinaryLogicalOperation}. These logical operations
permit \val{AND} or \val{OR} operators. Since condition
expressions are dynamically generated during experiments,
\metaclass{CompositeCondition} permits complex multi-level boolean
expressions of arbitrary depth.
%TO BE REVIEWED: MAY BE MOVED TO A FOLLOWING SECTION
%In this paper, a major aspect is to investigate how different DSL syntaxes permit the comprehension of deeply nested condition expressions, allowing the user to interpret their meanings. We will also investigate how the syntaxes allow the users to manually modify these syntaxes.
\subsection{Concrete Syntaxes}
The fuzzing-based testing DSL can be used through two concrete syntaxes: a graphical tree-based syntax and a hybrid syntax. The graphical syntax is provided by the default EMF-generated tree editor, whereas the hybrid syntax was developed using Graphite~\cite{predoaia2023streamlining}.
\vspace{1.2mm}\noindent
%The testing space contains two metrics (a \metaclass{FuzzingOperationTimesMetric} and a \metaclass{StreamMetric}), a \metaclass{FuzzingOperation} of type \metaclass{RandomValueFromSetOperation} named \val{attackJointPosR3200ProductMove\_Link0}, a \metaclass{TestCampaign}, and the definition of an MRS.
\textbf{Tree editor.} A subset of a testing space used in the case study is shown in Figure~\ref{FIG-TREE-EDITOR}. The \metaclass{TestCampaign} is selected in the editor, and its attributes (e.g., \attr{includedOperations} and \attr{metrics}) are displayed in the Properties View from Figure~\ref{FIG-TREE-EDITOR-PROPERTIES}. The test campaign contains three executed tests and two campaign result sets. Recall that a \metaclass{CampaignResultSet} represents a logical grouping of executed tests. The first test contains the \metaclass{RandomValueFromSetOperation} under \attr{includedOperations} in the \metaclass{TestCampaign}, and also two metric instances: one for \val{fuzzingOperationTimes} and one for \val{collisionOccurrence}, as specified by the \attr{metrics} attribute displayed in the Properties View from Figure~\ref{FIG-TREE-EDITOR-PROPERTIES}. Figure~\ref{FIG-TREE-EDITOR-CONDITIONS} expands the \metaclass{RandomValueFromSetOperation} named \val{attackJointPosR3200ProductMove\_Link0} and displays its child elements. This operation is activated when its start condition holds. The \metaclass{Condition} element is the root of the activation condition and it has a \metaclass{CompositeCondition}, in which the left side is a \metaclass{BasicCondition} and the right side is another \metaclass{CompositeCondition}. %Note that the \metaclass{Condition} element and its children conform to the metamodel fragment from Figure~\ref{FIG-CONDITIONS-DSL}.
% \begin{figure*}
% \centering
% \frame{\includegraphics[width=0.78\linewidth]{figures/hybrid_editor_test.png}}
% \vspace{-2mm}
% \caption{Hybrid Editor — Test with Condition Textual Expressions}
% \label{FIG-HYBRID-EDITOR-TEST}
% \vspace{-4mm}
% \end{figure*}
% \begin{figure*}
% \centering
% \frame{\includegraphics[width=0.9\linewidth]{figures/hybrid_editor_resultsets.png}}
% \vspace{-2mm}
% \caption{Hybrid Editor — Diagram of Campaign Result Sets}
% \label{FIG-HYBRID-EDITOR-RESULTSETS}
% \vspace{-1mm}
% \end{figure*}
\begin{figure*}
\hspace{2mm}
\begin{subfigure}{0.705\textwidth}
\frame{\includegraphics[height=4.7cm]{figures/hybrid_editor_test.png}}
% \caption{Hybrid Editor — Test with Condition Textual Expressions}
% \label{FIG-HYBRID-EDITOR-TEST}
\end{subfigure}
\begin{subfigure}{0.24\textwidth}
\frame{\includegraphics[height=4.7cm]{figures/hybrid_editor_resultsets_vertical.png}}
% \caption{Hybrid Editor — Diagram of Campaign Result Sets}
% \label{FIG-HYBRID-EDITOR-RESULTSETS}
\end{subfigure}
\vspace{-2mm}
\caption{Hybrid Editor --- a test with a condition textual expression (left diagram) and campaign result sets (right diagram)}
\label{FIG-HYBRID-EDITOR-RESULTSETS}
\vspace{-2mm}
\end{figure*}
\vspace{1.2mm}\noindent
\textbf{Hybrid editor.} The Graphite-based concrete syntax is shown in Figures~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} and %, \ref{FIG-HYBRID-EDITOR-TEST}, and~
\ref{FIG-HYBRID-EDITOR-RESULTSETS}.
A graphically defined testing space is illustrated in the diagram in Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}. The elements of the testing space are displayed as rectangles with various colours and symbols depending on each element type. Arrows highlight relationships between elements. To create new elements, users can drag and drop various symbols from the Palette on top of the diagram. The diagram displays the high-level elements of the testing space; however, users can view lower-level details by double-clicking on specific elements.
By double-clicking on a \metaclass{Test} contained in a \metaclass{TestCampaign}, a new diagram is instantiated with the child elements of the \metaclass{Test}. For instance, Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} (left diagram) shows a hybrid editor that contains the child elements of the first test from the test campaign. The activation condition is specified with a textual expression in an embedded textual editor, with developer assistance features such as syntax highlighting and auto-completion. Users can also double-click the \metaclass{TestCampaign} from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} to view its campaign result sets in a newly instantiated diagram. Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} (right diagram) illustrates the campaign result sets of the test campaign, ordered from first to last via arrows. By double-clicking a \metaclass{Test} from the campaign result sets, a new diagram similar to the one from Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} (left diagram) is instantiated, illustrating the details of the \metaclass{Test}.
We should note that the capabilities and behaviour of the hybrid editor, e.g., the instantiation of new diagrams via double-clicking, are shown to the participants of the user study via a video tutorial.
\subsection{Simulation-Based Testing}
The SESAME tools for simulation-based testing \cite{sesame-deliverable-d6.2,sesame-deliverable-d6.6} provide a model-driven
code generation engine. Input models conforming to the DSL configure an evolutionary optimisation
loop for a given \metaclass{TestCampaign}. The loop optimises a
population of fuzzing tests with the intent of maximising safety
requirement violations. For every \metaclass{Test} generated and added
to the model, the SESAME code generation engine generates a test
runner, including an MRS logical interface enabling two-way runtime
communication. The test runner launches the simulation,
monitors and manipulates fuzzing variables to perform the operations
defined for that test.
% \jrhchanged{When a fuzzing operation is active, all messages of the target variable will be modified}, delayed or deleted as specified by the fuzzing operation.
Performance metrics are computed from monitored variables, and their final values are logged under
\metaclass{Test} as a \metaclass{MetricInstance}.
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.7}
\caption{\label{tab:demographics}Breakdown of expertise in participants}
\vspace{-3mm}
\begin{tabular}{|>{\centering}m{1.5cm}|>{\centering}m{1.5cm}|
>{\centering}m{1.2cm}|
>{\centering}m{1.2cm}|>{\centering\arraybackslash}m{1.2cm}|}
\hline
\vspace{1.1mm}
\textbf{Expertise} & \vspace{1.1mm} \textbf{Modelling} & \vspace{1.1mm} \textbf{DSLs} & \vspace{1.1mm} \textbf{MDE} & \vspace{1.1mm} \textbf{MRS} \\[1.3mm]
\hline
\vspace{0.7mm} None\;/\;Low & \vspace{0.75mm} 6 & \vspace{0.75mm} 7 & \vspace{0.75mm} 8 & \vspace{0.75mm} 17 \\
\hline
\vspace{0.7mm} Medium & \vspace{0.75mm} 6 & \vspace{0.75mm} 8 & \vspace{0.75mm} 4 & \vspace{0.75mm} 3 \\
\hline
\vspace{0.7mm} High & \vspace{0.75mm} 10 & \vspace{0.75mm} 7 & \vspace{0.75mm} 10 & \vspace{0.75mm} 2 \\
\hline
\end{tabular}
\endgroup
\end{center}
% \vspace{2mm}
\vspace{-4mm}
\end{table}
\section{Study Methodology}
\label{SEC-METHODOLOGY}
% To answer the questions, w
% We conducted an empirical study, recruiting individuals to test the DSL concrete syntaxes for a diverse set of scenarios and tasks.
\subsection{Recruitment}
As the study was part of a wider research project with a broad range of industry and academic partners in modelling platforms, we emailed an invitation letter to target individuals with relevant experience, with a request to indicate their availability for the 2-hour-long study. We shared the same email invite within the Computer Science department of the University of York, UK, for undergraduate and postgraduate students and academics with experience in computer programming, modelling or MDE. The invitation was also shared with various research groups in academia and professionals working in the industry, with modelling expertise.
\subsection{Participants}
A total of 22 participants participated in the study. The participants currently work either in the industry or academia: 5 participants work in the industry (3 companies across the UK, Germany and Romania), and 17 participants come from academia across 8 universities in the UK, Spain, Austria and Germany. Two of the academic participants were computer science students. Participants had a range of years of programming experience, with 10 participants with more than 10 years, 5 participants with 7--10 years, 5 participants with 4--6 years, and 2 participants with less than 1. Table \ref{tab:demographics} shows the level of knowledge and expertise of participants.
\subsection{Procedures}
The study was pilot-tested with two users who did not participate in the final study. During the pilot, we identified practical issues and revised the study accordingly, e.g., to reduce the time needed for the study and improve the wording of task descriptions.
Once a participant agreed to join the study, the first author would contact them to arrange a time for the user study.
A facilitator, i.e., the first author, guided and observed participants during the sessions. The faculty research committee ethically approved the study.
Each participant joined a virtual meeting with video conferencing together with the facilitator via Zoom. %\footnote{a video conferencing tool that allows users to meet online through audio and video}.
At the beginning of the session, the facilitator described the context of the user study and shared two documents with the participant: an information sheet describing the study and a consent form. The participant read these documents and then signed the consent form if they agreed to continue with the user study session.
Next, the facilitator shared a 20-minute video tutorial, presenting the abstract and concrete syntaxes of the fuzzing-based testing DSL and practical examples of its usage in the Eclipse Integrated Development Environment (IDE) via the hybrid and tree editors. After watching the video, the participants could ask questions on the presented information.
Each participant completed a Google Forms %\footnote{web-based software for surveys and questionnaires that collects and stores responses}
survey, which contained the tasks %- for participants to complete -
and follow-up questions (e.g., regarding the correctness and confidence in their answers/solutions). Throughout the user study, the facilitator clarified any uncertainties related to the task descriptions. After completing all tasks, participants were asked to report their preferences, including features they liked and disliked about the editors. Lastly, the participants completed a demographic survey, including details of their years of experience and level of expertise in the areas of Table~\ref{tab:demographics}.% such as computer programming, modelling, MDE, DSLs and robotics software.
The participant connected remotely to the computer of the facilitator via TeamViewer. %\footnote{a tool that allows users to remotely access and control computers}.
The computer ran a Windows Operating System (OS). On the computer, two Eclipse IDE instances were running for defining and editing testing specifications: one containing the tree editor and another containing the hybrid editor. Participants executed given tasks through both editors using the same semantic model.
To reduce memorising a task solution,
% prevent them from memorising
participants were not told that the underlying semantic model is the same.
To reduce bias and raise the generalisability of the results, the order in which the two types of editors were used for a specific task was interchanged between participants. Half of the participants first carried out a given task with the hybrid editor, and then performed the same task once more with the tree editor. For the other half, the order was reversed, thus participants first executed the task with the tree editor. By alternating the order we can reduce its effects on performance, confidence, or preference, as a task is typically performed more efficiently and faster the second time than the first.
To meticulously analyse how users performed, all sessions were audio- and video-recorded.
The participants were encouraged to think aloud while completing the tasks, to record and later analyse any relevant comments.
Also, a tool for visually highlighting mouse clicks was used to display a red-coloured circle around the mouse pointer when a mouse click occurred. This allows for counting only relevant mouse clicks when analysing the video recordings. For example, mouse clicks outside the editors are not relevant.
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.888}
\caption{\label{tab:tasks}Tasks for Understanding (U) and Modelling (M)}
\vspace{-3mm}
\begin{tabular}{|>{\centering}m{0.5cm}|>{\raggedright\arraybackslash}m{7.2cm}|}
\hline
\vspace{1mm} \textbf{{ID}} & \vspace{1mm} \textbf{{Description}} \\[1mm]
\hline
\vspace{0.5mm} \hypertarget{U1}{U1} & \vspace{0.5mm} In the first test, find the \metaclass{Result Value} of the \metaclass{Metric Result} associated with \metaclass{FuzzingOperationTimesMetric}. \\
\hline
\vspace{0.5mm} \hypertarget{U2}{U2} & \vspace{0.5mm} In the first test, in the \metaclass{RandomValueFromSetOperation}, describe the activation starting condition. \\
\hline
\vspace{0.5mm} \hypertarget{U3}{U3} & \vspace{0.5mm} In the campaign result sets of the \metaclass{TestCampaign}, find the differences among the last 3 \val{NONDOM} sets. \\
\hline
\vspace{0.5mm} \hypertarget{U4}{U4} & \vspace{0.5mm} In the testing space, find the variable(s) related to the \metaclass{StreamMetric} named \val{collisionOccurrence}. \\
\hline
\vspace{0.5mm} \hypertarget{U5}{U5} & \vspace{0.5mm} In the testing space, find the fuzzing operation with the largest \metaclass{DoubleRange} interval. \\
\hline
\vspace{0.5mm} \hypertarget{M1}{M1} & \vspace{0.5mm} In the testing space, create a new \metaclass{TestCampaign} and set its name as \val{NewTestCampaign}. \\[0.25cm]
\hline
\vspace{0.5mm} \hypertarget{M2}{M2} & \vspace{0.5mm} Copy the first test from the prior existing \metaclass{TestCampaign} into the \val{NewTestCampaign}. \\
\hline
\vspace{0.5mm} \hypertarget{M3}{M3} & \vspace{0.5mm} In the \metaclass{RandomValueFromSetOperation}, modify the activation starting condition by replacing the operators. \\
\hline
\vspace{0.5mm} \hypertarget{M4}{M4} & \vspace{0.5mm} In the \metaclass{RandomValueFromSetOperation}, change the first referenced variable of the activation starting condition. \\
\hline
\vspace{0.5mm} \hypertarget{M5}{M5} & \vspace{0.5mm} Define from scratch the activation ending condition for the \metaclass{RandomValueFromSetOperation}. \\
\hline
\end{tabular}
\endgroup
\end{center}
\vspace{-4.484mm}
\end{table}
\subsection{Tasks}
Participants were given the tasks from Table~\ref{tab:tasks}, categorised into two types.
Tasks \hyperlink{U1}{U1}--\hyperlink{U5}{U5} focus on understanding a testing space, used to evaluate which type of model editor is better for comprehension.
Tasks \hyperlink{M1}{M1}--\hyperlink{M5}{M5} focus on modelling a testing space, used for evaluating which model editor provides better performance when modelling.
The table shows a summary of the tasks; the complete task descriptions used in the study are available in \cite{reproduction_package}.
%The tasks for understanding a testing space are used to evaluate which type of model editor is better for comprehension.
% Correspondingly, the tasks for modelling a testing space are used for evaluating which model editor provides superior performance when modelling.
All given tasks were driven by the industrial case study, aiming to reflect realistic modelling scenarios performed in practice by domain experts while covering a wide range of modelling activities such as finding specific elements, finding relationships and differences between elements, changing references, and creating and copying elements. Participants operate over the semantic model presented in Figure~\ref{FIG-TREE-EDITOR}, which contains a subset of the relevant model elements from the actual model used in the case study. All tasks of the same type were first executed with one editor, and then they were executed again with the other editor, e.g., after completing all modelling tasks with the hybrid editor, they were executed again with the tree editor.
\vspace{1mm}\noindent
\textbf{Understanding Tasks (\hyperlink{U1}{U1}\,--\,\hyperlink{U5}{U5}).}
To perform the tasks using the tree editor, participants have to carefully inspect all model elements, and accordingly expand them to view their child elements and scan their attributes displayed in the Properties View. We describe next how a user should execute the tasks via the hybrid editor.
Task \hyperlink{U1}{U1} requires participants to locate various model elements with specific values of interest. More specifically, the task asks participants to locate the first test contained in the test campaign from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} and view its details through a newly instantiated diagram by double-clicking the test. In the instantiated diagram, that looks like that in Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS}, participants must locate the \metaclass{Metric Result} that contains a \metaclass{FuzzingOperationTimesMetric}, and then identify its \metaclass{Result Value}. For the diagram from Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS}, the \metaclass{Result Value} is 0.01.
Task \hyperlink{U2}{U2} asks participants to describe conditions defined through a tree representation and a textual representation. The task entails viewing the details of the first test as presented above, locating the \metaclass{RandomValueFromSetOperation}, finding the activation starting condition, and describing it.
In the tree editor, the activation starting condition looks as in Figure~\ref{FIG-TREE-EDITOR-CONDITIONS}, whereas in the hybrid editor, it looks as the textual expression displayed in the embedded textual editor from Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS}. When using the tree editor, the participant was asked to construct an equivalent textual expression as pseudocode. Correspondingly, when using the hybrid editor, the participant was asked to find the textual expression, describe its meaning, and categorise conditions as either \metaclass{BasicCondition} or \metaclass{CompositeCondition}, and identify the left and right branches of \metaclass{CompositeConditions}.
Task \hyperlink{U3}{U3} asks participants to report differences or similarities among an ordered list of model elements.
Participants should locate the \metaclass{TestCampaign} from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} and analyse its campaign result sets using a newly instantiated diagram by double-clicking the \metaclass{TestCampaign}. In the instantiated diagram, like that in Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS}, participants must find the last 3 campaign result sets whose names are prefixed with \val{NONDOM}, and then notice their differences and similarities.
Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} contains a subset of campaign result sets from those used in the user study, which included 15 result sets.
Task \hyperlink{U4}{U4} asks participants to find relationships (references) between elements.
Participants should find the \val{collisionOccurrence} \metaclass{StreamMetric} from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}, and then identify the related variables. For instance, as indicated by the arrows in Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}, the \metaclass{StreamMetric} is related to the variables \val{safetyzones/Z1} and \val{safetyzones/Z2}.
Task \hyperlink{U5}{U5} asks participants to find specific types of model elements and perform minor mental computations. Participants should locate the fuzzing operation with the largest \metaclass{DoubleRange} interval. Note that the largest interval is that with the highest difference between the upper bound and the lower bound. Although multiple fuzzing operations existed in the testing space of the user study, in Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE} there is only one fuzzing operation: the \metaclass{RandomValueFromSetOperation}, which has a \metaclass{DoubleRange} interval of [-0.3; 0.4].
\vspace{1.2mm}
\noindent
\textbf{Modelling Tasks (\hyperlink{M1}{M1}\,--\,\hyperlink{M5}{M5}). }
In these tasks, participants edit existing and create new model elements, and change relevant property values via both editors. The modelling tasks are done after completing the understanding tasks; thus, participants have a good understanding of the testing space before engaging in these tasks.
Task \hyperlink{M1}{M1} requests participants to create a new model element and set one of its properties. For instance, in the hybrid editor from Figure~\ref{FIG-HYBRID-EDITOR-TESTINGSPACE}, the participant would have to identify in the Palette the symbol related to \metaclass{TestCampaign}, then drag and drop it on the diagram, and lastly set its name using the Properties View.
Task \hyperlink{M2}{M2} requires participants to execute operations within the model editors, specifically, the copy and paste operations. The participant would have to either use the context menu or a keyboard shortcut to trigger the copy operation of the first test, and then execute the paste operation into the new test campaign.
Task \hyperlink{M3}{M3} asks participants to modify an existing condition, through its tree representation as in Figure~\ref{FIG-TREE-EDITOR-CONDITIONS} and its textual representation as in Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS}. More specifically, the task requires the participant to replace all \val{AND} operators with \val{OR} operators, and replace the comparison operators \val{LESS\_THAN} and \val{EQUALS} with \val{GREATER\_THAN}.
Task \hyperlink{M4}{M4} requests participants to change a reference within a condition.
In particular, the task involves changing the first referenced metric from the activation starting condition with another metric. For instance, the first referenced metric in the condition from Figure~\ref{FIG-HYBRID-EDITOR-RESULTSETS} is \val{collisionOccurrence}, and participants had to use \val{fuzzingOperationTimes} instead.
This change would be realised in the tree editor by changing the reference via a selection menu, whereas in the hybrid editor, this would be done by either typing and deleting text manually, or by using the auto-completion menu that lists all metrics that can be referenced.
Task \hyperlink{M5}{M5} requires participants to define a condition from scratch that meets certain requirements.
Participants should define the activation ending condition such that it contains one root \metaclass{CompositeCondition} containing two \metaclass{BasicConditions}. When defining conditions as in Figure~\ref{FIG-TREE-EDITOR-CONDITIONS}, multiple clicks are required to create new elements, whereas via the hybrid editor users can leverage the auto-completion menu to define the conditions textually.
\vspace{-1mm}
\subsection{Data Collection and Analysis}
We analysed the video recordings and the surveys completed by the participants to derive the following metrics:
% Various metrics have been collected from the video recordings and from the surveys completed by the participants, as follows:
\vspace{1mm}\noindent\textbf{Duration}. The duration quantifies the amount of effort required to do a task, measuring the time-on-task, in seconds, within the editors.
The duration does not include time spent reading task descriptions, or
% It does not include time spent reading task descriptions.
time wasted when the remote access and control session via TeamViewer was lost. %Note that
Participants alternated %switched back and forth
between the Google Forms survey and the Eclipse IDE instances containing the editors, thus we were able to monitor the time spent on each.
\vspace{0.8mm}\noindent
\textbf{Correctness}. After performing all user study sessions, we examined whether participants completed the tasks correctly or made mistakes.
% we noticed that almost all participants completed the tasks correctly and completely. Otherwise, the participants made minor mistakes.
Thus, we marked solutions as either 1 (correct and complete) or 0 (mistakes). When a task was not solved correctly, we recorded the specific mistake(s) made.
\vspace{0.8mm}\noindent
\textbf{Clicks and Keystrokes}. Mouse clicks and keystrokes reflect modelling effort~\cite{hill2011measuring}. The number of clicks and keystrokes was recorded for the tasks related to conditions modelling, i.e., tasks \hyperlink{M3}{M3} and \hyperlink{M5}{M5}.
\vspace{0.8mm}\noindent
\textbf{Confidence}. After performing each task, participants self-reported their confidence level in their solution, ranging from 1 (very low confidence) to 5 (very high confidence).
\vspace{0.8mm}\noindent
\textbf{Errors Corrected}. We recorded the mistakes made in the context of a modelling task, which were later corrected by the user during the execution of the same task and before its conclusion. A mistake represents a performed action that is incorrect, redundant or certainly unrelated to the solution of a modelling task. %This metric was collected for task \hyperlink{M5}{M5}, where conditions are modelled from scratch.
Note that the clicks and keystrokes and the time spent while making mistakes and correcting them were included in the analysis.
% The scale for the self-reported confidence is between 1 (very low confidence) to 5 (very high confidence).
% \begin{itemize}[leftmargin=2mm]
% \item \textbf{Duration}. The duration quantifies the amount of effort required to do a task, measuring the time-on-task, in seconds, within the editors.
% The duration does not include time spent reading task descriptions. or
% % It does not include time spent reading task descriptions.
% time wasted when the remote access and control session via TeamViewer was lost. %Note that
% Participants alternated %switched back and forth
% between the Google forms survey and the Eclipse IDE instances containing the editors, thus we were able to monitor the time spent on each.
% \vskip 0.8mm
% \item \textbf{Correctness}. After performing all user study sessions, examined if participants completed the tasks correctly or made minor/major mistakes.
% % we noticed that almost all participants completed the tasks correctly and completely. Otherwise, the participants made minor mistakes.
% Thus, we marked solutions as either 1 (correct and complete) and 0 (minor mistakes). When a task was not solved correctly, we recorded the specific mistake(s) made.
% \vskip 0.8mm
% \item \textbf{Errors Corrected}. We recorded the mistakes made during modelling, which were spotted and corrected at a later point by participants. In this case, a mistake represents a performed action that is either incorrect, redundant or certainly unrelated to the solution of a modelling task. This metric was collected for task \hyperlink{M5}{M5}, in which conditions are modelled from scratch.
% \vskip 0.8mm
% \item \textbf{Clicks and Keystrokes}. Mouse clicks and keystrokes are metrics that reflect modelling effort \cite{hill2011measuring}. The number of clicks and keystrokes was recorded for the tasks related to the modelling of conditions, i.e., tasks \hyperlink{M3}{M3} and \hyperlink{M5}{M5}.
% \vskip 0.8mm
% \item \textbf{Confidence}. After performing each task, participants self-reported their confidence level on their solution, ranging from 1 (very low confidence) to 5 (very high confidence).
% % The scale for the self-reported confidence is between 1 (very low confidence) to 5 (very high confidence).
% \end{itemize}
\section{Results and Discussion}
\label{SEC-RESULTS}
% We discuss the results of the user study, considering performance, confidence and preference.
% Also, we provide aggregated insights and preferences of participants on the usage of the model editors.
\subsection{Performance (RQ1)}
We evaluate research question \hyperlink{link_rq1}{RQ1} when carrying out various tasks using the tree-based and hybrid editors by considering the metrics: duration, correctness, errors corrected and clicks and keystrokes.
% Research question RQ1 aims to evaluate the performance of
\vspace{1.2mm}\noindent
\textbf{Duration.}
Table~\ref{tab:durations_statistics} shows the absolute differences between the durations for completing the tasks via the tree ($T$) and hybrid ($H$) editors. The table is split into tasks for understanding (U) and modelling (M), and the five numeric columns indicate tasks \hyperlink{U1}{U1}--\hyperlink{U5}{U5} and \hyperlink{M1}{M1}--\hyperlink{M5}{M5}.
The delta ($\Delta$) subtracts the duration when using the hybrid editor from the duration taken with the tree editor, i.e., $Delta = Duration(Tree) - Duration(Hybrid)$. For each task, the delta of the mean and median durations are displayed. A positive value signifies a higher duration with the tree editor compared to the hybrid editor; the opposite holds for a negative value.
In addition, Figure~\ref{FIG-DURATION-BOXPLOTS} shows boxplots with the duration needed for executing the understanding and modelling tasks using both editors.
\begin{figure*}[t!]
\centering
\includegraphics[width=0.99\linewidth]{figures/testingDSLboxplotsH.jpg}
\vspace{-2.5mm}
\caption{Boxplots --- Durations (seconds) for understanding (U) and modelling (M) tasks using the tree (T) and hybrid (H) editors}
\label{FIG-DURATION-BOXPLOTS}
\vspace{-2mm}
\end{figure*}
\begin{table}[t]
\begin{center}
\begingroup
\caption{\label{tab:durations_statistics}Deltas of Mean and Median Durations (seconds)}
\vspace{-3mm}
\renewcommand*{\arraystretch}{0.5}
\begin{tabular}{|>{\centering}m{0.7cm}|>{\centering}m{1.3cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|>{\centering\arraybackslash}m{0.77cm}|}
\hline
\vspace{1.2mm}
\textbf{{Type}} & \vspace{1.2mm} $\mathbf{\Delta}$ $(\boldsymbol{T}-\boldsymbol{H})$ & \vspace{1.2mm} \textbf{{\#1}} & \vspace{1.2mm} \textbf{{\#2}} & \vspace{1.2mm} \textbf{{\#3}} & \vspace{1.2mm} \textbf{{\#4}} & \vspace{1.2mm} \textbf{{\#5}} \\[1.8mm]
\hline
%\vspace{2mm} \multirow{2}{*}{\textbf{U}}
\multirow{2}{*}{\parbox{1\linewidth}{\vspace{2mm} \textbf{\hspace{2.3mm}U}}}
& \vspace{0.8mm} Mean & \vspace{0.8mm} 1.55 & \vspace{0.8mm} 315.95 & \vspace{0.8mm} -15.32 & \vspace{0.8mm} 10 & \vspace{0.8mm} 36.45 \\[1mm]
\cline{2-7}
& \vspace{0.8mm} Median & \vspace{0.8mm} 1.50 & \vspace{0.8mm} 301 & \vspace{0.8mm} -23 & \vspace{0.8mm} 0.50 & \vspace{0.8mm} 32.50 \\[1mm]
\hline
\multirow{2}{*}{\parbox{1\linewidth}{\vspace{2mm} \textbf{\hspace{2.1mm}M}}} & \vspace{0.8mm} Mean & \vspace{0.8mm} 0.59 & \vspace{0.8mm} -9.50 & \vspace{0.8mm} 27.09 & \vspace{0.8mm} 8.94 & \vspace{0.8mm} 9.51 \\[1mm]
\cline{2-7}
& \vspace{0.8mm} Median & \vspace{0.8mm} -0.50 & \vspace{0.8mm} 2 & \vspace{0.8mm} 34 & \vspace{0.8mm} 3.50 & \vspace{0.8mm} 19.50 \\[1mm]
\hline
\end{tabular}
\endgroup
\end{center}
% \vspace{2mm}
\vspace{-5.3mm}
\end{table}
Completing task \hyperlink{U1}{U1} required on average (median) 1.55 (1.50) seconds more with the tree editor than with the hybrid editor.
% Correspondingly, completing task \hyperlink{U1}{U1} required a median duration that is higher by 1.50 seconds with the tree editor compared to the hybrid editor.
Thus, for solving task \hyperlink{U1}{U1}, which requires locating model elements with specific values of interest, the hybrid editor had slightly better performance; however, no significant difference between the two exists.
For task \hyperlink{U2}{U2}, the hybrid editor was faster on average by 315.95 seconds, indicating that using textual representations for expressing conditions improves comprehension significantly.
In task \hyperlink{U3}{U3}, the tree editor was faster by 15.32 seconds than the hybrid editor; the median value outlines a higher difference, of 23 seconds. Concretely, the relative difference reported to the mean and median durations of the hybrid editor was 15\% and 27\%, respectively. This shows that tree editors are more appropriate to explore and analyse ordered lists of model elements, as navigating long ordered element lists in diagrams often incurs overheads.
For task \hyperlink{U4}{U4}, the hybrid editor was faster by 10 seconds on average, however, this difference is negated by the delta median value, showing the existence of some outliers for this task.
We conclude that no significant difference exists in performance between the editors when executing a task that requires finding references between elements.
The hybrid editor yielded faster results in task \hyperlink{U5}{U5}, showing that finding elements that meet certain criteria is quicker in diagrams, as users have to meticulously expand and explore all elements in a tree editor.
Considering the modelling tasks, as indicated by the mean and median delta values of tasks \hyperlink{M1}{M1} and \hyperlink{M2}{M2}, we cannot conclude that there is a difference in performance between the editors for these tasks.
For task \hyperlink{M3}{M3}, the hybrid editor resulted in improved performance, indicating that modifying existing conditions via a textual editor with syntax-aware editing features is quicker than through a tree editor.
In task \hyperlink{M4}{M4}, the hybrid editor was faster on average by 8.94 seconds, however, the delta value of the median suggests a negligible difference between the performance of the editors.
Thus, for solving a task that requires changing a reference, the editors have similar performance, although the hybrid editor may be slightly faster. For task \hyperlink{M5}{M5}, the hybrid editor was faster on average (median) by 9.51 (19.50) seconds, with a relative difference of 8\% (16\%), thus the hybrid editor was more effective in modelling conditions.
Finally, we used the non-parametric Wilcoxon signed rank test to perform statistical analysis between the duration taken by participants to complete the understanding and modelling tasks using the editors. Our statistical analysis with ($\alpha$=0.05) demonstrated significant differences in understanding tasks \hyperlink{U2}{U2} (p=0.00001) and \hyperlink{U5}{U5} (p=0.0127) and modelling tasks \hyperlink{M3}{M3} (p=0.0002) and \hyperlink{M5}{M5} (p=0.0329). These results are also underpinned by the boxplots in Figure~\ref{FIG-DURATION-BOXPLOTS}.
\vspace{1.2mm}\noindent
\textbf{Correctness.}
All participants solved task \hyperlink{U1}{U1} correctly through the hybrid editor, whereas three participants solved the task incorrectly with the tree editor. From those, two participants reported an incorrect value as a solution, whereas one participant did not find a solution and abandoned the task. For task \hyperlink{U2}{U2}, two participants reported incorrect solutions with the hybrid editor, not being able to accurately state the left and right branches of the conditions represented textually. Correspondingly, seven participants solved the task incorrectly with the tree editor, as they constructed incorrect equivalent textual representations of the conditions from the tree editor by either grouping sub-conditions incorrectly, missing operators or using unsuitable operators.
All participants solved correctly tasks \hyperlink{U3}{U3}--\hyperlink{U5}{U5} using both editors.
The results evidence that the hybrid editor surpasses the tree editor in correctly locating specific model elements and in comprehending conditions.
In modelling tasks, one participant incorrectly performed task \hyperlink{M1}{M1} via the hybrid editor. The participant tried to set the name of a \metaclass{TestCampaign} by double-clicking its name on the diagram, and then entering the name in the pop-up menu displayed after double-clicking. The participant forgot a detail shown in the video tutorial: double-clicking specific elements in the hybrid editor instantiates new diagrams. As such, the participant instantiated a new diagram with the name entered in the pop-up menu, without noticing that the name of the \metaclass{TestCampaign} had not been set. Correspondingly, all participants solved task \hyperlink{M1}{M1} correctly through the tree editor.
Task \hyperlink{M2}{M2} was solved correctly by all participants through both editors. One and six participants solved task \hyperlink{M3}{M3} incorrectly, with the hybrid editor and tree editor, respectively. The most encountered mistake made through the tree editor was that participants did not expand all condition elements to view their child elements, to be able to locate the operators that needed to be replaced. All participants solved task \hyperlink{M4}{M4} correctly with the hybrid editor, while only one participant failed using the tree editor.
For task \hyperlink{M5}{M5}, two participants incompletely defined the required condition from scratch with the tree editor, while one participant made a mistake when textually defining the required condition; specifically, they used an \val{AND} operator instead of an \val{OR} operator.
Consequently, we conclude that there is no significant difference between the editors for solving the tasks, with the exception of task \hyperlink{M3}{M3}, for which the hybrid editor yielded better results. %clearly superior.
Fine-grained changes to conditions require a high level of attention to detail, and the results indicate that it is more appropriate to modify existing conditions through a textual representation rather than a tree representation.
\vspace{1.2mm}\noindent
\textbf{Errors Corrected. }
The errors corrected have been recorded for task \hyperlink{M5}{M5}, in which participants defined conditions from scratch.
During the condition definition, 14 and 8 participants made incorrect actions with the hybrid editor and tree editor, respectively; the mistakes were corrected later in both cases.
% , which they later corrected. Correspondingly, 8 participants made mistakes when using the tree editor, that they later corrected.
The results highlight that more than half of the participants faced challenges during modelling with the hybrid editor.
The reason for the high number of mistakes is that the embedded textual editor from the hybrid editor allows any invalid input, such as typos.
For all participants, in total, 8 errors have been corrected with the tree editor and 17 errors have been corrected with the hybrid editor. The mistakes made with the hybrid editor were minor, e.g, participants typed \val{LESS\_THAN} instead of \val{GREATER\_THAN}, typed lowercase \val{OR} instead of uppercase, and used curly braces instead of parentheses for defining \metaclass{BasicConditions}. However, the errors that have been corrected in the tree editor were of a higher magnitude. For example, participants created a \metaclass{BasicCondition} as a root element, then continued modelling the conditions, and later realised that the root should have been a \metaclass{CompositeCondition}. Next, participants deleted the \metaclass{BasicCondition}, and defined once again the conditions from scratch, as the tree editor does not allow replacing a model element.
\vspace{1mm}\noindent
\textbf{Clicks and Keystrokes.}
Table~\ref{tab:clicks_statistics} shows statistics for clicks and keystrokes, %. The number of clicks and keystrokes have been
recorded for tasks \hyperlink{M3}{M3} and \hyperlink{M5}{M5}, which involve modifying and modelling conditions. The delta subtracts the mean and median number of clicks or keystrokes when using the hybrid editor from the corresponding values %mean and median number of clicks or keystrokes
of the tree editor.
On average, more mouse clicks (24.32) and fewer keystrokes (32.55) were used for solving task \hyperlink{M3}{M3} with the tree editor. Similarly, more clicks and fewer keystrokes were needed to solve task \hyperlink{M5}{M5} via the tree editor.
The results are not surprising, %reflect what is typically expected,
as hybrid editors require more typing to define textual expressions, which are defined via clicks through a tree editor. %However, the way in which the editors are used will dictate the results. For instance, there has been a participant who completed task \hyperlink{M3}{M3} using the tree editor with 21 clicks and 0 keystrokes, whereas there has been another participant who completed the same task via the hybrid editor with 21 clicks and 11 keystrokes, as many unnecessary clicks have performed.
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.5}
\caption{\label{tab:clicks_statistics}Deltas of Mean and Median No. Clicks, Keystrokes}
\vspace{-3mm}
\begin{tabular}{|>{\centering}m{1.3cm}|>{\centering}m{1cm}|
>{\centering}m{1.65cm}|
>{\centering}m{1cm}|>{\centering\arraybackslash}m{1.65cm}|}
\hline
\vspace{0.8mm}
$\mathbf{\Delta}$ $(\boldsymbol{T}-\boldsymbol{H})$ & \vspace{0.8mm} \textbf{{Clicks \hyperlink{M3}{M3}}} & \vspace{0.8mm} \textbf{{Keystrokes \hyperlink{M3}{M3}}} & \vspace{0.8mm} \textbf{{Clicks \hyperlink{M5}{M5}}} & \vspace{0.8mm} \textbf{{Keystrokes \hyperlink{M5}{M5}}} \\[3.25mm]
\hline
\vspace{0.8mm} Mean & \vspace{0.8mm} 24.32 & \vspace{0.8mm} -32.55 & \vspace{0.8mm} 29.70 & \vspace{0.8mm} -53.49 \\[1mm]
\hline
\vspace{0.8mm} Median & \vspace{0.8mm} 25 & \vspace{0.8mm} -14.50 & \vspace{0.8mm} 30 & \vspace{0.8mm} -48 \\[1mm]
\hline
\end{tabular}
\endgroup
\end{center}
% \vspace{2mm}
\vspace{-3mm}
\end{table}
\begin{table}[t]
\begin{center}
\begingroup
\renewcommand*{\arraystretch}{0.5}
%\vspace{-3mm}
\caption{\label{tab:confidence_statistics}Deltas of Mean and Median Confidence Levels}
\vspace{-3mm}
\begin{tabular}{|>{\centering}m{0.7cm}|>{\centering}m{1.3cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|
>{\centering}m{0.77cm}|>{\centering\arraybackslash}m{0.77cm}|}
\hline
\vspace{1.2mm} \textbf{{Type}} & \vspace{1.2mm} $\mathbf{\Delta}$ $(\boldsymbol{T}-\boldsymbol{H})$ & \vspace{1.2mm} \textbf{{\#1}} & \vspace{1.2mm} \textbf{{\#2}} & \vspace{1.2mm} \textbf{{\#3}} & \vspace{1.2mm} \textbf{{\#4}} & \vspace{1.2mm} \textbf{{\#5}} \\[1.8mm]
\hline
\multirow{2}{*}{\parbox{1\linewidth}{\vspace{2mm} \textbf{\hspace{2.3mm}U}}} & \vspace{0.8mm} Mean & \vspace{0.8mm} -0.68 & \vspace{0.8mm} -1.18 & \vspace{0.8mm} -0.14 & \vspace{0.8mm} 0.03 & \vspace{0.8mm} -0.05 \\[1mm]
\cline{2-7}
& \vspace{0.8mm} Median & \vspace{0.8mm} -0.50 & \vspace{0.8mm} -1 & \vspace{0.8mm} 0 & \vspace{0.8mm} 0 & \vspace{0.8mm} 0 \\[1mm]
\hline
\multirow{2}{*}{\parbox{1\linewidth}{\vspace{2mm} \textbf{\hspace{2.1mm}M}}} & \vspace{0.8mm} Mean & \vspace{0.8mm} -0.09 & \vspace{0.8mm} 0.09 & \vspace{0.8mm} -0.18 & \vspace{0.8mm} -0.34 & \vspace{0.8mm} -0.32 \\[1mm]
\cline{2-7}
& \vspace{0.8mm} Median & \vspace{0.8mm} 0 & \vspace{0.8mm} 0 & \vspace{0.8mm} 0 & \vspace{0.8mm} 0 & \vspace{0.8mm} 0 \\[1mm]
\hline
\end{tabular}
\endgroup
\end{center}
\vspace{-1mm}
\end{table}
\subsection{Confidence (RQ2)}
We answer research question \hyperlink{link_rq2}{RQ2} concerning the users' confidence when using both editors to do the specified tasks by analysing their self-reported confidence level, ranging between 1 (very low confidence) to 5 (very high confidence).
% As a reminder, research question RQ2 aims to evaluate the confidence of users when using both editors for carrying out various tasks. Recall that the confidence is a self-reported metric on a scale between 1 (very low confidence) to 5 (very high confidence).
%
Table~\ref{tab:confidence_statistics} shows statistics of the confidence level of participants about their task solutions.
As before, the table is split into tasks for understanding (U) and modelling (M), and the five numeric columns are linked to tasks \hyperlink{U1}{U1}--\hyperlink{U5}{U5} and \hyperlink{M1}{M1}--\hyperlink{M5}{M5}. The delta subtracts the mean and median confidence when using the hybrid editor from the mean and median confidence when using the tree editor. For task \hyperlink{U1}{U1}, participants were on average slightly more confident with their solution when using the hybrid editor, signifying users' confidence
% This suggests that users are slightly more confident with their solutions when using a hybrid editor,
for tasks that require understanding a domain and locating various model elements with specific values of interest. Participants were on average more confident by 1.18 points, with their solution to task \hyperlink{U2}{U2} when using the hybrid editor, signifying their confidence in
% that users are more confident with their solutions when using a hybrid editor for
understanding the meaning of conditions. For the remaining tasks, the results show no significant differences in the confidence of the solutions.
We also analyse the correlation between incorrect solutions and confidence level, aiming to determine how confident participants are with their incorrect solutions. A mean confidence of 3.13 and median confidence of 3 has been reported for the incorrect solutions to tasks \hyperlink{U1}{U1}--\hyperlink{U5}{U5} when using the tree editor.
This shows that participants are neutral about the correctness of their partially incorrect solutions. Further, a mean and median confidence of 4.5 has been reported for the incorrect solutions to tasks \hyperlink{M1}{M1}--\hyperlink{M5}{M5} when using the tree editor. Similarly, a mean confidence of 4.67 and a median confidence of 5 has been reported for the incorrect solutions to tasks \hyperlink{M1}{M1}--\hyperlink{M5}{M5} when using the hybrid editor. These results show that participants were confident with their partially incorrect solutions, as they had not noticed any of their mistakes. This finding suggests that the confidence level is related to the ability of users to spot their mistakes. Participants could identify the understanding tasks they could not solve, and that was reflected in their confidence level for each understanding task. However, for the modelling tasks, the participants wrongly assumed that their solutions were correct, leading to a mistaken increased confidence level.
\subsection{Preference (RQ3)}
We answer research question \hyperlink{link_rq3}{RQ3} concerning the preferences of users with respect to the two editor types.
Before engaging in the study, participants were asked for their preferred editor type for modelling. As reported, 3 participants preferred a tree editor, 11 participants a hybrid editor, and 8 participants stated no preference. After performing the given tasks, the participants were asked again about their preferences. As reported, 2 participants preferred a tree editor, 16 participants (73\%) a hybrid editor, and 4 participants had no preference. The results show that one participant changed their preference from a tree editor to a hybrid editor, and half of the participants with no initial preference now prefer a hybrid editor.
Concerning understanding a testing space that does not contain condition expressions, 12 participants reported their preference for a diagrammatic editor, 7 participants for a tree editor, and 3 participants had no preference.
To explore and navigate a testing space that does not contain condition expressions, 13 participants preferred a diagrammatic editor, 7 participants a tree editor, and 2 participants had no preference.
To find relationships between elements of a testing space, 19 participants preferred a diagrammatic editor, 0 participants a tree editor, and 2 participants had no preference.
To understand the meaning of condition expressions, 17 participants preferred a textual representation, 3 participants a tree representation, and 2 participants have no preference.
For modelling conditions from scratch, 16 participants preferred a textual representation, 3 participants a tree representation, and 3 participants have no preference.
To modify existing conditions, 17 participants preferred a textual representation, 4 participants a tree representation, and 1 participant has no preference. As reported, 20 participants stated that adding a textual syntax for modelling conditions inside the tree editor would improve usability and modelling efficiency; 2 participants did not share this opinion.
The feature of the tree editor that participants particularly liked was its conciseness and hierarchical representation that evidences the order and containment of elements. Participants also noted that the tree editor provided a better sense of control over the model, enabling faster exploration, especially when using the keyboard. However, several participants reported difficulties in understanding and defining complex conditions with the tree editor. Many participants would have liked the tree editor to use graphical symbols and colour-coding to differentiate between element types.
Concerning the hybrid editor, participants particularly liked the flexibility in defining complex conditions and finding relationships between elements. Several participants liked the visual aspect of the hybrid editor, as they could relate colours and symbols to specific element types.
However, some participants did not like the need to switch between multiple diagrams and considered that diagrams would not be readable for large models. Also, the textual syntax of conditions was not favoured by all participants, as it contained multiple kinds of brackets for grouping sub-conditions rather than a single kind, and it did not provide any formatting (pretty printing).
\subsection{Threats to Validity}
\label{SEC-THREATS}
% This study is subject to a number of threats to validity that are discussed in the following.
\vspace{0.8mm}\noindent
\textbf{Construct Validity. }
A construct validity threat is that participants may misinterpret task descriptions, although they could eventually understand the descriptions with the help of the session facilitator.
If participants begin a task without completely understanding its description, they will need more time to solve it, and they will make more mistakes that they may later correct.
Thus, the performance metrics for duration, correctness, and errors corrected are, by design, influenced by the clarity and understandability of task descriptions.
\vspace{1.2mm}\noindent
\textbf{Internal Validity. }
The main internal validity threat is that the study was conducted by accessing a remote computer via Team\-View\-er.
For tasks \hyperlink{M3}{M3}--\hyperlink{M5}{M5}, participants used keyboard shortcuts, e.g., control+space to display an auto-completion menu in the textual editor from the hybrid editor.
Some participants faced challenges, either because TeamViewer randomly did not trigger the actions associated with the keyboard shortcuts or because they did not know how to trigger keyboard shortcuts correctly due to their limited knowledge of the Windows OS. In such instances, durations reported with the hybrid editor were higher than expected.
\vspace{1.2mm}\noindent
\textbf{External Validity. }
An external validity threat is that the tree and hybrid editors have been evaluated with a single concrete syntax.
Using a more appealing hybrid notation, with a different graphical representation and with a more intuitive textual syntax, may yield a better performance. Although the tasks covered a broad range of modelling activities, they were uncomplicated, as almost all participants could solve all tasks correctly.
Further experiments would be useful to strengthen the generalisability of our findings, involving hybrid editors that are not Sirius/Xtext-based and other kinds of non-EMF or custom EMF-based tree editors.
\vspace{-2mm}
\section{Related Work}
\label{SEC-RELATED-WORK}
\input{dslpapers}
%\textbf{Hybrid DSLs}
%\lipsum[3]
% ACM's consolidated article template, introduced in 2017, provides a
% consistent \LaTeX\ style for use across ACM publications, and
% incorporates accessibility and metadata-extraction functionality
% necessary for future Digital Library endeavors. Numerous ACM and
% SIG-specific \LaTeX\ templates have been examined, and their unique
% features incorporated into this single new template.
% \vspace{-2mm}
\section{Conclusions and Future Work}
% \vspace{-1mm}
\label{SEC-CONCLUSIONS}
% We explored the tradeoffs between tree-based and hybrid graphical-textual model editors.
% We executed an empirical user study in the context of an industrial use case, investigating the performance, confidence and preferences of users for both model editor types.
Our empirical study, using an industrial use case, aimed to analyse and understand users' performance, confidence, and preferences between using tree-based and hybrid graphical-textual model editors.
% To understand the tradeoffs between tree-based and hybrid graphical-textual model editors, we conducted an empirical study using an industrial use case, analysing users' performance, confidence, and preferences for both model editor types.
Concerning performance, there was a tradeoff between the speed of executing tasks using the editors.
% certain activities can be done faster via a hybrid editor, whereas other activities with a tree editor.
% In particular, t
The hybrid editor was better for comprehending and modelling complex conditions, while the tree editor was better for analysing ordered lists of model elements.
When modelling conditions, more corrected errors were found with the hybrid editor; however, the mistakes made were minor.
Also, more keystrokes and fewer clicks were made for modelling conditions with the hybrid editor.
Users were more confident with their solutions when using the hybrid editor to understand the meaning of conditions.
Finally, before the study, 50\% of participants preferred a hybrid editor; at the end, the percentage increased to 73\%, showing an increased preference towards the hybrid editor.
We plan to validate the generality of our findings by using multiple graphical tree-based and hybrid syntaxes. Also, tree-based and hybrid editors should be evaluated with other domains and case studies, by carrying out a wider range of modelling activities.
\vspace{2.5mm}
% \begin{acks}
\noindent
\textbf{Acknowledgements.}
We thank our SESAME partners and the study participants.
This work was supported by
the SCHEME InnovateUK project (\#10065634),
the EU Horizon 2020 project SESAME (\#101017258), and
the Horizon Europe project SOPRANO (\#101120990).%. This work was also partially supported by
%(Safety Critical Harsh Environment Micro-processing Evolution)
% \end{acks}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
\end{document}
\endinput
%%
%% End of file `sample-sigplan.tex'.
\subsubsection{Graphical/Tree Based Syntax}
\begin{itemize}
\item Good for quick exploration - especially faster with keyboard
\item Better for non-condition/simple expressions
\item Some users found it hard to understand modifying/defining complex conditions
\item Some users liked defining/modifying conditions
\item Makes ordering/containment of elements clearer
% From the user survey text
\item Concise and hierarchical order; lets you see entire system on single page
\item Would likely have better scalability as models enlarge
\item Some users commented on it giving more of a sense of control over the model
\item May be more useful than hybrid for non-experts in domain to understand metamodel
\item Potential improvements: graphical/meaningful icons, visual difference to different elements
\item Potential improvements: Clarify type of element, visually or by color coding
\item Potential improvements: more information visible at level of each element (without clicking on it)
\item Potential improvements: introduce textual representation into tree e.g. for conditions
\end{itemize}
\subsubsection{Hybrid Syntax}
\begin{itemize}
\item Generally text better for entering condition expressions, especially deep expressions
\item Diagram view switching/multiple boxes layout could be annoying
\item Some users not clear on why they had to double-click+confirm to enter new view
\item May not be scalable to larger results - automatic layout may help
% From the user survey text
\item Visual aspect of hybrid syntax (e.g. tool pallette, colours etc) helpful
\item Easier to find interdependencies / relationships between objects in DSL
\item Textual provides a representation similar to that of programming languages
\item Allows prioritisation of elements by re-ordering e.g. manually adjusting layout
\item Potential improvements; auto layout algorithms, e.g. arrange of place items clearly in specific zones
\item Potential improvements; remove brackets, or use single consistent bracket throughout
\item Potential improvements: Improved validation or tooltips for conditions
\item Potential improvements; add home button to return to root
\end{itemize}
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{54}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Addazi et~al\mbox{.}(2017)]%
{addazi2017towards}
\bibfield{author}{\bibinfo{person}{Lorenzo Addazi}, \bibinfo{person}{Federico Ciccozzi}, \bibinfo{person}{Philip Langer}, {and} \bibinfo{person}{Ernesto Posse}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Towards Seamless Hybrid Graphical--Textual Modelling for UML and Profiles}. In \bibinfo{booktitle}{\emph{Modelling Foundations and Applications: 13th European Conference, ECMFA 2017, Held as Part of STAF 2017, Marburg, Germany, July 19-20, 2017, Proceedings 13}}. Springer, \bibinfo{pages}{20--33}.
\newblock
\bibitem[Altran(2022)]%
{solution_altran}
\bibfield{author}{\bibinfo{person}{Altran}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{X}text {S}irius integration}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://altran-mde.github.io/xtext-sirius-integration.io}, (Last Accessed: 2024-07-20)}.
\bibitem[Bruyninckx et~al\mbox{.}(2013)]%
{bruyninckx2013brics}
\bibfield{author}{\bibinfo{person}{Herman Bruyninckx}, \bibinfo{person}{Markus Klotzb{\"u}cher}, \bibinfo{person}{Nico Hochgeschwender}, \bibinfo{person}{Gerhard Kraetzschmar}, \bibinfo{person}{Luca Gherardi}, {and} \bibinfo{person}{Davide Brugali}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{The BRICS component model: a model-based development paradigm for complex robotics software systems}. In \bibinfo{booktitle}{\emph{Proceedings of the 28th Annual ACM Symposium on Applied Computing}}. \bibinfo{pages}{1758--1764}.
\newblock
\bibitem[Burnette(2005)]%
{burnette2005eclipse}
\bibfield{author}{\bibinfo{person}{Ed Burnette}.} \bibinfo{year}{2005}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{E}clipse {IDE} {P}ocket {G}uide: {U}sing the {F}ull-{F}eatured {IDE}}}.
\newblock \bibinfo{publisher}{"O'Reilly Media"}, Chapter Workbench 101, \bibinfo{pages}{7--17}.
\newblock
\bibitem[Casalaro et~al\mbox{.}(2022)]%
{casalaro2021model}
\bibfield{author}{\bibinfo{person}{Giuseppina~Lucia Casalaro}, \bibinfo{person}{Giulio Cattivera}, \bibinfo{person}{Federico Ciccozzi}, \bibinfo{person}{Ivano Malavolta}, \bibinfo{person}{Andreas Wortmann}, {and} \bibinfo{person}{Patrizio Pelliccione}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Model-driven engineering for mobile robotic systems: a systematic mapping study}.
\newblock \bibinfo{journal}{\emph{Software and Systems Modeling}} \bibinfo{volume}{21}, \bibinfo{number}{1} (\bibinfo{year}{2022}), \bibinfo{pages}{19--49}.
\newblock
\bibitem[Cattivera and Casalaro(2015)]%
{cattivera2015model}
\bibfield{author}{\bibinfo{person}{G Cattivera} {and} \bibinfo{person}{GL Casalaro}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \emph{\bibinfo{title}{Model-Driven Engineering for Mobile Robot Systems: A Systematic Mapping Study}}.
\newblock \bibinfo{thesistype}{Master's\ thesis}. \bibinfo{school}{Mälardalen University}.
\newblock
\bibitem[Challenger et~al\mbox{.}(2016)]%
{mas-dsl-2016}
\bibfield{author}{\bibinfo{person}{Moharram Challenger}, \bibinfo{person}{Geylani Kardas}, {and} \bibinfo{person}{Bedir Tekinerdogan}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{A systematic approach to evaluating domain-specific modeling language environments for multi-agent systems}.
\newblock \bibinfo{journal}{\emph{Software Quality Journal}} \bibinfo{volume}{24}, \bibinfo{number}{3} (\bibinfo{year}{2016}), \bibinfo{pages}{755--795}.
\newblock
\showISSN{1573-1367}
\bibitem[Ciccozzi et~al\mbox{.}(2016)]%
{ciccozzi2016adopting}
\bibfield{author}{\bibinfo{person}{Federico Ciccozzi}, \bibinfo{person}{Davide Di~Ruscio}, \bibinfo{person}{Ivano Malavolta}, {and} \bibinfo{person}{Patrizio Pelliccione}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{{A}dopting {MDE} for {S}pecifying and {E}xecuting {C}ivilian {M}issions of {M}obile {M}ulti-{R}obot {S}ystems}.
\newblock \bibinfo{journal}{\emph{IEEE Access}} \bibinfo{volume}{4} (\bibinfo{year}{2016}), \bibinfo{pages}{6451--6466}.
\newblock
\bibitem[Ciccozzi et~al\mbox{.}(2017)]%
{ciccozzi2017engineering}
\bibfield{author}{\bibinfo{person}{Federico Ciccozzi}, \bibinfo{person}{Davide Di~Ruscio}, \bibinfo{person}{Ivano Malavolta}, \bibinfo{person}{Patrizio Pelliccione}, {and} \bibinfo{person}{Jana Tumova}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Engineering the software of robotic systems}. In \bibinfo{booktitle}{\emph{IEEE/ACM 39th International Conference on Software Engineering Companion}}. \bibinfo{pages}{507--508}.
\newblock
\bibitem[Ciccozzi et~al\mbox{.}(2019)]%
{ciccozzi2019blended}
\bibfield{author}{\bibinfo{person}{Federico Ciccozzi}, \bibinfo{person}{Matthias Tichy}, \bibinfo{person}{Hans Vangheluwe}, {and} \bibinfo{person}{Danny Weyns}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{B}lended {M}odelling - {W}hat, {W}hy and {H}ow}. In \bibinfo{booktitle}{\emph{2019 ACM/IEEE 22nd International Conference on Model Driven Engineering Languages and Systems Companion (MODELS-C)}}. IEEE, \bibinfo{pages}{425--430}.
\newblock
\bibitem[Cooper(2018)]%
{cooper2018msc}
\bibfield{author}{\bibinfo{person}{Justin Cooper}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \emph{\bibinfo{title}{{A} {F}ramework to {E}mbed {T}extual {D}omain {S}pecific {L}anguages in {G}raphical {M}odel {E}ditors}}.
\newblock \bibinfo{thesistype}{Master's\ thesis}. \bibinfo{school}{University of York}.
\newblock
\bibitem[Cooper and Kolovos(2019)]%
{cooper2019engineering}
\bibfield{author}{\bibinfo{person}{Justin Cooper} {and} \bibinfo{person}{Dimitris Kolovos}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{E}ngineering {H}ybrid {G}raphical-{T}extual {L}anguages with {S}irius and {X}text: {R}equirements and {C}hallenges}. In \bibinfo{booktitle}{\emph{{ACM/IEEE} 22nd {I}nternational {C}onference on {M}odel {D}riven {E}ngineering {L}anguages and {S}ystems {C}ompanion}}. \bibinfo{pages}{322--325}.
\newblock
\bibitem[Dhouib et~al\mbox{.}(2012)]%
{dhouib2012robotml}
\bibfield{author}{\bibinfo{person}{Saadia Dhouib}, \bibinfo{person}{Selma Kchir}, \bibinfo{person}{Serge Stinckwich}, \bibinfo{person}{Tewfik Ziadi}, {and} \bibinfo{person}{Mikal Ziane}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{{R}obotML, a {D}omain-{S}pecific {L}anguage to {D}esign, {S}imulate and {D}eploy {R}obotic {A}pplications}. In \bibinfo{booktitle}{\emph{International Conference on Simulation, Modeling, and Programming for Autonomous Robots}}. Springer, \bibinfo{pages}{149--160}.
\newblock
\bibitem[Dragule et~al\mbox{.}(2017)]%
{dragule2017generated}
\bibfield{author}{\bibinfo{person}{Swaib Dragule}, \bibinfo{person}{Bart Meyers}, {and} \bibinfo{person}{Patrizio Pelliccione}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{{A} {G}enerated {P}roperty {S}pecification {L}anguage for {R}esilient {M}ultirobot {M}issions}. In \bibinfo{booktitle}{\emph{Software Engineering for Resilient Systems: 9th International Workshop, SERENE 2017, Geneva, Switzerland, September 4--5, 2017, Proceedings 9}}. Springer, \bibinfo{pages}{45--61}.
\newblock
\bibitem[Dreossi et~al\mbox{.}(2019)]%
{verifai-cav19}
\bibfield{author}{\bibinfo{person}{Tommaso Dreossi}, \bibinfo{person}{Daniel~J. Fremont}, \bibinfo{person}{Shromona Ghosh}, \bibinfo{person}{Edward Kim}, \bibinfo{person}{Hadi Ravanbakhsh}, \bibinfo{person}{Marcell Vazquez{-}Chanlatte}, {and} \bibinfo{person}{Sanjit~A. Seshia}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{VerifAI:} {A} Toolkit for the Formal Design and Analysis of Artificial Intelligence-Based Systems}. In \bibinfo{booktitle}{\emph{31st International Conference on Computer Aided Verification (CAV)}}.
\newblock
\bibitem[Eclipse\;Foundation(2024a)]%
{capella_extension}
\bibfield{author}{\bibinfo{person}{Eclipse\;Foundation}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \bibinfo{booktitle}{\emph{{C}apella {T}extual {E}ditor {E}xtension}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://github.com/eclipse/capella-textual-editor}, (Last Accessed: 2024-07-20)}.
\bibitem[Eclipse\;Foundation(2024b)]%
{sirius_website}
\bibfield{author}{\bibinfo{person}{Eclipse\;Foundation}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \bibinfo{booktitle}{\emph{{S}irius {W}ebsite}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://eclipse.dev/sirius}, (Last Accessed: 2024-07-20)}.
\bibitem[Eclipse\;Foundation(2024c)]%
{xtext_website}
\bibfield{author}{\bibinfo{person}{Eclipse\;Foundation}.} \bibinfo{year}{2024}\natexlab{c}.
\newblock \bibinfo{booktitle}{\emph{{X}text {W}ebsite}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://eclipse.dev/Xtext}, (Last Accessed: 2024-07-20)}.
\bibitem[Fremont et~al\mbox{.}(2019)]%
{scenic-2019}
\bibfield{author}{\bibinfo{person}{Daniel~J. Fremont}, \bibinfo{person}{Tommaso Dreossi}, \bibinfo{person}{Shromona Ghosh}, \bibinfo{person}{Xiangyu Yue}, \bibinfo{person}{Alberto~L. Sangiovanni-Vincentelli}, {and} \bibinfo{person}{Sanjit~A. Seshia}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Scenic: a language for scenario specification and scene generation}. In \bibinfo{booktitle}{\emph{Proceedings of the 40th ACM SIGPLAN Conference on Programming Language Design and Implementation}}. \bibinfo{pages}{63–78}.
\newblock
\showISBNx{9781450367127}
\bibitem[Gascuena et~al\mbox{.}(2012)]%
{gascuena2012model}
\bibfield{author}{\bibinfo{person}{Jos{\'e}~M Gascuena}, \bibinfo{person}{Elena Navarro}, {and} \bibinfo{person}{Antonio Fern{\'a}ndez-Caballero}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Model-driven engineering techniques for the development of multi-agent systems}.
\newblock \bibinfo{journal}{\emph{Engineering Applications of Artificial Intelligence}} \bibinfo{volume}{25}, \bibinfo{number}{1} (\bibinfo{year}{2012}), \bibinfo{pages}{159--173}.
\newblock
\bibitem[Gerasimou et~al\mbox{.}(2019)]%
{gerasimou2019towards}
\bibfield{author}{\bibinfo{person}{Simos Gerasimou}, \bibinfo{person}{Nicholas Matragkas}, {and} \bibinfo{person}{Radu Calinescu}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{T}owards {S}ystematic {E}ngineering of {C}ollaborative {H}eterogeneous {R}obotic {S}ystems}. In \bibinfo{booktitle}{\emph{2019 IEEE/ACM 2nd International Workshop on Robotics Software Engineering (RoSE)}}. \bibinfo{publisher}{IEEE}, \bibinfo{pages}{25--28}.
\newblock
\bibitem[Godefroid(2020)]%
{godefroid2020fuzzing}
\bibfield{author}{\bibinfo{person}{Patrice Godefroid}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Fuzzing: Hack, Art, and Science}.
\newblock \bibinfo{journal}{\emph{Commun. ACM}} \bibinfo{volume}{63}, \bibinfo{number}{2} (\bibinfo{year}{2020}), \bibinfo{pages}{70--76}.
\newblock
\bibitem[Harbin et~al\mbox{.}(2021)]%
{harbin-models-2021}
\bibfield{author}{\bibinfo{person}{James Harbin}, \bibinfo{person}{Simos Gerasimou}, \bibinfo{person}{Nicholas Matragkas}, \bibinfo{person}{Athanasios Zolotas}, {and} \bibinfo{person}{Radu Calinescu}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Model-Driven Simulation-Based Analysis for Multi-Robot Systems}. In \bibinfo{booktitle}{\emph{24th International Conference on Model Driven Engineering Languages and Systems (MODELS' 21)}}. \bibinfo{publisher}{IEEE}, \bibinfo{pages}{331--341}.
\newblock
\bibitem[Harbin et~al\mbox{.}(2023)]%
{harbin2023model}
\bibfield{author}{\bibinfo{person}{James Harbin}, \bibinfo{person}{Simos Gerasimou}, \bibinfo{person}{Nicholas Matragkas}, \bibinfo{person}{Thanos Zolotas}, \bibinfo{person}{Radu Calinescu}, {and} \bibinfo{person}{Misael Alpizar~Santana}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Model-driven design space exploration for multi-robot systems in simulation}.
\newblock \bibinfo{journal}{\emph{Software and Systems Modeling}} \bibinfo{volume}{22}, \bibinfo{number}{5} (\bibinfo{year}{2023}), \bibinfo{pages}{1665--1688}.
\newblock
\bibitem[Hill(2011)]%
{hill2011measuring}
\bibfield{author}{\bibinfo{person}{James~H Hill}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{{M}easuring and {R}educing {M}odeling {E}ffort in {D}omain-{S}pecific {M}odeling {L}anguages with {E}xamples}. In \bibinfo{booktitle}{\emph{2011 18th IEEE International Conference and Workshops on Engineering of Computer-Based Systems}}. IEEE, \bibinfo{pages}{120--129}.
\newblock
\bibitem[Hoffmann et~al\mbox{.}(2022)]%
{athos-2022}
\bibfield{author}{\bibinfo{person}{Benjamin Hoffmann}, \bibinfo{person}{Neil Urquhart}, \bibinfo{person}{Kevin Chalmers}, {and} \bibinfo{person}{Michael Guckert}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{An empirical evaluation of a novel domain-specific language -- modelling vehicle routing problems with Athos}.
\newblock \bibinfo{journal}{\emph{Empirical Software Engineering}} \bibinfo{volume}{27}, \bibinfo{number}{7} (\bibinfo{year}{2022}), \bibinfo{pages}{180}.
\newblock
\showISSN{1573-7616}
\bibitem[Jak{\v{s}}i{\'{c}} et~al\mbox{.}(2014)]%
{familiar-jaksic-2014}
\bibfield{author}{\bibinfo{person}{Aleksandar Jak{\v{s}}i{\'{c}}}, \bibinfo{person}{Robert~B. France}, \bibinfo{person}{Philippe Collet}, {and} \bibinfo{person}{Sudipto Ghosh}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{Evaluating the Usability of a Visual Feature Modeling Notation}. In \bibinfo{booktitle}{\emph{Software Language Engineering: 7th International Conference, SLE 2014, V{\"a}ster{\aa}s, Sweden, September 15-16, 2014. Proceedings 7}}, \bibfield{editor}{\bibinfo{person}{Beno{\^i}t Combemale}, \bibinfo{person}{David~J. Pearce}, \bibinfo{person}{Olivier Barais}, {and} \bibinfo{person}{Jurgen~J. Vinju}} (Eds.). \bibinfo{publisher}{Springer}, \bibinfo{address}{Cham}, \bibinfo{pages}{122--140}.
\newblock
\showISBNx{978-3-319-11245-9}
\bibitem[{J}et{B}rains(2024)]%
{jetbrains_website}
\bibfield{author}{\bibinfo{person}{{J}et{B}rains}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{J}etBrains {MPS} {W}ebsite}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://www.jetbrains.com/mps}, (Last Accessed: 2024-07-20)}.
\bibitem[Jolak et~al\mbox{.}(2020)]%
{whispers-jolak-2020}
\bibfield{author}{\bibinfo{person}{Rodi Jolak}, \bibinfo{person}{Maxime Savary-Leblanc}, \bibinfo{person}{Manuela Dalibor}, \bibinfo{person}{Andreas Wortmann}, \bibinfo{person}{Regina Hebig}, \bibinfo{person}{Juraj Vincur}, \bibinfo{person}{Ivan Polasek}, \bibinfo{person}{Xavier Le~Pallec}, \bibinfo{person}{S\'{e}bastien G\'{e}rard}, {and} \bibinfo{person}{Michel R.~V. Chaudron}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Software engineering whispers: The effect of textual vs. graphical software design descriptions on software design communication}.
\newblock \bibinfo{journal}{\emph{Empirical software engineering}} \bibinfo{volume}{25}, \bibinfo{number}{6} (\bibinfo{year}{2020}), \bibinfo{pages}{4427–4471}.
\newblock
\showISSN{1382-3256}
\bibitem[Kahraman and Bilgen(2015)]%
{fqad-dsls-2013}
\bibfield{author}{\bibinfo{person}{G\"{o}khan Kahraman} {and} \bibinfo{person}{Semih Bilgen}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{A framework for qualitative assessment of domain-specific languages}.
\newblock \bibinfo{journal}{\emph{Software \& Systems Modeling}} \bibinfo{volume}{14}, \bibinfo{number}{4} (\bibinfo{year}{2015}), \bibinfo{pages}{1505–1526}.
\newblock
\showISSN{1619-1366}
\bibitem[KUKA(2024)]%
{kuka_website}
\bibfield{author}{\bibinfo{person}{KUKA}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Company Website}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://www.kuka.com}, (Last Accessed: 2024-07-20)}.
\bibitem[Liang et~al\mbox{.}(2018)]%
{liang2018fuzz}
\bibfield{author}{\bibinfo{person}{Jie Liang}, \bibinfo{person}{Mingzhe Wang}, \bibinfo{person}{Yuanliang Chen}, \bibinfo{person}{Yu Jiang}, {and} \bibinfo{person}{Renwei Zhang}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{{F}uzz testing in practice: {O}bstacles and solutions}. In \bibinfo{booktitle}{\emph{IEEE 25th International Conference on Software Analysis, Evolution and Reengineering}}. \bibinfo{pages}{562--566}.
\newblock
\bibitem[Miyazawa et~al\mbox{.}(2019)]%
{miyazawa2019robochart}
\bibfield{author}{\bibinfo{person}{Alvaro Miyazawa}, \bibinfo{person}{Pedro Ribeiro}, \bibinfo{person}{Wei Li}, \bibinfo{person}{Ana Cavalcanti}, \bibinfo{person}{Jon Timmis}, {and} \bibinfo{person}{Jim Woodcock}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{RoboChart: modelling and verification of the functional behaviour of robotic applications}.
\newblock \bibinfo{journal}{\emph{Software \& Systems Modeling}} \bibinfo{volume}{18}, \bibinfo{number}{5} (\bibinfo{year}{2019}), \bibinfo{pages}{3097--3149}.
\newblock
\bibitem[Obeo and TypeFox(2017)]%
{paper_obeo_typefox}
\bibfield{author}{\bibinfo{person}{Obeo} {and} \bibinfo{person}{TypeFox}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{X}text {S}irius integration - white paper}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://www.obeodesigner.com/resource/white-paper/WhitePaper_XtextSirius_EN.pdf}, (Last Accessed: 2024-07-20)}.
\bibitem[Ottensooser et~al\mbox{.}(2012)]%
{ottensooser-compare-2010}
\bibfield{author}{\bibinfo{person}{Avner Ottensooser}, \bibinfo{person}{Alan Fekete}, \bibinfo{person}{Hajo~A. Reijers}, \bibinfo{person}{Jan Mendling}, {and} \bibinfo{person}{Con Menictas}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Making sense of business process descriptions: An experimental comparison of graphical and textual notations}.
\newblock \bibinfo{journal}{\emph{Journal of Systems and Software}} \bibinfo{volume}{85}, \bibinfo{number}{3} (\bibinfo{year}{2012}), \bibinfo{pages}{596--606}.
\newblock
\showISSN{0164-1212}
\newblock
\shownote{Novel approaches in the design and implementation of systems/software architecture}.
\bibitem[Padhye et~al\mbox{.}(2019)]%
{padhye2019fuzzfactory}
\bibfield{author}{\bibinfo{person}{Rohan Padhye}, \bibinfo{person}{Caroline Lemieux}, \bibinfo{person}{Koushik Sen}, \bibinfo{person}{Laurent Simon}, {and} \bibinfo{person}{Hayawardh Vijayakumar}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Fuzzfactory: domain-specific fuzzing with waypoints}.
\newblock \bibinfo{journal}{\emph{Proceedings of the ACM on Programming Languages}} \bibinfo{volume}{3}, \bibinfo{number}{OOPSLA} (\bibinfo{year}{2019}), \bibinfo{pages}{1--29}.
\newblock
\bibitem[Paraschos et~al\mbox{.}(2012)]%
{paraschos2012model}
\bibfield{author}{\bibinfo{person}{Alexandros Paraschos}, \bibinfo{person}{Nikolaos~I Spanoudakis}, {and} \bibinfo{person}{Michail~G Lagoudakis}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Model-Driven Behavior Specification for Robotic Teams}. In \bibinfo{booktitle}{\emph{Proceedings of the 11th International Conference on Autonomous Agents and Multiagent Systems - Volume 1}} (Valencia, Spain) \emph{(\bibinfo{series}{AAMAS '12})}. \bibinfo{pages}{171--178}.
\newblock
\bibitem[Parker(2013)]%
{parker2012reliability}
\bibfield{author}{\bibinfo{person}{Lynne~E Parker}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{R}eliability and {F}ault {T}olerance in {C}ollective {R}obot {S}ystems}}.
\newblock \bibinfo{publisher}{Jenny Stanford Publishing}, Chapter~6, \bibinfo{pages}{167--205}.
\newblock
\bibitem[Partners(2022)]%
{sesame-deliverable-d6.2}
\bibfield{author}{\bibinfo{person}{SESAME~Project Partners}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{D6.2: Simulation-Based Testing Methodology for {EDDIs}}}.
\newblock \bibinfo{type}{{T}echnical {R}eport}. \bibinfo{institution}{The Open Group}.
\newblock
\bibitem[Partners(2023a)]%
{tts-dtwin-deliverable}
\bibfield{author}{\bibinfo{person}{SESAME~Project Partners}.} \bibinfo{year}{2023}\natexlab{a}.
\newblock \bibinfo{booktitle}{\emph{D6.5: Quality Assurance of {EDDI}-Enabled {MRS} Using Digital Twins}}.
\newblock \bibinfo{type}{{T}echnical {R}eport}. \bibinfo{institution}{The Open Group}.
\newblock
\bibitem[Partners(2023b)]%
{sesame-deliverable-d6.6}
\bibfield{author}{\bibinfo{person}{SESAME~Project Partners}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \bibinfo{booktitle}{\emph{D6.6: Multi-Stage Quality Assurance Methodology for {EDDIs}}}.
\newblock \bibinfo{type}{{T}echnical {R}eport}. \bibinfo{institution}{The Open Group}.
\newblock
\bibitem[Pinciroli and Beltrame(2016)]%
{pinciroli2016buzz}
\bibfield{author}{\bibinfo{person}{Carlo Pinciroli} {and} \bibinfo{person}{Giovanni Beltrame}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Buzz: An extensible programming language for heterogeneous swarm robotics}. In \bibinfo{booktitle}{\emph{2016 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)}}. IEEE, \bibinfo{pages}{3794--3800}.
\newblock
\bibitem[Predoaia(2023)]%
{predoaia2023towards}
\bibfield{author}{\bibinfo{person}{Ionut Predoaia}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{{T}owards {S}ystematic {E}ngineering of {H}ybrid {G}raphical-{T}extual {D}omain-{S}pecific {L}anguages}. In \bibinfo{booktitle}{\emph{2023 ACM/IEEE International Conference on Model Driven Engineering Languages and Systems Companion (MODELS-C)}}. \bibinfo{publisher}{IEEE}, \bibinfo{pages}{153--158}.
\newblock
\bibitem[Predoaia(2024)]%
{reproduction_package}
\bibfield{author}{\bibinfo{person}{Ionut Predoaia}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Reproduction Package for the User Study Tasks}}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.5281/zenodo.10892973}
\showDOI{\tempurl}
\bibitem[Predoaia et~al\mbox{.}(2023)]%
{predoaia2023streamlining}
\bibfield{author}{\bibinfo{person}{Ionut Predoaia}, \bibinfo{person}{Dimitris Kolovos}, \bibinfo{person}{Matthias Lenk}, {and} \bibinfo{person}{Antonio García-Domínguez}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{{S}treamlining the {D}evelopment of {H}ybrid {G}raphical-{T}extual {M}odel {E}ditors for {D}omain-{S}pecific {L}anguages}.
\newblock \bibinfo{journal}{\emph{Journal of Object Technology}} \bibinfo{volume}{22}, \bibinfo{number}{2} (\bibinfo{year}{2023}), \bibinfo{pages}{1--14}.
\newblock
\bibitem[Scheidgen(2008)]%
{scheidgen2008textual}
\bibfield{author}{\bibinfo{person}{Markus Scheidgen}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{{T}extual {M}odelling {E}mbedded into {G}raphical {M}odelling}. In \bibinfo{booktitle}{\emph{{E}uropean {C}onference on {M}odel {D}riven {A}rchitecture-{F}oundations and {A}pplications}}. Springer, \bibinfo{pages}{153--168}.
\newblock
\bibitem[Schlegel et~al\mbox{.}(2009)]%
{schlegel2009robotic}
\bibfield{author}{\bibinfo{person}{Christian Schlegel}, \bibinfo{person}{Thomas Ha{\ss}ler}, \bibinfo{person}{Alex Lotz}, {and} \bibinfo{person}{Andreas Steck}.} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{Robotic software systems: From code-driven to model-driven designs}. In \bibinfo{booktitle}{\emph{2009 International Conference on Advanced Robotics}}. IEEE, \bibinfo{pages}{1--8}.
\newblock
\bibitem[Schlegel et~al\mbox{.}(2010)]%
{schlegel2010design}
\bibfield{author}{\bibinfo{person}{Christian Schlegel}, \bibinfo{person}{Andreas Steck}, \bibinfo{person}{Davide Brugali}, {and} \bibinfo{person}{Alois Knoll}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{{D}esign {A}bstraction and {P}rocesses in {R}obotics: {F}rom {C}ode-{D}riven to {M}odel-{D}riven {E}ngineering}. In \bibinfo{booktitle}{\emph{International Conference on Simulation, Modeling, and Programming for Autonomous Robots}}. Springer, \bibinfo{pages}{324--335}.
\newblock
\bibitem[SESAME(2024)]%
{sesame_website}
\bibfield{author}{\bibinfo{person}{SESAME}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{P}roject {W}ebsite}}.
\newblock
\newblock
\shownote{[Online]. Available: \url{https://www.sesame-project.org}, (Last Accessed: 2024-07-20)}.
\bibitem[Skubch et~al\mbox{.}(2011)]%
{skubch2011modelling}
\bibfield{author}{\bibinfo{person}{Hendrik Skubch}, \bibinfo{person}{Michael Wagner}, \bibinfo{person}{Roland Reichle}, {and} \bibinfo{person}{Kurt Geihs}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{A modelling language for cooperative plans in highly dynamic domains}.
\newblock \bibinfo{journal}{\emph{Mechatronics}} \bibinfo{volume}{21}, \bibinfo{number}{2} (\bibinfo{year}{2011}), \bibinfo{pages}{423--433}.
\newblock
\bibitem[Timperley et~al\mbox{.}(2018)]%
{timperley-icst18}
\bibfield{author}{\bibinfo{person}{Christopher~Steven Timperley}, \bibinfo{person}{Afsoon Afzal}, \bibinfo{person}{Deborah~S Katz}, \bibinfo{person}{Jam~Marcos Hernandez}, {and} \bibinfo{person}{Claire Le~Goues}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{{C}rashing {S}imulated {P}lanes is {C}heap: {C}an {S}imulation {D}etect {R}obotics {B}ugs {E}arly?}. In \bibinfo{booktitle}{\emph{2018 IEEE 11th International Conference on Software Testing, Verification and Validation (ICST)}}. \bibinfo{publisher}{IEEE}, \bibinfo{pages}{331--342}.
\newblock
\bibitem[Wei et~al\mbox{.}(2020)]%
{wei2020automatic}
\bibfield{author}{\bibinfo{person}{Ran Wei}, \bibinfo{person}{Athanasios Zolotas}, \bibinfo{person}{Horacio Hoyos~Rodriguez}, \bibinfo{person}{Simos Gerasimou}, \bibinfo{person}{Dimitrios~S Kolovos}, {and} \bibinfo{person}{Richard~F Paige}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Automatic generation of UML profile graphical editors for Papyrus}.
\newblock \bibinfo{journal}{\emph{Software and Systems Modeling}} \bibinfo{volume}{19} (\bibinfo{year}{2020}), \bibinfo{pages}{1083--1106}.
\newblock
\bibitem[Wood et~al\mbox{.}(2020)]%
{wood2020supporting}
\bibfield{author}{\bibinfo{person}{Sophie Wood}, \bibinfo{person}{Nicholas Matragkas}, \bibinfo{person}{Dimitris Kolovos}, \bibinfo{person}{Richard Paige}, {and} \bibinfo{person}{Simos Gerasimou}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Supporting robotic software migration using static analysis and model-driven engineering}. In \bibinfo{booktitle}{\emph{Proceedings of the 23rd ACM/IEEE International Conference on Model Driven Engineering Languages and Systems}}. \bibinfo{pages}{154--164}.
\newblock
\bibitem[Zolotas et~al\mbox{.}(2018)]%
{zolotas2018towards}
\bibfield{author}{\bibinfo{person}{Athanasios Zolotas}, \bibinfo{person}{Ran Wei}, \bibinfo{person}{Simos Gerasimou}, \bibinfo{person}{Horacio Hoyos~Rodriguez}, \bibinfo{person}{Dimitrios~S Kolovos}, {and} \bibinfo{person}{Richard~F Paige}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{{T}owards {A}utomatic {G}eneration of {UML} {P}rofile {G}raphical {E}ditors for {P}apyrus}. In \bibinfo{booktitle}{\emph{Modelling Foundations and Applications: ECMFA, Proceedings 14}}. Springer, \bibinfo{pages}{12--27}.
\newblock
\end{thebibliography}