$\texttt{causalAssembly}$: Generating Realistic Production Data for Benchmarking Causal Discovery
Extracted source text · arXiv 2306.10816
\documentclass{article}
% if you need to pass options to natbib, use, e.g.:
% \PassOptionsToPackage{numbers, compress}{natbib}
% before loading neurips_data_2023
% to compile a preprint version, add the [preprint] option, e.g.:
% \usepackage[preprint]{neurips_data_2023}
% This will indicate that the work is currently under review.
% to compile a camera-ready version, add the [final] option, e.g.:
% \usepackage[final]{neurips_data_2023}
% to avoid loading the natbib package, add option nonatbib:
% \usepackage[nonatbib]{neurips_data_2023}
% Submissions to the datasets and benchmarks are typically non anonymous,
% but anonymous submissions are allowed. If you feel that you must submit
% anonymously, you can compile an anonymous version by adding the [anonymous]
% option, e.g.:
% \usepackage[anonymous]{neurips_data_2023}
% This will hide all author names.
%\usepackage{neurips_data_2023}
\PassOptionsToPackage{numbers, compress}{natbib}
\usepackage[preprint]{neurips_data_2023}
\bibliographystyle{unsrtnat}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{hyperref} % hyperlinks
\usepackage{url} % simple URL typesetting
\usepackage{booktabs}
\usepackage{tabularx } % professional-quality tables
\usepackage{amsfonts} % blackboard math symbols\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{bbm}
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{color}
\usepackage{graphicx}
\usepackage{todonotes}
\usepackage[ruled]{algorithm2e}
\usepackage{tikz}
\usepackage{pgfplots}
\pgfplotsset{compat=1.16}
\usepackage{wrapfig}
\usepackage{caption, subcaption}
\usepackage{array}
\usetikzlibrary{fit,positioning,calc,arrows,backgrounds}
\newcolumntype{C}[1]{>{\centering\arraybackslash}p{#1}}
\tikzset{
main/.style = {
draw,
circle,},
every node/.style={scale=.7}
}
%\usepackage{xr}
%\externaldocument{supplementary}
\usepackage{tkz-berge}
\renewcommand*{\VertexSmallMinSize}{4pt}
\usetikzlibrary{patterns}
\usetikzlibrary{positioning}
\usetikzlibrary{patterns}
\usetikzlibrary{decorations.pathreplacing}
\usetikzlibrary{graphs,graphs.standard}
\usetikzlibrary{calc}
\usetikzlibrary{shapes.geometric,backgrounds}
%Tikz
\GraphInit[vstyle=Classic]
\SetGraphUnit{1}
\SetVertexMath
\usepackage{color}
\definecolor{spgreen}{RGB}{46,139,87}
\definecolor{tored}{RGB}{220,20,60}
\title{\texttt{causalAssembly}: Generating Realistic Production Data for Benchmarking Causal Discovery}
\author{%
Konstantin Göbler$^{1,3}$ \quad Tobias Windisch$^{2}$ \quad Tim
Pychynski$^3$ \\\textbf{Steffen Sonntag$^3$ \quad Martin Roth$^3$ \quad Mathias Drton$^1$} \\
$^1$Technical University Munich \quad $^2$University of Applied
Sciences Kempten \\$^3$Robert Bosch GmbH\\
\texttt{\{konstantin.goebler,mathias.drton\}@tum.de}\\
\texttt{\{steffen.sonntag,tim.pychynski,martin.roth2\}@de.bosch.com}\\
\texttt{tobias.windisch@hs-kempten.de}
}
%-------------------------------------------------------------------------------$
%-------------------------------------------------------------------------------$
\begin{document}
\maketitle
\begin{abstract}
\input{sections/abstract.tex}
\end{abstract}
%-------------------------------------------------------------------------------$
\section{Introduction}\label{intro}
\input{sections/introduction.tex}
%-------------------------------------------------------------------------------$
\section{Identifiability and common benchmarking strategies}\label{sec:prelim}
\input{sections/preliminaries.tex}
%-------------------------------------------------------------------------------$
\section{Data generation procedure}\label{sec:datagen}
\input{sections/datageneration.tex}
%-------------------------------------------------------------------------------$
\section{Benchmark results}\label{sec:benchmark}
\input{sections/benchmarking.tex}
%-------------------------------------------------------------------------------$
\section{Conclusion and Limitations}\label{sec:conclusion} %broader impact of \texttt{causalAssembly}}
\input{sections/conclusion.tex}
%-------------------------------------------------------------------------------$
\begin{ack}
\input{sections/acknowledgements.tex}
\end{ack}
\clearpage
\newpage
\bibliography{bib}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\section*{Checklist}
%%% BEGIN INSTRUCTIONS %%%
%The checklist follows the references. Please
%read the checklist guidelines carefully for information on how to answer these
%questions. For each question, change the default \answerTODO{} to \answerYes{},
%\answerNo{}, or \answerNA{}. You are strongly encouraged to include a {\bf
%justification to your answer}, either by referencing the appropriate section of
%your paper or providing a brief inline description. For example:
%\begin{itemize}
%\item Did you include the license to the code and datasets? \answerYes{See Section~\ref{gen_inst}.}
%\item Did you include the license to the code and datasets? \answerNo{The code and the data are proprietary.}
%\item Did you include the license to the code and datasets? \answerNA{}
%\end{itemize}
%Please do not modify the questions and only use the provided macros for your
%answers. Note that the Checklist section does not count towards the page
%limit. In your paper, please delete this instructions block and only keep the
%Checklist section heading above along with the questions/answers below.
%%% END INSTRUCTIONS %%%
%\input{sections/checklist.tex}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\clearpage
\newpage
\section*{Appendix}
\appendix
\input{sections/appendix.tex}
\end{document}
%Assessing causal discovery outcomes still remains to be a major hurdle in algorithmic development due to the lack of realistically complex data with known ground truth.
Algorithms for causal discovery have recently undergone rapid advances and increasingly draw on flexible nonparametric methods to process complex data. With these advances comes a need for adequate empirical validation of the causal relationships learned by different algorithms. However, for most real data sources true causal relations remain unknown. This issue is further compounded by privacy concerns surrounding the release of suitable high-quality data. To help address these challenges, we gather a complex dataset comprising measurements from an assembly line in a manufacturing context. This line consists of numerous physical processes for which we are able to provide ground truth causal relationships on the basis of a detailed study of the underlying physics. We use the assembly line data and associated ground truth information to build a system for generation of semisynthetic manufacturing data that supports benchmarking of causal discovery methods. To accomplish this, we employ distributional random forests in order to flexibly estimate and represent conditional distributions that may be combined into joint distributions that strictly adhere to a causal model over the observed variables. The estimated conditionals and tools for data generation are made available in our Python library \texttt{causalAssembly}. Using the library, we showcase how to benchmark several well-known causal discovery algorithms.
%TLDR: The paper provides details regarding \texttt{causalAssembly} a causal disvovery benchmark data tool based on complex production data with partial ground truth made semisynthetic using distributional random forests.
%Keywords: Causal Discovery, Benchmarking, semisynthetic, production data, distributional random forest
Causal discovery is the process of learning causal relations among variables of interest from data. In recent years, the area has seen numerous theoretical and algorithmic developments; for an overview see, e.g., \citep[][]{Glymour2019,Heinze18,Maathuis2019,Vowels2022, Zanga2022}. Causal structure learning is also gaining traction in industrial domains, compare e.g.~the review in \citep[][]{Vukovic2022}, where knowledge about causal relations forms the basis for downstream operations such as root-cause analysis \citep{Budhathoki2022}.
Despite the recent advances and expanded application areas, it remains challenging to provide adequate empirical validation of the causal relationships inferred by causal discovery algorithms \citep{Gentzel2019,Eigenmann2020}. Ground truth causal relationships are unknown for most real-world examples that involve complex data sources. In addition, privacy concerns often hinder the release of high-quality data. As a result, research papers proposing novel procedures tend to evaluate structure learning performance in simplistic simulation setups that tend to have limited generalizability to real-world settings \citep{Huegle2021,Reisach2021}. When applied to real data and assessed by domain experts, results are often sobering \citep{Heinze18,Constantinou2021}.
To help address these challenges, we present a complex production dataset that consists of measurements from a highly automated assembly line in a manufacturing plant. The assembly line is composed of multiple production cells or stations (we use both terms interchangeably) along which individual components are joined together. Each individual manufacturing process highly depends on the state of the preceding processes as well as on the raw components added in earlier cells. Due to the intricate and often nonlinear nature of the physical processes---including, for example, the interaction of machine state, process control settings and component state---complex causal relationships arise across the entire assembly line. In the manufacturing plant, the individual processes are computer-monitored, and the resulting process measurements are stored throughout production.
% At the same time, the dependencies between the various process are not always obvious, despite the known line structure. The dataset we provide therefore is a realistic mix of complex linear and non-linear dependencies between the variables with different types of distributions. This diverse mix of properties enables benchmarking of different causal learning algorithms.
% To help address these challenges, we introduce a complex production data set comprising real measurements from a highly automated assembly line in a manufacturing plant. The assembly line consists of multiple production cells or stations along which individual components are joined together.
%To do so, the sub-assemblies produced in each individual joining process are sequentially passed to the next production cell according to the line structure.
% Often, one production cell performs the same process several times either in succession or in parallel to reduce overall cycle time. Raw components are added to the assembly at the respective stations. The quality of the assembly process depends on the respective machine state and settings (e.g. tool state, control parameters), the environmental conditions, and on the raw state of the components or intermediates. Therefore, dependencies between the various production steps are to be expected. The success of each production step is monitored using measured features extracted from each of the processes.
From the assembly line,
we select a subset of production cells within each of which individual processes take place for which we are able to provide ground truth causal relationships. We achieve this by carefully studying the physical processes that take place in each of the cells. In addition, in order to connect the measurements available for different processes in a flexible nonparametric manner we apply sparse additive models (SpAM) \citep{Ravikumar2009}.
Next, we utilize the real assembly line data and associated ground truth information to construct a semisynthetic data generation tool implemented in the Python library \texttt{causalAssembly}. Synthetization is performed by estimating the conditional distributions indicated by the ground truth causal structure using distributional random forests (DRF) \citep{Cevid2022,Gamella2022}. By combining the conditionals, we create a joint distribution that strictly obeys a causal model over the observed variables. %The semisynthetic data obey causal Markov and sufficiency conditions.
%In this paper, we argue in favor of a semisynthetic approach to obtain data that can be used to benchmark causal discovery methods. Next to privacy issues regrading real world and complex data sources the ground truth knowledge is often not available. Therefore, we advocate for an approach that carries real data characteristics over to semisynthetic data for which ground truth is available. The data generation strategy is as follows: First, settle on a ground truth (equivalent to most strategies). Second, train conditional distributions implied by the ground truth on real data via distributional random forests (DRF) \citep{Cevid2022}. Last, sample from these DRF similar to \citet{Gamella2022}. The resulting data obey the causal Markov and causal faithfulness conditions as well as causal sufficiency \citep[see e.g.][]{Heinze18}.
\textbf{Our contributions.} We summarize our main contributions below:
\begin{itemize}
\item We create a highly complex assembly line dataset with associated ground truth based on which we propose a semisynthetic data generation pipeline that is implemented in the Python library \texttt{causalAssembly}. Using the library, one can easily generate data of varying dimension and complexity in order to benchmark causal discovery algorithms.
\item The proposed procedure to synthesize real data is general
and can in principle be applied to any given dataset with
partially known domain knowledge regardless of the domain.
\item We provide first benchmark results of popular causal discovery algorithms such as the PC algorithm \citep{Spirtes1993}, DirectLiNGAM \citep{Shimizu2011}, NOTEARS \citep{Zheng2018}, and GraN-DAG \citep{Lachapelle2020}.
\end{itemize}
In the remainder of this paper, we first address identifiability concerns and provide an overview of related work (Section~\ref{sec:prelim}). In Section~\ref{sec:datagen}, we describe the data generation pipeline.
%and related prerequisites.
Section~\ref{sec:benchmark} demonstrates benchmarking results, and Section~\ref{sec:conclusion} concludes. Supplementary Sections~\ref{sec:app_a}-\ref{sec:app_c} provide additional information regarding data, algorithms and metrics used, as well as additional benchmark results.
\subsection{General background}\label{sec:general}
% We make use of the following concepts: Let \(G=(V,E)\) be an acyclic
% directed graph (DAG) with finite node set \(V = \{X_1, \dots, X_p\}\)
% and edge set \(E \subset V \times V\). We often write \(X_i
% \rightarrow X_j\) instead of \((X_i,X_j)\) to indicate the existence
% of an edge from $X_i$ to $X_j$ in $G$. Furthermore, we call \(X_i\) a \textit{parent} of
% \(X_j\) if \(X_i \rightarrow X_j \in E\) and denote the set of all
% parents of \(X_j\) by \(pa(X_j)\). A \textit{directed path} in $G$ from $X_i$
% to $X_j$ is a sequence of nodes starting at $X_i$ and ending at $X_j$
% where subsequent nodes are connected by an edge from $E$.
% Denote \(X_i\) an \textit{ancestor} of \(X_j\) if there
% exists a directed path from \(X_i\) to \(X_j\) and a
% \textit{descendant} if there exists a directed path from \(X_j\) to \(X_i\).
% The set of all ancestor and all descendant sets of node \(X_j\) is
% denoted by \(an(X_j)\) and
% \(de(X_j)\), respectively.
For natural number
$p\in\mathbb{N}$, let $[p]:=\{1,\ldots, p\}$. Defining the vertex set as \(V = [p]\), let
\(G=(V,E)\) be a directed
acyclic graph (DAG) with edge set \(E \subset V \times V\). We always assume that \(V\) indexes a
collection of jointly distributed random variables \((X_v)_{v\in V}\). For a subset
$S\subseteq V$, we write $X_S:=(X_s)_{s\in S}$ for the collection of
variables indexed by $S$.
We often write \(v
\rightarrow w\) or $X_v\to X_w$ instead of \((v,w)\) to indicate the existence
of an edge from $v$ to $w$ in $G$. Furthermore, a node \(v\in V\) is a \textit{parent} of
another node \(w\in V\) if \(v \rightarrow w \in E\). The set of all
parents of \(w\) in \(G\) is denoted by \(pa_G(w)\). A \textit{directed path} from $v$
to $w$ in $G$ is a sequence of nodes $v_1,\dots,v_k$ starting at $v_1=v$ and ending at $v_k=w$
with subsequent nodes connected by edges $v_i\to v_{i+1}\in E$.
Call \(v\) an \textit{ancestor} of \(w\) if there
exists a directed path from \(v\) to \(w\), and call $v$ a
\textit{descendant} of $w$ if there exists a directed path from \(w\) to \(v\).
The sets of all ancestors and all descendants of node \(w\) in \(G\) are
denoted \(an_G(w)\) and \(de_G(w)\), respectively. When it is clear which graph $G$ is being considered, we simply write $pa(w)$, $de(w)$ or
$an(g)$.
A permutation \(\pi: [p] \to [p]\) is a causal ordering if \(\pi(v) < \pi(w)\) for all
$(v,w)$ with \(w \in
de_G(v)\). Such an ordering exists for every DAG,
albeit not necessarily a unique one.
Throughout the remainder of this work, we assume that \(V\) is partitioned into
\(K\) subsets \(V_1, \dots, V_K\) such that for
all \(s,t \in [K]\) with \(s<t\), the nodes in
\(V_t\) cannot be ancestors of nodes in \(V_s\). A graph
with such a partitioned vertex set is sometimes called a
\textit{layered digraph} \citep{Healy2002}. Later, the partitions
$V_1,\ldots, V_K$ correspond to the measurements from $K$ sequentially performed
manufacturing processes in a given assembly line. Often
we have to consider all nodes up to a certain process and, thus,
define $V_{1:s}:=\cup_{i=1}^{s-1}V_i$ for $s\in[K]$. Note that
$V_s\cap V_{1:s}=\emptyset$.
The DAG \(G\) indicates causal relationships among the random
variables, and
we say that \(X_v\) is a direct cause of \(X_w\) if \(v \rightarrow
w\). Furthermore, \(X_v\) may be an indirect cause of \(X_w\) if there
is a directed path from \(v\) to \(w\); we refer
to~\cite{Pearl2009,Peters2017} for more background. Clearly, in a
layered digraph, no variable with index in \(V_t\) can be a direct or
indirect cause of any node in $V_s$ with $s< t$.
Each DAG \(G\) on $V$ gives rise to a statistical model for the joint distribution \(P_X\) of the random vector \(X=(X_v)_{v \in V}\). Here, the edges in
\(G\) encode the permitted conditional dependence relations among the
variables \citep[see e.g.][]{Drton2017,Lauritzen1996}. Any distribution
that satisfies the conditional independence constraints implied by
\(G\) is said to be \textit{Markov} with respect to \(G\). In
other words, the joint density of \(X\) factors into a product of
conditional densities as
\(
p(x) =
\prod_{v \in V} p(x_v \mid x_{pa_G(v)})
\)
with \(p(x_v\mid
x_\emptyset) = p(x_v)\).
The conditional independence relations holding in all such factorizing distributions can be obtained using the graphical criterion of \textit{d-separation}
\citep{Pearl2009}. A distribution \(P_X\) is \textit{Markov} and
\textit{faithful} with respect to the underlying DAG \(G\) if the conditional
independence relations in \(P_X\) correspond exactly to
d-separation relations in $G$.
% Consequently, if \(P_X\) satisfies the Markov condition
% and is faithful with respect to the underlying DAG \(G\) we have a
% one-to-one correspondence between graphical (d-separation) and
% probabilistic conditional independence statements.
Different DAGs may share the same set of
conditional independence statements. Such DAGs are
said to be Markov equivalent. All equivalent DAGs form a Markov
equivalence class (MEC) \citep[see e.g.][]{Spirtes1993}.
\subsection{Identifiability of the causal structure}\label{sec:identifiability}
In general, the DAG \(G\) is not identifiable from \(P_X\) alone. Learning the causal structure or its corresponding MEC from data
requires assumptions. Under the Markov and
faithfulness conditions regarding \(P_X\), \(G\) can be
recovered up to its MEC.
%\citep{Chickering1995}.
Prominent routines
that operate under this assumption are, for example, the PC
\citep{Spirtes1993} and the Greedy Equivalence Search (GES)
\citep{Chickering2003} algorithm.
Under additional assumptions, we may not only recover the MEC of \(G\)
but \(G\) itself. Note, that the graphical model that is associated
with \(G\) can also be expressed in terms of a structural equation
model (SEM) \citep[][]{Bollen1989}. That is, if \(P_X\) is Markov with
respect to \(G\) then there exist independent noise variables
\(\epsilon = (\epsilon_1,\ldots, \epsilon_p)\) and measurable function
\(g_{i}\) such that \(X_i = g_{i}(X_{pa(i)}), \epsilon_{i})\) for
all \(i \in [p]\).
In this framework, if all the \(g_{i}\) are linear and all \(\epsilon_{i}\) follow
non-Gaussian distributions---termed Linear Non-Gaussian Acyclic Model
(LiNGAM)---then the causal graph is identifiable from data
\citep{Shimizu2006}. A comprehensive treatment of this model and various extensions is given in \cite{Shimizu2022}.
%provides an overview of numerous extensions of this model.
%to
%cover hidden variables, time series data, nonlinear models, and
%discrete variables (see~\cite{Shimizu2022} for an overview).
In a different vein, identifiability of $G$ is also achieved under an additive noise model
(ANM) \cite{Hoyer2008}. In other words, as long as all
\(g_{i}\) are non-linear as defined in~\cite{Hoyer2008}, the noise \(\epsilon_{i}\) may follow
any distribution given that it is additive. A generalization to
the post-nonlinear model is proposed by \cite{Zhang2009}.
Linear SEMs with Gaussian noise terms are not
identifiable. However,
identifiability can be achieved in this setting under assumptions such as equal noise variances \cite{Peters2014b}, where a causal order is found by sorting (conditional) variances \cite{Chen2019}.
\subsection{Additive noise models for synthetic data generation}
Comprehensive assessment of methods for learning causal relations is challenging
due to the scarcity of real data where true causal relations are
known. Hence, the focus is then often on simulated synthetic data. In order to accommodate non-linearities and also to ensure identifiability of the true DAG, ANMs are often
employed for this purpose.
% Good performance of an algorithm depends on the accuracy of the
% assumptions made about the data-generating process as well as on
% implementation details and choices. To that end, ANMs are often
% employed to ensure identifiability of the true DAG.
However, synthetic
data generation can leave unintended traces hinting at the true
causal relations among the variables. In particular, the work of
\cite{Reisach2021} demonstrates that for commonly used simulation setups involving ANMs,
irrespective of the choice of the functions \(g_{i}\), ordering nodes by
increasing marginal variance tends to agree with the causal order of the
ANM. The authors introduce \textit{varsortability}, a
score that measures the degree of concordance between causal orders of
the ground truth DAG and the order of increasing marginal variance
\cite[Section 3.1]{Reisach2021}. When varsortability equals one
then all directed paths lead from lower variance nodes to higher
variance nodes.
%and the causal structure is identifiable.
This is connected to the aforementioned identifiability results for linear SEMs with equal variance \cite{Chen2019,Peters2014b}.
%and~\cite{Park2020}.
Many of the recently proposed continuous optimization based structure
learning algorithms report a similar data generative setup based on ANMs, see among others \cite{Zheng2018, Ng2020, Lachapelle2020, Yu2021, Zhang2022, Zhang2023}. The resulting high varsortability on the raw data scale contributes to often staggering performances of these methods as they often inadvertently make use of variance order. In real systems, on the
other hand, measurement scales are essentially irrelevant for the
causal order. Therefore, standardization would be a simple solution to
render data scale non-informative. Yet, remnants of high
varsortability with distinctive covariance patterns
may remain even after standardization~\cite{Reisach2021,Reisach2023}.
Ultimately, all synthetic data generation processes are
vulnerable to a lack of portraying common themes observable from
practice.
Evaluating the performance of algorithms
is a key factor that impacts which research directions the field is
gravitating towards. This can be seen, for instance, in the case of
continuous optimization-based structure learning. Following the
development of NOTEARS \citep{Zheng2018}, numerous extensions have
been proposed in short succession (see \cite{Vowels2022} for an
overview). However, when evaluated outside of ANMs, performance often drops notably \cite{Constantinou2021}.
Therefore, there is a clear necessity for more robust evaluation
techniques in the field of causal discovery as outlined
in~\cite{Eigenmann2020,Gentzel2019}.
This motivates the present work which seeks to support benchmarking of causal discovery by providing tools for semisynthetic data generation that draw on partial ground truth and real data from a manufacturing plant.
\subsection{Related work}
Focusing on observational data, the available tools for empirical illustration and benchmarking of causal discovery algorithm can be broadly classified into three tiers:
% there exist numerous datasets and benchmarking tools that allow to evaluate causal discovery algorithms. They broadly group into three tiers:
(1) Real datasets with expert-knowledge driven
ground truth, such as the cause-effect pairs of \cite{Mooij2016} and data pertaining to model organisms/already well understood systems in biology~\cite{Marbach2012, Replogle2022,Sachs2005}. (2)
Synthetic datasets generated based on well-established
parameterizations
\citep[][]{VandenBulcke2006, Schaffter2021, Scutari2010}. (3) Data
generated using functional specifications (see \cite{Chen2023,Huegle2021}
for recent benchmarking tools). In recent years, causal discovery benchmark challenges have also been hosted in cooperation with special conference tracks, e.g., \emph{GSK.ai CausalBench challenge} at ICLR 2023 or
\emph{Causality 4 Climate} at NeurIPS 2019.
% The vast majority of synthetic data generation is carried out
% without relying on datasets provided by dedicated causal benchmarking
% tools. Researchers typically generate data that fit the
% domain-specific requirements of the proposed causal discovery
% algorithm. This is particularly true for continuous optimization based
% structure learning algorithms. It has become common to generate data
% according to ANMs following a selection of different graph types,
% graph size and noise distributions, see for instance \cite{Zheng2018, Ng2020, Lachapelle2020, Yu2021, Zhang2022,
% Zhang2023}.
% In light of this, \cite{Reisach2021} demonstrate that while
% combinatorial-based causal discovery routines are usually scale
% agnostic many of the prominent continuous optimization based structure
% learning algorithms inadvertently make use of the high varsortability in ANMs. This results in often staggering performance which, as the authors show, dramatically drops as soon as the data is standardized.
% Therefore, there is a clear necessity for more robust evaluation
% techniques in the field of causal discovery as outlined
% in~\cite{Gentzel2019, Eigenmann2020}. Evaluating algorithmic quality
% is a key factor with respect to what research directions the field is
% gravitating towards. This can be seen very clearly in the case of
% continuous optimization based structure learning. Following the
% development of NOTEARS \citep{Zheng2018}, numerous extensions have
% been proposed in short succession (see \cite{Vowels2022} for an
% overview).
\subsection{Modelling expert knowledge}\label{sec:expertknowledge}
\paragraph{Establishing partial ground truth} In this
work, the real-world data come from an assembly line at a
manufacturing plant of Robert Bosch
GmbH. We focus on a selected subset of production stations at which press-in and staking processes are executed (see
Figure~\ref{fig:pline_schematic}). In the following, we give a brief overview of these
processes. A more in depth discussion of the relations among the central
features of the physical procedures is given in supplementary Section~\ref{sec:process_knowledge}.
\textit{Press-in: \quad} In each fitting process, two components are
joined in a pressing machine to form a mechanically strong and
sometimes even leakage tight connection between the components. An
inner cylindrical component (e.g., a valve, axle or pin) is pushed into
a slightly smaller bore of a larger component. This is achieved by
applying a mechanical force high enough to overcome the required
friction force. After the friction force is exceeded, the inner
component will move into the bore. The pressing machine stops as soon
as a certain predefined press-in force is exceeded for the first time.
\textit{Staking: \quad} A staking process is an extension of a press-in process. The objective of staking is to fixate the inner component in its final position by permanently deforming some outer component materials. This is achieved by applying a large enough force to the tool to create a mechanical stress-strain state in the material resulting in permanent plastic material deformation.
\begin{figure} %[ht!]
\centering
\begin{subfigure}[t]{0.45\textwidth}
\centering
\includegraphics[width=.9\linewidth]{figures/F_pressin_sketch.pdf}
\caption{Press-in}
\label{fig:pressin}
\end{subfigure}
\begin{subfigure}[t]{.45\textwidth}
\centering
\includegraphics[width=.9\linewidth]{figures/F_staking_sketch.pdf}
\caption{Staking}
\label{fig:staking}
\end{subfigure}
\caption{Illustration of the various phases of a press-in and staking process.}
\label{fig:pressin_and_staking}
\end{figure}
Figure~\ref{fig:pressin_and_staking} depicts these two processes in a
schematic manner. In order to control and monitor these processes in
real-time, important signals are measured and analyzed by a process control unit. We extract all relevant numeric measurements for each process, and together with the mapping detailed in Section~\ref{sec:process_knowledge} they build the foundation for our data generation scheme.
\subsection{Data generation in \texttt{causalAssembly}}
Suppose the random variables \((X_v)_{v\in V}\) represent the features extracted
from the manufacturing processes described above. Recall the layering introduced in Section \ref{sec:general}, where we assume that $V=V_1\dot\cup\cdots\dot\cup V_K$. We extract data drawn from $K=10$ processes carried out sequentially. These processes are lined up across
five process cells such that each cell accommodates two of the
production processes outlined in~Figure~\ref{fig:pline_schematic}. Overall we collect \(\lvert V \rvert = 98\) features for \(n = 15,581\) parts. The features distribute across production cells as follows: Station1 with \(\lvert V_1 \cup V_2 \rvert = 6\), Station2 with \(\lvert V_3 \cup V_4 \rvert = 34\), Station3 with\(\lvert V_5 \cup V_6 \rvert = 16\), Station4 with \(\lvert V_7 \cup V_8 \rvert = 26\), and Station5 with \(\lvert V_9 \cup V_{10} \rvert = 16\).
% Figure \todo{either leave this out or provide a small little schematic
% picture at this point} visualizes the assembly line setup with its
% five process cells.
\begin{figure} %[ht!]
\centering
%\includegraphics[width =\textwidth]{figures/line_schematic.pdf}
\begin{tikzpicture}[scale=0.7]
\foreach \i/\l in {1, 2, 3, 4, 5}{
\draw[rounded corners, fill=gray!15] (4*\i-4-0.15, -0.25) rectangle (4*\i-.5+0.15, 2.50) {};
\node[scale=1.2] at (4*\i-4+1.75, 2.2) {Station\i};
}
\foreach \i in {1, 2, 3, ..., 10}{
\draw[rounded corners, fill=gray!55] (2*\i-2-0.05, 0) rectangle (2*\i-0.5+0.05, 1.5) {};
\node[scale=1.0, align=center] at (2*\i-2+0.75, 1.15) {Process\i};
\node[scale=1.4, align=center] at (2*\i-2+0.75, 0.45) {$V_{\i}$};
};
\draw[->, line width=1.5pt] ( 6, -0.7) to node[midway, fill=white] {component flow} (14, -0.7);
\end{tikzpicture}
\caption{Illustration of the production line with five process
stations each containing two successive processes. The first cell
prepares components while cell two and four carry out staking tasks. The remaining cells perform press-in tasks.}
\label{fig:pline_schematic}
\end{figure}
Each individual process we chose to focus on is well understood, and we are able to state causal relations among the variables in $V_s$. In other words, we map the process knowledge into graphs
$G_s^*=(V_s, E_s)$ for $s\in[K]$.
Let $G^*:=G_{1}^{*} \oplus \dots \oplus G_{K}^*$ be the graph union of
all process graphs.
Let $P^*_k:=pa_{G^*}(k)$ be the set of parents for variable $X_k$ in $G^*$. Note that if $k\in V_s$, then $P^*_k\subset V_s$ as well.
%all
%parents of $X_k$ correspond to nodes in $G^*_s$.
For some measurements $X_k$ of the $s$-th process, the process experts are able to state functional relations in the form $X_k\approx f^*_k(X_{P^*_k})$.
However, due to their complex physical nature, our process knowledge
does not extend to how the $K$ processes may influence each other. Thus, we apply a learning step in order to find an overall causal graph $G=(V, E)$ on $V$, such that the induced
subgraphs coincide with the expert knowledge locally, i.e.,
$G[V_s]=G^*_s$ for all $s\in[K]$. In this learning step we account for the known within process graph and functional
structures when learning edges between the partition blocks
$V_1,\ldots, V_K$. In particular, we will not add further edges between nodes
of the same process and assume that the process knowledge is ground
truth. The cross-process connections will be learned from data as outlined below.
In our application, all processes are executed sequentially and process $t$ can be influenced by all processes
executed earlier. There are other assembly lines where this is not the case and
where domain knowledge also contains information about inter-process
influences. Such additional information could be
easily inserted in our framework by imposing the restriction that the graph $\mathcal{G}$
has no edges between $V_j$ and $V_t$ for the concerned indices $j<t$.
% on the nodes
% $[K]$ where the absence of an edge between $j$ and
% $t$ with $j<t$ tells that there there are no edges between $V_j$ and $V_t$
% in $G$. In the assembly line we studied, $\mathcal{G}$ contains all
% possible edges $(j, t)$ with $j<t$.
In order to infer edges across processes one could feed
in $G_{1}^{*} \oplus \dots \oplus G_{K}^*$
as prior knowledge into causal discovery tools. However,
all causal discovery algorithms impose more or less stringent
assumptions regarding the data generative process. Consequently, these assumptions may carry over to our proposed semisynthetic generator, i.e., they would become features of data produced by the generator. We demonstrate this behavior
in supplementary Section~\ref{sec:initial_influence}. In an attempt to minimize this effect, we make use of sparse additive models (SpAM) \citep{Ravikumar2009} to learn cross-process conditional distributions, taking care to account for the (graphical) within process dependence structure as noted below. These models are flexible and non-parametric
such that we minimize possible traces of the data generative procedure
that might carry over to the final sampling process. We set them up such
that we can easily obey the layered structure within and across cells
and can build upon existing functional process knowledge.
% Another important factor is to take care of the layered structure
% across tasks and cells. Often prior knowledge comes in the form of
% being aware of the existence of a relationship between to measurements
% but not necessarily their orientation. In our case it is the other way
% around.
\paragraph{Respecting prior knowledge in cross-process learning} Let $X_k$ be a variable from the $t$th process, i.e., $k\in V_t$. We want to establish which edges $j\to k$ for $j\in V_{1:t}$ ought to be included in the overall causal graph $G$. To this end, we consider the regression
function for the residual to the functional process
knowledge provided by domain experts:
\begin{equation*}
f_k(x_{V_{1:t}}) = \mathbb{E}(X_k -f^*_k(X_{P^*_k})\mid
X_{V_{1:t}}=x_{V_{1:t}}).
\end{equation*}
That way, we learn the gap between process knowledge and observation
using information from prior processes. If no
functional process knowledge $f^*_k$ is available for $X_k$, we
set $f^*_k=0$ and regress onto all variables with index in
\(V_{1:t}\cup P^*_k\). Sparse additive models use sums to approximate the regression
function of the form \(f_k(x_{V_{1:t}}) \approx \sum_{l\in
V_{1:t}}f_{k,l}(x_l)\) and induce sparsity via the penalty \(\sum_{l \in V_{1:t}}
\lVert f_{k,l} \rVert_2\). We refer to~\cite[Section 2]{Ravikumar2009}
for more details on sparse additive models.
Algorithm~\ref{alg:spam_search} outlines our
procedure to learn edges across processes.
% Furthermore, assume that a subset $\mathcal{S}_t\subset V_{1:t}$
% exists that admits a decomposition in which the variables of
% % $\mathcal{S}_t$ do not interact with each other:
% $$x_k-f^*_k(x_{P^*_{k}})\approx
% f_k(x_{V_{1:t}})=\sum_{l\in \mathcal{S}_t}f_{k,l}(x_l) $$
\begin{algorithm}
\caption{SpAM cross-process learning}\label{alg:spam_search}
\KwIn{Data \(\mathcal{D}\), \ DAGs \(G^{*}_{1},\ldots, G^{*}_{K}\),
\ functions $f^*_1,\ldots, f^*_p$, \ regularization parameter \(\lambda \geq 0\)}
\KwOut{Edge set \(E_{\text{across}}\subset V\times V\).}
\(E_{\text{across}} \gets \{\}\);\\
\For{\(t = 2, \dots, K\)}{
Compute a causal order $\pi_t$ of $G^*_t$;\\
\For{\(k\) along $\pi_t$}{
\(S_t \gets V_{1:t}\) \\
\If { $f^*_k = 0$}{
$S_t \gets S_t\cup P_k^*$
}
Compute $(\hat f_{k,l})_{l\in S_t}$ with $\hat
f_{k,l}\in\mathcal{F}_{k,l}$ that minimize:
\begin{equation*}
\min_{f_{k,l} \in \mathcal{F}_{kl}\text{ for }l\in
S_t}\frac{1}{|\mathcal{D}|}\sum_{x\in\mathcal{D}}
\left(x_k-f^*_k(x_{P_k^*})-\sum_{l\in
S_t}f_{k,l}(x_l)\right)^2+ \lambda\sum_{l \in S_t} \lVert f_{k,l} \rVert_2
\end{equation*}
\(E_{\text{across}} \gets E_{\text{across}}\cup\{(l,
k): l\in S_t, \hat{f}_{k,l} \neq 0\}\)
}
}
\end{algorithm}
Note that depending on the function classes $\mathcal{F}_{k,l}$, the
minimization problem in Algorithm~\ref{alg:spam_search} becomes a
convex program. Indeed, in practice, we express each \(X_k\) as a projection on to a set of cubic spline basis functions to ensure convexity. The tuning parameter \(\lambda \geq 0\) is selected via cross-validation.
Algorithm~\ref{alg:spam_search} admits a natural way to also
incorporate higher-level process knowledge. One such example would be that influences between measurements of process $j$ and $t$ with $j<t$ can be excluded. If such domain information is encoded as a graph $\mathcal{G}$ on $[K]$ as described above, we set \(S_t=\cup_{j\in pa_\mathcal{G}(t)}V_j\) instead of $S_t=V_{1:t}$.
Applying Algorithm \ref{alg:spam_search} to the given process knowledge graphs, we obtain the final ground truth DAG
\(G^{*}_{\text{partial}}:=(V, E\cup
E_{\text{across}})\). Figure \ref{fig:assemblyline_overview} illustrates \(G^{*}_{\text{partial}}\).
\begin{figure}[!ht]
\includegraphics[clip, trim=0cm 2.8cm 0cm 3cm,width = \linewidth]{figures/ground_truth_spam.pdf}
\centering
\caption{Assembly line ground truth after edges have been learned
between processes using the SpAM procedure. Note that only the
stations are shown, not the processes they are decomposed in. Node size increases with the number of out-edges. Node color gets brighter with the number of in-edges.}
\label{fig:assemblyline_overview}
\end{figure}
%In the data version of the SpAM search, each predictor \(X_k, k \in \tilde{S}\) is expressed as a projection on to a set of basis functions \(\{\psi_{kl}\}_{l=1}^{d_k}\) with knots chosen along the range of \(X_k\). Let \(\beta_{k}\) be the \(d_k\)-vector \(\{\beta_{kl}, l = 1, \dots, d_k \}\) and \(\text{supp}(\beta_S) = \{ k \in S: \beta_k \neq 0 \}\), where \(0\) is the \(d_k\)-dimensional zero vector.
%We use SpAMs in order to find the best sparse approximation of the regression function \(f(x_S) = \mathbb{E}(X_j \mid X_{S})\), where \(S \subset \{1, \dots, p\}\) is an index set and \(j \in \{1,\dots,p\} \setminus S\). More precisely we let \(f(x_S) \approx \sum_{k\in S} f_k(x_k)\) with \(f_k \in \mathcal{F}_k\) a set of univariate function classes. We consider the following penalized convex program for a given regularization parameter \(\lambda \geq 0\) as proposed by \citet{Ravikumar2009}:
%\begin{equation*}
% \min_{f_k \in \mathcal{F}_k, k \in S} \left\{ \mathbb{E}( X_j - \sum_{k\in S} f_k(X_k))^2 + \lambda\sum_{k\in S} \lVert f_k \rVert_2 \right\}, \quad j \in \{1,\dots,p\} \setminus S.
%\end{equation*}
%We further express each \(X_k, k \in S\) as a projection on to a set of basis functions \(\{\psi_{kl}\}_{l=1}^{d_k}\) with knots chosen along the range of \(X_k\). Consequently, we have for each \(k \in S\) that \(f_k(\cdot) = \sum_{l=1}^{d_k} \psi_{kl}(\cdot)\beta_{kl}\). Let \(\beta_{k}\) be the \(d_k\)-vector \(\{\beta_{kl}, l = 1, \dots, d_k \}\) and \(\text{supp}(\beta_S) = \{ k \in S: \beta_k \neq 0 \}\), where \(0\) is the \(d_k\)-dimensional zero vector. We are interested in the non-zero estimated components \(\text{supp}(\hat\beta)\) from which we will draw an edge to the response \(X_j\).
\textit{Example: } Consider an assembly line with two processes
and decomposition $V=V_1\cup V_2$, where we want to learn
connections between $V_1$ and $V_2$, each of which comprises three measured variables (see Figure~\ref{fig:spam}).
Ignoring already known parental relations for $X_6$ in the second process will
lead to inferring incorrect edges when running the regression with
predictors from the first process only, i.e., with predictor set
\(S^{\text{naive}}_6=\{1, 2, 3\}\); see Figure~\ref{fig:spam}(b). This is true unless both processes are in fact unconnected or the second process graph is empty. Including the parent set of $X_6$ in $G^*_2$ in the predictor set $S_6=\{1, 2, 3, 5\}$ leads to correct inference of the edges across processes; see~Figure~\ref{fig:spam}(c).
\begin{figure}
\begin{tikzpicture}
% a
\node[scale=1.5] at (0, 0) {
\begin{tikzpicture}
\draw[rounded corners, fill=gray!15] (0.35, -1.2) rectangle (1.65, 1.5) {};
\node[] at (1, 1.2) {$V_1$};
\tikzset{VertexStyle/.append style = {fill = blue!60}}
\Vertex[x=1, y=0.8, Lpos=180,Math, L=X_1]{1}
\Vertex[x=1, y=0, Lpos=180,Math, L=X_2]{2}
\Vertex[x=1, y=-0.8, Lpos=180,Math, L=X_3]{3}
\draw[rounded corners, fill=gray!15] (2.35, -1.2) rectangle (3.65, 1.5) {};
\node[] at (3, 1.2) {$V_2$};
\tikzset{VertexStyle/.append style = {fill = blue!60}}
\Vertex[x=3, y=0.8, Lpos=180,Math, L=X_4]{4}
\Vertex[x=3, y=0, Lpos=180,Math, L=X_5]{5}
\Vertex[x=3, y=-0.8, Lpos=180,Math, L=X_6]{6}
\tikzset{EdgeStyle/.append style={->}}
\Edge(1)(2)
\Edge(2)(3)
\Edge(5)(6)
\tikzset{EdgeStyle/.style={->, bend left, in=210, out=-40}}
\Edge(2)(4)
\Edge(3)(5)
% \tikzset{EdgeStyle/.append style={->, bend left, in=100, color=black}}
% \Edge(4)(6)
\node[align=center] at (2, -1.5) {(a)};
\end{tikzpicture}
};
% b
\node[scale=1.5] at (5, 0) {
\begin{tikzpicture}
\draw[rounded corners, fill=gray!15] (0.35, -1.2) rectangle (1.65, 1.5) {};
\node[] at (1, 1.2) {$V_1$};
\tikzset{VertexStyle/.append style = {fill = blue!60}}
\Vertex[x=1, y=0.8, Lpos=180,Math, L=X_1]{1}
\Vertex[x=1, y=0, Lpos=180,Math, L=X_2]{2}
\Vertex[x=1, y=-0.8, Lpos=180,Math, L=X_3]{3}
\draw[rounded corners, fill=gray!15] (2.35, -1.2) rectangle (3.65, 1.5) {};
\node[] at (3, 1.2) {$V_2$};
\tikzset{VertexStyle/.append style = {fill = blue!60}}
\Vertex[x=3, y=0.8, Lpos=180,Math, L=X_4]{4}
\Vertex[x=3, y=0, Lpos=180,Math, L=X_5]{5}
\Vertex[x=3, y=-0.8, Lpos=180,Math, L=X_6]{6}
\tikzset{EdgeStyle/.append style={->}}
\Edge(1)(2)
\Edge(2)(3)
\Edge(5)(6)
% \tikzset{EdgeStyle/.append style={->, bend left, out=20, in=100, color=black}}
% \Edge(4)(6)
\tikzset{EdgeStyle/.style={->, bend left, in=210, out=-40, color=green!45!black}}
\Edge(2)(4)
\Edge(3)(5)
% \tikzset{EdgeStyle/.style={->, bend right, in=250, out=-40, color=red!45!black}}
% \Edge(2)(6)
\tikzset{EdgeStyle/.style={->, bend right, in=230, out=-40, color=red!45!black}}
\Edge(3)(6)
\node[align=center] at (2, -1.5) {(b)};
\end{tikzpicture}
};
% c
\node[scale=1.5] at (10, 0) {
\begin{tikzpicture}
\draw[rounded corners, fill=gray!15] (0.35, -1.2) rectangle (1.65, 1.5) {};
\node[] at (1, 1.2) {$V_1$};
\tikzset{VertexStyle/.append style = {fill = blue!60}}
\Vertex[x=1, y=0.8, Lpos=180,Math, L=X_1]{1}
\Vertex[x=1, y=0, Lpos=180,Math, L=X_2]{2}
\Vertex[x=1, y=-0.8, Lpos=180,Math, L=X_3]{3}
\draw[rounded corners, fill=gray!15] (2.35, -1.2) rectangle (3.65, 1.5) {};
\node[] at (3, 1.2) {$V_2$};
\tikzset{VertexStyle/.append style = {fill = blue!60}}
\Vertex[x=3, y=0.8, Lpos=180,Math, L=X_4]{4}
\Vertex[x=3, y=0, Lpos=180,Math, L=X_5]{5}
\Vertex[x=3, y=-0.8, Lpos=180,Math, L=X_6]{6}
\tikzset{EdgeStyle/.append style={->}}
\Edge(1)(2)
\Edge(2)(3)
\Edge(5)(6)
% \tikzset{EdgeStyle/.append style={->, bend left, in=100, color=black}}
% \Edge(4)(6)
\tikzset{EdgeStyle/.style={->, bend left, in=210, out=-40, color=green!45!black}}
\Edge(2)(4)
\Edge(3)(5)
\node[align=center] at (2, -1.5) {(c)};
\end{tikzpicture}
};
\end{tikzpicture}
\vspace{-0.3cm}
\caption{SpAM search illustration. Figure (a) represents the
ground truth. Two scenarios are considered: In (b) naive search is performed, without taking into account the causal order of the DAG in the second station. In (c) the parental relations in Station 2 are used, resulting in the correct DAG.}
\label{fig:spam}
\end{figure}
\paragraph{Synthetization} In order to
ensure that the data used in benchmarking really come from a distribution that
factorizes according to the ground truth DAG \( G^{*}_{\text{partial}} \), we
explicitly estimate the conditional distributions under the factorization implied by the independence constraints in \( G^{*}_{\text{partial}} \). To that end, we
employ distributional random forests (DRFs) \citep{Cevid2022} to obtain
estimates \(\hat{P}^{\text{DRF}}_X(X_k\mid X_{pa_{G^{*}_{\text{partial}}}(k)})\), \(k \in V\),
along a causal order of \(G^{*}_{\text{partial}} \). This has been
first proposed by \cite{Gamella2022} in the context of the data from
\citep{Sachs2005}. Section \ref{sec:drf_background} details how DRFs are built, what splitting criterion is used, and
how to estimate conditional distributions. Given
\(\hat{P}^{\text{DRF}}_X(X_k\mid X_{pa_{G^{*}_{\text{partial}}}(k)})\)
we sample a new dataset
$\mathcal{D}_{\text{synth}}\subset\mathbb{R}^p$ from the production line as outlined in Algorithm \ref{alg:drf_sampling}.
By construction, \(\mathcal{D}_{\text{synth}}\) is drawn
from the estimated joint distribution
\(\hat{P}^{\text{DRF}}_X\) and
is guaranteed to be Markov with respect to the DAG
\(G^{*}_{\text{partial}}\).
In order to provide an option to consider data of different dimensionality, we also offer the option to focus on each one of the five process cells that constitute the natural subunits of the assembly line. Simply considering the marginal distribution of the variables in one such cell generally induces confounding (through effects of earlier cells). Hence, to allow sampling from a distribution Markov to the ground truth within-cell DAG $G^*_s$ for cell $s$, we refit the DRFs and apply
Algorithm \ref{alg:drf_sampling} to each cell individually to generate data the desired within-cell dependence structure.
% To allow sampling from chosen processes, recall that they are organized within five process cells. These cells are natural subunits of the assembly line. In order to be able to draw isolated samples from these cells we refit the DRFs and apply
% Algorithm \ref{alg:drf_sampling} to each cell individually. If we took
% \(\hat{P}^{\text{DRF}}_X\) to sample from the cells we likely
% introduce confounding in all but the first cell.
\begin{algorithm}\label{alg:sampling}
\caption{Sampling from the causal DRFs}\label{alg:drf_sampling}
\KwIn{Finite dataset \(\mathcal{D}_{\text{original}}\subset
\mathbb{R}^{p}\), DAG \(G\) with \(p\) nodes, number of samples \(n\) to be generated}
\KwOut{A dataset \(\mathcal{D}_{\text{synth}} \subset
\mathbb{R}^{p}\) of size $n$}
$\mathcal{D}_{synth}\gets\emptyset$;\\
Compute a causal order $\pi$ of $G$;\\
\While{$|\mathcal{D}_{\text{synth}}|<n$}{
\For{\(k\) along $\pi$}{
\eIf{\(pa_{G}(k) = \emptyset\)}{
sample $\hat x_k\in\mathbb{R}$ via smooth bootstrapping from
\(\{x_k\in\mathbb{R}:
x\in \mathcal{D}_{\text{original}}\}\);
}{
sample $\hat x_k\in\mathbb{R}$ from $
\hat{P}^{\text{DRF}}_X(X_k\mid
X_{pa_{G}(k)} = \hat{x}_{pa_{G}(k)}\}$;
}
$\mathcal{D}_{synth}\gets\mathcal{D}_{synth}\cup\{\hat x\}$;
}
}
\end{algorithm}
%Figure \ref{fig:drf_sampling} provides a visualization of the data generation pipeline in \texttt{causalAssembly}.
%\begin{figure}[ht]
% \includegraphics[width = .85\linewidth]{figures/drf_sampling.png}
% \centering
% \caption{Data generation pipeline (interim) for \texttt{causalAssembly}.}
% \label{fig:drf_sampling}
%\end{figure}
\subsection{Semisynthetic data description}
% \begin{wrapfigure}{r}{0.45\textwidth}
% \vspace{-3em}
% \includegraphics[width=0.45\textwidth]{figures/marginal_compare.pdf}
% \caption{Kernel density plots of the three best (a) and three worst variables (b) in terms of Kolmogorov-Smirnov statistic.}
% \label{fig:marginal}
% \end{wrapfigure}
\begin{figure}
\centering
\includegraphics[width=\textwidth]{figures/marginal_compare.pdf} \caption{Kernel density plots comparing the highest (first three) and lowest (last three) agreement in terms of Kolmogorov-Smirnov statistic.}
\label{fig:marginal}
\end{figure}
To demonstrate that the semisynthetic ``Markovian'' data obtained from our data generation pipeline preserves key characteristics of
the real data, we calculated the two-sample
Kolmogorov-Smirnov statistic \citep{Kolmogorov1933} between all
variables in the real and a semisynthetic dataset. Figure~\ref{fig:marginal} shows kernel density estimates of the variables
with largest and smallest Kolmogorov-Smirnov statistics among all
non-source nodes.
Even in unfavorable cases
where the absolute difference between the marginal distributions is
largest, the semisynthetic data only shows moderate deviations from the real data.
% \begin{figure}[ht]
% \includegraphics[clip, trim=0cm 1.0cm 0cm 1.5cm,scale=0.6]{figures/marginal_compare.pdf}
% \centering
% \caption{Kernel density plots of the three best (a) and three worst variables (b) in terms of Kolmogorov-Smirnov statistic.}
% \label{fig:marginal}
% \end{figure}
As a proof of concept, we present first causal discovery benchmark results based on data obtained from the semisynthetic data generative procedure in the \texttt{causalAssembly} library. In order to make sure that privacy concerns are respected, we proceed as follows: First, we apply the procedure to the real data from the manufacturing plant. Based on the fitted DRF we sample new data that matches the dimension of the original real data. Based on this first-level semisynthetic dataset, we conduct the benchmark study below, which establishes results for the full assembly line using the ground truth depicted in Figure \ref{fig:assemblyline_overview}. Additionally, we present further benchmark results based on the individual process cells in Section \ref{sec:addition_benchmarks}.
The purpose of the study we report on here is simply to demonstrate the versatility of the data generation scheme. A more elaborate set of experiments comprising more causal discovery routines, varying sample size and adjusting hyperparameters will be taken up in future work.
% Restrict to one sample size setting, limited number of runs etc. describe how the drf object for the full line is large and requires lots of memory... for convenience we provide a subsampled range of drf objects for code testing purposes before launching a big run!
\begin{figure} %[ht]
\includegraphics[width = \linewidth]{figures/full_benchmark.pdf}
\centering
\caption{Chosen metrics for assessing causal discovery of the full assembly line (\(\lvert V \rvert = 98\)) using \(100\) simulation runs on a sample size of \(n = 500\)}
\label{fig:full_benchmark}
\end{figure}
Figure \ref{fig:full_benchmark} reports evaluation
results based on causal discovery algorithms and metrics that we described explicitly in Section
\ref{sec:alg_metrics}. In this evaluation, the data is always standardized before structure learning,
and we perform \(100\) simulation runs. The sortnregress (snr) routine
serves as a reference point, as for standardized data it corresponds to
choosing some order at random and regressing (\textit{random regress})
each node on its ancestors (see also~\cite{Reisach2021}). Considering first the structural Hamming distance (SHD) to the ground truth, the studied default methods fail to beat outputting an empty graph. The fact that NOTEARS does better than the other methods here can be attributed to outputting very sparse graphs. Linear methods such as DirectLiNGAM and PC algorithm with standard partial correlation tests are on par with random regress. For GraN-DAG some higher variability in results is observed. For Precision, Recall, and F1 score, the PC algorithm fares surprisingly well. GraN-DAG shows better Recall and F1 score, but a highly variable Precision.
% anon We find that NOTEARS outperforms the other routines in terms of SHD and Precision while Recall suggests that this is driven by outputting very sparse graphs. DirectLiNGAM is on a par with random regress and GraN-DAG produces highly unstable results.
We emphasize that these results are based on off-the-shelf implementations with no attempts to tune the methods to the nature of the data. Moreover, the considered methods also all ignore the prior knowledge provided by the layering across processes.
% The layering across processes can already be regarded as prior knowledge and should be incorporated. As not all implementations of the algorithms considered allow an off-the-shelf inclusion of prior knowledge we refrain from making use of layering.
Although very limited, this first prototype of a benchmark study reveals that the data one may create with \texttt{causalAssembly} are challenging for standard off-the-shelf methods.
%Overall, benchmark results suggest that none of the algorithms is able
%to perform substantially better than random regress.
This largely
confirms results conducted on empirical data in the sense that
learning accuracy presented in the literature on the basis of
synthetic data overestimates performance on more complex datasets \cite{Constantinou2021,Rios2022}.
% Authors are required to include a statement of the broader impact of their work, including its ethical aspects and future societal consequences.
% Authors should discuss both positive and negative outcomes, if any. For instance, authors should discuss a)
% who may benefit from this research, b) who may be put at disadvantage from this research, c) what are the consequences of failure of the system, and d) whether the task/method leverages
% biases in the data. If authors believe this is not applicable to them, authors can simply state this.
% Use unnumbered first level headings for this section, which should go at the end of the paper. {\bf Note that this section does not count towards the eight pages of content that are allowed.}
Empirical validation is key in understanding how causal discovery algorithms perform outside their laboratory settings. In this work, we introduce a complex dataset comprising measurements from a highly automated assembly line in a manufacturing plant, for which we are able to put together partial ground truth causal relationships on the basis of a detailed study of the underlying physics. Based on this dataset, we use the associated ground truth and develop a versatile synthetization pipeline based on distributional random forests that helps overcome common real world data problems such as lack of ground truth, confounding, and privacy concerns. The resulting data generator is implemented in the Python library \texttt{causalAssembly} and can be used to benchmark causal discovery algorithms based on complex data. We demonstrate this by providing initial benchmarking results for some well-known causal discovery algorithms.
An important limitation is that at the moment \texttt{causalAssembly} supports the generation of continuous data only. This is because features extracted from staking and press-in processes are continuous. However, the pipeline presented is more general and supports the incorporation of mixed data. We plan to include discrete features by selecting a wider range of processes in future releases of \texttt{causalAssembly}. Furthermore, the fitted conditional distributions require substantial memory, and we plan to optimize the data architecture in future releases. In addition, it would be interesting to take advantages of opportunities to involve interventional samples.
%\todo{Maybe future interventional samples if available?}
% When real world In non-synthetic settings it is often difficult to
% ascertain whether erroneous learning results stem from poor
% algorithmic quality or from violating critical assumptions \citep[see][Section 5]{Heinze18}.
% Often
% What's more, even in those scenarios\todo{To which szenarios is
% \textbf{those} referring to?} it is often difficult to
% ascertain whether erroneous learning results stem from poor
% algorithmic quality or from violating critical assumptions \citep[see][Section 5]{Heinze18}. \todo{Here the point from \citep{Heinze18}.}
% We believe that the tools described in this work and provided in \texttt{causalAssembly} help address challenges in causal discovery regarding lack of ground truth in complex datasets.
\textbf{Ethical aspects and societal consequences} The authors do not foresee negative societal consequences of this work.
We would like to thank Jens Ackermann for helpful comments and support.
The research for this project was partly conducted by authors employed at Robert Bosch GmbH. Furthermore, this project has received funding from the European Research Council (ERC) under the European Union's Horizon 2020 research and innovation programme (grant agreement No 883818).
% Include extra information in the appendix. This section will often be part of the supplemental material. Please see the call on the NeurIPS website for links to additional guides on dataset publication.
% \begin{enumerate}
% \item Submission introducing new datasets must include the following in the supplementary materials:
% \begin{enumerate}
% \item Dataset documentation and intended uses. Recommended documentation frameworks include datasheets for datasets, dataset nutrition labels, data statements for NLP, and accountability frameworks.
% \item URL to website/platform where the dataset/benchmark can be viewed and downloaded by the reviewers.
% \item Author statement that they bear all responsibility in case of violation of rights, etc., and confirmation of the data license.
% \item Hosting, licensing, and maintenance plan. The choice of hosting platform is yours, as long as you ensure access to the data (possibly through a curated interface) and will provide the necessary maintenance.
% \end{enumerate}
% \item To ensure accessibility, the supplementary materials for datasets must include the following:
% \begin{enumerate}
% \item Links to access the dataset and its metadata. This can be hidden upon submission if the dataset is not yet publicly available but must be added in the camera-ready version. In select cases, e.g when the data can only be released at a later date, this can be added afterward. Simulation environments should link to (open source) code repositories.
% \item The dataset itself should ideally use an open and widely used data format. Provide a detailed explanation on how the dataset can be read. For simulation environments, use existing frameworks or explain how they can be used.
% \item Long-term preservation: It must be clear that the dataset will be available for a long time, either by uploading to a data repository or by explaining how the authors themselves will ensure this.
% \item Explicit license: Authors must choose a license, ideally a CC license for datasets, or an open source license for code (e.g. RL environments).
% \item Add structured metadata to a dataset's meta-data page using Web standards (like schema.org and DCAT): This allows it to be discovered and organized by anyone. If you use an existing data repository, this is often done automatically.
% \item Highly recommended: a persistent dereferenceable identifier (e.g. a DOI minted by a data repository or a prefix on identifiers.org) for datasets, or a code repository (e.g. GitHub, GitLab,...) for code. If this is not possible or useful, please explain why.
% \end{enumerate}
% \item
% \end{enumerate}
% For benchmarks, the supplementary materials must ensure that all results are easily reproducible. Where possible, use a reproducibility framework such as the ML reproducibility checklist, or otherwise guarantee that all results can be easily reproduced, i.e. all necessary datasets, code, and evaluation procedures must be accessible and documented.
\section{Assembly line dataset}\label{sec:app_a}
We introduce a complex production dataset comprising real measurements from a highly automated assembly line in a manufacturing plant. The assembly line consists of multiple production cells or stations along which individual components are joined together. Parts are processed in production stations or cells and are sequentially passed to the next according to the line structure.
Often, one production cell performs the same process several times either in succession or in parallel to reduce overall cycle time. Raw components are added to the assembly at the respective stations. The quality of the assembly process depends on the respective machine state and settings (e.g. tool state, control parameters), the environmental conditions, and on the raw state of the components or intermediates. We carefully study the physical processes that take place in each of the cells.
\subsection{Mapping process knowledge}\label{sec:process_knowledge}
\textbf{Press-in \quad} In order for the inner cylindrical component to be pressed into the bore, mechanical force has to exceed friction force. The friction force depends on component properties, such as the geometrical dimensions (especially diameters and axial lengths), structural stiffness as well as the surface conditions (e.g. surface roughness). Also, machine properties such as misalignment of the pressing machine tool with respect to the work piece holder and cylindrical bore axis can affect the process. None of these properties is measured directly for each component. Depending on the product type, the nominal values of the component properties may differ.
\begin{table}%[!ht]
\centering
\caption{Description of relevant force and position features extracted from the measurements during the press-in process.}
\begin{tabularx}{\textwidth}{lX}
\toprule
Parameter & Description \\ \midrule
\(F_{\text{offs}}\) & Average force measured during closing of the gap between tool and work piece. This depends on the acceleration and mass of the tool as well as the force sensor measurement error. \\ \addlinespace
\(F_1\) & Press-in force (friction force) required to reach the block position (\(S_1\)) as extracted from gradient detection algorithm. \\ \addlinespace
\(F_2\) & Maximum force at the end of the process. This force depends on the structural stiffness and the difference between the positions \(S_2\) and \(S_1\). \\ \addlinespace
\(S_0\) & Axial position of tool at first force increase as extracted from gradient detection algorithm. Position depends on geometrical tolerances of the machine and components as well as on the calibration of the position sensor. \\ \addlinespace
\(S_1\) & Axial position of tool at reaching the block position. Position depends on the relative displacement \(S_0\). \(S_1\) is affected by the length of the bore as well as by the elastic deformation of the machine. Deformation is influenced by the required press-in force \(F_1\) and by the structural machine and component stiffness. Once this gradient position is detected by the process control unit during process execution the command for reversing the tool movement is given. \\ \addlinespace
\(S_2\) & Maximum tool position where the tool direction of movement is effectively reversed. This is affected by the position \(S_1\), by the time required for computation of the gradient point, the tool speed, by the inertial mass of the tool and by the effective structural stiffness. \\
\bottomrule
\end{tabularx}%
\label{table:vardesc_pressin}\\
\end{table}%
During the fitting process the force acting on the pressing machine tool and its current vertical position are measured jointly using special sensors. In order to control and monitor the process both signals are analyzed during and after the process by the process control unit, for example using gradient detection algorithms. The change in measured position is the sum of both the elastic deformation of the pressing machine (driven by the force and structural machine stiffness) and the actual movement of the inner component relative to the bore. Figure \ref{fig:force_displacement} demonstrates how the machine tool force behaves along the vertical position. The most important features extracted from this procedure are stored in the manufacturing execution system.
\begin{figure}%[!ht]
\includegraphics[width=.5\textwidth]{figures/F_pressin.pdf}
\centering
\caption{A typical Force (y-axis) and Position (x-axis) curve during press-in illustrating the characteristics of the process.}
\label{fig:force_displacement}
\end{figure}
\textbf{Staking \quad} In staking the deformation process of the material is mainly driven by two factors. One factor is the material yield strength and hardening behavior. The other factor is the compression stress. The latter is imposed by the force acting on the tool and by the contact area between tool and component. If several staking processes are happening on the same component with several bore positions, the unknown material state is likely to be similar, confounding the observed force values.
% \begin{figure}[ht]
% \includegraphics[width = .5\textwidth]{figures/F_staking_sketch.pdf}
% \centering
% \caption{Illustration of the various phases of the Staking process: (a) Initial state, where the inner component has previously been pressed into the bore, (b) first contact of tool with surface of bore component and start of staking, (c) end position of staking process where the material has been pushed down entirely onto the inner component by plastic material deformation resulting in a sudden increase of the press-in force due to the increased structural stiffness.}
% \label{fig:staking}
% \end{figure}
\begin{figure}%[!ht]
\includegraphics[width=.5\textwidth]{figures/F_staking.pdf}
\centering
\caption{A typical Force (y-axis) and Position (x-axis) curve during staking illustrating the characteristics of the process.}
\label{fig:force_displacement_staking}
\end{figure}
\begin{table}%[!ht]
\centering
\caption{Description of relevant force and position features extracted from the measurements during the staking process.}
\begin{tabularx}{\textwidth}{lX}
\toprule
Parameter & Description \\ \midrule
\(F^*_1\) & Staking force required to reach the turning point of increasing structural stiffness as extracted from the detection algorithm. Once this turning point is detected by the process control unit during process execution the process command for reversing the tool movement is given after an additional, pre-defined force delta has been exceeded. \\ \addlinespace
\(F^*_2\) & Maximum force at the end of the process. This force depends on the structural stiffness and the difference between the positions \(S^*_2\) and \(S^*_1\). \\ \addlinespace
\(S^*_0\) & Axial position of tool at first force increase as extracted from gradient detection algorithm. Position depends on geometrical tolerances of the machine and components as well as on the calibration of the position sensor. \\ \addlinespace
\(S^*_1\) & Axial position of tool at reaching the turning point. Position depends on the relative displacement \(S_0\). \(S_0\) is affected by the length of the bore as well as by the elastic deformation of the machine. Deformation is influenced by the required press-in force \(F^*_1\) and by the structural machine and component stiffness. \\ \addlinespace
\(S^*_2\) & Maximum tool position where the tool direction of movement is effectively reversed. This is affected by the position \(S^*_1\), by the time required for computation of the turning point, the tool speed, by the inertial mass of the tool and by the effective structural stiffness. \\ \addlinespace
\(\Delta S^*\) & Difference in axial position of the tool after removing the pressing force measured at a pre-defined remaining force value. This indicates the true deformation length in the bore without the elastic machine deformation. \\
\bottomrule
\end{tabularx}%
\label{table:vardesc_staking}\\
\end{table}%
Often, press-in and staking processes are run on the same machine. Unobserved machine properties such as structural stiffness, tool misalignment and force sensor positions may impact both processes. Furthermore, confounding can be caused by similarities of the component properties (same component with several bores) or due to batch effects.
\section{Semisynthetic data generator}\label{sec:app_b}
The semisynthetic data generation tool is made available through the Python library \texttt{causalAssembly} available at \url{https://github.com/boschresearch/causalAssembly}. We also provide the semisynthetic data itself (and for convenience one set of datasets with sample size $n=500$) as well as the partial ground truth. All benchmark results presented both in the main text and the supplementary sections are also accessible through \texttt{causalAssembly}.
\subsection{Distributional random forests}\label{sec:drf_background}
We make use of distributional random forests \citep{Cevid2022} in order to estimate conditional distributions. The procedure builds on random forests \citep{Breiman2001} where the original splitting criterion is replaced with a distributional metric in order to ensure homogeneity in each of the tree's leaf nodes. To that end, the authors use the maximal mean discrepancy (MMD) statistic \citep{Gretton2006}. The resulting leaf nodes in the forest give rise to neighborhoods of relevant training data points. Just as in the standard random forest this induces weighting functions for each tree that quantify importance of the given training data points. The average of the tree-specific weighting functions can then be used to estimate the conditional distribution given some test data points.
In our case the conditional distribution of \(X_k\) in graph \(G\) is given by \(P(X_k \mid X_{pa_G(k)} = x)\) for all \(k \in V\). Assume we have grown \(N\) trees \(\mathcal{T}_1, \dots, \mathcal{T}_N\) and denote \(\mathcal{L}_j(x)\) the set of training data points that are assigned to the same leaf node as the test data point \(x\) in tree \(\mathcal{T}_j\). \cite{Cevid2022} define the weighting function by the average of the tree-specific weighting functions i.e. \(w_x(x_i) = 1/N \sum_{j=1}^N \mathbbm{1}(x_i \in \mathcal{L}_j(x))/\lvert \mathcal{L}_j(x) \rvert\). Estimating the conditional distribution \(P(X_k \mid X_{pa_G(k)} = x)\) boils down to assigning weights to the point mass \(\delta_{x_{ik}}\) at \(x_{ik}\) in the empirical distribution of \(X_k\), i.e.
\begin{equation*}
\hat{P}(X_k \mid X_{pa_G(k)} = x) = \sum_{i=1}^n w_x(x_i) \cdot \delta_{x_{ik}},
\end{equation*}
where \(n\) is the sample size of the training data.
As recommended in \cite{Cevid2022} for each conditional distribution we grow \(N = 2000\) trees and choose the Gaussian kernel for the MMD splitting criterion with bandwidth chosen according to the \textit{median heuristic} \citep{Gretton2012}. Table \ref{table:tuning} gives an overview over the remaining hyperparameters.
\subsection{Influence of initial learning}\label{sec:initial_influence}
As indicated in the main text, learning edges across processes using causal
discovery algorithms leaves distinct traces. Benchmarking outcomes are then
tilted in favor of the routine employed in this initial step. We
demonstrate this on processes that take place in Station \(3\) consisting of \(16\) nodes.
As not all methods considered are able to handle the inclusion of prior knowledge,
we ignore existing process knowledge and learn from the data only.
Figures \ref{fig:initial_learning} and \ref{fig:initial_learning_f1} report SHD and \(F_1\) score between the ground truth DAG learned by the initial causal discovery procedure and
results based on corresponding semisynthetic data.
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/initial_impact.pdf}
\centering
\caption{SHD for initial learning based on chosen causal discovery algorithms. Simulation run with \(50\) repetitions.}
\label{fig:initial_learning}
\end{figure}
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/initial_impact_f1.pdf}
\centering
\caption{\(F_1\) score for initial learning based on chosen causal discovery algorithms. Simulation run with \(50\) repetitions.}
\label{fig:initial_learning_f1}
\end{figure}
Next to overall patterns left by the initial learning step, we present sensitivity results with respect to measurement scale. The real data obtained from the process control unit carry a wide range of different measurement scales. These scales should ideally be
irrelevant with respect to causal relationships.
In a first step, we keep the raw data scale both for the initial learning and for the
benchmark run based on the obtained ground truth. We report the average varsortability across
simulation runs. In a second step, we standardize all data. We also report results for the
empty graph to keep track of the number of edges inferred during the
initial graph learning step. If the SHD of the empty graph changes
under standardization then data scale has an effect on the initial
ground truth.
DirectLiNGAM, NOTEARS, and GraN-DAG show a clear performance edge when
the ground truth is originally obtained using the respective routine. This pattern is most clear in Figure \ref{fig:initial_learning_f1}.
The PC algorithm does not seem to benefit too greatly which can to
some degree be attributed to the random DAG completion. In contrast to
the remaining methods, the PC algorithm is agnostic to measurement
scale. DirectLiNGAM is in theory also scale agnostic, however scaling
may influence the adjacency matrix estimation via adaptive lasso when
predictors are not standardized. Regarding NOTEARS and GraN-DAG
measurement units play a substantial role. NOTEARS obtains a ground
truth with varsortability close to one on unstandardized data and
recovers a very sparse DAG on standardized data. This suggests that NOTEARS essentially orders variables according to their variances when
learned on raw data. GraN-DAG results are highly variable throughout
all initial learning methods. The instability becomes particularly
severe when applied to standardized data.
Overall, except for the PC algorithm there are clear traces left by initial learning algorithms applied to process cell structure learning discovery.
\section{Benchmarking}\label{sec:app_c}
\subsection{Algorithms and metrics}\label{sec:alg_metrics}
\textbf{Algorithms \quad} The following well-known structure learning algorithms are employed in our work.
\textit{PC algorithm} \citep{Spirtes1993} (\texttt{pc})\\
A constraint based algorithm that in a first step performs conditional
independence tests in a resource efficient way returning a skeleton
and in a second step orients as many edges as possible. We use the
stable version of the algorithm proposed by \cite{Colombo2014}.
Conditional independence tests are performed using Fisher
z-transformed (partial) Pearson correlation coefficients. Since the PC
algorithm outputs a completed partially directed acyclic graph
(CPDAG), we choose in each run a random DAG that is a member of the
MEC implied by the CPDAG.
\textit{DirectLiNGAM} \citep{Shimizu2011, Hyvarinen2013} (\texttt{lingam})\\
The direct method to learning the LiNGAM model \citep{Shimizu2006}
referred to in Section \ref{sec:identifiability}. First, the procedure
detects source variables by employing entropy-based measures to find
independence between the error variables. The \textit{most
independent} variable is placed first in the causal order and its
effect removed from the other variables. As the ordering of the
residuals resembles that of the variables themselves the algorithm
continues to find the next variable in the ordering by finding the
next \textit{source}-residual.
\textit{NOTEARS} \citep{Zheng2018, Zheng2020} (\texttt{notears})\\
A score-based method that recasts the discrete DAG constraint as a
continuous equality constraint. This allows for the application of a
wide range of techniques for non-convex programs. In NOTEARS a linear
SEM is assumed and for each variable, a feed-forward neural network is
trained that gets all other variables as input. The graph structure is
then dynamically read-off from the networks during training by
aggregating the weights of the input layer of each network. Acyclicity
of the graph is enforced by training the networks simultaneously in an
augmented Lagrangian schema such that the matrix exponential of the
adjacency matrix has small trace.
\textit{GraN-DAG } \citep{Lachapelle2020} (\texttt{grandag})\\
A score-based method and an extension of NOTEARS. In contrast to
NOTEARS, the relation structure is read-off from the networks by
aggregating all the weights till the output layer of the neural
networks individually.
\textit{sortnregress} \citep{Reisach2021} (\texttt{snr})\\
A diagnostic tool to expose data scenarios with high varsortability.
The procedure sorts variables by increasing marginal variance and then
regresses each variable on their ancestors adding an \(\ell\)-1 type
penalty to induce sparsity.
\textbf{Metrics \quad} The choice of evaluation metrics for assessing causal discovery outcomes matters \citep[see][]{Gentzel2019}. We employ a selection of common structural performance metrics and note that in a full scale benchmark these should be extended to include a greater variety of evaluation measures.
\textit{Structural hamming distance (SHD):} \\
The SHD counts how many edge insertions, deletions, and reversals are
necessary to turn the estimated DAG into the ground truth DAG.
\textit{Precision, Recall and \(F_1\) score:} \\
Precision in the graphical context refers to the fraction of correctly
identified edges among all edges inferred. Recall, sets these
correctly identified edges in relation to all edges present in the
ground truth. The \(F_1\) score is the harmonic mean of Precision and
Recall, i.e. \(F_1 =
2\frac{\text{Precision}\cdot\text{Recall}}{\text{Precision}+\text{Recall}}\).
\subsection{Additional benchmarking results}\label{sec:addition_benchmarks}
Table \ref{table:compute} states compute time for each of the benchmark scenarios.
\begin{table}%[htbp!]
\caption{All benchmarks are run on a Linux machine with 32 GB Memory and 8 cores.}
\label{table:compute}
\centering
\begin{tabular}{l c}
\toprule
Result & Compute time \\ \midrule
Full assembly line benchmark Fig \ref{fig:full_benchmark} in the main text & 16.90 hours \\
Station \(1\) benchmark Fig \ref{fig:station1_benchmark} & 1.07 hours \\
Station \(2\) benchmark Fig \ref{fig:station2_benchmark} & 4.39 hours \\
Station \(3\) benchmark Fig \ref{fig:station3_benchmark} & 2.22 hours \\
Station \(4\) benchmark Fig \ref{fig:station4_benchmark} & 3.54 hours \\
Station \(5\) benchmark Fig \ref{fig:station5_benchmark} & 2.33 hours \\
\bottomrule
\end{tabular}
\end{table}
We present additional benchmarks for the remaining production stations. Each simulation run, we sample from the DRF regrown on the corresponding cell graphs. We find that results vary substantially across stations. Figure \ref{fig:station1_benchmark} reports results for the first station. DirectLiNGAM performs best while NOTEARS is not able to detect any edges. GraN-DAG and PC Algorithm results are very unstable throughout all benchmarks. In Figure \ref{fig:station2_benchmark} station two benchmarks are depicted. In this case, NOTEARS performs best. Figure \ref{fig:station3_benchmark} reports station three benchmarks where DirectLiNGAM and NOTEARS demonstrate similar performance. In Figure \ref{fig:station4_benchmark} station four benchmarks suggest that NOTEARS outperforms DirectLiNGAM in terms of SHD and precision. In terms of recall and F1 score however, DirectLiNGAM performs best. On station five (Figure \ref{fig:station5_benchmark}) DirectLiNGAM overall performs best.
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/station1_benchmark.pdf}
\centering
\caption{Chosen metrics for assessing causal discovery outcomes on Station \(1\) (\(\lvert V_1 \cup V_2\rvert = 6\)) using \(100\) simulation runs on a sample size of \(n = 500\).}
\label{fig:station1_benchmark}
\end{figure}
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/station2_benchmark.pdf}
\centering
\caption{Chosen metrics for assessing causal discovery outcomes on Station \(2\) (\(\lvert V_3 \cup V_4 \rvert = 34\)) using \(100\) simulation runs on a sample size of \(n = 500\).}
\label{fig:station2_benchmark}
\end{figure}
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/station3_benchmark.pdf}
\centering
\caption{Chosen metrics for assessing causal discovery outcomes on Station \(3\) (\(\lvert V_5 \cup V_6 \rvert = 16\)) using \(100\) simulation runs on a sample size of \(n = 500\).}
\label{fig:station3_benchmark}
\end{figure}
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/station4_benchmark.pdf}
\centering
\caption{Chosen metrics for assessing causal discovery outcomes on Station \(4\) (\(\lvert V_7 \cup V_8 \rvert = 26\)) using \(100\) simulation runs on a sample size of \(n = 500\).}
\label{fig:station4_benchmark}
\end{figure}
\begin{figure}%[ht]
\includegraphics[width = \linewidth]{figures/station5_benchmark.pdf}
\centering
\caption{Chosen metrics for assessing causal discovery outcomes on Station \(5\) (\(\lvert V_9 \cup V_10 \rvert = 16\)) using \(100\) simulation runs on a sample size of \(n = 500\).}
\label{fig:station5_benchmark}
\end{figure}
\subsection{Hyper and tuning parameters}\label{sec:tuning}
Table \ref{table:tuning} reports all hyperparameters and tuning parameters used in this work.
\begin{table}%[ht!]
\caption{Hyper and tuning parameter choices for routines employed in this work.}
\label{table:tuning}
\centering
\begin{tabular}{l l}
\toprule
& Parameters \\ \midrule
SpAM & \texttt{num\_folds} = \(5\) \\
& \texttt{cv\_loss} = MSE \\
& \texttt{lambda\_choice} = 1se \\
& \texttt{num\_bsplines} = \(6\) \\ [2em]
DRF & \texttt{min\_node\_size} = \(15\) \\
& \texttt{num\_trees} = \(2000\) \\
& \texttt{splitting\_rule} = FourierMMD \\ [2em]
PC Algorithm & \texttt{indep\_test} : fisherz \\
& \texttt{alpha} = \(0.05\) \\ [2em]
NOTEARS & \texttt{lambda\_1} = $0.1$ \\
& \texttt{loss\_type} = $\ell_2$ \\
& \texttt{max\_iter} = \(100\) \\
& \texttt{h\_tol} = \(1\mathrm{e}-8\) \\
& \texttt{rho\_max} = \(1\mathrm{e}+16\) \\
& \texttt{w\_threshold} = \(0.3\) \\ [2em]
GraN-DAG & \texttt{input\_dim} = \(V\) \\
& \texttt{hidden\_num} = $2$ \\
& \texttt{hidden\_dim} = $10$ \\
& \texttt{batch\_size} =$64$ \\
& \texttt{lr} = $0.001$ \\
& \texttt{iterations} = $10000$ \\
& \texttt{model\_name} = NonLinGaussANM \\
& \texttt{nonlinear} = leaky-relu \\
& \texttt{optimizer} = rmsprop \\
& \texttt{h\_threshold} = $1\mathrm{e}-8$ \\
& \texttt{lambda\_init} = $0.0$ \\
& \texttt{mu\_init} = $0.001$ \\
& \texttt{omega\_lambda} = $0.0001$ \\
& \texttt{omega\_mu} = $0.9$ \\
& \texttt{stop\_crit\_win} = $100$ \\
& \texttt{edge\_clamp\_range} = $0.0001$ \\
\bottomrule
\end{tabular}
\end{table}
\begin{thebibliography}{53}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Glymour et~al.(2019)Glymour, Zhang, and Spirtes]{Glymour2019}
Clark Glymour, Kun Zhang, and Peter Spirtes.
\newblock Review of causal discovery methods based on graphical models.
\newblock \emph{Frontiers in Genetics}, 10, June 2019.
\newblock \doi{10.3389/fgene.2019.00524}.
\newblock URL \url{https://doi.org/10.3389/fgene.2019.00524}.
\bibitem[Heinze-Deml et~al.(2018)Heinze-Deml, Maathuis, and
Meinshausen]{Heinze18}
Christina Heinze-Deml, Marloes~H. Maathuis, and Nicolai Meinshausen.
\newblock Causal structure learning.
\newblock \emph{Annual Review of Statistics and Its Application}, 5\penalty0
(1):\penalty0 371--391, 2018.
\newblock \doi{10.1146/annurev-statistics-031017-100630}.
\newblock URL \url{https://doi.org/10.1146/annurev-statistics-031017-100630}.
\bibitem[Maathuis et~al.(2019)Maathuis, Drton, Lauritzen, and
Wainwright]{Maathuis2019}
Marloes Maathuis, Mathias Drton, Steffen Lauritzen, and Martin Wainwright,
editors.
\newblock \emph{Handbook of graphical models}.
\newblock Chapman \& Hall/CRC Handbooks of Modern Statistical Methods. CRC
Press, Boca Raton, FL, 2019.
\newblock ISBN 978-1-4987-8862-5.
\bibitem[Vowels et~al.(2022)Vowels, Camgoz, and Bowden]{Vowels2022}
Matthew~J. Vowels, Necati~Cihan Camgoz, and Richard Bowden.
\newblock D'ya like {DAGs}? {A} survey on structure learning and causal
discovery.
\newblock \emph{{ACM} Computing Surveys}, 55\penalty0 (4):\penalty0 1--36,
2022.
\newblock \doi{10.1145/3527154}.
\newblock URL \url{https://doi.org/10.1145/3527154}.
\bibitem[Zanga et~al.(2022)Zanga, Ozkirimli, and Stella]{Zanga2022}
Alessio Zanga, Elif Ozkirimli, and Fabio Stella.
\newblock A survey on causal discovery: Theory and practice.
\newblock \emph{International Journal of Approximate Reasoning}, 151:\penalty0
101--129, 2022.
\newblock ISSN 0888-613X.
\newblock \doi{https://doi.org/10.1016/j.ijar.2022.09.004}.
\newblock URL
\url{https://www.sciencedirect.com/science/article/pii/S0888613X22001402}.
\bibitem[Vuković and Thalmann(2022)]{Vukovic2022}
Matej Vuković and Stefan Thalmann.
\newblock Causal discovery in manufacturing: A structured literature review.
\newblock \emph{Journal of Manufacturing and Materials Processing}, 6\penalty0
(1), 2022.
\newblock ISSN 2504-4494.
\newblock \doi{10.3390/jmmp6010010}.
\newblock URL \url{https://www.mdpi.com/2504-4494/6/1/10}.
\bibitem[Budhathoki et~al.(2022)Budhathoki, Minorics, Bloebaum, and
Janzing]{Budhathoki2022}
Kailash Budhathoki, Lenon Minorics, Patrick Bloebaum, and Dominik Janzing.
\newblock Causal structure-based root cause analysis of outliers.
\newblock In \emph{Proceedings of the 39th International Conference on Machine
Learning}, volume 162 of \emph{Proceedings of Machine Learning Research},
pages 2357--2369. PMLR, 17--23 Jul 2022.
\newblock URL \url{https://proceedings.mlr.press/v162/budhathoki22a.html}.
\bibitem[Gentzel et~al.(2019)Gentzel, Garant, and Jensen]{Gentzel2019}
Amanda Gentzel, Dan Garant, and David Jensen.
\newblock The case for evaluating causal models using interventional measures
and empirical data.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~32. Curran Associates, Inc., 2019.
\newblock URL
\url{https://proceedings.neurips.cc/paper_files/paper/2019/file/a87c11b9100c608b7f8e98cfa316ff7b-Paper.pdf}.
\bibitem[Eigenmann et~al.(2020)Eigenmann, Mukherjee, and
Maathuis]{Eigenmann2020}
Marco Eigenmann, Sach Mukherjee, and Marloes Maathuis.
\newblock Evaluation of causal structure learning algorithms via risk
estimation.
\newblock In \emph{Proceedings of the 36th Conference on Uncertainty in
Artificial Intelligence (UAI)}, volume 124 of \emph{Proceedings of Machine
Learning Research}, pages 151--160. PMLR, 2020.
\newblock URL \url{https://proceedings.mlr.press/v124/eigenmann20a.html}.
\bibitem[Huegle et~al.(2021)Huegle, Hagedorn, Boehme, Poerschke, Umland, and
Schlosser]{Huegle2021}
Johannes Huegle, Christopher Hagedorn, Lukas Boehme, Mats Poerschke, Jonas
Umland, and Rainer Schlosser.
\newblock Manm-cs: Data generation for benchmarking causal structure learning
from mixed discrete-continuous and nonlinear data.
\newblock In \emph{WHY-21 @ NeurIPS 2021}, 2021.
\bibitem[Reisach et~al.(2021)Reisach, Seiler, and Weichwald]{Reisach2021}
{Alexander G.} Reisach, Christof Seiler, and Sebastian Weichwald.
\newblock Beware of the simulated {DAG}! {C}ausal discovery benchmarks may be
easy to game.
\newblock In \emph{Advances in Neural Information Processing Systems 34
(NeurIPS)}, pages 1--13. NeurIPS Proceedings, 2021.
\bibitem[Constantinou et~al.(2021)Constantinou, Liu, Chobtham, Guo, and
Kitson]{Constantinou2021}
Anthony~C. Constantinou, Yang Liu, Kiattikun Chobtham, Zhigao Guo, and
Neville~K. Kitson.
\newblock Large-scale empirical validation of {B}ayesian network structure
learning algorithms with noisy data.
\newblock \emph{International Journal of Approximate Reasoning}, 131:\penalty0
151--188, 2021.
\newblock \doi{10.1016/j.ijar.2021.01.001}.
\newblock URL \url{https://doi.org/10.1016/j.ijar.2021.01.001}.
\bibitem[Ravikumar et~al.(2009)Ravikumar, Lafferty, Liu, and
Wasserman]{Ravikumar2009}
Pradeep Ravikumar, John Lafferty, Han Liu, and Larry Wasserman.
\newblock Sparse additive models.
\newblock \emph{J. R. Stat. Soc. Ser. B Stat. Methodol.}, 71\penalty0
(5):\penalty0 1009--1030, 2009.
\newblock ISSN 1369-7412,1467-9868.
\newblock \doi{10.1111/j.1467-9868.2009.00718.x}.
\newblock URL \url{https://doi.org/10.1111/j.1467-9868.2009.00718.x}.
\bibitem[Cevid et~al.(2022)Cevid, Michel, Näf, Bühlmann, and
Meinshausen]{Cevid2022}
Domagoj Cevid, Loris Michel, Jeffrey Näf, Peter Bühlmann, and Nicolai
Meinshausen.
\newblock Distributional random forests: Heterogeneity adjustment and
multivariate distributional regression.
\newblock \emph{Journal of Machine Learning Research}, 23\penalty0
(333):\penalty0 1--79, 2022.
\newblock URL \url{http://jmlr.org/papers/v23/21-0585.html}.
\bibitem[Gamella et~al.(2022)Gamella, Taeb, Heinze-Deml, and
B\"uhlmann]{Gamella2022}
Juan~L. Gamella, Armeen Taeb, Christina Heinze-Deml, and Peter B\"uhlmann.
\newblock Characterization and greedy learning of {G}aussian structural causal
models under unknown interventions.
\newblock \emph{arXiv preprint arXiv:2211.14897}, 2022.
\bibitem[Spirtes et~al.(1993)Spirtes, Glymour, and Scheines]{Spirtes1993}
Peter Spirtes, Clark Glymour, and Richard Scheines.
\newblock \emph{Causation, prediction, and search}, volume~81 of \emph{Lecture
Notes in Statistics}.
\newblock Springer-Verlag, New York, 1993.
\newblock ISBN 0-387-97979-4.
\newblock \doi{10.1007/978-1-4612-2748-9}.
\newblock URL \url{https://doi.org/10.1007/978-1-4612-2748-9}.
\bibitem[Shimizu et~al.(2011)Shimizu, Inazumi, Sogawa, Hyv{{\"a}}rinen,
Kawahara, Washio, Hoyer, and Bollen]{Shimizu2011}
Shohei Shimizu, Takanori Inazumi, Yasuhiro Sogawa, Aapo Hyv{{\"a}}rinen,
Yoshinobu Kawahara, Takashi Washio, Patrik~O. Hoyer, and Kenneth Bollen.
\newblock {DirectLiNGAM}: A direct method for learning a linear non-{G}aussian
structural equation model.
\newblock \emph{Journal of Machine Learning Research}, 12\penalty0
(33):\penalty0 1225--1248, 2011.
\newblock URL \url{http://jmlr.org/papers/v12/shimizu11a.html}.
\bibitem[Zheng et~al.(2018)Zheng, Aragam, Ravikumar, and Xing]{Zheng2018}
Xun Zheng, Bryon Aragam, Pradeep~K Ravikumar, and Eric~P Xing.
\newblock Dags with no tears: Continuous optimization for structure learning.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~31. Curran Associates, Inc., 2018.
\newblock URL
\url{https://proceedings.neurips.cc/paper_files/paper/2018/file/e347c51419ffb23ca3fd5050202f9c3d-Paper.pdf}.
\bibitem[Lachapelle et~al.(2020)Lachapelle, Brouillard, Deleu, and
Lacoste-Julien]{Lachapelle2020}
Sébastien Lachapelle, Philippe Brouillard, Tristan Deleu, and Simon
Lacoste-Julien.
\newblock Gradient-based neural {DAG} learning.
\newblock In \emph{International Conference on Learning Representations}, 2020.
\newblock URL \url{https://openreview.net/forum?id=rklbKA4YDS}.
\bibitem[Healy and Nikolov(2002)]{Healy2002}
Patrick Healy and Nikola~S. Nikolov.
\newblock A branch-and-cut approach to the directed acyclic graph layering
problem.
\newblock In \emph{Graph drawing}, volume 2528 of \emph{Lecture Notes in
Comput. Sci.}, pages 98--109. Springer, Berlin, 2002.
\newblock ISBN 3-540-00158-1.
\newblock \doi{10.1007/3-540-36151-0\{_}10}.
\newblock URL \url{https://doi.org/10.1007/3-540-36151-0_10}.
\bibitem[Pearl(2009)]{Pearl2009}
Judea Pearl.
\newblock \emph{Causality}.
\newblock Cambridge University Press, Cambridge, second edition, 2009.
\newblock ISBN 978-0-521-89560-6; 0-521-77362-8.
\newblock \doi{10.1017/CBO9780511803161}.
\newblock URL \url{https://doi.org/10.1017/CBO9780511803161}.
\newblock Models, reasoning, and inference.
\bibitem[Peters et~al.(2017)Peters, Janzing, and Sch{\"o}lkopf]{Peters2017}
Jonas Peters, Dominik Janzing, and Bernhard Sch{\"o}lkopf.
\newblock \emph{Elements of causal inference: Foundations and learning
algorithms}.
\newblock Adaptive Computation and Machine Learning. MIT Press, Cambridge, MA,
2017.
\newblock ISBN 978-0-262-03731-0.
\newblock URL \url{https://mitpress.mit.edu/books/elements-causal-inference}.
\bibitem[Drton and Maathuis(2017)]{Drton2017}
Mathias Drton and Marloes~H. Maathuis.
\newblock Structure learning in graphical modeling.
\newblock \emph{Annual Review of Statistics and Its Application}, 4\penalty0
(1):\penalty0 365--393, 2017.
\newblock \doi{10.1146/annurev-statistics-060116-053803}.
\newblock URL \url{https://doi.org/10.1146/annurev-statistics-060116-053803}.
\bibitem[Lauritzen(1996)]{Lauritzen1996}
Steffen~L. Lauritzen.
\newblock \emph{Graphical models}, volume~17 of \emph{Oxford Statistical
Science Series}.
\newblock The Clarendon Press, Oxford University Press, New York, 1996.
\newblock ISBN 0-19-852219-3.
\newblock Oxford Science Publications.
\bibitem[Chickering(2003)]{Chickering2003}
David~Maxwell Chickering.
\newblock Optimal structure identification with greedy search.
\newblock \emph{J. Mach. Learn. Res.}, 3:\penalty0 507--554, 2003.
\newblock ISSN 1532-4435.
\newblock \doi{10.1162/153244303321897717}.
\newblock URL \url{https://doi.org/10.1162/153244303321897717}.
\bibitem[Bollen(1989)]{Bollen1989}
Kenneth~A. Bollen.
\newblock \emph{Structural equations with latent variables}.
\newblock Wiley Series in Probability and Mathematical Statistics: Applied
Probability and Statistics. John Wiley \& Sons, Inc., New York, 1989.
\newblock ISBN 0-471-01171-1.
\newblock \doi{10.1002/9781118619179}.
\newblock URL \url{https://doi.org/10.1002/9781118619179}.
\newblock A Wiley-Interscience Publication.
\bibitem[Shimizu et~al.(2006)Shimizu, Hoyer, Hyv\"{a}rinen, and
Kerminen]{Shimizu2006}
Shohei Shimizu, Patrik~O. Hoyer, Aapo Hyv\"{a}rinen, and Antti Kerminen.
\newblock A linear non-{G}aussian acyclic model for causal discovery.
\newblock \emph{J. Mach. Learn. Res.}, 7:\penalty0 2003--2030, 2006.
\newblock ISSN 1532-4435,1533-7928.
\bibitem[Shimizu(2022)]{Shimizu2022}
Shohei. Shimizu.
\newblock \emph{Statistical causal discovery: {LiNGAM} approach}.
\newblock JSS Research Series in Statistics. Springer Japan, Tokyo, 1st ed.
2022. edition, 2022.
\newblock ISBN 9784431557845.
\bibitem[Hoyer et~al.(2008)Hoyer, Janzing, Mooij, Peters, and
Sch\"{o}lkopf]{Hoyer2008}
Patrik Hoyer, Dominik Janzing, Joris~M Mooij, Jonas Peters, and Bernhard
Sch\"{o}lkopf.
\newblock Nonlinear causal discovery with additive noise models.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~21. Curran Associates, Inc., 2008.
\newblock URL
\url{https://proceedings.neurips.cc/paper_files/paper/2008/file/f7664060cc52bc6f3d620bcedc94a4b6-Paper.pdf}.
\bibitem[Zhang and Hyv\"{a}rinen(2009)]{Zhang2009}
Kun Zhang and Aapo Hyv\"{a}rinen.
\newblock On the identifiability of the post-nonlinear causal model.
\newblock In \emph{Proceedings of the Twenty-Fifth Conference on Uncertainty in
Artificial Intelligence}, UAI '09, pages 647--655, Arlington, Virginia, USA,
2009. AUAI Press.
\newblock ISBN 9780974903958.
\bibitem[Peters and B\"{u}hlmann(2014)]{Peters2014b}
J.~Peters and P.~B\"{u}hlmann.
\newblock Identifiability of {G}aussian structural equation models with equal
error variances.
\newblock \emph{Biometrika}, 101\penalty0 (1):\penalty0 219--228, 2014.
\newblock ISSN 0006-3444,1464-3510.
\newblock \doi{10.1093/biomet/ast043}.
\newblock URL \url{https://doi.org/10.1093/biomet/ast043}.
\bibitem[Chen et~al.(2019)Chen, Drton, and Wang]{Chen2019}
Wenyu Chen, Mathias Drton, and Y.~Samuel Wang.
\newblock On causal discovery with an equal-variance assumption.
\newblock \emph{Biometrika}, 106\penalty0 (4):\penalty0 973--980, 2019.
\newblock ISSN 0006-3444.
\newblock \doi{10.1093/biomet/asz049}.
\newblock URL \url{https://doi-org.eaccess.tum.edu/10.1093/biomet/asz049}.
\bibitem[Ng et~al.(2020)Ng, Ghassami, and Zhang]{Ng2020}
Ignavier Ng, AmirEmad Ghassami, and Kun Zhang.
\newblock On the role of sparsity and {DAG} constraints for learning linear
{DAGs}.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~33, pages 17943--17954. Curran Associates, Inc., 2020.
\newblock URL
\url{https://proceedings.neurips.cc/paper_files/paper/2020/file/d04d42cdf14579cd294e5079e0745411-Paper.pdf}.
\bibitem[Yu et~al.(2021)Yu, Gao, Yin, and Ji]{Yu2021}
Yue Yu, Tian Gao, Naiyu Yin, and Qiang Ji.
\newblock {DAGs} with no curl: An efficient {DAG} structure learning approach.
\newblock In \emph{Proceedings of the 38th International Conference on Machine
Learning}, volume 139 of \emph{Proceedings of Machine Learning Research},
pages 12156--12166. PMLR, 2021.
\newblock URL \url{https://proceedings.mlr.press/v139/yu21a.html}.
\bibitem[Zhang et~al.(2022)Zhang, Ng, Gong, Liu, Abbasnejad, Gong, Zhang, and
Shi]{Zhang2022}
Zhen Zhang, Ignavier Ng, Dong Gong, Yuhang Liu, Ehsan~M Abbasnejad, Mingming
Gong, Kun Zhang, and Javen~Qinfeng Shi.
\newblock Truncated matrix power iteration for differentiable {DAG} learning.
\newblock In \emph{Advances in Neural Information Processing Systems}, 2022.
\newblock URL \url{https://openreview.net/forum?id=I4aSjFR7jOm}.
\bibitem[Zhang et~al.(2023)Zhang, Liu, Ma, Cai, Wang, and Chua]{Zhang2023}
An~Zhang, Fangfu Liu, Wenchang Ma, Zhibo Cai, Xiang Wang, and Tat-Seng Chua.
\newblock Boosting causal discovery via adaptive sample reweighting.
\newblock In \emph{The Eleventh International Conference on Learning
Representations}, 2023.
\newblock URL \url{https://openreview.net/forum?id=LNpMtk15AS4}.
\bibitem[Reisach et~al.(2023)Reisach, Tami, Seiler, Chambaz, and
Weichwald]{Reisach2023}
Alexander~G. Reisach, Myriam Tami, Christof Seiler, Antoine Chambaz, and
Sebastian Weichwald.
\newblock Simple sorting criteria help find the causal order in additive noise
models.
\newblock \emph{arXiv preprint arXiv:2303.18211}, 2023.
\bibitem[Mooij et~al.(2016)Mooij, Peters, Janzing, Zscheischler, and
Sch{{\"o}}lkopf]{Mooij2016}
Joris~M. Mooij, Jonas Peters, Dominik Janzing, Jakob Zscheischler, and Bernhard
Sch{{\"o}}lkopf.
\newblock Distinguishing cause from effect using observational data: Methods
and benchmarks.
\newblock \emph{Journal of Machine Learning Research}, 17\penalty0
(32):\penalty0 1--102, 2016.
\newblock URL \url{http://jmlr.org/papers/v17/14-518.html}.
\bibitem[Marbach et~al.(2012)Marbach, Costello, K\"{u}ffner, Vega, Prill,
Camacho, Allison, Kellis, Collins, and Stolovitzky]{Marbach2012}
Daniel Marbach, James~C Costello, Robert K\"{u}ffner, Nicole~M Vega, Robert~J
Prill, Diogo~M Camacho, Kyle~R Allison, Manolis Kellis, James~J Collins, and
Gustavo Stolovitzky.
\newblock Wisdom of crowds for robust gene network inference.
\newblock \emph{Nature Methods}, 9\penalty0 (8):\penalty0 796--804, 2012.
\newblock \doi{10.1038/nmeth.2016}.
\newblock URL \url{https://doi.org/10.1038/nmeth.2016}.
\bibitem[Replogle et~al.(2022)Replogle, Saunders, Pogson, Hussmann, Lenail,
Guna, Mascibroda, Wagner, Adelman, Lithwick-Yanai, Iremadze, Oberstrass,
Lipson, Bonnar, Jost, Norman, and Weissman]{Replogle2022}
Joseph~M. Replogle, Reuben~A. Saunders, Angela~N. Pogson, Jeffrey~A. Hussmann,
Alexander Lenail, Alina Guna, Lauren Mascibroda, Eric~J. Wagner, Karen
Adelman, Gila Lithwick-Yanai, Nika Iremadze, Florian Oberstrass, Doron
Lipson, Jessica~L. Bonnar, Marco Jost, Thomas~M. Norman, and Jonathan~S.
Weissman.
\newblock Mapping information-rich genotype-phenotype landscapes with
genome-scale perturb-seq.
\newblock \emph{Cell}, 185\penalty0 (14):\penalty0 2559--2575.e28, 2022.
\newblock \doi{10.1016/j.cell.2022.05.013}.
\newblock URL \url{https://doi.org/10.1016/j.cell.2022.05.013}.
\bibitem[Sachs et~al.(2005)Sachs, Perez, Pe'er, Lauffenburger, and
Nolan]{Sachs2005}
Karen Sachs, Omar Perez, Dana Pe'er, Douglas~A. Lauffenburger, and Garry~P.
Nolan.
\newblock Causal protein-signaling networks derived from multiparameter
single-cell data.
\newblock \emph{Science}, 308\penalty0 (5721):\penalty0 523--529, 2005.
\newblock \doi{10.1126/science.1105809}.
\newblock URL \url{https://www.science.org/doi/abs/10.1126/science.1105809}.
\bibitem[den Bulcke et~al.(2006)den Bulcke, Leemput, Naudts, van Remortel, Ma,
Verschoren, Moor, and Marchal]{VandenBulcke2006}
Tim~Van den Bulcke, Koenraad~Van Leemput, Bart Naudts, Piet van Remortel,
Hongwu Ma, Alain Verschoren, Bart~De Moor, and Kathleen Marchal.
\newblock {SynTReN}: A generator of synthetic gene expression data for design
and analysis of structure learning algorithms.
\newblock \emph{{BMC} Bioinformatics}, 7\penalty0 (1), 2006.
\newblock \doi{10.1186/1471-2105-7-43}.
\newblock URL \url{https://doi.org/10.1186/1471-2105-7-43}.
\bibitem[Schaffter et~al.(2011)Schaffter, Marbach, and Floreano]{Schaffter2021}
Thomas Schaffter, Daniel Marbach, and Dario Floreano.
\newblock {GeneNetWeaver: In silico benchmark generation and performance
profiling of network inference methods}.
\newblock \emph{Bioinformatics}, 27\penalty0 (16):\penalty0 2263--2270, 2011.
\newblock ISSN 1367-4803.
\newblock \doi{10.1093/bioinformatics/btr373}.
\newblock URL \url{https://doi.org/10.1093/bioinformatics/btr373}.
\bibitem[Scutari(2010)]{Scutari2010}
M.~Scutari.
\newblock Learning {B}ayesian networks with the {bnlearn} {R} package.
\newblock \emph{Journal of Statistical Software}, 35\penalty0 (3):\penalty0
1--22, 2010.
\newblock URL \url{http://www.jstatsoft.org/v35/i03/}.
\bibitem[Chen and Fayek(2023)]{Chen2023}
Jarry Chen and Haytham~M. Fayek.
\newblock The structurally complex with additive parent causality (scary)
dataset.
\newblock In \emph{2nd Conference on Causal Learning and Reasoning}, 2023.
\bibitem[Kolmogorov(1933)]{Kolmogorov1933}
A.~N Kolmogorov.
\newblock Sulla determinazione empirica di una legge di distribuzione.
\newblock \emph{Giornale dell'Istituto Italiano degli Attuari}, 4:\penalty0
83--91, 1933.
\newblock URL \url{https://cir.nii.ac.jp/crid/1571135650766370304}.
\bibitem[Rios et~al.(2022)Rios, Moffa, and Kuipers]{Rios2022}
Felix~L. Rios, Giusi Moffa, and Jack Kuipers.
\newblock Benchpress: A scalable and versatile workflow for benchmarking
structure learning algorithms.
\newblock \emph{arXiv preprint arXiv:2107.03863}, 2022.
\bibitem[Breiman(2001)]{Breiman2001}
Leo Breiman.
\newblock Random forests.
\newblock \emph{Machine Learning}, 45\penalty0 (1):\penalty0 5--32, 2001.
\newblock \doi{10.1023/a:1010933404324}.
\newblock URL \url{https://doi.org/10.1023/a:1010933404324}.
\bibitem[Gretton et~al.(2006)Gretton, Borgwardt, Rasch, Sch\"{o}lkopf, and
Smola]{Gretton2006}
Arthur Gretton, Karsten Borgwardt, Malte Rasch, Bernhard Sch\"{o}lkopf, and
Alex Smola.
\newblock A kernel method for the two-sample-problem.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~19. MIT Press, 2006.
\newblock URL
\url{https://proceedings.neurips.cc/paper_files/paper/2006/file/e9fb2eda3d9c55a0d89c98d6c54b5b3e-Paper.pdf}.
\bibitem[Gretton et~al.(2012)Gretton, Sejdinovic, Strathmann, Balakrishnan,
Pontil, Fukumizu, and Sriperumbudur]{Gretton2012}
Arthur Gretton, Dino Sejdinovic, Heiko Strathmann, Sivaraman Balakrishnan,
Massimiliano Pontil, Kenji Fukumizu, and Bharath~K. Sriperumbudur.
\newblock Optimal kernel choice for large-scale two-sample tests.
\newblock In \emph{Advances in Neural Information Processing Systems},
volume~25. Curran Associates, Inc., 2012.
\newblock URL
\url{https://proceedings.neurips.cc/paper_files/paper/2012/file/dbe272bab69f8e13f14b405e038deb64-Paper.pdf}.
\bibitem[Colombo and Maathuis(2014)]{Colombo2014}
Diego Colombo and Marloes~H. Maathuis.
\newblock Order-independent constraint-based causal structure learning.
\newblock \emph{J. Mach. Learn. Res.}, 15:\penalty0 3741--3782, 2014.
\newblock ISSN 1532-4435,1533-7928.
\bibitem[Hyv\"{a}rinen and Smith(2013)]{Hyvarinen2013}
Aapo Hyv\"{a}rinen and Stephen~M. Smith.
\newblock Pairwise likelihood ratios for estimation of non-{G}aussian
structural equation models.
\newblock \emph{J. Mach. Learn. Res.}, 14:\penalty0 111--152, 2013.
\newblock ISSN 1532-4435,1533-7928.
\bibitem[Zheng et~al.(2020)Zheng, Dan, Aragam, Ravikumar, and Xing]{Zheng2020}
Xun Zheng, Chen Dan, Bryon Aragam, Pradeep Ravikumar, and Eric Xing.
\newblock Learning sparse nonparametric {DAGs}.
\newblock In \emph{Proceedings of the Twenty Third International Conference on
Artificial Intelligence and Statistics}, volume 108 of \emph{Proceedings of
Machine Learning Research}, pages 3414--3425. PMLR, 2020.
\newblock URL \url{https://proceedings.mlr.press/v108/zheng20a.html}.
\end{thebibliography}