Type Information-Assisted Self-Supervised Knowledge Graph Denoising
Extracted source text · arXiv 2503.09916
\documentclass[twoside]{article}
% for toc for appendix
\usepackage{scrwfile}
\TOCclone[\contentsname~(\appendixname)]{toc}{atoc}
\newcommand\StartAppendixEntries{}
\AfterTOCHead[toc]{%
\renewcommand\StartAppendixEntries{\value{tocdepth}=-10000\relax}%
}
\AfterTOCHead[atoc]{%
\edef\maintocdepth{\the\value{tocdepth}}%
\value{tocdepth}=-10000\relax%
\renewcommand\StartAppendixEntries{\value{tocdepth}=\maintocdepth\relax}%
}
\newcommand*\appendixwithtoc{%
% \cleardoublepage
\addtocontents{toc}{\protect\StartAppendixEntries}
\listofatoc
}
%
\usepackage[accepted]{aistats2025}
% If your paper is accepted, change the options for the package
% aistats2024 as follows:
%
%\usepackage[accepted]{aistats2024}
%
% This option will print headings for the title of your paper and
% headings for the authors names, plus a copyright note at the end of
% the first column of the first page.
% If you set papersize explicitly, activate the following three lines:
%\special{papersize = 8.5in, 11in}
%\setlength{\pdfpageheight}{11in}
%\setlength{\pdfpagewidth}{8.5in}
% If you use natbib package, activate the following three lines:
%\usepackage[round]{natbib}
%\renewcommand{\bibname}{References}
%\renewcommand{\bibsection}{\subsubsection*{\bibname}}
% If you use BibTeX in apalike style, activate the following line:
%\bibliographystyle{apalike}
\usepackage{hyperref}
\usepackage{url}
\usepackage{amsmath, amsthm, amssymb}
\usepackage{algpseudocode}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% THEOREMS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}
% [section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
\usepackage[utf8]{inputenc} % allow utf-8 input
\usepackage[T1]{fontenc} % use 8-bit T1 fonts
\usepackage{hyperref} % hyperlinks
\usepackage{url} % simple URL typesetting
\usepackage{booktabs} % professional-quality tables
\usepackage{amsfonts} % blackboard math symbols
\usepackage{nicefrac} % compact symbols for 1/2, etc.
\usepackage{microtype} % microtypography
\usepackage{xcolor} % colors
\usepackage{adjustbox}
\usepackage{xcolor, color, colortbl}
\usepackage{nicematrix}
\usepackage{amsmath}
\usepackage{multicol, multirow}
% \usepackage[tableposition=top]{caption}
\usepackage{subfigure, graphicx}
\usepackage{floatrow}
% \floatsetup[table]{capposition=top}
% Table float box with bottom caption, box width adjusted to content
% \newfloatcommand{capbtabbox}{table}[][\FBwidth]
% \usepackage{blindtext}
% \usepackage[toc,page,header]{appendix}
% \usepackage{minitoc}
% colors
\definecolor{pink_bg}{HTML}{ffcbcb}
% \definecolor{Gray}{gray}{0.05}%#CDESD2
\definecolor{Gray}{gray}{0.95}
\definecolor{green_bg}{HTML}{f1f7b5}
\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}
\begin{document}
% If your paper is accepted and the title of your paper is very long,
% the style will print as headings an error message. Use the following
% command to supply a shorter title of your paper so that it can be
% used as headings.
%
%\runningtitle{I use this title instead because the last one was very long}
% If your paper is accepted and the number of authors is large, the
% style will print as headings an error message. Use the following
% command to supply a shorter version of the authors names so that
% they can be used as headings (for example, use only the surnames)
%
%\runningauthor{Surname 1, Surname 2, Surname 3, ...., Surname n}
\twocolumn[
\aistatstitle{Type Information-Assisted Self-Supervised Knowledge Graph Denoising}
\aistatsauthor{ Jiaqi Sun$^{1,2}$ \And Yujia Zheng$^1$ \And Xinshuai Dong$^1$ \And Haoyue Dai$^1$ \And Kun Zhang$^{1,2}$ }
\aistatsaddress{ $^1$Carnegie Mellon University \quad\quad $^2$Mohamed bin Zayed University of Artificial Intelligence } ]
\input{tex/0_abstract}
\input{tex/1_intro}
\input{tex/2_preliminary}
\input{tex/3_method}
\input{tex/4_experiments}
\input{tex/7_related_work}
\input{tex/5_conclusion}
\input{tex/8_acknowledgements}
\bibliographystyle{apalike}
\bibliography{cr-main}
% \begin{thebibliography}{}
% \setlength{\itemindent}{-\leftmargin}
% \makeatletter\renewcommand{\@biblabel}[1]{}\makeatother
% \bibitem{} J.~Alspector, B.~Gupta, and R.~B.~Allen (1989).
% \newblock Performance of a stochastic learning microchip.
% \newblock In D. S. Touretzky (ed.),
% \textit{Advances in Neural Information Processing Systems 1}, 748--760.
% San Mateo, Calif.: Morgan Kaufmann.
% \bibitem{} F.~Rosenblatt (1962).
% \newblock \textit{Principles of Neurodynamics.}
% \newblock Washington, D.C.: Spartan Books.
% \bibitem{} G.~Tesauro (1989).
% \newblock Neurogammon wins computer Olympiad.
% \newblock \textit{Neural Computation} \textbf{1}(3):321--323.
% \end{thebibliography}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CHECKLIST
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \newpage
\section*{Checklist}
% % %%% BEGIN INSTRUCTIONS %%%
% The checklist follows the references. For each question, choose your answer from the three possible options: Yes, No, [Not Applicable] You are encouraged to include a justification to your answer, either by referencing the appropriate section of your paper or providing a brief inline description (1-2 sentences).
% Please do not modify the questions. Note that the Checklist section does not count towards the page limit. Not including the checklist in the first submission won't result in desk rejection, although in such case we will ask you to upload it during the author response period and include it in camera ready (if accepted).
% \textbf{In your paper, please delete this instructions block and only keep the Checklist section heading above along with the questions/answers below.}
% % %%% END INSTRUCTIONS %%%
\begin{enumerate}
\item For all models and algorithms presented, check if you include:
\begin{enumerate}
\item A clear description of the mathematical setting, assumptions, algorithm, and/or model. [Yes]
\item An analysis of the properties and complexity (time, space, sample size) of any algorithm. [Yes]
\item (Optional) Anonymized source code, with specification of all dependencies, including external libraries. [Yes]
\end{enumerate}
\item For any theoretical claim, check if you include:
\begin{enumerate}
\item Statements of the full set of assumptions of all theoretical results. [Yes]
\item Complete proofs of all theoretical results. [Yes]
\item Clear explanations of any assumptions. [Yes]
\end{enumerate}
\item For all figures and tables that present empirical results, check if you include:
\begin{enumerate}
\item The code, data, and instructions needed to reproduce the main experimental results (either in the supplemental material or as a URL). [Yes]
\item All the training details (e.g., data splits, hyperparameters, how they were chosen). [Yes]
\item A clear definition of the specific measure or statistics and error bars (e.g., with respect to the random seed after running experiments multiple times). [Yes]
\item A description of the computing infrastructure used. (e.g., type of GPUs, internal cluster, or cloud provider). [Yes]
\end{enumerate}
\item If you are using existing assets (e.g., code, data, models) or curating/releasing new assets, check if you include:
\begin{enumerate}
\item Citations of the creator If your work uses existing assets. [Yes]
\item The license information of the assets, if applicable. [Yes]
\item New assets either in the supplemental material or as a URL, if applicable. [Not Applicable]
\item Information about consent from data providers/curators. [Yes]
\item Discussion of sensible content if applicable, e.g., personally identifiable information or offensive content. [Not Applicable]
\end{enumerate}
\item If you used crowdsourcing or conducted research with human subjects, check if you include:
\begin{enumerate}
\item The full text of instructions given to participants and screenshots. [Not Applicable]
\item Descriptions of potential participant risks, with links to Institutional Review Board (IRB) approvals if applicable. [Not Applicable]
\item The estimated hourly wage paid to participants and the total amount spent on participant compensation. [Not Applicable]
\end{enumerate}
\end{enumerate}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \newpage
\onecolumn
% \aistatstitle{Supplementary Materials for Type Information-Assisted Self-Supervised Knowledge Graph Refinement}
% \vspace{-3cm}
\appendix
\label{apd}
\appendixwithtoc
\input{tex/6_appendix}
\end{document}
\begin{abstract}
% \looseness=-1
Knowledge graphs serve as critical resources supporting intelligent systems, but they can be noisy due to imperfect automatic generation processes.
%
Existing approaches to noise detection often rely on external facts, logical rule constraints, or structural embeddings.
%
These methods are often challenged by imperfect entity alignment, flexible knowledge graph construction, and overfitting on structures.
%
In this paper, we propose to exploit the consistency between entity and relation type information for noise detection, resulting a novel self-supervised knowledge graph denoising method that avoids those problems.
%
We formalize \textit{type inconsistency} noise as triples that deviate from the majority with respect to type-dependent reasoning along the topological structure.
%
Specifically, we first extract a compact representation of a given knowledge graph via an encoder that models the type dependencies of triples.
Then, the decoder reconstructs the original input knowledge graph based on the compact representation.
%
It is worth noting that, our proposal has the potential to address the problems of knowledge graph compression and completion, although this is not our focus.
For the specific task of noise detection, the discrepancy between the reconstruction results and the input knowledge graph provides an opportunity for denoising, which is facilitated by the type consistency embedded in our method.
Experimental validation demonstrates the effectiveness of our approach in detecting potential noise in real-world data.
\end{abstract}
% trash
% \looseness=-1
% Knowledge graphs are vital for intelligent systems but often require refinement due to imperfections in their automatic generation, including noise detection and missing links completion.
% %
% Existing approaches to refinement often rely on external facts, logical rule constraints or topological embeddings.
% %
% However, despite their effectiveness, these methods are often challenged by imperfect entity alignment, incompatibility with flexible knowledge graph constructions, or lack of interpretability.
% %
% In this paper, we propose to exploit the consistency between entity and relation type information for refinement, resulting in a novel self-supervised method.
% %
% Specifically, we first extract a compact representation of a given knowledge graph with a sparsity constrain on the output.
% Then, the decoder reconstructs the original input knowledge graph based on the compact representation.
% %
% Therefore, both the missing and the noisy edges can be revealed by the discrepancy between the original knowledge graph and its reconstruction, which is facilitated by the type-consistency coding in our method.
% %
% Theoretically, we provide asymptotic identifiability results for the uniqueness of the compact representation.
% Experimental validation demonstrates the effectiveness of our approach in denoising and completing knowledge graphs on real-world data.
% trash
% Knowledge graphs serve as critical resources supporting intelligent systems, yet they can be noisy due to imperfect automatic generation processes, requiring refinement, including noise detection and missingness completion.
% %
% Existing approaches to noise detection often rely on external facts or strict logical rule constraints.
% %
% Despite their effectiveness, however,
% these methods are often challenged by
% imperfect entity alignment or incompatibility with flexible knowledge graph constructions.
% %
% In this paper, we propose to leverage the consistency between entity and relation type information for noise detection, which gives rise to a novel self-supervised knowledge graph denoising method that does not suffer from the alignment and incompatibility issues.
% %
% We formalize \textit{type inconsistency} noise as triples that deviate from the majority with respect to type-dependent reasoning and make use of type information to assist the self-supervised denoising framework.
% %
% % In designing this framework, we specifically draw inspiration from single image denoising, where we rely on higher-level type abstractions to construct similar ``local'' patches, enabling us to learn a generalized denoiser over a single knowledge graph.
% To be specific, we first extract a compact representation of a given knowledge graph via an encoder that models type-dependencies of triples.
% Then, the decoder reconstructs the original input knowledge graph based on the compact representation.
% % 大格局
% We would like to note that, our proposal has the potential to address the problems of knowledge graph compression and completion, although it is not our focus.
% For the specific noise detection task, the discrepancy between the reconstruction results and the input knowledge graph provides an opportunity for denoising, which is facilitated by the type consistency embedded in our method.
% Experimental validation demonstrates the effectiveness of our approach in avoiding fitting noise and detecting potential noise in real-world data.
% trash
% Unlike previous discussions of noise, which rely on external facts that face entity alignment problems or strict logical rule constraints that are incompatible with flexible knowledge graph constructions, we exploit the consistency between entity and relation type information to detect noise: we formalize \textit{type inconsistency} noise as triples that are inconsistent with the majority with respect to type-dependent reasoning.
% for noise detection that
%Our method differs from those solutions, instead, we attempt to
%
%usually structured as sets of triples containing head entities, relations, and tail entities,
% In this paper, we focus on denoising knowledge graphs by detecting potential noise.
%
% In this paper, we address the challenge of denoising knowledge graphs by identifying potential noise.
%
%Existing approaches to noise detection rely on external facts that face entity alignment challenges or strict logical rule constraints that may be incompatible with flexible knowledge graph constructions.
% version 1
% Knowledge graphs are widely used to represent external expert knowledge, where the triples are assumed to be correct.
% However, knowledge graphs can be noisy due to imperfect automatic generation processes that often lack rigorous expert oversight.
% We recognize that perceptible noise is semantically inconsistent triples, i.e., those that contradict most existing triples, or those that contradict existing triples obtained through semantic reasoning.
% Unfortunately, most existing knowledge graph embedding methods ignore this noise because they focus only on matching syntactic graph structures.
% In this work, we utilize the most accessible semantic information, i.e., the type information of entities and relations, to achieve the goal of knowledge graph denoising.
% Due to the lack of noiseless counterpart data, we draw inspiration from self-supervised single-image denoising methods that learn denoisers in a localized manner and use an autoencoder architecture.
% Specifically, we first extract a compact representation of a given knowledge graph, i.e., a subset of the triple, using an encoder that considers both local semantic and structural information.
% Then, we use the decoder to reconstruct the original input knowledge graph based on the compact representation.
% The difference between the reconstructed result and the input knowledge graph provides an opportunity for denoising, which is ensured by the semantic consistency embedded in our proposal.
% The experimental results validate the performance of our proposed approach in knowledge graph denoising and general knowledge graph completion.
% trash
% ------------- another version of abstract ----------------
% Knowledge graphs (KGs) play a crucial role in intelligent systems like chatbots and recommendation systems by providing structured expert knowledge.
% However, KGs can be noisy due to imperfect automatic generation processes and manual biases, which can be exacerbated by multi-hop connections and message-passing methods.
% This paper addresses the challenge of denoising knowledge graphs. We propose a new definition of noise in KGs as semantically inconsistent triples that contradict the majority of existing triples or those obtained through semantic reasoning.
% This definition is independent of external input and supports self-supervised denoising.
% Our approach leverages an auto-encoder architecture to extract a compact representation of the KG that retains most of the information while excluding noisy triples.
% The encoder considers both individual semantics and dependencies between triples, while the decoder reconstructs the original KG from the compact representation.
% The difference between the reconstructed and original KGs is used to identify and remove noise, ensuring semantic consistency.
% We utilize type information for entities and relations to enhance the denoising process, inspired by single image denoising techniques.
% We validate our approach through experiments, demonstrating its effectiveness in avoiding the fitting of noise triples and improving performance on KG completion tasks compared to existing embedding methods.
% Our method represents a self-supervised, information-assisted strategy for KG denoising that enhances the reliability of knowledge-based intelligent systems.
% We manage to denoise the knowledge graph by decomposing it into three mutually exclusive parts, viz,
% Noise, Compact, and Derivative, and propose a two-step method.
% First, we argue that the Derivative part is semantically dependent on the remaining part,
% i.e., it can be functionally generated from the disjunction of Compact and Noise, given the type information.
% Second, Compact and Noise can be separated by using the mutual information criterion between the extracted Derivative.
% , i.e., each link should be generatively consistent with the existence of the rest.
% More essentially, the semantic contradiction comes from the incompatability between high-level features of the connected entities and relations.
% Given these considerations, we consider using the accessible type information as such high-level information, and propose a consitency assumption as \textit{}
% In order to verify the denoising capability of the proposal, experiments are conducted on both simulation datasets and real-world knowledge graphs.
% In addition, the performance on downstream knowledge graph completion can be improved.
\section{INTRODUCTION}
\label{sec:1_intro}
Knowledge graphs are widely used to provide expert knowledge support for intelligent systems, such as chatbots, recommendation systems, etc.~\cite{guo2020kg_rec_survey, ji2021kgsurvey, pan2024kg_llm_survey}.
A set of triples is commonly used to represent a knowledge graph, where each triple contains a head entity, a relation, and a tail entity.
%
Despite the fact that knowledge graphs are used as expert knowledge, they can be noisy due to the imperfect automatic generation process, which often lacks strict expert supervision~\cite{deng2023gold, ma2023learning_denoise}.
Examples of such imperfections include computational errors during the construction of knowledge graphs from text extractions and construction bias when multiple people contribute to the cosntruction process.
These problems can be further exacerbated when using multi-hop connections over knowledge graphs, which is particularly detrimental when precise retrieval is required~\cite{guo2020kg_rec_survey, ji2021kgsurvey, chetoui2022kg_gnn_survey}.
In this paper, we will focus on denoising knowledge graphs by detecting potential noise in real-world data.
Before discussing related proposals, it is necessary to clarify what kind of noise we are trying to tackle in the context of knowledge graphs.
Previous work on commonsense knowledge graphs defines noise as triples that are inconsistent with objective facts or logical reasoning based on those facts~\cite{deng2023gold}.
However, detecting such noise is often impractical because external facts are usually missing in most knowledge graph applications, and there are entity alignment challenges if the external database is assumed to be correct~\cite{zhao2020entity_align_survey, zeng2021entity_align_survey}.
Some rule-based approaches take into account the incompleteness, conflict, and redundancy of knowledge graphs~\cite{cheng2018rule1, belth2020normal}, but their strict definition of noise, based on a rigorous formulation of logical rules, is likely to be incompatible with the flexible construction process of real-world knowledge graphs.
Other embedding-based methods attempt to generate a confidence score for each triple based on topological consistency, while potentially suffering from overfitting in the structures~\cite{zhang2022contrastive, zhang2023integrating_attr}.
% Other works roughly treat noise as false positive triples but often lack formal definitions or descriptions, leaving a large space of possible noise~\cite{neil2018interpretable_noise, zhao2019embedding_noise, dong2023contrast_denoise, ma2023learning_denoise}.
%
Given the above shortcomings, we try to find a more general and practical way to uncover the noisy triples, which should go beyond topological consistencies and external facts.
% We focus on a single knowledge graph without considering external supervision.
% We find that inconsistencies within the triples of a given knowledge graph help reveal noisy relationships, without any external supervision.
Consider a triple, \texttt{(head\_entity}, \texttt{relation}, \texttt{tail\_entity)}, from the NELL-995 dataset~\cite{xiong2017deeppath_nell}:
\texttt{(concept\_city\_murdoch, concept:agentcontrols, concept\_personasia\_news\_corporation)}.
This triple does not seem sensible because Rupert Murdoch, who the triple is supposed to represent, is a businessman, which does not logically fit the entity type \texttt{city}.
This inconsistency can be caught by noticing that the entity type \texttt{city} does not typically appear with the relation type \texttt{agentcontrols} in the data set.
Based on this discovery, in this paper, we aim to detect noise by leveraging the consistency between the type information of entities and relations.
We define \textit{type inconsistency} noise: \textit{Given a knowledge graph, type inconsistency noise is a set of triples whose type information is inconsistent with that of the majority of existing triples or with triples derived through type-dependent reasoning.}
This definition implies a reasonable assumption that most links are correct.
%
In particular, this definition of noise does not rely on any external input, allowing us to denoise the knowledge graph in a self-supervised manner.
Moreover, the type information along with the structural relations gives rise to the constraints that the legit triples should obey.
% Moreover, it provides two valuable pieces of information that we can exploit: the type information of the triples (i.e., entities and relations) and the structural dependencies between the triples, which are essential for the denoising task.
These considerations form the general idea of our proposal: a type-information assisted self-supervised knowledge graph denoising approach.
In the methodology, we introduce an auto-encoder architecture to implement this idea.
Specifically, we first employ an encoder that extracts a compact representation of the entire knowledge graph, considering only the type information of triples and their structural dependencies.
The decoder then reconstructs the original input knowledge graph based on this compact representation.
The difference between the reconstructed result and the input knowledge graph provides an opportunity for denoising, which is facilitated by the type consistency embedded in our approach. It should be noted that the proposed method has the potential to address the problems of knowledge graph compression and completion, although this is not our focus~\cite{sachan2020kgcompression_survey, zamini2022kgcompletion_survey}.
In summary, our contributions are as follows: \textbf{i)} We investigate type information as a direct and effective source of revealing noisy triples in knowledge graphs. \textbf{ii)} Accordingly, we design a type information-assisted self-supervised denoising approach. \textbf{iii)} We manage to reveal noisy triples directly from real data.
% In addition, we draw the analogy between our framework and single-image denoising techniques, in particular those that use local denoisers to derive generalized denoising patterns~\cite{quan2020self2self, huang2021neighbor2neighbor, ko2023self2self+}.
% In our approach, we use type abstraction to obtain generalized denoising patterns across different detailed triples.
% %
% Finally, we provide experimental investigation of our proposal to validate its ability to avoid fitting potential noise triples while existing knowledge graph embedding methods fail to do so, and its capacity in detecting potential noise.
% \textbf{Contributions.}
% trash
% we
% we exploit the consistency between entity and relation type information to detect noise
% In this context, noise is generally understood as triples that are inconsistent with specialized objective facts.
% what is noise in knowledge graph, xxx considers xxx, in this paper, we consider it as...
%
% we focus on the scenario when the data base is isolated from external information, and propose the perceivable noise are the semantic contraidctories; these inconsistencies cannot be detect by leveraging external knowledge
% so, we propose that
%
% a compact representation of a given knowledge graph, i.e. a subset of the triples, is extracted by an encoder that considers both individual semantics and semantic dependencies between triples.
% The decoder is then used to reconstruct the original input knowledge graph based on the compact representation.
% The difference between the reconstructed result and the input knowledge graph provides an opportunity for denoising, which is ensured by the semantic consistency embedded in our proposal.
% Specifically, the semantics we use on the knowledge graph is the type information, which for relations usually means the index itself, while for entities it is an additional abstraction of the entity index, i.e., multiple distinctively indexed entities would share the same type index.
% This type information does not require much effort, which usually comes with the dataset, but is of little use.
\section{NOTATIONS}
\label{sec:2_pre}
% \subsection{Notations}
\textbf{Knowledge graph and type information.}\quad
A knowledge graph consists of entities and the relations between them, typically represented as a set of triples: $K = \{(h_i,r_i,t_i)\}_{i=1}^n$, where each triple $(h_i,r_i,t_i)$ comprises a head entity $h_i$, a relation $r_i$, and a tail entity $t_i$ and there are $n$ triples in the graph.
Each entity is from an entity token domain $V$, i.e., $\forall i\in z(n), \ h_i, t_i \in V$.
% which could be an entity index or a proper name like \texttt{concept\_city\_murdoch}.
We use $z(n) = \{1,2,\cdots,n\}$ to denote the index set.
Similarly, each relation $r_i$ belongs to a relation domain $R$.
A knowledge graph can alternatively be represented as a three-dimensional cube, i.e., $\textbf{A} \in \{0, 1\}^{|V|\times|R|\times|V|}$, where $\textbf{A}[{h,r,t}] = 1$ if $(h,r,t)\in K$, otherwise $\textbf{A}[{h,r,t}] = 0$.
Both $(h,r,t)$ and $(h_i,r_i,t_i)$ can represent a knowledge graph triple, and the latter is preferable when the index is needed.
Moreover, in this work we focus especially on the knowledge graph that comes with type information of entities, represented by a type function $c: V\vee R\mapsto C$.
% For entity input, type function is not surjective;
The cardinality of the entity type domain is strictly less than that of the entity domain, i.e., $|c[V]| < |V|$.
% ~\footnote{Such type information is usually contained in datasets, but has received less attention in mainstream work.}.
However, in most knowledge graphs,
the type domain and the token domain are the same for relations.
Therefore, for relations, the type function is an identity mapping.
% , i.e., $\forall r \in R, c(r) = r$ and $R\subsetneq C$.
% the type domain and the token domain are indistinguishable in terms of relations,
% e.g., $r_k$ means the same whether $(h_i, r_k, t_i)$ or $(h_i, r_k, t_j)$, if $i\neq j$.
In this paper, a knowledge graph with entity type information is denoted as $G=(K,c)$ or $G=(\textbf{A},c)$.
\textbf{Other notation convention.}\quad
We use uppercase letters to denote a set, and its member is marked as lowercase or indexed, i.e., $x$ or $x_i$.
% We also abuse the use of set and domain.
The cardinality of the set $X$ is denoted by $|X|$.
Bold capital letters denote matrices or a three-dimensional cube (i.e., a collection of matrices). Bold lowercase letters denote a vector.
The entries of a given matrix are marked by using square brackets, i.e., $\textbf{X}[h,r,t]$.
Functions are denoted using lowercase letters.
% , while a script letter indicates cases where the formulation of the functions has multiple choices.
The output of a function for a singleton input is denoted as $f(x)$, and
$f[X]$ donates the case where the input and the output are set.
% Finally, the indexing set is denoted as $z(n) = \{1,2,\cdots,n\}$, where $n\in\mathbb{N}$.
% \subsection{Knowledge Graph Embedding}
% % \quad
% To enhance computational efficiency and utility in working with knowledge graphs,
% % \textcolor{blue}{To efficiently and effectively encode the structural }
% knowledge graph representation learning is commonly employed.
% The primary objective is to transform entities and relations from their discrete token domains into continuous real numbers by learning an embedding function.
% We denote the embedding function as $e_{\psi}: V\vee R \mapsto \mathbb{R}^{d}$, where $\mathbb{R}^{d}$ is a $d$-dimensional real space and ${\theta}$ is the learnable parameters.
% Here, for simplicity, we have aligned the two target domains to the same dimension, although there exists research that delves into more complex representation space modeling~\cite{cao2024knowledge_space_survey}.
% The main idea of learning the embeddings is to encode the triples that are contained in the dataset, e.g., $K$, differently from those absent from the dataset, e.g., $K' \subseteq V\times R\times V \setminus K$.
% This process first requires a scoring function to project the triples into an one-dimensional ordinal space, e.g., $\mathbb{R}$, to make comparisons.
% In this paper, we focus on the wildly used formation of
% % $s: V\times R\times V \mapsto \mathbb{R}$,
% \textcolor{blue}{$s: \mathbb{R}^d\times \mathbb{R}^d\times \mathbb{R}^d \mapsto \mathbb{R}$, where each input entry $\mathbb{R}^d$ denotes the representation of an entity or a relation.}
% % i.e., $s(k_i) \in \mathbb{R}$, where $k_i=(h_i,r_i,t_i)$.
% Choices of non-parametric scoring functions include the ones adopted in the models like DistMult~\cite{yang2015distmult} and ConvE~\cite{dettmers2018conve}; an overview of the scoring functions can be found in review literature~\cite{shen2022comprehensive}.
% %
% All the triples are scored, then, the embedding function can be obtained by optimizing over the following.
% %
% \begin{equation}
% % \mathcal{L} = \frac{1}{|K|} \sum_{K_i=(h_i,r_i,t_i)\in K} \frac{1}{|K_i'|} \sum_{(h'_i,r'_i,t'_i)} \mathcal{L}_0(s(h_i,r_i,t_i), s(h_i',r_i',t_i')) + \mathcal{R}(\theta)
% % \mathcal{L} = \frac{1}{|K|} \sum_{(h_i,r_i,t_i)\in K} \mathcal{L}_0(s(h_i,r_i,t_i), \{s(h_i',r_i',t_i'); (h_i',r_i',t_i') \in K'_i \}) + \mathcal{R}(\theta)
% \min_{\psi\in\Psi} \frac{1}{|K|} \sum_{k_i\in K} {L}_0\left(s(e_{\psi}[k_i]), \{s(e_{\psi}[k_i']); k_i' \in K'_i \}\right) + \mathcal{R}(\psi),
% \label{eq2: kge_obj}
% \end{equation}
% %
% where $K'_i \in V\times R\times V \setminus K$ is the set of triples that does not exist in the dataset, obtained from negative sampling.
% A regularization term, $\mathcal{R}(\psi)$, is often to stabilize the optimization process. %, i.e., $l^2$-norm over the representations.
% ${L}_0$ is the distance comparison function, for instance, sigmoid loss~\cite{sun2019rotate}:
% \begin{equation}
% L_0 (s_i, s_i') = -\log\sigma \left(\gamma-s_i\right) - \sum_{s_{i'_j}\in s_i'} p_i\log \textcolor{blue}{\sigma} \left(s_{i'_j}-\gamma\right),
% \end{equation}
% \looseness=-1
% where $s_i$ is the score of the a triple $k_i$, $s_k'$ is the set of the scores of all the negative triples corresponding to $k_i$, and $s_{i'_j}$ denotes the score of one specific negative triple indexed by $j$.
% Besides, $\sigma$ is the sigmoid function, $\gamma$ is the margin given as a hyper-parameter, and $p_i$ is a weight.
% Notably, there are broadly two ways to encode the triples for the embedding function $e_{\psi}$.
% The first treats each triple independently, using the embedding functions straightforward as a dictionary, and the structural dependencies between the entities and relations can only be seen as input into the overall objective function, such as many of the approaches focusing on representation space~\cite{cao2024knowledge_space_survey}.
% The other way is to encode the structural dependencies before the objective function, i.e., utilizing Message-Passing Neural Networks~(MPNN) to explicitly model the structural dependencies, like R-GCN~\cite{schlichtkrull2018rgcn} and KBGAT~\cite{nathani2019kbgat}.
% The comparisons between these two branches of work can be found in the empirical analysis work~\cite{li2023message}.
% In this work, we will deploy the MPNN-based method to explicitly model the structural dependencies between the triples.
% ---------- remark begin -----------
% %\textbf{Remark.}\quad
% Based on the above formulations, the primary goal of most existing knowledge graph embedding methods is to encapsulate all triples in the dataset through modeling procedures by adjusting the parameters of the embedding function $e_{\psi}$.
% Since there is no semantic input to the learning model, the task is essentially to decompose the topological knowledge graph cube in a way such that all entries (or triples) can be recovered through computations over the continuous representations.
% %
% This typically requires a more expressive framework with a larger parameter space and a more flexible architecture to facilitate optimization.
% However, such modeling approaches do not inherently capture noise in a semantic sense, whether in the sense of commonsense contradictory noise~\cite{deng2023gold} or our proposed \textit{type-inconsistency} noise, since topological information is orthogonal to noise signals. It's important to note that structural noise definitions over high frequency spectra on homogeneous graphs are not suitable in the context of knowledge graphs~\cite{ortega2018gsp}, mainly because nodes have different meanings and the graph itself is not complete.
% %
% Therefore, detecting noise within a knowledge graph requires additional input beyond structural information. In the remainder of this paper, we explore the feasibility of using type information from knowledge graphs to detect noise.
% ---------- remark end -----------
% From the formulation of knowledge graph embedding above, it is straightforward that the goal of the modeling is to remember the input knowledge graph, by forcing the parameters of the embedding function $\psi$ fitting the space shaped by the scoring function and the input knowledge graph.
% From those conclusions, we can derive that the neighboring relationships in knowledge graph can be useless in resolving knowledge graph embedding, when the embedding function is sufficiently expressive.
%\textcolor{blue}{However, for graph-structured data, the correlation of the edges shouldn't be ignored, since it may serve essential roles as a generative sense of knowledge graph. }
% \textcolor{red}{TODO: both noisy and missing relations, comparisons, found out that only KGC won't fit in the contradictory relations.}
% Figure~\ref{fig:sec2_pre} shows that, by directly applying the knowledge graph embedding methods over a semantically contradictory knowledge graph, they still can reconstruct the whole problematic graph.
% It indicates that the contradictory noise (or missingness) should not be unveiled from the data, however, since the existing knowledge graph embedding approaches overlooked the relationships between the triples, they still can be fitted.
% Nonetheless, our proposed relational compression representation considers the correlational message
% According to ~\cite{shen2022comprehensive}, the regularizer often has a non-convex component to guide the learning process of the relations' representation, and an $l2$-norm regularization on the whole parameter space of $\theta$.
% \textbf{Knowledge graph completion.}\quad
% Most of the knowledge graph embedding are evaluated by
\section{METHODOLOGY}
\label{sec:3_method}
\subsection{Observations and Motivation}
\looseness=-1
\textbf{Type information.}\quad
First, let us elaborate on our motivation by demonstrating the correlation between type information and triples in a knowledge graph. Here, we provide quantitative evidence to illustrate the relationship between type information and the frequency of observing the corresponding triples.
Table~\ref{tab:1_pre_stats} summarizes the statistics of three widely used datasets (with detailed descriptions provided in Section~\ref{sec:4_exp}).
The cardinality of the type domain is significantly smaller than that of the token domain, as introduced in Section~\ref{sec:2_pre}.
More importantly, we calculate the proportion of legitimate triple types~(\% LTT)—defined as the existing combinations of head entity type, relation~(type), and tail entity type—relative to all possible combinations of entity and relation types in the entire dataset, as follows:
\begin{align}
\text{\% LTT} = \frac{|\{(c(h), c(r), c(t)); (h,r,t)\in K\}|}{|c[V]|\times|c[R]|\times|c[V]|},
\end{align}
where $c$ is the type function within a given knowledge graph $G=(K,c)$.
As shown in the table, legitimate triple types represent only a small fraction of all possible triple type combinations.
To better illustrate the concentration of type distribution, Figure~\ref{fig:3_pre_rel_dist} shows the distribution of entity types for a randomly chosen relation type in the least concentrated dataset, WN18RR.
In the figure, the darker green indicates a higher frequency of the relation type occurring between the given connected entity types.
The results show that the possible entity types associated with a given relation type are very limited.
Such observations are general across different datasets, which can be found in the Appendix.
This observation suggests that as the order of the link increases, the type constraints become stricter.
For instance, given two entities with known types, the number of possible relation type combinations in the intermediate multi-hop link becomes more centrally distributed.
Consequently, if we reasonably assume that the majority of triples in a knowledge graph are correct, and therefore their triple types are also correct, we can use this concentrated distributed type information to detect noise.
Specifically, we can identify noisy triples when their type information deviates from the majority, as indicated by multi-hop constraints.
%
% \textcolor{red}{TODO: compare the order of the networks used. Here, the logic is that we can even enforce type constraints, so it is a balance between computational cost and effectiveness.}
Given the feasibility of using type information as a guide for noise detection, we propose a method to model type dependencies over triples to unveil hidden noise in knowledge graphs.
\begin{table}[]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tabular}{lccc}
\toprule
% \bf{Dataset} & \bf{NELL-995}~\cite{xiong2017deeppath_nell} & \bf{WN18RR}~\cite{bordes2013translating_wn18rr} & \bf{FB15k-237}~\cite{toutanova2015observed_fb15k} \\
\bf{Dataset} & \bf{NELL-995} & \bf{WN18RR} & \bf{FB15k-237} \\
\midrule
\# entities & 75,492 & 40,943 & 14,541 \\
\# entity types & 267 & 11 & 237 \\
\# relations~(=types) & 200 & 11 & 237 \\
\rowcolor{Gray}
\% LTT & 0.13 & 21.11 & 0.05 \\
\bottomrule
\end{tabular}
\caption{Type information of three real-world datasets.}\label{tab:1_pre_stats}
\end{adjustbox}
\end{table}
% \begin{figure}[t]
% \centering
% \begin{adjustbox}{width=0.5\textwidth}
% \includegraphics[width=0.8\textwidth, trim={0cm 0cm 0cm 0.1cm}, clip]{figs/3_pre_rel4_wn18rr.png}
% \caption{\small Distribution of entity types given a relation.}\label{fig:3_pre_rel_dist}
% \end{adjustbox}
% \end{figure}
% \begin{figure*}[t]
% \CenterFloatBoxes
% % \killfloatstyle
% \begin{floatrow}
% \ttabbox{%
% \begin{adjustbox}{width=0.5\textwidth}
% \begin{tabular}{lccc}
% \toprule
% % \bf{Dataset} & \bf{NELL-995}~\cite{xiong2017deeppath_nell} & \bf{WN18RR}~\cite{bordes2013translating_wn18rr} & \bf{FB15k-237}~\cite{toutanova2015observed_fb15k} \\
% \bf{Dataset} & \bf{NELL-995} & \bf{WN18RR} & \bf{FB15k-237} \\
% \midrule
% \# entities & 75,492 & 40,943 & 14,541 \\
% \# entity types & 267 & 11 & 237 \\
% \# relations~(=types) & 200 & 11 & 237 \\
% \rowcolor{Gray}
% \% legitimate triple types & 0.13 & 21.11 & 0.05 \\
% \bottomrule
% \end{tabular}
% \end{adjustbox}
% }
% {
% \caption{Statistics about type information of three real-world datasets.}\label{tab:1_pre_stats}%
% }
% \ffigbox[\FBwidth]{%
% \begin{adjustbox}{width=0.25\textwidth}
% \includegraphics[width=0.9\textwidth, trim={0cm 0cm 0cm 0.1cm}, clip]{figs/3_pre_rel4_wn18rr.png}
% \end{adjustbox}
% }
% {%
% \caption{\small Dist. of entity types given a relation.}\label{fig:3_pre_rel_dist}%
% }
% \end{floatrow}
% \end{figure*}
\textbf{Compact triple set and denoising.}\quad
Since the topology of the knowledge graph allows for information dependencies between triples, to model the type dependencies of triples, we need to consider the type information of each triple and propagate these dependencies along the structure of the graph.
Furthermore, reduction along these topological dependencies confirms the existence of a subset of triples on which the type information of all other triples depends.
%
Given the above analysis, we propose to extract a compact representation of the knowledge graph, i.e., a subset of all existing triples.
This compact set is to be inferred from the existing triples and used to infer other legitimate triples.
In this way, noisy triples are not kept in the compact set, since they are not type-dependent on most of the existing triples.
Moreover, for the same reason, these noisy triples cannot be constructed from the compact representation set.
%
Accordingly, we design a self-supervised architecture in which the encoder extracts the compact set based on type information and structural dependencies, and the decoder reconstructs the knowledge graph. The difference between the original knowledge graph triples and their reconstruction provides an avenue to detect type-inconsistency noise.
\begin{figure}[t]
\centering
\begin{adjustbox}{width=0.5\textwidth}
\includegraphics[width=0.8\textwidth, trim={0cm 0cm 0cm 0.1cm}, clip]{figs/3_pre_rel4_wn18rr.png}
\caption{\small Entity types distribution w.r.t. a given relation in WN18RR.}\label{fig:3_pre_rel_dist}
\end{adjustbox}
\end{figure}
\subsection{General Proposal}
% \textbf{General proposal.}\quad
Given a knowledge graph $G=(K,\textbf{A}, c)$ containing $n$ triples, i.e., $K=\{(h_i, r_i, t_i)\}_{i=1}^n$, its three-dimensional cube representation $\textbf{A}\in\{0,1\}^{|V|\times|R|\times|V|}$, and the type information function of the entities, i.e., $c: V\mapsto C$ (here we focus on the case when the relation type domain is identical to its token domain), we want to find the most compact triples that can be used to reconstruct the original graph.
To simplify the notation, we use $\mathcal{B}$ to denote the discrete three-dimensional space: $\{0,1\}^{|V|\times|R|\times|V|}$.
And we use $(h,r,t)$ to denote arbitrary triple if not otherwise specified.
We consider the following formulation.
%
\begin{subequations}
\begin{align}
& \min_{\substack{\textbf{B}\in\mathcal{B} \\ \text{supp}(\textbf{B}) \subseteq K} }||\textbf{B}||_0, \label{eq: opt_sparsity_goal}\\
&\text{s.t.}\quad \textbf{B} = \arg\min_{\substack{\textbf{B}\in \mathcal{B}\\ \text{supp}(\textbf{B}) \subseteq K\\ \theta\in\Theta}}
d(f_{\theta}(\textbf{B}, c), \textbf{A}). \label{eq: opt_recons_goal}
\end{align}
\end{subequations}
%
The above formulation corresponds to a hierarchical optimization problem~\cite{anandalingam1992hierarchical}.
The first-level objective, i.e., $||\textbf{B}||_0$, represents the $l^0$ norm of $\textbf{B}$, and $\text{supp}(\textbf{B})$ denotes the support set of the matrix $\textbf{B}$, e.g., $\text{supp}(\textbf{A}) = K$~(definition can be found in Appendix~\ref{apd-def}).
The reconstruction function that takes into account the structure of the graph and the type information of the triples is denoted as $f_{\theta}: \mathcal{B} \times c\mapsto \mathcal{B}$, with a learnable parameter $\theta$ searched from $\Theta$.
% Note that the type function can take both $V\vee R$, as we defined before, and the indices of the knowledge graph cube, i.e., \textcolor{blue}{$z(|V|+|R|)$}, as input domain.
% For the sake of conciseness, we slightly abuse the notation for both scenarios.
The reconstruction results are measured by measuring the distance of two matrices, i.e., $d: \mathcal{B} \times \mathcal{B} \mapsto \mathbb{R}$.
To ensure that the compact representation $\textbf{B}$ does not degrade into an oversimplified matrix, e.g., a zero matrix, $\textbf{0}^{|V|\times|R|\times|V|}$, rendering it meaningless, and to confirm that the reconstruction process relies on the type information $c$ rather than overfitting to the structural information $\textbf{A}$, we consider $f$ to belong to a specific function class.
% \textcolor{red}{TODO: add sparsity constraints here}
%
In other words, we want each entry in the function's output, i.e. $f_{\theta}(\textbf{B}, c)[h,r,t]$, to depend on the type information of $h, r, t$ and their topological neighbours.
This wish is consistent with the idea of message-passing based knowledge graph embedding methods, and here we implement this desired function $f$ using Relational Graph Convolutional Networks~(R-GCN)~\cite{schlichtkrull2018rgcn}.
In general, the $[h,r,t]-$entry of the output $f_{\theta}(\textbf{B}, c)$ is produced:
\begin{align}
f_{\theta}(\textbf{B}, c)[h,r,t] = s(\textbf{Z}[h],\textbf{Z}[r], \textbf{Z}[t]), \label{eq: recon}
\end{align}
where $s$ denotes any function that maps the embeddings of the triple, i.e., $\textbf{Z}[h], \textbf{Z}[r], \textbf{Z}[t] \in \mathbb{R}^{d}$ into the real range of $[0,1]$.
Consequently, the domain of the distance function $d$ extends to $[0,1]^{|V|\times|R|\times|V|} \times \mathcal{B}$.
The embeddings, i.e., $\textbf{Z}[h], \textbf{Z}[r], \textbf{Z}[t]$, are all aggregations of the propagated type information along the graph structure by using multi-layer R-GCN.
And the first-layer representations of the entities are initialized as the type information of the entities, i.e., $\textbf{Z}[h]^{(0)} = c[h]$.
%
After the initialization, each R-GCN layer outputs an updated representation of entities, for example:
\begin{align}
\textbf{Z}[h]^{(l+1)} = &\sigma ( \sum_{r\in R} \sum_{j\in \mathcal{N}_{h}^{r}} \frac{1}{c_{h,r}} \textbf{W}_r^{(l+1)} \textbf{Z}[j]^{(l)} \\
& + \textbf{W}_0^{(l+1)} \textbf{Z}[h]^{(l)} ),
\label{eq:rgcn_layer}
\end{align}
where $\sigma$ is the activation function. $\mathcal{N}_h^r$ represents the structural neighbor entities of a given entity $h$ that are connect by a specific relation type $r$.
Thus, the parameter space of $f$, i.e., $\Theta$, consists of all the transformation parameter matrices $\{\textbf{W}_{r}^{(l)}; r\in R, l=1,\cdots,L\}$, given a hyper-parameter $L$ representing the depth of the network, the self linear transformation $\textbf{W}_0^{l}$ for each layer, a learnable representation for each relation type, i.e., $\textbf{Z}[r]\in \mathbb{R}^{d}; r\in R$, and the possible parameters of $s$.
% This class models each entity as an aggregation of type information propagated along the graph structures.
% Specifically, we consider the following formulation for $f$.
%
% \begin{align}
% f_{\theta}(\textbf{B}, c)[i,j, k] = s(\textbf{Z}[i],\textbf{Z}[j], \textbf{Z}[k]), \label{eq: recon}
% \end{align}
%
% where $s$ is a non-parametric function, which transforms the representations of a triple, i.e., $(\textbf{Z}[i], \textbf{Z}[k], \textbf{Z}[j])$ into a real number score in $[0,1]$.
% This score indicates how likely the corresponding triple $[i,j,k]$ is to be reconstructed from the compact set $\textbf{B}$, such as the scoring functions used in knowledge graph embeddings~\cite{shen2022comprehensive}.
%
\subsection{Parameterization and Overall Objective}
\textbf{Parameterized search.}\quad
As presented in Eq.~(\ref{eq: opt_sparsity_goal}) and (\ref{eq: opt_recons_goal}), the search space for $\textbf{B}$ is discrete and vast, that is, $(\textbf{B}\in\mathcal{B}) \wedge (\text{supp}(\textbf{B}) \subseteq K)$, making direct optimization of this space challenging.
To facilitate computation, we consider parameterizing the search space, transforming the originally discrete space into a continuous one.
%
To achieve this goal, recall that $\textbf{B}$ is the most compact representation of $\textbf{A}$.
Hence, we naturally assume that the entry value of $\textbf{B}[h,r,t]$, which indicates whether the corresponding triple $(h,r,t)$ belongs to the compact representation of the knowledge graph, should be inferred from the neighboring connections and the type information $c$.
In simple terms, a triple is retained in the compact representation of a knowledge graph if it semantically aligns with the local connections.
We implement this by implementing a masking function on the given knowledge graph, i.e. $m: \mathcal{B} \times c\mapsto \mathcal{B}$, which determines whether a triple should be retained in the compact representation.
We use a design similar to the reconstruction function, as shown below:
\begin{align}
m_{\phi}(\textbf{A}, c)[h,r,t] = s(\textbf{H}[h],\textbf{H}[r], \textbf{H}[t]), \label{eq: mask}
\end{align}
where $\phi$ is the learnable parameter in the masking function, and its parameter space includes all parameters involved in calculating the embeddings denoted by $\textbf{H}$ and the possibly potential parameters in $s$.
The vector $\textbf{H}[h]\in \mathbb{R}^d$ is recursively calculated using Eq.~(\ref{eq:rgcn_layer}), initialized in the same way as above.
In the implementation of $s$, we deploy a Multi-Layer Perceptron~(MLP) to enhance the flexibility of inferring the compact set from all triplets.
% And $\mathtt{Cat}$ denotes the row-wise concatenation of the matrices.
Also note that the parameters of $f_{\theta}$ and $m_{\phi}$ are independent, as indicated by the different symbols used, i.e., $\textbf{Z}$ and $\textbf{H}$ are different, although their internal function classes are identical.
% Additionally, we note that the parameters of $f_{\theta}$ and $m_{\phi}$ are independent, as indicated by the different symbols used, i.e., $\textbf{Z}$ and $\textbf{H}$, even though their internal function classes are identical.
Given this efficient parameterization of the search space, the original optimization problem, as demonstrated in Eqs.(\ref{eq: opt_sparsity_goal}) and (\ref{eq: opt_recons_goal}), is modified into the following form:
%
\begin{subequations}
\begin{align}
& \min_{\phi \in \Phi} || m_{\phi}(\textbf{A}, c) ||_0, \label{eq: opt_sparsity_goal_mod}\\
& \text{s.t.}\quad m_{\phi}(\textbf{A}, c) = \arg\min_{\substack{\phi \in \Phi \\ \theta\in\Theta}}
d\left(f_{\theta}\left(m_{\phi}(\textbf{A}, c\right), c\right), \textbf{A}). \label{eq: opt_recons_goal_mod}
\end{align}
\end{subequations}
%
%
\textbf{Asymptotic discretization.}\quad
The parameterization outlined above facilitates continuous optimization using gradient descent.
However, it is worth noting that in the modified framework, i.e., Eqs.(\ref{eq: opt_sparsity_goal_mod}) and (\ref{eq: opt_recons_goal_mod}), both the reconstruction function and the masking function produce continuous output, creating a discrepancy between the original problem and its modified version.
To address this, we employ suitable techniques to discretize the output.
First, we apply a sigmoid transformation to the output of both functions to polarize the values.
Furthermore, we utilize Gumbel-Softmax~\cite{jang2017gumbel} to further discretize the output of the masking function.
This is particularly important because the output of the masking function serves as the input to the reconstruction function, which requires stricter discretization.
%
For instance, if $q_i$ represents an entry of the output of the original masking function, i.e., Eq.~(\ref{eq: mask}), the actual input to the reconstruction function is given by:
% For instance, if $q_i$ represents the output of the original masking function, i.e., Eq.~(\ref{eq: mask}) for the triple $(h_i, r_i, t_i)$, the input to the reconstruction function is given by:
\begin{align}
% \begin{adjustbox}{width=\textwidth}
% m_{\phi}(\textbf{A}, c)[h_i, r_i, t_i] =
\frac{\exp(\frac{\log(\frac{1}{1+\exp(-q_i)} + p_i)}{\tau} )}
{\exp(\frac{\log(\frac{1}{1+\exp(-q_i)} + p_i)}{\tau} ) + \exp(\frac{\log(\frac{1}{1+\exp(-(1-q_i))} + p_i')}{\tau})},
\label{eq: gumble}
% \end{adjustbox}
\end{align}
where $p_i$ and $p_i'$ are independently sampled from a Gumbel distribution with a location parameter of $0$ and a scale parameter of $1$, and $\tau$ is a hyper-parameter controlling the sparsity of the outputs.
\textbf{The overall optimization objective and the sparsity constraint.}\quad
We consider the following objective to solve the constrained optimization problem formulated in Eqs.\ref{eq: opt_sparsity_goal_mod} and \ref{eq: opt_recons_goal_mod}.
%
\begin{align}
\min_{\substack{\phi \in \Phi \\ \theta\in\Theta}}
d\left(f_{\theta}\left(m_{\phi}(\textbf{A}, c\right), c\right), \textbf{A}) + \gamma \rho(m_{\phi}(\textbf{A}, c) ), \label{eq: final_obj}
\end{align}
where $\rho(m_{\phi}(\textbf{A}, c) )$ is a sparsity regularizer with a non-zero hyper-parameter $\gamma$ controlling the strength, for replacing the original $l^0$ norm constrain, because the original one is the discrete component, making continuous optimization of the whole problem difficult.
Considering the relevant discussions on sparsity constraints~\cite{ng2024sparse}, we adopt the minimax concave penalty regularizers~\cite{zhang2010mcp}, where both the $l^1$ and $l^2$ norms are involved:
\[
\rho(\textbf{X}[i,j,k]) =
\begin{cases}
\lambda|\textbf{X}[i,j,k]|-\frac{\textbf{X}[i,j,k]^2}{2\alpha},\text{if } |\textbf{X}[i,j,k]|\leq \alpha\lambda,\\
\frac{\alpha\lambda^2}{2}, \text{ otherwise}.\\
\end{cases}
\]
\subsection{Denoising Based on the Discrepancy}
Given the objective formulated by Eq.~\eqref{eq: final_obj}, the optimized functions $m_{\phi}$ and $f_{\theta}$ can be obtained.
Respectively, $f_{\theta^*}$ is the general mechanism that generates the links by the minimal fundamental set of triples, which is marked by $m_{\phi^*}(\textbf{A}, c)$.
Obviously.
Our idea is that noisy triples should not be contained in the reconstruction $f_{\theta^*}(m_{\phi^*}(\textbf{A}, c), c)$.
Therefore, we can detect noisy triples simply by comparing the discrepancy between the original input $\textbf{A}$ and the reconstruction, with a chosen threshold $0.5$.
More specifically, given a triple $(h,r,t)$ that appears in the original knowledge graph, that is, $\textbf{A}[h,r,t] = 1$, the noise label $y((h, r, t))$ is determined in the following form:
\[
y((h,r,t)) =
\begin{cases}
1,& \text{if } f_{\theta^*}\left(m_{\phi^*}(\textbf{A}, c) \right)[h,r,t]\geq 0.5 \\
0,& \text{otherwise}.\\
\end{cases}
\]
Similarly, the task of completing the knowledge graph can be accomplished by introducing the triples that have entry values greater than or equal to the threshold.
And the compression task is already done by obtaining the fundamental representation of the knowledge graph, i.e., $m_{\phi^*}(\textbf{A}, c)$.
The sensitivity analysis of choosing the threshold is provided in the Appendix.
In addition, the diagram demonstration of the training and inference phases can be found in Figure~\ref{fig:model-train} and Figure~\ref{fig:model-denoise}.
\section{DISCUSSIONS}
\textbf{The analogy to single image denoising.}\quad
To make it more intuitive, let us draw the analogy between our knowledge graph denoising framework and single image denoising techniques~\cite{quan2020self2self, huang2021neighbor2neighbor, ko2023self2self+}.
In these methods, a self-supervised approach is employed, where no reference image is provided, and thereby, the original image is used as input to the denoiser, which outputs an image of the same size.
The assumption is that local patches of the image, regardless of their location, should preserve the same noise pattern.
This allows the denoiser to be trained on multiple local patches.
%
Similarly, our knowledge graph denoising proposal assumes a shared noise pattern at the type information level, which abstracts the token-level information we observe.
In this way, ``local patches'' can be constructed using the different token-level embodiments of the abstract type information. We would like to add that the proposed framework is expected to help in knowledge graph compression and completion as well, although we do not focus on them~\cite{zhang2020kgcompression1, sachan2020kgcompression_survey, zamini2022kgcompletion_survey}.
\textbf{Sample size and the type abstraction.}\quad
Just as in image denoising, where a larger image provides more local patches to train a more stable and generalizable denoising model (while avoiding converging to identity mapping), knowledge graph denoising also benefits from a sufficiently large knowledge graph.
Thus, our experiments focus on real-world knowledge graphs to ensure a sufficient scale for effective denoising.
%
It is crucial that the abstraction of the knowledge graph should not be too high for an effective denoising, as overly abstract type information can make it difficult for the model to fit the input graph information.
Finally, we would like to note that the compact representation proposed in our approach is essential for denoising tasks, since it reduces the chance that the model performs identity mapping by storing the most informative part of the graph, ensuring that noise is not preserved.
% trash
% \[
% y((h_i,r_i,t_i)) =
% \begin{cases}
% 1,& \text{if } t_i = \textcolor{blue}{\arg\min_{t\in V} s(\textbf{Z}[h_i], \textbf{Z}[r_i], \textbf{Z}[t]),} \\
% 0,& \text{otherwise}.\\
% \end{cases}
% \]
% \[
% y((h_i,r_i,t_i)) =
% \begin{cases}
% 1,& \text{if } t_i = \textcolor{blue}{\arg\min_{t\in V} s(\textbf{Z}[h_i], \textbf{Z}[r_i], \textbf{Z}[t]),} \\
% 0,& \text{otherwise}.\\
% \end{cases}
% \]
% \[
% y((h_i,r_i,t_i)) =
% \begin{cases}
% 1,& \text{if } t_i = \arg\min_{t\in V} \sigma(s(\textbf{Z}[h_i], \textbf{Z}[r_i], \textbf{Z}[t])), \\
% 0,& \text{otherwise}.\\
% \end{cases}
% \]
% \textcolor{blue}{The intuition behind why we fix the choice of head entity and relation and apply the ranking to the tail entity follows from our choice of the evaluation function as a simplified bilinear DistMult~\cite{yang2015distmult}, which we will elaborate in the experiment implementations. In DistMult, the transformed representation of the head entity should be close to the original representation of the tail entity.}
% \textcolor{blue}{todo: more heuristics/intuition about why it works? comparing to the ideas behind other approaches}
% According to Eq.~\eqref{eq: recon}, how likely the triple $(i,k,j)$ in the original knowledge graph (i.e., $\textbf{A}[i,k,j]=1$), can be retained in the denoised version is indicated by $f_{\theta}(\textbf{B}, c)[i, k, j] = s(\textbf{Z}[i],\textbf{Z}[k], \textbf{Z}[j])$, where $\textbf{Z}[i]$, $\textbf{Z}[k]$, and $\textbf{Z}[j]$ denote the optimized representation.
% Instead of giving a general threshold to be applied to each entry of $f_{\theta}$, which might miss the difference in each case, we introduce here a more stringent local solution as follows.
% We employ an absolute ranking method to unveil the noise.
% Specifically, for each triple in the dataset, e.g., $(h_i, r_i, t_i)$, we use the optimized representations of the head entity and the relation, i.e., $\textbf{Z}[h_i]$ and $\textbf{Z}[r_i]$.
% Given the non-parametric scoring function $s$ in Eq.~(\ref{eq: recon}), we compute the plausibility of all possible triples over the entire set of entities.
% Noise triples are identified as those where the tail entity obtains the lowest score among all entities when paired with the given head entity and relation.
% Similarly, our proposal for knowledge graph denoising assumes a shared noise pattern at the type information level, which abstracts the token-level information we observe.
% In this way, ``local patches'' can be constructed by the different token-level embodiments of the abstract type information. Just as in image denoising, where a larger image provides more local patches to train a more stable and generalizable denoising model (while avoiding converging to identity mapping), knowledge graph denoising also benefits from a sufficiently large knowledge graph.
% Thus, our experiments focus on real-world knowledge graphs to ensure a sufficient scale for effective denoising.
% %
% It is crucial that the abstraction of categories in the knowledge graph is not too high, as overly abstract categories can make it difficult for the model to fit the input graph information. Finally, we would like to note that the compact representation proposed in our approach, i.e., the output of the encoder, is essential for denoising tasks.
% This compact representation explicitly reduces the chance of the model performing identity mapping, but storing the most informative part of the graph, ensuring that noise is not preserved.
% To better understand our proposal for knowledge graph denoising, we draw inspiration from single image denoising techniques~\cite{quan2020self2self, huang2021neighbor2neighbor, ko2023self2self+}, where no reference image is provided, a self-supervised approach is typically employed.
% This involves using the original image as input to the denoiser, which outputs an image of the same size.
% The assumption is that local patches of different location of the given image should preserve the same noise pattern, allowing the denoiser to be trained on multiple local patches.
% \textbf{The analogy to single image denoising.}\quad
% To better understand our proposal for knowledge graph denoising, we draw inspiration from single image denoising techniques~\cite{quan2020self2self, huang2021neighbor2neighbor, ko2023self2self+}.
% In single image denoising tasks, where no reference image is provided, a self-supervised approach is typically employed, i.e., the original image as input to the denoiser, which outputs an image of the same size.
% The assumption is that local patches of the image should preserve the same noise pattern, allowing the denoiser to be trained self-supervised on multiple local patches.
% %
% Similarly, our proposal for knowledge graph denoising involves abstracting token information from knowledge graphs into type-level information, such abstraction allows to construct `local patches' by the different embodiment of the same type information.
% %
% Just as in image denoising, where a larger image provides more local patches to train a more stable and generalizable denoising model (while avoiding proficiency in identity mapping and thereby preserving noise), knowledge graph denoising also benefits from a large enough knowledge graph.
% Thus, our experiments focus on real-world knowledge graphs to ensure a sufficient scale for effective denoising.
% %
% It is crucial that the abstraction of categories in the knowledge graph is not too high, as overly abstract categories can make it difficult for the model to fit the input graph information. The compressed representation proposed in our approach, i.e., the output of the encoder, is essential for denoising tasks. This compressed representation reduces the likelihood of the model performing identity mapping on the abstracted category information, ensuring that noise is not preserved.
% To better understand our proposal of knowledge graph denoising, we elaborate the inspiration we draw from single image denoising~\cite{quan2020self2self, huang2021neighbor2neighbor, ko2023self2self+}.
% In single image denoising task, since no reference image is given, the remaining optional is usually self-supervised manner, i.e., extract the denoiser that with the original image as the input, and output a same size of image.
% During the process, with the assumption that the local patches should preserve the same noise pattern, then, the denoiser can be obtained by doing self-supervised training on multiple local patches.
% %
% Similarly, our proposal on knowledge graph denoising can be understood as abstracting the token information of knowledge graphs into type level, then, those `multiple local patches' can be captured as higher level type extract
% \textbf{Sample size and the type numbers.}\quad
% \textbf{How it can help in denoising and completing the graphs?}\quad
% Recall the original goal of representation learning: to extract useful patterns for downstream tasks, e.g., classification. In the scenario of knowledge graph, the goal is to reconstruct the original knowledge graph by modifying the learnable parametric embeddings to fit the formation of the scoring function.
% Therefore, the main point of this work is to propose a way to enforce the correlation between relations explicitly, for a more \textcolor{red}{consistent?} representation of the knowledge graph.
% If we still do it in an implicit way, then the embedding function is still potentially degrading into being significantly expressive to fit in any kind of dataset.
% By `consistency', here I mean the representation should be resistant to fit the contradictory noise and has the potential to fulfill the missingness.
% % \textcolor{red}{TODO: what problem do we solve, just avoiding overfitting in a principled way?}
% \input{tabs/kge_class}
%
% \textcolor{red}{TODO: three dim way of representing KG, define 3-d norm, like the way how eq.1 in yujia's paper; roughly speaking, it's a asymmetric, and semantic complete generalization of structural learning in a linear case.}
% \subsection{Relevance to Causal Discovery on Linear Structural Equivalence Model}
% It is nearly impossible to assume the compressed knowledge graph of
% \textcolor{red}{TODO: how to understand that by leveraging assumptions, causal discovery method can go beyond correlations? How can it avoid fitting in the noise? (It may leave some noise part)}
% \textcolor{red}{TODO: The compact set might not be unique, how to explain it? (Will I think it can be a unique set)}
% \subsection{Relevance to Single Image Denoising Tasks}
% % \textbf{Preliminary experiments.}\quad
% \input{tabs/3_pre_stat}
% \begin{figure}
% \centering
% \includegraphics[width=0.3\textwidth, trim={0cm 0cm 0.1cm 0.5cm}, clip]{figs/3_pre_rel4_wn18rr.png}
% \caption{Caption}
% \label{fig:enter-label}
% \end{figure}
% \textbf{Compact triple set and denoising.}\quad
% These considerations form the general idea of our proposal: the type dependencies between triples suggest the existence of a compact set of triples that encapsulates most of the knowledge graph's information. This compact set can be distilled from the existing complete set of triples and used to infer other legitimate triples. Consequently, the noise in the knowledge graph will not be part of this compact set, as it is inconsistent with the majority of the triples, and similarly, it will not appear in the reconstructed set.
% The above parameterization makes it convenient to deploy continuous optimization, and we manege to use gradient descent to solve the problem.
% However, it is noticing that, in the modified framework above (i.e., Eq.(\ref{eq: opt_sparsity_goal_mod}) and (\ref{eq: opt_recons_goal_mod})), both the reconstruction function and the masking function produces continuous outputs, leaving a gap between the original problem and the modified version.
% To enhance the incontinuity and meanwhile maintain continuous optimization, we incorporate the following two techniques to discrete the output asymptotically.
% We attach a sigmoid transformation to the output of both functions, on one hand.
% On the other hand, we utilize Gumbel-Softmax~\cite{jang2017gumbel} to further discretize the output of the masking function, since it also serves as the input of the reconstruction function, which demands discretization more strictly.
% It is a very general framework, that the type information gathering functions, i.e., Eq.~\ref{eq: mask} and Eq.~\ref{eq: recon} can be replaced by any functions that are capable of modeling the type information and the type dependencies passed by the structure of the knowledge graph.
\section{EXPERIMENTS}
\label{sec:4_exp}
To evaluate our proposal for detecting noise, we focus on uncovering noise directly from the original dataset, given that the actual distributions of noise in real-world knowledge graphs are typically unknown.
This approach, which aims to reveal noise in real-world datasets, has been relatively unexplored in existing research.
%
Our experiments focus on two sets of questions.
\textbf{RQ1}: Is the identified noise reasonable? Can other knowledge graph embedding methods achieve similar noise detection?
\textbf{RQ2}: Does the proposed method avoid fitting noisy data? How do the individual components of our proposed method contribute to this attribute?
\textbf{RQ3}: Is our proposal robust to mild corruption of type information?
Due to page limit, we save i) noise detection verification of general setting (i.e., uncover the additionally added abnormal triples from the original knowledge graph) and the results on larger-scale datasets, ii) hyper-parameter sensitivity analysis, iii)
computational complexity examination, and iv) preliminary evaluation of our proposal on knowledge graph completion and compression task in the Appendix.
\subsection{Settings}
% \looseness=-1
\textbf{Dataset and baselines.}\quad
We conduct experiments mainly on three datasets: WN18RR~\cite{bordes2013translating_wn18rr}, FB15k-237~\cite{toutanova2015observed_fb15k}, and NELL-995~\cite{xiong2017deeppath_nell}.
In addition, the larger-scale datasets are also considered, namely, ogbl-biokg~\cite{hu2020open}, DBpedia~\cite{lehmann2015dbpedia}, and Yago~\cite{mahdisoltani2013yago3}, which are put in the Appendix due to page limit.
The entity type information for the WN18RR and FB15k-237 datasets was obtained from the publicly available OpenKE project\footnote{\href{https://github.com/thunlp/OpenKE}{https://github.com/thunlp/OpenKE}}, where the entity type information is extracted based on the relation type.
For FB15k-237, we obtained the entity types from the corresponding text descriptions.
% More details about the dataset are included in our code attached to the submission.
For the baselines, here we choose embedding-based denoising approaches for fair comparisons and also due to the limit of page. For other types of baseline, please refer to the Appendix.
Among them, the most related are the MPNN-based embedding approaches: R-GCN~\cite{schlichtkrull2018rgcn}, KBGAT~\cite{nathani2019kbgat}, and the denoising methods GCNN~\cite{jia2019gcnn}, and TRUST~\cite{neil2018interpretable_noise}.
We also compare MLP-enhanced distance measurement methods, namely DistMult~\cite{yang2015distmult} and ConvE~\cite{dettmers2018conve}; Their superiority compared to vanilla representation space modeling methods has been revealed~\cite{li2023message}; therefore, we do not additionally include those methods as baselines.
%
In these methods, negative sampling is usually employed to enhance robustness.
Because we are concerned with revealing noise in real-world datasets, rather than revealing additional noise added to the original dataset, these suggestions for combating robustness are already embodied in the knowledge graph embedding methods described above.
\textbf{Implementations.}\quad
Like most MPNN-based knowledge graph embedding approaches that do not rely on strict logical rules, we typically process triples by appending a reverse version of each.
This practice fully leverages the asymmetric dependencies between triples. For example, the example we give in Section~\ref{sec:1_intro}, \texttt{(concept\_personasia\_news\_corporation, concept:agentcontrols\_reverse, concept\_city\_murdoch)} is also included in the training set of these approaches.
In our implementation, we adopt this practice, considering that type dependencies are also asymmetric. This modification further strengthens the type dependencies our method relies on.
For the hyper-parameters of the masking and reconstruction functions, we use the optimized settings suggested by recent work~\cite{li2023message}, and we provide detailed settings in Appendix~\ref{apd-hyper}.
% For the non-parametric scoring function $s$ in Eq.(\ref{eq: recon}), we use DistMult~\cite{yang2015distmult}, which often shows better performance when combined with MPNN-based approaches compared to other scoring functions.
Since our proposed method is an auto-encoder architecture based on R-GCN, the abbreviation RAE (i.e., R-GCN Auto-Encoder) denotes our proposal.
More implementation details regarding RAE and baselines are provided in the Appendix~\footnote{The link to our code repository is \href{https://github.com/sajqavril/Code-Repo-for-R-GCN-Auto-Encoder.git}{https://github.com/sajqavril/Code-Repo-for-R-GCN-Auto-Encoder.git}.}.
\textbf{Evaluation metrics.}\quad
We first focus on revealing the noise triples from real-world data, where the true error label is unknown. In this case, we report the number of uncovered noisy triples for \textbf{RQ1} and \textbf{RQ3}, which is computed as: $\#E = |\{f_{\theta^*}\left(m_{\phi^*}(\textbf{A}, c) \right)[h,r,t]\geq 0.5 : \textbf{A}[h,r,t] = 1\}|$.
In the Appendix, we elaborate the metrics we use for evaluating the performance of artificially added error triples (i.e., RQ i)), when all the observed triples are assumed to be correct and the noise labels are known, including true negative rate given a bandit.
% \textcolor{blue}{
% \subsection{Simulation Experiments}
% To precisely evaluate the features that we claimed for the proposal in the previous sections, we simulate some knowledge graph data sets as follows.
% }
\input{tabs/4_exp1_noise_detection}
\input{tabs/4_exp_rgcnae-SP0.1-14}
\subsection{Experimental Results on Real-World Data}
\looseness=-1
\textbf{RQ1: noise detection and case study on NELL-995.}\quad
We employ the reconstruction discrepancy-based noise detection method outlined in Section~\ref{sec:3_method} to identify potential noise in three real-world datasets.
Table~\ref{tab:exp1_noise_detection} shows that our proposed method consistently detects a stable number of noisy triples with lower variance, except for WN18RR.
A possible reason is that this dataset is characterized by overly abstract type information (as shown in Table~\ref{tab:1_pre_stats}), which poses challenges to our proposal to effectively fit the triples.
%
In contrast, the baselines either vary widely in the number of detected noisy triples, such as R-GCN, or converge on nearly all the triples, leaving little room for denoising, such as DistMult, KBGAT, GCNN, and TRUST.
We also include the DistMult training process in Appendix~\ref{apd-exp1-process} to confirm its fit to all triples.
%
Table~\ref{tab:exp4-rgcnae-sp0.1-ep14} shows the noise detected from NELL-995 by our proposal.
For readability, we remove the ``concept'' prefix.
Obvious noise is highlighted (some non-noise triples are omitted; please refer to Appendix~\ref{apd-full-tab} for the complete list).
% The scores are calculated using the DistMult scoring function with the embeddings from the decoder, followed by a sigmoid transformation. For instance, a triple $(h_i,r_i,t_i)$ has a score of $\sigma(s(\textbf{Z}[h_i], \textbf{Z}[r_i] \textbf{Z}[t_i]))$, where $\sigma$ denotes the sigmoid transformation, and $s$ is the scoring function.
%
In particular, most noise triples are detected by their reverse counterparts.
For example, the noisy triple \texttt{(ceo\_robert\_iger}, \texttt{agentcontrol}, \texttt{person\_disney)} is originally captured by its reverse counterpart \texttt{(person\_disney}, \texttt{agentcontrol\_reverse}, \texttt{ceo\_robert\_iger)}.
This finding further supports our approach of reversing links to better utilize the type constraints contained in the triples.
\input{tabs/apdx_robust_correpted_type}
\looseness=-1
\textbf{RQ2: avoiding fitting on noise.}\quad
As we analyze in Section~\ref{sec:3_method}, the triples are distributed centralized with respect to the types. Generally speaking, if a triple has a type that agrees with the majority of the triples' type, then it is less likely to be a noisy triple.
Given that the ground truth of noisy triples is unknown in real-world datasets, we indirectly demonstrate how the model avoids fitting noise by showing how well it fits the more frequently occurring types of triple.
In Figure~\ref{fig: exp2_fit_noise}, we compare the performance of R-GCN, our proposed RAE, RAE without type information guidance (RAE w/o T), and RAE without Gumbel-Softmax (RAE w/o GS).
RAE w/o T treats each entity as a different token like R-GCN, maintaining the auto-encoder framework. RAE w/o GS removes the Gumbel-Softmax component.
Each point in the figure represents a triple type, and the y-axis score for each type is the average score of all triples of that type minus the scores of 10 randomly sampled negative triples (created by replacing the tail entities).
For example, given a triple type $(c_h,r,c_t)$, the score is calculated as:
\begin{align}
\#F = \frac{1}{|S|} \sum_{(h,r,t)\in S}(\hat{\textbf{A}}[h,r,t] - \frac{1}{10}\sum_{(h',r',t')\in S'}\hat{\textbf{A}}[h',r',t']),
\end{align}
where $S=\{(h,r,t) : c[h] = c_h, c[r]=r, c[t]=c_t\}$ marks the set of all the triples that have the type of $(c_h,r,c_t)$, and $\hat{\textbf{A}}=f_{\theta^*}(m_{\phi^*}(\textbf{A}, c), c)$ is the reconstruction.
$(h',r',t')$ is the corresponding corrupted negative triple, each of which is from $S'$, whose entity or relation is randomly corrupted.
% $\frac{1}{|K_{hrt}|} \sum_{k_{hrt}} s(k_{hrt}) - \frac{1}{10}\sum_{k_{hrt}'} s(k_{hrt}')$, where $K_{hrt}$ denotes all triples of the specific type, $k_{hrt}$ is such a triple, and $k_{hrt}'$ is a corresponding negative triple.
From the results, we find that only the complete RAE model effectively captures high-frequency type triples.
The comparison between R-GCN and RAE w/o GS shows that the asymptotic discretization achieved through the Gumbel-Softmax contributes to better capturing the type constraints among the triples.
The procedure of drawing this picture is put in the Appendix.
% As shown in Figure~\ref{fig: exp2_fit_noise}, we compare the performance of R-GCN, our proposal RAE, RAE w/o T, and RAE w/o GS.
% RAE w/o T means the original RAE with removing the type information guidance, but treat each entity as different tokens like R-GCN. Compared with R-GCN, the auto-encoder framework is maintained.
% RAE w/o GS denotes the variant of RAE when the Gumbel-Softmax is removed.
% Each scattered points on the figure represents a triple type, and the y-axis score of the corresponding triple type is calculated by the average of the scores of all the triples of the given type minus $10$ randomly sampled negative triples by replacing the tail entities.
% From the results, we find only the RAE with all the components ensures to model the high-frequency type triples well. And from the comparison between R-GCN and RAE w/o GE, we can find that the asymptotic discretization of getting the compact representation contributes to better capture the type constrain among the triples.
\textbf{RQ3: robustness for corrupted type information}
Since we explicitly reveal the type-inconsistent noise in our method, the robustness of our proposal with respect to type corruption is worth investigating.
Consequently, in Table~\ref{tab:rebuttal-robustness-type} we compare the performance ($\#E$) when different fractions of the entitytype labels are corrupted, from $0$ to $1.0\%$, where we find that the proposal is quite robust to the acceptable fraction of corruption.
\begin{figure*}
\centering
\includegraphics[width=0.95\textwidth, trim={1cm 0cm 0cm 0cm}, clip]{figs/exp2-noise-fiting.png}
\vspace{-0.3cm}
\caption{\small Comparisons of different methods on fitting triples with various type frequencies ~(\#F).}
\label{fig: exp2_fit_noise}
\end{figure*}
% trash
% \subsection{Asymptotical Identifiability of the Compressed Knowledge Graph}
% Conditions: expressiveness of the masking function and the reconstruct function, preventing degradation (resulting the decoder to be a sole KGE model), sparsity constrain, and type constrain.
% \subsection{Attempting to Infer Type Information by Unsupervised Learning \textcolor{red}{Not sure if we can finish this. }}
% For the page limits, we put the ablation analysis of \textbf{RQ3} in the Appendix~\ref{apd-exp-ablation}.
% \textbf{RQ3: ablation studies.}\quad
\begin{table}[t]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tabular}{lccc}
\toprule
% \textbf{Type} &
\textbf{Method} & \textbf{NELL-995} & \textbf{WN18RR} & \textbf{FB15k-237} \\
\midrule
% \multirow{2}{*}{MLP-based}
DistMult & $1.50 \pm 1.58$ & $0.50 \pm 0.85$ & $3.20 \pm 1.23$ \\
ConvE & $1.90 \pm 0.24$ & $1.22 \pm 0.24$ & $4.60 \pm 3.24$ \\
% \midrule
% \multirow{3}{*}{MPNN-based}
KBGAT & $0.50 \pm 0.76$ & $2.83 \pm 1.72$ & $15.00 \pm 22.49$ \\
R-GCN & $1184.50 \pm 675.65$ & $442.00 \pm 124.45$ & $343.00 \pm 87.88$ \\
GCNN & $0.65 \pm 0.27$ & $2.25 \pm 1.90$ & $12.23 \pm 3.44$\\
TRUST & $1.84 \pm 0.73$ & $3.15 \pm 1.55$ & $13.51 \pm 15.43$ \\
\cellcolor{Gray}RAE~(Ours) & \cellcolor{Gray}$25.00 \pm 1.00$ & \cellcolor{Gray}$49.25 \pm 42.17$ & $120.67 \pm 5.03$\cellcolor{Gray} \\
% & \cellcolor{Gray}RAE w/o T & & & \\
% & \cellcolor{Gray}RAE w/o GS & $32.50 \pm 1.50$ & $1.20 \pm 0.53$ & \\
\bottomrule
\end{tabular}
\end{adjustbox}
\caption{\small Comparisons of the number of detected noise among different approaches~($\#E$).}
\label{tab:exp1_noise_detection}
\end{table}
\begin{table*}[h]
\killfloatstyle
\centering
\caption{\small Detected noise from the NELL-995 dataset~(best viewed in color).}
\begin{adjustbox}{width=0.8\columnwidth}
\begin{tabular}{ccc}
\toprule
\textbf{Head Entity} & \textbf{Relation} & \textbf{Tail Entity} \\
\midrule
person\_larry\_page & agentcontrol & university\_google \\
stateorprovince\_idaho & stateorprovinceisborderedbystateorprovince & stateorprovince\_south\_dakota \\
\rowcolor{pink_bg}
personasia\_news\_corporation & agentcontrol & city\_murdoch \\
\rowcolor{Gray}
\multicolumn{3}{c}{Eight non-noisy triples have been omitted due to page limitations.}\\
% biotechcompany\_delta\_air\_lines\_inc & organizationterminatedperson & journalist\_richard\_anderson & 0.0262 \\
% magazine\_gucci & organizationterminatedperson & ceo\_domenico\_de\_sole & 0.0289 \\
% stateorprovince\_idaho & stateorprovinceisborderedbystateorprovince & visualizablescene\_washington & 0.0337 \\
% stateorprovince\_idaho & stateorprovinceisborderedbystateorprovince & stateorprovince\_oregon & 0.0410 \\
% sportsteam\_detroit\_tigers & teamplayssport & sport\_baseball & 0.0419 \\
% person\_wendy001 & persongraduatedfromuniversit & university\_state\_university & 0.0433 \\
% musicartist\_the\_jam & agentcollaborateswithagent & male\_bruce\_foxton & 0.0449 \\
% % terroristorganization\_popular\_front\_for\_the\_liberation\_of\_palestine & agentcontrol & terroristorganization\_george\_habash & 0.0472 \\
% bank\_bear\_stearns & organizationterminatedperson & ceo\_james\_cayne & 0.0496 \\
\rowcolor{pink_bg}
ceo\_robert\_iger & agentcontrol & person\_disney \\
company\_apple002 & headquarteredin & city\_london\_city \\
\rowcolor{pink_bg}
personmexico\_m\_s & agentcontrol & director\_stuart\_rose \\
% school\_shichahai\_school & agentcontrol & island\_zhang & 0.0643 \\
% website\_technorati & competeswith & blog\_google & 0.0766 \\
% sportsteam\_arizona\_state\_sun\_devils & agentcollaborateswithagent & personmexico\_ncaa & 0.0785 \\
% city\_erie & proxyfor & stateorprovince\_pennsylvania & 0.0818 \\
\rowcolor{Gray}
\multicolumn{3}{c}{Four non-noisy triples have been omitted due to page limitations.}\\
\rowcolor{pink_bg}
profession\_parts & proxyfor & book\_new \\
\rowcolor{pink_bg}
sportsleague\_irl & agentcontrol & personmexico\_tony\_george \\
\rowcolor{Gray}
\multicolumn{3}{c}{Four non-noisy triples have been omitted due to page limitations.}\\
% professor\_richard\_stallman & personleadsorganization & nonprofitorganization\_free\_software\_foundation & 0.1078 \\
% website\_cnn\_\_fox & competeswith & newspaper\_times & 0.1084 \\
% ceo\_william\_r\_\_klesse & agentcontrol & petroleumrefiningcompany\_valero\_energy & 0.1361 \\ company\_pimco & agentcontrol & wine\_gross & 0.1376 \\
% website\_youtube & competeswith & website\_yahoo & 0.1922 \\
\rowcolor{pink_bg}
movie\_repo & synonymfor & airport\_epel \\
recordlabel\_roc\_a\_fella & organizationterminatedperson & ceo\_damon\_dash \\
\bottomrule
\end{tabular}
\end{adjustbox}
\label{tab:exp4-rgcnae-sp0.1-ep14}
\end{table*}
% university\_google & agentcontrols\_reverse & person\_larry\_page & 0.0065 \\
% stateorprovince\_south\_dakota & stateorprovinceisborderedbystateorprovince\_reverse & stateorprovince\_idaho & 0.0096 \\
% \rowcolor{pink_bg}
% city\_murdoch & agentcontrols\_reverse & personasia\_news\_corporation & 0.0113 \\
% journalist\_richard\_anderson & organizationterminatedperson\_reverse & biotechcompany\_delta\_air\_lines\_inc\_ & 0.0262 \\
% ceo\_domenico\_de\_sole & organizationterminatedperson\_reverse & magazine\_gucci & 0.0289 \\
% visualizablescene\_washington & stateorprovinceisborderedbystateorprovince\_reverse & stateorprovince\_idaho & 0.0337 \\
% stateorprovince\_oregon & stateorprovinceisborderedbystateorprovince\_reverse & stateorprovince\_idaho & 0.0410 \\
% sport\_baseball & teamplayssport\_reverse & sportsteam\_detroit\_tigers & 0.0419 \\
% university\_state\_university & persongraduatedfromuniversity\_reverse & person\_wendy001 & 0.0433 \\
% male\_bruce\_foxton & agentcollaborateswithagent\_reverse & musicartist\_the\_jam & 0.0449 \\
% terroristorganization\_george\_habash & agentcontrols\_reverse & terroristorganization\_popular\_front\_for\_the\_liberation\_of\_palestine & 0.0472 \\
% ceo\_james\_cayne & organizationterminatedperson\_reverse & bank\_bear\_stearns & 0.0496 \\
% \rowcolor{pink_bg}
% person\_disney & agentcontrols\_reverse & ceo\_robert\_iger & 0.0561 \\
% city\_london\_city & headquarteredin\_reverse & company\_apple002 & 0.0569 \\
% director\_stuart\_rose & agentcontrols\_reverse & personmexico\_m\_s & 0.0606 \\
% island\_zhang & agentcontrols\_reverse & school\_shichahai\_school & 0.0643 \\
% blog\_google & competeswith\_reverse & website\_technorati & 0.0766 \\
% personmexico\_ncaa & agentcollaborateswithagent\_reverse & sportsteam\_arizona\_state\_sun\_devils & 0.0785 \\
% stateorprovince\_pennsylvania & proxyfor\_reverse & city\_erie & 0.0818 \\
% \rowcolor{pink_bg}
% book\_new & proxyfor\_reverse & profession\_parts & 0.0921 \\
% personmexico\_tony\_george & agentcontrols\_reverse & sportsleague\_irl & 0.0952 \\
% nonprofitorganization\_free\_software\_foundation & personleadsorganization\_reverse & professor\_richard\_stallman & 0.1078 \\
% newspaper\_times & competeswith\_reverse & website\_cnn\_\_fox & 0.1084 \\
% petroleumrefiningcompany\_valero\_energy & agentcontrols\_reverse & ceo\_william\_r\_\_klesse & 0.1361 \\
% wine\_gross & agentcontrols\_reverse & company\_pimco & 0.1376 \\
% website\_yahoo & competeswith\_reverse & website\_youtube & 0.1922 \\
% \rowcolor{pink_bg}
% airport\_epel & synonymfor\_reverse & movie\_repo & 0.2216 \\
% ceo\_damon\_dash & organizationterminatedperson\_reverse & recordlabel\_roc\_a\_fella & 0.2286 \\
\begin{table}[t]
\centering
\begin{adjustbox}{width=\textwidth}
\begin{tabular}{cccc}
\toprule
Corrupt. ratio & NELL-995 & WN18RR & FB15k-237 \\
\midrule
w/o & $25.00 \pm 1.00$ & $49.25 \pm 42.17$ & $120.67 \pm 5.03$ \\
w/ 0.01\% & $23.37 \pm 1.37$ & $45.73 \pm 51.32$ & $118.94 \pm 3.11$\\
w/ 0.1\% & $26.09 \pm 2.02$ & $44.25 \pm 52.87$ & $124.85 \pm 6.36$\\
w/ 1.0\% & $21.57 \pm 4.29$ & $52.36 \pm 58.66$ & $110.61 \pm 10.30$\\
\bottomrule
\end{tabular}
\caption{\small Robustness w.r.t. corrupted type labels~($\# E$).}
\label{tab:rebuttal-robustness-type}
\end{adjustbox}
\end{table}
% \newpage
\section{RELATED WORK}
\label{sec:7_related}
\textbf{Knowledge graph denosing.}\quad
There are mainly three types of methods to deal with noise in knowledge graphs.
The first is the use of logical rules, which considers the noisy triples as those that violate the predefined rules~\cite{galarraga2013rule2, pellissier2017rule3, cheng2018rule1}.
However, these methods make too strict assumptions about the correct triples of following the logical reasoning rules, which may not be compatible with real-world knowledge graph constructions.
Except for such hard rule constraints, some work exploits soft rules, that is, inductive combinations of the rules defined by the type or attributes information of the triples~\cite{belth2020normal}.
In both cases, it is still possible that there are unknown additional constraints between edges other than explicit rules, so it is desirable to develop an automatic approach to discover all such constraints from the data.
% For instance, redundancies and incompleteness also contain correct knowledge, when logical rules are not followed.
%
Recently, a framework for denoising common-sense knowledge graphs based on pre-train language models~\cite{deng2023gold} has emerged, aiming to use external knowledge to denoise common-sense knowledge graphs.
For such methods to work, except for the assumption that the additional supervision input is correct, it requires a high quality of entity alignment for the external knowledge to help, and the cost of obtaining such knowledge is not trivial.
%
The third class of methods uses knowledge graph embedding techniques to give rise to a confidence score (sometimes called trustworthiness) of the triples, which is generated based on the structure of the knowledge graph~\cite{paulheim2015emb1,neil2018interpretable_noise,jia2019emb2,xu2022emb3,zhang2022contrastive, zhang2023integrating_attr}. These works either rely on topological information without semantic support, which may lead to overfitting on the structure if a given knowledge graph (e.g., see the results shown in Table~\ref{tab:exp1_noise_detection}). Or, expensive attributes of the entities are required~\cite{zhang2023integrating_attr}, which is not feasible for most real-world scenarios.
\textbf{Remark.}\quad
From the related work discussed, except for those that require external knowledge, we can see that many of them are trying to reveal noise by imposing regularities among the triples, by logical rules~\cite{cheng2018rule1, belth2020normal}, or by extracting the invariant among the changes~\cite{dong2023active, zhang2022contrastive}.
In addition, the importance of semantic information beyond the topological relationships captured by embedding-based approaches~\cite{zhao2019embedding_noise} has been pointed out~\cite{zhang2023integrating_attr, belth2020normal}.
Therefore, our proposal integrates these merits and uses type information to reveal the noisy triples by enforcing type-consistent constraints on knowledge graphs.
Finally, our explicit assumptions about the noise we are trying to tackle allow us to evaluate directly on real knowledge graphs without error labels, which is harder than revealing random false positive triples.
% where negative samples are constructed and the contrast between them and those appearing in the given knowledge graphs is increased by using different training techniques and embedding variants.
%
% \textcolor{blue}{Additionally, it is worthy to mention that this work is still rule-based, even though the higher-level type information is used - strict constraints on the form of the rules are still applied. These constraints are critical to their proposal: all token-level triples are encoded based on the rules, and the noisy ones can be revealed based on whether the triples are expensive to encode. The format of the rules is explicitly defined, and they are assumed to be recursively defined. Thus, how far the information can lead us to the underlying rules is strongly limited by the function class (i.e. the way the rules are formulated). However, it is also possible that there are unknown additional constraints between edges other than explicit rules, so it is desirable to develop an automatic approach to discover all such constraints from the data. \cite{belth2020normal}}
%
% In contrast, our approach makes no assumptions about the distribution of noise triples and does not artificially introduce noise.
% Instead, it focuses solely on detecting noise present in real-world datasets.
% It is worth noting that almost all related literature overlooks the noise inherent in real-world datasets, opting instead to inject synthetic noise and evaluate their performance on these additional noisy triples.
% This approach can introduce selection bias in the noise distribution between training and test sets, as the added noise in both sets is generated from a large set of false positive triples.
% trash
% \textbf{Knowledge graph completion.}\quad
% %
% Knowledge graph completion is the most common downstream task to evaluate the knowledge graph embedding functions or models.
% It assumes the existing knowledge graph is incomplete, i.e., some triples are missing, and
% by learning low-dimensional representations of the relations and entities on a given knowledge graph to predict possible missing links on itself.
% Due to the lack of ground truth of missingness, the current pipeline is to divide the triples in to training, validation, and test sets first; then, finish the task in an unsupervised manner.
% %
% There are two main streams of knowledge graph representation learning methods, with respect to how the dependencies between the triples are modelled.
% One considers the dependencies by using an explicit scoring function only, which can be further divided into bi-linear and distance-based methods by whether the relations are modelled as matrices or vectors~\cite{li2022star, cao2024knowledge_space_survey}.
% The other type incorporates more expressive neural networks to enhance modeling the dependencies between the triples, before input the representations into the scoring functions.
% In this sort, MPNN-based approaches are often seen, such as R-GCN~\cite{schlichtkrull2018rgcn}, KGAT~\cite{wang2019kgat}, KBGAT~\cite{nathani2019kbgat}.
% Compared to the first type, they are more flexible for including more dependencies.
% \textbf{Tackling noise on knowledge graphs.}\quad
% Additionally, our approach is based on more formal and general assumptions about noise in knowledge graphs, specifically type-inconsistency noise. This foundation makes our method more consistent with the underlying nature of noise in real-world data and enhances its applicability and reliability.
% It is worth noting that almost all related literature ignores to reveal the noise from the original real-world dataset, instead they project noise into the dataset and evaluate their performance on detecting those additional triples.
% In such a way, there might be a selection bias in the noise distribution in the training and test sets, since the noise added in the training process and the noise added in the test set are all generated from a large set of false positive triples.
% Therefore, in this work, we do not make any assumptions about the distribution of noise triples, so we do not construct the noise ourselves.
% Instead, we focus on detecting noise in real-world datasets.
% Furthermore, our proposal is based on more formal and general assumptions about noise in knowledge graphs, i.e., type-inconsistency noise, making our approach more consistent and reasonable with these assumptions.
% trash
% \textbf{Logical rules extraction.}\quad
% \textbf{Other knowledge graph denoising approaches.}\quad
\section{CONCLUSIONS}
\label{sec:5_con}
% This work investigates the feasibility of incorporating type information to unveil the noise of knowledge graph.
% Given the definition of type-inconsistency noise, we propose a type-information assisted self-supervised approach to accomplish knowledge graph denoise. This approach enjoys the ease of access to type information in the dataset, and shows capacity in avoiding adapting to the noisy triples and detecting potential noise in real-world dataset.
This work investigates the feasibility of incorporating type information to identify noise in knowledge graphs.
Given the definition of type-inconsistency noise, we propose a self-supervised approach based on type information to denoise knowledge graphs.
This approach leverages the readily available type information in datasets and demonstrates its ability to avoid fitting noisy triples and effectively detect potential noise in real-world datasets.
\textbf{Limitations and future work.}\quad
Our proposal still faces challenges in distinguishing isolated facts, such as triples that are independent of existing triples, and noise of type inconsistency, which is consistent with our analysis. More research is needed to differentiate these facts, possibly by analyzing the distributional differences between the two sets.
%
In scenarios where the embedded type information in the dataset is limited—either in quantity or quality—we plan to explore multi-hop combinations of relations and entity types to enhance the support from type information. In addition, the interpretability of the masking and reconstruction functions needs to be improved.
% \textcolor{blue}{cannot deal with missing entities}
% Future work could include approximate linearization analysis and asymptotic identifiability of the compact representations to better understand and refine these functions.
% Our proposal still has difficulties in distinguishing isolated facts, i.e., the triples independent from existing triples, and type-inconsistency noise, which is consistent with our definition and corresponding proposal of this paper, but further inverstigation to remove these facts are needed, i.e., distributionzl difference between those two sets.
% For the scenarios when the embeded type information in the dataset is limited, i.e., the number of types are limited or of low-quality, we will try to explore the combinations of multi-hop combinations of relations and entities type to improve the support from type information.
% In addition, the interpretation of the mask and reconstruction functions should be further improved, e.g., approximate linearization analysis and asymptotic identifiability of the compact representations.
\section{ACKNOWLEDGEMENTS}
All the authors would like to thank the anonymous reviewers for their generous feedback. We would also like to acknowledge the support from NSF Award No. 2229881, AI Institute for Societal Decision Making (AI-SDM), the National Institutes of Health (NIH) under Contract R01HL159805, and grants from Quris AI, Florin Court Capital, and MBZUAI-WIS Joint Program.
\setcounter{theorem}{0}
\section{MORE EXPERIMENTS ON DENOISING TASK}
\subsection{Experimental Settings}
% \subsubsection{Datasets}
% \subsubsection{Baselines}
% \subsubsection{Link to the Implementation of RAE}
% The link to the overall code repository is provided
\subsubsection{Hyper-Parameters Settings}
\label{apd-hyper}
In the experiments, we refer to the framework proposed by the analytical work between MPNN-based and MLP-based knowledge graph embedding framework~\cite{li2023message}.
The hyper-parameters of the models are listed as below:
\begin{itemize}
\item R-GCN:
\begin{itemize}
\item FB15k-237: negative\_sampling=10, score\_function=DistMult, num\_blocks=100, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, hidden\_dropout=0.1.
\item NELL-995: negative\_sampling=10, score\_function=DistMult, num\_blocks=100, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, hidden\_dropout=0.1.
\item WN18RR: negative\_sampling=10, score\_function=DistMult, num\_blocks=100, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, hidden\_dropout=0.1.
\end{itemize}
\item RAE~(Ours):
\begin{itemize}
\item FB15k-237: negative\_sampling=10, score\_function=DistMult, num\_blocks=100, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, hidden\_dropout=0.1, sparsity\_constrain=0.5.
\item NELL-995:negative\_sampling=10, score\_function=DistMult, num\_blocks=100, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, hidden\_dropout=0.1, sparsity\_constrain=0.5.
\item WN18RR: negative\_sampling=10, score\_function=DistMult, num\_blocks=100, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, hidden\_dropout=0.1, sparsity\_constrain=0.5.
\end{itemize}
\item KBGAT:
\begin{itemize}
\item FB15k-237: negative\_sampling=0, score\_function=ConvE, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, num\_heads=2, hidden\_dropout=0.3.
\item NELL-995: negative\_sampling=0, score\_function=ConvE, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, num\_heads=8, hidden\_dropout=0.3.
\item WN18RR: negative\_sampling=0, score\_function=ConvE, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, gcn\_layer=2, num\_heads=8, hidden\_dropout=0.3.
\end{itemize}
\item MLP-DistMult:
\begin{itemize}
\item FB15k-237: negative\_sampling=0, score\_function=DistMult, learning\_rate=0.0001, batch=512, l2=0, num\_workers=3, layer=2, hidden\_dropout=0.05.
\item NELL-995: negative\_sampling=0, score\_function=DistMult, learning\_rate=0.0001, batch=512, l2=0, num\_workers=3, layer=2, hidden\_dropout=0.05.
\item WN18RR: negative\_sampling=0, score\_function=DistMult, learning\_rate=0.0001, batch=512, l2=0, num\_workers=3, layer=2, hidden\_dropout=0.05.
\end{itemize}
\item MLP-ConvE:
\begin{itemize}
\item FB15k-237: negative\_sampling=0, score\_function=ConvE, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, layer=1, hidden\_dropout=0.3.
\item NELL-995: negative\_sampling=0, score\_function=ConvE, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, layer=1, hidden\_dropout=0.3.
\item WN18RR: negative\_sampling=0, score\_function=ConvE, learning\_rate=0.001, batch=512, l2=0, num\_workers=3, layer=1, hidden\_dropout=0.3.
\end{itemize}
\end{itemize}
In addition, for our method, the hyper-parameters $\alpha$ and $\lambda$ in the minimax concave regularizer are set to $10$ and $1$, respectively.
For the strength of the sparsity constrain, we conducted comparisons between $\{0.1, 0.5, 1.0\}$ without observing significant differences, and we chose $0.5$ as the default setting when no specific note is provided.
In addition, for all the reported numbers, we ran the experiments five times independently with random seeds chosen from $\{41504, 42, 0, 1, 2\}$ and calculated the mean and variance values.
Exceptions were made for the case study, as it is challenging to calculate the average and variance across different runs.
However, we additionally provide the results from other runs, see the Tables below, where we found the differences between different runs were not significant, as demonstrated in Table~\ref{tab:exp1_noise_detection}.
\subsubsection{Computation Architecture and Optimization Methods}
On the hardware side, all experiments were conducted on a machine equipped with a 24 vCPU Intel(R) Xeon(R) Platinum 8352V CPU @ 2.10GHz and 2 GPUs (RTX 4090, each with 24GB of memory).
On the software side, CUDA 12.1 was used, and the computational platform primarily consisted of PyTorch 2.1.0 and Python 3.10 running on Ubuntu 22.04.
All optimization problems were solved using the Adam optimizer with a default weight decay of 0.00005.
The learning rate varied across different baselines and datasets, and detailed learning rate settings are provided in the appended hyper-parameters settings section~\ref{apd-hyper}.
\subsection{Comprehensive Visualization of the Type Distribution}
\label{apd-more-type}
\begin{figure}[h]
\centering
\includegraphics[width=\textwidth, trim={7cm 3cm 6cm 4cm}, clip]{figs/supplymentary_wn18rr_rel8_type_distribution.png}
\label{fig:pre_rel_dist_more_type}
\caption{Eight relation type distributions on WN18RR dataset.}
\end{figure}
Except for Figure~\ref{fig:3_pre_rel_dist}, which we showed to motivate our proposal in the main content, Figure~\ref{fig:pre_rel_dist_more_type} shows more relation type distributions corresponding to the entity types, where we can see the legitimate relation types are tightly distributed across all possible combinations of entity types.
\subsection{Converging Process of Compared Methods}
\label{apd-exp1-process}
As shown in Figure~\ref{fig:apd-distmult-converge}, Figure~\ref{fig:apd-kgbat-converge} and Figure~\ref{fig:apd-rgcnae-converge}, the convergence processes of the difference methods are compared when applied to the three data sets, i.e. WN18RR, NELL-995 and FB15k-237. From the results we can observe that both DistMult and KBGAT suffer from an overfitting problem, as the number of unfitted triples tends to zero, especially DistMult. This confirms our previous claim that embedding-based methods tend to fit all topological information and cannot effectively use higher-level semantic information, e.g. type information, to detect noise.
% \subsection{Comparisons of R-GCN with Other Relational Encoders}
\begin{figure}[h]
\centering
\subfigure[Convergence process of DistMult]{
\includegraphics[width=\textwidth]{figs/exp1_distmult_type_converge.png}
\label{fig:apd-distmult-converge}
}
\centering
\subfigure[Convergence process of KBGAT]{
\includegraphics[width=\textwidth]{figs/exp1_kbgat_type_converge.png}
\label{fig:apd-kgbat-converge}
}
\centering
\subfigure[Convergence process of RAE]{
\includegraphics[width=0.7\textwidth]{figs/exp1_rgcnae_type_converge.png}
\label{fig:apd-rgcnae-converge}
}
\caption{Comparisons of convergence processes of different models}
\label{fig:apd-converge-three}
\end{figure}
\subsection{Full Set of Detected Noisy Triples from NELL-995}
\label{apd-full-tab}
\input{tabs/4_full_tab}
Table~\ref{tab:full} shows all the detected noisy triples.
% \subsection{Computation Complexity Analysis and Results on Larger-Scale Datasets}
\subsection{Parameter Sensitivity Analysis and Robustness Study}
We additionally study the sensitivity of our proposal, with respect to the depth of the R-GCN, and the strength of the sparsity constraint.
From Table~\ref{tab:rebuttal-sensitivity-depth} we can see that the different values (e.g. the choice of 0.1, 0.5 and 1.0) of $\gamma$ do not affect the noise detection performance of the RAE in different data sets. Table~\ref{tab:rebuttal-sensitivity-sparsity} shows how the depth of the R-GCN model used in both the encoder and the decoder can affect the noise detection performance of the RAE. From the results we can see that as long as the expressiveness of the model becomes sufficient, e.g. two layers for the datasets used in the experiments, the noise detection remains quite stable thereafter.
\input{tabs/apdx_sensitivity_on_RGCN_depth}
\input{tabs/apdx_sensitivity_sparse}
% \input{AISTATS2025-KGF/tabs/apdx_more_baselines}
% \input{AISTATS2025-KGF/tabs/apdx_robust_correpted_type}
% \section{Other Knowledge Graph Tasks and A Universal View}
% Except for knowledge graph refining, which compresses three tasks all together, namely, completion, denoising, and compress. In the following, we provide more comprehensive related work on these three tasks, respectively.
% \subsection{Knowledge Graph Denoising}
% \subsection{Knowledge Graph Completion}
% % %
% Knowledge graph completion is the most common downstream task to evaluate the knowledge graph embedding functions or models.
% It assumes the existing knowledge graph is incomplete, i.e., some triples are missing, and
% by learning low-dimensional representations of the relations and entities on a given knowledge graph to predict possible missing links on itself.
% Due to the lack of ground truth of missingness, the current pipeline is to divide the triples in to training, validation, and test sets first; then, finish the task in an unsupervised manner.
% %
% There are two main streams of knowledge graph representation learning methods, with respect to how the dependencies between the triples are modelled.
% One considers the dependencies by using an explicit scoring function only, which can be further divided into bi-linear and distance-based methods by whether the relations are modelled as matrices or vectors~\cite{li2022star, cao2024knowledge_space_survey}.
% The other type incorporates more expressive neural networks to enhance modeling the dependencies between the triples, before input the representations into the scoring functions.
% In this sort, MPNN-based approaches are often seen, such as R-GCN~\cite{schlichtkrull2018rgcn}, KGAT~\cite{wang2019kgat}, KBGAT~\cite{nathani2019kbgat}.
% Compared to the first type, they are more flexible for including more dependencies.
% \subsection{Knowledge Graph Compression}
% \subsection{A Unified View}
% \section{Boarder Experimental Results on Completion and Compression Tasks}
% \section{Asymptotic Identifiability Results and Definitional Clarifications}
% \subsection{Asymptotic Identifiability Theory}
% \subsubsection{Data Model}
% Firstly, we'd like to present the data model
% \subsubsection{General Result}
% We provide asymptotic identifiability results on the provided
% \begin{theorem}[Global view.]
% Given the
% \end{theorem}
% \begin{proof}
% \end{proof}
% \textbf{Feasibility of the assumptions.}\quad
% \subsubsection{Local Approximation}
% \begin{theorem}[Local view.]
% \end{theorem}
% \begin{proof}
% \end{proof}
% \textbf{Better feasibility of the modified assumptions.}\quad
% \textbf{Connection to our implementation.}\quad
% Connection to
% \subsection{Definitional Clarifications}
% \label{apd-def}
% \begin{definition}{(\textbf{$l^p$ norm of a matrix.})} Given a matrix $\textbf{X}$, the norm of the matrix is defined as
% \begin{align*}
% ||\textbf{X}||_p = \sqrt[p]{\sum_{i,j} \textbf{X}[i,j]^p}.
% \end{align*}
% It can easily be generalized to a higher dimensional tensor, e.g. $||\textbf{Y}||_p = \sqrt[p]{\sum_{i,j,k} \textbf{Y}[i,j,k]^p}$ is the $l^p$ norm of a three-dimensional cube $\textbf{Y}$.
% \end{definition}
% \begin{definition}{(\textbf{Support set of a matrix.})} Given a matrix $\textbf{X}$, define the support set of the matrix:
% \begin{align*}
% \text{supp}(\textbf{X}) = \{(i,j); \textbf{X}[i,j]\neq 0\}.
% \end{align*}
% Similarly, this definition can be generalized to higher dimensional tensors, e.g., $\{(i,j,k); \textbf{Y}[i,j,k]\neq 0\}$ is the support set of a three-dimensional cube $\textbf{Y}$.
% \end{definition}
\section{DIAGRAM DEMONSTRATION OF THE PROPOSED RAE}
% \subsection{Visualization of RAE Workflow}
The training and inference phases are demonstrated in Figure~\ref{fig:model-train} and Figure~\ref{fig:model-denoise}. Due to some adjustments in the structure of the appendix, the Figure~7 and Figure~8 that we put in the main content are actually Figure~4(a) and Figure~(b) here.
\begin{figure}[h]
\centering
\subfigure[Training Phase of RAE]{
\label{fig:model-train}
\includegraphics[width=\textwidth, trim={0cm 3cm 0cm 0cm}, clip]{figs/kgc_rebuttal_training_pic.pdf}
}
\subfigure[Denoising Phase of RAE]{
\label{fig:model-denoise}
\includegraphics[width=\textwidth, trim={0cm 3cm 0cm 0cm}, clip]{figs/kgc_rebuttal_denoise_pic.pdf}
}
\caption{Diagram demonstration of the training and inference phases of our proposal RAE.}
\label{fig:model-training-and-denoise}
\end{figure}
% \subsection{Algorithm Summarization}
% \begin{algorithmic}[Training process of RAE]
% \State $i \gets 10$
% \If{$i\geq 5$}
% \State $i \gets i-1$
% \Else
% \If{$i\leq 3$}
% \State $i \gets i+2$
% \EndIf
% \EndIf
% \end{algorithmic}
\section{BOARDER IMPACTS}
This work has the potential to enhance the reliability of AI applications that incorporate knowledge graphs as external expert knowledge support. However, since the interpretability of the model is not confirmed, it might be used to deliberately reject some facts by a third party. Therefore, the usage of this method should ensure the integrity of the original dataset, preventing any malicious manipulations.
\begin{table}[h]
\killfloatstyle
\centering
\caption{Full set of detected noise from NELL-995 dataset~(better view in color).}
\begin{adjustbox}{width=\columnwidth}
\begin{tabular}{ccc}
\toprule
\textbf{Head Entity} & \textbf{Relation} & \textbf{Tail Entity} \\
\midrule
person\_larry\_page & agentcontrol & university\_google \\
stateorprovince\_idaho & stateorprovinceisborderedbystateorprovince & stateorprovince\_south\_dakota \\
\rowcolor{pink_bg}
personasia\_news\_corporation & agentcontrol & city\_murdoch \\ biotechcompany\_delta\_air\_lines\_inc & organizationterminatedperson & journalist\_richard\_anderson \\
magazine\_gucci & organizationterminatedperson & ceo\_domenico\_de\_sole\\
stateorprovince\_idaho & stateorprovinceisborderedbystateorprovince & visualizablescene\_washington \\
stateorprovince\_idaho & stateorprovinceisborderedbystateorprovince & stateorprovince\_oregon \\
sportsteam\_detroit\_tigers & teamplayssport & sport\_baseball \\
person\_wendy001 & persongraduatedfromuniversit & university\_state\_university \\
musicartist\_the\_jam & agentcollaborateswithagent & male\_bruce\_foxton \\
% terroristorganization\_popular\_front\_for\_the\_liberation\_of\_palestine & agentcontrol & terroristorganization\_george\_habash & 0.0472 \\
bank\_bear\_stearns & organizationterminatedperson & ceo\_james\_cayne\\
\rowcolor{pink_bg}
ceo\_robert\_iger & agentcontrol & person\_disney \\
company\_apple002 & headquarteredin & city\_london\_city \\
\rowcolor{pink_bg}
personmexico\_m\_s & agentcontrol & director\_stuart\_rose \\
school\_shichahai\_school & agentcontrol & island\_zhang \\
website\_technorati & competeswith & blog\_google \\
sportsteam\_arizona\_state\_sun\_devils & agentcollaborateswithagent & personmexico\_ncaa \\
city\_erie & proxyfor & stateorprovince\_pennsylvania \\
\rowcolor{pink_bg}
profession\_parts & proxyfor & book\_new \\
\rowcolor{pink_bg}
sportsleague\_irl & agentcontrol & personmexico\_tony\_george \\
professor\_richard\_stallman & personleadsorganization & nonprofitorganization\_free\_software\_foundation \\
website\_cnn\_\_fox & competeswith & newspaper\_times \\
ceo\_william\_r\_\_klesse & agentcontrol & petroleumrefiningcompany\_valero\_energy \\ company\_pimco & agentcontrol & wine\_gross \\
website\_youtube & competeswith & website\_yahoo \\
\rowcolor{pink_bg}
movie\_repo & synonymfor & airport\_epel \\
recordlabel\_roc\_a\_fella & organizationterminatedperson & ceo\_damon\_dash \\
\bottomrule
\end{tabular}
\end{adjustbox}
\label{tab:full}
\end{table}
% university\_google & agentcontrols\_reverse & person\_larry\_page & 0.0065 \\
% stateorprovince\_south\_dakota & stateorprovinceisborderedbystateorprovince\_reverse & stateorprovince\_idaho & 0.0096 \\
% \rowcolor{pink_bg}
% city\_murdoch & agentcontrols\_reverse & personasia\_news\_corporation & 0.0113 \\
% journalist\_richard\_anderson & organizationterminatedperson\_reverse & biotechcompany\_delta\_air\_lines\_inc\_ & 0.0262 \\
% ceo\_domenico\_de\_sole & organizationterminatedperson\_reverse & magazine\_gucci & 0.0289 \\
% visualizablescene\_washington & stateorprovinceisborderedbystateorprovince\_reverse & stateorprovince\_idaho & 0.0337 \\
% stateorprovince\_oregon & stateorprovinceisborderedbystateorprovince\_reverse & stateorprovince\_idaho & 0.0410 \\
% sport\_baseball & teamplayssport\_reverse & sportsteam\_detroit\_tigers & 0.0419 \\
% university\_state\_university & persongraduatedfromuniversity\_reverse & person\_wendy001 & 0.0433 \\
% male\_bruce\_foxton & agentcollaborateswithagent\_reverse & musicartist\_the\_jam & 0.0449 \\
% terroristorganization\_george\_habash & agentcontrols\_reverse & terroristorganization\_popular\_front\_for\_the\_liberation\_of\_palestine & 0.0472 \\
% ceo\_james\_cayne & organizationterminatedperson\_reverse & bank\_bear\_stearns & 0.0496 \\
% \rowcolor{pink_bg}
% person\_disney & agentcontrols\_reverse & ceo\_robert\_iger & 0.0561 \\
% city\_london\_city & headquarteredin\_reverse & company\_apple002 & 0.0569 \\
% director\_stuart\_rose & agentcontrols\_reverse & personmexico\_m\_s & 0.0606 \\
% island\_zhang & agentcontrols\_reverse & school\_shichahai\_school & 0.0643 \\
% blog\_google & competeswith\_reverse & website\_technorati & 0.0766 \\
% personmexico\_ncaa & agentcollaborateswithagent\_reverse & sportsteam\_arizona\_state\_sun\_devils & 0.0785 \\
% stateorprovince\_pennsylvania & proxyfor\_reverse & city\_erie & 0.0818 \\
% \rowcolor{pink_bg}
% book\_new & proxyfor\_reverse & profession\_parts & 0.0921 \\
% personmexico\_tony\_george & agentcontrols\_reverse & sportsleague\_irl & 0.0952 \\
% nonprofitorganization\_free\_software\_foundation & personleadsorganization\_reverse & professor\_richard\_stallman & 0.1078 \\
% newspaper\_times & competeswith\_reverse & website\_cnn\_\_fox & 0.1084 \\
% petroleumrefiningcompany\_valero\_energy & agentcontrols\_reverse & ceo\_william\_r\_\_klesse & 0.1361 \\
% wine\_gross & agentcontrols\_reverse & company\_pimco & 0.1376 \\
% website\_yahoo & competeswith\_reverse & website\_youtube & 0.1922 \\
% \rowcolor{pink_bg}
% airport\_epel & synonymfor\_reverse & movie\_repo & 0.2216 \\
% ceo\_damon\_dash & organizationterminatedperson\_reverse & recordlabel\_roc\_a\_fella & 0.2286 \\
\begin{table}[h]
\centering
\begin{tabular}{cccc}
\toprule
$L$ & NELL-995 & WN18RR & FB15k-237 \\
\midrule
1 & $44.92 \pm 16.06$ & $113.48 \pm 32.90$ & $90.65 \pm 15.62$ \\
2 & $25.00 \pm 1.00$ & $49.25 \pm 42.17$ & $120.67 \pm 5.03$ \\
3 & $22.70 \pm 1.43$ & $58.21 \pm 31.69$ & $114.55 \pm 4.58$ \\
4 & $23.80 \pm 0.61$ & $41.77 \pm 29.09$ & $126.54 \pm 3.90$ \\
\bottomrule
\end{tabular}
\caption{Sensitivity study on the depth of R-GCN $L$}
\label{tab:rebuttal-sensitivity-depth}
\end{table}
\begin{table}[h]
\centering
\begin{tabular}{cccc}
\toprule
$\gamma$ & NELL-995 & WN18RR & FB15k-237 \\
\midrule
0.1 & $23.32 \pm 1.57$ & $43.77 \pm 22.91$ & $131.44 \pm 4.81$ \\
0.5 & $25.00 \pm 1.00$ & $49.25 \pm 42.17$ & $120.67 \pm 5.03$ \\
1.0 & $21.56 \pm 2.03$ & $58.74 \pm 36.22$ & $122.62 \pm 7.40$ \\
\bottomrule
\end{tabular}
\caption{Sensitivity study on the sparsity constraint $\gamma$}
\label{tab:rebuttal-sensitivity-sparsity}
\end{table}
\begin{thebibliography}{}
\bibitem[Anandalingam and Friesz, 1992]{anandalingam1992hierarchical}
Anandalingam, G. and Friesz, T.~L. (1992).
\newblock Hierarchical optimization: An introduction.
\newblock {\em Annals of Operations Research}, 34:1--11.
\bibitem[Belth et~al., 2020]{belth2020normal}
Belth, C., Zheng, X., Vreeken, J., and Koutra, D. (2020).
\newblock What is normal, what is strange, and what is missing in a knowledge graph: Unified characterization via inductive summarization.
\newblock In {\em Proceedings of The Web Conference 2020}, pages 1115--1126.
\bibitem[Bordes et~al., 2013]{bordes2013translating_wn18rr}
Bordes, A., Usunier, N., Garcia-Duran, A., Weston, J., and Yakhnenko, O. (2013).
\newblock Translating embeddings for modeling multi-relational data.
\newblock {\em Advances in neural information processing systems}, 26.
\bibitem[Cheng et~al., 2018]{cheng2018rule1}
Cheng, Y., Chen, L., Yuan, Y., and Wang, G. (2018).
\newblock Rule-based graph repairing: Semantic and efficient repairing methods.
\newblock In {\em 2018 IEEE 34th International Conference on Data Engineering (ICDE)}, pages 773--784. IEEE.
\bibitem[Chetoui et~al., 2022]{chetoui2022kg_gnn_survey}
Chetoui, I., El~Bachari, E., El~Adnani, M., and El~Hassan, A. (2022).
\newblock Graph neural networks to improve knowledge graph embedding: A survey.
\newblock In {\em International Conference on Advanced Intelligent Systems for Sustainable Development}, pages 15--25. Springer.
\bibitem[Deng et~al., 2023]{deng2023gold}
Deng, Z., Wang, W., Wang, Z., Liu, X., and Song, Y. (2023).
\newblock Gold: A global and local-aware denoising framework for commonsense knowledge graph noise detection.
\newblock In {\em Findings of the Association for Computational Linguistics: EMNLP 2023}, pages 3591--3608.
\bibitem[Dettmers et~al., 2018]{dettmers2018conve}
Dettmers, T., Minervini, P., Stenetorp, P., and Riedel, S. (2018).
\newblock Convolutional 2d knowledge graph embeddings.
\newblock In {\em Proceedings of the AAAI conference on artificial intelligence}, volume~32.
\bibitem[Dong et~al., 2023]{dong2023active}
Dong, J., Zhang, Q., Huang, X., Tan, Q., Zha, D., and Zihao, Z. (2023).
\newblock Active ensemble learning for knowledge graph error detection.
\newblock In {\em Proceedings of the sixteenth ACM international conference on web search and data mining}, pages 877--885.
\bibitem[Gal{\'a}rraga et~al., 2013]{galarraga2013rule2}
Gal{\'a}rraga, L.~A., Teflioudi, C., Hose, K., and Suchanek, F. (2013).
\newblock Amie: association rule mining under incomplete evidence in ontological knowledge bases.
\newblock In {\em Proceedings of the 22nd international conference on World Wide Web}, pages 413--422.
\bibitem[Guo et~al., 2020]{guo2020kg_rec_survey}
Guo, Q., Zhuang, F., Qin, C., Zhu, H., Xie, X., Xiong, H., and He, Q. (2020).
\newblock A survey on knowledge graph-based recommender systems.
\newblock {\em IEEE Transactions on Knowledge and Data Engineering}, 34(8):3549--3568.
\bibitem[Hu et~al., 2020]{hu2020open}
Hu, W., Fey, M., Zitnik, M., Dong, Y., Ren, H., Liu, B., Catasta, M., and Leskovec, J. (2020).
\newblock Open graph benchmark: Datasets for machine learning on graphs.
\newblock {\em Advances in neural information processing systems}, 33:22118--22133.
\bibitem[Huang et~al., 2021]{huang2021neighbor2neighbor}
Huang, T., Li, S., Jia, X., Lu, H., and Liu, J. (2021).
\newblock Neighbor2neighbor: Self-supervised denoising from single noisy images.
\newblock In {\em Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 14781--14790.
\bibitem[Jang et~al., 2017]{jang2017gumbel}
Jang, E., Gu, S., and Poole, B. (2017).
\newblock Categorical reparametrization with gumble-softmax.
\newblock In {\em International Conference on Learning Representations (ICLR 2017)}. OpenReview. net.
\bibitem[Ji et~al., 2021]{ji2021kgsurvey}
Ji, S., Pan, S., Cambria, E., Marttinen, P., and Philip, S.~Y. (2021).
\newblock A survey on knowledge graphs: Representation, acquisition, and applications.
\newblock {\em IEEE transactions on neural networks and learning systems}, 33(2):494--514.
\bibitem[Jia et~al., 2019a]{jia2019gcnn}
Jia, S., Xiang, Y., Chen, X., and Wang, K. (2019a).
\newblock Triple trustworthiness measurement for knowledge graph.
\newblock In {\em The World Wide Web Conference}, pages 2865--2871.
\bibitem[Jia et~al., 2019b]{jia2019emb2}
Jia, S., Xiang, Y., Chen, X., and Wang, K. (2019b).
\newblock Triple trustworthiness measurement for knowledge graph.
\newblock In {\em The World Wide Web Conference}, pages 2865--2871.
\bibitem[Ko and Lee, 2023]{ko2023self2self+}
Ko, J. and Lee, S. (2023).
\newblock Self2self+: Single-image denoising with self-supervised learning and image quality assessment loss.
\newblock {\em arXiv preprint arXiv:2307.10695}.
\bibitem[Lehmann et~al., 2015]{lehmann2015dbpedia}
Lehmann, J., Isele, R., Jakob, M., Jentzsch, A., Kontokostas, D., Mendes, P.~N., Hellmann, S., Morsey, M., Van~Kleef, P., Auer, S., et~al. (2015).
\newblock Dbpedia--a large-scale, multilingual knowledge base extracted from wikipedia.
\newblock {\em Semantic web}, 6(2):167--195.
\bibitem[Li et~al., 2023]{li2023message}
Li, J., Shomer, H., Ding, J., Wang, Y., Ma, Y., Shah, N., Tang, J., and Yin, D. (2023).
\newblock Are message passing neural networks really helpful for knowledge graph completion?
\newblock In {\em Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, pages 10696--10711.
\bibitem[Ma et~al., 2023]{ma2023learning_denoise}
Ma, T., Chen, Y., Tao, W., Zheng, D., Lin, X., Pang, P. C.-l., Liu, Y., Wang, Y., Song, B., and Zeng, X. (2023).
\newblock Learning to denoise unreliable interactions for link prediction on biomedical knowledge graph.
\newblock {\em arXiv preprint arXiv:2312.06682}.
\bibitem[Mahdisoltani et~al., 2013]{mahdisoltani2013yago3}
Mahdisoltani, F., Biega, J., and Suchanek, F.~M. (2013).
\newblock Yago3: A knowledge base from multilingual wikipedias.
\newblock In {\em CIDR}.
\bibitem[Nathani et~al., 2019]{nathani2019kbgat}
Nathani, D., Chauhan, J., Sharma, C., and Kaul, M. (2019).
\newblock Learning attention-based embeddings for relation prediction in knowledge graphs.
\newblock In {\em Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, pages 4710--4723.
\bibitem[Neil et~al., 2018]{neil2018interpretable_noise}
Neil, D., Briody, J., Lacoste, A., Sim, A., Creed, P., and Saffari, A. (2018).
\newblock Interpretable graph convolutional neural networks for inference on noisy knowledge graphs.
\newblock {\em arXiv preprint arXiv:1812.00279}.
\bibitem[Ng et~al., 2024]{ng2024sparse}
Ng, I., Zheng, Y., Dong, X., and Zhang, K. (2024).
\newblock On the identifiability of sparse ica without assuming non-gaussianity.
\newblock {\em Advances in Neural Information Processing Systems}, 36.
\bibitem[Pan et~al., 2024]{pan2024kg_llm_survey}
Pan, S., Luo, L., Wang, Y., Chen, C., Wang, J., and Wu, X. (2024).
\newblock Unifying large language models and knowledge graphs: A roadmap.
\newblock {\em IEEE Transactions on Knowledge and Data Engineering}.
\bibitem[Paulheim and Gangemi, 2015]{paulheim2015emb1}
Paulheim, H. and Gangemi, A. (2015).
\newblock Serving dbpedia with dolce--more than just adding a cherry on top.
\newblock In {\em The Semantic Web-ISWC 2015: 14th International Semantic Web Conference, Bethlehem, PA, USA, October 11-15, 2015, Proceedings, Part I 14}, pages 180--196. Springer.
\bibitem[Pellissier~Tanon et~al., 2017]{pellissier2017rule3}
Pellissier~Tanon, T., Stepanova, D., Razniewski, S., Mirza, P., and Weikum, G. (2017).
\newblock Completeness-aware rule learning from knowledge graphs.
\newblock In {\em The Semantic Web--ISWC 2017: 16th International Semantic Web Conference, Vienna, Austria, October 21--25, 2017, Proceedings, Part I 16}, pages 507--525. Springer.
\bibitem[Quan et~al., 2020]{quan2020self2self}
Quan, Y., Chen, M., Pang, T., and Ji, H. (2020).
\newblock Self2self with dropout: Learning self-supervised denoising from single image.
\newblock In {\em Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages 1890--1898.
\bibitem[Sachan, 2020]{sachan2020kgcompression_survey}
Sachan, M. (2020).
\newblock Knowledge graph embedding compression.
\newblock In {\em Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, pages 2681--2691.
\bibitem[Schlichtkrull et~al., 2018]{schlichtkrull2018rgcn}
Schlichtkrull, M., Kipf, T.~N., Bloem, P., Van Den~Berg, R., Titov, I., and Welling, M. (2018).
\newblock Modeling relational data with graph convolutional networks.
\newblock In {\em The semantic web: 15th international conference, ESWC 2018, Heraklion, Crete, Greece, June 3--7, 2018, proceedings 15}, pages 593--607. Springer.
\bibitem[Toutanova and Chen, 2015]{toutanova2015observed_fb15k}
Toutanova, K. and Chen, D. (2015).
\newblock Observed versus latent features for knowledge base and text inference.
\newblock In {\em Proceedings of the 3rd workshop on continuous vector space models and their compositionality}, pages 57--66.
\bibitem[Xiong et~al., 2017]{xiong2017deeppath_nell}
Xiong, W., Hoang, T., and Wang, W.~Y. (2017).
\newblock Deeppath: A reinforcement learning method for knowledge graph reasoning.
\newblock {\em arXiv preprint arXiv:1707.06690}.
\bibitem[Xu et~al., 2022]{xu2022emb3}
Xu, Z., Huang, X., Zhao, Y., Dong, Y., and Li, J. (2022).
\newblock Contrastive attributed network anomaly detection with data augmentation.
\newblock In {\em Pacific-Asia Conference on Knowledge Discovery and Data Mining}, pages 444--457. Springer.
\bibitem[Yang et~al., 2015]{yang2015distmult}
Yang, B., Yih, S. W.-t., He, X., Gao, J., and Deng, L. (2015).
\newblock Embedding entities and relations for learning and inference in knowledge bases.
\newblock In {\em Proceedings of the International Conference on Learning Representations (ICLR) 2015}.
\bibitem[Zamini et~al., 2022]{zamini2022kgcompletion_survey}
Zamini, M., Reza, H., and Rabiei, M. (2022).
\newblock A review of knowledge graph completion.
\newblock {\em Information}, 13(8):396.
\bibitem[Zeng et~al., 2021]{zeng2021entity_align_survey}
Zeng, K., Li, C., Hou, L., Li, J., and Feng, L. (2021).
\newblock A comprehensive survey of entity alignment for knowledge graphs.
\newblock {\em AI Open}, 2:1--13.
\bibitem[Zhang, 2010]{zhang2010mcp}
Zhang, C.-H. (2010).
\newblock Nearly unbiased variable selection under minimax concave penalty.
\newblock {\em The Annals of Statistics}, pages 894--942.
\bibitem[Zhang et~al., 2022]{zhang2022contrastive}
Zhang, Q., Dong, J., Duan, K., Huang, X., Liu, Y., and Xu, L. (2022).
\newblock Contrastive knowledge graph error detection.
\newblock In {\em Proceedings of the 31st ACM International Conference on Information \& Knowledge Management}, pages 2590--2599.
\bibitem[Zhang et~al., 2023]{zhang2023integrating_attr}
Zhang, Q., Dong, J., Tan, Q., and Huang, X. (2023).
\newblock Integrating entity attributes for error-aware knowledge graph embedding.
\newblock {\em IEEE Transactions on Knowledge and Data Engineering}.
\bibitem[Zhang et~al., 2020]{zhang2020kgcompression1}
Zhang, S., Zhang, Z., Zhuang, F., Shi, Z., and Han, X. (2020).
\newblock Compressing knowledge graph embedding with relational graph auto-encoder.
\newblock In {\em 2020 IEEE 10th International Conference on Electronics Information and Emergency Communication (ICEIEC)}, pages 366--370. IEEE.
\bibitem[Zhao et~al., 2020]{zhao2020entity_align_survey}
Zhao, X., Zeng, W., Tang, J., Wang, W., and Suchanek, F.~M. (2020).
\newblock An experimental study of state-of-the-art entity alignment approaches.
\newblock {\em IEEE Transactions on Knowledge and Data Engineering}, 34(6):2610--2625.
\bibitem[Zhao et~al., 2019]{zhao2019embedding_noise}
Zhao, Y., Feng, H., and Gallinari, P. (2019).
\newblock Embedding learning with triple trustiness on noisy knowledge graph.
\newblock {\em Entropy}, 21(11):1083.
\end{thebibliography}