Generative Benchmark Creation for Table Union Search
Extracted source text · arXiv 2308.03883
% VLDB template version of 2020-08-03 enhances the ACM template, version 1.7.0:
% https://www.acm.org/publications/proceedings-template
% The ACM Latex guide provides further information about the ACM template
%\documentclass[sigconf, nonacm, review]{acmart}
\documentclass[sigconf, nonacm]{acmart}
\input{datalabmacros} % includes various useful macros (keep macros separate from main content files)
\input{individualmacros} %
%% The following content must be adapted for the final version
% paper-specific
\newcommand\vldbdoi{}
\newcommand\vldbpages{}
% issue-specific
\newcommand\vldbvolume{}
\newcommand\vldbissue{}
\newcommand\vldbyear{2023}
% should be fine as it is
\newcommand\vldbauthors{\authors}
\newcommand\vldbtitle{\shorttitle}
% leave empty if no availability url should be set
\newcommand\vldbavailabilityurl{https://github.com/northeastern-datalab/alt-gen}
% whether page numbers should be shown or not, use 'plain' for review versions, 'empty' for camera ready
\newcommand\vldbpagestyle{empty}
\begin{document}
\title{Generative Benchmark Creation for Table Union Search}
%after Discovery \roee{do we need the "after Discovery" in the title?} \aamod{we need to fix the paper name too}
%%
%% The "author" command and its associated commands are used to define the authors and their affiliations.
\author{Koyena Pal}
\affiliation{%
\institution{Northeastern University}
%\streetaddress{P.O. Box 1212}
\city{Boston}
\state{Massachusetts, USA}
%\postcode{43017-6221}
}
%\email{trovato@corporation.com}
\email{pal.k@northeastern.edu}
\author{Aamod Khatiwada}
\affiliation{%
\institution{Northeastern University}
%\streetaddress{P.O. Box 1212}
\city{Boston}
\state{Massachusetts, USA}
%\postcode{43017-6221}
}
%\email{trovato@corporation.com}
\email{khatiwada.a@northeastern.edu}
\author{Roee Shraga}
\affiliation{%
\institution{Northeastern University}
\city{Boston}
\state{Massachusetts, USA}
%\streetaddress{P.O. Box 1212}
%\city{Dublin}
%\state{Ireland}
%\postcode{43017-6221}
}
\email{r.shraga@northeastern.edu}
\author{Ren\'ee J. Miller}
\affiliation{%
\institution{Northeastern University}
\city{Boston}
\state{Massachusetts, USA}
}
%\email{trovato@corporation.com}
\email{miller@northeastern.edu}
%\input{sections/cover_letter}
\input{sections/abstract.tex}
\maketitle
%\setcounter{page}{1} % reset page numbering to 1
%%% do not modify the following VLDB block %%
%%% VLDB block start %%%
\pagestyle{\vldbpagestyle}
\begingroup\small\noindent\raggedright\textbf{PVLDB Reference Format:}\\
\vldbauthors. \vldbtitle. PVLDB, \vldbvolume(\vldbissue): \vldbpages, \vldbyear.\\
\href{https://doi.org/\vldbdoi}{doi:\vldbdoi}
\endgroup
\begingroup
\renewcommand\thefootnote{}\footnote{\noindent
This work is licensed under the Creative Commons BY-NC-ND 4.0 International License. Visit \url{https://creativecommons.org/licenses/by-nc-nd/4.0/} to view a copy of this license. For any use beyond those covered by this license, obtain permission by emailing \href{mailto:info@vldb.org}{info@vldb.org}. Copyright is held by the owner/author(s). Publication rights licensed to the VLDB Endowment. \\
\raggedright Proceedings of the VLDB Endowment, Vol. \vldbvolume, No. \vldbissue\ %
ISSN 2150-8097. \\
\href{https://doi.org/\vldbdoi}{doi:\vldbdoi} \\
}\addtocounter{footnote}{-1}\endgroup
%%% VLDB block end %%%
%%% do not modify the following VLDB block %%
%%% VLDB block start %%%
\ifdefempty{\vldbavailabilityurl}{}{
\vspace{.3cm}
\begingroup\small\noindent\raggedright\textbf{PVLDB Artifact Availability:}\\
The source code, data, and/or other artifacts have been made available at \url{\vldbavailabilityurl}.
\endgroup
}
\input{sections/introduction}
\input{sections/relatedworks}
\input{sections/benchmarkgen}
\input{sections/analysiseval}
\input{sections/conclusion}
\begin{acks}
This work was supported in part by NSF
under award numbers IIS-1956096 and IIS-2107248.
\end{acks}
% \section{Open source contributions}
% % \textbf{take from NSF proposal}
% The core component of the proposed work is the creation of new benchmarks to be shared with the research community.
% Ensuring easy access to the data created as a part of this project, ensuring reproducibility and reusability, and accordingly, impact, is a high priority for our group. All benchmarks will be made available via the Lab's GitHub account\footnote{\label{git}\url{https://github.com/northeastern-datalab}}. Where practical, example data will be stored in standard formats such as CSV. Documentation will be stored in plaintext, HTML, Markdown, and/or PDF. We will use Creative Commons licenses\footnote{\url{http://creativecommons.org/licenses/}}
% for the re-use, re-distribution, and the production of derivatives of the project data, and seek to use open access journals and publishing venues.
% %while respecting the limitations on copyrighted material from published journals.
% Source code will also be made available in our lab's GitHub$^{\ref{git}}$. We will use Apache licenses\footnote{\url{http://www.apache.org/licenses/}}
% or similarly permissive licenses for the re-use, re-distribution, and the production of derivatives of all project source code, while respecting the licenses of any library dependencies.
\newpage
\bibliographystyle{ACM-Reference-Format}
\bibliography{proposal}
\end{document}
%==============================================================================================
% General packages before hyperref (no personalized macros here)
%=============================================================================================
\usepackage{amsmath}
% \usepackage{amsfonts}
% \usepackage{amssymb} % throws errors with ACM template
% \usepackage{amsopn} % allows to define new operators, e.g., \argmin
\usepackage{mathtools}
\usepackage{graphicx} % can better scale and rotate graphics than graphic package
\usepackage{url}
\usepackage{enumitem} % remove spaces inside 'itemize' [nolistsep]
% \usepackage{bm} % bold Greek letters
\usepackage{stmaryrd} % for symbol \llceil
% max 16 math family slots in latex; commented out bm to allow stmaryrd to work
\usepackage{upgreek} % upright Greek letters; together with package "bm" -> upright bold letters
\usepackage{bbding} % for the \CheckmarkBold symbol
\usepackage{listings} % to get sql etc nicely formatted (source code listings)
% \usepackage{fontspec} % to change the SQL font in listings
\lstset{
basicstyle=\linespread{0.9}\ttfamily,
columns=fullflexible,
keepspaces=true,
mathescape,
escapechar={|} % https://tex.stackexchange.com/questions/345694/change-color-of-some-items-in-lstlisting
}
\usepackage{tabularx} % table environment that allows to span a whole line
\usepackage{booktabs} % \toprule
\usepackage{multirow}
\usepackage{hhline} % cannot be overwritten by cellcolor
\usepackage{diagbox}
\usepackage{pgfplotstable} % powerful numerical table macros
% \usepackage[usenames,table,dvipsnames,x11names]{xcolor} % not needed as options are passed directoy to documentclass 'acmart'
% \PassOptionsToPackage{table}{xcolor} % xcolor needs to come before tikz which calls xcolor as well. Solution: pass the options before calling tikz
% for colored cells in tables
\usepackage{xcolor,colortbl} % !!! for some reason, overleaf needs xcolor separately, although that should already have been called by colortbl (10/2019)
\usepackage{tikz} % drawing figures
\usetikzlibrary{matrix}
\usetikzlibrary{calc}
\usetikzlibrary{math}
\usetikzlibrary{arrows.meta,positioning}
\usetikzlibrary{intersections, pgfplots.fillbetween}
\usepackage{easybmat} % \begin{BMAT} alternative syntax for math tabulars and also includes the command addpath which allows to draw a path between matrix elements.
\usepackage{adjustbox}
\def\eox{\unskip\kern 10pt{\unitlength1pt\linethickness{.4pt}$\diamondsuit${}}} % "\eox" command for end of example
\usepackage{rotating} % for rotated text in tables, \begin{turn}{75}Accuracy\end{turn}
\sloppy % Latex sometimes creates text lines that go into the margin, just to avoid ``ugly'' large spacing. The following prevents this:
\usepackage{xspace} % for macros
% \usepackage[bottom]{footmisc} % for footnotes
%==============================================================================================
% subfigures
% use package 'subcaption'
% Package 'subfigure' is deprecated and superseded by 'subfig'
% But 'subfig' is not working properly with hyperref.
% 'subcaption' package is most recent
% See: https://tex.stackexchange.com/questions/13625/subcaption-vs-subfig-best-package-for-referencing-a-subfigure
%=============================================================================================
\usepackage{subcaption} % recommended new package for subfigures (package subfig deprecated)
\captionsetup[figure]{labelfont=bf} % textfont=bf, labelfont=bf, ,font+=bf
\captionsetup[subfigure]{font+=small}
\captionsetup[table]{labelfont=bf} % font+=small, textfont=bf, labelfont=b
% \captionsetup[subfigure]{justification=centering}
\captionsetup[figure]{aboveskip=2pt}
\captionsetup[subfigure]{aboveskip=1pt, belowskip=1pt} % aboveskip = distance fig and subcaption)
\captionsetup[figure]{belowskip=-3pt} % space after figures
%==============================================================================================
% Comments and ways to hide them
% macro \note{W: bla bla}: In blue. Start your comment with a short author identifier, e.g. 'W' for Wolfgang. Can be kept in the paper even during and after submission
% macro \todo{W: bla bla}: In red. Those are important issues that need to be fixed. Once they are fixed, replace the \todo with \note
% macro \mkclean: hides the blue comments (not the red ones)
%=============================================================================================
\newcommand{\todo}[1]{{{\color{red}{[\textbf{#1}]}}}} % important comments
\newcommand{\note}[1]{{{\color{magenta}{[#1]}}}} % general comments
\newcommand{\renee}[1]{{{\color{red}{#1}}}}
\newcommand{\rjm}[1]{{{\color{purple}{[\textbf{RJM}: #1]}}}}
\newcommand{\wolf}[1]{{{\color{magenta}{[\textbf{wg}: #1]}}}}
\newcommand{\wg}[1]{{{\color{blue}{[\textbf{wg-detail}: #1]}}}}
\newcommand{\mirek}[1]{{{\color{cyan}{[\textbf{MR}: #1]}}}}
\newcommand{\nikos}[1]{{{\color{magenta}{[\textbf{N}: #1]}}}}
\newcommand{\aamod}[1]{{{\color{violet}{[\textbf{AK}: #1]}}}}
\newcommand{\grace}[1]{{{\color{blue}{[\textbf{GF}: #1]}}}}
\newcommand{\zixuan}[1]{{{\color{olive}{[\textbf{ZC}: #1]}}}}
\newcommand{\roee}[1]{{{\color{brown}{[\textbf{RS}: #1]}}}}
\newcommand{\koyena}[1]{{{\color{teal}{[\textbf{KP}: #1]}}}}
\newcommand{\ak}[1]{{\color{violet}{#1}}}
\newcommand{\revision}[1]{{\color{blue}{#1}}}
\newcommand{\new}[1]{{\color{blue}{#1}}}
% important comments
% \newcommand{\todo}[1]{{\sffamily{\color{red}{[\small #1]}}}} % needs to be fixed
% \newcommand{\note}[1]{{\sffamily{\color{blue}{[\small #1]}}}} % comment
% \newcommand{\new}[1]{{\color{violet}{#1}}}
\newcommand{\hide}[1]{}
\newcommand{\mkclean}{
\renewcommand{\note}{\hide}
\renewcommand{\todo}{\hide}
\renewcommand{\wolf}{\hide}
\renewcommand{\nikos}{\hide}
\renewcommand{\mirek}{\hide}
\renewcommand{\new}{\hide}
}
%\mkclean % comment out this line, to hide all note comments except for those that *need* to be fixed
%==============================================================================================
% ALGORITHM2e
%=============================================================================================
\usepackage[ruled,noend,linesnumbered]{algorithm2e} % boxruled, linesnumbered,ruled
\usepackage{setspace} % for setting line spacing
\DontPrintSemicolon % does not print ";" at end of each line
% \LinesNotNumbered
% \LinesNumbered
\SetNlSty{}{}{} % style of line numbers "\Setnlsty{small}{}{:}"
\SetAlgoInsideSkip{smallskip} % distance between caption rule and beginning of algorithm
% FOLLOWING REMOVED FOR VLDB
%\SetAlCapSkip{1.5mm} % left margin of caption
%\setlength{\algomargin}{1.5mm} % (1) left margin of algorithm content, but also (!) right margin of caption; also puts line numbers into margin
% \SetNlSkip{2mm} % (2) therefore put line numbers back and use "\Indp" twice inside algorithm to put algorithm further to the right
% \SetAlgoVlined
%\SetInd{1.5mm}{1.5mm} % corrects for the twice usage of "\Indp"; otherwise vlines are wrong
\SetAlFnt{\small} % size of actual algorithm text\scriptsize\sf\rm
\SetAlCapFnt{\small} % size of name of algorithm
\SetAlCapNameFnt{\small}
%==============================================================================================
% Hyperref, Cleveref, Aliascnt etc. (algorithm2e, hyperref need to come before cleveref)
%=============================================================================================
% NT 7/13/20: already loaded from acm template
%\usepackage[breaklinks=true]{hyperref} % use for hypertext links, including those to external documents answerd URLs
\hypersetup{ % for both, DVI and PDF
pdfpagemode=UseNone, % pdfpagemode=UseOutlines
pdfstartview=Fit,
pdfpagelayout=SinglePage, % pdfpagelayout=SinglePage
bookmarks=false,
bookmarksnumbered=true,
bookmarksopen=false, % bookmarkstree expanded
pdftex=true,
unicode,
colorlinks=true, % false: boxed links; true: colored links
linkcolor=blue, % color of internal links (change box color with linkbordercolor)
citecolor=cyan, % color of links to bibliography
filecolor=magenta, % color of file links
urlcolor=blue % color of external links
% plainpages=false, % necessary for correct backref
% hypertexnames=true, % necessary for correct backref [??? false might be necessary in order to get the hyper-links to the figures right]
% pdfpagelabels=true, % necessary for correct backref
% hyperindex=true,
% naturalnames=false % !!! default, necessary for correct reference to subfigures (August 2010)
% former true: (necessary for not confusing e.g. roman page ii with later arabic 2
% use LaTeX-computed names for links (default: false))
}
%==============================================================================================
% CLEVERREF
% better than hyperref's autoref, needs to come after hyperref and algorithm2e (if referencing lines)(Sept 2015)
% \usepackage[nameinlink]{cleveref} % option clash with new ACM class (7/2017)
%=============================================================================================
\usepackage[capitalise,nameinlink]{cleveref} %nameinlink makes whole word colored
% \crefname{algocf}{alg.}{algs.}
% \Crefname{algocf}{Algorithm}{Algorithms}
% \crefalias{AlgoLine}{line}%
% \crefname{proposition}{prop.}{prop.}
% \crefname{figure}{Figure}{Figures}
% \crefname{observation}{Observation}{Observations}
% \crefformat{equation}{Eq.~(#2#1#3)} % show reference to equation as (1)
% \crefrangeformat{equation}{(#3#1#4) to~(#5#2#6)}
% \crefmultiformat{equation}{(#2#1#3)}%
% { and~(#2#1#3)}{, (#2#1#3)}{ and~(#2#1#3)}
% % \crefformat{equation}{(#2#1#3)} % show reference to equation as (1)
% % \crefrangeformat{equation}{(#3#1#4) to~(#5#2#6)}
% % \crefmultiformat{equation}{(#2#1#3)}%
% % { and~(#2#1#3)}{, (#2#1#3)}{ and~(#2#1#3)}
%
% % \Crefname{equation}{Equation}{Equations}
% \Crefformat{equation}{Equation~(#2#1#3)} % show reference to equation as (1)
% %==============================================================================
% % !!! following does not work if \amsthm is used !!!
% %
% % These lines fix some annyong property of the ACM template that first lines of definitions
% % swap over into the margin. Example
% % DEFINITION 2.4 (Actual cause). blabla
% % instead of
% % DEFINITION 2.4 (Actual cause). blabla
% %
% % Solution is to tweak some distance measure in the ACM template. Below separate not to have to tweak the cls file
% % still has to be tweaked with "~\!\!\!\!" and "~\,\," the ~ is important here (Wolf Dec 2009)
% % remoeved for new ACM template (7/2017)
% %==============================================================================
% \makeatletter
% \renewcommand{\@opargbegintheorem}[3]{%
% \parskip 0pt % GM July 2000 (for tighter spacing)
% \trivlist
% \item[%
% \hskip 10\p@
% \hskip \labelsep
% {\sc #1\ #2\ %
% \setbox\@tempboxa\hbox{(#3)} %
% \ifdim \wd\@tempboxa>\z@ %
% \hskip -2\p@\relax % This number has to be reduced from 5 !!!
% \box\@tempboxa %
% \fi.}%
% ]
% \it
% % \sloppypar % prevents the overrun into margins of first line for theorem environments for paragraph
% \setlength{\emergencystretch}{6pt}
% }
% \makeatother
%==============================================================================================
% New environments (notice that question and answer have slightly different formats)
% NOT NEEDED FOR acmart
% aliascnt still needed for VLDB!
%=============================================================================================
\usepackage{aliascnt}
% ``hyperref’s \autoref command does not work well with theorems that share a counter:
% it’ll always think it’s a Lemma even if it’s a Remark that shares the Lemma counter.
% Load this package to fix it. No further intervention needed.''
% Source: http://absatzen.de/thmtools.html (Jan 2009)
% better: http://www.tug.org/applications/hyperref/manual.html (Nov 2009)
% needs also: thm-patch.sty, parseargs.sty, aliasctr.sty ???
% see section below the three steps for usage
% 1 define a new alias counter
% 2 define the environment (e.g., example)
% 3 reset the counter
\newtheorem{theorem}{Theorem} % ...[section]
\newaliascnt{corollary}{theorem}
\newtheorem{corollary}[corollary]{Corollary}
\aliascntresetthe{corollary}
\newaliascnt{example}{theorem}
\newtheorem{example}[example]{Example}
\aliascntresetthe{example}
\newaliascnt{definition}{theorem}
\newtheorem{definition}[definition]{Definition}
\aliascntresetthe{definition}
\newaliascnt{proposition}{theorem}
\newtheorem{proposition}[proposition]{Proposition}
\aliascntresetthe{proposition}
\newaliascnt{lemma}{theorem}
\newtheorem{lemma}[lemma]{Lemma}
\aliascntresetthe{lemma}
\newaliascnt{conjecture}{theorem}
\newtheorem{conjecture}[conjecture]{Conjecture}
\aliascntresetthe{conjecture}
% \newaliascnt{remark}{theorem}
% \newtheorem{remark}[remark]{Remark}
% \aliascntresetthe{remark}
%
% \newaliascnt{axiom}{theorem}
% \newtheorem{axiom}[axiom]{Axiom}
% \aliascntresetthe{axiom}
%
% \newaliascnt{fact}{theorem}
% \newtheorem{fact}[fact]{Fact}
% \aliascntresetthe{fact}
%
% \newaliascnt{claim}{theorem}
% \newtheorem{claim}[claim]{Claim}
% \aliascntresetthe{claim}
\newtheorem{algo}[theorem]{Algorithm}
\newtheorem{observation}{Observation}
\newtheorem{problem}{Problem}
\newtheorem{answer}{Answer}
\newtheorem{question}{Question}
\newtheorem{questionW}{Question}
\newtheorem{resultW}{Result}
\renewenvironment{questionW}[1][]
{% begin code
{\textit{Question \arabic{questionW} #1}
\stepcounter{questionW}}%
}
{% end code
}
\setcounter{questionW}{1}
\renewenvironment{resultW}[1][]
{% begin code
% \par\medskip\noindent
\par\noindent
{\textbf{Result \arabic{resultW} #1}
\stepcounter{resultW}}%
\begin{itshape}%
}% end code
{\end{itshape}
% \medskip
}
\setcounter{resultW}{1}
%==============================================================================================
% Make footnotesizes smaller for VLDB template
%=============================================================================================
% \let\footnotesize\scriptsize
% \makeatletter
% \renewcommand\footnotesize{%
% \@setfontsize\footnotesize\@ixpt{11}%
% \abovedisplayskip 8\p@ \@plus2\p@ \@minus4\p@
% \abovedisplayshortskip \z@ \@plus\p@
% \belowdisplayshortskip 4\p@ \@plus2\p@ \@minus2\p@
% \def\@listi{\leftmargin\leftmargini
% \topsep 4\p@ \@plus2\p@ \@minus2\p@
% \parsep 2\p@ \@plus\p@ \@minus\p@
% \itemsep \parsep}%
% \belowdisplayskip \abovedisplayskip
% }
% \makeatother
%==============================================================================================
% Change the color use if 'screen=true' (used in acmart.cls around line 482 (4/2019)
%==============================================================================================
\AtEndPreamble{%
\hypersetup{colorlinks,
linkcolor=purple,
citecolor=blue,
urlcolor=ACMDarkBlue,
filecolor=ACMDarkBlue}}
%==============================================================================
% These lines fix some annyong property of the old ACM template (= VLDB template) that first lines of definitions
% swap over into the margin. Example
% DEFINITION 2.4 (Actual cause). blabla
% instead of
% DEFINITION 2.4 (Actual cause). blabla
% Solution is to tweak some distance measure in the ACM template. Below separate not to have to tweak the cls file
% still has to be tweaked with "~\!\!\!\!" and "~\,\," the ~ is important here (Wolf Dec 2009)
%
% New fix to remove the indentation for theorems environments (Wolf July 2014)
%==============================================================================
% \makeatletter
%
% \renewcommand{\@opargbegintheorem}[3]{%
% \parskip 0pt % GM July 2000 (for tighter spacing): 0 pt, 5 pt
% \trivlist
% \item[%
% \hskip 10\p@ % this number was change to 0, former 10
% \hskip \labelsep
% {\sc #1\ #2\ %
% \setbox\@tempboxa\hbox{(#3)} %
% \ifdim \wd\@tempboxa>\z@ %
% \hskip 0\p@\relax % This number has to be reduced from 5 !!!
% \box\@tempboxa %
% \fi.}%
% ]
% \it
% }
%
% \def\@begintheorem#1#2{%
% \parskip 0pt % GM July 2000 (for tighter spacing)
% \trivlist
% \item[%
% \hskip 10\p@ % this number was change to 0, former 10
% \hskip \labelsep
% {{\sc #1}\hskip 5\p@\relax#2.}%
% ]
% \it
% }
%
% \makeatother
%=============================================================================================
% Nice backgroundcolor around environments and results
%=============================================================================================
\usepackage{tcolorbox} % also allows to have shaded backgrounds around environments
\tcbuselibrary{breakable,skins} % allow to break boxes, allow to use "enhanced jigsaw"
% === Style used for blue example environment
\tcbset{examplestyle/.style={
enhanced jigsaw, % if box is broken, don't show border at continuation
colback=blue!08, % !20 gray!10
colframe=blue!08, % !40
arc=2mm,
boxrule=0pt, % 0pt
% colframe=red!75!black,
left=1mm,
right=1mm,
left skip= 0mm, % breaks ACM template unless specified for specific style (4/2019)
right skip= 0mm, % breaks ACM template unless specified for specific style (4/2019)
top=-0.5mm, % 0mm, -1mm
bottom=1mm, % problem in package with eq at end of env. just add: \tcbset{bottom=2mm} \tcbset{bottom=0mm}
breakable, % allows page break
parbox = false, % restores normal text behavior. ELSE: paragraphs are formatted slightly different as the main text. DOWNSIDE: unwanted side effects
% nobeforeafter,
% before={\par\pagebreak[0]\smallskip\parindent=0pt},
before={\par\pagebreak[0]\vspace{1mm}\parindent=0pt},
% before={\vspace{1mm}},
after={\par\pagebreak[0]\vspace{1mm}\parindent=0pt},
% after=5mm,
bottomrule = 0mm,
boxsep = 0mm, % common padding of hlengthi between the text content and the frame of the box, added to left, right, top, ...
topsep at break=0pt, % Additional vertical space at the top of middle and last parts in a break sequence
bottomsep at break=0pt, % Additional vertical space at the first and middle parts in a break sequence
pad at break=0mm,
pad before break=1mm,
pad after break=1mm,
bottomrule at break=0mm,
toprule at break=0mm,
}}
\usepackage{footnote}
\tcolorboxenvironment{example}{examplestyle}
%\usepackage{footnote}% allows to wrap an example in "\begin{savenotes}" to force footnotes to the bottom of page
% \tcbset{
% enhanced jigsaw, % if box is broken, don't show border at continuation
% % colback=gray!20,
% % colframe=gray!40,
% arc=0mm,
% boxrule=1pt, % 0pt
% % colframe=red!75!black,
% left=1pt,
% right=1pt,
% % left skip=-1pt, % breaks ACM template (7/2018)
% % right skip=-1pt, % breaks ACM template (7/2018)
% topsep at break=1mm, % Additional vertical space at the top of middle and last parts in a break sequence
% top=1pt, % 0mm, -1mm
% bottom=0mm, % problem in package with eq at end of env. just add: \tcbset{bottom=2mm} \tcbset{bottom=0mm}
% breakable, % allows page break
% parbox = false % restores normal text behavior. ELSE: paragraphs are formatted slightly different as the main text. DOWNSIDE: unwanted side effects
% }
% \tcolorboxenvironment{example}{}
% === Style used for red result box
\newcommand{\resultbox}[1]{
\begin{tcolorbox}[
enhanced jigsaw, % if box is broken, don't show border at continuation
colback=red!5,
colframe=red!75!black,
arc=0mm,
% boxrule=1pt,
% boxrule=0pt,
% colframe=red!75!black,
left skip=-1mm,
right skip=-1mm,
left=0mm,
topsep at break=1mm, % Additional vertical space at the top of middle and last parts in a break sequence
% enlarge top by=0mm, % before box
% enlarge top at break by=5mm, % above box, after pagebreak
right=0mm,
top=0mm,
bottom=0mm, % problem in package with eq at end of env. justadd:\tcbset{bottom=2mm}\tcbset{bottom=0mm}
% after skip=2mm,
breakable, % allows page break
% before upper={\parindent15pt}
parbox = false % restores normal text behavior. ELSE: paragraphs are formatted slightly different as the main text. DOWNSIDE: unwanted side effects
]
\emph{#1}
\end{tcolorbox}
}
%==============================================================================
% defines upright \ell character as \uell (10/2014)
% Source: http://tex.stackexchange.com/questions/175236/typeset-an-upright-ell
%==============================================================================
\makeatletter
\DeclareRobustCommand*\uell{\mathpalette\@uell\relax}
\newcommand*\@uell[2]{
% We need to adjust the width of \uell to be the same as \ell
\setbox0=\hbox{$#1\ell$}
\setbox1=\hbox{\rotatebox{10}{$#1\ell$}}
\dimen0=\wd0 \advance\dimen0 by -\wd1 \divide\dimen0 by 2
\mathord{\lower 0.1ex \hbox{\kern\dimen0\unhbox1\kern\dimen0}}
}
\makeatother
%==============================================================================
% allow figures and table to cover 100% of page
% See more: https://aty.sdsu.edu/bibliog/latex/floats.html
%==============================================================================
\renewcommand\dbltopfraction{1} % max fraction of a two-column page that may be occupied by a (double-wide) float at top
\setcounter{dbltopnumber}{5}% max nubmers of floats at top of two-column page (dbl = double)
%=============================================================================================
% Rebuttal
% Create marging comments for rebuttal
% needs \usepackage{soul} for highlighting (12/2018)
% CHANGE: now replaced soul with ulem
% "\marginpar" does not work for vldb.cls template; requires redefining it with marginnote package "\marginnote" (5/2019)
%=============================================================================================
% \usepackage{soul} % allows strike-through and highlighting, but very fragile
% \usepackage[normalem]{ulem} % use normalem to protect \emph; \hl command is more robust
% \newcommand\hl{\bgroup\markoverwith
% {\textcolor{yellow}{\rule[-.5ex]{2pt}{2.5ex}}}\ULon}
% \newcommand{\rebuttal}[2]{\marginpar{\textbf{\textcolor{black}{#1}}}\hl{#2}}
% \usepackage{marginnote}
% \newcommand{\rebuttalx}[2]{\marginnote{\textbf{#1}}\hl{#2}}
% \newcommand{\rebuttalx}[2]{\marginpar{\textbf{#2}}\hl{#1}} # for some reason does not work: puts text in margin into yellow
% \newcommand{\rebuttalx}[2]{\marginpar{\textcolor{black}{\textbf{#2}}}\hl{#1}} # does not work either: puts the comment one line above
%% FOR VLDB.CLS (5/2019)
\marginparwidth 25pt % Margin pars set to 0pt by vldb template
\usepackage{marginnote}
\newcommand{\rebuttal}[2]{\marginnote{\textbf{\textcolor{blue}{#1}}}{\color{blue}#2}}
% \newcommand{\rebuttal}[2]{\marginnote{\textbf{\textcolor{black}{#1}}}\hl{#2}}
\newcommand{\reviewercomment}[1]{
\begin{tcolorbox}[
enhanced jigsaw, % if box is broken, don't show border at continuation
colback=white!5,
% colback=blue!20,
colframe=white!75!black,
% colframe=gray!40,
arc=0mm,
% boxrule=1pt,
% boxrule=0pt,
% colframe=red!75!black,
left skip=-1mm,
right skip=1mm,
left=0mm,
topsep at break=1mm, % Additional vertical space at the top of middle and last parts in a break sequence
% enlarge top by=0mm, % before box
% enlarge top at break by=5mm, % above box, after pagebreak
right=0mm,
top=0mm,
bottom=0mm, % problem in package with eq at end of env. justadd:\tcbset{bottom=2mm}\tcbset{bottom=0mm}
% after skip=2mm,
breakable, % allows page break
% before upper={\parindent15pt}
parbox = false % restores normal text behavior. ELSE: paragraphs are formatted slightly different as the main text. DOWNSIDE: unwanted side effects
]
\emph{#1}
\end{tcolorbox}
}
\newcommand{\high}[1]{\textup{\textbf{#1.}}}
%==============================================================================================
% Other macros (add here your personalized macros)
%=============================================================================================
\newcommand{\smallsection}[1]{\vspace{2mm}\noindent\textbf{#1.}} % new subsection type: noindent, bold
\newcommand{\introparagraph}[1]{\textbf{#1.}} % define own new subsection type: noindent, bold (textsc)
\newcommand{\ourmethod}{\textsc{OurMethod}\xspace}
\renewcommand{\epsilon}{\varepsilon} % nicer epsilon symbol
\newcommand{\set}[1]{\{#1\}} % Set (as in \set{1,2,3}).
\newcommand{\setof}[2]{\{{#1}\mid{#2}\}} % Set (as in \setof{x}{x>0}).
\newcommand{\define}{:=} % define symbol
% \newcommand{\define}{\stackrel{\textup{\tiny def}}{=}}
% \newcommand{\define}{\triangleq} % define symbol
\newcommand{\datarule}{{\,:\!\!-\,}} % datalog command (March 2010)
\renewcommand{\vec}[1]{\boldsymbol{\mathbf{#1}}}
\newcommand{\transpose}{\intercal} % alternatives: \top \intercal
\newcommand{\N}{\mathbb{N}} % Natural numbers
\newcommand{\R}{{\mathbb{R}}} % Real numbers
\newcommand{\bigO}{{\mathcal{O}}} % Big O notation
\newcommand{\bigTH}{{\mathcal{\Theta}}} % Big O notation
\renewcommand{\O}{{\mathcal{O}}} % Big O notation DEPRECATED
\renewcommand{\P}{{\mathbb{P}}} % Probability \P
\newcommand{\PP}[1]{\mathbb{P} [ #1 ]} % Probability \PP{} -> P[], hollow
\newcommand{\E}{\mathbb{E}}
\newcommand{\sql}[1]{\textup{\textsf{\small#1}}}
\newcommand{\datastyle}[1]{\texttt{#1}}
% \newcommand{\val}{\textup{\textrm{cost}}}
\newcommand{\val}{\textup{\textrm{val}}}
% \newcommand{\val}{w}
\newcommand{\var}[1]{{\textup{\texttt{var}}} ( #1 )} % May 2011 (note naming conflict with variables !!!)
\newcommand{\Var}{\textup{\texttt{Var}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% ORCID icons for authors. Needs to have the image file too in dunnemacros/
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% % Use to create orcid icons and links for authors
% % Usage: \orcidicon{<NAME>}{<ORCID>}
% \newcommand{\orcidicon}[2]{%
% %\hypersetup{urlcolor=black}}{}% don't add colored links to author names
% % \begingroup\normalfont
% \href{#2}{#1 \includegraphics[height=\fontcharht\font`\B]{figs/orcid_16x16.png}}%
% }
\usepackage{scalerel}
\usepackage{tikz}
\usetikzlibrary{svg.path}
\definecolor{orcidlogocol}{HTML}{A6CE39}
\tikzset{
orcidlogo/.pic={
\fill[orcidlogocol] svg{M256,128c0,70.7-57.3,128-128,128C57.3,256,0,198.7,0,128C0,57.3,57.3,0,128,0C198.7,0,256,57.3,256,128z};
\fill[white] svg{M86.3,186.2H70.9V79.1h15.4v48.4V186.2z}
svg{M108.9,79.1h41.6c39.6,0,57,28.3,57,53.6c0,27.5-21.5,53.6-56.8,53.6h-41.8V79.1z M124.3,172.4h24.5c34.9,0,42.9-26.5,42.9-39.7c0-21.5-13.7-39.7-43.7-39.7h-23.7V172.4z}
svg{M88.7,56.8c0,5.5-4.5,10.1-10.1,10.1c-5.6,0-10.1-4.6-10.1-10.1c0-5.6,4.5-10.1,10.1-10.1C84.2,46.7,88.7,51.3,88.7,56.8z};
}
}
\RequirePackage{etoolbox}
\DeclareRobustCommand\orcidicon[1]{\href{https://orcid.org/#1}{\mbox{\scalerel*{
\begin{tikzpicture}[yscale=-1,transform shape]
\pic{orcidlogo};
\end{tikzpicture}
}{|}}}}
%rjm outerjoins
\def\ojoin{\setbox0=\hbox{$\bowtie$}%
\rule[-.02ex]{.25em}{.4pt}\llap{\rule[\ht0]{.25em}{.4pt}}}
\def\leftouterjoin{\mathbin{\ojoin\mkern-5.8mu\bowtie}}
\def\rightouterjoin{\mathbin{\bowtie\mkern-5.8mu\ojoin}}
\def\fullouterjoin{\mathbin{\ojoin\mkern-5.8mu\bowtie\mkern-5.8mu\ojoin}}
%rjm nulls
%\def\missing{\mathbin{\perp \perp}}
%\def\missing{\mathbin{\hat{\perp}}}
\def\missing{\mathbin{\pm}}
\def\produced{\mathbin{\perp}}
\def\schema{\mathbin{\mathcal{A}}}
%\def\schema{\mathbin{sc}}
% ---- For tables ----
\usepackage{array}
\usepackage{makecell}
\usepackage[utf8]{inputenc}
\def\ojoin{\setbox0=\hbox{$\bowtie$}%
\rule[-.02ex]{.25em}{.4pt}\llap{\rule[\ht0]{.25em}{.4pt}}}
\def\leftouterjoin{\mathbin{\ojoin\mkern-5.8mu\bowtie}}
\def\rightouterjoin{\mathbin{\bowtie\mkern-5.8mu\ojoin}}
\def\fullouterjoin{\mathbin{\ojoin\mkern-5.8mu\bowtie\mkern-5.8mu\ojoin}}
% ---- OPERATORS (requires amsmath) ----
% ---- NAMES ----
\newcommand{\Birds}{\textsc{OceaniaBirds}\xspace}
\newcommand{\predicate}{integration ID\xspace}
\newcommand{\Predicate}{Integration ID\xspace}
\newcommand{\predicates}{integration IDs\xspace}
\newcommand{\Predicates}{Integration IDs\xspace}
\newcommand{\ournull}{\perp}
%\newcommand{\labeled}[]{}
% ---- Algorithm comments ----
\newcommand{\cc}{olive}
\newcommand{\algocomment}[1]{\textcolor{\cc}{{//#1}}}
% ---- Notation ----
\newcommand{\attribute}{\mathcal{A}}
\newcommand{\join}{Join\xspace}
\newcommand{\updated}[1]{{{\color{red}{[\textbf{NEW TEXT}: #1]}}}}
\newcommand{\tname}{ALT-Gen\xspace}
\newcommand{\bname}{UGEN-V1\xspace}
\newcommand{\fullname}{Align and Integrate\xspace}
% ---- Symbols ----
\newcommand{\FDset}{{FD}$_{\textrm{tuple-set}}$\xspace}
\newcommand{\ra}[1]{\renewcommand{\arraystretch}{#1}}
\begin{abstract}
% rjm moved to intro due to citation needed
% David Paterson states that “when a field has good benchmarks, we settle debates and the field makes rapid progress”.
Data management has traditionally relied on synthetic data generators to generate structured benchmarks, like the TPC suite, where we can control important parameters like data size and its distribution precisely. These benchmarks were central to the success and adoption of database management systems. But more and more, data management problems are of a semantic nature.
%rjm we aren't doing EM anymore
%For example, determine if two records (tuples) in different databases refer to the same %real world entity (entity matching).
An important example is finding tables that can be unioned. While any two tables with the same cardinality can be unioned, table union search is the problem of finding tables whose union is semantically coherent.
Semantic problems cannot be benchmarked using synthetic data. Our current methods for creating benchmarks involve the manual curation and labeling of real data. These methods are not robust or scalable and perhaps more importantly, it is not clear how robust the created benchmarks are.
We propose to use generative AI models to
%rjm augment and
create structured data benchmarks for table union search. We
%will develop
present a
novel method for using generative models to create
%synthetic data sets
tables with specified properties. Using this method, we create a new benchmark containing
%realistic (not necessarily real) and we generate
pairs of tables
that are both unionable and non-unionable but related.
We thoroughly evaluate recent existing table union search methods over existing benchmarks and our new benchmark. We also present and evaluate a new table search methods based on recent large language models over all benchmarks.
%the existing benchmarks and newly generated benchmark.
We show that the new benchmark is more challenging for all methods than hand-curated benchmarks, specifically,
%In the new benchmark,
the top-performing method achieves a Mean Average Precision of around 60\%, over 30\% less than its performance on existing manually created benchmarks. We examine why this is the case and show that the new benchmark permits more detailed analysis of methods, including a study of both false positives and false negatives that were not possible with existing benchmarks. We discuss how our benchmark and generation method sheds more light into the successes and failures of table union search methods sparking new insights into further research on more robust approaches.
%\rjm{add specific results... include summary of experiments}
%\aamod{added. please make a pass once.}
\end{abstract}
%\roee{Please validate VLDB review format}
%aamod: done, copied from alite.
\section{Introduction}
David Paterson states that “when a field has good benchmarks, we settle debates and the field makes rapid progress”~\cite{patterson2012technical}.
%rjm I'm a little afraid we may offend...
%Database benchmarking is an area of tremendous importance, but one that has largely been static for decades. While the billion-dollar DBMS (Data Base Management System) market owes much to the sophistication and competitive use of standard benchmarks like the TPC suite~\cite{DBLP:books/mk/Gray93}, there has been little systematic advancement in benchmark generation for structured data in decades.
Traditionally, benchmark generation is done using synthetic data generators that precisely control parameters such as data size, data distributions, and correlations in data~\cite{DBLP:books/mk/Gray93}.
%\roee{Do we have a reference for the last statement?}
These parameters influence the standard DBMS performance metrics: response time and through-put. However today, more and more data management challenges require not only the fast and scalable processing of data, but also the understanding of semantics. A common example that we %will
consider in this %project
paper is \emph{table union} -- do two tables contain attributes and relationships with the same semantics so that they can be meaningfully unioned~\cite{nargesian2018table,2023_khatiwada_santos}. For this problem, the most important performance metrics are accuracy and robustness over different data types.
As the semantics of structured data is important in evaluation, synthetic data generators are ineffective. As a result, researchers find and curate (label) real datasets manually. %Examples include for entity matching a dataset derived from bibliographic data, specifically from DBLP and from ACM~\cite{kopcke2010evaluation} and a product dataset containing both Amazon and Google representations of products~\cite{2018_mudgal_deepmatcher}. For table union,
Nargesian et al. created labeled
benchmarks (which we will call TUS-Small and TUS-Large in this work) using real open data (from government open data portals)~\cite{nargesian2018table} which has been reused
for diverse applications, not just the table union problem~\cite{DBLP:conf/icde/BogatuFP020, DBLP:conf/icde/KoutrasSIPBFLBK21, DBLP:conf/edbt/LeventidisRMRG21}.
They took several large tables and sliced them horizontally and vertically to create tables that are unionable on some or all attributes. This approach was later extended by Khatiwada et al.~\cite{2023_khatiwada_santos} (SANTOS-Small and SANTOS-LARGE) and used in other work~\cite{DBLP:journals/pvldb/FanWLZM23, 2023_helali_linked_data, 2023_hu_autotus}.
Another example is the T2K Gold Benchmark, which was created from a large web table corpus~\cite{WDC_t2d} matched with properties from DBpedia and hand labeled. All these efforts require considerable human effort to find appropriate real data and to label the ground truth. For example, while TUS-Large benchmark~\cite{nargesian2018table} offers thousands of ``labeled'' tables, they originate from only 10 seed tables which
%cannot represent a wide variety of topics and data types.
are necessarily limited in the variety of semantics they capture (for example, the topics and data types in the tables). The SANTOS-Large~\cite{2023_khatiwada_santos}, which is large in size and based on real open data tables, offers only 80 human-labeled samples. Also, none of these benchmarks offer a set of non-unionable table pairs and assumes that table pairs that are not labeled as unionable are not unionable. This assumption is made because the original tables are very different.
% but may not always be accurate.
% Furthermore, they are not always reusable, for example, \roee{do we have a good example about table union that can replace the following commented-out text?:} %schema-agnostic entity matching techniques had to create new benchmarks that differ from the standard matching benchmarks that use tables with identical schemas~\cite{DBLP:conf/icde/Simonini0PB18}.
Despite these limitations, the hand-labeled benchmarks are still valuable to the community and have been reused widely.
Importantly, it is not possible to systematically vary many important independent parameters (such as varying the type or semantics of data to test robustness, the problem complexity, or data size) using these human-gathered and hand-labeled benchmarks. In addition, the need for new benchmarks also comes from the effectiveness of state-of-the-art models (Starmie~\cite{DBLP:journals/pvldb/FanWLZM23}) which stands at around 95\% MAP over existing benchmarks. It is possible that these benchmarks are too easy.
Generative AI models are machine learning algorithms that are tasked to predict the next text or related image (or other types of output) based on their input instruction.
They have become increasingly popular in recent years, especially in NLP, where models such as GPT3~\cite{brown2020language} and ChatGPT~\cite{openaichat}
are used by millions of users on a daily basis. These models can be used as is (aka zero-shot learning~\cite{xian2018zero}) or by providing a (small) set of examples that steer the model in the right direction with respect to the task at hand (aka In-Context Learning (ICL)~\cite{brown2020language,dong2022survey}).
Based on their performance on such tasks, these models could provide the key to making innovative new advancements in benchmarking structured data.
\noindent\textbf{Contributions:} In this paper, we employ the capabilities of generative AI models to create and improve benchmark datasets for the table union task. Our contributions can be summarized as follows.
\begin{enumerate}
\item \textbf{Automated LLM Table Generator (\tname):} an automated framework to generate tables for table union search tasks based on a set of desired properties.
\item \textbf{A New Table Union Search Benchmark (\bname):} we generate and share a new table union search benchmark featuring 1050 tables. The benchmark offers a variety of 50 topics ranging from World Geography to Veterinary Medicine, 1000 labeled table pairs (500 unionable/500 non-unionable) allowing fine-grained effectiveness analysis.% and configurable sparsity.
% \item \textbf{Sparsity:} in addition to providing a standard for evaluation and comparison of table union search methods, we show that ALT-Gen can also be used to vary systematically independent variables such as the sparsity of the benchmark (how many null values it contains) or the textuality of the benchmark (whether it contains short text values vs. longer values with many words). \rjm{do we consider domain sizes?}
% The new benchmark is also more challenging with best MAP values around 60\%, potentially triggering further research into better methods.
%\koyena{resolved Roee's comment on specifying this method is currently for table union search}
\item \textbf{Table Union Search Evaluation:}
% \textbf{A Through Experimental Analysis:}
we evaluate and analyze existing table union search methods over our new benchmark and the existing TUS and SANTOS benchmarks. While the best search methods achieve a MAP of over 90\% on existing benchmarks, we show that the new benchmark is more challenging. The best MAP values are around 60\%, potentially triggering research into more robust methods.
\item \textbf{A More Thorough Experimental Analysis:} previous work has used hand-labeled benchmarks as-is. We show that our \tname methodology lends itself to more robust benchmarking by considering other important parameters. Specifically, we vary the sparsity of the \bname benchmark (how many null values it contains) and evaluate methods on a variety of topics.
%the textuality of the benchmark (whether it contains short text values vs. longer values with many words).
%\rjm{do we consider domain sizes?}
%aamod/koyena: we do not have textuality experiments. updated.
We show that by using \tname, we can do ablation studies on these parameters to understand the table union search methods even better, something that has not been done before.
%\rjm{please improve my wording...}
%aamod: corrected typo and reworded.
\item \textbf{New Table Union Search Method:}
in generating a benchmark using an LLM for table union search, an important question is whether an LLM is better at table union search (especially on its own benchmark) than the best existing methods.
An LLM can easily be used to classify two tables as unionable or non-unionable, but not for table union search (given a query table, find tables within a massive repository that are unionable with it). So to answer this question, we present a new table union search method that uses a state-of-the-art search technique to find a set of candidates tables (Starmie~\cite{DBLP:journals/pvldb/FanWLZM23}) and then uses an LLM to classify table pairs among the candidates as unionable/non-unionable.
Our in depth analysis of the performance of this new method with existing methods highlights some important and interesting areas for future work and improvement.
% Our effectiveness analysis indicates that the new benchmark is, in fact, challenging and out-of-the-box LLMs are not sufficient to outperform state-of-the-art table search methods.
% \item We develop a method to extend table union search tasks to utilize LLMs for another task, i.e., table union classification.
% \item We share a new benchmark created using ALT-Gen and evaluate it against state-of-the-art methods for a particular inter-table task, i.e., table unionability.
\end{enumerate}
In Section~\ref{sec:related}, we survey related work. Section~\ref{sec:benchmarkgen} presents our methodology to generate benchmarks and the new benchmark. Section~\ref{sec:exp} provides the analysis and evaluation.
% \roee{Add a general description of what we do in this paper.} Our contributions can be summarized as the following. \roee{Polish the contributions.}\roee{I would go with (1) a method to generate tables for table union search, (2) a new table union search benchmark UGEN-V1 that offers more labeled table pairs, a selection of pre-defined topics, variable sparsity, ...., (3) a through analysis of table union search methods over the traditional and the new benchmark including an LLM-based method}
% In what follows, we employ the capabilities of generative AI models to create and improve benchmark datasets for the table union task.
%Roee: the intro should be spesific for our paper
%structured data management tasks.
%Moreover, we will develop the science of {\em semantic benchmark creation and augmentation}.
% We use generative AI to systematically enhance existing benchmarks or real datasets in a way that permits a data scientist to systematically vary independent variables (properties) of interest.
% Generative AI models can create realistic (somewhere between real and synthetic) data samples in a systematic manner (via prompting).
% We will develop techniques for prompting that enable control over desired properties of the generated data. Specifically, our main objectives for this project are to:
% \noindent$\bullet$ develop a prompting methodology for augmenting existing structured data benchmarks;\\
% \noindent$\bullet$ understand how to use generative AI to create new benchmarks for structured data tasks; and \\
% \noindent$\bullet$ wrap a generative AI model with a user friendly prompted-based framework that is able to generate on-demand (challenging) data samples for structured data tasks.\\
% Our contributions can be summarized as the following.
% \begin{enumerate}
% \item We propose ALT-Gen (Automated LLM Table Generator), an automated framework to generate tables for table union search tasks. \koyena{resolved Roee's comment on specifying this method is currently for table union search}
% \item We develop a method to extend table union search tasks to utilize LLMs for another task, i.e., table union classification.
% \item We share a new benchmark created using ALT-Gen and evaluate it against state-of-the-art methods for a particular inter-table task, i.e., table unionability.
% \end{enumerate}
\section{Related Work}\label{sec:related}
We first present the state-of-the-art methods in table union search (including methods that appear in our experimental study). We then consider related work on benchmark creation in data management tasks, including benchmark generation for semantic tasks like data cleaning and data integration. Finally, we introduce generative models that we use for our new benchmark creation methodology and the way they are used in the literature.
% \subsection{Union Table Search}
\subsection{Table Union Search}
Given a query table by the user, %the unionable
table union search techniques intend to find the data lake tables that can add new rows to the query table~\cite{2023_khatiwada_santos}. We cover several search techniques that were developed over the last few years aiming at finding %There are existing works that search for the
unionable tables over the data lakes.
\citet{nargesian2018table} considered two tables to be unionable if they have a subset of unionable columns. The column unionability is determined by using an ensemble of three statistical tests based on value overlap, semantic overlap, and word embedding of the column values.
\textbf{D$^3$L}~\cite{DBLP:conf/icde/BogatuFP020} extended that work~\cite{nargesian2018table} by considering, in addition to value overlap and word embedding measures, three additional attribute unionability measures (column header similarity, numerical value distribution, and regular expression similarity). \textbf{SANTOS}~\cite{2023_khatiwada_santos} considered the similarity of both columns and the binary relationship between the columns to make table unionability decisions. The binary relationships help SANTOS to understand table context better and omit the unioning of the tables having similar columns but different contexts~\cite{2023_khatiwada_santos}. %Recently,
\textbf{Starmie}~\cite{DBLP:journals/pvldb/FanWLZM23} uses a contrastive-learning approach to capture the contexts of the entire table rather than just the binary relationships, and use them to search for the unionable tables. Starmie captures the table context in the form of column embeddings. In very recent work, \citet{2023_hu_autotus} used the contextualized representation of the relationship between the column pairs to capture the table contexts and use them to find the unionable tables from the data lakes (however, open code is not available for this work).
%Unlike column-based approaches, SANTOS~\cite{2023_khatiwada_santos} considered the similarity of both columns and the binary relationship between the columns to make table unionability decisions. The binary relationships help SANTOS to understand table context better and omit the unioning of the tables having similar columns but different contexts~\cite{2023_khatiwada_santos}. Recently, Fan et al.~\cite{DBLP:journals/pvldb/FanWLZM23} introduced Starmie, which uses a contrastive-learning approach to capture the contexts of the entire table rather than just the binary relationships, and use them to search for the unionable tables. Starmie captures the table context in the form of column embeddings. More recently, Hu et al. used the contextualized representation of relationship between the column pairs to capture the table contexts and use them to find the unionable tables from the data lakes~\cite{2023_hu_autotus}.
The benchmarks used (and created) in the respective papers and their limitations are discussed in the introduction. Importantly, their creation and evaluation involve a significant amount of manual annotation. In this work, we present a principled way to use generative models, aiming to eliminate such manual workload during benchmark creation.
% Note that all these works create or use benchmarks for their evaluation that involves a significant amount of manual annotation. In this work, we aim to present a principled way to use generative language models to eliminate such manual workload during benchmark creation.
%- Describe 4 or 5 methods (TUS, D3L, SANTOS, Starmie, (Pylon?))
%SANTOS~\cite{2023_khatiwada_santos}
%\subsection{Manually Curated Benchmarks}
% \begin{enumerate}
% \item Manually Labeled (Real data that is manually labeled for a specific task)
% \item Synthetically Created (more like ablation?)
% \item Bart (injecting errors? dirty data generator)
% \end{enumerate}
%aamod: completed bart and ibench details.
% \aamod{As we have already covered the manually curated benchmarks in the intro section,I did not find additional papers than LakeBench to cite in this section. Should we include redundant discussion? I think we can safely merge sections 2.2 and section 2.3 and call that Benchmark Curation.} \rjm{Yes go ahead}
%aamod: resolved.
% A recent data discovery benchmark includes Lakebench which contains 8 open datasets for joinability and unionability tasks~\cite{srinivas2023lakebench}. Different from our work where we generate tables for \emph{table search} task using a generative model, Lakebench benchmarks existing open datasets and new datasets created using Knowledge graphs, for fine tuning large language models.
\subsection{Benchmark Generation in Databases}
Benchmark generation has been considered for many data management tasks.
In data cleaning, \citet{2015_arocena_bart} introduced BART which can be used to add errors into clean databases and evaluate data-repairing algorithms.
%aamod: completed.
In data integration, iBench~\cite{2015_arocena_ibench} was introduced to generate schemas and schema constraints with arbitrarily large and complex mappings.
%In data integration, iBench~\cite{} did Y.
%aamod: added.
These systems start with real data and metadata and systematically vary specific parameters that can influence the performance of cleaning or integration systems. In a similar spirit, we show how \tname can generate realistic data and how we can use \tname to vary certain parameters (such as textuality) during the generation process. We also consider how to post-process the generated result to vary other parameters such as sparsity.
Others have used knowledge graphs~\cite{2021_cutrona_semtab}, Git repositories~\cite{2023_hulsebos_gittables}, and the web~\cite{brinkmann2023web} to generate tabular benchmarks.
For example, the SemTab challenge~\cite{2021_cutrona_semtab}, which focuses on the semantic annotation of the tabular datasets using knowledge graphs, generates benchmarks using Wiki data
%\rjm{please explain this better...}
%aamod: added the details on what semtab challenge is about.
for important intra-table tasks such as column type annotation and column-column relationship annotation. However, limited value coverage in the knowledge graph~\cite{2023_khatiwada_santos} could pose a challenge for creating diverse benchmarks and this is an area where generative models could be helpful. Another example is The Web Data Commons project~\cite{brinkmann2023web} which extracts schema.org~\cite{guha2016schema} data from the Web using Common Crawl~\cite{commoncrawl}. This project yielded several successful benchmark for multiple tasks such as Product Matching. The extraction is limited to a specific schema and still requires manual annotation.
A recent data discovery benchmark includes Lakebench which contains 8 open datasets for joinability and unionability tasks~\cite{srinivas2023lakebench}. Different from our work where we generate tables for \emph{table search} task using a generative model, Lakebench benchmarks existing open datasets and new datasets created using Knowledge graphs, for fine tuning large language models.
\subsection{Generative Models and LLMs}
Generative models are machine learning models that are trained to learn their training data distribution. Based on this learned pattern and given input, they generate text, images, videos, and more. They have been used to augment training data in various related tasks such as commonsense reasoning~\cite{yang2020generative}, event detection~\cite{Veyseh2021AugmentingOE}, text classification and summarization~\cite{ chintagunta2021medically}. More recently, researchers have coined the term large language models (LLMs) which refers to generative language models in general. In this paper, we use Open AI's GPT3~\cite{brown2020language} for benchmark generation and also experiment with GPT2-xl, Alpaca~\cite{alpaca} and Vicuna~\cite{vicuna} which are open source via Hugging Face~\cite{wolf2020transformers}.
LLMs have been used within the data management community extensively. For example,~\citet{arora2023language} use LLMs to generate structured views of semi-structure data lakes. Others use LLMs to extract knowledge graphs~\cite{west-etal-2022-symbolic, cohen-etal-2023-crawling}. Trummer used GPT-3 to generate code for query processing~\cite{trummer2022bert,trummer2022codexdb}. Recently there were some additional initial attempts to solve other data management tasks such as information extraction~\cite{brinkmann2023product} and entity matching~\cite{peeters2023using} using prompting and in-context learning. But to the best of our knowledge, generative models have not be used in generating benchmarks for semantic data management tasks.
% In terms of relational data, there have been previous works where authors try to extract knowledge graphs~\cite{west-etal-2022-symbolic, cohen-etal-2023-crawling}, particularly those within large language models. With respect to database management, Arora et al. ~\cite{arora2023language} utilises LLMs by prompting it to generate structured views of semi-structure data lakes.
% Generative models are machine learning models that are trained to learn their training data distribution. Based on this learned pattern and its input, it would generate content, which can be in the form of textual, audio, or visual in nature. They have been used to augment training data in various related tasks such as commonsense reasoning~\cite{yang2020generative}, event detection~\cite{Veyseh2021AugmentingOE}, text classification, and summarization~\cite{ chintagunta2021medically}. In terms of relational data, there have been previous works where authors try to extract knowledge graphs~\cite{west-etal-2022-symbolic, cohen-etal-2023-crawling}, particularly those within large language models. With respect to database management, Arora et al. ~\cite{arora2023language} utilises LLMs by prompting it to generate structured views of semi-structure data lakes.
% \roee{We need to add here a description of what was done in the DB community with LLMs and how it is related. Specifically works by Immanuel Trummer} -- KP: CodexDB (https://openproceedings.org/2021/conf/edbt/p82.pdf); From BERT to GPT-3 Codex: Harnessing the Potential of Very Large Language Models for Data Management;
% LLMs have been used within the data management community extensively. For example, Arora et al. ~\cite{arora2023language} use LLMs to generate structured views of semi-structure data lakes. Others use LLMs to extract knowledge graphs~\cite{west-etal-2022-symbolic, cohen-etal-2023-crawling}. Trummer...
% \rjm{Yes I agree, @Roee can you take a shot at completing this?}
% But to the best of our knowledge, generative models have not be used in generating benchmarks for semantic data management tasks.
\section{Generative Union Benchmarks}\label{sec:benchmarkgen}
\begin{figure*}
\centering
\includegraphics[scale = 0.55]{figures/pipeline.pdf}
\caption{Pipeline for generating benchmarks through Large Language Models}
\label{fig:pipeline}
\end{figure*}
Before presenting our benchmark generation method, we first consider what properties a good benchmark for table union search should have. We then present our generative benchmark methodology \tname and %. Finally, we present
our new benchmark \bname.
%\rjm{Please make names macros so we can change them consistently throughout paper.}
%aamod: done!
%We now consider the problem of benchmarking table union searhc and what properties a good benchmark should have.
% \roee{I would use a different color (and/or color blind friendly gesture) to denote the parameters in the prompt.} \roee{Assuming the example is based on real data, I would make it available in the repository and refer the readers to the full versions of the tables.}\roee{Also, although I like the examples, I think it may be more "impressive" or representative wrt our contributions to have tables with Nulls and numeric values. This refers also to my above comment about the properties that we are looking to satisfy.}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/unionable_example.pdf}
% \caption{Example of generating an unionable table pair using GPT-3}
% \label{fig:unionable_example}
% \end{figure*}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/nonunionable_example.pdf}
% \caption{Example of generating a non-unionable table pair using GPT-3 \roee{see notes in above figure}}
% \label{fig:nonunionable_example}
% \end{figure*}
\subsection{Desired Properties for Union Benchmarks}\label{sec:properties}
% \roee{Add a description of what we are trying to achieve here. Also rethink the title -- we are focusing on table union. Also we need to think whether we actually do something that aims to achieve each objective we state.}
%rjm resolved \rjm{need to tie into TUS problem}
Following Primpeli and Bizer~\cite{primpeli2020profiling}, who presented a set of properties that should be considered within (hand curated) entity matching benchmarks, we present a set of properties the union benchmarks should have. The first (realism) is a required property. Beyond this, for the other properties, we argue that a benchmark generator should be able to vary the degree or value of the property systematically.
\noindent$\bullet$ \textbf{Realism:} the ability to generate ``realistic'' and semantically meaningful (not necessarily ``real'') tuples and structured tables. Due to this property, all union benchmarks to date have been created from real open data sets rather than using synthetic data. We require that a benchmark generator be able to create realistic data rather than randomly associating values as a synthetic data generator might. \\
\noindent$\bullet$ \textbf{Typing:} the ability to generate a variety of types of values (e.g., numeric and textual). This is important as solutions for semantic data management problems often differ based on types. A benchmark generator should be able to generate tables with a subset of types (for example, only categorical values), or with varying degrees of different types (for example, half numeric and half strings). \\
% \noindent$\bullet$ \textbf{Cardinality:} the ability to generate attributes with a specified cardinality. Note that uniqueness (a key attribute) is a specific case of this property. A benchmark generator should be able to create tables with different cardinalities from keys to binary values attributes.\\
% \rjm{again please update this is we don't do this on cardinality}
\noindent$\bullet$ \textbf{Incompleteness:} the ability to generate incomplete data containing nulls. A benchmark generator should be able to create complete tables and systematically vary the degree of incompleteness (number of nulls) contained in a table.
% \rjm{is any of the following in this paper? If so, edit to reflect what we do. If not, remove or put in future work}
% \noindent$\bullet$ \textbf{Distribution:} the ability to generate a distribution of values in a column/table.\\
% \noindent$\bullet$ \textbf{Functional dependencies:} the ability
% to generate data satisfying a functional constraint.\\
% \noindent$\bullet$ \textbf{Noise:} the ability to generate data not satisfying (or partially satisfying) a constraint such as a functional dependency.\\
% \noindent$\bullet$ \textbf{Responsible Generation:} An additional property we aim to explore is whether we can also make sure the generated tables follow certain properties of fairness (unbiased data)~\cite{le2022survey}, confidentiality (non-private)~\cite{dwork2011firm}, accuracy (the data in the tables is correct), and transparency (some form of explanation to why the tables were generated the way they were)~\cite{arrieta2020explainable}.
To evaluate whether the benchmark generator is able to achieve these properties, we generate a benchmark through this generator and evaluate the effectiveness of the benchmark itself. We adopt the ~\citet{primpeli2020profiling} dataset profiling approach (for ours and existing benchmarks) and consider the following factors:
\noindent$\bullet$ \textbf{Schema Complexity:} This factor refers to the number of attributes contributing to solving the unionability task. \\%an inter-table task. \\%\koyena{for these properties, do we leave it at inter-table or change it to unionability?} \roee{Yes. I will update.} \\
\noindent$\bullet$ \textbf{Textuality:} This refers to the average length of words of the %top-contributing
attribute values. \\
\noindent$\bullet$ \textbf{Sparsity:} This dimension refers to the ratio of missing attribute values in the tables considered for unionability. \\ %to all relevant attribute values of records that appear %in the correspondence set of an inter-table task. \\
\noindent$\bullet$ \textbf{Development Set Size:} This refers to the amount of positive and negative record pairs in the
%rjm we don't have training and optimization here, can we just say benchmark?
%training and optimization sets
benchmark for the unionability task.\\%of an inter-table task.\\
\noindent$\bullet$ \textbf{Corner Cases:} This factor refers to the ratio of false positive and false negative cases, which are referred to as corner cases. This is a method-specific that largely looks at the ratio of challenging cases in a benchmark for each method tested on the benchmark. \\%compared to all inter-table task instances in the benchmark. \\
%aamod: \aamod{if previous works have not shortlisted properties in this systematic way, we can claim this as an important contribution.} \koyena{the profiling benchmark paper already did for entity matching task -- repurposing that here}
%aamod: sounds good.
% \begin{example}
% \label{ex:example_example}
% Example example @koyena, please remove it.
% \end{example}
We propose to use a generative model (specifically an LLM) to automate annotation and ensure inter-table unionability, thereby improving benchmark diversity and scalability.
The benchmark will be comprised of two sets of tables, representing data lake and query tables. A ground truth file will also be included, which will provide information about which data lake tables can be unioned with each given query table and which cannot.
% \subsection{Benchmark Generation}\label{sec:benchmarkgentemplate}
\subsection{Unionability Benchmark Generation}\label{sec:benchmarkgentemplate}
Existing hand-curated benchmarks are fairly homogeneous and relatively small (the largest fully labeled benchmark has 5,000 tables~\cite{nargesian2018table}) as they are generated from a few large \emph{seed} tables that are then sliced (horizontally and vertically) to create unionable tables~\cite{nargesian2018table}. Using generative models, we do not have to rely on \emph{seed} tables. Instead, we will navigate the model to generate pairs of unionable (and non-unionable) tables via prompting and in-context learning.
\subsubsection{\tname: Automated LLM Table Generator}\label{sec:benchmarkgentemplate}
\Cref{fig:pipeline} illustrates the pipeline of generating tables using LLMs.
We prompt an LLM (for instance, GPT3) to generate pairs of tables with the following set of features customized for each table: topic, number of rows and columns (shape), and %finally,
the textuality rate. To generate tables for a unionability benchmark, we add additional properties to the prompt such as whether the table pairs are supposed to be unionable or not. In cases where they are unionable, we prompt the LLM to generate both the tables as well as the key column name in the query table (table 1), which act as the primary column containing shared data points with a paired data lake (table 2) column.\footnote{A key or intent column is required by one of the search techniques that we will evaluate~\cite{2023_khatiwada_santos}.} % \rjm{I don't understand last sentence and I don't see it in the figure.} \koyena{rewrote the last couple sentences. It is shown in Figure 2 (left hand side) as `key: spacecraft'} The template of such a prompt looks like the following: \\
\resultbox{
\texttt{Create 2 tables with cells separated by |. Table 1 has \{$T1_{row}$\} rows, \{$T1_{col}$\} columns and \{${T1_{textuality}}$\} words, related to \{${T1_{topic}}$\}. Table 2 has \{$T2_{row}$\} rows, \{$T2_{col}$\} columns and \{${T2_{textuality}}$\} words, related to \{${T2_{topic}}$\}.}
}
\noindent In the prompt \{$T1_{row}$\}, \{$T1_{col}$\}, \{${T1_{textuality}}$\} and \{${T1_{topic}}$\} (and similarly for $T2$) are parameters. These parameters are currently instantiated randomly during the `create table-pair properties' phase in \Cref{fig:pipeline}, but can be set by users if a table-pair needs to have certain user-specific conditions. %\roee{@Koyena, please add here where and how we instantiate them} \koyena{added a sentence to describe a bit} \roee{resolved.}
Depending on whether these tables should be unioned or not, we add either of the relevant sentences to the prompt.
% \begin{center}
\resultbox{
\texttt{ They can be unioned because they have only \{$T_{unionable\_col}$\} semantically common columns, and at least 1 related row values across the tables.}
}
% \end{center}
% \begin{center}
\resultbox{
\texttt{ They cannot be unioned because they have $0$ semantically common columns.}
}
% \end{center}
The unionability-specific instructions can be switched to another inter-table task (such as entity matching or joinability). For instance, in the case of entity matching, we could state that the data points ``can be entity-matched" or ``cannot be entity-matched." This generalizes the pipeline while creating specialized tables for various inter-table tasks.
To allow for increased consistency in the format of the output generated by an LLM%\rjm{should we be saying GPT-3 here or LLM? Is LLM specific enough or should be say "generative model"?}\koyena{changed to LLM. I think LLM is specific enough is it is a subtype of generative model i believe}
, we add the following instruction to the prompt template:
% \begin{center}
\resultbox{
\texttt{Answer the above task in the following format: \\
Table 1: \{table 1\} \\
Table 2: \{table 2\} \\
Key: \{key column in Table 1\}}
}
% \end{center}
\noindent In this prompt, ``Key:..." is included only if the table pair should be unionable. Note that relevant methods can use this key when necessary, for example, SANTOS~\cite{2023_khatiwada_santos} can use the key as the intent column of the table.
% \rjm{explain why we ask for a key}\koyena{after making the change in an earlier comment in 3.2, it includes reasoning for the key. Should we move that down here?}
After we create this prompt, we input this to the LLM, which then outputs the tables (and key column value, if unionable) as a string. We can then implement a post-processing script to convert the string-formatted tables into CSV files.
Apart from the size of the tables and the benchmark itself, we also consider the sparsity present in each table. Because this feature involves removing information instead of adding any new information, we automate the process of generating such tables through an algorithm that replaces cell values with a null (empty) value. Although sparse tables can be generated by LLM, it is more consistent and cost-efficient to create non-sparse tables using the model and multiple sparse ones using an algorithm.
\begin{figure*}[h]
{
% \centering
\begin{minipage}[t]{0.5\textwidth}
\includegraphics[width=\linewidth]{figures/unionable_example.pdf}
\end{minipage}%
% \centering
\begin{minipage}[t]{0.5\textwidth}
\includegraphics[width=\linewidth]{figures/nonunionable_example.pdf}
\end{minipage}%
\caption{Example of generating pairs of unionable and non-unionable table pairs through \tname.}
\label{fig:generation-example}
}
\end{figure*}
To create diverse and realistic benchmark, we prompt the LLM to generate %a set of $n$
tables of different topics. An important property, which is not
%rjm apparent
%rjm apparent implies present but just not visible/obvious
present
in other benchmarks, is the ability to generate non-unionable tables of the same topic.
%that these tables should describe.
%r4jm Hence, it comes up with $n$ topics where $n$ is the number of topics that we prompt for while requesting for topic themes.
% \rjm{Is the following true? Above we say we prompt the model for non-unionable pairs.} \koyena{we do prompt for non-unionability pairs, but for same domains...although we didn't ultimately augment it...but we could do so by pairing different topic tables to create non-unionable ones}
% Similar to the case of generating sparse tables, LLM is capable of producing non-unionable pairs of tables with different topics. %However, since we are already generating such tables in other runs, we have the option to augment our benchmark by pairing existing generated tables from different topics to create non-unionable table pairs. \koyena{we can alternatively comment this part out -- the one talking about non-unionability between 2 different topic themed tables -- since we didn't ultimately augment it}
% \rjm{is the following correct?} \koyena {yes}
Unlike hand-curated benchmarks where any pair of tables that are not explicitly created to be unionable (that is two tables that are created from slicing and dicing the same large seed table) are assumed to be non-unionable, our generation method, \tname,
%\rjm{do we have name for the generation method?} \koyena{yes just added that}
explicitly creates pairs of non-unionable tables.
If each seed table is selected to be very different from the others (and on a different topic) as done in both the TUS~\cite{nargesian2018table} and SANTOS~\cite{2023_khatiwada_santos} benchmarks that we compare against, then the non-unionable tables may be unrealistically easy to determine.
The benefits of our approach is we get varied examples from a variety of topics that would allow for more realistic unionable and non-unionable table-pair examples (including tables on the same topic as a query table that do not union with it) that we can test for unionability. This creates opportunities for current and future table union search method to consider harder cases. %\roee{complete} \koyena{not sure if this is what you wanted to say, but added a 'benefit' or two}
We now provide an example illustrating prompts and the corresponding generated tables.
\begin{example}
\label{ex:running_example}
\Cref{fig:generation-example} provides two generation examples including a prompt and a pair of tables. On the left a pair of unionable tables is generated and on the right a pair of non-unionable tables is generated. Each prompt provides a set of properties for the model to satisfy and whether the generated tables can (left) or cannot (right) be unioned. In this case both tables contain real data. For example, \texttt{Apollo 11}'s launch system is \texttt{Saturn V} and its orbit is \texttt{Earth-Moon}. Potentially the generative model can generate realistic data that is not real. For example, a record about the \texttt{Hubble Space} spacecraft with the launch system \texttt{Solaris Rocket} that has a \texttt{Low Earth Orbit} and \texttt{Orion MPCV} payload. While the orbit and payload are real, the Hubble Space spacecraft and its launch system are made up. This can be important in real-world applications where real data can be considered as a privacy leakage~\cite{carlini2021extracting,carlini2022quantifying}.
%\roee{Is there a way to generate an example row that has realistic (non-real) information? we don't need to add it to the table, just to give here an in-line example here.} \koyena{I am little confused by this...isn't it a realistic information already? or did u mean a completely false one but looks really real?}
The unionable tables (left) are related to \texttt{Astronautics} and share (at least) 4 unionable (semantically common) columns, out of which \texttt{Spacecraft} and \texttt{Orbit} are visible in the figure. The provided key of the tables (denoted under Table 2) is \texttt{Spacecraft}.
% \footnote{\roee{???} Please see the repository for the full tables}. [Removed bc not sure where that is yet...but can add back if i find it]
Notably, the columns overlap (e.g., \texttt{Appollo 11}) but not completely and contain multiple types (e.g., \texttt{RPM} is numeric and \texttt{Date Launched} is a date). %\roee{Anything else?} \koyena{there is technically one more...duration of mission and mission duration, it appears later in the table...so didn't include}
The non-unionable tables (right) are both related to \texttt{Astronomy} and do not share any unionable (semantically common) columns. Importantly, these two tables are not unionable but are of the same topic i.e. \texttt{Astronomy}.
%Such samples do not exist in any of the traditional benchmarks.
Such samples are not labeled explicitly in the traditional benchmarks.
% \roee{???} \koyena{not sure what you meant for ???} \roee{Making sure that my statement is true. @Aamod? @Renee?}
% \aamod{We cannot claim that such samples were not there, it's just that they were not labeled.}
We envision such examples to be the most challenging ones for contemporary methods which are built on top of identifying semantics, which can be mistakenly thought of as topic. Also here we see several data types such as distances (\texttt{Distance} and \texttt{Equatorial Radius}) and, interestingly missing values (last row in \texttt{Mean Temperature}). The latter being another property we aimed for in a benchmark (incompleteness).
% \roee{Anything else?} all good
\end{example}
% \subsection{Unionability Benchmark (\bname)}
\subsubsection{\bname: A New Unionability Benchmark}\label{sec:new_unionability_benchmark}
\begin{table*}
% \small
% \centering
\ra{1.1}
\caption{Overview of Union Table Search Benchmarks}
\begin{tabular}{@{}ll l l l l l@{}}
\toprule
\multicolumn{2}{@{}l@{}}{\textbf{Benchmark}} &
\textbf{SANTOS-Small} &
\textbf{SANTOS-Large} &
\textbf{TUS-Small} &
\textbf{TUS-Large} &
\textbf{UGEN-V1} \\
\hline
\multicolumn{2}{@{}l@{}}{\textbf{\# Tables}} & 550 & 11166 & 1655 & 4295 & 1050 \\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{2}{*}{\makecell[l]{\textbf{\# Average Shape}\\ \texttt{rows x columns}}}} &
\textbf{Query} &
21401.70 x 12.30 &
12917.23 x 12.71 &
4423.20 x 12.88 &
1996.22 x 11.85 &
8.34 x 11.00 \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{Datalake} & 6921.38 x 11.49 & 7684.54 x 10.98 & 4457.10 x 9.68 & 1996.22 x 11.85 & 7.81 x 10.41\\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{2}{*}{\textbf{Size}}} & \textbf{Query} & 127.7 MB & 143.1 MB & 139 M & 1.5 GB & 205KB \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{Datalake} & 442.4 MB & 11.48 MB & 1.2 GB & 1.5 GB & 4.1MB \\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{2}{*}{\textbf{\# Labeled Pair}}} & \textbf{Unionable} & 695 & 80 & 230921 & 1202826 & 500 \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{Non-unionable} & 0 & 0 & 0 & 0 & 500 \\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{8}{*}{\makecell[l]{\textbf{Attributes}\\ \texttt{(query,datalake)}}}} &
\textbf{\# Short Str} &
(79, 568) &
(60, 8541) &
(168,3752) &
15740 &
(5, 193) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{\# Medium Str} & (90, 1048) & (198, 26081) & (156,1785) & 10394 & (56, 904) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{\# Long Str} & (299, 2995) & (491, 55313) & (1107,7909) & 21092 & (441, 8559) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{\# Num} & (139, 1496) & (169, 22350) & (179,1364) & 3208 & (32, 400) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{Avg. Density} & (99.94\%, 99.92\%) & (99.83\%, 99.78\%) & (99.94\%,99.96\%) & 99.87\% & (94.02\%, 93.50\%) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{\# Sparsely Unique} & (481, 4777) & (764, 89375) & (1173,10505) & 39787 & (388, 6463) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{\# Moderately Unique} & (117, 1380) & (240, 28938) & (428,3623) & 9177 & (161, 3943) \\
% \cline{2-7}
\multicolumn{1}{@{}l@{}}{} & \textbf{\# Densely Unique} & (17, 165) & (13,3371) & (9,682) & 1928 & (1, 3) \\
\bottomrule
\end{tabular}%
% }
\label{tab:union-overview}
\end{table*}
% \rjm{This paragraph is good but belongs at start of 3.2? Let's include in this subsection only a description of WHAT we created (not how) - i.e., describe UGEN} \koyena{moved the paragraph to the start of 3.2}
Using GPT3 (as the LLM model) and prompt template described in \Cref{sec:benchmarkgentemplate}, we generated 1000 pairs of tables (half unionable, half non-unionable), which took approximately 10 hours and \$18 USD. These tables covered 50 topics ranging from World Geography and Art History to Genealogy and Veterinary Medicine. For each topic, we generated 1 query table and 20 other data lake tables that are either unionable or non-unionable with the query table. If we create $I$ versions of each table with different levels of sparsity then we have $1050 * I$ tables and at least $1000 * I$ labeled pairs of tables in the benchmark. For example, if $I = 5$, we create 5 versions of each table which may, for illustration, include a complete table, one with 5\% nulls, one with 10\% nulls, one with 15\% nulls, and one with 20\% nulls.
\bname and our generation code \tname are publicly available at \url{https://github.com/northeastern-datalab/alt-gen}. %\url{https://github.com/northeastern-datalab/alt-gen/tree/main/data/ugen_v1}. %\roee{@Koyena, maybe generate a sub-folder in the main repo and refer there instead.}.
% To include table pairs that cover non-unionability in different domains, we pair query tables from different topics as well as query tables with data lake tables from a different topic.
% \rjm{we don't pair data lake topics from two different topics?} \koyena{no we don't} This adds 15,295 more non-unionable pairs for a total of XXX non-unionable pairs in our benchmark.
% \rjm{Why 15,750 what are the other 455? I thought we "generated" 500 non-unionable pairs, not 455???} \koyena{we do generate 500}
% The results in the total of 15,750.
% \rjm{Upper bound on what? And don't use k since k is for Top-k. Do you mean instead: "If we create $I$ versions of each table with different levels of sparsity then we have 1,050 tables and $15,295 * I$ labeled pairs of tables in the benchmark. For example, if $I = 3$, we create 3 versions of each table which may, for illustration, include a complete table, one with 10\% nulls and one with 20\% nulls.} \koyena{yes we mean this} We can further create an upper bound of $15295 \cdot k$ tables where $k$ is the number of sparse tables we create for each original table in the benchmark.
% \rjm{if we $15295 \dot k$ pairs, why doesn't the table say that? Is this another version of UGEN?} \koyena{we don't have 15295...in this section, i believe we were just trying to indicate how this can be scaled up even more with a couple of different ablations}
% \koyena{commented out the other bits since we didn't augment those and hence, might be confusing}
\subsection{Comparison of Unionability Benchmarks}
In ~\cref{tab:union-overview}, we provide an overview of the new benchmark (\bname) as well as other unionability benchmarks. We provide the following features for each benchmark:
\begin{enumerate}
\item \# Tables: Total number of tables in the benchmark.
\item \# Average Shape: Average number of rows and columns in the tables, including query tables and data lake tables.
\item Size: The total size/storage space of the benchmark.
\item \# Labeled Pairs: For each benchmark, there is a ground truth file that indicates which pairs of tables are unionable or not. It can be a non-exhaustive list since non-unionability, for instance, can be inferred by permuting and pairing all other tables that are not stated to be unionable (though this may include false negatives). Nevertheless, we record the number of unionable and non-unionable pairs mentioned in the ground truth file.
\item Attributes: In the attributes section of the table, we give the following properties of the query tables (first number) and data lake tables (second numbe) in the benchmarks:
\begin{enumerate}
\item \# Short Str: Number of columns in the benchmark that have an average length of strings that is less than 3.
\item \# Medium Str: Number of columns in the benchmark that have an average length of strings less than 6, but greater than or equal to 3.
\item \# Long Str: Number of columns in the benchmark that have an average length of strings greater than or equal to 6.
\item \# Num: Number of columns in the benchmark with a numerical data type.
\item Avg Density: Average percentage of non-null values present in columns of the benchmark.
\item \# Sparsely Unique: Number of columns that have less than 10\% of the unique values in the benchmark.
\item \# Moderately Unique: Number of columns that have less than 50\% but greater than 10\% of the unique values in the benchmark.
\item \# Densely Unique: Number of columns that have greater than 50\%, of the unique values present in the benchmark.
\end{enumerate}
%rjm moved earlier The table shares the above attribute-related statistics in the form of a tuple, where the first one describes the query table dataset, while the second part illustrates the datalake table set of the benchmark.
\end{enumerate}
One of the unique features included in the new benchmark is the presence of labeled non-unionable pairs. Past benchmarks have inferred non-unionability by considering pairs of tables that are not otherwise labeled as unionable. While we can infer additional non-unionable tables in the \bname benchmark as well, we have labeled non-unionable pairs that allow us to perform more in-depth analysis with respect to non-unionability (see \Cref{sec:nonunionability-analysis}). Compared to past benchmarks, \bname has the smallest size of tables in terms of average number of rows and columns in both query and data lake sets of tables. Despite its size, it is a more challenging benchmark as shown in the next section. Unlike previous benchmarks, \bname has higher textuality rate, which is evident in the comparatively higher long string ratio, and higher sparsity, which is indicated with a comparatively lower density value. These are contributing factors that make \bname more difficult. Finally, we note that the new benchmark also provides topics and, specifically, non-unionable table pairs of the same topic.
% \roee{TODO: Add a few insights from the table. what are the important things? (1) labeled non-unionable tables, (2) smaller tables (which we show are more challenging in the next section), (3) topics! (not in the table, should we add?), (4) density?, (5) longer strings? anything else?} \koyena{made a pass for it} looks great!
\section{Analysis and Evaluation}\label{sec:exp}
We now perform empirical analysis over the existing benchmarks and our new benchmark created using generative model. We start by explaining the evaluation tasks~(\cref{sec:tasks}) followed by the details on experimental setups~(\cref{section:experimental_setup}), competing methods~(\cref{section:baselines}), benchmarks used for evaluation~(\cref{section:exp_benchmark}) and evaluation measures~(\cref{section:evaluation_measures}). Then we report the effectiveness of different methods in table union search task~(\cref{section:union_search_effectiveness}) followed by different ablation studies~(\cref{section:ablation_study}).
\subsection{Tasks}\label{sec:tasks}
% \roee{As we eventually use a two phase method for the "new" method, maybe the right way to phrase it here would be to introduce a two-phase table union search -- search (which is the "traditional" way) and classify (which is an additional for methods that cannot search). This is very common in an IR setup where a fairly quick method first gathers a set of documents for a query and then a more sophisticated method is applied over the retrieved set.} \roee{In what follows, we can start with the traditional problem def. (the Union Table Search paragraph) and then say we extend it. You can also use some of the text that is already used in "Starmie-GPT3"}
We analyze the efficacy of the benchmark for the following two tasks.
The first is the traditional search problem for which there are numerous solutions in the literature~\cite{nargesian2018table,DBLP:conf/icde/BogatuFP020,2023_khatiwada_santos,DBLP:journals/pvldb/FanWLZM23}.
\introparagraph{Table Union Search:}~\cite{nargesian2018table,2023_khatiwada_santos} Given a query table, $Q$ and a set of data lake tables, $\mathcal T = \{t_1, t_2, ...t_m\}$ where $|\mathcal T| = m$, find the top-$k$ data lake tables in $\mathcal T$ that are unionable with $Q$.
%rjm I agree with Roee here as we have no idea what a LLM thinks unionable means
%\aamod{We should define "unionable" too. Maybe: two columns are unionable if they belong to the same domain and two tables are unionable if they have a subset of columns that are unionable?} \roee{I would actually try to avoid that. I would either say we follow XXX unionabilty definition or just avoid a definition.}
\introparagraph{Union Table Classification}
Given a pair of tables $T_1$ and $T_2$, determine if they are unionable or not. The second problem is motivated by our use of LLMs to generate a unionability benchmark. Since we are claiming LLMs can produce unionable tables, can we also use an LLM to classify whether two tables we give to it are unionable or not. For example, we can use labeled pair of tables from our own \bname benchmark or hand-labeled pairs from any of the existing benchmarks. The latter can be viewed as a sanity check on whether the notion of unionability used by an LLM is a reasonable one conforming to definitions used in current research.
%Table Union Search methods look for top-$k$ relevant tables in terms of unionability, to classify each potential table in the top-$k$ as unionable.
% The second problem is motivated by our use of LLMs to generate a unionability benchmark. Since we are claiming LLMs can produce unionable tables, can we also use an LLM to classify whether two tables we give to it are unionable or not. For example, we can use labeled pair of tables from our own \bgen benchmark or hand-labeled pairs from any of the existing benchmarks. The latter can be viewed as a sanity check on whether the notion of unionability used by an LLM is a reasonable one conforming to definitions used in current research.
%we can utilize LLMs by prompting and asking if the two tables, the query table and the data lake table in question, is unionable or not. Due to the token size limit present in LLMs, we sample a row to represent each table in the prompts.
%ak reworded: instead of providing the entire set of tables in the prompts.
% \roee{TODO: add two phrase table union search as a third method that we build on top of with Starmie-LLM}
% \introparagraph{Two Phase Table Union Search}
% These two tasks can be further combined as a two phase table union search, namely search and classification. This is beneficial in cases where a method cannot be applied in large-scale. For example, in cases where using the method is expensive (either computationally or financially). This approach is very common for information retrieval applications~\cite{dehghani2017neural,shraga2020web} in which two models are applied consecutively.
% \roee{complete.}
% \subsection{Table Union Classification Method}
% While Table Union Search methods look for top-$k$ relevant tables in terms of unionability, to classify each potential table in the top-$k$ as unionable, we can utilize LLMs by prompting and asking if the two tables, the query table and the datalake table in question, is unionable or not. Due to token size limit present in LLMs, we sample one or few rows of each table instead of providing the entire set of tables in the prompts.
% \subsection{Experiments}
% \roee{Consider re-framing of the following sections. As the name of the section is "Analysis and Evaluation" I think we can skip the subsection "Experiments" and make all the following subsubsections subsections instead. This would also emphsize better the experimental nature of our paper. Another option is to have a section called "bla bla setup" and have the relevant subsections there (Tasks, Methods, Setup and Measures) and then the next section would include the experiments themselves (can still keep the name "Analysis and Evaluation")}
\subsection{Experimental Setup}
\label{section:experimental_setup}
We run experiments using Python 3.8 on a server with A100 80GB PCIe. We host the large language models locally in the aforementioned GPU. For the models unavailable to host locally, we use their open APIs. %\roee{I did not understand the final sentence}.
%\aamod{reworded to clarify. For example, as vicuna's model is available to download, we host vicuna on our own server for which we do not use API. For Chatgpt, since the model is not open source, we use its api.}
Our experiments aim to answer specific questions.
\begin{itemize}
\item How do the current union search methods perform on the new \bname benchmark created using the generative model?
\item Can generative models understand unionability?
\item How is the performance of union search techniques impacted by the sparsity (see ~\cref{sec:properties})?
\item How is the performance of union search techniques on specific topics?
\end{itemize}
% \roee{Do we want to add a question about topics?}
%aamod: added
\subsection{Union Search Methods}
%\roee{I would avoid calling it "Baselines" especially since out "new" method is not better... I would simply call it methods}
%aamod: agreed and resolved.
\label{section:baselines}
We evaluate the publicly available recent union search methods over the existing and new benchmarks.
\introparagraph{D$^3$L~\cite{DBLP:conf/icde/BogatuFP020}} Bogatu et al. extended TUS~\cite{nargesian2018table} to use not only word embeddings, knowledge graph mappings, or value overlap, but also
%rjm this is irrelevant right? we use it for union
%for the broader problem of finding related tables including joinable search and unionable search from the data lakes. They first assess the column similarity using five metrics based on column header similarity, value overlap,
column header similarity, distributions for numerical columns, and regular expressions.
%, and word embeddings of string columns. Then they aggregate the column similarity to search for the related tables.
We implement $D^3L$ using publicly available code.\footnote{\url{https://github.com/alex-bogatu/d3l}} For a fair comparison, we do not use the column header similarity metric since the existing benchmarks and other baselines are schema-agnostic.
\introparagraph{SANTOS~\cite{2023_khatiwada_santos}} SANTOS uses column semantics and the semantics of relationships between column pairs to search for the unionable tables. %Note that
SANTOS only uses column values and does not use metadata like column headers. To find column semantics and relationships semantics, SANTOS uses an external knowledge base and a synthesized knowledge base created using the data lake itself. To run SANTOS, we use the public code provided with the paper.\footnote{\url{https://github.com/northeastern-datalab/santos}}
\introparagraph{Starmie~\cite{DBLP:journals/pvldb/FanWLZM23}} Starmie is a recent state-of-the-art self-supervised table union search technique based on contrastive learning. Starmie captures the table contexts in the form of column embeddings and uses them to perform table union search. We reproduced Starmie following the instruction in its open implementation.\footnote{\url{https://github.com/megagonlabs/starmie}}
%We run pre-trained Starmie using its open implementation.\footnote{\url{https://github.com/megagonlabs/starmie}} %\roee{We use a ``pre-trained'' Starmie, right? I am mainly referring to the new benchmark, which was obviously not included in the original paper. Do we use the same model that was created in the original paper? For SANTOS it is obvious as we search an existing KB and have to create a synthesized one for coverage.}
% \aamod{@Roee, I am not sure either. @koyena is it pre-trained version?} \koyena{no, so i trained a model for every benchmark combination i tested it out with using the open implementation}
\introparagraph{Starmie-LLM}
%\roee{Modify based on the changes in Section~\ref{sec:tasks}:}
%aamod: done, please modify section 4.1 accordingly.
The LLMs cannot be used directly for the search task but can as mentioned be used for union classification. Therefore, to assess LLMs in the table union search task, we use an existing table union search method to search for a set of candidate unionable tables for each query table. Then we prompt LLMs to classify whether the query table with each candidate table are unionable or not. This two-phase approach is very common for information retrieval applications~\cite{dehghani2017neural,shraga2020web} in which two models are applied consecutively. In our experimental setup, we use
%state-of-the-art
Starmie to search for a reasonable number of candidate unionable tables. Then we prompt an LLM to classify each query-candidate table pairs are truly unionable by asking the following question:
% \begin{center}
\resultbox{
\texttt{Are the following tables unionable? Answer in the following format: \\
Unionable: \{yes/no\}}
}
% \end{center}
We use recent LLMs that have shown promising performance in other generative tasks.
Specifically, we use GPT2-xl\footnote{\url{https://huggingface.co/gpt2-xl}}, GPT3 (text-davinci-003 model with 175 billion parameters)\footnote{ \url{https://platform.openai.com/docs/models/gpt-3}}, Alpaca (7 Billion parameters)\footnote{\url{https://huggingface.co/circulus/alpaca-7b}}, and Vicuna (7 billion parameters)\footnote{\url{https://huggingface.co/lmsys/vicuna-7b-v1.3}} models in our experiments.
We denote respective LLM variations using Starmie-GPT2-XL, Starmie-GPT3, Starmie-Alpaca, and Starmie-Vicuna.
% Notice that the LLMs can be used as-is (aka zero-shot) or steered to specific tasks by providing a few examples (aka few-shot learning).
% So, to compare with other methods, we use Starmie-LLM's zero-shot version denoted by Starmie-LLM$_{\text{Zero}}$ and the few-shot version with an optimal number of examples (Starmie-LLM$_{\text{Optim}}$).
%Note that
LLMs can function either as they are (zero-shot) or can be directed towards specific tasks by providing a few examples (in-context learning). For comparison with other methods, we utilize Starmie-LLM's zero-shot versions denoted by Starmie-LLM$_{\text{Zero}}$ and the in-context versions with an optimal number of examples (Starmie-LLM$_{\text{Optim}}$), which gives the best performance against its other in-context versions.
%\aamod{please provide a footnote/citation to text-davinci-003.}
%aamod: added
% \introparagraph{Vicuna}:
\subsection{Benchmarks}
\label{section:exp_benchmark}
Along with the newly created \textbf{\bname} benchmark, we use the existing union search benchmarks for our experiments.
The details of each benchmark are given in~\cref{tab:union-overview}.
We use \textbf{SANTOS-Small} and \textbf{TUS-Small} benchmarks to measure the effectiveness of union search methods. Note that we do not use \textbf{SANTOS-Large} because its ground truth is not annotated~\cite{2023_khatiwada_santos}. %Furthermore, w
We also exclude the %larger TUS benchmark (
\textbf{TUS-Large} from the experiments as its tables do not have intent columns annotated, a requirement for the SANTOS method.
\subsection{Evaluation measures}
\label{section:evaluation_measures}
Following the previous table union search work~\cite{nargesian2018table, 2023_khatiwada_santos, 2023_hu_autotus}, we use Precision@k (P@k), Recall@k (R@k) and Mean Average Precision (MAP@k) to evaluate the effectiveness of table union search techniques. We use the binary label for unionability (unionable/non-unionable), and each benchmark's ground truth contains a set of unionable data lake tables for each query table.
Given a Query Table $Q$, let $\mathcal{T}_{Q}$ represent the ground truth set of its unionable tables and $\hat{\mathcal{T}}_{Q}$ represents the set of top-k unionable tables returned by some method. Then, $P@k$, $R@k$ and $MAP@k$ with respect to $Q$ are given by~\cite{2023_khatiwada_santos}:
\begin{equation}\label{eq:P_R}
P@k = \frac{\mathcal{T}_{Q}\cap\hat{\mathcal{T}}_{Q}}{\hat{\mathcal{T}}_{Q}},
R@k = \frac{\mathcal{T}_{Q}\cap\hat{\mathcal{T}}_{Q}}{\mathcal{T}_{Q}},
MAP@k = \frac{1}{|\hat{\mathcal{T}}_{Q}|}\sum_{k=1}^{|\hat{\mathcal{T}}_{Q}|} P@k
\end{equation}
Consistent with the prior work~\cite{2023_khatiwada_santos, DBLP:journals/pvldb/FanWLZM23}, we run experiments with k less than the ground truth size to ensure that there are enough true results available in the data lake when searching for top-k unionable tables per query table. In such cases, a Recall@k of 1 is not possible since all unionable tables cannot be returned.
%and we refer to the best possible Recall@k as the IDEAL.
Furthermore, unlike existing benchmarks, the new \bname Benchmark also contains the labeled non-unionable pairs. Therefore, we also measure Accuracy (ACC) and Corner Case Ratio (CCR) in this benchmark.
Let, TP, FP, TN, and FN represent the True Positives, False Positives, True Negatives, and False Negatives by a method over a benchmark. Then,
\begin{equation}\label{eq:accuracy}
ACC = \frac{TP + TN}{TP + TN + FP + FN},
CCR = 1 - ACC
\end{equation}
%Moreover, to analyze the nature of false positive predictions by the methods, we use False Positive Ratio (FPR) which is given by:
%specifically, whether such predictions are from the same topic or different topics. \rjm{Hmm the following equation doesn't mention topics. Why is FPR specific to a topic?}
%\aamod{In our ground truth, we explicitly label "non-unionable" tables only within the same topic. For example, we have 1050 tables which mean there could be 50 * 1000 possible labels (50 query and 1000 data lake tables). In this analysis, we consider }
% \begin{equation}\label{eq:accuracy}
% FPR = \frac{FP}{TN + FP}
% \end{equation}
%aamod: I removed FPR because this yields inconsistencies.
% \roee{Suggestion: Since one of the new features of the new benchmark is having labeled "non-unionable" tables, should we also include some sort of accuracy for that benchmark? We can claim here that we are able to evaluate the ability of methods to also identify non-unionable tables as non-unionable.}
% \roee{TODO: Add another analysis to the ablation study in which we will explore table union classification and potentially present a confusion matrix for the methods emphasizing the ability to have a full analysis of the results (including saying no/yes when the real answer is no) with the new benchmark.}
% \aamod{@koyena @Roee, I am not sure if I understood this metric correctly. Could you please take the lead and add the metric here?}
% \aamod{@koyena @Roee , please verify the equations and see if I resolved your comments (commented out just before this comment)}
\noindent We further use a confusion matrix to illustrate the specific TP, FP, TN, and FN values.
\subsection{Union Search Effectiveness}
\label{section:union_search_effectiveness}
\begin{table}
% \centering
%\ra{1.2}
\caption{$\text{P@k}$, $\text{MAP@k}$ and $\text{R@k}$ of table union search methods over different benchmarks.}
\begin{tabular}{@{}lllll@{}}
\toprule
\multicolumn{1}{@{}l@{}}{\textbf{Benchmark}} &
\textbf{Method} &
\textbf{$MAP@k$} &
\textbf{$P@k$} &
\textbf{$R@k$} \\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{TUS-Small}\\ \texttt{k=60}}}} &
D$^3$L &
0.79 &
0.77 &
0.21 \\
\multicolumn{1}{@{}l@{}}{} &
SANTOS &
0.88 &
0.81 &
0.23 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie &
\textbf{0.94} &
\textbf{0.82} &
\textbf{0.27} \\
\multicolumn{1}{@{}l@{}}{} &
Starmie-Vicuna$_{\text{Zero}}$ &
0.89 &
0.70 &
0.24 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie-Vicuna$_{\text{Optim}}$ &
\textbf{0.94} &
0.80 &
\textbf{0.27} \\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{SANTOS-Small}\\ \texttt{k=10}}}} &
D$^3$L &
0.52 &
0.58 &
0.42 \\
\multicolumn{1}{@{}l@{}}{} &
SANTOS &
0.94 &
\textbf{0.91} &
\textbf{0.69} \\
\multicolumn{1}{@{}l@{}}{} &
Starmie &
\textbf{0.95} &
\textbf{0.91} &
0.68 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie-Vicuna$_{\text{Zero}}$ &
0.81 &
0.68 &
0.50 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie-Vicuna$_{\text{Optim}}$ &
\textbf{0.95} &
0.90 &
0.67 \\
\hline
\multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{UGEN-V1}\\ \texttt{k=10}}}} &
D$^3$L &
0.26 &
0.19 &
0.19 \\
\multicolumn{1}{@{}l@{}}{} &
SANTOS &
0.56 &
0.46 &
0.46 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie &
\textbf{0.61} &
\textbf{0.51} &
\textbf{0.51} \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-GPT3 &
% 0.56 &
% 0.39 &
% 0.39 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie-Vicuna$_{\text{Zero}}$ &
0.44 &
0.32 &
0.32 \\
\multicolumn{1}{@{}l@{}}{} &
Starmie-Vicuna$_{\text{Optim}}$ &
0.57 &
0.48 &
0.48 \\
\bottomrule
\end{tabular}%
% }
\label{tab:result-overview}
\end{table}
\begin{figure}[h]
{
\centering
\begin{minipage}[t]{\textwidth}
\includegraphics[width=0.48\linewidth]{figures/legend_ugen-v1.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $P@k$ on TUS-Small]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/precision_in_tus.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $R@k$ on TUS-Small]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/recall_in_tus.pdf}
\end{minipage}%
}
\hfill
% \centering
\subfloat[Average $P@k$ on Santos-Small]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/precision_in_santos.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $R@k$ on Santos-Small]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/recall_in_santos.pdf}
\end{minipage}%
}
\hfill
% \centering
\subfloat[Average $P@k$ on \bname]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/precision_in_ugen-v1.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $R@k$ on \bname]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/recall_in_ugen-v1.pdf}
\end{minipage}%
}
\caption{Effectiveness of baselines in different benchmarks}
\label{fig:pr_rc_all}
\end{figure}
Now we evaluate the performance of various table union search methods (described in \cref{section:baselines}) using both existing and new benchmarks~(detailed in \cref{section:exp_benchmark}).
Specifically, we %will
compare D$^3$L, SANTOS, Starmie, and Starmie-LLMs.
Among Starmie-LLMs, we select the best-performing Starmie-Vicuna's zero-shot version (Starmie-Vicuna$_{\text{Zero}}$) and
%\rjm{This is the first use of the word optimal in the paper. We need to define optimal-shot before this.}
%\aamod{resolved by adding optimal shot baseline in secion 4.3}
optimal-shot version (Starmie-Vicuna$_{\text{Optim}}$) as it shows a balanced performance in both versions. %Later i
In the ablation study (\cref{section:ablation_study}), we %will
discuss the results of other Starmie-LLM variations and their different versions.
The evaluation metrics used are MAP@k, P@k, and R@k.
Following previous work~\cite{2023_khatiwada_santos, nargesian2018table}, the maximum value of k is chosen for each benchmark, based on the number of unionable tables available in the data lake for query tables. For TUS-small, we select k up to 60. For SANTOS-small, we go up to k = 10.
The effectiveness result for the maximum value of k on each benchmark is presented in \cref{tab:result-overview}. We \textbf{bold} the score of the best-performing method on each measure and benchmark.
Further results for other smaller values of k are plotted in \cref{fig:pr_rc_all} with different values of k in horizontal axes, the evaluation metrics in vertical axes, and the methods encoded using different colors and line styles. As noted, the recall cannot be perfect if k is smaller than the ground truth size~\cite{2023_khatiwada_santos}. So, we show the IDEAL-RECALL line that indicates the maximum possible recall for each value of k.
% \rjm{In graph can we call this IDEAL-Recall so people don't think there is an IDEAL precision?} \koyena{done!}
%\aamod{@Koyena, please add precision and recall graph for P@k and R@k for santos-small and TUS-small too. I have added a template for that, please see the comments in the figure.} \koyena{added figures in the figure folder}
%It is seen that t
\introparagraph{Existing Benchmarks} The performances of all methods follow a similar trend in both TUS-Small and SANTOS-small benchmarks. Specifically, Starmie mostly stands out in terms of MAP@k, P@k, and R@k followed by
Starmie-Vicuna$_{\text{Optim}}$, Starmie-Vicuna$_{\text{Zero}}$,
%$Starmie-Vicuna_{Optim}$, $Starmie-Vicuna_{Zero}$,
SANTOS and D$^3$L.
As Starmie captures the entire table context using contrastive learning, it seems to understand the table semantics better in both benchmarks.
Furthermore, when we steer Starmie-Vicuna to understand unionability by providing an optimal number of in-context examples, its MAP increases by 5 \% over the zero-shot version in the TUS-small benchmark, and by 14 \% in the SANTOS-small benchmark matching the performance of the best-performing Starmie. This indicates that the out-of-the-box LLMs are not good enough to understand unionability, but they can be taught to do so by using in-context learning.
Moreover, SANTOS also shows a comparable performance against Starmie and Starmie-Vicuna$_{\text{Optim}}$ in both benchmarks and is the best method in SANTOS-small in terms of P@10 and R@10. Mainly, SANTOS seems to be benefited from the use of the binary relationships between the columns which helps it to better understand the table context.
\introparagraph{\bname} In the new \bname benchmark, Starmie continues to outperform all other methods across all three evaluation metrics, followed by Starmie-Vicuna$_{\text{Optim}}$, SANTOS, Starmie-Vicuna$_{\text{Zero}}$, and D$^3$L. However, it is important to note that the performance of all methods shows a significant drop when compared to their performance on the SANTOS-Small and TUS-Small benchmarks.
For instance, Starmie's MAP@k on
%UGEN-V1
\bname for all values of k drops by over 33 \% compared to its MAP@k on both the SANTOS-Small and TUS-Small benchmarks. This drop in performance can be attributed to the difficulty the methods face in distinguishing non-unionable tables from tables within the same topic. Note the other benchmarks %~\cite{2023_khatiwada_santos, nargesian2018table}
do not include this case due to the way they slice and dice tables each of which is from a different topic (see \cref{fig:generation-example}).
In contrast, \bname contains both unionable and non-unionable table pairs for each query table.
This scenario is quite common in data lakes, but it is challenging to include in hand labeled benchmarks~\cite{2023_khatiwada_santos, nargesian2018table}.
Hence, the systematic benchmark creation using a generative model enables more control over the diversity of table topics and facilitates the construction of a challenging yet realistic benchmark.
In \cref{section:ablation_study}, we conduct an error analysis and delve into the performance of different techniques in each domain.
Finally, the lower performance achieved by Starmie-Vicuna, the best among the Starmie-LLM variations, in comparison to the classical table union search methods, indicates that even advanced %pre-trained language models (
LLMs
%)
like Vicuna %, Alpaca, and GPT3
face challenges in the new %union
benchmark. This shows ample opportunity for
%enhancing the LLMs' performance on the union search task.
creating better table union search methods.
%\aamod{bookmark; start from here}
% \begin{figure*}[ht]
% \centering
% \includegraphics[width=0.7\textwidth]{figures/ugenv1_effectiveness.pdf}
% \caption{Effectiveness of UGEN\_V1 benchmark against different baselines. The left figure illustrates the average $P@k$ on UGEN\_V1 while the right figure showcases $R@k$ for the same. \aamod{@Koyena, for completeness, I would prefer to add precision and recall graph for P@k and R@k for santos-small and TUS-small too. I added a template just below this block of figure that needs pdfs for the following 6 figures in figures/ folder: tus benchmark precision, tus benchmark recall, santos benchmark precision, santos benchmark recall, ugen-v1 precision, ugen-v1 recall. Could you please upload them?}}
% \label{fig:prec_at_k}
% \end{figure*}
\subsection{Ablation Study}
\label{section:ablation_study}
% \begin{table}
% % \centering
% %\ra{1.2}
% \caption{Comparison of $P@k$, $MAP@k$ and $R@k$ against different number of in-context learning examples provided during Starmie-LLM prompting for identifying unionability in UGEN-V1 Benchmark.}
% \begin{tabular}{@{}lllll@{}}
% \toprule
% \multicolumn{1}{@{}l@{}}{\makecell[l]{\textbf{Number of}\\ \textbf{ICL Examples}}} &
% \textbf{{\makecell[l]{\textbf{Starmie-LLM}\\ \textbf{Variation}}}} &
% \textbf{$MAP@k$} &
% \textbf{$P@k$} &
% \textbf{$R@k$} \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{4}{*}{\makecell[l]{\textbf{0 (Zero-Shot)}\\ \texttt{k=10}}}} &
% Starmie-GPT2-XL &
% 0.01 &
% 0.004 &
% 0.004 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-GPT3 &
% 0.43 &
% 0.27 &
% 0.27 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Alpaca &
% \textbf{0.46} &
% \textbf{0.37} &
% \textbf{0.37} \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna &
% 0.44 &
% 0.32 &
% 0.32 \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{4}{*}{\makecell[l]{\textbf{1}\\ \texttt{k=10}}}} &
% Starmie-GPT2-XL &
% \textbf{0.60} &
% \textbf{0.49} &
% \textbf{0.49} \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-GPT3 &
% 0.50 &
% 0.34 &
% 0.34 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Alpaca &
% 0.42 &
% 0.33 &
% 0.33 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna &
% 0.57 &
% 0.48 &
% 0.48 \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{4}{*}{\makecell[l]{\textbf{2}\\ \texttt{k=10}}}} &
% Starmie-GPT2-XL &
% 0.45 &
% 0.36 &
% 0.36 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-GPT3 &
% \textbf{0.53} &
% 0.35 &
% 0.35 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Alpaca &
% 0.43 &
% 0.35 &
% 0.35 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna &
% 0.49 &
% \textbf{0.40} &
% \textbf{0.40} \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{4}{*}{\makecell[l]{\textbf{3}\\ \texttt{k=10}}}} &
% Starmie-GPT2-XL &
% 0.26 &
% 0.20 &
% 0.20 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-GPT3 &
% \textbf{0.56} &
% 0.39 &
% 0.39 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Alpaca &
% 0.45 &
% 0.35 &
% 0.35 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna &
% 0.54 &
% \textbf{0.44} &
% \textbf{0.44} \\
% \bottomrule
% \end{tabular}%
% \label{tab:icl-overview}
% \end{table}
Now we perform fine-grained analyses over the new benchmark to get better insights into the impact of different factors in the table unionability search.
\subsubsection{In-Context learning}
% \begin{figure}
% \centering
% \includegraphics[width=\linewidth]{figures/icl_all.pdf}
% \caption{Comparison of $MAP@10$, $P@10$ and $R@10$ against different numbers of in-context learning examples provided during Starmie-LLM prompting for identifying unionability in \bname Benchmark.}
% \label{fig:icl-overview}
% \end{figure}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/sparsity_all.pdf}
% \caption{Comparison of $MAP@10$, $P@10$ and $R@10$ of different sparsity variations of UGEN-V1 Benchmarks \roee{Can you flip it to be vertical as well? To be consistent with Figure~\ref{fig:icl-overview}}}
% \label{fig:sparsity-overview}
% \end{figure*}
\begin{figure}[t]
{
\centering
\begin{minipage}[t]{\textwidth}
\hspace{1.5cm}
\includegraphics[width=0.3\linewidth]{figures/legend_icl_in_ugen_v1.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $MAP@k$ on \bname]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/MAP_icl_in_ugen_v1.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $P@k$ on \bname]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/precision_icl_in_ugen_v1.pdf}
\end{minipage}%
}
\caption{$\text{MAP@10}$ and $\text{P@10}$ for different numbers of in-context learning examples provided during Starmie-LLM prompting for identifying unionability in \bname.}
\label{fig:icl-overview}
\end{figure}
In-Context learning is
%a phenomenon that emergent
used in LLMs where a model is able to learn a task based on a few demonstrations provided in the prompt~\cite{brown2020language}. This method does not require any model parameter training or updates, which makes it a quick adoption strategy to improve a model's performance. However, this method is sensitive to the number, order, and content of the examples~\cite{zhao2021calibrate}. Hence, we follow a variation of the setup described by~\citet{liu2021makes} to create good in-context learning examples. This variation uses the Roberta-Large model~\cite{liu2019roberta}\footnote{\url{https://huggingface.co/roberta-large}}
to encode the potential in-context examples, and employs average Euclidean distance for each example with respect to all query examples of the \bname benchmark. We sort these distance values and get $n$ closest distance values to create $n$ examples for in-context learning.
To prevent data leakage, we conduct evaluations on both the \bname and TUS-Small benchmarks, employing in-context examples exclusively from the SANTOS-Small benchmark. Likewise, when testing the methods on the SANTOS-Small benchmark, we exclusively utilize examples from the TUS-Small benchmark. Each example in this evaluation consists of a pair of either unionable or non-unionable tables.
In \Cref{fig:icl-overview}, we illustrate how in-context learning affects the effectiveness of the Starmie-LLM method on the \bname Benchmark. On the horizontal axis, we vary the number of examples given to the method and in the vertical axis, we show the performance metrics MAP@10 and P@10.
%, and R@10.
Note that P@10 and R@10 are the same because we report performance at k = 10, which is the number of unionable tables in ground truth for each query table, hence we only show P@10.
%\rjm{I suggest removing the recall graph if it is identical to precision!} \koyena{done!}
We encode each method using different colors and line styles.
As the performance trend of each method is almost the same in all the metrics, we focus our discussion on MAP@10 (top-most graph).
%Note that the graphs for P@10 and R@10 are the same because we report performance at k = 10, which is the number of unionable tables in ground truth for each query table.
%It is seen that
All but Starmie-Alpaca benefit significantly when they are provided with an in-context example to understand unionability. For instance, Starmie-GPT2-XL's MAP@10 increased by 59 \% when provided with an in-context example (one-shot learning). This implies that GPT2-XL possibly does not understand table unionability, but learns very easily with examples. Similarly, the one-shot version of Starmie-GPT3 and Starmie-Vicuna also enjoy an increase of up to 13 \% in MAP@10 with one-shot learning. Notice however, the performance of all the models saturates or even goes down if we provide more examples. For example, Starmie-GPT2-XL's MAP@10 drops from 60 \% on one-shot learning to 26 \% on three-shot learning.
This indicates that the LLM models can learn unionability with examples but may start to underfit the provided examples if we increase their numbers.
\subsubsection{Sparsity}
\label{section:sparsity}
% \begin{table}
% % \centering
% %\ra{1.2}
% \caption{Comparison of $P@k$, $MAP@k$ and $R@k$ of different sparsity variations of UGEN-V1 Benchmarks}
% \begin{tabular}{@{}lllll@{}}
% \toprule
% \multicolumn{1}{@{}l@{}}{\textbf{Sparsity}} &
% \textbf{Method} &
% \textbf{$MAP@k$} &
% \textbf{$P@k$} &
% \textbf{$R@k$} \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{0\%}\\ \texttt{k=10}}}} &
% D$^3$L &
% 0.26 &
% 0.19 &
% 0.19 \\
% \multicolumn{1}{@{}l@{}}{} &
% SANTOS &
% 0.56 &
% 0.46 &
% 0.46 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie &
% 0.61 &
% 0.51 &
% 0.51 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Zero}$ &
% 0.44 &
% 0.32 &
% 0.32 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Optim}$ &
% 0.57 &
% 0.48 &
% 0.48 \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{5\%}\\ \texttt{k=10}}}} &
% D$^3$L &
% 0.14 &
% 0.10 &
% 0.10 \\
% \multicolumn{1}{@{}l@{}}{} &
% SANTOS &
% 0.55 &
% 0.46 &
% 0.46 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie &
% 0.63 &
% 0.53 &
% 0.53 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Zero}$ &
% 0.44 &
% 0.32 &
% 0.32 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Optim}$ &
% 0.57 &
% 0.48 &
% 0.48 \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{10\%}\\ \texttt{k=10}}}} &
% $D^3L$ &
% 0.13 &
% 0.09 &
% 0.09 \\
% \multicolumn{1}{@{}l@{}}{} &
% SANTOS &
% 0.55 &
% 0.46 &
% 0.46 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie &
% 0.59 &
% 0.52 &
% 0.52 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Zero}$ &
% 0.40 &
% 0.33 &
% 0.33 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Optim}$ &
% 0.57 &
% 0.46 &
% 0.46 \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{15\%}\\ \texttt{k=10}}}} &
% $D^3L$ &
% 0.12 &
% 0.10 &
% 0.10 \\
% \multicolumn{1}{@{}l@{}}{} &
% SANTOS &
% 0.54 &
% 0.45 &
% 0.45 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie &
% 0.59 &
% 0.52 &
% 0.52 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Zero}$ &
% 0.43 &
% 0.32 &
% 0.32 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Optim}$ &
% 0.58 &
% 0.49 &
% 0.49 \\
% \hline
% \multicolumn{1}{@{}l@{}}{\multirow{5}{*}{\makecell[l]{\textbf{20\%}\\ \texttt{k=10}}}} &
% $D^3L$ &
% 0.11 &
% 0.10 &
% 0.10 \\
% \multicolumn{1}{@{}l@{}}{} &
% SANTOS &
% 0.55 &
% 0.45 &
% 0.45 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie &
% 0.61 &
% 0.53 &
% 0.53 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Zero}$ &
% 0.41 &
% 0.31 &
% 0.31 \\
% \multicolumn{1}{@{}l@{}}{} &
% Starmie-Vicuna$_{Optim}$ &
% 0.55 &
% 0.45 &
% 0.45 \\
% \bottomrule
% \end{tabular}%
% \label{tab:sparsity-overview}
% \end{table}
One of the benchmark properties that we care about is sparsity, which relates to the amount of null or missing attribute values %present
in the tables. Within the \tname framework, this property is controlled by a script that randomly removes values in a table until the desired sparsity is reached. %In ~
\cref{fig:sparsity-overview} illustrates how different methods can handle benchmarks with different rates of sparsity. We vary sparsity from 0\% to 20\% in X-axes and report MAP@10 and P@10 in y-axes.
%It is seen that
SANTOS, Starmie, and Starmie-Vicuna are not impacted by sparsity very much. However, the performance of the column-based approach (D$^3$L) goes down significantly when we increase sparsity. For example, its MAP@10 reduces by 15 \% when we increase sparsity to 20 \%. This gives us an interesting insight that the methods which make unionability decisions by capturing the overall table context rather than considering individual columns independently are more tolerant towards sparsity.
\begin{figure}[t]
{
\centering
\begin{minipage}[t]{\textwidth}
\includegraphics[width=0.48\linewidth]{figures/legend_sparse_in_ugen_v1.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $MAP@k$ on \bname]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/MAP_sparse_in_ugen_v1.pdf}
\end{minipage}%
}
% \centering
\subfloat[Average $P@k$ on \bname]{
\begin{minipage}[t]{0.49\linewidth}
\includegraphics[width=\linewidth]{figures/precision_sparse_in_ugen_v1.pdf}
\end{minipage}%
}
\caption{$\text{MAP@10}$ and $\text{P@10}$ in different sparsity variations of UGEN-V1 Benchmark}
\label{fig:sparsity-overview}
\end{figure}
\subsubsection{Topic-Based Analysis}
% \begin{figure*}[hbt!]
% \centering
% \includegraphics[width=.495\textwidth]{figures/topic_analysis.pdf}
% \caption{Topic-based MAP@k (k=10) by (a) D$^3$L (b) SANTOS (c) Starmie (d) Starmie-Vicuna$_{Zero}$ (e) Starmie-Vicuna$_{Optim}$ methods in \bname benchmark \roee{Consider making it double column if we have space.}}
% \label{fig:topic_specific_all}
% \end{figure*}
\begin{table*}
% \centering
%\ra{1.2}
\caption{Top-5 Topics where each method performs the best.}
\begin{tabular}{@{}llllll@{}}
\toprule
\textbf{Method} &
\textbf{Top 1} &
\textbf{Top 2} &
\textbf{Top 3} &
\textbf{Top 4} &
\textbf{Top 5} \\
\hline
\textbf{D$^3$L} &
Ecology &
Gardening &
Engineering &
Astronomy &
Computer Science \\
\textbf{SANTOS} &
Culture &
Law &
Gardening &
Photography &
Veterinary Medicine \\
\textbf{Starmie} &
Biology &
Climatology &
Economics &
Veterinary Medicine &
Photography \\
\textbf{Starmie-Vicuna} &
Biology &
Climatology &
Economics &
Photography &
Computer Science \\
\bottomrule
\end{tabular}
\label{tab:topic-top5-overview}
\end{table*}
\begin{table*}
% \centering
%\ra{1.2}
\caption{Bottom-5 Topics where each method performs the least.}
\begin{tabular}{@{}llllll@{}}
\toprule
\textbf{Method} &
\textbf{Bottom 1} &
\textbf{Bottom 2} &
\textbf{Bottom 3} &
\textbf{Bottom 4} &
\textbf{Bottom 5} \\
\hline
\textbf{D$^3$L} &
Psychology &
Sociology &
Sports &
Veterinary Medicine &
World Geography \\
\textbf{SANTOS} &
Politics &
Engineering &
Fashion &
Geology &
World History \\
\textbf{Starmie} &
Architecture &
Horticulture &
Environment &
Science &
Religion \\
\textbf{Starmie-Vicuna} &
Astronomy &
Geophysics &
Religion &
Business &
Genealogy \\
\bottomrule
\end{tabular}
\label{tab:topic-bottom5-overview}
\end{table*}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/legend_d3l_+query_specific.pdf}
% \caption{Topic-based MAP@k using $D^3L$ method on UGEN-V1 benchmark}
% \label{fig:d3l_topic_specific}
% \end{figure*}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/legend_santos_+query_specific.pdf}
% \caption{Topic-based MAP@k using SANTOS method on UGEN-V1 benchmark}
% \label{fig:santos_topic_specific}
% \end{figure*}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/legend_starmie_+query_specific.pdf}
% \caption{Topic-based MAP@k using Starmie method on UGEN-V1 benchmark}
% \label{fig:starmie_topic_specific}
% \end{figure*}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/legend_starmie_gpt3_+query_specific.pdf}
% \caption{Topic-based MAP@k using Starmie-GPT3 method on UGEN-V1 benchmark}
% \label{fig:starmie_gpt3_topic_specific}
% \end{figure*}
% \begin{figure*}
% \centering
% \includegraphics[width=\textwidth]{figures/legend_starmie_vicuna_+query_specific.pdf}
% \caption{Topic-based MAP@k using Starmie-Vicuna method on UGEN-V1 benchmark}
% \label{fig:starmie_vicuna_topic_specific}
% \end{figure*}
%\aamod{let's choose topic between topic and domain and use the same everywhere.}
One of the benefits of \tname framework is that we can generate unionable and non-unionable table pairs based on topics. This enables us to perform a topic-based analysis and understand how each baseline performs on various topics.
In ~\cref{tab:topic-top5-overview} and ~\cref{tab:topic-bottom5-overview}, we show the top-5 and bottom-5 topics out of 50 topics based on MAP@10 for each method.
For a comprehensive analysis, we also report MAP@10 by each union search method on all 50 topics in ~\cref{fig:topic_specific_all}.
Except for Starmie and its LLM variation (Starmie-Vicuna), it is interesting to see that there is not much overlap in the top-5 best topics of each method. Even more interesting, there is an overlap between the top-5 topics of one method and the bottom-5 topics of another method. For example, on one hand, \texttt{Engineering} is the third best topic for D$^3$L but the second worst topic for \texttt{SANTOS}. On the other hand, D$^3$L has \texttt{Veterinary Medicine} in its bottom-5, while the same topic is among SANTOS' top-5 best topics.
This signifies that each method captures different properties in the table to infer table unionability and an easy topic for one method could be difficult for another method. This result suggests that future work on building domain-specific search methods could be useful along with combining the strength of multiple methods to build a single highly-effective table union search method.
Additional interesting insights are visible in \cref{fig:topic_specific_all} which allows a comparison among methods for a given topic. For example, \texttt{World History} achieves a fair MAP@10 score among all methods except SANTOS which achieves a score of 0. This can be attributed to poor coverage of their knowledge base in this topic, which is key for a successful utilization of SANTOS~\cite{2023_khatiwada_santos}. Another apparent example is the topic of \texttt{Climatology} for which D$^3$L fails to find a single unionable table (MAP@10=0) while others were very successful with high MAP@10 scores. Finally, another interesting insight is the topic-specific benefit of in-context learning, comparing Starmie-Vicuna$ _{Zero}$ (\cref{fig:topic_specific_all} (d)) Starmie-Vicuna$ _{Optim}$ (\cref{fig:topic_specific_all} (e)). For example, along side topics such as \texttt{Bioinformatics} and \texttt{Language} which are unaffected, topics such as \texttt{Gardening} and \texttt{Geology} are significantly improved with ICL and, interestingly, the performance over topics such as \texttt{Cooking} and \texttt{Geophysics} actually decline.
\begin{figure}[t]
\centering
\includegraphics[width=.495\textwidth]{figures/topic_analysis.pdf}
\caption{Topic-based MAP@k (k=10) by (a) D$^3$L (b) SANTOS (c) Starmie (d) Starmie-Vicuna$_{Zero}$ (e) Starmie-Vicuna$_{Optim}$ methods in \bname benchmark} %\roee{Consider making it double column if we have space.}}
\label{fig:topic_specific_all}
\end{figure}
%\roee{If we will go ahead with the full bar plot for the topic analysis I would align the plots on top of each other (single X-axis, multiple Y-axes) to allow the reader the comparison. I would still go with (or add) a straightforward table showing top and bottom 5 topics for each method.}
%aamod: resolved
% \koyena{Added the straightforward table}
% \aamod{@koyena, if you merge the topic figures, please update the figure Crefs in the text accordingly.}
\subsubsection{Non-Unionable Pair Analysis}\label{sec:nonunionability-analysis}
%\aamod{yet to be polished, only the first draft.}
\begin{figure*}[hbt!]
% \centering
\includegraphics[width=\textwidth]{figures/confusion_matrix.pdf}
\caption{Confusion matrices for (a) D$^3$L (b) SANTOS (c) Starmie (d) Starmie-Vicuna$_{Zero}$ (e) Starmie-Vicuna$_{Optim}$ methods on distinguishing unionable and non-unionable pairs in \bname Benchmark. A legend in the top-right corner is for all the confusion matrices. (f) Accuracy and Corner Case Ratio of different methods in \bname Benchmark}
\label{fig:confusion_matrix}
\end{figure*}
\tname has the ability to generate labeled non-unionable table pairs from the same topic,
something that was previously overlooked in unionability benchmarks~\cite{nargesian2018table, 2023_khatiwada_santos}.
As reported in~\cref{tab:union-overview}, we have 50 query tables and each query has 10 unionable data lake tables and 10 non-unionable data lake tables, all of which are on the same topic and labeled in the ground truth.
The other tables that are unlabeled with respect to the query are non-unionable as they are semantically different, not even from the same topic. In this section, we analyze the potential impact of non-unionable table pairs from the same topic on the performance of the union search methods by creating confusion matrices using the 1000 labeled unionable and non-unionable pairs and reporting accuracy over them vs overall non-unionable tables.
To create a confusion matrix, we need true labels, false labels, and predicted labels. True labels and false labels are "unionable" and "non-unionable" table pairs from the ground truth respectively. Note that we use search methods that returns top-k results for each query. As the total number of unionable table pairs for each query is 10 in the ground truth, we search for the top-10 unionable tables for each query. With 50 query tables, we get 500 predicted true labels. The table pairs that are in the ground truth, but do not make it to the top-10 are false (non-unionable) predicted labels. We compare predicted labels with the ground truth to construct confusion matrices.
% Since the methods we consider are for table search and the total number of unionable table pairs for each query is 10, we run the search methods to find the top-10 unionable tables for each query table which are true (unionable) predicted labels. The tables that do not make it to the top-10 are false (non-unionable) predicted labels. So for 50 queries, we find at most 500 predicted labels.\footnote{The predicted labels could be less than 500 if a method returns less than 10 tables for a query.}
We report confusion matrix and accuracy in ~\cref{fig:confusion_matrix} for each method in \bname.
If we see correct non-unionable predictions (bottom-right of confusion matrices), D$^3L$ seems to be the best at detecting non-unionable tables (predicts 437 out of 500 non-unionable pairs accurately).
Notice however, if we look at true positives (top-left of confusion matrix in~\cref{fig:confusion_matrix}(a)), D$^3$L has the least number, around two times
%less
fewer
than the second-least performing Starmie-Vicuna$_{Zero}$~(\cref{fig:confusion_matrix} (d)).
This means that the false positives in D$^3$L's results are other unlabeled non-unionable tables from random topics rather than the labeled "non-unionable" tables from the same topic.
But if we look at SANTOS, Starmie, and Starmie's LLM variations that capture the table context to make unionability decisions, they have high true positives and relatively lower true negatives. This means that their false positives are mostly non-unionable tables from the same topic, as we discussed in~\cref{ex:running_example}.
This is also reflected in the accuracy numbers~\cref{fig:confusion_matrix} (f). For instance, SANTOS seems to be the most balanced method in differentiating unionable and non-unionable among the same topic, achieving the highest accuracy. This is because SANTOS captures fine-grained table context by looking at the binary relationship between the column pairs and getting less confused on the tables from the same topic. On the other hand, Starmie and its LLM variations capture the overall table context in the form of column embeddings without necessarily looking at the binary relationships between the columns explicitly. This may have deprived them of separating the non-unionable from unionable tables within the same topic.
Hence, we get a very interesting insight that table unionability is not just about finding the tables from the same topics. Instead, the search methods should also separate non-unionability among the same topic tables, an observation which has been overlooked in the current benchmarking literature.
%\aamod{todo: bookmark, continue from here.}
% In the past table-union search benchmarks, there are no table pairs that were specifically labeled to be non-unionability. Hence, there aren't any ...
% \begin{table}
% % \centering
% %\ra{1.2}
% \caption{Accuracy and Corner Case Ratio on \bname benchmark}
% \begin{tabular}{@{}lll@{}}
% \toprule
% \textbf{Method} &
% \textbf{Accuracy} &
% \textbf{Corner Case Ratio} \\
% \hline
% \textbf{D$^3$L} &
% 53.2\% &
% 0.47 \\
% \textbf{SANTOS} &
% 58.3\% &
% 0.42 \\
% \textbf{Starmie} &
% 56.3\% &
% 0.44 \\
% \textbf{Starmie-Vicuna$_{Zero}$} &
% 53.1\% &
% 0.47 \\
% \textbf{Starmie-Vicuna$_{Optim}$} &
% 54.7\% &
% 0.45 \\
% \bottomrule
% \end{tabular}
% \label{tab:acc_corner-overview}
% \end{table}
% \begin{figure}[h]
% \centering
% \includegraphics[width=\linewidth]{figures/cm_d3l.pdf}
% \caption{Confusion Matrix using $D^3L$ on \bname}
% \label{fig:cm_d3l}
% \end{figure}
% \begin{figure}[h]
% \centering
% \includegraphics[width=\linewidth]{figures/cm_santos.pdf}
% \caption{Confusion Matrix using SANTOS on \bname}
% \label{fig:cm_santos}
% \end{figure}
% \begin{figure}[h]
% \centering
% \includegraphics[width=\linewidth]{figures/cm_starmie.pdf}
% \caption{Confusion Matrix using Starmie on \bname}
% \label{fig:cm_starmie}
% \end{figure}
% \begin{figure}[h]
% \centering
% \includegraphics[width=\linewidth]{figures/cm_starmie_vicuna_zero.pdf}
% \caption{Confusion Matrix using Starmie-Vicuna$_{Zero}$ on \bname}
% \label{fig:cm_starmie_vic_zero}
% \end{figure}
% \begin{figure}[h]
% \centering
% \includegraphics[width=\linewidth]{figures/cm_starmie_vicuna_opt.pdf}
% \caption{Confusion Matrix using Starmie-Vicuna$_{Optim}$ on \bname}
% \label{fig:cm_starmie_vic_opt}
% \end{figure}
\subsection{Discussion and Future Work}
In \tname, we use the massive knowledge present in LLMs, namely GPT3, to generate table pairs that are either unionable or non-unionable for specified topics. However, there are certain limitations posed by this framework. Due to the dependence on LLMs to generate the data, it is not guaranteed that the size and type of table pairs would be exactly as
%asked for
requested in a prompt. For instance, it is possible
%to generate
that an LLM generates
a table with a different number of rows than requested in the prompt. Furthermore, due to the token limit size, in the case of the Starmie-LLM variations, we rely on a single table row each to identify unionability.
In~\Cref{sec:new_unionability_benchmark}, we reported that the generation of \bname took about 10 hours and \$18 USD.
%To infer this benchmark for unionability, $D^3L$, SANTOS, and Starmie
%performed as expected and reported in their respective works.
%\koyena{do we cite these methods here too?}
%rjm removing accuracy and only taking about runtime
%The accuracy of the existing table union search methods was lower on \bname than other existing benchmarks (\cref{section:union_search_effectiveness}), but their relative accuracy and runtime performance was that same, e.g., Starmie has better performance than SANTOS.
%resolved \rjm{koyenna is this what you meant -- take a look at the sentence above.} \koyena{yes!}
The total runtimes of existing methods on \bname for all 50 queries were about 7 minutes for D$^3$L, 3 minutes for SANTOS and 2 minutes for Starmie.
The Starmie-LLM variation methods had varying query-time based on their model sizes, length of the prompt, and platform for running them. In zero-shot case, Starmie-GPT-2XL took about 0.20 seconds per query (one of 50 queries), Starmie-Vicuna took about 0.53 seconds per query, and Starmie-Alpaca took around 0.54 seconds per query. GPT2-XL is a 1.5 Billion parameter model while Vicuna and Alpaca models are around 7 Billion parameter models. Hence, GPT2-XL had a relatively faster inference time than Vicuna and Alpaca. Furthermore, since Vicuna and Alpaca were similarly sized, they have similar query-times. These query times increase non-linearly.
%growth pattern.
Between zero-shot and one-shot, there is a 56\% increase in average query time. Between two-shot and one-shot, there is about 39\% increase in average query time. Lastly, between three-shot and two-shot, there is about 34\% increase in query time. This is consistent for both Vicuna and Alpaca. For GPT2-XL, these rate of increase are about 19\%, 25\%, and 23\% respectively. While GPT3 is the largest model amongst these models (175 Billion parameter), Starmie-GPT3 took an average of 0.3 seconds per query regardless of the size of the prompt. However, the cost of each prompt inference increased by $\$0.005 \cdot ICL$ for each query where $ICL$ refers to the number of In-Context learning examples provided in the prompt.
\begin{figure}[hbt!]
\centering
\includegraphics[width=\linewidth]{figures/joinable_example.pdf}
\caption{Unionable and Joinable example in \bname}
\label{fig:joinable_example}
\end{figure}
In \cref{fig:generation-example}, we illustrated a relatively challenging example generated by \tname framework. While such examples do exist in \bname, making it a relatively challenging benchmark, there are also table-pair instances that are relatively easier or are more related to other kinds of inter-table tasks. In \cref{fig:joinable_example}, we illustrate a unionable instance that has a large overlap between the first two columns of each table. Since the overlap includes exact column names, row values, and their relative positions, this example is not a challenging one. Furthermore, this is more related to a joinable table task than a unionable one. Since the prompts in \tname are zero-shot in nature, LLM interprets the unionability task to various degrees. As future work, including examples for each table-pair generation could be more helpful to create more unionability-specific tasks. While this can be limited due by token size limits in LLM prompting, there are many new LLMs such as GPT4~\cite{openai2023gpt4}, that have very recently been released and have higher token limit tolerance. Apart from this direction, the example showcases the ability for LLM to generate other inter-table tasks such as joinability. Hence, \tname framework can be used to generate other inter-table task benchmarks for tasks such as entity-matching and joinability. Lastly, using LLMs as new table union search method relies on current methods such as Starmie to provide a small set of candidate unionable tables and then classify whether an LLM understands these are unionable or not. As future work, we would like to have a framework where an LLM can be a standalone method to perform table-union search.
% Starmie-GPT3: 5 minutes for 1000 queries (regardless of how many shots)
% Starmie-Vicuna: average 526 seconds for 1000 queries (zero-shot)
% Starmie-GPT2XL: average 198 seconds for 1000 queries (zero-shot)
% Starmie-Alpaca: average 540 seconds for 1000 queries (zero-shot)
% Starmie-Vicuna: average 526 seconds for 1000 queries (zero-shot)
% Starmie-GPT2XL: average 198 seconds for 1000 queries (zero-shot)
% Starmie-Alpaca: average 540 seconds for 1000 queries (zero-shot)
% - Benefits of the new benchmark (why the llm thinks this is unionable for instance...)
\section{Conclusion}
\label{sec:conclusion}
We present \tname, a framework that uses Large Language Models to automatically generate tables for the table union search task. Using \tname, we generate the \bname benchmark, which we showed to be a realistic but more challenging benchmark than existing benchmarks for state-of-the-art table union search methods. It also allows for a more in-depth analysis of these methods, since it generates labeled non-unionable and allows the user to control the topics used. Our new methodology enabled the first topic-based analysis of table union search and the first in-depth analysis of all methods true/false positive/negative rates. Finally, we have made both \tname and \bname publicly available for other researchers and practitioners to utilize in their experiments and analysis.
% \roee{Remember to make a clear statement about the fact that we use gpt-3 and will move to gpt-4 in future work.}\koyena{done in discussion}
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{45}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[\protect\citeauthoryear{??}{com}{[n.d.]}]%
{commoncrawl}
\bibinfo{year}{[n.d.]}\natexlab{}.
\newblock \bibinfo{title}{Common Crawl}.
\newblock \bibinfo{howpublished}{https://commoncrawl.org/}.
\newblock
\bibitem[\protect\citeauthoryear{Arocena, Glavic, Ciucanu, and Miller}{Arocena
et~al\mbox{.}}{2015a}]%
{2015_arocena_ibench}
\bibfield{author}{\bibinfo{person}{Patricia~C. Arocena}, \bibinfo{person}{Boris
Glavic}, \bibinfo{person}{Radu Ciucanu}, {and}
\bibinfo{person}{Ren{\'{e}}e~J. Miller}.} \bibinfo{year}{2015}\natexlab{a}.
\newblock \showarticletitle{The iBench Integration Metadata Generator}.
\newblock \bibinfo{journal}{\emph{Proc. {VLDB} Endow.}} \bibinfo{volume}{9},
\bibinfo{number}{3} (\bibinfo{year}{2015}), \bibinfo{pages}{108--119}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.14778/2850583.2850586}
\showDOI{\tempurl}
\bibitem[\protect\citeauthoryear{Arocena, Glavic, Mecca, Miller, Papotti, and
Santoro}{Arocena et~al\mbox{.}}{2015b}]%
{2015_arocena_bart}
\bibfield{author}{\bibinfo{person}{Patricia~C. Arocena}, \bibinfo{person}{Boris
Glavic}, \bibinfo{person}{Giansalvatore Mecca},
\bibinfo{person}{Ren{\'{e}}e~J. Miller}, \bibinfo{person}{Paolo Papotti},
{and} \bibinfo{person}{Donatello Santoro}.} \bibinfo{year}{2015}\natexlab{b}.
\newblock \showarticletitle{Messing Up with {BART:} Error Generation for
Evaluating Data-Cleaning Algorithms}.
\newblock \bibinfo{journal}{\emph{Proc. {VLDB} Endow.}} \bibinfo{volume}{9},
\bibinfo{number}{2} (\bibinfo{year}{2015}), \bibinfo{pages}{36--47}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.14778/2850578.2850579}
\showDOI{\tempurl}
\bibitem[\protect\citeauthoryear{Arora, Yang, Eyuboglu, Narayan, Hojel,
Trummer, and Ré}{Arora et~al\mbox{.}}{2023}]%
{arora2023language}
\bibfield{author}{\bibinfo{person}{Simran Arora}, \bibinfo{person}{Brandon
Yang}, \bibinfo{person}{Sabri Eyuboglu}, \bibinfo{person}{Avanika Narayan},
\bibinfo{person}{Andrew Hojel}, \bibinfo{person}{Immanuel Trummer}, {and}
\bibinfo{person}{Christopher Ré}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Language Models Enable Simple Systems for Generating
Structured Views of Heterogeneous Data Lakes}.
\newblock
\newblock
\showeprint[arxiv]{2304.09433}~[cs.CL]
\bibitem[\protect\citeauthoryear{Bogatu, Fernandes, Paton, and
Konstantinou}{Bogatu et~al\mbox{.}}{2020}]%
{DBLP:conf/icde/BogatuFP020}
\bibfield{author}{\bibinfo{person}{Alex Bogatu}, \bibinfo{person}{Alvaro A.~A.
Fernandes}, \bibinfo{person}{Norman~W. Paton}, {and}
\bibinfo{person}{Nikolaos Konstantinou}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Dataset Discovery in Data Lakes}. In
\bibinfo{booktitle}{\emph{ICDE}}. \bibinfo{pages}{709--720}.
\newblock
\bibitem[\protect\citeauthoryear{Brinkmann, Primpeli, and Bizer}{Brinkmann
et~al\mbox{.}}{2023a}]%
{brinkmann2023web}
\bibfield{author}{\bibinfo{person}{Alexander Brinkmann}, \bibinfo{person}{Anna
Primpeli}, {and} \bibinfo{person}{Christian Bizer}.}
\bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{The Web Data Commons Schema. org Data Set Series}.
In \bibinfo{booktitle}{\emph{Companion Proceedings of the ACM Web Conference
2023}}. \bibinfo{pages}{136--139}.
\newblock
\bibitem[\protect\citeauthoryear{Brinkmann, Shraga, Der, and Bizer}{Brinkmann
et~al\mbox{.}}{2023b}]%
{brinkmann2023product}
\bibfield{author}{\bibinfo{person}{Alexander Brinkmann}, \bibinfo{person}{Roee
Shraga}, \bibinfo{person}{Reng~Chiz Der}, {and} \bibinfo{person}{Christian
Bizer}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{Product Information Extraction using ChatGPT}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2306.14921}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Brown, Mann, Ryder, Subbiah, Kaplan, Dhariwal,
Neelakantan, Shyam, Sastry, Askell, et~al\mbox{.}}{Brown
et~al\mbox{.}}{2020}]%
{brown2020language}
\bibfield{author}{\bibinfo{person}{Tom Brown}, \bibinfo{person}{Benjamin Mann},
\bibinfo{person}{Nick Ryder}, \bibinfo{person}{Melanie Subbiah},
\bibinfo{person}{Jared~D Kaplan}, \bibinfo{person}{Prafulla Dhariwal},
\bibinfo{person}{Arvind Neelakantan}, \bibinfo{person}{Pranav Shyam},
\bibinfo{person}{Girish Sastry}, \bibinfo{person}{Amanda Askell},
{et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Language models are few-shot learners}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing
systems}} \bibinfo{volume}{33} (\bibinfo{year}{2020}),
\bibinfo{pages}{1877--1901}.
\newblock
\bibitem[\protect\citeauthoryear{Carlini, Ippolito, Jagielski, Lee, Tramer, and
Zhang}{Carlini et~al\mbox{.}}{2022}]%
{carlini2022quantifying}
\bibfield{author}{\bibinfo{person}{Nicholas Carlini}, \bibinfo{person}{Daphne
Ippolito}, \bibinfo{person}{Matthew Jagielski}, \bibinfo{person}{Katherine
Lee}, \bibinfo{person}{Florian Tramer}, {and} \bibinfo{person}{Chiyuan
Zhang}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Quantifying Memorization Across Neural Language
Models}. In \bibinfo{booktitle}{\emph{The Eleventh International Conference
on Learning Representations}}.
\newblock
\bibitem[\protect\citeauthoryear{Carlini, Tramer, Wallace, Jagielski,
Herbert-Voss, Lee, Roberts, Brown, Song, Erlingsson, et~al\mbox{.}}{Carlini
et~al\mbox{.}}{2021}]%
{carlini2021extracting}
\bibfield{author}{\bibinfo{person}{Nicholas Carlini}, \bibinfo{person}{Florian
Tramer}, \bibinfo{person}{Eric Wallace}, \bibinfo{person}{Matthew Jagielski},
\bibinfo{person}{Ariel Herbert-Voss}, \bibinfo{person}{Katherine Lee},
\bibinfo{person}{Adam Roberts}, \bibinfo{person}{Tom Brown},
\bibinfo{person}{Dawn Song}, \bibinfo{person}{Ulfar Erlingsson},
{et~al\mbox{.}}} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Extracting training data from large language
models}. In \bibinfo{booktitle}{\emph{30th USENIX Security Symposium (USENIX
Security 21)}}. \bibinfo{pages}{2633--2650}.
\newblock
\bibitem[\protect\citeauthoryear{Chintagunta, Katariya, Amatriain, and
Kannan}{Chintagunta et~al\mbox{.}}{2021}]%
{chintagunta2021medically}
\bibfield{author}{\bibinfo{person}{Bharath Chintagunta}, \bibinfo{person}{Namit
Katariya}, \bibinfo{person}{Xavier Amatriain}, {and} \bibinfo{person}{Anitha
Kannan}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Medically aware GPT-3 as a data generator for
medical dialogue summarization}. In \bibinfo{booktitle}{\emph{Machine
Learning for Healthcare Conference}}. PMLR, \bibinfo{pages}{354--372}.
\newblock
\bibitem[\protect\citeauthoryear{Cohen, Geva, Berant, and Globerson}{Cohen
et~al\mbox{.}}{2023}]%
{cohen-etal-2023-crawling}
\bibfield{author}{\bibinfo{person}{Roi Cohen}, \bibinfo{person}{Mor Geva},
\bibinfo{person}{Jonathan Berant}, {and} \bibinfo{person}{Amir Globerson}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Crawling The Internal Knowledge-Base of Language
Models}. In \bibinfo{booktitle}{\emph{Findings of the Association for
Computational Linguistics: EACL 2023}}. \bibinfo{publisher}{Association for
Computational Linguistics}, \bibinfo{address}{Dubrovnik, Croatia},
\bibinfo{pages}{1856--1869}.
\newblock
\urldef\tempurl%
\url{https://aclanthology.org/2023.findings-eacl.139}
\showURL{%
\tempurl}
\bibitem[\protect\citeauthoryear{Cutrona, Chen, Efthymiou, Hassanzadeh,
Jim{\'{e}}nez{-}Ruiz, Sequeda, Srinivas, Abdelmageed, Hulsebos, Oliveira, and
Pesquita}{Cutrona et~al\mbox{.}}{2021}]%
{2021_cutrona_semtab}
\bibfield{author}{\bibinfo{person}{Vincenzo Cutrona}, \bibinfo{person}{Jiaoyan
Chen}, \bibinfo{person}{Vasilis Efthymiou}, \bibinfo{person}{Oktie
Hassanzadeh}, \bibinfo{person}{Ernesto Jim{\'{e}}nez{-}Ruiz},
\bibinfo{person}{Juan Sequeda}, \bibinfo{person}{Kavitha Srinivas},
\bibinfo{person}{Nora Abdelmageed}, \bibinfo{person}{Madelon Hulsebos},
\bibinfo{person}{Daniela Oliveira}, {and} \bibinfo{person}{Catia Pesquita}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Results of SemTab 2021}. In
\bibinfo{booktitle}{\emph{Semantic Web Challenge on Tabular Data to Knowledge
Graph Matching}}, Vol.~\bibinfo{volume}{3103}. \bibinfo{pages}{1--12}.
\newblock
\bibitem[\protect\citeauthoryear{Dehghani, Zamani, Severyn, Kamps, and
Croft}{Dehghani et~al\mbox{.}}{2017}]%
{dehghani2017neural}
\bibfield{author}{\bibinfo{person}{Mostafa Dehghani}, \bibinfo{person}{Hamed
Zamani}, \bibinfo{person}{Aliaksei Severyn}, \bibinfo{person}{Jaap Kamps},
{and} \bibinfo{person}{W~Bruce Croft}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Neural ranking models with weak supervision}. In
\bibinfo{booktitle}{\emph{Proceedings of the 40th international ACM SIGIR
conference on research and development in information retrieval}}.
\bibinfo{pages}{65--74}.
\newblock
\bibitem[\protect\citeauthoryear{Dong, Li, Dai, Zheng, Wu, Chang, Sun, Xu, and
Sui}{Dong et~al\mbox{.}}{2022}]%
{dong2022survey}
\bibfield{author}{\bibinfo{person}{Qingxiu Dong}, \bibinfo{person}{Lei Li},
\bibinfo{person}{Damai Dai}, \bibinfo{person}{Ce Zheng},
\bibinfo{person}{Zhiyong Wu}, \bibinfo{person}{Baobao Chang},
\bibinfo{person}{Xu Sun}, \bibinfo{person}{Jingjing Xu}, {and}
\bibinfo{person}{Zhifang Sui}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{A Survey for In-context Learning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2301.00234}}
(\bibinfo{year}{2022}).
\newblock
\bibitem[\protect\citeauthoryear{Fan, Wang, Li, Zhang, and Miller}{Fan
et~al\mbox{.}}{2023}]%
{DBLP:journals/pvldb/FanWLZM23}
\bibfield{author}{\bibinfo{person}{Grace Fan}, \bibinfo{person}{Jin Wang},
\bibinfo{person}{Yuliang Li}, \bibinfo{person}{Dan Zhang}, {and}
\bibinfo{person}{Ren{\'{e}}e~J. Miller}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Semantics-aware Dataset Discovery from Data Lakes
with Contextualized Column-based Representation Learning}.
\newblock \bibinfo{journal}{\emph{{PVDLB}}} \bibinfo{volume}{16},
\bibinfo{number}{7} (\bibinfo{year}{2023}), \bibinfo{pages}{1726--1739}.
\newblock
\bibitem[\protect\citeauthoryear{Gray}{Gray}{1993}]%
{DBLP:books/mk/Gray93}
\bibfield{editor}{\bibinfo{person}{Jim Gray}} (Ed.).
\bibinfo{year}{1993}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{The Benchmark Handbook for Database and
Transaction Systems (2nd Edition)}}.
\newblock \bibinfo{publisher}{Morgan Kaufmann}.
\newblock
\showISBNx{1-55860-292-5}
\bibitem[\protect\citeauthoryear{Guha, Brickley, and Macbeth}{Guha
et~al\mbox{.}}{2016}]%
{guha2016schema}
\bibfield{author}{\bibinfo{person}{Ramanathan~V Guha}, \bibinfo{person}{Dan
Brickley}, {and} \bibinfo{person}{Steve Macbeth}.}
\bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Schema. org: evolution of structured data on the
web}.
\newblock \bibinfo{journal}{\emph{Commun. ACM}} \bibinfo{volume}{59},
\bibinfo{number}{2} (\bibinfo{year}{2016}), \bibinfo{pages}{44--51}.
\newblock
\bibitem[\protect\citeauthoryear{Helali, Vashisth, Carrier, Hose, and
Mansour}{Helali et~al\mbox{.}}{2023}]%
{2023_helali_linked_data}
\bibfield{author}{\bibinfo{person}{Mossad Helali}, \bibinfo{person}{Shubham
Vashisth}, \bibinfo{person}{Philippe Carrier}, \bibinfo{person}{Katja Hose},
{and} \bibinfo{person}{Essam Mansour}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Linked Data Science Powered by Knowledge Graphs}.
\newblock \bibinfo{journal}{\emph{CoRR}} \bibinfo{volume}{abs/2303.02204}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Hu, Wang, Qin, Lei, Shen, Faloutsos,
Katsifodimos, Karypis, Wen, and Yu}{Hu et~al\mbox{.}}{2023}]%
{2023_hu_autotus}
\bibfield{author}{\bibinfo{person}{Xuming Hu}, \bibinfo{person}{Shen Wang},
\bibinfo{person}{Xiao Qin}, \bibinfo{person}{Chuan Lei},
\bibinfo{person}{Zhengyuan Shen}, \bibinfo{person}{Christos Faloutsos},
\bibinfo{person}{Asterios Katsifodimos}, \bibinfo{person}{George Karypis},
\bibinfo{person}{Lijie Wen}, {and} \bibinfo{person}{Philip~S. Yu}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Automatic Table Union Search with Tabular
Representation Learning}. In \bibinfo{booktitle}{\emph{Findings of the
Association for Computational Linguistics: {ACL} 2023, Toronto, Canada, July
9-14, 2023}}. \bibinfo{publisher}{Association for Computational Linguistics},
\bibinfo{pages}{3786--3800}.
\newblock
\urldef\tempurl%
\url{https://aclanthology.org/2023.findings-acl.233}
\showURL{%
\tempurl}
\bibitem[\protect\citeauthoryear{Hulsebos, Demiralp, and Groth}{Hulsebos
et~al\mbox{.}}{2023}]%
{2023_hulsebos_gittables}
\bibfield{author}{\bibinfo{person}{Madelon Hulsebos},
\bibinfo{person}{{\c{C}}agatay Demiralp}, {and} \bibinfo{person}{Paul
Groth}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{GitTables: {A} Large-Scale Corpus of Relational
Tables}.
\newblock \bibinfo{journal}{\emph{Proc. {ACM} Manag. Data}}
\bibinfo{volume}{1}, \bibinfo{number}{1} (\bibinfo{year}{2023}),
\bibinfo{pages}{30:1--30:17}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.1145/3588710}
\showDOI{\tempurl}
\bibitem[\protect\citeauthoryear{Khatiwada, Fan, Shraga, Chen, Gatterbauer,
Miller, and Riedewald}{Khatiwada et~al\mbox{.}}{2023}]%
{2023_khatiwada_santos}
\bibfield{author}{\bibinfo{person}{Aamod Khatiwada}, \bibinfo{person}{Grace
Fan}, \bibinfo{person}{Roee Shraga}, \bibinfo{person}{Zixuan Chen},
\bibinfo{person}{Wolfgang Gatterbauer}, \bibinfo{person}{Renée~J. Miller},
{and} \bibinfo{person}{Mirek Riedewald}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{SANTOS: Relationship-based Semantic Table Union
Search}. In \bibinfo{booktitle}{\emph{Accepted to appear in SIGMOD
Conference}}. \bibinfo{publisher}{ACM}.
\newblock
\urldef\tempurl%
\url{https://arxiv.org/pdf/2209.13589.pdf}
\showURL{%
\tempurl}
\bibitem[\protect\citeauthoryear{Koutras, Siachamis, Ionescu, Psarakis, Brons,
Fragkoulis, Lofi, Bonifati, and Katsifodimos}{Koutras et~al\mbox{.}}{2021}]%
{DBLP:conf/icde/KoutrasSIPBFLBK21}
\bibfield{author}{\bibinfo{person}{Christos Koutras}, \bibinfo{person}{George
Siachamis}, \bibinfo{person}{Andra Ionescu}, \bibinfo{person}{Kyriakos
Psarakis}, \bibinfo{person}{Jerry Brons}, \bibinfo{person}{Marios
Fragkoulis}, \bibinfo{person}{Christoph Lofi}, \bibinfo{person}{Angela
Bonifati}, {and} \bibinfo{person}{Asterios Katsifodimos}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Valentine: Evaluating Matching Techniques for
Dataset Discovery}. In \bibinfo{booktitle}{\emph{{ICDE}}}.
\bibinfo{pages}{468--479}.
\newblock
\bibitem[\protect\citeauthoryear{Leventidis, Rocco, Gatterbauer, Miller, and
Riedewald}{Leventidis et~al\mbox{.}}{2021}]%
{DBLP:conf/edbt/LeventidisRMRG21}
\bibfield{author}{\bibinfo{person}{Aristotelis Leventidis},
\bibinfo{person}{Laura~Di Rocco}, \bibinfo{person}{Wolfgang Gatterbauer},
\bibinfo{person}{Ren{\'{e}}e~J. Miller}, {and} \bibinfo{person}{Mirek
Riedewald}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{DomainNet: Homograph Detection for Data Lake
Disambiguation}. In \bibinfo{booktitle}{\emph{{EDBT}}},
\bibfield{editor}{\bibinfo{person}{Yannis Velegrakis},
\bibinfo{person}{Demetris Zeinalipour{-}Yazti}, \bibinfo{person}{Panos~K.
Chrysanthis}, {and} \bibinfo{person}{Francesco Guerra}} (Eds.).
\bibinfo{pages}{13--24}.
\newblock
\bibitem[\protect\citeauthoryear{Liu, Shen, Zhang, Dolan, Carin, and Chen}{Liu
et~al\mbox{.}}{2021}]%
{liu2021makes}
\bibfield{author}{\bibinfo{person}{Jiachang Liu}, \bibinfo{person}{Dinghan
Shen}, \bibinfo{person}{Yizhe Zhang}, \bibinfo{person}{Bill Dolan},
\bibinfo{person}{Lawrence Carin}, {and} \bibinfo{person}{Weizhu Chen}.}
\bibinfo{year}{2021}\natexlab{}.
\newblock \bibinfo{title}{What Makes Good In-Context Examples for GPT-$3$?}
\newblock
\newblock
\showeprint[arxiv]{2101.06804}~[cs.CL]
\bibitem[\protect\citeauthoryear{Liu, Ott, Goyal, Du, Joshi, Chen, Levy, Lewis,
Zettlemoyer, and Stoyanov}{Liu et~al\mbox{.}}{2019}]%
{liu2019roberta}
\bibfield{author}{\bibinfo{person}{Yinhan Liu}, \bibinfo{person}{Myle Ott},
\bibinfo{person}{Naman Goyal}, \bibinfo{person}{Jingfei Du},
\bibinfo{person}{Mandar Joshi}, \bibinfo{person}{Danqi Chen},
\bibinfo{person}{Omer Levy}, \bibinfo{person}{Mike Lewis},
\bibinfo{person}{Luke Zettlemoyer}, {and} \bibinfo{person}{Veselin
Stoyanov}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Roberta: A robustly optimized bert pretraining
approach}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1907.11692}}
(\bibinfo{year}{2019}).
\newblock
\bibitem[\protect\citeauthoryear{Nargesian, Zhu, Pu, and Miller}{Nargesian
et~al\mbox{.}}{2018}]%
{nargesian2018table}
\bibfield{author}{\bibinfo{person}{Fatemeh Nargesian}, \bibinfo{person}{Erkang
Zhu}, \bibinfo{person}{Ken~Q Pu}, {and} \bibinfo{person}{Ren{\'e}e~J
Miller}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Table union search on open data}.
\newblock \bibinfo{journal}{\emph{PVLDB}} \bibinfo{volume}{11},
\bibinfo{number}{7} (\bibinfo{year}{2018}), \bibinfo{pages}{813--825}.
\newblock
\bibitem[\protect\citeauthoryear{OpenAI}{OpenAI}{[n.d.]}]%
{openaichat}
\bibfield{author}{\bibinfo{person}{OpenAI}.} \bibinfo{year}{[n.d.]}\natexlab{}.
\newblock \bibinfo{title}{Chat GPT}.
\newblock \bibinfo{howpublished}{https://chat.openai.com/chat}.
\newblock
\bibitem[\protect\citeauthoryear{OpenAI}{OpenAI}{2023}]%
{openai2023gpt4}
\bibfield{author}{\bibinfo{person}{OpenAI}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{GPT-4 Technical Report}.
\newblock
\newblock
\showeprint[arxiv]{2303.08774}~[cs.CL]
\bibitem[\protect\citeauthoryear{Patterson}{Patterson}{2012}]%
{patterson2012technical}
\bibfield{author}{\bibinfo{person}{David Patterson}.}
\bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Technical perspective for better or worse,
benchmarks shape a field}.
\newblock \bibinfo{journal}{\emph{Commun. ACM}} \bibinfo{volume}{55},
\bibinfo{number}{7} (\bibinfo{year}{2012}).
\newblock
\bibitem[\protect\citeauthoryear{Peeters and Bizer}{Peeters and Bizer}{2023}]%
{peeters2023using}
\bibfield{author}{\bibinfo{person}{Ralph Peeters} {and}
\bibinfo{person}{Christian Bizer}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Using ChatGPT for Entity Matching}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2305.03423}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Primpeli and Bizer}{Primpeli and
Bizer}{2020}]%
{primpeli2020profiling}
\bibfield{author}{\bibinfo{person}{Anna Primpeli} {and}
\bibinfo{person}{Christian Bizer}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Profiling entity matching benchmark tasks}. In
\bibinfo{booktitle}{\emph{CIKM}}. \bibinfo{pages}{3101--3108}.
\newblock
\bibitem[\protect\citeauthoryear{Shraga, Roitman, Feigenblat, and
Cannim}{Shraga et~al\mbox{.}}{2020}]%
{shraga2020web}
\bibfield{author}{\bibinfo{person}{Roee Shraga}, \bibinfo{person}{Haggai
Roitman}, \bibinfo{person}{Guy Feigenblat}, {and} \bibinfo{person}{Mustafa
Cannim}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Web table retrieval using multimodal deep
learning}. In \bibinfo{booktitle}{\emph{Proceedings of the 43rd international
ACM SIGIR conference on research and development in information retrieval}}.
\bibinfo{pages}{1399--1408}.
\newblock
\bibitem[\protect\citeauthoryear{Srinivas, Dolby, Abdelaziz, Hassanzadeh,
Kokel, Khatiwada, Pedapati, Chaudhury, and Samulowitz}{Srinivas
et~al\mbox{.}}{2023}]%
{srinivas2023lakebench}
\bibfield{author}{\bibinfo{person}{Kavitha Srinivas}, \bibinfo{person}{Julian
Dolby}, \bibinfo{person}{Ibrahim Abdelaziz}, \bibinfo{person}{Oktie
Hassanzadeh}, \bibinfo{person}{Harsha Kokel}, \bibinfo{person}{Aamod
Khatiwada}, \bibinfo{person}{Tejaswini Pedapati}, \bibinfo{person}{Subhajit
Chaudhury}, {and} \bibinfo{person}{Horst Samulowitz}.}
\bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{LakeBench: Benchmarks for Data Discovery over Data
Lakes}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2307.04217}}
(\bibinfo{year}{2023}).
\newblock
\bibitem[\protect\citeauthoryear{Taori, Gulrajani, Zhang, Dubois, Li, Guestrin,
Liang, and Hashimoto}{Taori et~al\mbox{.}}{2023}]%
{alpaca}
\bibfield{author}{\bibinfo{person}{Rohan Taori}, \bibinfo{person}{Ishaan
Gulrajani}, \bibinfo{person}{Tianyi Zhang}, \bibinfo{person}{Yann Dubois},
\bibinfo{person}{Xuechen Li}, \bibinfo{person}{Carlos Guestrin},
\bibinfo{person}{Percy Liang}, {and} \bibinfo{person}{Tatsunori~B.
Hashimoto}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Stanford Alpaca: An Instruction-following LLaMA
model}.
\newblock
\bibinfo{howpublished}{\url{https://github.com/tatsu-lab/stanford_alpaca}}.
\newblock
\bibitem[\protect\citeauthoryear{Trummer}{Trummer}{2022a}]%
{trummer2022codexdb}
\bibfield{author}{\bibinfo{person}{Immanuel Trummer}.}
\bibinfo{year}{2022}\natexlab{a}.
\newblock \showarticletitle{CodexDB: Synthesizing code for query processing
from natural language instructions using GPT-3 Codex}.
\newblock \bibinfo{journal}{\emph{Proceedings of the VLDB Endowment}}
\bibinfo{volume}{15}, \bibinfo{number}{11} (\bibinfo{year}{2022}),
\bibinfo{pages}{2921--2928}.
\newblock
\bibitem[\protect\citeauthoryear{Trummer}{Trummer}{2022b}]%
{trummer2022bert}
\bibfield{author}{\bibinfo{person}{Immanuel Trummer}.}
\bibinfo{year}{2022}\natexlab{b}.
\newblock \showarticletitle{From BERT to GPT-3 codex: harnessing the potential
of very large language models for data management}.
\newblock \bibinfo{journal}{\emph{Proceedings of the VLDB Endowment}}
\bibinfo{volume}{15}, \bibinfo{number}{12} (\bibinfo{year}{2022}),
\bibinfo{pages}{3770--3773}.
\newblock
\bibitem[\protect\citeauthoryear{Veyseh, Nguyen, Min, and Nguyen}{Veyseh
et~al\mbox{.}}{2021}]%
{Veyseh2021AugmentingOE}
\bibfield{author}{\bibinfo{person}{Amir Pouran~Ben Veyseh},
\bibinfo{person}{Minh~Le Nguyen}, \bibinfo{person}{Bonan Min}, {and}
\bibinfo{person}{Thien~Huu Nguyen}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Augmenting Open-Domain Event Detection with
Synthetic Data from GPT-2}. In \bibinfo{booktitle}{\emph{ECML/PKDD}}.
\newblock
\bibitem[\protect\citeauthoryear{{WDC}}{{WDC}}{2017}]%
{WDC_t2d}
{WDC} \bibinfo{year}{2017}\natexlab{}.
\newblock \bibinfo{title}{{T2D Gold Standard}}.
\newblock
\newblock
\newblock
\shownote{http://webdatacommons.org/webtables/goldstandard.html.}
\bibitem[\protect\citeauthoryear{West, Bhagavatula, Hessel, Hwang, Jiang,
Le~Bras, Lu, Welleck, and Choi}{West et~al\mbox{.}}{2022}]%
{west-etal-2022-symbolic}
\bibfield{author}{\bibinfo{person}{Peter West}, \bibinfo{person}{Chandra
Bhagavatula}, \bibinfo{person}{Jack Hessel}, \bibinfo{person}{Jena Hwang},
\bibinfo{person}{Liwei Jiang}, \bibinfo{person}{Ronan Le~Bras},
\bibinfo{person}{Ximing Lu}, \bibinfo{person}{Sean Welleck}, {and}
\bibinfo{person}{Yejin Choi}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Symbolic Knowledge Distillation: from General
Language Models to Commonsense Models}. In
\bibinfo{booktitle}{\emph{Proceedings of the 2022 Conference of the North
American Chapter of the Association for Computational Linguistics: Human
Language Technologies}}. \bibinfo{publisher}{Association for Computational
Linguistics}, \bibinfo{address}{Seattle, United States},
\bibinfo{pages}{4602--4625}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.18653/v1/2022.naacl-main.341}
\showDOI{\tempurl}
\bibitem[\protect\citeauthoryear{Wolf, Debut, Sanh, Chaumond, Delangue, Moi,
Cistac, Rault, Louf, Funtowicz, et~al\mbox{.}}{Wolf et~al\mbox{.}}{2020}]%
{wolf2020transformers}
\bibfield{author}{\bibinfo{person}{Thomas Wolf}, \bibinfo{person}{Lysandre
Debut}, \bibinfo{person}{Victor Sanh}, \bibinfo{person}{Julien Chaumond},
\bibinfo{person}{Clement Delangue}, \bibinfo{person}{Anthony Moi},
\bibinfo{person}{Pierric Cistac}, \bibinfo{person}{Tim Rault},
\bibinfo{person}{R{\'e}mi Louf}, \bibinfo{person}{Morgan Funtowicz},
{et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Transformers: State-of-the-art natural language
processing}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020 conference
on empirical methods in natural language processing: system demonstrations}}.
\bibinfo{pages}{38--45}.
\newblock
\bibitem[\protect\citeauthoryear{Xian, Lampert, Schiele, and Akata}{Xian
et~al\mbox{.}}{2018}]%
{xian2018zero}
\bibfield{author}{\bibinfo{person}{Yongqin Xian}, \bibinfo{person}{Christoph~H
Lampert}, \bibinfo{person}{Bernt Schiele}, {and} \bibinfo{person}{Zeynep
Akata}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Zero-shot learning—a comprehensive evaluation of
the good, the bad and the ugly}.
\newblock \bibinfo{journal}{\emph{IEEE transactions on pattern analysis and
machine intelligence}} \bibinfo{volume}{41}, \bibinfo{number}{9}
(\bibinfo{year}{2018}), \bibinfo{pages}{2251--2265}.
\newblock
\bibitem[\protect\citeauthoryear{Yang, Malaviya, Fernandez, Swayamdipta, Bras,
Wang, Bhagavatula, Choi, and Downey}{Yang et~al\mbox{.}}{2020}]%
{yang2020generative}
\bibfield{author}{\bibinfo{person}{Yiben Yang}, \bibinfo{person}{Chaitanya
Malaviya}, \bibinfo{person}{Jared Fernandez}, \bibinfo{person}{Swabha
Swayamdipta}, \bibinfo{person}{Ronan~Le Bras}, \bibinfo{person}{Ji-Ping
Wang}, \bibinfo{person}{Chandra Bhagavatula}, \bibinfo{person}{Yejin Choi},
{and} \bibinfo{person}{Doug Downey}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Generative data augmentation for commonsense
reasoning}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2004.11546}}
(\bibinfo{year}{2020}).
\newblock
\bibitem[\protect\citeauthoryear{Zhao, Wallace, Feng, Klein, and Singh}{Zhao
et~al\mbox{.}}{2021}]%
{zhao2021calibrate}
\bibfield{author}{\bibinfo{person}{Tony~Z. Zhao}, \bibinfo{person}{Eric
Wallace}, \bibinfo{person}{Shi Feng}, \bibinfo{person}{Dan Klein}, {and}
\bibinfo{person}{Sameer Singh}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \bibinfo{title}{Calibrate Before Use: Improving Few-Shot Performance
of Language Models}.
\newblock
\newblock
\showeprint[arxiv]{2102.09690}~[cs.CL]
\bibitem[\protect\citeauthoryear{Zheng, Chiang, Sheng, Zhuang, Wu, Zhuang, Lin,
Li, Li, Xing, Zhang, Gonzalez, and Stoica}{Zheng et~al\mbox{.}}{2023}]%
{vicuna}
\bibfield{author}{\bibinfo{person}{Lianmin Zheng}, \bibinfo{person}{Wei-Lin
Chiang}, \bibinfo{person}{Ying Sheng}, \bibinfo{person}{Siyuan Zhuang},
\bibinfo{person}{Zhanghao Wu}, \bibinfo{person}{Yonghao Zhuang},
\bibinfo{person}{Zi Lin}, \bibinfo{person}{Zhuohan Li},
\bibinfo{person}{Dacheng Li}, \bibinfo{person}{Eric.~P Xing},
\bibinfo{person}{Hao Zhang}, \bibinfo{person}{Joseph~E. Gonzalez}, {and}
\bibinfo{person}{Ion Stoica}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Judging LLM-as-a-judge with MT-Bench and Chatbot
Arena}.
\newblock
\newblock
\showeprint[arxiv]{2306.05685}~[cs.CL]
\end{thebibliography}