A Control Function Framework for Mitigating Position Bias in Learning to Rank Systems
Extracted source text · arXiv 2506.06989
%%
%% This is file `sample-sigconf.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `all,proceedings,bibtex,sigconf')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%%
%% Commands for TeXCount
%TC:macro \cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%% The first command in your LaTeX source must be the \documentclass
%% command.
%%
%% For submission and review of your manuscript please change the
%% command to \documentclass[manuscript, screen, review]{acmart}.
%%
%% When submitting camera ready or to TAPS, please change the command
%% to \documentclass[sigconf]{acmart} or whichever template is required
%% for your publication.
%%
%%
\documentclass[sigconf, nonacm]{acmart}
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
Bib\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmlicensed}
\copyrightyear{2018}
\acmYear{2018}
\acmDOI{XXXXXXX.XXXXXXX}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[Conference acronym 'XX]{Make sure to enter the correct
conference title from your rights confirmation email}{June 03--05,
2018}{Woodstock, NY}
%%
%% Uncomment \acmBooktitle if the title of the proceedings is different
%% from ``Proceedings of ...''!
%%
%%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
%% June 03--05, 2018, Woodstock, NY}
\acmISBN{978-1-4503-XXXX-X/2018/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
\usepackage{graphicx}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{bm}
\usepackage{tabularx}
\usepackage{multirow}
\usepackage{makecell}
\usepackage{xr}
% \usepackage{appendix}
\usepackage{placeins}
\usepackage{enumitem}
\usepackage{array}
\usepackage{float}
\usepackage{dblfloatfix}
\usepackage{makecell}
\usepackage{amsmath}
\usepackage{algorithm}
\usepackage{algpseudocode}
\newcolumntype{C}[1]{>{\centering\arraybackslash}m{#1}}
\fontsize{7.8}{8}\selectfont
\newcommand{\figsize}{0.25}
% \hypersetup{
% colorlinks=true,
% citecolor=black % citations will appear black
% }
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
\thanks{This is a preprint version.}
\input{0-variables}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{Correcting for Position Bias in Learning to Rank: \\A Control Function Approach}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Md Aminul Islam }
\affiliation{%
\institution{University of Illinois Chicago}
\country{}
}
\email{mislam34@uic.edu}
\author{Kathryn Vasilaky}
\affiliation{%
\institution{California Polytechnic State University}
\country{}
}
\email{kvasilak@calpoly.edu}
\author{Elena Zheleva}
\affiliation{%
\institution{University of Illinois Chicago}
\country{}
}
\email{ezheleva@uic.edu}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Islam et al.}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
\input{0-abstract}
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>00000000.0000000.0000000</concept_id>
<concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>00000000.00000000.00000000</concept_id>
<concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
<concept_significance>300</concept_significance>
</concept>
<concept>
<concept_id>00000000.00000000.00000000</concept_id>
<concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
<concept_significance>100</concept_significance>
</concept>
<concept>
<concept_id>00000000.00000000.00000000</concept_id>
<concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
<concept_significance>100</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Information systems~Learning to rank}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords \keywordList
% remove copyright information
\setcopyright{none}
\pagestyle{plain}
\settopmatter{printacmref=false} % Removes citation information below abstract
\renewcommand\footnotetextcopyrightpermission[1]{} % removes footnote with conference information in first column
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
% \input{0-abstract}
\input{1-introduction}
\input{2-related-works}
\input{3-problem-description}
\input{4-methodology}
\input{5-experimental-setup}
\input{6-results-discussions}
\input{7-discussions}
\input{8-conclusion}
%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{main}
\newpage
%%
%% If your work has an appendix, this is the place to put it.
\appendix
\input{9-appendix}
\end{document}
\endinput
%%
%% End of file `sample-sigconf.tex'.
\newcommand{\keywordList}{unbiased learning to rank, position bias, control function}
% \begin{abstract}
Implicit feedback data, such as user clicks, is commonly used in learning-to-rank (LTR) systems because it is easy to collect and it often reflects user preferences. However, this data is prone to various biases, and training an LTR system directly on biased data can result in suboptimal ranking performance. One of the most prominent and well-studied biases in implicit feedback data is position bias, which occurs because users are more likely to interact with higher-ranked documents regardless of their true relevance. In this paper, we propose a novel control function-based method that accounts for position bias in a two-stage process. The first stage uses exogenous variation from the residuals of the ranking process to correct for position bias in the second stage click equation. Unlike previous position bias correction methods, our method does not require knowledge of the click or propensity model and allows for nonlinearity in the underlying ranking model. Moreover, our method is general and allows for debiasing any state-of-the-art ranking algorithm by plugging it into the second stage. We also introduce a technique to debias validation clicks for hyperparameter tuning to select the optimal model in the absence of unbiased validation data. Experimental results demonstrate that our method outperforms state-of-the-art approaches in correcting for position bias.
% \end{abstract}
% The first stage estimates the contribution of exogenous factors to the ranking of documents in the biased data, and the second stage accounts for this exogeneity in order to correct for position bias.
% In this paper, we introduce a novel control function-based method to mitigate position bias in LTR systems. We model the system's previous document ranking process using a control function and estimate parameters specific to each query-document pair to account for position bias in the ranking model.
% particularly position bias, where users are more likely to interact with higher-ranked documents regardless of their true relevance.
% Most position bias correction methods depend on click models, propensity estimation, or econometric techniques.
% Click modeling methods depend on assumptions about complex browsing behavior, which can be incorrectly specified. Estimating propensities from click data can be challenging when documents receive insufficient clicks or when clicks are sparse.
% Existing econometrics-based approaches for bias correction assume linear ranking models, which are unsuitable for LTR tasks. To overcome the limitations of existing methods,
% Control functions have been widely used in econometrics to correct for different types of biases and endogeneity problems.
% We refer to the residual of this control function as a correction term for position bias, which captures the unobserved factors influencing a document's rank for a given query. The residual is then used as an additional feature for each query-document pair in the ranking model to account for position bias.
\section{Introduction} \label{sec:introduction}
Learning-to-rank (LTR) systems have become a core component in many online applications, from search engines to recommender systems. These systems typically use implicit feedback data, such as user clicks, as a proxy for user preferences due to its ease of collection compared to costly manual human annotations. While valuable, implicit feedback data is prone to biases~\cite{ai-sigir18, yue-www10}, with position bias being particularly pervasive. Position bias is the tendency of users to interact more with higher-ranked documents, regardless of their true relevance~\cite{craswell-wsdm08, joachims-tois07}. This skews click data, causing LTR models to overestimate higher-ranked documents and underestimate lower ones. Training on such biased data degrades ranking performance and reinforces a feedback loop that negatively affects user experience.
State-of-the-art methods that address position bias can be broadly categorized as click models, propensity-based approaches, and econometrics-based models. Click models~\cite{chapelle-www09, craswell-wsdm08, dupret-sigir08, wang-www13} estimate true relevance of documents by hypothesizing how users browse and maximizing the likelihood of observed clicks. Click models rely on assumptions about complex user click behavior, which sometimes may not hold in practice. Two-tower models~\cite{guo-recsys19, yan-sigir22}, following the position-based click model (PBM)~\cite{richardson-www07}, attempt to disentangle relevance and bias from clicks by modeling them separately, but may fail when confounding exists between the two. Propensity-based methods treat clicks as counterfactual events and re-weight them using inverse propensity weighting (IPW) to achieve relevance-equivalent training loss~\cite{joachims-wsdm17, wang-sigir16}. Propensity estimation often relies on result randomization~\cite{joachims-wsdm17, oosterhuis-sigir20, wang-sigir16}, which can harm user experience~\cite{ai-sigir18, wang-wsdm18}. Some methods~\cite{ai-sigir18, hu-www19, luo-sigir24, vardasbi-cikm20, zhu-recsys20} jointly learn propensities and rankers, but coupling the two raises concerns about the unbiasedness if either model is misspecified~\cite{agarwal-wsdm19, fang-sigir19, tian-sigir20}.
% Several works~\cite{saito-recsys20, kiyohara-wsdm22, yuan-cikm19, zou-cikm22} apply doubly robust (DR) estimators to position-biased clicks, but address different problem settings than unbiased learning to rank (ULTR) and use action propensity corrections similar to standard counterfactual methods~\cite{oosterhuis-tis23}.
Doubly robust (DR) IPW-based bias correction methods~\cite{luo-wsdm23, oosterhuis-tis23} have also been proposed recently, where the DR estimator combines an IPW estimate with predictions from a regression model and becomes unbiased if either the estimated propensity or the regression model is accurate. Most click modeling and propensity-based methods require modifying the ranker's loss function, which makes them tightly coupled with a specific LTR algorithm and limits their generalization to other LTR algorithms. Alternative approaches~\cite{ovaisi-www20, ovaisi-sigir21}, based on Heckman’s two-stage econometric model~\cite{heckman-econometrica79}, avoid propensity estimation but assume linear ranking models. However, linear models often struggle to capture complex feature interactions and handle the sparse click data typical in LTR systems.
Addressing data bias has a long tradition in the econometrics literature. A prominent class of approaches for addressing various types of biases is control functions~\cite{wooldridge-jhr15} which includes a number of related well-known econometric techniques,
such as Heckman selection~\cite{heckman-econometrica79} and endogenous switching models~\cite{maddala-75sa}; shares the same conceptual foundations with
instrumental variables~\cite{reiersol-metrica41}; and exploits identification via orthogonal moments akin to, the more recent,
double machine learning~\cite{chernozhukov-ej18}. Control functions are two-stage methods which model bias through the endogeneity in the error term. In this paper, we introduce a control function for addressing position bias. In the first stage, we model the system’s previous ranking process, which generated the user feedback data based on observable features, to determine document positions in a ranked list for a query.
%In real-world systems unobserved factors can also play role in document positions and they should be accounted for even if they are not related to document relevance. Examples of such factors include system constraints, biases in prior data, limited training data, or external conditions that are not captured by observable features. These unobserved factors can push more relevant documents to lower positions or less relevant ones to higher positions, which in turn affects their likelihood of being clicked, affecting position bias.}
%In this paper, we introduce an alternative propensity-independent approach for addressing position bias in LTR systems that relies on control functions. Control function methods are a large class of econometric methods that use two-stage approaches for addressing various types of biases and endogeneity~\cite{wooldridge-jhr15}, and include a number of well-known econometric techniques such as instrumental variables, Heckman selection, and endogenous switching models. In the first stage, we model the system’s previous ranking process, which generated the user feedback data based on observable features, to determine document positions in a ranked list for a query. \modifynew{However, unobserved factors can also influence document positions, such as system constraints, biases in prior data, limited training data, or external conditions that are not captured by these observable features. These unobserved factors, which are not related to document relevance, can push more relevant documents to lower positions or less relevant ones to higher positions, which in turn affects their likelihood of being clicked, introducing position bias.}
In the second stage, we leverage the exogenous variation in first‐stage residuals %induced by heteroskedasticity—
as a control function within the ranking model to adjust for position bias. We introduce residual transformation as a hyperparameter in the second stage and suggest several carefully designed transformation options.
% In the second stage, the residuals from the first-stage model, which capture unobserved factors or exogenous variation of previous ranking process beyond the observable features, are used as a ``control function'' in the ranking model to account for position bias.
% In the second stage, the residuals from the control function, which capture unobserved factors beyond observable features, are used as correction terms to account for position bias in learning the ranking model.
%In contrast to existing methods for addressing position bias in LTR systems,
A control function approach for addressing position bias in LTR systems offers a number of advantages over existing approaches. %greater flexibility in modeling the data-generating process than existing , which can lead to improved learning to rank.
Unlike Heckman-based methods~\cite{ovaisi-www20, ovaisi-sigir21}, which require linear models in both stages, our control function method allows for any linear or non-linear models in both stages. Additionally, by regularizing the feature space during the modeling of the document ranking process, our method addresses multicollinearity issues with high-dimensional feature spaces that Heckman-based methods do not. While propensity-based methods~\cite{ai-sigir18, hu-www19, joachims-wsdm17, luo-sigir24, luo-wsdm23, oosterhuis-sigir20, vardasbi-cikm20, wang-sigir16, wang-wsdm18} reweight click data using IPW to account for bias, our method circumvents the need for propensity estimation. We estimate query-document-specific residuals by modeling the system's previous document ranking process to account for position bias. Our method also does not require result randomization~\cite{joachims-wsdm17, oosterhuis-sigir20, wang-sigir16}. Unlike click modeling methods~\cite{chapelle-www09, craswell-wsdm08, dupret-sigir08, wang-www13}, our method makes no assumptions about user browsing behavior in either of its stages. Moreover, our method addresses position bias without any modification to the ranker's loss function, which generalizes its application to any pointwise, pairwise, or listwise LTR algorithm.
% By avoiding these limitations, our method can handle the complexities of LTR tasks, including high-dimensional, nonlinear feature interactions and sparse clicks.
In addition, our work addresses a gap in the literature: the challenge of hyperparameter tuning for unbiased ranking models in the absence of unbiased ground truth or true relevance of documents. Most existing bias correction methods~\cite{ai-sigir18, joachims-wsdm17, luo-sigir24, luo-wsdm23, oosterhuis-sigir20, wang-wsdm18} either assume the availability of unbiased ground truth data~\cite{ai-sigir18, luo-sigir24, luo-wsdm23}, which is sometimes unrealistic, or rely on biased click data for hyperparameter tuning~\cite{joachims-wsdm17, oosterhuis-sigir20, wang-wsdm18}. However, learning to rank with biased validation data can exhibit significant declines in performance due to the distribution shift of unbiased test data~\cite{wang-www24, zhao-www25}.
% In contrast, we introduce a technique to debias validation clicks, ensuring that unbiased LTR methods maintaineurpn robust performance during hyperparameter tuning.
Here, we introduce a technique to debias validation clicks for hyperparameter tuning, ensuring that our method maintains robust performance when true relevance is unavailable.
% Our empirical evaluation on benchmark datasets demonstrates the strength of this approach. Our method consistently outperforms state-of-the-art methods across many setups including various severities of position bias and noisy clicks.
% We present a thorough empirical evaluation of our method, comparing it to state-of-the-art approaches under varying levels of position bias, noisy click assumptions, and the assumption of the availability and unavailability of true relevance for hyperparameter tuning.}
% In this paper, we present a new control function method for mitigating position bias in LTR systems. Control function methods ~\cite{wooldridge-jhr15} are two-stage methods that control for bias and have their roots in econometrics. By modeling the bias with a control function in the first stage, and then accounting for it in the second stage, these methods can account for endogeneity ~\cite{wooldridge-jhr15}. In the context of LTR systems, we model a control function to represent the system's previous document ranking process to determine its position in the ranked list for a given query. We recover this ranking process via the residuals of the control function. The control function represents the previous ranking process based on observable features, while the residuals capture all other unobserved components (e.g., biases, previous rankings, limited training data, or hidden features) that contribute to bias in the rankings. This query-document-specific residual serves as a correction term to account for the position bias for each query-document pair when learning the ranking model. Note that the Heckman model ~\cite{heckman-econometrica79} is an instance of the control function methods ~\cite{wooldridge-jhr15} that imposes specific assumptions and focuses on a particular form of selection bias correction. Whereas the Heckman model controls for the probability of being selected into the outcome pool, control function methods are more general in that they control for the unobserved components of the selection process, in our case, the process that dictates document rankings.
% In contrast to existing methods for addressing position bias in LTR systems, our approach does not require user browsing models ~\cite{chapelle-www09, craswell-wsdm08, dupret-sigir08, wang-www13}, propensity model specifications ~\cite{ai-sigir18, hu-www19, vardasbi-cikm20, zhu-recsys20, luo-wsdm23}, online result randomization ~\cite{agarwal-sigir19, joachims-wsdm17}, or linearity assumptions ~\cite{ovaisi-www20, ovaisi-sigir21}. Therefore, our method can address the limitations of existing position bias correction methods. In addition, many existing bias correction methods ~\cite{wang-wsdm18, wang-sigir16, luo-wsdm23, ai-sigir18, hu-www19, tran-cikm21} overlook the issue of hyperparameter tuning for an unbiased ranking model when true relevance is unavailable for validation. We develop a method to debias validation clicks using click data and show that unbiased learning to rank methods experience a significant decline in performance when using click data directly for hyperparameter tuning. Our empirical evaluation on benchmark datasets demonstrates that our method outperforms state-of-the-art approaches while remaining robust to different severities of position bias and noisy clicks.
\section{Related Work}
Position bias in LTR systems is mainly addressed through click models, propensity-based methods, and econometrics-based approaches. Several studies~\cite{chapelle-www09, chuklin-wsdm16, craswell-wsdm08, dupret-sigir08} aim to model clicks via user browsing behavior to capture true relevance feedback. Craswell et al.~\cite{craswell-wsdm08} propose the Cascade model, assuming users view results sequentially. Dupret et al.~\cite{dupret-sigir08} introduce the User Browsing Model (UBM), which estimates examination probabilities based on position and prior clicks, allowing users to skip results. Chapelle et al.~\cite{chapelle-www09} propose a Dynamic Bayesian Network (DBN) model that accounts for search abandonment by modeling user satisfaction by clicks. Wang et al.~\cite{wang-wsdm18} jointly train a ranking model and estimate examination propensity using a graphical model and an Expectation-Maximization (EM) algorithm.
Guo et al.~\cite{guo-recsys19} use a two-tower model based on PBM~\cite{richardson-www07} model, where one tower models relevance probability and the other models observation bias. While this approach aims to decouple relevance from position bias, it may be ineffective when the two are confounded, leading to biased estimates. A key drawback of click-based models is their reliance on assumptions about user browsing behavior, which may not always hold and can lead to inaccurate relevance estimation.
Propensity-based methods use counterfactual learning, treating bias as a counterfactual factor and debiasing user clicks through IPW. Several approaches~\cite{joachims-wsdm17, oosterhuis-sigir20, wang-sigir16} estimate propensities through result randomization. Wang et al.~\cite{wang-sigir16} randomize search results to estimate query-level position bias and apply IPW during training. Joachims et al.~\cite{joachims-wsdm17} show that, with correctly specified propensities, models trained on click data converge to those trained on true relevance. Ai et al.~\cite{ai-sigir18} and Hu et al.~\cite{hu-www19} propose learning algorithms that jointly learn the propensity model and the ranker. Vardasbi et al.~\cite{vardasbi-cikm20} develop a cascade model-based inverse propensity scoring method for estimating propensities under cascade click behavior. Oosterhuis et al.~\cite{oosterhuis-sigir20} propose a policy-aware propensity scoring method that uses a stochastic ranking policy to address selection and position bias. Luo et al.~\cite{luo-sigir24} show that strong position-relevance correlations can cause existing methods to overestimate propensities. They propose a causal approach that treats relevance as a confounder and use a two-step optimization for unconfounded propensity estimation. Ovaisi et al.~\cite{ovaisi-www20, ovaisi-sigir21} develop methods for correcting selection and position bias in LTR systems using the Heckman econometric model~\cite{heckman-econometrica79}, which do not require propensity estimation. However, Heckman-based approaches require linear models in both stages to satisfy their underlying assumptions.
Position-biased clicks has also been studied beyond LTR. IPW-based bandit algorithms for displaying ads in multiple positions simultaneously have been proposed by Komiyama et al.~\cite{komiyama-icml15} and Lagrée et al.~\cite{lagree-neurips16}. Several works apply DR estimators to position-biased clicks. Saito~\cite{saito-recsys20} proposes a DR estimator for post-click conversions by modeling user behavior after a click. For policy evaluation under cascading click behavior, Kiyohara et al.~\cite{kiyohara-wsdm22} propose a DR estimator. Yuan et al.~\cite{yuan-cikm19} develop a DR estimator for ad click prediction, accounting for both shown and unshown documents. Zou et al.~\cite{zou-cikm22} propose a unified framework for unbiased relevance estimation by combining counterfactual learning with a DR estimator. However, these methods address different problem settings than unbiased LTR. Unlike in unbiased LTR, where examination propensities are required, general DR estimators rely on action propensity corrections similar to standard counterfactual methods~\cite{oosterhuis-tis23}. Recent studies by Oosterhuis~\cite{oosterhuis-tis23} and Luo et al.~\cite{luo-wsdm23} introduce bias correction methods for LTR using IPW in a doubly robust way. Unbiased estimation in DR methods can be achieved by combining an IPW estimate with a regression-based prediction, provided that at least one of the two models is correctly specified.
% To mitigate the high variance problem of IPW, Luo et al. ~\cite{luo-wsdm23} propose a method that generates pseudo clicks for unobserved ranked lists and applies IPW with these pseudo clicks.
% However, the propensity is embedded implicitly within the user behavior data, such as user clicks ~\cite{wang-wsdm18}. Click data is often sparse ~\cite{saito-recsys20}, especially for long-tail queries, which can result in misspecification of the propensity model ~\cite{tian-sigir20}. The high sparsity of click data may cause propensity models to inaccurately capture the relationship between click behavior and position.
% \modify{However, as mentioned earlier, the high sparsity of click data may cause propensity models to inaccurately represent the relationship between click behavior and position.}
\section{Task Formulation}
We first present the formal definition of LTR systems following Ai et al.~\cite{ai-sigir18}. Let $\bm{\mathcal{Q}}$ be a universal set of independent and identically distributed (i.i.d.) queries, with each query $q$ sampled from the distribution $P(q)$. Given a ranking model $g \in \bm{\mathcal{G}}$, let $l(g, q)$ represent the loss incurred for query $q$ when all associated documents are annotated. The goal of an LTR system is to find the optimal model $g$ that minimizes the total empirical loss $\hat{\bm{\mathcal{L}}}(g)$ over all models in $\bm{\mathcal{G}}$.
\begin{equation}
\hat{\bm{\mathcal{L}}}(g) = \frac{1}{|\bm{\mathcal{Q}}|} \sum_{q \in \bm{\mathcal{Q}}} l(g, q).
\end{equation}
Although human-annotated labels are reliable indicators of true relevance, manually annotating every query-document pair is costly, especially with millions of documents. Instead, LTR systems often rely on implicit feedback, such as user clicks, which is easier to collect and better reflects user intent~\cite{joachims-kdd02}.
A document click can be modeled based on its relevance to the query using query-document features.
\begin{equation} \label{eq:biased_c}
C^{y}_{q} = g(\bm{x}) + \epsilon^{y}_{q,g}.
\end{equation}
$C^{y}_{g}$ denotes whether document $y$ associated with query $q$ has been clicked or not. $g()$ is a ranking model that predicts click probabilities from which documents can be ranked, and it depends on the query-document features $\bm{x}$, while $\epsilon^{y}_{q,g}$ represents the error term for document $y$ with query $q$. $g()$ can be learned from click data using equation~\eqref{eq:biased_c}. However, since the likelihood of clicking a document is also influenced by the document's position in the ranking,
%However, since clicks are biased toward higher-ranked documents,
directly learning from clicks will embed position bias into the ranking model $g()$.
Let us consider a deterministic ranking policy $\pi$, and let $r(y|\pi_q)$ be the position of document $y$ in the system's previous ranking $\pi_q$ presented for the query $q$. Not accounting for the document position ($r(y|\pi_q)$) in the click equation~\eqref{eq:biased_c} causes its effect to be absorbed into the error term $\epsilon^{y}_{q,g}$. However, because the document's position depends on the features and because its unobserved influences are correlated with that of clicks, the residual error in the click model is not independent of the features, i.e. $E(\epsilon^{y}_{q,g} \mid \bm{x}) \neq 0$, which violates the standard Gauss–Markov assumption for unbiased estimation~\cite{amemiya-1985advanced}. One could naively consider directly including the document rank parameter $r(y|\pi_q)$ as a control variable in the ranking model $g()$ to account for position bias. However, both document clicks and the document rank depend on the same underlying features $\bm{x}$. Including document rank directly in $g()$ as an additional feature would introduce statistical bias in the estimation because a confounder, the features, affects both document ranking and document clicks. Therefore, to achieve unbiased estimation, we require isolating the parts of document rank that are orthogonal to the features. In the following section, we model the system's previous ranking process to extract the parts of its exogenous variation not explained by the features, in order to account for position bias in the click equation~\eqref{eq:biased_c}.
%Let us consider a deterministic ranking policy $\pi$, and let $r(y|\pi_q)$ be the system's previous ranking or position of document $y$ in the ranked list $\pi_q$ presented for the query $q$. The likelihood of clicking a document is also influenced by the document's ranking in the query. \modifynew{Therefore, not accounting for the document rank or position ($r(y|\pi_q)$) in the click equation~\eqref{eq:biased_c}} causes its effect to be absorbed into the error term $\epsilon^{y}_{q,g}$. \katya{That is, even after conditioning on covariates $\bm{x}$ the residual error in the outcome model still has a systematic process that is driven by selection and $E(\epsilon^{y}_{q,g} \mid \bm{x}) \neq 0$, which violates standard Gauss–Markov assumptions~\cite{amemiya-1985advanced}. This systematic component exists precisely because the shock that drives selection (the error in the selection equation) is correlated with the shock that drives the outcome (the error in the outcome equation).}
% As a result, $\epsilon^{y}_{q,g}$ becomes correlated with both the document click variable $C^{y}_{q}$ and the features $\bm{x}$ \modifynew{due to document rank or position, such that} $E(\epsilon^{y}_{q,g} \mid \bm{x}) \neq 0$.
% This violates the Gauss–Markov assumption, which requires zero correlation between independent variables and the error term~\cite{amemiya-1985advanced}, resulting in biased estimates of $g()$.
% Document ranking process is the source of position bias because the current generation of biased clicks results from this ranking process, and accounting for the variations created by the document's previous ranking $r(y|\pi_q)$ will account for this bias.
% The document ranking process that generated the ranks ($r(y|\pi_q)$) is the source of position bias because the current generation of biased clicks results from this process.
% Document ranking process is the source of position bias because the current generation of biased clicks results from this ranking process, and accounting for the variations created by the document's previous ranking $r(y|\pi_q)$ will account for this bias. In the following section, we formulate a control function to model the system's previous ranking $r(y|\pi_q)$.
% We can now rewrite the click equation ~\eqref{eq:biased_c} including the system's previous document ranking as:
% \begin{equation} \label{eq:unbiased_c}
% C^{y}_{q} = g(\bm{x}, r(y|\pi_q)) + \varepsilon^{y}_{q,g}.
% \end{equation}
% Equation \eqref{eq:unbiased_c} now includes $r(y|\pi_q)$ in the ranker $g()$, thereby accounting for the system's previous ranking process.
% From the control function, we estimate the part of $r(y|\pi_q)$ that is absorbed into the error term $\varepsilon^{y}_{q,g}$ in click equation ~\eqref{eq:biased_c} to account for the \modify{bias}.
% \subsection{Partial-information Setting}
% When the true relevance of each query-document pair is available, the phenomenon is referred to as a full-information setting. In a full-information setting, the assumption is that the true relevance of every query-document pair is available, typically obtained through human-annotated labels. However, generating such labels for all possible query-document pairs is highly impractical and expensive, especially given that modern recommender systems usually contain millions of documents. Therefore, LTR systems generally avoid reliance on human-annotated labels for training. Instead, they use implicit user feedback data, such as click data, which is more readily obtainable and aligned with user intents ~\cite{joachims-kdd02}. Training an LTR system using implicit feedback data is considered a partial-information setting. Since we use a partial-information setting using historical click data as a proxy for true relevance during training, our method belongs to the category of counterfactual learning to rank (CLTR).
% Features $\bm{x}$ impact a document's ranking $r(y|\pi_q)$ in a query because the ranking of a document in a query depends on its relevance to the query, which, in turn, depends on the query-document features. Moreover, ranker $g()$ also depends on features $\bm{x}$. Hence, omitting $r(y|\pi_q)$ in learning $g()$ results in the residual $\epsilon^{y}_{q,g}$ capturing part of the effect of $r(y|\pi_q)$. As a result $E(\epsilon^{y}_{q,g}, \bm{x}) \neq 0$, which results in a statistically biased estimation of $g()$ and suboptimal predictions of document click probabilities.
\section{Position Bias Correction Using Control Function}
We now introduce our framework for the two-stage Control Function-based Correction (CFC) of position bias in LTR systems.
In the first stage, we model the system’s previous ranking to recover residuals that capture exogenous variation in the unobserved determinants of ranking. In the second stage, we include those residuals as a control function to correct for position bias.
% \modifynew{In the first stage, we model the system's previous ranking process to estimate its exogenous variation, which is then accounted for in the second stage to correct for position bias in the ranking model.}
\subsection{Control function for document ranking} \label{control_function_section}
In the first stage, we model the previous ranking process that generated the training data, where users were more likely to click on higher-ranked documents, leading to position bias in the clicks. Whether a document is relevant to a query depends on the features of the query, document, and context. These features are the observable attributes that determine the document's position in the ranked list for that query. Therefore, we can model the previous document rankings as follows:
\begin{equation} \label{eq:control_function}
r(y|\pi_q) = m(\bm{x}) + \epsilon^{y}_{q,m}.
\end{equation}
% The previous ranking process that generated the rankings ($r(y|\pi_q)$) affects the current document clicks in equation~\eqref{eq:biased_c}. Therefore, the unobserved factors contributing to position bias that influence document rankings also affect the current clicks in equation~\eqref{eq:biased_c}. This results in a correlation between the error terms of the ranking process ($\epsilon^{y}_{q,m}$) and that of the click equation ($\epsilon^{y}_{q,m}$), such that $E[\epsilon^{y}_{q,m},\epsilon^{y}_{q,g}] \neq 0$. Without accounting for this correlation, the estimation of the ranking model using equation~\eqref{eq:biased_c} will be biased.
Here, $m()$ is the ranking function that captures the previous ranking process in terms of document position, which depends on the features $\bm{x}$, and $\epsilon^{y}_{q,m}$, the error term for document $y$ associated with query $q$. %Importantly, $m()$ can be learned using any supervised machine learning algorithm, with features as inputs and document ranks as target outputs.
An advantage of a nonparametric control function approach is the use of any linear or non-linear model in the first stage~\citep{blundell-res04, das-res03}.
% While the Heckman method~\cite{heckman-econometrica79} models selection probability in the first stage, it relies on OLS or probit regression, assuming a linear relationship between features and rank. However, ranking is rarely strictly linear, and OLS or probit struggles with multicollinearity in high-dimensional feature spaces, leading to \katya{noisy parameter estimation. In contrast,
%\katya{A nonparametric control function method does not require the joint error distribution and reduced form regression functions to be parametrically specified. As such, we can apply nonparametric linear or non-linear model in this first stage to estimate $m()$. For example, Blundell et al.~\cite{blundell-res04} use a Nadaraya-Watson kernel regression in the first stage.}
This flexibility also enables us to include model selection via regularization, which leads to more stable parameter estimation.
%\footnote{\katya{Regularization will introduce an estimation bias into the estimators, however, the non parametric control function approach does not depend on unbiased estimation of the covariate effects in the first stage, nor parametric assumptions on the residuals.}} Furthermore, with a non-linear and nonparametric first stage, the residuals are no longer assumed to be normal nor parametric \citep{blundell-res04, das-res03}.}
If the previous ranker that generated the rankings in the training data is known, a similar model could be used to model $m()$.
%\footnote{\katya{Note, however, that in canonical LTR algorithms $m()$ typically models clicks, however, here it models the rank of a document.}}
%However, one has to keep in mind that unlike typical LTR algorithms which model a document's relevance, $m()$ models the exact rank position of a document in a ranking, so they have slightly different objective functions.
% Our objective is to estimate the residuals of the system's previous document ranking model $m()$.
%\katya{As previously noted, in our framework, residual variation from the historical ranking process directly influences user clicks, creating correlation between the ranking error \(\epsilon^{y}_{q,m}\) and the click error \(\epsilon^{y}_{q,g}\). As a result, estimators in the click equation~\eqref{eq:biased_c} become biased, which—as we will demonstrate—undermines out‐of‐sample predictive performance.}
% However, the previous ranking process that generated the rankings ($r(y|\pi_q)$) affects the current document clicks in equation~\eqref{eq:biased_c}. Therefore, the document click and its position are correlated, such that \katya{$Cov[\epsilon^{y}_{q,m},\epsilon^{y}_{q,g}] = E[\epsilon^{y}_{q,m},\epsilon^{y}_{q,g}] \neq 0$.} Without correcting for this correlation, the estimators, in expectation, of the ranking model using equation~\eqref{eq:biased_c} will be biased.}
To disentangle the endogeneity of previous rankings from that of clicks, we exploit heteroskedastic variation in the residuals of the first‐stage ranking model \(m()\) \cite{lewbel-jbec12}. This gives us internally generated exogenous variation from the ranking process, which is orthogonal to the observable features and to clicks. %This gives us internally generated variation from the ranking process, which is orthogonal to the observable features and to clicks, that exogenously drives observed rankings \(r(y\mid \pi_q)\).
% To remove this correlation, we use the residual variation from the previous ranking process $m()$, which influences both the previous rankings ($r(y|\pi_q)$) and the current clicks in equation~\eqref{eq:biased_c}.}
% \katya{ Importantly, $Cov[\epsilon^{y}_{q,m},\epsilon^{y}_{q,g}] = E[\epsilon^{y}_{q,m},\epsilon^{y}_{q,g}] \neq 0$. That is, the document click and its position are correlated and not accounting for this wil introduce a ranking bias in the first stage.}
% To remove this correlation, we use the residual variation from the ranking process $m()$ that affects the previous rankings and current clicks in equation~\eqref{eq:biased_c}. By incorporating the residual variation from the ranking process—capturing unobserved factors that influence both document position and clicks—we break this correlation between the error terms. This enables unbiased estimation in the second-stage ranking model.
Let $\hat{\epsilon}^{y}_{q,m} = r(y|\pi_q) - \hat{r}(y|\pi_q)$ be the estimated residual for document $y$ associated with query $q$ %We can estimate $\hat{\epsilon}^{y}_{q,m}$ as:
%\begin{equation} \label{eq:reqiduals}
% \hat{\epsilon}^{y}_{q,m} = r(y|\pi_q) - \hat{r}(y|\pi_q),
%\end{equation}
where $\hat{r}(y|\pi_q)$ represents the predicted ranking from $m()$ for document $y$, given that users are shown the ranked list $\pi_q$ for query $q$. Since $m()$ captures the effect of features on ranking, the residual $\hat{\epsilon}^{y}_{q,m}$ is orthogonal to these feature effects. Therefore, $\hat{\epsilon}^{y}_{q,m}$ represents unobserved factors beyond the features that influence the placement of document $y$ in the particular position $r(y|\pi_q)$ within the ranked list $\pi_q$.
Examples of such factors include system constraints, biases in prior data, limited training data, or external conditions that are not captured by observable features. These unobserved factors can push more relevant documents to lower positions or less relevant ones to higher positions, which in turn affects their position and likelihood of being clicked.
%Factors not explained by the features represent parts of the previous ranking process unrelated to relevance, \modifynew{which can place more relevant documents in lower positions or less relevant ones in higher positions, introducing position bias in click data}. For example, the previous ranking process may not be fully deterministic, as unobserved factors can still influence document positions, including system constraints, biases in prior data, limited training data, or external conditions, which the residuals are intended to capture. The larger the absolute value of a residual, the greater the influence of unobserved factors on the previous ranking process. A positive residual indicates that unobserved factors beyond the features placed the document above its feature-predicted rank, while a negative residual indicates it was placed below that rank. A zero residual means the document’s position is fully explained by the features. \katya{These unobserved factors can influence previous rankings, which can include a system promoting items because they had been previously clicked on, or appeared more frequently across several queries. This, in turn, affects current document clicks and, if not accounted for in the ranking model, can bias the set of items shown in future recommendations.}
Following a control function approach, we will use the estimated residuals ($\hat{\epsilon}^{y}_{q,m}$) as a ``control function'' to address the latter bias in ranking each current query-document pair.
% \textit{, controlling for any possible confounding \modify{or endogeneity} from the previous ranking process that would otherwise bias the current outcome click equation~\eqref{eq:biased_c}.
\subsection{Position bias correction in LTR model}\label{subsection_unbiased_ltr}
In this second stage, we train the ranking model incorporating the estimated residuals from the first stage. The ranking model can be written as:
\begin{equation} \label{eq:unbiased_c_temp}
C^{y}_{q} = g(\bm{x}, \hat{\epsilon}^{y}_{q,m}) + \epsilon^{y}_{q,g},
\end{equation}
where $\hat{\epsilon}^{y}_{q,m}$ is an additional variable, serving as the control function, used alongside the features.
% \modify{Note that our goal is not to account for document position in the ranking model, but to account for any confounding factors or endogenous components that affect the previous ranking process and, in turn, the current clicks, which the residuals aim to capture. This is why we use the control function approach and incorporate residuals to correct for position bias.}
% As previously mentioned, we cannot directly include document position or rank ($r(y|\pi_q)$) in the ranking model because both document rank and clicks depend on the same features $\bm{x}$. Instead,
This accounts for any exogenous variation that influences the previous ranking process beyond the features and, in turn, affects current clicks, which the residuals capture. %This is why we adopt a control function approach to isolate the exogenous variation of the previous document ranking process that contributes to position bias and is correlated with the click outcome, and to account for it in the ranking model to correct for position bias. This also breaks the correlation between the error terms of the previous ranking process and the current click equation.}
% While using the residuals in this manner makes sense from a control function perspective, it does not account for potential normalization issues and additional non-linearities introduced by position bias. \modifynew{Blundell et al.~\cite{blundell-res04} address the problem of endogeneity or confounding in binary response models using control function methods, where they estimate some non-linear functions (e.g., kernel function) to the first-stage residuals.
%\katya{In the ranking model literature there are different approaches for modeling the residuals, from including them directly in the second stage to including their transforms as in~\cite{lee-econometrica83} who use a generalized hazard ratio tailored to the error distribution, \modifynew{or applying some non-linear functions (e.g., kernel function) as in~\cite{blundell-res04}}. Similar to ~\cite{lee-econometrica83}, we also use a generalized hazard ratio tailored to the error distribution of the first-stage residuals. \modifynew{Additionally, we apply a parametric hazard ratio and a parametric density function for transforming the residuals, as well as a normalization method.}}
% Therefore, before using the estimated residuals as additional feature in the ranking model in equation~\eqref{eq:unbiased_c_temp}, we apply different functions to the residuals, which we explain next.
There are different approaches for modeling the residuals. Most studies directly include them in the second stage %, but there is some evidence of transforming residuals as in \cite{rivers-econometrica88}, who develop a scored-based residual by converting the first stage residuals into a likelihood score, or more flexible approximations of the residual using polynomials and splines as in \cite{newey-je97}.
but a few also show the value of using residual transformations, such as %a generalized hazard ratio as in~\cite{lee-econometrica83},
converting the first stage residuals into a likelihood score~\cite{rivers-econometrica88} and estimating a kernel function of the residuals~\cite{blundell-res04}.
%and a non-linear function~\cite{blundell-res04}.
%series expansions of the residuals (Newwy 1997).
Instead of deciding on a specific transformation form, we introduce residual transformation as a hyperparameter in the learning of the second stage function $g()$ which has not been considered in previous work. This allows us to optimize the transformation for ranking performance using validation data.
We propose four carefully designed transformations of the residuals, each capturing a unique aspect of position bias correction, but our framework can easily incorporate others:
\begin{enumerate}[leftmargin=12pt, nosep]
\item \textbf{Min-Max normalization:} We apply min-max normalization to the residuals ($\hat{\epsilon}^{y}_{q,m}$), and use the normalized values in the ranking model (equation~\eqref{eq:unbiased_c_temp}) instead of the raw residuals. This aligns with typical feature normalization and avoids disproportionate impact from the residuals.
% \modifynew{Note that this transformation, in fact, uses the residuals directly but with a normalization.}
\item \label{PDF-Normal} \textbf{PDF transformation (assuming normal
distribution):} We estimate the probability density function (PDF) of the residual ($\hat{\epsilon}^{y}_{q,m}$) for each query-document pair and use the PDF in the ranking model (equation~\eqref{eq:unbiased_c_temp}) instead of using the residuals directly. The PDF captures how typical or unusual a residual is for a query-document pair relative to the overall residual distribution. In contrast, residuals are absolute measures and do not capture the distributional information of unobserved factors. Therefore, instead of directly using the residuals, the PDF of the residuals can be used as an additional feature. This can improve the consistency of the ranker’s estimation by leveraging distributional information that is not captured by the residuals.
\item \label{IMR-normal} \textbf{Hazard ratio
transformation (assuming normal distribution):} This transformation under the normality assumption is also known as inverse Mills ratio (IMR). We transform the residual ($\hat{\epsilon}^{y}_{q,m}$) for each query-document pair into a hazard ratio, assuming normally distributed residuals. The hazard ratio is the ratio of the PDF to the cumulative distribution function (CDF). Let $\psi^{y}_{q}$ be the hazard ratio of the estimated residual $\hat{\epsilon}^{y}_{q, m}$ for document $y$ associated with query $q$, then the hazard function is defined as:
\begin{equation} \label{eq:IMR}
\psi^{y}_{q} = \frac{f(\hat{\epsilon}^{y}_{q, m})} {F(\hat{\epsilon}^{y}_{q, m})},
\end{equation}
where $f()$ and $F()$ are PDF and CDF, respectively. We use the estimated hazard ratio or IMR in the ranking model (equation~\eqref{eq:unbiased_c_temp}) instead of using the residuals directly. We choose IMR because it is typically used in regression models to address non-random missing data.
% When data is missing not at random (MNAR), the observed sample no longer reflects the true data distribution, leading to bias in data.
Documents at lower ranks are less likely to be observed and clicked, even if relevant, leading to missing relevance feedback. This missingness is not random because it depends on the position of the document, which, in turn, is influenced by previous ranking process. Consequently, the observed clicks form a biased and non-random sample, introducing position bias in the click data. The IMR accounts for the non-random missingness of documents by modeling the probability that a document is selected into a sample based on the underlying distribution of residuals. IMR has also been used in Heckman-based methods~\cite{ovaisi-www20, ovaisi-sigir21} to model the document selection probability for correcting selection bias, while we apply the IMR of the estimated residuals from document ranking process to correct for position bias. We provide a discussion of the differences between our method and Heckman-based methods in
Appendix~\ref{sec:heckman_diff}. The difference between the PDF transformation and this transformation is that the latter is scaled by the CDF.
\item \textbf{Hazard ratio transformation:} Instead of assuming normality as in the PDF and IMR transformations, we use a more general nonparametric Kernel Density Estimation (KDE) to estimate the PDF ($f(\hat{\epsilon}^{y}_{q, m})$) of each residual ($\hat{\epsilon}^{y}_{q,m}$).
% For example, we can use fastKDE~\cite{o-csda16} to estimate the PDF.
The estimated PDF of the residual is used to calculate the hazard
ratio using equation~\eqref{eq:IMR} by scaling with the CDF. We then use the estimated hazard ratio in the ranking model (equation~\eqref{eq:unbiased_c_temp}) instead of using the residuals directly. Note that, nonparametric control function
methods~\cite{blundell-res04, das-res03} do not require parametric assumptions about the distribution of residuals. %Following this, we also apply nonparametric distributions of the residuals of the document ranking process.
\end{enumerate}
% \modifynew{While we use the four mentioned transformations, future work could explore alternative parametric forms (e.g., log-normal, Weibull) or more advanced non-parametric methods with bandwidth tuning.}
Let $T(\hat{\epsilon}^{y}_{q,m}) \in \mathcal{T}$ be any transformation of the estimated residuals, with $\mathcal{T}$ being the set of all transformations. Now, we can rewrite the final ranking model from equation~\eqref{eq:unbiased_c_temp} as:
\begin{equation} \label{eq:unbiased_c_final}
C^{y}_{q} = g(\bm{x}, T(\hat{\epsilon}^{y}_{q,m})) + \epsilon^{y}_{q,g}.
\end{equation}
We can train the ranker $g()$ with an LTR algorithm with features and transformed residuals as inputs and document clicks as target outputs. One of the main advantages of our approach is that $g()$ can be any linear or non-linear pointwise, pairwise, or listwise ranking model, making it broadly applicable as bias correction with any state-of-the-art LTR method.
%We consider the residual transformation function as a hyperparameter and optimize it for ranking performance using validation data for the given ranking model $g()$.
% Note that we do not make any specific assumptions about the residual distribution. Instead, we transform the residuals using various parametric, nonparametric, and normalization techniques to automatically select the best one based on the residual distribution.
% That is, we tune the hyperparameters to find the optimal transformation $T^*$ automatically that provides the minimum ranking loss for $g()$ on the validation set:
% \begin{equation}
% T^* = \arg \min_{T \in \mathcal{T}} \mathcal{\hat {\bm{L}}} \left(g(x, T(\hat{\epsilon}^{y}_{q,m})); \text{validation data}\right).
% \end{equation}
After learning the optimal ranking model $g()$, we can predict click probabilities for all documents with each new query and rank them accordingly.
% Since the residual of the control function captures unobserved factors that influenced both the previous ranking process and, in turn, the current clicks, weighting each document in the ranking model using the transformed residual from the previous ranking process helps adjust the model by correcting non-random, position-driven current click patterns that stem from the previous ranking process.
% The probabilities predicted by $g()$ are free from the effect of position bias, as we account for the bias through the estimated residuals during the learning of $g()$.
Since the unbiased ranking model $g()$ estimates the unbiased feature effects on clicks, the transformed residual $T(\hat{\epsilon}^{y}_{q,m})$ for each query-document pair is no longer required during inference. As a result, we can predict unbiased click probabilities solely based on features and set $T(\hat{\epsilon}^{y}_{q,m})$ to zero during inference.
We provide the CFC algorithm in Appendix~\ref{cfc_algorithm}, along with its computational complexity, data requirements, and differences from click model-based methods in Appendix~\ref{computational_complexity},\ref{sec:data_requirements}, and~\ref{sec:click_model_diff}, respectively.
% Algorithm~\ref{alg:cfc_bias_correction} shows the procedure of our CFC method.
% For example, the Ridge Classifier for the control function takes $O(n d^2)$ time ~\cite{zhang-mls15}, where the total number of documents is $n$, and $d$ is the feature dimensions.
\subsection{Debiasing clicks for hyperparameter tuning} \label{sec:debiased_clicks}
In typical machine learning, train, validation and test data are assumed to come from the same distribution (i.i.d. assumption)~\cite{wang-www24, zhao-www25}. In unbiased LTR, however, training uses biased click data, while proper evaluation relies on unbiased test data. To tune hyperparameters for an unbiased ranking model, it is ideal to use unbiased ground truth data for validation. However, such data is rarely available in real-world scenarios and when the validation set is sampled from the training data, as is typically the case, hyperparameter tuning relies on biased click data, which is suboptimal. We would like to point out that many existing works on unbiased LTR~\cite{ai-sigir18, joachims-wsdm17, luo-sigir24, luo-wsdm23, oosterhuis-sigir20, wang-wsdm18} overlook the issue of appropriate validation in the absence of unbiased data, likely because popular benchmark datasets include unbiased validation data. To address the problem of hyperparameter tuning when the true relevance of documents to a query is unavailable for validation, which is likely to occur in real-world LTR applications, we propose a method for debiasing click data.
%In typical machine learning, train, validation and test data are assumed to come from the same distribution (i.i.d. assumption)~\cite{wang-www24, zhao-www25}. In ULTR, however, training relies on biased click data corrected by a debiasing method, while evaluation uses unbiased test data. If validation is based on biased clicks, a distribution shift arises between validation and testing, leading to potentially suboptimal hyperparameter tuning. Therefore, to tune hyperparameters for an unbiased ranking model, it is ideal to use unbiased ground truth or true relevance for validation rather than the biased click data used in training. However, when the validation set is sampled from the training data, as is typically the case, true relevance is unavailable. In such cases, hyperparameter tuning relies on biased click data, which is suboptimal. We would like to point out that many existing works on unbiased learning to rank (ULTR)~\cite{ai-sigir18, joachims-wsdm17, luo-sigir24, luo-wsdm23, oosterhuis-sigir20, wang-wsdm18} overlook the issue of appropriate validation in the absence of unbiased data, where a distributional discrepancy exists between the biased validation data and the unbiased test data. To circumvent this problem, we propose using debiased validation click data as a proxy for true relevance during hyperparameter tuning. Next, we describe our method for debiasing click data, which we use for hyperparameter tuning when the true relevance of documents is unavailable for validation.
In equation~\eqref{eq:unbiased_c_final}, we model the click data as dependent on the features $\bm{x}$ and the corresponding estimated transformed residual $T(\hat{\epsilon}^{y}_{q,m})$. $T(\hat{\epsilon}^{y}_{q,m})$ accounts for the position bias of document $y$ associated with query $q$. To debias the clicks from the position bias effect, we need to partial out the effect of $T(\hat{\epsilon}^{y}_{q,m})$ from clicks. To achieve this, we consider $T(\hat{\epsilon}^{y}_{q,m})$ as the only independent variable affecting clicks, and learn a function $f()$ as follows:
\begin{equation} \label{eq:valid_clicks}
C^{y}_{q} = f(T(\hat{\epsilon}^{y}_{q,m})) + \epsilon^{y}_{q,f},
\end{equation}
where $\epsilon^{y}_{q,f}$ is the error term for document $y$ associated with query $q$ for the function $f()$. Therefore, the effect of the features on a click is absorbed in the error term $\epsilon^{y}_{q,f}$. Using the predicted clicks from the function $f()$, we calculate the estimated residual $\hat{\epsilon}^{y}_{q,f}= C^{y}_{q} - \hat{C}^{y}_{q}$ for each document $y$ associated with query $q$ in the validation dataset %as:
%\begin{equation} \label{eq:unbiased_clicks}
% \hat{\epsilon}^{y}_{q,f} = C^{y}_{q} - \hat{C}^{y}_{q},
%\end{equation}
where $\hat{C}^{y}_{q}$ is the predicted value from $f()$ for document $y$ associated with query $q$. The residual $\hat{\epsilon}^{y}_{q,f}$ represents the parts of a click that is separated from the position bias effect, capturing only unbiased features' effect on a click. Thus, when validation data with true relevance is not available, we can use $\hat{\epsilon}^{y}_{q,f}$ as a proxy for true relevance during validation.
\subsection{Unbiasedness of the CFC method} \label{sec:unbiasedness}
%\katya{As mentioned above, a document's click depends on its position, such that $Cov[\epsilon^{y}_{q,m},\epsilon^{y}_{q,g}] \neq 0 $. Not accounting for document ranking $r(y|\pi_q)$ in equation \eqref{eq:biased_c} results in a biased estimation of ranker $g()$, as higher‑ranked documents receive disproportionately more exposure and clicks—causing $g()$ to conflate position‑induced variation with true relevance.}
In selection‐type control‐function models, to statistically identify the parameters of the outcome (click) equation in the presence of confounding, one typically needs a variable (an instrument) that exogenously affects the endogenous first stage (ranking) process but does not directly affect the outcome (clicks). Then, by conditioning on this variable in the first stage, one can isolate and estimate the part of the residual that is correlated with the outcome residual, leaving only the exogenous variation in the ranking variable. We then control for this estimated residual in the second stage. This breaks the covariance of the errors between the first and second stage equations: $E[\epsilon^{y}_{q,g}, \epsilon^{y}_{q,m}] = 0$.
%\begin{equation*}
% E[\epsilon^{y}_{q,g}, \epsilon^{y}_{q,m}] = 0.
%\end{equation*}
However, when no external exclusion restrictions are available, one can follow Lewbel et al.~\cite{lewbel-jbec12} who exploit heteroskedasticity in the error terms when applicable—i.e. if the residuals' variance from the rank equation (equation~\eqref{eq:control_function}) varies systematically with the features—one can use this as an internal source of exogenous variation in the ranking equation.
%THIS COULD BE WHERE WE GO INTO THE USE OF HETERSKADASTIC ERRORS THEORETICALLY
We find that heteroskedasticity in the first‐stage residuals is true in common LTR benchmark datasets (see Subsection~\ref{sec:heteroskedasticity} in Appendix), and we exploit this residual variation as our source of exogenous shocks in the ranking error. In particular, Lewbel et al.~\cite{lewbel-jbec12} show that interacting each demeaned covariate with the ranking residual produces valid control‐function variables: they are correlated with the endogenous ranking process but orthogonal to the click‐error~\cite{lewbel-jbec12}. %Intuitively, these interactions capture heteroskedastic shocks whose magnitude varies with feature levels, while any constant, endogenous correlation with the outcome residual washes out because each covariate is mean‐zero after demeaning. However, implementing the full set of covariate–residual interactions in the click equation would double our feature dimensionality and become impractical.
Since our primary goal is out‑of‑sample predictive accuracy rather than formally unbiased parameter estimates, we instead include only the estimated first‐stage residual as a single control function. This sacrifices some theoretical unbiasedness but still injects the crucial exogenous variation needed to improve predictive performance.
%\katya{Concretely, we first regress the observed document ranking on the feature vector $\bm{x}$ to recover residuals whose conditional variance depends on $\bm{x}$. We then include a suitable transformation of those heteroskedastic residuals as our control function in the outcome click equation. By conditioning on both the features and this control function, the remaining click‐error has zero expectation, ensuring unbiased estimation of the feature effects in the final ranking model $g()$ in equation~\eqref{eq:unbiased_c_final}.}
% \subsection{Discussions} \label{sec:discussions}
% \modifynew{The CFC method corrects for bias at the feature level without modifying the ranker’s loss function, which can enable its integration into other bias correction frameworks for joint debiasing. For example, it can be extended to jointly address position bias and selection bias, which we will explore as a future research direction. A limitation of our method is that we do not have an exclusion restriction or an external instrument in the first stage to satisfy the standard control function identification strategy. Instead, as described previously, we follow Lewbel et al.~\cite{lewbel-jbec12} and exploit heteroskedasticity in the residuals of the ranking process as a source of exogenous variation in the first stage. While for the residual transformation in the first stage we assume normally distributed residuals for two parametric transformations and apply one
% nonparametric transformation, future work can also explore other parametric forms and more advanced nonparametric or kernel-based transformation techniques.}
% \modifynew{Bias can arise from historical data that reflect societal prejudices~\cite{wu-tkdd23}, and training recommender systems on such data may propagate unfair treatment of underrepresented users or content~\cite{bojic-futures24}. Therefore, mitigating these biases is essential for promoting fairness and preventing marginalization. Bias correction strategies can help ensure equitable treatment and prevent the marginalization of underrepresented users and content~\cite{bojic-futures24}. However, for practitioners, it is important that these systems are used in a way that benefits society, respecting fairness and privacy~\cite{fan-sigkdd23}.}
% \modifynew{Bias can emerge from historical data, which may inherently reflect pre-existing societal prejudices~\cite{wu-tkdd23}. When such data are used to train recommender systems, these biases can be propagated, potentially leading to the unfair treatment of underrepresented users or contents~\cite{bojic-futures24}. Therefore, mitigating bias in such systems plays a vital role in promoting fairness and reducing the risk of reinforcing unfairness. By addressing biases that originate from historical data and user interactions, such bias mitigation strategy can help ensure equitable treatment and prevent the marginalization of underrepresented users and content.~\cite{bojic-futures24}. However, for practitioners, it is important that these systems are used in a way that benefits society, respecting fairness and privacy~\cite{fan-sigkdd23}.}
% However, in LTR systems, our primary goal is not to recover the structural parameters but to improve predictive ranking performance.
% While we provide an identification argument that ensures unbiased estimation of feature effects under control function assumptions, our primary objective is not the recovery of structural parameters but accurate prediction of relevance. Therefore, the emphasis is on improving ranking performance rather than ensuring theoretical unbiasedness of estimators.
% While we provide an identification argument that supports unbiased estimation of feature effects under control function assumptions, our primary goal in LTR systems is not to recover structural parameters but to improve predictive ranking performance. Therefore, the emphasis is on achieving accurate relevance prediction rather than ensuring theoretical unbiasedness of estimators.
% While we do not have an exclusion restriction or an external instrument in the first stage to satisfy the standard control function identification strategy, we instead exploit heteroskedasticity in the residuals of the ranking process as a source of exogenous variation. Our goal is not to recover structural parameters or guarantee theoretical unbiasedness of the estimators, but rather to improve predictive accuracy in document ranking.
% \modifynew{Note that an assumption in our method is the presence of heteroskedasticity in the residuals of the first-stage ranking process, which allows the identification of exogenous variation without requiring external instruments or exclusion restrictions. While this assumption underlies our identification strategy in the first stage, it may not be a restrictive limitation in practice. As described earlier, there can be many unobserved factors influencing the previous ranking process apart from the features, which can introduce variation in the residuals of this process.}
% as the expectation of the error term is not zero.
% \begin{equation*}
% E(\epsilon^{y}_{q,g}| \bm{x}) \neq 0.
% \end{equation*}}
% \katya{Delete the rest}
% To achieve an unbiased estimation of feature parameters within a general control function approach, two key assumptions should be satisfied. First, the explanatory variables should be correlated with the endogenous variable.
% % \modify{The document ranking variable $r(y|\pi_q)$ is endogenous because it is influenced by features $\bm{x}$, which also affect the ranking model $g()$.}
% Explanatory variables (features $\bm{x}$) are correlated with the endogenous variable $r(y|\pi_q)$ in the control function in equation~\eqref{eq:control_function} \modify{because features $\bm{x}$ influence document ranking variable $r(y|\pi_q)$}. Second, the explanatory variables should be uncorrelated with the error term of the outcome equation. \modify{Since, by construction, residual $\hat{\epsilon}^{y}_{q, m}$ is orthogonal to features $\bm{x}$, the transformed residual $T(\hat{\epsilon}^{y}_{q,m})$, derived from the residual, is also orthogonal to features. Our outcome equation for clicks (equation~\eqref{eq:unbiased_c_final}) controls for the features and the transformed residual. By controlling for the transformed residual into the ranking model, we account for the \modifynew{exogenous variation} of the document ranking process that would otherwise be absorbed into the error term $\epsilon^{y}_{q,g}$. As a result, by accounting for the variation of the document ranking process, the expectation of the error term in the click equation will be zero, and features $\bm{x}$ become uncorrelated with the error term $\epsilon^{y}_{q,g}$, ensuring an unbiased estimation of $g()$.
% \begin{equation*}
% E[\epsilon^{y}_{q,g} | x, T(\hat{\epsilon}^{y}_{q,m})] = 0.
% \end{equation*}
% }
% % Satisfying these two assumptions, we are able to estimate unbiased feature effects of the ranker $g()$ using click equation~\eqref{eq:unbiased_c_final}.
% \modifynew{General control function methods typically require an exclusion restriction for identification~\cite{wooldridge-jhr15}, where the first-stage document ranking process (equation~\eqref{eq:control_function}) should include at least one variable that influences the previous ranking process but does not directly affect the outcome click equation~\eqref{eq:unbiased_c_final} in the second stage. However, Lewbel et al.~\cite{lewbel-jbec12} show that heteroskedasticity— the variance of the residuals is not constant across observations ($\text{Var}(\hat{\epsilon}^{y}_{q,m} | \bm{x}) = \sigma^2(\bm{x})$)—can be used as a source of identification, when exclusion restriction is not satisfied. Following this, identification is possible through the variance of the residuals under the assumption of heteroskedasticity. In our case, we follow this principle by using the residuals from the document ranking process, assuming heteroskedasticity in these residuals, which enables the estimation of an unbiased ranking model without satisfying the exclusion restriction.}
% \modify{One might ask why we do not directly include the document rank parameter $r(y|\pi_q)$ as a control variable in the ranking model instead of computing residuals. The reason is that both document clicks and the document ranking process depend on the same underlying features. Including document ranking directly in the click outcome would introduce statistical bias in estimation. This bias occurs because a confounder, the features themselves, influences both document ranking and document clicks. The control function addresses this by modeling the document ranking process and isolating the parts of that process not explained by the features. This isolated endogenous component is captured by the residual of the control function. By controlling for this, we account for position bias stemming from the previous document ranking process while removing the effects of shared features.}
% \modifynew{Our control function approach does not rely on exclusion restrictions, which are often difficult to justify in real-world ranking systems. Instead, identification is achieved by leveraging heteroskedasticity in the residuals of the prior ranking model, following the framework of Lewbel et al.~\cite{lewbel-jbec12}.}
% \modifynew{Blundell et al.~\cite{blundell-res04} and Das et al.~\cite{das-res03} demonstrate that control function methods can be implemented in a nonparametric framework without requiring specific distributional assumptions on the error terms, unlike the Heckman model. Building on this insight, our control function approach for correcting position bias similarly can allow for flexible modeling of residuals terms without assuming a specific distributional form.}
% \modifynew{Blundell et al.~\cite{blundell-res04} demonstrate that control function methods in binary response models can flexibly address endogeneity by estimating a nonparametric function of first-stage residuals, rather than assuming a specific parametric form. Inspired by this, our method incorporates nonlinear transformations of residuals, such as the inverse Mills ratio and standardized PDF, as correction terms in the ranking model. This allows us to adjust for position bias without imposing strong distributional assumptions, following the same semiparametric principle.}
% without directly incorporating document ranking into the final ranking model $g()$.}
% The control function approach corrects this by first modeling the ranking mechanism through a function $m(x)$, which predicts the document’s position based on observed features $\bm{x}$. The residuals from this model capture the unobserved components influencing position—such as latent popularity or interface bias—that are not accounted for by $\bm{x}$. In the second stage, these residuals (or their transformations) are included as additional inputs to the click prediction model. By conditioning on these residuals, the approach removes the correlation between the features and the error term in the click model, satisfying the conditions for consistent estimation of relevance.
% First of all, the explanatory variables need to be correlated with the endogenous variable. In our case, in the control function, as outlined in equation~\eqref{eq:control_function}, the explanatory variables $\bm{x}$ are correlated with the endogenous variable $r(y|\pi_q)$. Second, the endogenous variable should not be correlated with the error term of the outcome equation.
% % In our case, the estimated residuals ($\hat{\epsilon}^{y}_{q,m}$) capture the unobserved factors beyond features that is absorbed in the error term $\epsilon^{y}_{q,g}$ in equation ~\eqref{eq:biased_c}.
% % In our case, in equation ~\eqref{eq:biased_c}, the residual $\epsilon^{y}_{q,g}$ captures part of the effect of $r(y|\pi_q)$ that is orthogonal to features $\bm{x}$. The estimated residuals ($\hat{\epsilon}^{y}_{q,m}$) from the control function capture these unobserved factors beyond the features.
% In our case, we control for the features in the outcome equation ~\eqref{eq:unbiased_c_final} for clicks by residualizing the outcome of features’ effects. This ensures that explanatory variables $\bm{x}$ are uncorrelated with the error term $\epsilon^{y}_{q,g}$ in the outcome click equation~\eqref{eq:unbiased_c_final}.
% % We control for these estimated residuals ($\hat{\epsilon}^{y}_{q,m}$) in our outcome click equation ~\eqref{eq:unbiased_c_final}, which ensures that the explanatory variables $\bm{x}$ are uncorrelated with the error term $\epsilon^{y}_{q,g}$ in the outcome click equation ~\eqref{eq:unbiased_c_final}.
% Therefore, we satisfy the assumptions of the general control-function-based methods in our CFC approach for correcting position bias. This allows us to estimate the unbiased feature effects on clicks for the ranker $g()$ using click equation~\eqref{eq:unbiased_c_final}.
\section{Experiments}
% {\renewcommand{\arraystretch}{1.0}
\begin{table*} [ht]
% \setlength{\belowcaptionskip}{5pt}
\setlength{\tabcolsep}{2.0pt}
% \scriptsize
\fontsize{7.8}{8}\selectfont
\captionsetup{justification=raggedright, width=1.0\textwidth}
\caption{A comparison between CFC and other methods across various datasets using click data for hyperparameter tuning. CFC and PIJD can be implemented with both LambdaMART and DNN rankers. PairD is coupled with LambdaMART, while UPE, MULTR, PAL, REM, and DLA are implemented with DNN rankers and do not have LambdaMART implementations. * denotes a statistically significant improvement of the best-performing CFC method over the best baseline methods with ${\bm{p < 0.05}}$. The best performance among all methods, excluding Oracle, is bolded, and the second-best is underlined.}
\centering
\scalebox{1.0}{%
\begin{tabular}{cccccccccccccccccc}
\toprule
\multirow{2}{*}{Datasets} & \multirow{2}{*}{Metrics} & \multicolumn{6}{c} {LambdaMART} & \multicolumn{10}{c}{DNN} \\ \cmidrule(r){3-8} \cmidrule(r){9-18}
& & CFC-DC & CFC-BC & PIJD & PairD & LambdaMART & Oracle & CFC-DC & CFC-BC & PIJD & UPE & MULTR & PAL & REM & DLA & DNN & Oracle \\
\midrule
% \cline{1-11}
\multirow{2}[0]{*}{Yahoo}
& ERR@10 & \textbf{0.460}${^*}$ & 0.437 & 0.452 & 0.452 & 0.446 & 0.474 & \underline{0.455} & 0.444 & 0.447 & 0.437 & 0.452 & 0.441 & 0.444 & 0.433 & 0.441 & 0.464 \\
& NDCG@10 & \textbf{0.794}${^*}$ & 0.783 & 0.780 & 0.785 & 0.789 & 0.817 & \underline{0.792} & 0.784 & 0.785 & 0.772 & 0.789 & 0.782 & 0.768 & 0.777 & 0.781 & 0.805\\
\midrule
\multirow{2}[0]{*}{\makecell{MSLR-\\WEB10k}}
& ERR@10 & \textbf{0.307}${^*}$ & 0.278 & 0.275 & 0.285 & 0.268 & 0.342 & \underline{0.292} & 0.291 & 0.259 & 0.232 & 0.291 & 0.253 & 0.241 & 0.256 & 0.253 & 0.325 \\
% \cline{2-11}
& NDCG@10 & \textbf{0.482}${^*}$ & \underline{0.465} & 0.438 & 0.463 & 0.446 & 0.498 & \underline{0.465} & 0.464 & 0.448 & 0.362 & 0.451 & 0.436 & 0.419 & 0.447 & 0.446 & 0.476 \\
% \cline{1-11}
\midrule
\multirow{2}[0]{*}{Istella-S}
& ERR@10 & \textbf{0.716}${^*}$ & \underline{0.698} & 0.680 & 0.689 & 0.690 & 0.737 & 0.679 & 0.676 & 0.672 & 0.671 & 0.675 & 0.669 & 0.653 & 0.676 & 0.669 & 0.702 \\
% \cline{2-11}
& NDCG@10 & \textbf{0.765}${^*}$ & \underline{0.752} & 0.692 & 0.739 & 0.733 & 0.777 & 0.710 & 0.709 & 0.706 & 0.690 & 0.691 & 0.708 & 0.685 & 0.706 & 0.705 & 0.745 \\
\bottomrule
\end{tabular}
}
\label{tab:main_results_with_biased_clicks_short_version}
\end{table*}
% }
We aim to address the following research questions in evaluation:
\begin{itemize}[leftmargin=9pt, nosep]
\item \textbf{RQ1:} How does CFC perform when the true relevance of the validation data is unavailable?
\item \textbf{RQ2:} How does CFC perform when the true relevance of the validation data is available?
\item \textbf{RQ3:} Is CFC robust to different degrees of position bias severity?
\item \textbf{RQ4:} How does the performance of CFC change as the number of clicks increases?
\item \textbf{RQ5:} Is CFC robust to different levels of click noise?
\item \textbf{RQ6:} How does the number of queries used for hyperparameter tuning affect the performance of different ranking models?
\item \textbf{RQ7:} How does the choice of model for learning the document ranking function in the first stage (equation~\eqref{eq:control_function}) affect the performance of CFC?
\end{itemize}
\subsection{Experimental setup} \label{sec:experimental_setup}
% In this section, we describe the experimental setup used to measure the performance of our CFC method.
\textbf{Datasets and evaluation metrics.} We use three widely used benchmark datasets for LTR: Yahoo! Learning to Rank Challenge (C14B)~\cite{chapelle-pmlr11}
% \footnote{\label{yahoo_link}\url{https://webscope.sandbox.yahoo.com/catalog.php?datatype=c}}
, MSLR-WEB10k~\cite{qin-arXiv13}
% \footnote{\label{mslr_link}\url{https://www.microsoft.com/en-us/research/project/mslr}}
, and Istella-S~\cite{lucchese-sigir16}.
% \footnote{\label{istella_link}\url{http://quickrank.isti.cnr.it/istella-dataset}}
We provide detailed descriptions of the three datasets in Appendix~\ref{datasets_details}. We evaluate all methods using Expected Reciprocal Rank (ERR) and Normalized Discounted Cumulative Gain (NDCG), denoted as $ERR@p$ and $NDCG@p$, where $p$ is the rank position up to which the performance of a ranked list is evaluated.
\textbf{Click simulation.} We generate click data using a two-step process following Ai et al.\cite{ai-sigir18} and Joachims et al.\cite{joachims-wsdm17}, which is a widely adopted approach in unbiased LTR. First, we train a RankSVM~\cite{joachims-kdd06} model on $1\%$ of the training data using true relevance to produce initial ranked lists $\pi_q$ for each query $q$ over the remaining $99\%$ training data. An initial ranker is used as it outperforms random ranking while leaving room for improvement. In the second phase, we generate synthetic clicks for each ranked list $\pi_q$ using PBM~\cite{richardson-www07}, where a document is clicked if it is observed ($O^{y}_{q}$) and perceived as relevant ($R^{y}_{q}$). Hence, the click probability can be written as: $Pr(C^{y}_{q} = 1) = Pr(O^{y}_{q}=1|\pi_q) \ Pr(R^{y}_{q}=1|\pi_q)$. Following Joachims et al.~\cite{joachims-wsdm17}, the observation probability can be expressed as: $Pr(O^{y}_{q}=1|\pi_q) = {(\frac{1}{r(y|\pi_q)})}^\eta$, where $\eta$ controls the severity of position bias. If not explicitly mentioned, we set $\eta = 1.0$, which is considered the standard position bias. According to Chapelle et al.~\cite{chapelle-cikm09}, the relevance probability is: $Pr(R^{y}_{q}=1|\pi_q) = \epsilon + (1-\epsilon) \frac{2^{rel(q,y)}-1}{2^{{rel(q,y)}_{max}} - 1}$, where the true relevance for document $y$ with query $q$ is denoted by $rel(q,y) \in [0,4]$. The maximum relevance $rel(q,y)_{max}$ is $4$ in all our datasets. The parameter $\epsilon$ controls click noise, indicating that irrelevant documents can still have a chance of being mistakenly perceived as relevant and clicked. By default, we set $\epsilon = 0.0$ unless stated otherwise.
% Next, we generate whether a click occurs for each query-document pair by sampling from a binomial distribution using the corresponding click probability.
Generating clicks this way for a full pass over the training data is called one sampling pass. Following Ovaisi et al.~\cite{ovaisi-www20, ovaisi-sigir21}, we use 10 passes over the training dataset to generate clicks, unless stated otherwise. This method of click generation captures a typical user pattern, where some documents receive several clicks and others may not be clicked at all~\cite{ovaisi-www20}. We use the clicks generated in this step for all our experiments. A detailed explanation of why online evaluation is not feasible and challenging to implement for unbiased evaluation in unbiased LTR is provided in Appendix~\ref{online_eval}.
% {\renewcommand{\arraystretch}{1.0}
\begin{table*} [hb]
\setlength{\tabcolsep}{4.0pt}
% \scriptsize
\fontsize{7.8}{8}\selectfont
\captionsetup{justification=raggedright, width=1.0\textwidth}
\caption{A comparison between CFC and other methods across various datasets using true relevance for hyperparameter tuning. * denotes a statistically significant improvement of the best-performing CFC method over the best baseline methods with ${\bm{p < 0.05}}$. The best performance among all methods, excluding Oracle, is bolded, and the second-best is underlined.}
\centering
\scalebox{1.0}{%
\begin{tabular}{cccccccccccccccc}
\toprule
\multirow{2}{*}{Datasets} & \multirow{2}{*}{Metrics} & \multicolumn{5}{c} {LambdaMART} & \multicolumn{8}{c}{DNN} \\ \cmidrule(r){3-7} \cmidrule(r){8-16}
& & CFC & PIJD & PairD & LambdaMART & Oracle & CFC & PIJD & UPE & MULTR & PAL & REM & DLA & DNN & Oracle \\
\midrule
% \cline{1-11}
\multirow{2}[0]{*}{Yahoo}
& ERR@10 & \textbf{0.462}${^*}$ & 0.455 & 0.452 & 0.447 & 0.474 & \underline{0.456} & 0.448 & 0.449 & 0.453 & 0.446 & 0.452 & 0.452 & 0.444 & 0.464 \\
% \cline{2-11}
& NDCG@10 & \textbf{0.796}${^*}$ & 0.788 & 0.789 & 0.786 & 0.817 & \underline{0.792} & 0.787 & 0.783 & 0.789 & 0.786 & 0.775 & 0.788 & 0.782 & 0.805\\
\midrule
\multirow{2}[0]{*}{\makecell{MSLR-\\WEB10k}}
& ERR@10 & \textbf{0.317}${^*}$ & 0.277 & 0.285 & 0.270 & 0.342 & \underline{0.306} & 0.260 & 0.265 & 0.303 & 0.252 & 0.277 & 0.301 & 0.256 & 0.325 \\
% \cline{2-11}
& NDCG@10 & \textbf{0.486}${^*}$ & 0.440 & 0.464 & 0.457 & 0.498 & \underline{0.466} & 0.451 & 0.422 & 0.462 & 0.440 & 0.426 & 0.457 & 0.448 & 0.476 \\
% \cline{1-11}
\midrule
\multirow{2}[0]{*}{Istella-S}
& ERR@10 & \textbf{0.714}${^*}$ & 0.681 & 0.690 & \underline{0.694} & 0.737 & 0.683 & 0.674 & 0.645 & 0.689 & 0.675 & 0.679 & 0.681 & 0.672 & 0.702 \\
% \cline{2-11}
& NDCG@10 & \textbf{0.765}${^*}$ & 0.694 & \underline{0.741} & 0.740 & 0.777 & 0.712 & 0.709 & 0.657 & 0.715 & 0.709 & 0.699 & 0.710 & 0.705 & 0.745 \\
\bottomrule
\end{tabular}
}
\label{tab:main_results_with_true_validation_data_short_version}
\end{table*}
% }
\textbf{Implementation details.}
To train the ranker $g()$ using equation~\eqref{eq:unbiased_c_final}, we use an LTR algorithm. Our CFC method is flexible and can be integrated with any state-of-the-art LTR algorithm. LTR literature commonly uses two types of models for ranking: gradient-boosted decision trees (GBDTs) and neural networks (NNs). As a GBDT-based method, we use LambdaMART~\cite{burges-learning10} as the ranker $g()$, which is the most widely used LTR algorithm~\cite{lyzhin-icml23}. As an NN-based method, we use a deep neural network (DNN) implemented in~\cite{ai-sigir18}, as the ranker $g()$. Following Ai et al.~\cite{ai-sigir18}, we compute statistical significance using the Fisher randomization test ~\cite{smucker-cikm07} with $\bm{p} < 0.05$. We conduct our experiments on machines with 64GB RAM, 1TB storage, and a GPU with 16GB CUDA-enabled memory. The click generation, first-stage ranking, and residual transformations, LambdaMART run on CPU, while DNN-based models are trained on GPU. We provide additional implementation details in Appendix~\ref{implementation_details} and a detailed description of the training parameters in Appendix~\ref{training_paramters}.
\textbf{Baselines.} We compare CFC with several position bias correction baselines, including PIJD~\cite{ovaisi-sigir21}, PairD~\cite{hu-www19}, UPE~\cite{luo-sigir24}, MULTR~\cite{luo-wsdm23}, PAL~\cite{guo-recsys19}, REM~\cite{wang-wsdm18}, and DLA~\cite{ai-sigir18}. Since CFC method is implemented with the LambdaMART and the DNN rankers, we include comparisons with the LambdaMART and DNN ranking algorithms that do not correct for any bias. The Oracle method is trained with human-annotated labels, representing the upper-bound performance of any ranker. For PIJD, we use both LambdaMART and DNN as the ranker, as PIJD is flexible to integrate with both of these rankers. For PairD, we use LambdaMART, as PairD is coupled with LambdaMART in the original implementation. We use DNN ranker for UPE, MULTR, PAL, REM, and DLA, as these methods are implemented with DNNs. On the other hand, our method is flexible enough to integrate with both of the rankers. We provide detailed descriptions of the baselines in Appendix~\ref{baselines}.
% % {\renewcommand{\arraystretch}{1.0}
% \begin{table*}
% % \setlength{\belowcaptionskip}{5pt}
% \setlength{\tabcolsep}{1.4pt}
% % \scriptsize
% \fontsize{6.2}{7}\selectfont
% \captionsetup{justification=centering, width=1.0\textwidth}
% \caption{A comparison between CFC and other methods across various datasets using click data for hyperparameter tuning. \modify{CFC and PIJD can be implemented with both LambdaMART and DNN rankers. PairD is coupled with LambdaMART, while UPE, MULTR, PAL, REM, and DLA are implemented with DNN rankers and do not have LambdaMART implementations. * denotes a statistically significant improvement of the best-performing CFC method over the best baseline methods with ${\bm{p} < 0.05}$. The best performance among all methods, excluding Oracle, is bolded, and the second-best is underlined.}}
% \centering
% \scalebox{1.0}{%
% \begin{tabular}{cccccccccccccccccc}
% \toprule
% \multirow{2}{*}{Datasets} & \multirow{2}{*}{Metrics} & \multicolumn{6}{c} {LambdaMART} & \multicolumn{10}{c}{DNN} \\ \cmidrule(r){3-8} \cmidrule(r){9-18}
% & & CFC-DC & CFC-BC & PIJD & PairD & LambdaMART & Oracle & CFC-DC & CFC-BC & PIJD & UPE & MULTR & PAL & REM & DLA & DNN & Oracle \\
% \midrule
% % \cline{1-11}
% \multirow{2}[0]{*}{Yahoo}
% & ERR@10 & \textbf{0.460}${^*}$ & 0.437 & 0.452 & 0.452 & 0.446 & 0.474 & \underline{0.455} & 0.444 & 0.447 & 0.437 & 0.452 & 0.441 & 0.444 & 0.433 & 0.441 & 0.464 \\
% & NDCG@10 & \textbf{0.794}${^*}$ & 0.783 & 0.780 & 0.785 & 0.789 & 0.817 & \underline{0.792} & 0.784 & 0.785 & 0.772 & 0.789 & 0.782 & 0.768 & 0.777 & 0.781 & 0.805\\
% \midrule
% \multirow{2}[0]{*}{\makecell{MSLR-\\WEB10k}}
% & ERR@10 & \textbf{0.307}${^*}$ & 0.278 & 0.275 & 0.285 & 0.268 & 0.342 & \underline{0.292} & 0.291 & 0.259 & 0.232 & 0.291 & 0.253 & 0.241 & 0.256 & 0.253 & 0.325 \\
% % \cline{2-11}
% & NDCG@10 & \textbf{0.482}${^*}$ & \underline{0.465} & 0.438 & 0.463 & 0.446 & 0.498 & \underline{0.465} & 0.464 & 0.448 & 0.362 & 0.451 & 0.436 & 0.419 & 0.447 & 0.446 & 0.476 \\
% % \cline{1-11}
% \midrule
% \multirow{2}[0]{*}{Istella-S}
% & ERR@10 & \textbf{0.716}${^*}$ & \underline{0.698} & 0.680 & 0.689 & 0.690 & 0.737 & 0.679 & 0.676 & 0.672 & 0.671 & 0.675 & 0.669 & 0.653 & 0.676 & 0.669 & 0.702 \\
% % \cline{2-11}
% & NDCG@10 & \textbf{0.765}${^*}$ & \underline{0.752} & 0.692 & 0.739 & 0.733 & 0.777 & 0.710 & 0.709 & 0.706 & 0.690 & 0.691 & 0.708 & 0.685 & 0.706 & 0.705 & 0.745 \\
% \bottomrule
% \end{tabular}
% }
% \label{tab:main_results_with_biased_clicks_short_version}
% \end{table*}
% % }
% % {\renewcommand{\arraystretch}{1.0}
% \begin{table*}
% \setlength{\tabcolsep}{2.0pt}
% % \scriptsize
% \fontsize{7}{7}\selectfont
% \captionsetup{justification=centering, width=1.0\textwidth}
% \caption{A comparison between CFC and other methods across various datasets using true relevance for hyperparameter tuning. \modify{* denotes a statistically significant improvement of the best-performing CFC method over the best baseline methods with ${\bm{p} < 0.05}$. The best performance among all methods, excluding Oracle, is bolded, and the second-best is underlined.}}
% \centering
% \scalebox{1.0}{%
% \begin{tabular}{cccccccccccccccc}
% \toprule
% \multirow{2}{*}{Datasets} & \multirow{2}{*}{Metrics} & \multicolumn{5}{c} {LambdaMART} & \multicolumn{8}{c}{DNN} \\ \cmidrule(r){3-7} \cmidrule(r){8-16}
% & & CFC & PIJD & PairD & LambdaMART & Oracle & CFC & PIJD & UPE & MULTR & PAL & REM & DLA & DNN & Oracle \\
% \midrule
% % \cline{1-11}
% \multirow{2}[0]{*}{Yahoo}
% & ERR@10 & \textbf{0.462}${^*}$ & 0.455 & 0.452 & 0.447 & 0.474 & \underline{0.456} & 0.448 & 0.449 & 0.453 & 0.446 & 0.452 & 0.452 & 0.444 & 0.464 \\
% % \cline{2-11}
% & NDCG@10 & \textbf{0.796}${^*}$ & 0.788 & 0.789 & 0.786 & 0.817 & \underline{0.792} & 0.787 & 0.783 & 0.789 & 0.786 & 0.775 & 0.788 & 0.782 & 0.805\\
% \midrule
% \multirow{2}[0]{*}{\makecell{MSLR-\\WEB10k}}
% & ERR@10 & \textbf{0.317}${^*}$ & 0.277 & 0.285 & 0.270 & 0.342 & \underline{0.306} & 0.260 & 0.265 & 0.303 & 0.252 & 0.277 & 0.301 & 0.256 & 0.325 \\
% % \cline{2-11}
% & NDCG@10 & \textbf{0.486}${^*}$ & 0.440 & 0.464 & 0.457 & 0.498 & \underline{0.466} & 0.451 & 0.422 & 0.462 & 0.440 & 0.426 & 0.457 & 0.448 & 0.476 \\
% % \cline{1-11}
% \midrule
% \multirow{2}[0]{*}{Istella-S}
% & ERR@10 & \textbf{0.714}${^*}$ & 0.681 & 0.690 & \underline{0.694} & 0.737 & 0.683 & 0.674 & 0.645 & 0.689 & 0.675 & 0.679 & 0.681 & 0.672 & 0.702 \\
% % \cline{2-11}
% & NDCG@10 & \textbf{0.765}${^*}$ & 0.694 & \underline{0.741} & 0.740 & 0.777 & 0.712 & 0.709 & 0.657 & 0.715 & 0.709 & 0.699 & 0.710 & 0.705 & 0.745 \\
% \bottomrule
% \end{tabular}
% }
% \label{tab:main_results_with_true_validation_data_short_version}
% \end{table*}
% % }
% While neural networks (NNs) have made significant advances across different machine learning areas, gradient-boosted decision trees (GBDTs) remain the preferred method for ranking tasks~\cite{qin-iclr20}.
% In the second stage of our method, as depicted by equation ~\eqref{eq:unbiased_c_final}, we train $g()$ using an LTR algorithm. Any state-of-the-art LTR algorithm can be used for training $g()$ while accounting for position bias through our CFC method. The literature on recommender systems commonly utilizes two machine learning models for ranking: gradient-boosted decision trees (GBDTs) and neural networks (NNs). Even with the notable progress in NN-based methods across different machine learning areas, GBDTs continue to be the preferred method for tabular data~\cite{shwartz-inf22} and ranking tasks~\cite{qin-iclr20}. LambdaMART~\cite{burges-learning10, wu-springer10}, based on GBDTs, is considered the most widely used state-of-the-art algorithm for LTR tasks ~\cite{lyzhin-icml23}. We use the LambdaMART algorithm in the ranker $g()$ in our CFC method.
% Detailed descriptions of the training parameters and hyperparameter tuning for our method are included in Appendix ~\ref{sec:training_params}.
% In the first stage of our method, as formulated in equation ~\eqref{eq:control_function}, we train the control function $m()$ using a supervised machine learning algorithm. For learning $m()$, we explore various machine learning models, such as Ridge Classifier, Logistic Regression, Support Vector Machine (SVM), and eXtreme Gradient Boosting (XGBoost). If $m()$ is overfitted, it may achieve near-zero training error, leaving no residuals to correct for position bias, even when such bias is present in the data. Thus, we select $m()$ based on how well incorporating the residuals from that $m()$ into $g()$ performs for ranker $g()$. While all four classifiers we explore to train $m()$ provide comparable performance on ranker $g()$, we observe that the Ridge Classifier generally yields the better ranking performance on $g()$ across all three datasets. Therefore, we use Ridge Classifier to learn $m()$.
% In the first stage of our method, as formulated in equation ~\eqref{eq:control_function}, we train the control function $m()$ using a supervised machine learning algorithm to predict the ranking of query-document pairs.
% For learning $m()$, we explore various machine learning models, such as Ridge Classifier, Logistic Regression, Support Vector Machine (SVM), and eXtreme Gradient Boosting (XGBoost). We choose $m()$ based on whether calculating residuals with that $m()$ provides the best performance for
% ranker $g()$. If the model $m()$ is overfitted, it may achieve a near-zero training error, leaving no residuals to correct for position bias, even when position bias exists in the data. Therefore, we select $m()$ based on how the residuals from that $m()$ performs on the ranker $g()$. While all of these four classifiers we used to train $m()$ provides comparable performance on the ranker $g()$, we observe that Ridge Classifier for $m()$ generally yields the best ranking performance on $g()$ for all three datasets. Therefore, we use Ridge Classifier to learn $m()$.
% We use the Ridge classifier that uses the Ridge regression loss function. This loss function is the mean squared error (MSE) with an $L_2$ regularization term. We use a classification model for $m()$ instead of a regression model because $m()$ directly predicts document rankings, which are discrete rather than continuous.
%Note that while we use a Ridge classifier, other machine learning algorithms can also be used to learn $m()$. If the model $m()$ is overfitted, it may achieve a near-zero training error, leaving no residuals to correct for position bias, even when position bias exists in the data. Therefore, the model for $m()$ should be selected based on the performance of the final ranker $g()$ on the validation data.
\subsection{Results}
In this section, we show the results of our proposed CFC method in comparison with existing methods.
\textbf{Performance comparison without access to true relevance in validation data (RQ1).}
In response to RQ1, we assume true relevance is unavailable in the validation dataset and use click data for hyperparameter tuning. When using click data for validation, we have two variants: CFC-DC, which uses debiased clicks, and CFC-BC, which uses biased clicks directly for hyperparameter tuning. Since the baselines do not have a mechanism to debias the click data, we use the click data directly for hyperparameter tuning in the baselines. Results for $ERR@10$ and $NDCG@10$ are shown in Table~\ref{tab:main_results_with_biased_clicks_short_version}. CFC-DC with LambdaMART outperforms all baseline methods using either LambdaMART or DNN rankers across all datasets. Moreover, CFC-DC with DNN achieves the best performance among all DNN-based baselines on all datasets. CFC-BC, with both LambdaMART and DNN rankers, underperforms compared to the CFC-DC variants. This is because tuning hyperparameters for an unbiased ranking model $g()$ with biased clicks may not yield the optimal ranker. This highlights that using biased clicks for validation can lead to suboptimal ranking performance on unbiased test data due to distribution shift. Both variants of CFC perform better with LambdaMART as the ranker than with their DNN counterparts, as GBDT-based rankers like LambdaMART are generally considered stronger LTR models than neural rankers~\cite{hager-sigir24}. While CFC-DC with DNN is the second-best method on Yahoo and MSLR-WEB10K datasets, CFC-BC with LambdaMART outperforms it on Istella-S dataset, likely due to LambdaMART’s stronger ranking capability.
% Nevertheless, our CFC-DC with DNN achieves the best performance among all DNN-based baselines on Istella-S dataset.
In summary, CFC-DC with LambdaMART achieves the best performance, surpassing all baselines and CFC-BC variants for both rankers, while CFC-DC with DNN outperforms all other DNN-based baselines.
% This highlights the effectiveness of our position bias correction methods for both rankers.
% The Oracle method, using human-annotated labels, achieves the best results, indicating room for further improvements in ULTR.
Results for $ERR@p$ and $NDCG@p$ at
$p = 5$ and $15$ are provided in Table~\ref{tab:main_results_with_biased_clicks_full_version} in Appendix.
\begin{figure*} [hb]
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-lm.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-severity-lm.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-severity-lm.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-severity-lm.pdf}
\caption{Istella-S}
\end{subfigure}
% \vspace{-0.3cm}
\captionsetup{width=0.98\textwidth}
\caption{The performance of different methods by varying position bias severity using LambdaMART as the ranker.\label{fig:bias_severity-lm}}
% \vspace{-0.7cm}
\end{figure*}
\begin{figure*} [hb]
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-lm.png}
\end{subfigure}
\begin{subfigure}{\figsize \textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-passes-lm.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize \textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-passes-lm.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize \textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-passes-lm.pdf}
\caption{Istella-S}
\end{subfigure}
\captionsetup{width=0.98\textwidth}
\caption{The performance of different methods with varying the number of passes for clicks generation using LambdaMART as the ranker.\label{fig:passes_variations-lm}}
\end{figure*}
\textbf{Performance comparison assuming access to true relevance in validation data (RQ2).} \label{sec:performance_true}
For question RQ2, we use true relevance in the validation set for hyperparameter tuning for all methods. Results using $ERR@10$ and $NDCG@10$ metrics are shown in Table~\ref{tab:main_results_with_true_validation_data_short_version}. The CFC method with LambdaMART achieves the best performance across all datasets, outperforming all baseline methods using either LambdaMART or DNN ranker. CFC with DNN achieves the best performance among all DNN-based baselines on the Yahoo and MSLR-WEB10K datasets. On the Istella-S dataset, LambdaMART or PairD achieves the second-best results, surpassing DNN-based methods, likely due to LambdaMART’s stronger ranking capability. Nevertheless, on Istella-S dataset, CFC with DNN outperforms all other DNN-based methods, except MULTR.
On the MSLR-WEB10K and Istella-S datasets, the PDF and IMR transformations assuming normality yield the best ranking performance, likely because the residuals from $m()$ in these datasets approximately follow a normal distribution. In contrast, for the Yahoo dataset, where the residuals do not follow a normal distribution, the min-max transformation performs best. This highlights that CFC method can automatically find the best residual transformation to achieve optimal ranking performance. Clicks in the MSLR-WEB10k and Istella-S datasets are sparser than in Yahoo, with click densities of approximately $0.36\%$ and $1.10\%$, respectively, compared to $2.72\%$ in Yahoo under standard position bias. CFC method surpasses the baselines, showing its robustness in scenarios with such highly sparse click data.
\begin{figure*} [hb]
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-lm.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-noisy-lm.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-noisy-lm.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-noisy-lm.pdf}
\caption{Istella-S}
\end{subfigure}
\captionsetup{width=0.98\textwidth}
\caption{The performance of different methods up to $\bm{10 \ (p)}$ positions with varying percentages of noisy clicks using LambdaMART as the ranker.\label{fig:noisy_clicks-lm}}
\end{figure*}
\begin{figure*} [hb]
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-lm.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-validation-lm.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-validation-lm.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-validation-lm.pdf}
\caption{Istella-S}
\end{subfigure}
\captionsetup{width=0.98\textwidth}
% \caption{The performance of different methods for up to $\bm{10 \ (p)}$ positions across various datasets with varying numbers of validation queries.\label{fig:validation_variety}}
\caption{The performance of different methods up to $\bm{10 \ (p)}$ positions with varying numbers of validation queries using LambdaMART as the ranker.\label{fig:validation_variety-lm}}
% \vspace{-.7cm}
\end{figure*}
\textbf{The effect of bias severity (RQ3).}
We evaluate CFC under varying the severity of position bias (\( \eta \in \{0.5, 1.0, 1.5, 2.0\} \)) in click generation. Figure~\ref{fig:bias_severity-lm} shows the $ERR@10$ results with LambdaMART, while Figure~\ref{fig:bias_severity-dnn} in the Appendix presents results for the DNN ranker. Across all datasets and bias severities, CFC with LambdaMART outperforms all baselines. As \( \eta \) increases, the performance gap between Oracle and other methods increases due to increased sparsity in clicks. Nevertheless, our method is more robust than baselines across bias severity levels. CFC with DNN also performs best among DNN-based baselines on Yahoo and MSLR-WEB10K for all \( \eta \), except at \( \eta = 2.0 \) on MSLR-WEB10K. On the Istella-S dataset, CFC with DNN performs best among DNN-based methods at \( \eta = 0.5 \), while MULTR outperforms it at other bias severities. While results for LambdaMART and DNN-based methods are shown separately for clarity of visualization, CFC with LambdaMART consistently outperforms all baselines across datasets and bias levels, regardless of ranker.
\textbf{Performance trends with increasing clicks (RQ4).} \label{sec:passes_variations} We evaluate the performance of the CFC method by varying the amount of click data. To achieve this, we generate clicks by performing 1, 10, and 20 passes over the training data. For $1$, $10$, and $20$ training passes, we obtain approximately $13$k, $127$k, and $253$k clicks on Yahoo dataset; $3$k, $27$k, and $54$k on MSLR-WEB10k dataset; and $22$k, $223$k, and $446$k on Istella-S dataset. Figure~\ref{fig:passes_variations-lm} shows the $ERR@10$ performance with LambdaMART, while the results for the DNN ranker are presented in Figure~\ref{fig:passes_variations-dnn} in the Appendix. Across all three datasets and all passes, CFC with LambdaMART consistently outperforms all baseline methods. As the number of clicks (passes) increases, the performance of our method improves and gets closer to the performance of the Oracle method. With more clicks (passes), the model benefits from a bias-variance tradeoff, with reduced bias from unbiased method and lower variance due to more stable training data, leading to improved performance. The performance of CFC with DNN also improves with the number of clicks, achieving the best results on the Yahoo and MSLR-WEB10K datasets among all DNN-based baselines, except for pass $1$ on MSLR-WEB10K dataset. On the Istella-S dataset, CFC with DNN surpasses all other DNN-based methods except MULTR. Note that while we do not display LambdaMART and DNN-based methods in the same figure for clarity of visualization, CFC with LambdaMART outperforms all LambdaMART and DNN-based baselines across all three datasets and for all numbers of passes.
\textbf{The effect of click noise (RQ5).}
Noisy clicks often arise in real-world settings when users interact with irrelevant documents. To evaluate the robustness of our method against noisy clicks, we vary the probability of clicking on irrelevant documents ($\epsilon = 5\%$, $10\%$, $15\%$, and $20\%$). The $ERR@10$ results for LambdaMART are shown in Figure~\ref{fig:noisy_clicks-lm}, while Figure~\ref{fig:noisy_clicks-dnn} in the Appendix displays the results for the DNN ranker. CFC with LambdaMART outperforms all the baselines across different levels of noisy clicks on all three datasets. With an increase in noisy clicks, the performance difference between Oracle and all other methods increases, gradually lessening the effectiveness of bias correction methods. For the DNN-based ranker, CFC with DNN outperforms all other DNN-based methods on the Yahoo and MSLR-WEB10k datasets, except for slightly lower performance compared to MULTR with $15\%$ noisy clicks on the MSLR-WEB10k dataset. On the Istella-S dataset, CFC with DNN outperforms all DNN-based methods except MULTR. Note that, for clarity of visualization, we present LambdaMART and DNN-based methods in separate figures. However, CFC with LambdaMART outperforms all LambdaMART and DNN-based baselines across all three datasets and noise levels. Therefore, our method demonstrates greater robustness to noisy clicks compared to baselines using either LambdaMART or DNN as the ranker.
\textbf{The effect of the number of queries on hyperparameter tuning (RQ6).}
We conduct a study to analyze how the number of queries with true relevance in the validation data affects the performance of different methods. We randomly sample $150$, $300$, and $600$ queries from the validation dataset and also use the entire validation dataset for hyperparameter tuning. Figure~\ref{fig:validation_variety-lm} presents the $ERR@10$ performance with LambdaMART, whereas the corresponding results for the DNN ranker are shown in Figure~\ref{fig:validation_variety-dnn} in the Appendix. Across all datasets, CFC with LambdaMART outperforms the baselines for varying numbers of queries used in hyperparameter tuning. CFC with DNN surpasses all DNN-based methods on the Yahoo and MSLR-WEB10K datasets, and also outperforms them on Istella-S, except for MULTR. While LambdaMART and DNN-based methods are shown in separate figures for clarity of visualization, CFC with LambdaMART outperforms all LambdaMART and DNN-based baselines across all three datasets and different number of queries used for hyperparameter tuning.
Moreover, the results indicate that hyperparameter tuning of CFC—whether using LambdaMART or DNN as the ranker—is not sensitive to the number of validation queries, as it achieves comparable performance across different number of queries used for hyperparameter tuning. Moreover, nearly all baselines also exhibit comparable performance across different numbers of queries used for hyperparameter tuning.
% Moreover, the results indicate that hyperparameter tuning of the CFC method is not sensitive to the number of validation queries while outperforming baselines in almost all cases, with the exception of the DLA method. This could be because the parameters of the DLA method are sensitive to the quality and representativeness of the sampled validation dataset.
This experimental evaluation is beneficial for hyperparameter tuning with true relevance, as it can eliminate the need for annotating the entire validation dataset. The evaluation demonstrates that a small subset of queries can be randomly sampled, annotated, and used for hyperparameter tuning.
\begin{table} [htbp]
\fontsize{7.8}{8}\selectfont
\captionsetup{justification=raggedright, width=0.49\textwidth}
\caption{The performance of the CFC method using LambdaMART as the ranker with different machine learning algorithms used to estimate the previous ranking process in first stage.}
\centering
\begin{tabular}{ccccc}
\toprule
\multirow{2}{*}{Datasets} & \makecell{Ridge} & \makecell{Logistic Regression} & \makecell{SVM} & \makecell{XGBoost} \\ \cmidrule(r){2-5} & \multicolumn{4}{c} {$ERR@10$} \\
\midrule
Yahoo & \textbf{0.462} & 0.459 & 0.457 & 0.457 \\
\makecell{MSLR-WEB10k} & \textbf{0.317} & 0.301 & 0.301 & 0.304 \\
Istella-S & \textbf{0.714} & 0.713 & 0.715 & 0.714 \\
\bottomrule
\end{tabular}
\label{tab:cf_variations}
% \vspace{-2em}
\end{table}
\textbf{The effect of model choice in estimating the previous ranking process (RQ7).}
For question RQ7, we investigate how the choice of machine learning model used to train $m()$—from which the residuals are estimated and included in the second stage—affects the ranking performance of the CFC method. We experiment with four machine learning models to train $m()$: Ridge, Logistic Regression, SVM, and XGBoost, and incorporate the resulting residuals into $g()$. The performance of the CFC method with LambdaMART using each of these models for learning $m()$ is shown in Table~\ref{tab:cf_variations}. Our results show that all models used to estimate $m()$ yield comparable ranking performance for $g()$, demonstrating the flexibility of the CFC method and its insensitivity to model specification. However, using Ridge for the control function yields the best ranking performance on all datasets. This may be because the linear structure of Ridge Regression, along with its ability to handle multicollinearity through regularization, is well-suited for modeling residual variation within the control function framework on these
datasets. Our findings suggest that the effectiveness of our approach is not heavily dependent on the choice of model used in the first-stage training. Unlike Heckman-based methods~\cite{ovaisi-www20, ovaisi-sigir21}, which require ordinary least squares (OLS) in the first stage to satisfy their statistical assumptions, our control function-based method imposes no such restrictions and allows both linear and non-linear machine learning models in the first stage. Thus, unlike the Heckman-based methods, the CFC method is not tightly coupled with any specific type of learning model in its first stage.
% \textbf{RQ4-RQ7.} In question RQ4, we evaluate the performance of the CFC method by varying the amount of click data, generating clicks through $1$, $10$, and $20$ passes over the training set. As the number of clicks (passes) increases, the performance of CFC improves (Figure~\ref{fig:passes_variations}) and tends to get closer to that of the Oracle method. This improvement can be attributed to the bias-variance tradeoff—reduced bias from the debiasing method and lower variance due to more click training data. In RQ5, we evaluate the robustness of our method against noisy clicks by varying the noise levels in the click data. CFC method shows greater robustness to noisy clicks (Figure~\ref{fig:noisy_clicks}), outperforming the baselines. For question RQ6, we evaluate how the number of queries used for hyperparameter tuning affects model performance to assess whether optimal tuning is possible with limited validation queries containing true relevance. The CFC method can achieve comparable performance using only a subset of the validation set with true relevance labels (Figure~\ref{fig:validation_variety}), which can reduce the need for extensive human annotation required for full validation datasets. For question RQ7, We assess whether the choice of machine learning model for learning the document ranking process \( m() \), from which we estimate the residual variation, affects the final ranking performance of \( g() \). Using Ridge, Logistic Regression, SVM, and XGBoost for $m()$, we find that all achieve comparable final ranking performance on $g()$ (Table~\ref{tab:cf_variations}), indicating that CFC method is not sensitive to model choice or specification for estimating the residuals. We provide detailed results and analysis for RQ4–RQ7 in Appendix~\ref{additional_results}.
% With more clicks (passes), the variance in the model's estimates decreases while the bias is reduced, resulting in improved performance due to the bias-variance trade-off.
% As the number of clicks increases, the model benefits from a favorable tradeoff between bias and variance, with reduced bias from richer relevance signals and lower variance due to more stable training data, resulting in improved generalization.
% The Oracle method exhibits a flat line because it is trained on true relevance data.
% which does not have the impact of position bias. When $\eta$ is comparatively lower, our method and the baselines achieve performance closer to the Oracle.
% We evaluate the performance of our CFC method under different levels of position bias severity. We vary the values of $\eta$ ($\eta=0.5, 1.0, 1.5$, and $2.0$) in the click data generation process, which controls the severity of position bias. Figure~\ref{fig:bias_severity-lm} presents the $ERR@10$ results using LambdaMART as the ranker, and Figure~\ref{fig:bias_severity-dnn} in the Appendix shows the results for the DNN-based ranker. On all three datasets, CFC with LambdaMART outperforms all the baselines across all four values of position bias severity. As $\eta$ increases, the performance gap between Oracle and all other methods generally increases. This is because an increase in $\eta$ leads to sparser clicks, which gradually decreases the effectiveness of bias correction methods compared to Oracle. Nonetheless, when $\eta$ increases, our method demonstrates better performance than the baselines, making it more robust across various severities of position bias. The results also demonstrate that our method exhibits greater robustness in scenarios with sparser click data. CFC with DNN achieves the best performance among all DNN-based baselines on the Yahoo and MSLR-WEB10K datasets for all values of $\eta$, except at $\eta=2.0$ on the MSLR-WEB10k dataset. On the Istella-S dataset, CFC with DNN achieves the second-best performance at $\eta0.5$, while MULTR outperforms CFC with DNN at other bias severity levels. However, although visualized separately for clarity, CFC with LambdaMART outperforms all LambdaMART and DNN-based baselines across all datasets and bias severities.
% The LambdaMART method, which does not account for position bias, sometimes outperforms certain bias correction methods, particularly when the severity of position bias is low. This may occur due to a mismatch between the propensity estimation and the click bias in a correction method. When position bias is lower, the correction method may overestimate the click bias, leading to suboptimal performance compared to LambdaMART.
% As an example, CFC shows percentage improvements of $+1.12\%$, $+1.75\%$, and $+0.89\%$ for $ERR@10$ compared to the strongest baselines under $10\%$ noisy clicks on the Yahoo, MSLR-WEB10k, and Istella-S datasets, respectively. Because the Oracle method uses true relevance for training, it does not have any effect from noisy clicks.
% The Heckman-based PIJD method performs poorly, especially on the Istella-S dataset, due to its reliance on linear models that may fail to capture the non-linear relationships between high-dimensional features and sparse clicks. In contrast, the CFC method does not require linearity assumptions, and as a result, shows notably improved performance compared to PIJD.
% The LambdaMART method, which does not address position bias, sometimes outperforms certain bias correction methods. Our explanation for this is that when the ranking algorithm used in a bias correction method is weaker than LambdaMART and the improvement by applying the method is small, the method may still not outperform LambdaMART, which does not correct for bias.
% For example, the PIJD method uses a linear model to satisfy its assumptions, which can lead to lower performance compared to LambdaMART, which is a non-linear ranking algorithm. The extended results, using $p = 1, 3, 5, 10,$ and $15$ with $ERR@p$ and $NDCG@p$, are provided in Appendix ~\ref{sec:additional_results}.
% Note that we also evaluate the performance for $p = 1, 3, 5,$ and $15$ with $ERR@p$ and $NDCG@p$. However, due to space constraints, we do not include those results. Our CFC method also outperforms all the baselines across all datasets for these values of $p$ as well.
% For example, the percentage improvements of CFC over the strongest baselines for $ERR@10$ are $+1.12\%$, $+1.96\%$, and $+1.43\%$ on the Yahoo, MSLR-WEB10k, and Istella-S datasets, respectively.
% For example, the percentage improvements of CFC over the strongest baselines for $NDCG@10$ are $+1.40\%$ for Yahoo, $+4.33\%$ for MSLR-WEB10k, and $+2.32\%$ for the Istella-S datasets.
% \begin{table} [htbp]
% \centering
% \begin{tabularx}{0.249\textwidth} {|c|c|}
% \hline
% Dataset & Click density \\ \hline
% Yahoo & 2.72\% \\ \hline
% MSLR-WEB10k & 0.36\% \\ \hline
% Istella-S & 1.10\% \\ \hline
% \end{tabularx}
% \captionsetup{justification=centering, font=small}
% \caption{\modify{Click density of three datasets for standard position bias.}}
% \label{tab:density}
% \vspace{-0.8cm}
% \end{table}
% We find that for the MSLR-WEB10k and Istella-S datasets, applying the PDF transformation to the residuals of the control function with the assumption of normally distributed residuals yields the optimal ranker. This is likely because the residuals in these two datasets approximately follow a normal distribution, and estimating the PDF using a normal distribution captures the underlying distributional information of the residuals. However, for the Yahoo dataset, the min-max transformation of the residuals yields the optimal ranker. This can be due to the fact that the residuals in this dataset do not follow a normal or approximately normal distribution, making the use of a normal distribution for approximation less effective for capturing their distributional information.
% For example, CFC-DC shows percentage improvements over the strongest baselines in $ERR@10$ by $+2.24\%$ on Yahoo, $+1.96\%$ on MSLR-WEB10k, and $+3.05\%$ on Istella-S datasets.
% For example, CFC-DC shows percentage improvements over the strongest baselines in $NDCG@10$ by $+1.02\%$ on Yahoo, $+4.78\%$ on MSLR-WEB10k, and $+2.87\%$ on Istella-S datasets.
% The performance of several baseline methods declines significantly. For instance, on the MSLR-WEB10k dataset, CFC-DC achieves an $ERR@10$ of $0.307$, while DLA only reaches $0.130$. On the Istella-S dataset, CFC-DC achieves $0.709$, while MULTR drops to $0.302$. In some cases, some baselines perform worse than LambdaMART, which does not correct for position bias. These findings suggest that using click data directly for hyperparameter tuning can lead to unintended performance degradation in bias correction methods. In contrast, CFC-DC shows consistent performance with debiased clicks for hyperparameter tuning.
% The performance of several baseline methods declines significantly. For example, on the MSLR-WEB10k dataset, while CFC-DC achieves an $ERR@10$ of $0.307$, the DLA method reaches only $0.130$. On the Istella-S dataset, CFC-DC achieves an $ERR@10$ of $0.709$, whereas the performance of the MULTR method drops to $0.302$. Moreover, in most of the cases, the performance of some baselines falls below that of the LambdaMART method, which does not correct for position bias. These observations indicate that directly using click data for hyperparameter tuning can lead to unintended performance degradation in bias correction methods. In contrast, our CFC-DC method demonstrates consistent performance using the debiased clicks derived from the biased click data for hyperparameter tuning. The Oracle method, trained using human-annotated labels and serving as an upper bound for rankers, achieves the best results. This implies that there is still scope for further performance improvements in unbiased learning to rank.
% \modify{\subsection{Discussion on Computational Complexity}
% We now discuss the computational complexity and scalability of our implemented method. Let the total number of documents be $n$, and let $d$ represent the the feature dimensions. The Ridge Classifier for the control function takes $O(n d^2)$ time ~\cite{zhang-mls15}. Min-max normalization, parametric PDF transformation, and parametric IMR transformation are computed using closed-form solutions, each requiring $O(n)$ time. The nonparametric IMR transformation is estimated using fastKDE ~\cite{o-csda16}, which has a complexity of $O(n + p \log p)$ ~\cite{heer-vis21}, where $p$ is the bin size. $p$ is typically $256$ or $512$, and in LTR datasets, $p \ll n$. Therefore, this step can be approximated as $O(n)$. For the LambdaMART ranker, the computational complexity is $O(n (T + L))$ ~\cite{li-mpe18}, where $T$ is the number of trees in the ensemble, and $L$ is the max depth of trees. Thus, the total complexity of the CFC method can be expressed as: $O(n d^2) + 4O(n) + O(n (T + L))$. This complexity can be more simplified as: $O(n (T + L + d^2))$. Therefore, the overall runtime is polynomial. In LTR datasets, where millions of documents are present, $T$ (typically between $100$ and $500$) and $L$ (usually between $4$ and $16$) are very small compared to the total number of documents. Therefore, they do not impact the overall runtime. Additionally, the feature dimensions $d$ are usually constant for a given dataset. As a result, adding more documents or queries leads to a linear increase in computation. This linear increase makes the CFC method scalable and suitable for large-scale industrial applications. For instance, we randomly sample half of the training queries across all three datasets and execute our method using both the sampled half and the full set of queries. We observe that increasing the number of queries from half to the full set results in a runtime increase of approximately $70\%$ to $80\%$ across the three datasets.}
% \modify{Since our CFC method uses the LambdaMART LTR algorithm, we compare its runtime performance with the LambdaMART method that does not correct for position bias. Our analysis shows that the CFC method requires approximately $85\%$ to $90\%$ more time compared to LambdaMART on the three datasets. This additional overhead arises from training the control function, estimating residuals, and applying different transformations to the residuals. However, this overall increase corresponds to a linear time overhead relative to LambdaMART, which is manageable.}
% Summaries of our findings:
% \begin{itemize}[leftmargin=9pt, nosep]
% \item On the Yahoo dataset, our method outperforms all the baselines in terms of ERR and NDCG. The improvements of CFC over the strongest baselines are $+2.07\%$, $+1.20$, $+1.13$, $+1.12\%$, $+1.09\%$ in terms of $ERR@p$ for $p = 1,3,5,10, \text{and} \ 15$, respectively. In terms of $NDCG@p$ for $p = 1,3,5,10, \text{and} \ 15$, the improvements of CFC over the strongest baselines are $+2.47\%$, $+1.51\%$, $+0.81\%$, $+0.76\%$, $+0.62\%$, respectively.
% \item Our method outperforms all the baselines on the MSLR-WEB10k dataset in both ERR and NDCG metrics. The CFC method shows improvements of $+4.10\%$, $+2.23$, $+2.73$, $+1.92\%$, $+1.92\%$ over the best baselines for $ERR@p$ at $p = 1,3,5,10, \text{and} \ 15$, respectively. For $NDCG@p$, the improvements are $+5.42\%$, $+4.64\%$, $+4.85\%$, $+4.33\%$, $+3.65\%$, respectively.
% \item On the Istella-S dataset, our method outperforms all the baselines in both ERR and NDCG metrics. CFC improves upon the best baselines by $+2.10\%$, $+1.63$, $+1.59$, $+1.43\%$, $+1.41\%$ for $ERR@p$ at $p = 1,3,5,10, \text{and} \ 15$, respectively. n terms of $NDCG@p$ for $p = 1,3,5,10, \text{and} \ 15$, the improvements are $+2.60\%$, $+2.87\%$, $+2.85\%$, $+3.01\%$, $+3.13\%$, respectively.
% \item We find that for the MSLR-WEB10k and Istella-S datasets, applying PDF transformation with the assumption of normally distributed residuals yields the optimal ranker. This is likely because the residuals in these datasets closely follow a normal distribution, and estimating the PDF using a normal distribution captures the underlying distributional information of the residuals. However, for the Yahoo dataset, the min-max transformation of the residuals yields the optimal ranker. This can be due to the fact that the residuals in this dataset do not follow a normal or approximately normal distribution, making the use of a normal distribution for approximation less effective for capturing their distributional information.
% \item The Oracle method, trained using human-annotated labels and serving as an upper bound for rankers, achieves the best results. This implies that there is still scope for further performance improvements in unbiased learning to rank.
% \end{itemize}
% Another contributing factor could be the ranking algorithm used in any correction method. If the ranking algorithm is weaker than LambdaMART, and the performance improvements are minimal with bias correction, the method may still fail to outperform LambdaMART, which does not account for position bias. For instance, the PIJD method employs a linear model as the ranking algorithm to satisfy its assumptions, which can result in lower performance compared to LambdaMART.
% Summaries of our findings:
% \begin{itemize}[leftmargin=9pt, nosep]
% \item Our method outperforms all baselines on the Yahoo dataset. CFC achieves percentage improvements in $ERR@10$ of $1.02\%$, $1.12\%$, $1.36\%$, and $0.40\%$ over the best baselines under position bias severity levels of $\eta = 0.5$, $1.0$, $1.5$, and $2.0$, respectively.
% \item On the MSLR-WEB10k dataset, CFC outperforms the best baselines, improving $ERR@10$ by $9.83\%$, $1.92\%$, $0.72\%$, and $0.72\%$ at position bias severity levels of $\eta = 0.5$, $1.0$, $1.5$, and $2.0$, respectively.
% \item Our method shows enhanced performance on the Istella-S dataset. CFC increases $ERR@10$ by $3.19\%$, $1.43\%$, $0.43\%$, and $0.44\%$ relative to the strongest baselines at position bias severity levels of $\eta = 0.5$, $1.0$, $1.5$, and $2.0$, respectively.
% \item The LambdaMART method, which does not account for position bias, sometimes outperforms certain bias correction methods, particularly when the severity of position bias is low. This may occur due to a mismatch between the propensity estimation and the actual click bias in the correction method. When position bias is lower, the correction method may overestimate the click bias, leading to suboptimal performance compared to LambdaMART. Another contributing factor could be the ranking algorithm used in any correction method. If the ranking algorithm is weaker than LambdaMART, and the performance improvements are minimal with bias correction, the method may still fail to outperform LambdaMART, which does not account for position bias. For instance, the PIJD method employs a linear model as the ranking algorithm to satisfy its assumptions, which can result in lower performance compared to LambdaMART.
% \item The Oracle method, which represents the upper bound of any ranking model, exhibits a flat line across varying levels of position bias severity. This is because it is trained on true relevance data, which eliminates the impact of position bias that would otherwise arise from using click data.
% \end{itemize}
% Summaries of our findings:
% \begin{itemize}[leftmargin=9pt, nosep]
% \item Our method demonstrates improved performance on the Yahoo dataset at different noise levels. CFC shows percentage improvements in $ERR@10$ of $6.38\%$, $7.18\%$, $6.84\%$, and $5.15\%$ over the best baselines for noise levels $5\%$, $10\%$, $15\%$, and $20\%$, respectively.
% \item On the MSLR-WEB10k dataset, our method shows improved performance for different noise levels. CFC achieves improvements in $ERR@10$ of $0.73\%$, $0.83\%$, $0.29\%$, and $0.15\%$ compared to the best baselines under $5\%$, $10\%$, $15\%$, and $20\%$ noisy clicks, respectively.
% \item Across multiple noise levels, our method demonstrates enhanced performance on the Istella-S dataset. CFC improves $ERR@10$ by $1.32\%$, $0.45\%$, $0.66\%$, and $0.22\%$ over the best baselines at $5\%$, $10\%$, $15\%$, and $20\%$ noisy clicks, respectively.
% \item Because the Oracle method uses true relevance for training and not click data, it does not have any effect from noisy clicks. As a result, its performance remains constant.
% \end{itemize}
\section{Discussion} \label{sec:discussions}
% The experimental results highlight several key takeaways. First, the CFC method with LambdaMART consistently outperforms all baselines across the three benchmark datasets, including both LambdaMART and DNN-based methods, under both true relevance and biased click validation settings. CFC with DNN also surpasses all other DNN-based baselines on all datasets, except in some cases on the Istella-S dataset where MULTR performs slightly better. Second, CFC shows superior robustness under severe position bias, high click sparsity, and varying levels of click noise. Third, its performance improves with increased volumes of click data, indicating a favorable bias-variance tradeoff. Fourth, CFC achieves comparable performance even when using only a limited subset of annotated validation data, and remains agnostic to the choice of model used for estimating residual variation. Taken together, these findings affirm the reliability and adaptability of CFC in realistic, biased learning-to-rank settings.
The experimental results highlight several key takeaways. First, the CFC method outperforms various baselines across the three benchmark datasets under both true relevance and biased click validation settings. Second, it corrects for position bias with both LambdaMART and DNN rankers, highlighting its flexibility and general applicability. Third, CFC demonstrates robustness under varying levels of position bias severity, sparse click data, and noise. Fourth, its performance improves with more click data, reflecting a favorable bias-variance tradeoff. Finally, CFC maintains comparable performance even with limited annotated validation data and is agnostic to the choice of residual estimation model. While we expected to see better results with kernel-based transformations of the residuals, future work could explore better tuning of these transformations, as well as other parametric and nonparametric transformation techniques.
%The CFC method corrects for bias at the feature level without modifying the ranker’s loss function, which can enable its integration into other bias correction frameworks for joint debiasing. For example, it can be extended to jointly address position bias and selection bias, which we will explore as a future research direction. A limitation of our method is that we do not have an exclusion restriction or an external instrument in the first stage to satisfy the standard control function identification strategy. Instead, as described previously, we follow Lewbel et al.~\cite{lewbel-jbec12} and exploit heteroskedasticity in the residuals of the ranking process as a source of exogenous variation in the first stage. While for the residual transformation in the first stage we assume normally distributed residuals for two parametric transformations and apply one nonparametric transformation, future work can also explore other parametric forms and more advanced nonparametric or kernel-based transformation techniques.
Our work has broader societal implications. When recommender systems are trained on biased historical data, they risk perpetuating unfair outcomes, particularly for underrepresented users or content~\cite{bojic-futures24,wu-tkdd23}. While our work is a step towards promoting more equitable outcomes, it does not put any guardrails on how this system is used in practice. It is very important that practitioners mitigate any possible negative effects on society by deploying recommender systems responsibly, with careful attention to fairness, privacy, and broader principles of trustworthiness~\cite{fan-sigkdd23}.
%Bias can arise from historical data that reflect societal prejudices~\cite{wu-tkdd23}, and training recommender systems on such data may propagate unfair treatment of underrepresented users or content~\cite{bojic-futures24}. Therefore, mitigating these biases is essential for promoting fairness and preventing marginalization. Bias correction strategies can help ensure equitable treatment and prevent the marginalization of underrepresented users and content~\cite{bojic-futures24}. To combat potentially negative consequences arising from recommender systems, it is important that practitioners use these systems in a way that benefits society, respecting fairness, privacy and other aspects of trustworthiness~\cite{fan-sigkdd23}.
\section{Conclusion}
% In this paper, we introduce a new control function-based method to correct for position bias in LTR systems. Our approach models system's previous document ranking process to capture the residua. We refer to the residuals of this control function as correction terms for position bias, which captures the unobserved factors influencing a document's rank for a given query. We incorporate this query-document-specific residual in the ranking model to account for position bias. We also develop a method to debias validation clicks using click data for hyperparameter tuning. Our evaluation on benchmark datasets shows that CFC corrects for position bias and outperforms state-of-the-art methods, while also being robust to varying levels of bias severity and noisy clicks.
% By incorporating the residual variation from the system’s previous ranking process that contributes to position bias and incorporating it into the current ranking model, our method corrects for the bias.
% \modify{Future work can extend the control function approach to jointly address position and selection bias by modeling both document selection and ranking processes, and explore advanced nonparametric residual transformations and tuning the bandwidth of these nonparametric transformations.}
In this paper, we introduce a new control function-based method to correct for position bias in LTR systems. By incorporating the residual variation—which captures the unobserved ranking factors responsible for position bias—from the previous ranking process into current click model, we correct for that bias.
Our method requires no propensity estimation or click model assumptions. It also avoids modifications to the ranker's loss function, making it applicable for bias correction with any state-of-the-art LTR method. Another innovation of this study is that we also propose a residual-based click debiasing approach for hyperparameter tuning in the absence of annotated validation data. Our evaluation shows that CFC outperforms state-of-the-art methods, while also being robust to varying levels of position bias severity and noisy clicks. The CFC method is flexible and can be extended to incorporate more biases encountered in LTR systems, such as selection bias and trust bias.
% \input{2-related-works}
\section{More Analysis of the CFC Method}
\subsection{CFC algorithm} \label{cfc_algorithm}
The complete CFC algorithm is provided below.
% \vspace{-2.2cm}
\begin{algorithm}
\caption{CFC method for position bias correction}
\label{alg:cfc_bias_correction}
\begin{algorithmic}[1]
\Require Features set $\bm{X}$, document clicks indicators $C$, and document rankings $\pi$
\Ensure An unbiased ranking model
\State Train a document ranking model $m()$ using Eq.~\eqref{eq:control_function}
\State Compute the residuals ($\hat{\epsilon}^{y}_{q, m}$) for each query-document pair using Eq.~\eqref{eq:reqiduals}
% \State Define residual transformations:
% \begin{itemize}
\State $T_1 \leftarrow$ apply min-max normalization of residuals
\State $T_2 \leftarrow$ apply parametric IMR transformation of residuals
\State $T_3 \leftarrow$ apply parametric PDF transformation of residuals
\State $T_4 \leftarrow$ apply nonparametric IMR transformation of residuals
% \end{itemize}
\State Initialize the minimum loss: $\text{min\_loss} \leftarrow \infty$
\State Initialize the best model: $\text{best\_model} \leftarrow NULL$
\For{each transformation $T_i$ (from $T_1$ to $T_4$)}
\State Train a ranking model $g()$ with both clicked and unclicked documents and the transformed residuals $T_i$ using Eq.~\eqref{eq:unbiased_c_final}
\State Estimate the ranking loss $\hat{\bm{\mathcal{L}}}$ on validation data for the trained model $g()$
\If{$\hat{\bm{\mathcal{L}}} < \text{min\_loss}$}
\State $\text{min\_loss} \leftarrow \hat{\bm{\mathcal{L}}}$
\State $\text{best\_model} \leftarrow g()$
\EndIf
\EndFor
\State \Return best\_model
\end{algorithmic}
\end{algorithm}
\subsection{Computational complexity analysis} \label{computational_complexity}
The CFC method involves training the previous ranking process $m()$, estimating residuals, and applying various transformations to the residuals. Computing residuals using $m()$ requires $O(n)$ time, where $n$ is the total number of documents. Min-max normalization, parametric PDF transformation, and parametric hazard ratio transformation can all be computed using closed-form solutions, each with a runtime of $O(n)$. The generalized nonparametric hazard ratio using KDE transformation can be estimated in $O(n + p \log p)$ or $O(n + p)$~\cite{heer-vis21}, depending on the method, where $p$ is the bin size, typically $256$ or $512$. Since LTR datasets contain millions of documents, where $p \ll n$, this step can be approximated as $O(n)$. Thus, estimating residuals and applying various transformations collectively require $O(n)$, making it linear. The runtime for training the ranking process $m()$ and the current ranking model $g()$ depends on the specific algorithm used. Using efficient algorithms (e.g., Ridge, LambdaMART) for both can keep the overall runtime manageable, making it suitable for large-scale applications.
\subsection{Data requirements of our method} \label{sec:data_requirements}
% The assumptions behind the CFC method are that document positions are influenced by observable features but are not fully explained by them in the previous ranking process. By modeling the unexplained parts (residuals) and incorporating them into the click equation, the method corrects for potential confounding or endogeneity arising from the previous ranking process and enables unbiased relevance estimation under standard control function theory. The method assumes that position is the only omitted variable influencing clicks, and hence, its effect is captured in the error term of the click equation. We make this assumption because our current work aims to address position bias. As a result, we incorporate the residuals from the control function, which estimates document rank positions in queries, removes the correlation between the error term and the observed features in the final outcome click equation.
Regarding the data requirements of our method, in addition to clicks $(C^{y}_{q})$ and feature vectors $\bm{x}$ for query-document pairs, our method requires the document rankings ($r(y|\pi_q)$) that were presented to users and from which the current click data were generated. These rankings are readily available, as users are presented with ranked lists for each query, from which document positions for queries can be extracted. The document rankings are required in the first stage to model the previous ranking process $m()$. Importantly, our method does not require randomized exposures, estimated propensity scores, user browsing assumptions, or linearity assumptions, making it more practical and compatible with existing logged data.
% \subsection{Discussions about the CFC method} ~\label{discussions}
% For propensity-based methods~\cite{ai-sigir18, hu-www19, vardasbi-cikm20, zhu-recsys20, luo-wsdm23, luo-sigir24}, estimating correct propensity scores is challenging for highly sparse clicks or for lower-ranked documents that typically receive few or no clicks. On the other hand, we estimate residuals from the document ranking process to account for position bias, and thus, we do not require clicks to estimate the correction terms for position bias. Moreover, propensity-based methods estimate propensity only for clicked documents, overlooking unclicked yet relevant documents of lower rankings~\cite{tian-sigir20}. In contrast, we estimate residuals for both clicked and unclicked documents and incorporate them into learning the ranking model. Furthermore, click modeling methods~\cite{chapelle-www09, craswell-wsdm08, dupret-sigir08, wang-www13} require assumptions about the user browsing model. In contrast, we do not rely on any such assumptions in our method.
\subsection{Difference with Heckman method} \label{sec:heckman_diff}
Although the Heckman method is a specific subset of control function techniques, its assumptions and applications differ significantly from other approaches within this family, as do many of the econometric techniques that fall within the control function umbrella.
% For example, Instrumental Variable (IV) methods, also a control function method, have distinct assumptions and proofs for statistical unbiasedness that are not interchangeable with the Heckman two-step. \modify{For the same reason, control function methods are also not interchangeable with the Heckman method.}
While we acknowledge works~\cite{ovaisi-www20, ovaisi-sigir21} for their contributions to Heckman-based bias corrections, our methodology extends beyond these models. The assumptions needed for unbiased estimation of feature parameters within a general control function approach are less stringent than for the specific sub-case of Heckman-based methods. The Heckman model relies on linearity assumptions in both stages for unbiased estimation of feature parameters, which makes it highly unsuitable for LTR tasks that involve high-dimensional, non-linear feature interactions, and highly sparse clicks. On the other hand, our control function approach, which leverages the residual variation from the modeled document ranking process, does not require linearity assumptions in either stage and can handle multicollinearity in high-dimensional feature spaces by selecting flexible models with regularization. Moreover, Heckman-based methods use the same features $\bm{x}$ in both stages and estimate the IMR of the first-stage predictions based on features to include it as an additional variable in the second stage. This contributes to the already high likelihood of multicollinearity in a high-dimensional feature space in the second stage. Our CFC method avoids this issue by using residuals, which are orthogonal to the features, thereby evading this multicollinearity.
\subsection{Difference with click models} \label{sec:click_model_diff}
Basic click models ~\cite{chapelle-www09, craswell-wsdm08, mao-sigir18} typically aim to separate the effects of document position and document relevance (feature effects) and use the relevance component for an unbiased ranking model. Like click models, we can also separate the unbiased feature effects. In equation~\eqref{eq:unbiased_c_final}, we model the click data as dependent on the features and the transformed residual. To separate the feature effects on clicks, we need to partial out the effect of the transformed residual from clicks. To achieve this, we can consider $T(\hat{\epsilon}^{y}_{q,m})$ as the only independent variable affecting clicks to learn a model. The estimated residual from this model will capture the feature effects, which can be used as document relevance. However, extracting document relevance by separating the effect of $T(\hat{\epsilon}^{y}_{q,m})$ from clicks or directly incorporating $T(\hat{\epsilon}^{y}_{q,m})$ into the ranking model to obtain an unbiased ranking model is statistically equivalent. Since separating the effect of $T(\hat{\epsilon}^{y}_{q,m})$ requires training an additional model, we directly use equation~\eqref{eq:unbiased_c_final} to obtain the unbiased ranking model $g()$.
% \textbf{Flexibility of our method.} Since the CFC method does not require modifying the loss function of the ranker $g()$, it can be used for position bias correction with any pointwise, pairwise, or listwise LTR algorithm. Moreover, the CFC method operates at the feature level, which enables its integration into other optimization frameworks for joint optimization of multiple objectives. This allows the CFC method to integrate with other bias correction techniques to address multiple biases simultaneously, which we will explore as a future research direction.
% \textbf{Future extensions.} \modify{The current residual transformation strategy uses standard parametric methods such as the normal distribution. It also considers non-parametric approaches like kernel density estimation. Future work can explore other parametric forms, including log-normal, Weibull, and exponential distributions. Future work can also explore more advanced non-parametric techniques and tuning the bandwidth in these methods. These may lead to more flexible residual transformation and improved ranking accuracy. Future research can explore control function methods to jointly address multiple biases—such as selection bias, where only a subset of relevant documents is shown for a query—with position bias in LTR systems by simultaneously modeling the document selection and ranking processes.}
\begin{table*}[ht]
% \small
\fontsize{7.8}{8}\selectfont
% \setlength{\belowcaptionskip}{5pt}
\captionsetup{justification=raggedright, width=1.0\textwidth}
\caption{Dataset descriptions.}
\centering
\begin{tabular}{ccccccccc}
\toprule
Dataset & \makecell{Train \\ queries} & \makecell{Train \\ docs.} & \makecell{Valid \\ queries} & \makecell{Valid \\ docs.} & \makecell{Test \\ queries} & \makecell{Test \\ docs.} & \makecell{Avg. docs. \\ per query} & Feature \\
\midrule
Yahoo (Set1) & 19,944 & 473,134 & 2,994 & 71,083 & 6,983 & 165,660 & 24 & 700 \\
\makecell{MSLR-WEB10k} & 6,000 & 723,412 & 2,000 & 235,260 & 2,000 & 241,521 & 121 & 136 \\
Istella-S & 19,245 & 2,043,304 & 7,211 & 684,076 & 6,562 & 681,250 & 103 & 220 \\
\bottomrule
\end{tabular}
\label{tab:dataset_description}
% \vspace{-2em}
\end{table*}
\section{Experimental Details} \label{experimental_details}
\subsection{Datasets} \label{datasets_details}
We now provide detailed descriptions of the three benchmark datasets used for evaluation: Learning to Rank Challenge (C14B)\footnote{\label{yahoo_link}\url{https://webscope.sandbox.yahoo.com/catalog.php?datatype=c}}~\cite{chapelle-pmlr11}, MSLR-WEB10K\footnote{\label{mslr_link}\url{https://www.microsoft.com/en-us/research/project/mslr}}~\cite{qin-arXiv13}, and Istella-S\footnote{\label{istella_link}\url{http://quickrank.isti.cnr.it/istella-dataset}}~\cite{lucchese-sigir16}. Each dataset contains human-annotated relevance labels for each query-document pair $\in [0,4]$. In the Yahoo dataset, the feature values are provided in the normalized form $\in[0, 1]$. We use min-max normalization to normalize the features $\in[0, 1]$ for the MSLR-WEB10k and Istella-S datasets.
\begin{itemize}[leftmargin=10pt, nosep]
\item \textbf{Yahoo! Learning to Rank Challenge} dataset includes $19,944$ training queries with $473,134$ documents. It also contains $2,994$ validation queries with $71,083$ documents, and $6,983$ test queries with $165,660$ documents. On average, there are $24$ documents per query and each document has a $700$-dimensional feature vector.
\item \textbf{MSLR-WEB10k} dataset has five folds. We use fold-1 for our evaluations from the MSLR-WEB10k dataset. Fold-1 consists of $6,000$ training queries with $723,412$ documents, $2,000$ validation queries with $235,260$ documents, and $2,000$ test queries with $241,521$ documents. Each document has a $136$-dimensional feature vector with an average of $121$ documents per query.
\item \textbf{Istella-S} dataset consists of $19,245$ training queries with $2,043,304$ documents, $7,211$ validation queries with $684,076$ documents, and $6,562$ test queries with $681,250$ documents. There is an average of $103$ documents per query where each document has a $220$-dimensional feature vector.
\end{itemize}
We present a summary of the descriptions of these three datasets in Table \ref{tab:dataset_description}.
\subsection{Baselines} \label{baselines}
We compare our CFC method with baseline methods commonly used for correcting position bias in LTR systems.
% \begin{itemize}[label={}, leftmargin=0pt, nosep]
\begin{itemize}[leftmargin=9pt, nosep]
\item $\textbf{UPE:}$ The Unconfounded Propensity Estimation~\cite{luo-sigir24} method corrects position bias with a causal method using document relevance as a confounder and addresses propensity overestimation.
\item $\textbf{MULTR:}$ The Model-based Unbiased Learning to Rank~\cite{luo-wsdm23} method uses a context-aware simulator to generate pseudo clicks and addresses the discrepancy between pseudo and actual clicks through IPW with pseudo labels in a doubly robust manner.
\item $\textbf{PIJD:}$ The Propensity-Independent Joint Debiasing~\cite{ovaisi-sigir21} corrects for both position and selection bias using Heckman's method.
\item $\textbf{PairD:}$ The Pairwise Debiasing~\cite{hu-www19} model extends the IPW that jointly estimates position bias and trains a pairwise ranker.
\item $\textbf{PAL:}$ The Position-bias Aware Learning~\cite{guo-recsys19} framework uses a two-tower model to disentangle relevance and bias factors.
\item $\textbf{REM:}$ The regression-based EM~\cite{wang-wsdm18} model uses a position bias click model to estimate propensity scores and
trains a ranker.
\item $\textbf{DLA:}$ The Dual Learning Algorithm~\cite{ai-sigir18} jointly learns an unbiased propensity model and unbiased ranker from biased click data.
\item $\textbf{LambdaMART:}$ Using LambdaMART~\cite{burges-learning10} as the LTR algorithm, we train a model directly on biased click data without any position bias correction. This serves as a baseline benchmark to assess whether our CFC method, which also uses the LambdaMART LTR algorithm as its ranker, can outperform a LambdaMART algorithm that does not correct for position bias.
\item $\textbf{DNN:}$ We train a Deep Neural Network model directly on biased click data, following the implementation in~\cite{ai-sigir18}, without applying any position bias correction. This baseline allows us to evaluate whether our CFC method, which also uses a DNN ranker, can surpass this method that does not account for position bias.
\item $\textbf{Oracle:}$ This model is trained using human-annotated labels. Because it relies on the true relevance of each query-document pair, its performance represents the upper bound of the ranker.
\end{itemize}
\subsection{Difficulty of online evaluation} \label{online_eval}
Online evaluation, though often expected, is costly, hard to implement in academic settings, and requires real-time user participation in an existing LTR system. Moreover, in unbiased LTR, unbiased evaluation of methods requires unbiased ground truth or true relevance of documents, which is typically not possible to obtain in real-world data. As a result, unbiased LTR methods depend on benchmark datasets for unbiased evaluation, where true relevance is provided. Following state-of-the-art approaches, we evaluate our method using benchmark datasets and synthetic click data for all experiments. Using the synthetic click generation process, we can simulate clicks with varying experimental conditions that mimic real-world scenarios and user behaviors, including different degrees of position bias, numbers of clicks, and levels of click noise. By doing so, we evaluate the performance and robustness of our method across various scenarios, ensuring that the evaluation aligns with real-world challenges rather than favoring our approach.
\subsection{More implementation details} \label{implementation_details}
In the first stage of our method, as formulated in equation~\eqref{eq:control_function}, we train $m()$ using a supervised machine learning algorithm. For learning $m()$, we explore various machine learning models, such as Ridge, Logistic Regression, SVM, and XGBoost. While all four classifiers used to train \( m() \) yield comparable ranking performance on \( g() \) when residuals from \( m() \) are incorporated into the ranker, we observe that the Ridge classifier generally leads to better ranking performance on \( g() \) across all three datasets. Therefore, we report all results using Ridge to learn \( m() \) in the first stage and incorporate the residuals from this \( m() \) into \( g() \). The impact of using different machine learning models to estimate $m()$ is discussed in detail in RQ7. For nonparametric hazard ratio transformation of the first-stage residuals, we use fastKDE~\cite{o-csda16} to estimate the PDF.
% We cannot determine whether the estimated residuals from $m()$, trained using a particular machine learning algorithm, properly capture the unobserved factors until we incorporate those residuals into ranker $g()$ and evaluate the performance of $g()$.
% % If $m()$ is overfitted, it may achieve near-zero training error, leaving no residuals to correct for position bias, even when such bias is present in the data.
% Thus, we select $m()$ based on how well the ranker $g()$ performs when incorporating the residuals from that $m()$ into $g()$. While all four classifiers we explore to train $m()$ provide comparable performance on ranker $g()$, we observe that Ridge Classifier generally yields better ranking performance on $g()$ across all three datasets.
% This is likely due to the high-dimensional feature space in these LTR datasets, where features are often highly correlated. OLS, used in Heckman-based methods~\cite{ovaisi-www20, ovaisi-sigir21}, to meet their assumptions, cannot handle multicollinearity. In contrast, the Ridge Classifier tackles this by adding an $L_2$-norm to the loss function, allowing it to capture unobserved factors more effectively. As a result, using a Ridge Classifier in the control function yields improved ranking performance by providing more accurate residual estimates. Therefore, we use Ridge Classifier to learn $m()$.
For question RQ1, we obtain debiased clicks for hyperparameter tuning and learn the function $f()$ with Ridge Regression. We train $f()$ using training data and predict on validation data. Next, we calculate the debiased clicks ($\hat{\epsilon}^{y}_{q,f}$) for validation data using the predictions from $f()$. We use these debiased clicks for hyperparameter tuning when we assume that true relevance is unavailable for validation. As previously described, We define two CFC variants based on how we use clicks for hyperparameter tuning: CFC-DC, where the CFC method uses debiased clicks in hyperparameter tuning, and CFC-BC, where it uses biased clicks directly for hyperparameter tuning. For questions RQ2-RQ7, we use the true relevance of the validation dataset for hyperparameter tuning for both our method and the baselines.
\subsection{Training parameters} \label{training_paramters}
To facilitate reproducibility, we outline the training parameters for our method. For the LambdaMART ranker, we use the LightGBM~\cite{ke-neurips17} implementation, which is a pairwise LTR method. We choose $NDCG$ as the optimization metric while setting the minimum number of data points in a leaf node to $2$, and the maximum number of tree leaves to $255$. We keep the number of boosted trees and the learning rate as hyperparameters. The number of boosted trees is tuned between $100$ and $500$, while the learning rate $\alpha$ is tuned between $0.05$ and $0.1$. We leave all other parameters of LambdaMART at their default settings. For DNN-based ranker, we follow the implementation by Ai et al.~\cite{ai-sigir18} and integrate DNN with our CFC method. We train the model using a batch size of $256$ for $10$k steps, using the AdaGrad optimizer~\cite{duchi-jmlr11}. The neural network architecture consists of three hidden layers with sizes [$512$, $256$, $128$], each using the ELU activation function~\cite{clevert-iclr16}. We use softmax cross-entropy loss function, which is a listwise objective function. Additionally, as discussed in section~\ref{subsection_unbiased_ltr}, we have a hyperparameter for the transformations of the estimated residuals used in the ranker. We tune this hyperparameter on validation data to select the best ranking model $g()$, whether using LambdaMART or DNN as the ranker.
% We use all the default settings of the Ridge Classifier and Ridge Regression to train the models $m()$ and $f()$.
\subsection{Heteroskedasticity test on first-stage residuals} \label{sec:heteroskedasticity}
To validate the presence of heteroskedasticity in the first-stage residuals, we conduct nonparametric variance tests on the Yahoo, MSLR-WEB10K, and Istella-S datasets. Specifically, we use the Fligner–Killeen test~\cite{fligner-jasa76}, a nonparametric test that does not assume any specific distribution. Our results show statistically significant test statistics with $\bm{p}$-values very close to zero (approximately in the
$10^{-6} \sim 10^{-5}$ ranges) across all three datasets. Using a significance threshold of $\bm{p} < 0.05$, this provides strong evidence of heteroskedasticity in the first-stage residuals of these three benchmark LTR datasets. This finding supports the use of heteroskedastic residuals as a valid source of exogenous variation to correct for position bias in the second stage of our control function framework.
% {\renewcommand{\arraystretch}{1.0}
\begin{table*} [htbp]
% \scriptsize
% \setlength{\belowcaptionskip}{5pt}
\captionsetup{justification=raggedright, width=1.0\textwidth}
\caption{A detailed comparison between CFC and other methods across various datasets using click data for hyperparameter tuning. * denotes a statistically significant improvement of the best-performing CFC method over the best baseline methods with ${\bm{p < 0.05}}$. The best performance among all methods, excluding Oracle, is bolded, and the second-best is underlined.}
\centering
\begin{tabular}{ccccccccc}
% \cline{1-8}
\toprule
\multirow{2}{*}{Datasets} & \multirow{2}{*}{Ranker} & \multirow{2}{*}{Methods} & \multicolumn{3}{c} {ERR@p} & \multicolumn{3}{c}{NDCG@p} \\ \cmidrule(r){4-6} \cmidrule(r){7-9}
& & & $p=5$ & $p=10$ & $p=15$ & $p=5$ & $p=10$ & $p=15$ \\
% \cline{1-8}
\midrule
\multirow{16}[0]{*}{Yahoo} & \multirow{6}{*}{LambdaMART}
& CFC-DC & \textbf{0.444}${^*}$ & \textbf{0.460}${^*}$ & \textbf{0.463}${^*}$ & \textbf{0.756}${^*}$ & \textbf{0.794}${^*}$ & \textbf{0.816}${^*}$ \\
& & CFC-BC & 0.420 & 0.437 & 0.440 & 0.739 & 0.783 & 0.806 \\
& & PIJD & 0.438 & 0.452 & 0.457 & 0.740 & 0.780 & 0.803 \\
& & PairD & 0.437 & 0.452 & 0.456 & 0.745 & 0.785 & 0.808 \\
& & LambdaMART & 0.430 & 0.446 & 0.449 & 0.748 & 0.789 & 0.812 \\
& & Oracle & 0.460 & 0.474 & 0.477 & 0.785 & 0.817 & 0.837 \\
\cmidrule(r){2-9}
& \multirow{10}{*}{DNN}
& CFC-DC & \underline{0.439} & \underline{0.455} & \underline{0.458} & \underline{0.752} & \underline{0.792} & \underline{0.815} \\
& & CFC-BC & 0.428 & 0.444 & 0.448 & 0.742 & 0.784 & 0.809 \\
& & PIJD & 0.430 & 0.447 & 0.449 & 0.745 & 0.785 & 0.806 \\
& & UPE & 0.421 & 0.437 & 0.441 & 0.726 & 0.772 & 0.797 \\
& & MULTR & 0.437 & 0.452 & 0.456 & 0.749 & 0.789 & 0.812 \\
& & PAL & 0.425 & 0.441 & 0.444 & 0.739 & 0.782 & 0.807 \\
& & REM & 0.428 & 0.444 & 0.448 & 0.723 & 0.768 & 0.793 \\
& & DLA & 0.417 & 0.433 & 0.437 & 0.734 & 0.777 & 0.802 \\
& & DNN & 0.425 & 0.441 & 0.445 & 0.740 & 0.781 & 0.806 \\
& & Oracle & 0.449 & 0.464 & 0.467 & 0.769 & 0.805 & 0.827 \\
\midrule
% \cline{1-12}
\multirow{16}[0]{*}{\makecell{MSLR-\\WEB10k}} & \multirow{6}{*}{LambdaMART}
& CFC-DC & \textbf{0.287}${^*}$ & \textbf{0.307}${^*}$ & \textbf{0.313}${^*}$ & \textbf{0.474}${^*}$ & \textbf{0.482}${^*}$ & \textbf{0.492}${^*}$ \\
& & CFC-BC & 0.256 & 0.278 & 0.285 & 0.451 & 0.465 & 0.480 \\
& & PIJD & 0.255 & 0.275 & 0.284 & 0.422 & 0.438 & 0.456 \\
% \cline{2-8}
& & PairD & 0.263 & 0.285 & 0.292 & 0.451 & 0.463 & 0.477 \\
& & LambdaMART & 0.246 & 0.268 & 0.276 & 0.430 & 0.446 & 0.462 \\
% \cline{2-8}
& & Oracle & 0.323 & 0.342 & 0.348 & 0.495 & 0.498 & 0.507 \\
\cmidrule(r){2-9}
& \multirow{10}{*}{DNN}
& CFC-DC & \underline{0.271} & \underline{0.292} & \underline{0.299} & \underline{0.456} & \underline{0.465} & \underline{0.477} \\
& & CFC-BC & \underline{0.271} & 0.291 & 0.298 & \underline{0.456} & 0.464 & 0.476 \\
& & PIJD & 0.237 & 0.259 & 0.266 & 0.433 & 0.448 & 0.464 \\
& & UPE & 0.209 & 0.232 & 0.240 & 0.339 & 0.362 & 0.382 \\
& & MULTR & 0.270 & 0.291 & 0.298 & 0.438 & 0.451 & 0.466 \\
& & PAL & 0.230 & 0.253 & 0.260 & 0.419 & 0.436 & 0.453 \\
& & REM & 0.218 & 0.241 & 0.249 & 0.399 & 0.419 & 0.436 \\
& & DLA & 0.233 & 0.256 & 0.263 & 0.429 & 0.447 & 0.463 \\
& & DNN & 0.230 & 0.253 & 0.260 & 0.430 & 0.446 & 0.462 \\
& & Oracle & 0.301 & 0.325 & 0.328 & 0.469 & 0.476 & 0.487 \\
\midrule
% \cline{1-8}
\multirow{16}[0]{*}{Istella-S} & \multirow{6}{*}{LambdaMART}
& CFC-DC & \textbf{0.709}${^*}$ & \textbf{0.716}${^*}$ & \textbf{0.717}${^*}$ & \textbf{0.732}${^*}$ & \textbf{0.765}${^*}$ & \textbf{0.803}${^*}$ \\
& & CFC-BC & \underline{0.691} & \underline{0.698} & \underline{0.699} & \underline{0.715} & \underline{0.752} & \underline{0.791} \\
& & PIJD & 0.673 & 0.680 & 0.683 & 0.672 & 0.692 & 0.729 \\
& & PairD & 0.682 & 0.689 & 0.691 & 0.705 & 0.739 & 0.778 \\
& & LambdaMART & 0.682 & 0.690 & 0.691 & 0.700 & 0.733 & 0.773 \\
& & Oracle & 0.731 & 0.737 & 0.738 & 0.746 & 0.777 & 0.815 \\
\cmidrule(r){2-9}
& \multirow{10}{*}{DNN}
& CFC-DC & 0.670 & 0.679 & 0.680 & 0.681 & 0.710 & 0.749 \\
& & CFC-BC & 0.668 & 0.676 & 0.678 & 0.679 & 0.709 & 0.748 \\
& & PIJD & 0.663 & 0.672 & 0.673 & 0.676 & 0.706 & 0.746 \\
& & UPE & 0.662 & 0.671 & 0.673 & 0.663 & 0.690 & 0.729 \\
& & MULTR & 0.667 & 0.675 & 0.676 & 0.667 & 0.691 & 0.727 \\
& & PAL & 0.660 & 0.669 & 0.670 & 0.675 & 0.708 & 0.748 \\
& & REM & 0.644 & 0.653 & 0.655 & 0.657 & 0.685 & 0.724 \\
& & DLA & 0.668 & 0.676 & 0.678 & 0.677 & 0.706 & 0.745 \\
& & DNN & 0.660 & 0.669 & 0.670 & 0.674 & 0.705 & 0.744 \\
& & Oracle & 0.696 & 0.702 & 0.704 & 0.711 & 0.745 & 0.784 \\
\bottomrule
\end{tabular}
\label{tab:main_results_with_biased_clicks_full_version}
% \vspace{-0.5cm}
\end{table*}
% }
\begin{figure*} [ht]
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-dnn.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-severity-dnn.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-severity-dnn.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-severity-dnn.pdf}
\caption{Istella-S}
\end{subfigure}
% \vspace{-0.3cm}
\captionsetup{width=0.98\textwidth}
\caption{The performance of different methods with varying levels of position bias severity using DNN as the ranker.\label{fig:bias_severity-dnn}}
% \vspace{-0.7cm}
\end{figure*}
\begin{figure*}
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-dnn.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-passes-dnn.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-passes-dnn.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-passes-dnn.pdf}
\caption{Istella-S}
\end{subfigure}
% \vspace{-0.4cm}
\captionsetup{width=0.98\textwidth}
\caption{The performance of different methods with varying the number of passes for clicks generation using DNN as the ranker.\label{fig:passes_variations-dnn}}
% \vspace{-.3cm}
\end{figure*}
\subsection{Additional experimental results} \label{additional_results}
\textbf{Detailed results for RQ1}. For question RQ1, we assume that true relevance labels are unavailable for the validation data and instead use click data for hyperparameter tuning. In Table~\ref{tab:main_results_with_biased_clicks_short_version}, we report the results based on $ERR@10$ and $NDCG@10$. Detailed results for $p = 5$, $10$, and $15$ are provided here in Table~\ref{tab:main_results_with_biased_clicks_full_version}. The CFC-DC method with LambdaMART, which uses debiased clicks for hyperparameter tuning, consistently outperforms all baselines across all datasets and all values of \( p \), whether using LambdaMART or DNN
ranker. Among the DNN-based methods, CFC-DC with DNN achieves the best performance across all values of \( p \) on all three datasets. While CFC-DC with DNN is the second-best method on the Yahoo and MSLR-WEB10K datasets, CFC-BC with LambdaMART outperforms it for all values of $p$ on the Istella-S dataset, likely due to LambdaMART’s stronger ranking capability. However, CFC-DC with DNN still achieves the best performance among DNN-based methods on the Istella-S dataset for all values of $p$.
\begin{figure*} [ht]
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-dnn.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-noisy-dnn.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-noisy-dnn.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-noisy-dnn.pdf}
\caption{Istella-S}
\end{subfigure}
\captionsetup{width=0.98\textwidth}
\caption{The performance of different methods up to $\bm{10 \ (p)}$ positions with varying percentages of noisy clicks using DNN as the ranker.\label{fig:noisy_clicks-dnn}}
% \vspace{-.3cm}
\end{figure*}
\begin{figure*}
\centering
\captionsetup{justification=raggedright, margin=0cm}
\captionsetup[subfigure]{labelfont=normalfont, textfont=normalfont}
\begin{subfigure}{0.98\textwidth}
\includegraphics[width=\linewidth]{figures/legends-bold-dnn.png}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Yahoo-Set1-validation-dnn.pdf}
\caption{Yahoo}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/MSLR-WEB10k-validation-dnn.pdf}
\caption{MSLR-WEB10k}
\end{subfigure}
\begin{subfigure}{\figsize\textwidth}
\includegraphics[width=\linewidth]{figures/Istella-S-validation-dnn.pdf}
\caption{Istella-S}
\end{subfigure}
\captionsetup{width=0.98\textwidth}
% \caption{The performance of different methods for up to $\bm{10 \ (p)}$ positions across various datasets with varying numbers of validation queries.\label{fig:validation_variety}}
\caption{The performance of different methods up to $\bm{10 \ (p)}$ positions with varying numbers of validation queries using DNN as the ranker.\label{fig:validation_variety-dnn}}
% \vspace{-.7cm}
\end{figure*}
% \textbf{More results for RQ3}.
% For question RQ3, we evaluate the performance of the CFC method by varying the number of passes used to generate clicks. We present the results using LambdaMART as the ranker for the \( \text{ERR@10} \) metric in the main body of the paper (Figure~\ref{fig:passes_variations-lm}). Here, we show the corresponding results for the DNN-based ranker in Figure~\ref{fig:passes_variations-dnn}. As the number of clicks increases, CFC with DNN shows performance improvements, outperforming all other DNN-based baselines on the Yahoo and MSLR-WEB10K datasets—except at pass $1$ on MSLR-WEB10K dataset. On the Istella-S dataset, CFC with DNN outperforms all other DNN-based methods, with the exception of MULTR.
\textbf{More results for RQ3}. For question RQ3, we evaluate the performance of the CFC method by varying the position bias severity
\( \eta \). We present the results using LambdaMART as the ranker for the
\( ERR@10 \) metric in the main body of the paper (Figure~\ref{fig:bias_severity-lm}). The corresponding results for the DNN-based ranker are shown here in Figure~\ref{fig:bias_severity-dnn}. Across different values of \( \eta \), CFC with DNN outperforms all other DNN-based baselines on the Yahoo and MSLR-WEB10K datasets, except at \( \eta = 2.0 \) on MSLR-WEB10K dataset. For the Istella-S dataset, CFC with DNN provides the best results among DNN-based methods at the bias severity (\( \eta = 0.5 \)), but is outperformed by MULTR at other bias levels.
\textbf{More results for RQ4}. For question RQ4, we evaluate the performance of the CFC method by varying the amount of click data (passes). We present the results using LambdaMART as the ranker for the
\( ERR@10 \) metric in the main body of the paper (Figure~\ref{fig:passes_variations-lm}). The corresponding results for the DNN-based ranker are shown here in Figure~\ref{fig:passes_variations-dnn}. CFC with DNN shows improved performance as the number of clicks increases, outperforming all DNN-based baselines on the Yahoo and MSLR-WEB10K datasets, except at pass $1$ on MSLR-WEB10K. CFC with DNN achieves the second-best performance on the Istella-S dataset, surpassed only by MULTR.
\textbf{More results for RQ5}. For question RQ5, we evaluate the robustness of our method against noisy clicks by varying the probability of clicking on irrelevant documents. We present the results using LambdaMART as the ranker for the
\( ERR@10 \) metric in the main body of the paper (Figure~\ref{fig:noisy_clicks-lm}). The corresponding results for the DNN-based ranker are shown here in Figure~\ref{fig:noisy_clicks-dnn}. CFC with DNN consistently outperforms all other DNN-based methods on the Yahoo and MSLR-WEB10K datasets, with the exception of a marginally lower performance compared to MULTR under $15\%$ noisy clicks on MSLR-WEB10K dataset. On the Istella-S dataset, CFC with DNN ranks second, with only MULTR performing better.
\textbf{More results for RQ6}. For question RQ6, we analyze how the number of queries with true relevance in the validation data affects the performance of different methods. We present the results using LambdaMART as the ranker for the
\( ERR@10 \) metric in the main body of the paper (Figure~\ref{fig:validation_variety-lm}). The corresponding results for the DNN-based ranker are shown here in Figure~\ref{fig:validation_variety-dnn}. CFC with DNN achieves the best results among DNN-based methods on Yahoo and MSLR-WEB10K datasets across different numbers of queries used for hyperparameter tuning. On Istella-S dataset, CFC is outperformed only by MULTR.
% % {\renewcommand{\arraystretch}{1.0}
% \begin{table*} [hb]
% % \setlength{\tabcolsep}{4.0pt}
% % \scriptsize
% % \fontsize{7.8}{8}\selectfont
% \captionsetup{justification=raggedright, width=1.0\textwidth}
% \caption{The performance of the CFC method using LambdaMART as the ranker is compared with DirectRank, CFC\&DirectRank and LambdaMART. DirectRank method directly includes the document's rank variable ($r(y|\pi_q)$) as an additional feature in the ranking model instead of using residuals from document ranking process. CFC\&DirectRank method includes both the document's rank variable ($r(y|\pi_q)$) and the residuals from document ranking process as an additional features in the ranking model. For each ranker, the best-performing method is bolded, and the second-best is underlined.}
% \centering
% \scalebox{1.0}{%
% \begin{tabular}{ccccccccccc}
% \toprule
% \multirow{2}{*}{Datasets} & \multirow{2}{*}{Metrics} & \multicolumn{4}{c} {LambdaMART} & \multicolumn{4}{c}{DNN} \\ \cmidrule(r){3-6} \cmidrule(r){7-10}
% & & CFC & DirectRank & CFC\&DirectRank & LambdaMART & CFC & DirectRank & CFC\&DirectRank & DNN \\
% \midrule
% \multirow{2}[0]{*}{Yahoo}
% & ERR@10 & \textbf{0.462} & 0.455 & \underline{0.456} & 0.447 & \underline{0.456} & \textbf{0.458} & 0.453 & 0.444 \\
% & NDCG@10 & \textbf{0.796} & 0.776 & \underline{0.791} & 0.786 & \underline{0.792} & \textbf{0.794} & 0.777 & 0.782 \\
% \midrule
% \multirow{2}[0]{*}{\makecell{MSLR-\\WEB10k}}
% & ERR@10 & \textbf{0.317} & 0.287 & \underline{0.291} & 0.270 & \underline{0.306} & 0.294 & \textbf{0.312} & 0.256 \\
% & NDCG@10 & \textbf{0.486} & 0.416 & 0.414 & \underline{0.457} & \underline{0.466} & 0.453 & \textbf{0.478} & 0.448 \\
% \midrule
% \multirow{2}[0]{*}{Istella-S}
% & ERR@10 & \textbf{0.714} & 0.702 & 0.694 & \underline{0.694} & 0.683 & 0.694 & 0.645 & 0.672 \\
% & NDCG@10 & \textbf{0.765} & 0.715 & 0.740 & 0.740 & 0.712 & 0.728 & 0.708 & 0.705 \\
% \bottomrule
% \end{tabular}
% }
% \label{tab:cfc_without_residuals_but_ranking_and_with_both}
% \end{table*}
% % }
% \begin{table*}[ht]
% % \setlength{\belowcaptionskip}{5pt}
% \captionsetup{justification=raggedright, width=1.0\textwidth}
% \caption{The performance of the CFC method using LambdaMART as the ranker is compared with DirectRank, CFC\&DirectRank and LambdaMART. DirectRank method directly includes the document's rank variable ($r(y|\pi_q)$) as an additional feature in the ranking model instead of using residuals from document ranking process. CFC\&DirectRank method includes both the document's rank variable ($r(y|\pi_q)$) and the residuals from document ranking process as an additional features in the ranking model.}
% \centering
% \begin{tabular}{ccccccccc}
% \toprule
% \multirow{2}{*}{Datasets} & \makecell{CFC} & \makecell{DirectRank} & \makecell{CFC\&DirectRank} & \makecell{LambdaMART} \\ \cmidrule(r){2-5} & \multicolumn{4}{c} {$ERR@10$} & \multicolumn{4}{c}{NDCG@10} \\ \cmidrule(r){4-6} \cmidrule(r){7-9} \\
% \midrule
% Yahoo & \textbf{0.462} & \underline{0.455} & - & 0.446 \\
% \makecell{MSLR-WEB10k} & \textbf{0.317} & \underline{0.287} & - & 0.270 \\
% Istella-S & \textbf{0.714} & \underline{0.702} & - & 0.694 \\
% \bottomrule
% \end{tabular}
% \label{tab:cfc_without_residuals_but_ranking}
% \end{table*}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
% \title{Appendix: Correcting Position Bias in Learning to Rank: \\A Control Function Approach}
% \begin{center}
% \Huge\textbf{Appendix}
% \end{center}
% \FloatBarrier
% \appendix
% \section*{\hspace{10pt} \Huge {Appendix}}
% \vspace{10pt}
% \section{Dataset Description} \label{sec:datasets}
% We now provide detailed descriptions of the three benchmark datasets that we use for evaluation purposes.
% \begin{itemize}[leftmargin=11pt, nosep]
% \item \textbf{Yahoo! Learning to Rank Challenge} dataset includes $19,944$ training queries with $473,134$ documents. It also contains $2,994$ validation queries with $71,083$ documents, and $6,983$ test queries with $165,660$ documents. On average, there are $24$ documents per query and each document has a $700$-dimensional feature vector.
% \item \textbf{MSLR-WEB10k} dataset has five folds. We use fold-1 for our evaluations from the MSLR-WEB10k dataset. Fold-1 consists of $6,000$ training queries with $723,412$ documents, $2,000$ validation queries with $235,260$ documents, and $2,000$ test queries with $241,521$ documents. Each document has a $136$-dimensional feature vector with an average of $121$ documents per query.
% \item \textbf{Istella-S} dataset consists of $19,245$ training queries with $2,043,304$ documents, $7,211$ validation queries with $684,076$ documents, and $6,562$ test queries with $681,250$ documents. There is an average of $103$ documents per query where each document has a $220$-dimensional feature vector.
% \end{itemize}
% We present a summary of the descriptions of these three datasets in Table \ref{tab:dataset_description}.
% \section{Training Parameters} \label{sec:training_params}
% To facilitate the reproducibility, we describe the different training parameters associated with our method. For LambdaMART implementation as the ranker $g()$, we use the implementation provided by the open-source LightGBM module~\cite{ke-neurips17}. We choose Normalized Discounted Cumulative Gain ($NDCG$) as the optimization metric while setting the minimum number of data points in a leaf node to $2$, and the maximum number of tree leaves to $255$. We keep the number of boosted trees and the learning rate as hyperparameters. The number of boosted trees is tuned between $100$ and $500$, while the learning rate $\alpha$ is tuned between $0.05$ and $0.1$. We leave all other parameters of LambdaMART at their default settings. Note that there is an additional hyperparameter discussed in section ~\ref{subsection_unbiased_ltr}, which involves the different transformations of the estimated residuals to be used in the ranker $g()$. Thus, we have a total of three hyperparameters, and we select the best ranking model $g()$ by tuning these hyperparameters on validation data. We use all the default settings of the Ridge Classifier and Ridge Regression to train the models $m()$ and $f()$.
% \begin{table*}[b]
% \setlength{\belowcaptionskip}{5pt}
% \centering
% \begin{tabularx}{0.97\textwidth}{|>{\centering\arraybackslash}p{2.97cm}|>{\centering\arraybackslash}p{0.96cm}|>{\centering\arraybackslash}p{1.40cm}|>{\centering\arraybackslash}p{0.96cm}|>{\centering\arraybackslash}p{1.40cm}|>{\centering\arraybackslash}p{0.96cm}|>{\centering\arraybackslash}p{1.40cm}|>{\centering\arraybackslash}p{2.80cm}|>{\centering\arraybackslash}c|}
% \hline
% Dataset & Train \newline queries & Train \newline documents & Valid \newline queries & Valid \newline documents & Test \newline queries & Test \newline documents & Average documents \newline per query & Features \\
% \hline
% Yahoo (Set1) & 19,944 & 473,134 & 2,994 & 71,083 & 6,983 & 165,660 & 24 & 700 \\
% \hline
% MSLR-WEB10k (Fold 1) & 6,000 & 723,412 & 2,000 & 235,260 & 2,000 & 241,521 & 121 & 136 \\
% \hline
% Istella-S & 19,245 & 2,043,304 & 7,211 & 684,076 & 6,562 & 681,250 & 103 & 220 \\
% \hline
% \end{tabularx}
% \captionsetup{justification=centering, font=small}
% \caption{Dataset descriptions.}
% \label{tab:dataset_description}
% % \vspace{-2em}
% \end{table*}
% \section{Additional Experimental Results} \label{sec:additional_results}
% In Section~\ref{sec:performance_true}, we evaluate the performance of different methods under the assumption that the true relevance of documents in the validation data is available and present the results using $ERR@10$ and $NDCG@10$. Table~\ref{tab:main_results_with_true_validation_data_full_version} presents the extended results under this assumption using the $ERR@p$ and $NDCG@p$ metrics, with $p$ values set to $1, 3, 5, 10, \text{and} 15$. The CFC method achieves the best performance on all three datasets compared to all baselines in terms of $ERR@p$ and $NDCG@p$ across various values of $p$. For example, the percentage improvements of CFC over the strongest baselines for $ERR@10$ are $+1.11\%$, $+1.96\%$, and $+1.43\%$ on the Yahoo, MSLR-WEB10k, and
% Istella-S datasets, respectively.
% % \vspace{0.5cm}
% \begin{table*}
% \setlength{\belowcaptionskip}{5pt}
% \centering
% \begin{tabularx}{0.96\textwidth}{|c|c|X|X|X|X|X|X|X|X|X|X|}
% \cline{1-12}
% \multirow{2}{*}{Datasets} & \multirow{2}{*}{Methods} & \multicolumn{5}{c|} {$ERR@p$} & \multicolumn{5}{c|}{$NDCG@p$} \\ \cline{3-12}
% & & $p=1$ & $p=3$ & $p=5$ & $p=10$ & $p=15$ & $p=1$ & $p=3$ & $p=5$ & $p=10$ & $p=15$ \\ \cline{3-12}
% \cline{1-12}
% \multirow{9}[0]{*}{Yahoo}
% & CFC & \textbf{0.345} & \textbf{0.421} & \textbf{0.443} & \textbf{0.459} & \textbf{0.462} & \textbf{0.747} & \textbf{0.738} & \textbf{0.751} & \textbf{0.792} & \textbf{0.814} \\
% \cline{2-12}
% & \modify{UPE ~\cite{luo-sigir24}} & 0.325 & 0.407 & 0.430 & 0.446 & 0.449 & 0.709 & 0.718 & 0.737 & 0.780 & 0.805 \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.338 & 0.416 & 0.438 & 0.454 & 0.457 & 0.727 & 0.724 & 0.739 & 0.781 & 0.805 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.327 & 0.409 & 0.431 & 0.447 & 0.450 & 0.721 & 0.722 & 0.738 & 0.779 & 0.802 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.323 & 0.406 & 0.429 & 0.445 & 0.448 & 0.721 & 0.723 & 0.741 & 0.783 & 0.807 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.326 & 0.406 & 0.428 & 0.445 & 0.448 & 0.692 & 0.702 & 0.722 & 0.766 & 0.792 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.313 & 0.397 & 0.421 & 0.437 & 0.441 & 0.712 & 0.715 & 0.735 & 0.778 & 0.803 \\
% \cline{2-12}
% & LambdaMART & 0.322 & 0.405 & 0.428 & 0.444 & 0.447 & 0.729 & 0.727 & 0.745 & 0.786 & 0.809 \\
% \cline{2-12}
% & Oracle & 0.364 & 0.442 & 0.463 & 0.477 & 0.480 & 0.778 & 0.779 & 0.791 & 0.823 & 0.842 \\
% \cline{1-12}
% \cline{1-12}
% \multirow{9}[0]{*}{MSLR-WEB10k}
% & CFC & \textbf{0.195} & \textbf{0.269} & \textbf{0.293} & \textbf{0.312} & \textbf{0.319} & \textbf{0.486} & \textbf{0.474} & \textbf{0.475} & \textbf{0.482} & \textbf{0.493} \\
% \cline{2-12}
% & \modify{UPE ~\cite{luo-sigir24}} & 0.155 & 0.218 & 0.243 & 0.265 & 0.272 & 0.381 & 0.381 & 0.394 & 0.414 & 0.432 \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.187 & 0.263 & 0.285 & 0.306 & 0.313 & 0.461 & 0.453 & 0.453 & 0.462 & 0.475 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.126 & 0.199 & 0.225 & 0.248 & 0.254 & 0.392 & 0.401 & 0.412 & 0.434 & 0.449 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.141 & 0.217 & 0.242 & 0.265 & 0.274 & 0.424 & 0.430 & 0.440 & 0.455 & 0.471 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.148 & 0.221 & 0.243 & 0.266 & 0.273 & 0.375 & 0.385 & 0.391 & 0.414 & 0.431 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.185 & 0.261 & 0.285 & 0.304 & 0.311 & 0.447 & 0.445 & 0.449 & 0.461 & 0.475 \\
% \cline{2-12}
% & LambdaMART & 0.148 & 0.225 & 0.250 & 0.272 & 0.279 & 0.434 & 0.442 & 0.445 & 0.460 & 0.474 \\
% \cline{2-12}
% & Oracle & 0.234 & 0.312 & 0.333 & 0.353 & 0.359 & 0.535 & 0.518 & 0.514 & 0.517 & 0.524 \\
% \cline{1-12}
% \cline{1-12}
% \multirow{9}[0]{*}{Istella-S}
% & CFC & \textbf{0.570} & \textbf{0.684} & \textbf{0.702} & \textbf{0.709} & \textbf{0.710} & \textbf{0.748} & \textbf{0.716} & \textbf{0.721} & \textbf{0.751} & \textbf{0.789} \\
% \cline{2-12}
% & \modify{UPE ~\cite{luo-sigir24}} & 0.532 & 0.653 & 0.673 & 0.681 & 0.683 & 0.697 & 0.669 & 0.673 & 0.700 & 0.739 \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.558 & 0.673 & 0.691 & 0.699 & 0.700 & 0.729 & 0.696 & 0.701 & 0.729 & 0.765 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.476 & 0.605 & 0.627 & 0.636 & 0.638 & 0.655 & 0.629 & 0.633 & 0.656 & 0.693 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.523 & 0.645 & 0.665 & 0.674 & 0.675 & 0.704 & 0.672 & 0.678 & 0.711 & 0.749 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.530 & 0.649 & 0.669 & 0.677 & 0.678 & 0.703 & 0.667 & 0.671 & 0.699 & 0.736 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.544 & 0.661 & 0.680 & 0.688 & 0.689 & 0.712 & 0.680 & 0.681 & 0.710 & 0.749 \\
% \cline{2-12}
% & LambdaMART & 0.540 & 0.661 & 0.680 & 0.688 & 0.689 & 0.726 & 0.694 & 0.700 & 0.734 & 0.773 \\
% \cline{2-12}
% & Oracle & 0.606 & 0.717 & 0.732 & 0.738 & 0.738 & 0.778 & 0.748 & 0.752 & 0.784 & 0.822 \\
% \cline{1-12}
% \end{tabularx}
% \captionsetup{justification=centering, width=0.75\textwidth}
% \caption{A detailed comparison between CFC and other methods across various datasets using true relevance for hyperparameter tuning.} \label{tab:main_results_with_true_validation_data_full_version}
% \end{table*}
% \section{Performance Comparison}
% \begin{table*}
% \setlength{\belowcaptionskip}{5pt}
% \centering
% \begin{tabularx}{0.96\textwidth}{|C{1.9cm}|C{1.4cm}|C{0.8cm}|C{0.8cm}|C{1.4cm}|C{1.0cm}|C{1.2cm}|C{1.1cm}|C{0.9cm}|C{1.8cm}|C{1.0cm}|}
% \cline{1-11}
% Datasets & Metrics & CFC-DCD & CFC-BCD & MULTR ~\cite{luo-wsdm23} & PIJD ~\cite{ovaisi-sigir21} & PairD ~\cite{hu-www19} & REM ~\cite{wang-wsdm18} & DLA ~\cite{ai-sigir18} & LambdaMART & Oracle \\ \cline{3-10}
% \cline{1-11}
% \multirow{2}[0]{*}{Yahoo}
% & $ERR@10$ & \textbf{0.457} & 0.437 & 0.446 & 0.447 & 0.438 & 0.442 & 0.430 & 0.447 & 0.477 \\
% \cline{2-11}
% & $NDCG@10$ & \textbf{0.790} & 0.784 & 0.780 & 0.770 & 0.780 & 0.763 & 0.772 & 0.786 & 0.823\\
% \cline{1-11}
% \multirow{2}[0]{*}{MSLR- \\ WEB10k}
% & $ERR@10$ & \textbf{0.307} & 0.278 & 0.301 & 0.248 & 0.247 & 0.241 & 0.130 & 0.272 & 0.353 \\
% \cline{2-11}
% & $NDCG@10$ & \textbf{0.482} & 0.465 & 0.460 & 0.434 & 0.445 & 0.414 & 0.460 & 0.460 & 0.517 \\
% \cline{1-11}
% \multirow{2}[0]{*}{Istella-S}
% & $ERR@10$ & \textbf{0.709} & 0.691 & 0.302 & 0.636 & 0.667 & 0.627 & 0.657 & 0.688 & 0.738 \\
% \cline{2-11}
% & $NDCG@10$ & \textbf{0.752} & 0.735 & 0.303 & 0.656 & 0.704 & 0.643 & 0.674 & 0.731 & 0.784 \\
% \cline{1-11}
% \end{tabularx}
% \captionsetup{justification=centering, width=0.85\textwidth}
% \caption{A comparison between CFC and other methods across various datasets using click data for validation. CFC-DCD indicates that the CFC method uses debiased click data for validation, while CFC-BCD denotes that the CFC method uses biased click data directly for validation.} \label{tab:main_results_with_biased_clicks_short_version}
% \vspace{-0.4cm}
% \end{table*}
% \begin{table*}
% \setlength{\belowcaptionskip}{5pt}
% \centering
% \begin{tabularx}{0.96\textwidth}{|c|c|X|X|X|X|X|X|X|X|X|X|}
% \cline{1-12}
% \multirow{2}{*}{Datasets} & \multirow{2}{*}{Methods} & \multicolumn{5}{c|} {$ERR@p$} & \multicolumn{5}{c|}{$NDCG@p$} \\ \cline{3-12}
% & & $p=1$ & $p=3$ & $p=5$ & $p=10$ & $p=15$ & $p=1$ & $p=3$ & $p=5$ & $p=10$ & $p=15$ \\ \cline{3-12}
% \cline{1-12}
% \multirow{12}[-12]{*}{Yahoo}
% & CFC & \textbf{0.341} & \textbf{0.419} & \textbf{0.441} & \textbf{0.457} & \textbf{0.460} & \textbf{0.743} & \textbf{0.737} & \textbf{0.752} & \textbf{0.790} & \textbf{0.814} \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.334 & 0.413 & 0.435 & 0.451 & 0.454 & 0.726 & 0.723 & 0.743 & 0.783 & 0.806 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.327 & 0.409 & 0.431 & 0.447 & 0.450 & 0.721 & 0.722 & 0.738 & 0.779 & 0.802 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.310 & 0.396 & 0.420 & 0.436 & 0.440 & 0.710 & 0.716 & 0.735 & 0.777 & 0.802 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.323 & 0.404 & 0.424 & 0.442 & 0.446 & 0.699 & 0.701 & 0.719 & 0.763 & 0.790 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.328 & 0.410 & 0.433 & 0.449 & 0.452 & 0.729 & 0.729 & 0.746 & 0.787 & 0.810 \\
% \cline{2-12}
% & LambdaMART & 0.322 & 0.405 & 0.428 & 0.440 & 0.447 & 0.727 & 0.728 & 0.745 & 0.787 & 0.810 \\
% \cline{2-12}
% & Oracle & 0.364 & 0.442 & 0.463 & 0.477 & 0.480 & 0.778 & 0.779 & 0.791 & 0.823 & 0.842 \\
% \cline{1-12}
% \cline{1-12}
% \multirow{11}[-12]{*}{MSLR-WEB10k}
% & CFC & \textbf{0.198} & \textbf{0.271} & \textbf{0.294} & \textbf{0.314} & \textbf{0.321} & \textbf{0.490} & \textbf{0.477} & \textbf{0.473} & \textbf{0.483} & \textbf{0.493} \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.182 & 0.249 & 0.272 & 0.294 & 0.300 & 0.437 & 0.426 & 0.429 & 0.448 & 0.461 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.126 & 0.199 & 0.225 & 0.248 & 0.254 & 0.392 & 0.401 & 0.412 & 0.434 & 0.449 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.130 & 0.205 & 0.230 & 0.253 & 0.261 & 0.412 & 0.425 & 0.431 & 0.449 & 0.466 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.157 & 0.225 & 0.249 & 0.270 & 0.278 & 0.377 & 0.381 & 0.390 & 0.411 & 0.429 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.180 & 0.254 & 0.277 & 0.298 & 0.305 & 0.459 & 0.454 & 0.455 & 0.468 & 0.481 \\
% \cline{2-12}
% & LambdaMART & 0.148 & 0.225 & 0.250 & 0.272 & 0.279 & 0.434 & 0.442 & 0.445 & 0.460 & 0.474 \\
% \cline{2-12}
% & Oracle & 0.234 & 0.311 & 0.334 & 0.353 & 0.359 & 0.532 & 0.517 & 0.516 & 0.518 & 0.525 \\
% \cline{1-12}
% \cline{1-12}
% \multirow{11}[-12]{*}{Istella-S}
% & CFC & \textbf{0.569} & \textbf{0.683} & \textbf{0.701} & \textbf{0.708} & \textbf{0.709} & \textbf{0.747} & \textbf{0.714} & \textbf{0.719} & \textbf{0.751} & \textbf{0.790} \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.544 & 0.658 & 0.676 & 0.684 & 0.686 & 0.732 & 0.698 & 0.699 & 0.728 & 0.765 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.476 & 0.605 & 0.627 & 0.636 & 0.638 & 0.655 & 0.629 & 0.633 & 0.656 & 0.693 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.517 & 0.640 & 0.661 & 0.670 & 0.671 & 0.700 & 0.669 & 0.675 & 0.707 & 0.747 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.544 & 0.658 & 0.676 & 0.685 & 0.686 & 0.711 & 0.670 & 0.669 & 0.693 & 0.730 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.553 & 0.668 & 0.687 & 0.695 & 0.696 & 0.724 & 0.690 & 0.694 & 0.725 & 0.763 \\
% \cline{2-12}
% & LambdaMART & 0.543 & 0.663 & 0.682 & 0.690 & 0.691 & 0.728 & 0.694 & 0.702 & 0.737 & 0.776 \\
% \cline{2-12}
% & Oracle & 0.608 & 0.718 & 0.733 & 0.740 & 0.740 & 0.779 & 0.747 & 0.752 & 0.783 & 0.821 \\
% \cline{1-12}
% \end{tabularx}
% \captionsetup{justification=centering}
% \caption{Comparison of various methods across different datasets using $\bm{300}$ queries with true labels for validation.} \label{tab:main_results_with_300_queries}
% % \vspace{-0.4cm}
% \end{table*}
% \begin{table*}
% \setlength{\belowcaptionskip}{5pt}
% \centering
% \begin{tabularx}{0.96\textwidth}{|c|c|X|X|X|X|X|X|X|X|X|X|}
% \cline{1-12}
% \multirow{2}{*}{Datasets} & \multirow{2}{*}{Methods} & \multicolumn{5}{c|} {$ERR@p$} & \multicolumn{5}{c|}{$NDCG@p$} \\ \cline{3-12}
% & & $p=1$ & $p=3$ & $p=5$ & $p=10$ & $p=15$ & $p=1$ & $p=3$ & $p=5$ & $p=10$ & $p=15$ \\ \cline{3-12}
% \cline{1-12}
% \multirow{8}[0]{*}{Yahoo}
% & CFC & \textbf{0.345} & \textbf{0.421} & \textbf{0.443} & \textbf{0.459} & \textbf{0.462} & \textbf{0.747} & \textbf{0.738} & \textbf{0.751} & \textbf{0.792} & \textbf{0.814} \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.338 & 0.416 & 0.438 & 0.454 & 0.457 & 0.727 & 0.724 & 0.739 & 0.781 & 0.805 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.327 & 0.409 & 0.431 & 0.447 & 0.450 & 0.721 & 0.722 & 0.738 & 0.779 & 0.802 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.323 & 0.406 & 0.429 & 0.445 & 0.448 & 0.721 & 0.723 & 0.741 & 0.783 & 0.807 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.326 & 0.406 & 0.428 & 0.445 & 0.448 & 0.692 & 0.702 & 0.722 & 0.766 & 0.792 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.313 & 0.397 & 0.421 & 0.437 & 0.441 & 0.712 & 0.715 & 0.735 & 0.778 & 0.803 \\
% \cline{2-12}
% & LambdaMART & 0.322 & 0.405 & 0.428 & 0.444 & 0.447 & 0.729 & 0.727 & 0.745 & 0.786 & 0.809 \\
% \cline{2-12}
% & Oracle (Valid-150) & 0.137 & 0.226 & 0.258 & 0.283 & 0.289 & 0.486 & 0.534 & 0.575 & 0.650 & 0.689 \\
% \cline{2-12}
% & Oracle (Valid-300) & 0.332 & 0.412 & 0.434 & 0.450 & 0.453 & 0.730 & 0.727 & 0.742 & 0.781 & 0.805 \\
% \cline{2-12}
% & Oracle (Valid Full) & 0.350 & 0.428 & 0.449 & 0.465 & 0.468 & 0.753 & 0.751 & 0.766 & 0.803 & 0.823 \\
% \cline{2-12}
% & Oracle & 0.364 & 0.442 & 0.463 & 0.477 & 0.480 & 0.778 & 0.779 & 0.791 & 0.823 & 0.842 \\
% \cline{1-12}
% \cline{1-12}
% \multirow{8}[0]{*}{MSLR-WEB10k}
% & CFC & \textbf{0.195} & \textbf{0.269} & \textbf{0.293} & \textbf{0.312} & \textbf{0.319} & \textbf{0.486} & \textbf{0.474} & \textbf{0.475} & \textbf{0.482} & \textbf{0.493} \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.187 & 0.263 & 0.285 & 0.306 & 0.313 & 0.461 & 0.453 & 0.453 & 0.462 & 0.475 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.126 & 0.199 & 0.225 & 0.248 & 0.254 & 0.392 & 0.401 & 0.412 & 0.434 & 0.449 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.141 & 0.217 & 0.242 & 0.265 & 0.274 & 0.424 & 0.430 & 0.440 & 0.455 & 0.471 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.148 & 0.221 & 0.243 & 0.266 & 0.273 & 0.375 & 0.385 & 0.391 & 0.414 & 0.431 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.185 & 0.261 & 0.285 & 0.304 & 0.311 & 0.447 & 0.445 & 0.449 & 0.461 & 0.475 \\
% \cline{2-12}
% & LambdaMART & 0.148 & 0.225 & 0.250 & 0.272 & 0.279 & 0.434 & 0.442 & 0.445 & 0.460 & 0.474 \\
% \cline{2-12}
% & Oracle (Valid-150) & 0.135 & 0.204 & 0.229 & 0.251 & 0.259 & 0.393 & 0.400 & 0.409 & 0.429 & 0.447 \\
% \cline{2-12}
% & Oracle (Valid-300) & 0.197 & 0.274 & 0.298 & 0.319 & 0.325 & 0.494 & 0.484 & 0.486 & 0.496 & 0.506 \\
% \cline{2-12}
% & Oracle (Valid Full) & 0.211 & 0.283 & 0.305 & 0.324 & 0.331 & 0.482 & 0.466 & 0.465 & 0.472 & 0.483 \\
% \cline{2-12}
% & Oracle & 0.234 & 0.312 & 0.333 & 0.353 & 0.359 & 0.535 & 0.518 & 0.514 & 0.517 & 0.524 \\
% \cline{1-12}
% \cline{1-12}
% \multirow{8}[0]{*}{Istella-S}
% & CFC & \textbf{0.570} & \textbf{0.684} & \textbf{0.702} & \textbf{0.709} & \textbf{0.710} & \textbf{0.748} & \textbf{0.716} & \textbf{0.721} & \textbf{0.751} & \textbf{0.789} \\
% \cline{2-12}
% & MULTR ~\cite{luo-wsdm23} & 0.558 & 0.673 & 0.691 & 0.699 & 0.700 & 0.729 & 0.696 & 0.701 & 0.729 & 0.765 \\
% \cline{2-12}
% & PIJD ~\cite{ovaisi-sigir21} & 0.476 & 0.605 & 0.627 & 0.636 & 0.638 & 0.655 & 0.629 & 0.633 & 0.656 & 0.693 \\
% \cline{2-12}
% & PairD ~\cite{hu-www19} & 0.523 & 0.645 & 0.665 & 0.674 & 0.675 & 0.704 & 0.672 & 0.678 & 0.711 & 0.749 \\
% \cline{2-12}
% & REM ~\cite{wang-wsdm18} & 0.530 & 0.649 & 0.669 & 0.677 & 0.678 & 0.703 & 0.667 & 0.671 & 0.699 & 0.736 \\
% \cline{2-12}
% & DLA ~\cite{ai-sigir18} & 0.544 & 0.661 & 0.680 & 0.688 & 0.689 & 0.712 & 0.680 & 0.681 & 0.710 & 0.749 \\
% \cline{2-12}
% & LambdaMART & 0.540 & 0.661 & 0.680 & 0.688 & 0.689 & 0.726 & 0.694 & 0.700 & 0.734 & 0.773 \\
% \cline{2-12}
% & Oracle (Valid-150) & 0.458 & 0.593 & 0.616 & 0.626 & 0.628 & 0.644 & 0.626 & 0.635 & 0.670 & 0.714 \\
% \cline{2-12}
% & Oracle (Valid-300) & 0.530 & 0.649 & 0.669 & 0.677 & 0.678 & 0.706 & 0.673 & 0.679 & 0.709 & 0.748 \\
% \cline{2-12}
% & Oracle (Valid Full) & 0.590 & 0.702 & 0.718 & 0.725 & 0.726 & 0.764 & 0.731 & 0.736 & 0.767 & 0.805 \\
% \cline{2-12}
% & Oracle & 0.606 & 0.717 & 0.732 & 0.738 & 0.738 & 0.778 & 0.748 & 0.752 & 0.784 & 0.822 \\
% \cline{1-12}
% \end{tabularx}
% \captionsetup{justification=centering}
% \caption{Comparison of various methods across different datasets using full validation dataset with true labels (test-table). Summary: Valid-Full is best compared to any methods on all datasets. For Yahoo, valid-300 is better than most of the baselines. For MSLR-WEB10k, valid-300 is better than all. For Istella-S, valid-300 is not better than most of the baselines. Valid-150 is worst on all datasets than any methods.} \label{tab:main_results_test}
% \end{table*}
% \appendix
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{64}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{} and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showURL}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showURL #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Agarwal et~al\mbox{.}(2019)]%
{agarwal-wsdm19}
\bibfield{author}{\bibinfo{person}{Aman Agarwal}, \bibinfo{person}{Ivan Zaitsev}, \bibinfo{person}{Xuanhui Wang}, \bibinfo{person}{Cheng Li}, \bibinfo{person}{Marc Najork}, {and} \bibinfo{person}{Thorsten Joachims}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Estimating position bias without intrusive interventions}. In \bibinfo{booktitle}{\emph{Proceedings of the twelfth ACM international conference on web search and data mining}}. \bibinfo{pages}{474--482}.
\newblock
\bibitem[Ai et~al\mbox{.}(2018)]%
{ai-sigir18}
\bibfield{author}{\bibinfo{person}{Qingyao Ai}, \bibinfo{person}{Keping Bi}, \bibinfo{person}{Cheng Luo}, \bibinfo{person}{Jiafeng Guo}, {and} \bibinfo{person}{W~Bruce Croft}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Unbiased learning to rank with unbiased propensity estimation}. In \bibinfo{booktitle}{\emph{The 41st international ACM SIGIR conference on research \& development in information retrieval}}. \bibinfo{pages}{385--394}.
\newblock
\bibitem[Amemiya(1985)]%
{amemiya-1985advanced}
\bibfield{author}{\bibinfo{person}{Takeshi Amemiya}.} \bibinfo{year}{1985}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Advanced econometrics}}.
\newblock \bibinfo{publisher}{Harvard university press}.
\newblock
\bibitem[Blundell and Powell(2004)]%
{blundell-res04}
\bibfield{author}{\bibinfo{person}{Richard~W Blundell} {and} \bibinfo{person}{James~L Powell}.} \bibinfo{year}{2004}\natexlab{}.
\newblock \showarticletitle{Endogeneity in semiparametric binary response models}.
\newblock \bibinfo{journal}{\emph{The Review of Economic Studies}} \bibinfo{volume}{71}, \bibinfo{number}{3} (\bibinfo{year}{2004}), \bibinfo{pages}{655--679}.
\newblock
\bibitem[Bojic(2024)]%
{bojic-futures24}
\bibfield{author}{\bibinfo{person}{Ljubisa Bojic}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{{AI} alignment: Assessing the global impact of recommender systems}.
\newblock \bibinfo{journal}{\emph{Futures}} (\bibinfo{year}{2024}), \bibinfo{pages}{103383}.
\newblock
\bibitem[Burges(2010)]%
{burges-learning10}
\bibfield{author}{\bibinfo{person}{Christopher~JC Burges}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{From ranknet to lambdarank to lambdamart: An overview}.
\newblock \bibinfo{journal}{\emph{Learning}} \bibinfo{volume}{11}, \bibinfo{number}{23-581} (\bibinfo{year}{2010}), \bibinfo{pages}{81}.
\newblock
\bibitem[Chapelle and Chang(2011)]%
{chapelle-pmlr11}
\bibfield{author}{\bibinfo{person}{Olivier Chapelle} {and} \bibinfo{person}{Yi Chang}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{Yahoo! learning to rank challenge overview}. In \bibinfo{booktitle}{\emph{Proceedings of the learning to rank challenge}}. PMLR, \bibinfo{pages}{1--24}.
\newblock
\bibitem[Chapelle et~al\mbox{.}(2009)]%
{chapelle-cikm09}
\bibfield{author}{\bibinfo{person}{Olivier Chapelle}, \bibinfo{person}{Donald Metlzer}, \bibinfo{person}{Ya Zhang}, {and} \bibinfo{person}{Pierre Grinspan}.} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{Expected reciprocal rank for graded relevance}. In \bibinfo{booktitle}{\emph{Proceedings of the 18th ACM Conference on Information and Knowledge Management}} (Hong Kong, China). \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{621–630}.
\newblock
\showISBNx{9781605585123}
\bibitem[Chapelle and Zhang(2009)]%
{chapelle-www09}
\bibfield{author}{\bibinfo{person}{Olivier Chapelle} {and} \bibinfo{person}{Ya Zhang}.} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{A dynamic bayesian network click model for web search ranking}. In \bibinfo{booktitle}{\emph{Proceedings of the 18th international conference on World wide web}}. \bibinfo{pages}{1--10}.
\newblock
\bibitem[Chernozhukov et~al\mbox{.}(2018)]%
{chernozhukov-ej18}
\bibfield{author}{\bibinfo{person}{Victor Chernozhukov}, \bibinfo{person}{Denis Chetverikov}, \bibinfo{person}{Mert Demirer}, \bibinfo{person}{Esther Duflo}, \bibinfo{person}{Christian Hansen}, \bibinfo{person}{Whitney Newey}, {and} \bibinfo{person}{James Robins}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Double/debiased machine learning for treatment and structural parameters}.
\newblock \bibinfo{journal}{\emph{The Econometrics Journal}} \bibinfo{volume}{21}, \bibinfo{number}{1} (\bibinfo{year}{2018}), \bibinfo{pages}{C1--C68}.
\newblock
\bibitem[Chuklin et~al\mbox{.}(2016)]%
{chuklin-wsdm16}
\bibfield{author}{\bibinfo{person}{Aleksandr Chuklin}, \bibinfo{person}{Ilya Markov}, {and} \bibinfo{person}{Maarten de Rijke}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Click models for web search and their applications to {IR: WSDM} 2016 Tutorial}. In \bibinfo{booktitle}{\emph{Proceedings of the Ninth ACM International Conference on Web Search and Data Mining}}. \bibinfo{pages}{689--690}.
\newblock
\bibitem[Clevert et~al\mbox{.}(2016)]%
{clevert-iclr16}
\bibfield{author}{\bibinfo{person}{Djork-Arn{\'e} Clevert}, \bibinfo{person}{Thomas Unterthiner}, {and} \bibinfo{person}{Sepp Hochreiter}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Fast and accurate deep network learning by exponential linear units (elus)}. In \bibinfo{booktitle}{\emph{International Conference on Learning Representations}}.
\newblock
\bibitem[Craswell et~al\mbox{.}(2008)]%
{craswell-wsdm08}
\bibfield{author}{\bibinfo{person}{Nick Craswell}, \bibinfo{person}{Onno Zoeter}, \bibinfo{person}{Michael Taylor}, {and} \bibinfo{person}{Bill Ramsey}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{An experimental comparison of click position-bias models}. In \bibinfo{booktitle}{\emph{Proceedings of the 2008 international conference on web search and data mining}}. \bibinfo{pages}{87--94}.
\newblock
\bibitem[Das et~al\mbox{.}(2003)]%
{das-res03}
\bibfield{author}{\bibinfo{person}{Mitali Das}, \bibinfo{person}{Whitney~K Newey}, {and} \bibinfo{person}{Francis Vella}.} \bibinfo{year}{2003}\natexlab{}.
\newblock \showarticletitle{Nonparametric estimation of sample selection models}.
\newblock \bibinfo{journal}{\emph{The Review of Economic Studies}} \bibinfo{volume}{70}, \bibinfo{number}{1} (\bibinfo{year}{2003}), \bibinfo{pages}{33--58}.
\newblock
\bibitem[Duchi et~al\mbox{.}(2011)]%
{duchi-jmlr11}
\bibfield{author}{\bibinfo{person}{John Duchi}, \bibinfo{person}{Elad Hazan}, {and} \bibinfo{person}{Yoram Singer}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{Adaptive subgradient methods for online learning and stochastic optimization.}
\newblock \bibinfo{journal}{\emph{Journal of machine learning research}} \bibinfo{volume}{12}, \bibinfo{number}{7} (\bibinfo{year}{2011}).
\newblock
\bibitem[Dupret and Piwowarski(2008)]%
{dupret-sigir08}
\bibfield{author}{\bibinfo{person}{Georges~E Dupret} {and} \bibinfo{person}{Benjamin Piwowarski}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{A user browsing model to predict search engine click data from past observations.}. In \bibinfo{booktitle}{\emph{Proceedings of the 31st annual international ACM SIGIR conference on Research and development in information retrieval}}. \bibinfo{pages}{331--338}.
\newblock
\bibitem[Fan et~al\mbox{.}(2023)]%
{fan-sigkdd23}
\bibfield{author}{\bibinfo{person}{Wenqi Fan}, \bibinfo{person}{Xiangyu Zhao}, \bibinfo{person}{Lin Wang}, \bibinfo{person}{Xiao Chen}, \bibinfo{person}{Jingtong Gao}, \bibinfo{person}{Qidong Liu}, {and} \bibinfo{person}{Shijie Wang}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Trustworthy recommender systems: Foundations and frontiers}. In \bibinfo{booktitle}{\emph{Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}}. \bibinfo{pages}{5796--5797}.
\newblock
\bibitem[Fang et~al\mbox{.}(2019)]%
{fang-sigir19}
\bibfield{author}{\bibinfo{person}{Zhichong Fang}, \bibinfo{person}{Aman Agarwal}, {and} \bibinfo{person}{Thorsten Joachims}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Intervention harvesting for context-dependent examination-bias estimation}. In \bibinfo{booktitle}{\emph{Proceedings of the 42nd international ACM SIGIR conference on research and development in information retrieval}}. \bibinfo{pages}{825--834}.
\newblock
\bibitem[Fligner and Killeen(1976)]%
{fligner-jasa76}
\bibfield{author}{\bibinfo{person}{Michael~A Fligner} {and} \bibinfo{person}{Timothy~J Killeen}.} \bibinfo{year}{1976}\natexlab{}.
\newblock \showarticletitle{Distribution-free two-sample tests for scale}.
\newblock \bibinfo{journal}{\emph{J. Amer. Statist. Assoc.}} \bibinfo{volume}{71}, \bibinfo{number}{353} (\bibinfo{year}{1976}), \bibinfo{pages}{210--213}.
\newblock
\bibitem[Guo et~al\mbox{.}(2019)]%
{guo-recsys19}
\bibfield{author}{\bibinfo{person}{Huifeng Guo}, \bibinfo{person}{Jinkai Yu}, \bibinfo{person}{Qing Liu}, \bibinfo{person}{Ruiming Tang}, {and} \bibinfo{person}{Yuzhou Zhang}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{{PAL}: a position-bias aware learning framework for {CTR} prediction in live recommender systems}. In \bibinfo{booktitle}{\emph{Proceedings of the 13th ACM Conference on Recommender Systems}}. \bibinfo{pages}{452--456}.
\newblock
\bibitem[Hager et~al\mbox{.}(2024)]%
{hager-sigir24}
\bibfield{author}{\bibinfo{person}{Philipp Hager}, \bibinfo{person}{Romain Deffayet}, \bibinfo{person}{Jean-Michel Renders}, \bibinfo{person}{Onno Zoeter}, {and} \bibinfo{person}{Maarten de Rijke}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Unbiased Learning to Rank Meets Reality: Lessons from {Baidu's} Large-Scale Search Dataset}. In \bibinfo{booktitle}{\emph{Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval}}. \bibinfo{pages}{1546--1556}.
\newblock
\bibitem[Heckman(1979)]%
{heckman-econometrica79}
\bibfield{author}{\bibinfo{person}{James~J. Heckman}.} \bibinfo{year}{1979}\natexlab{}.
\newblock \showarticletitle{Sample Selection Bias as a Specification Error}.
\newblock \bibinfo{journal}{\emph{Econometrica}} \bibinfo{volume}{47}, \bibinfo{number}{1} (\bibinfo{year}{1979}), \bibinfo{pages}{153--161}.
\newblock
\showISSN{00129682, 14680262}
\bibitem[Heer(2021)]%
{heer-vis21}
\bibfield{author}{\bibinfo{person}{Jeffrey Heer}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Fast \& accurate gaussian kernel density estimation}. In \bibinfo{booktitle}{\emph{2021 IEEE Visualization Conference (VIS)}}. IEEE, \bibinfo{pages}{11--15}.
\newblock
\bibitem[Hu et~al\mbox{.}(2019)]%
{hu-www19}
\bibfield{author}{\bibinfo{person}{Ziniu Hu}, \bibinfo{person}{Yang Wang}, \bibinfo{person}{Qu Peng}, {and} \bibinfo{person}{Hang Li}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Unbiased lambdamart: an unbiased pairwise learning-to-rank algorithm}. In \bibinfo{booktitle}{\emph{The World Wide Web Conference}}. \bibinfo{pages}{2830--2836}.
\newblock
\bibitem[Joachims(2002)]%
{joachims-kdd02}
\bibfield{author}{\bibinfo{person}{Thorsten Joachims}.} \bibinfo{year}{2002}\natexlab{}.
\newblock \showarticletitle{Optimizing search engines using clickthrough data}. In \bibinfo{booktitle}{\emph{Proceedings of the eighth ACM SIGKDD international conference on Knowledge discovery and data mining}}. \bibinfo{pages}{133--142}.
\newblock
\bibitem[Joachims(2006)]%
{joachims-kdd06}
\bibfield{author}{\bibinfo{person}{Thorsten Joachims}.} \bibinfo{year}{2006}\natexlab{}.
\newblock \showarticletitle{Training linear {SVMs} in linear time}. In \bibinfo{booktitle}{\emph{Proceedings of the 12th ACM SIGKDD international conference on Knowledge discovery and data mining}}. \bibinfo{pages}{217--226}.
\newblock
\bibitem[Joachims et~al\mbox{.}(2007)]%
{joachims-tois07}
\bibfield{author}{\bibinfo{person}{Thorsten Joachims}, \bibinfo{person}{Laura Granka}, \bibinfo{person}{Bing Pan}, \bibinfo{person}{Helene Hembrooke}, \bibinfo{person}{Filip Radlinski}, {and} \bibinfo{person}{Geri Gay}.} \bibinfo{year}{2007}\natexlab{}.
\newblock \showarticletitle{Evaluating the accuracy of implicit feedback from clicks and query reformulations in web search}.
\newblock \bibinfo{journal}{\emph{ACM Transactions on Information Systems (TOIS)}} \bibinfo{volume}{25}, \bibinfo{number}{2} (\bibinfo{year}{2007}), \bibinfo{pages}{7--es}.
\newblock
\bibitem[Joachims et~al\mbox{.}(2017)]%
{joachims-wsdm17}
\bibfield{author}{\bibinfo{person}{Thorsten Joachims}, \bibinfo{person}{Adith Swaminathan}, {and} \bibinfo{person}{Tobias Schnabel}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Unbiased learning-to-rank with biased feedback}. In \bibinfo{booktitle}{\emph{Proceedings of the tenth ACM international conference on web search and data mining}}. \bibinfo{pages}{781--789}.
\newblock
\bibitem[Ke et~al\mbox{.}(2017)]%
{ke-neurips17}
\bibfield{author}{\bibinfo{person}{Guolin Ke}, \bibinfo{person}{Qi Meng}, \bibinfo{person}{Thomas Finley}, \bibinfo{person}{Taifeng Wang}, \bibinfo{person}{Wei Chen}, \bibinfo{person}{Weidong Ma}, \bibinfo{person}{Qiwei Ye}, {and} \bibinfo{person}{Tie-Yan Liu}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Lightgbm: A highly efficient gradient boosting decision tree}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}} \bibinfo{volume}{30} (\bibinfo{year}{2017}).
\newblock
\bibitem[Kiyohara et~al\mbox{.}(2022)]%
{kiyohara-wsdm22}
\bibfield{author}{\bibinfo{person}{Haruka Kiyohara}, \bibinfo{person}{Yuta Saito}, \bibinfo{person}{Tatsuya Matsuhiro}, \bibinfo{person}{Yusuke Narita}, \bibinfo{person}{Nobuyuki Shimizu}, {and} \bibinfo{person}{Yasuo Yamamoto}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Doubly robust off-policy evaluation for ranking policies under the cascade behavior model}. In \bibinfo{booktitle}{\emph{Proceedings of the Fifteenth ACM International Conference on Web Search and Data Mining}}. \bibinfo{pages}{487--497}.
\newblock
\bibitem[Komiyama et~al\mbox{.}(2015)]%
{komiyama-icml15}
\bibfield{author}{\bibinfo{person}{Junpei Komiyama}, \bibinfo{person}{Junya Honda}, {and} \bibinfo{person}{Hiroshi Nakagawa}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{Optimal regret analysis of thompson sampling in stochastic multi-armed bandit problem with multiple plays}. In \bibinfo{booktitle}{\emph{International Conference on Machine Learning}}. PMLR, \bibinfo{pages}{1152--1161}.
\newblock
\bibitem[Lagr{\'e}e et~al\mbox{.}(2016)]%
{lagree-neurips16}
\bibfield{author}{\bibinfo{person}{Paul Lagr{\'e}e}, \bibinfo{person}{Claire Vernade}, {and} \bibinfo{person}{Olivier Cappe}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Multiple-play bandits in the position-based model}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} \bibinfo{volume}{29} (\bibinfo{year}{2016}).
\newblock
\bibitem[Lewbel(2012)]%
{lewbel-jbec12}
\bibfield{author}{\bibinfo{person}{Arthur Lewbel}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Using heteroscedasticity to identify and estimate mismeasured and endogenous regressor models}.
\newblock \bibinfo{journal}{\emph{Journal of business \& economic statistics}} \bibinfo{volume}{30}, \bibinfo{number}{1} (\bibinfo{year}{2012}), \bibinfo{pages}{67--80}.
\newblock
\bibitem[Lucchese et~al\mbox{.}(2016)]%
{lucchese-sigir16}
\bibfield{author}{\bibinfo{person}{Claudio Lucchese}, \bibinfo{person}{Franco~Maria Nardini}, \bibinfo{person}{Salvatore Orlando}, \bibinfo{person}{Raffaele Perego}, \bibinfo{person}{Fabrizio Silvestri}, {and} \bibinfo{person}{Salvatore Trani}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Post-learning optimization of tree ensembles for efficient ranking}. In \bibinfo{booktitle}{\emph{Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval}}. \bibinfo{pages}{949--952}.
\newblock
\bibitem[Luo et~al\mbox{.}(2024)]%
{luo-sigir24}
\bibfield{author}{\bibinfo{person}{Dan Luo}, \bibinfo{person}{Lixin Zou}, \bibinfo{person}{Qingyao Ai}, \bibinfo{person}{Zhiyu Chen}, \bibinfo{person}{Chenliang Li}, \bibinfo{person}{Dawei Yin}, {and} \bibinfo{person}{Brian~D Davison}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Unbiased Learning-to-Rank Needs Unconfounded Propensity Estimation}. In \bibinfo{booktitle}{\emph{Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval}}. \bibinfo{pages}{1535--1545}.
\newblock
\bibitem[Luo et~al\mbox{.}(2023)]%
{luo-wsdm23}
\bibfield{author}{\bibinfo{person}{Dan Luo}, \bibinfo{person}{Lixin Zou}, \bibinfo{person}{Qingyao Ai}, \bibinfo{person}{Zhiyu Chen}, \bibinfo{person}{Dawei Yin}, {and} \bibinfo{person}{Brian~D Davison}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Model-based unbiased learning to rank}. In \bibinfo{booktitle}{\emph{Proceedings of the Sixteenth ACM International Conference on Web Search and Data Mining}}. \bibinfo{pages}{895--903}.
\newblock
\bibitem[Lyzhin et~al\mbox{.}(2023)]%
{lyzhin-icml23}
\bibfield{author}{\bibinfo{person}{Ivan Lyzhin}, \bibinfo{person}{Aleksei Ustimenko}, \bibinfo{person}{Andrey Gulin}, {and} \bibinfo{person}{Liudmila Prokhorenkova}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Which tricks are important for learning to rank?}. In \bibinfo{booktitle}{\emph{International Conference on Machine Learning}}. PMLR, \bibinfo{pages}{23264--23278}.
\newblock
\bibitem[Maddala and Nelson(1975)]%
{maddala-75sa}
\bibfield{author}{\bibinfo{person}{G.~S. Maddala} {and} \bibinfo{person}{F. Nelson}.} \bibinfo{year}{1975}\natexlab{}.
\newblock \showarticletitle{Switching regression models with exogenous and endogenous switching}. In \bibinfo{booktitle}{\emph{Proceedings of the American Statistical Association, Business and Economics Section}}. American Statistical Association, \bibinfo{pages}{423--426}.
\newblock
\bibitem[Mao et~al\mbox{.}(2018)]%
{mao-sigir18}
\bibfield{author}{\bibinfo{person}{Jiaxin Mao}, \bibinfo{person}{Cheng Luo}, \bibinfo{person}{Min Zhang}, {and} \bibinfo{person}{Shaoping Ma}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Constructing click models for mobile search}. In \bibinfo{booktitle}{\emph{The 41st international ACM SIGIR conference on research \& development in information retrieval}}. \bibinfo{pages}{775--784}.
\newblock
\bibitem[Oosterhuis(2023)]%
{oosterhuis-tis23}
\bibfield{author}{\bibinfo{person}{Harrie Oosterhuis}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Doubly robust estimation for correcting position bias in click feedback for unbiased learning to rank}.
\newblock \bibinfo{journal}{\emph{ACM Transactions on Information Systems}} \bibinfo{volume}{41}, \bibinfo{number}{3} (\bibinfo{year}{2023}), \bibinfo{pages}{1--33}.
\newblock
\bibitem[Oosterhuis and de~Rijke(2020)]%
{oosterhuis-sigir20}
\bibfield{author}{\bibinfo{person}{Harrie Oosterhuis} {and} \bibinfo{person}{Maarten de Rijke}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Policy-Aware Unbiased Learning to Rank for Top-k Rankings}. In \bibinfo{booktitle}{\emph{Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval}}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{pages}{489–498}.
\newblock
\bibitem[Ovaisi et~al\mbox{.}(2020)]%
{ovaisi-www20}
\bibfield{author}{\bibinfo{person}{Zohreh Ovaisi}, \bibinfo{person}{Ragib Ahsan}, \bibinfo{person}{Yifan Zhang}, \bibinfo{person}{Kathryn Vasilaky}, {and} \bibinfo{person}{Elena Zheleva}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Correcting for selection bias in learning-to-rank systems}. In \bibinfo{booktitle}{\emph{Proceedings of The Web Conference 2020}}. \bibinfo{pages}{1863--1873}.
\newblock
\bibitem[Ovaisi et~al\mbox{.}(2021)]%
{ovaisi-sigir21}
\bibfield{author}{\bibinfo{person}{Zohreh Ovaisi}, \bibinfo{person}{Kathryn Vasilaky}, {and} \bibinfo{person}{Elena Zheleva}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Propensity-independent bias recovery in offline learning-to-rank systems}. In \bibinfo{booktitle}{\emph{Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval}}. \bibinfo{pages}{1763--1767}.
\newblock
\bibitem[O’Brien et~al\mbox{.}(2016)]%
{o-csda16}
\bibfield{author}{\bibinfo{person}{Travis~A O’Brien}, \bibinfo{person}{Karthik Kashinath}, \bibinfo{person}{Nicholas~R Cavanaugh}, \bibinfo{person}{William~D Collins}, {and} \bibinfo{person}{John~P O’Brien}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{A fast and objective multidimensional kernel density estimation method: fastKDE}.
\newblock \bibinfo{journal}{\emph{Computational Statistics \& Data Analysis}} \bibinfo{volume}{101} (\bibinfo{year}{2016}), \bibinfo{pages}{148--160}.
\newblock
\bibitem[Qin and Liu(2013)]%
{qin-arXiv13}
\bibfield{author}{\bibinfo{person}{Tao Qin} {and} \bibinfo{person}{Tie-Yan Liu}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{Introducing {LETOR} 4.0 datasets}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:1306.2597}} (\bibinfo{year}{2013}).
\newblock
\bibitem[Reiers{\o}l(1941)]%
{reiersol-metrica41}
\bibfield{author}{\bibinfo{person}{Olav Reiers{\o}l}.} \bibinfo{year}{1941}\natexlab{}.
\newblock \showarticletitle{Confluence Analysis by Means of Lag Moments and Other Methods of Confluence Analysis}.
\newblock \bibinfo{journal}{\emph{Econometrica}} \bibinfo{volume}{9}, \bibinfo{number}{1} (\bibinfo{year}{1941}), \bibinfo{pages}{1--24}.
\newblock
\bibitem[Richardson et~al\mbox{.}(2007)]%
{richardson-www07}
\bibfield{author}{\bibinfo{person}{Matthew Richardson}, \bibinfo{person}{Ewa Dominowska}, {and} \bibinfo{person}{Robert Ragno}.} \bibinfo{year}{2007}\natexlab{}.
\newblock \showarticletitle{Predicting clicks: estimating the click-through rate for new ads}. In \bibinfo{booktitle}{\emph{Proceedings of the 16th international conference on World Wide Web}}. \bibinfo{pages}{521--530}.
\newblock
\bibitem[Rivers and Vuong(1988)]%
{rivers-econometrica88}
\bibfield{author}{\bibinfo{person}{Douglas Rivers} {and} \bibinfo{person}{Quang~H. Vuong}.} \bibinfo{year}{1988}\natexlab{}.
\newblock \showarticletitle{Limited information estimators and exogeneity tests for simultaneous probit models}.
\newblock \bibinfo{journal}{\emph{Journal of Econometrics}} \bibinfo{volume}{39}, \bibinfo{number}{3} (\bibinfo{year}{1988}), \bibinfo{pages}{347--366}.
\newblock
\href{https://doi.org/10.1016/0304-4076(88)90063-2}{doi:\nolinkurl{10.1016/0304-4076(88)90063-2}}
\bibitem[Saito(2020)]%
{saito-recsys20}
\bibfield{author}{\bibinfo{person}{Yuta Saito}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Doubly robust estimator for ranking metrics with post-click conversions}. In \bibinfo{booktitle}{\emph{Proceedings of the 14th ACM Conference on Recommender Systems}}. \bibinfo{pages}{92--100}.
\newblock
\bibitem[Smucker et~al\mbox{.}(2007)]%
{smucker-cikm07}
\bibfield{author}{\bibinfo{person}{Mark~D Smucker}, \bibinfo{person}{James Allan}, {and} \bibinfo{person}{Ben Carterette}.} \bibinfo{year}{2007}\natexlab{}.
\newblock \showarticletitle{A comparison of statistical significance tests for information retrieval evaluation}. In \bibinfo{booktitle}{\emph{Proceedings of the sixteenth ACM conference on Conference on information and knowledge management}}. \bibinfo{pages}{623--632}.
\newblock
\bibitem[Tian et~al\mbox{.}(2020)]%
{tian-sigir20}
\bibfield{author}{\bibinfo{person}{Mucun Tian}, \bibinfo{person}{Chun Guo}, \bibinfo{person}{Vito Ostuni}, {and} \bibinfo{person}{Zhen Zhu}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Counterfactual Learning to Rank using Heterogeneous Treatment Effect Estimation}.
\newblock \bibinfo{journal}{\emph{Proceedings of ACM SIGIR Workshop on eCommerce (SIGIR eCom)}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Vardasbi et~al\mbox{.}(2020)]%
{vardasbi-cikm20}
\bibfield{author}{\bibinfo{person}{Ali Vardasbi}, \bibinfo{person}{Harrie Oosterhuis}, {and} \bibinfo{person}{Maarten de Rijke}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{When inverse propensity scoring does not work: Affine corrections for unbiased learning to rank}. In \bibinfo{booktitle}{\emph{Proceedings of the 29th ACM International Conference on Information \& Knowledge Management}}. \bibinfo{pages}{1475--1484}.
\newblock
\bibitem[Wang et~al\mbox{.}(2024)]%
{wang-www24}
\bibfield{author}{\bibinfo{person}{Bohao Wang}, \bibinfo{person}{Jiawei Chen}, \bibinfo{person}{Changdong Li}, \bibinfo{person}{Sheng Zhou}, \bibinfo{person}{Qihao Shi}, \bibinfo{person}{Yang Gao}, \bibinfo{person}{Yan Feng}, \bibinfo{person}{Chun Chen}, {and} \bibinfo{person}{Can Wang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Distributionally robust graph-based recommendation system}. In \bibinfo{booktitle}{\emph{Proceedings of the ACM Web Conference 2024}}. \bibinfo{pages}{3777--3788}.
\newblock
\bibitem[Wang et~al\mbox{.}(2013)]%
{wang-www13}
\bibfield{author}{\bibinfo{person}{Hongning Wang}, \bibinfo{person}{ChengXiang Zhai}, \bibinfo{person}{Anlei Dong}, {and} \bibinfo{person}{Yi Chang}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{Content-aware click modeling}. In \bibinfo{booktitle}{\emph{Proceedings of the 22nd international conference on World Wide Web}}. \bibinfo{pages}{1365--1376}.
\newblock
\bibitem[Wang et~al\mbox{.}(2016)]%
{wang-sigir16}
\bibfield{author}{\bibinfo{person}{Xuanhui Wang}, \bibinfo{person}{Michael Bendersky}, \bibinfo{person}{Donald Metzler}, {and} \bibinfo{person}{Marc Najork}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Learning to Rank with Selection Bias in Personal Search}. In \bibinfo{booktitle}{\emph{Proceedings of the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval}}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{115–124}.
\newblock
\bibitem[Wang et~al\mbox{.}(2018)]%
{wang-wsdm18}
\bibfield{author}{\bibinfo{person}{Xuanhui Wang}, \bibinfo{person}{Nadav Golbandi}, \bibinfo{person}{Michael Bendersky}, \bibinfo{person}{Donald Metzler}, {and} \bibinfo{person}{Marc Najork}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Position Bias Estimation for Unbiased Learning to Rank in Personal Search}. In \bibinfo{booktitle}{\emph{Proceedings of the 11th ACM International Conference on Web Search and Data Mining (WSDM)}}. \bibinfo{pages}{610--618}.
\newblock
\bibitem[Wooldridge(2015)]%
{wooldridge-jhr15}
\bibfield{author}{\bibinfo{person}{Jeffrey~M Wooldridge}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{Control function methods in applied econometrics}.
\newblock \bibinfo{journal}{\emph{Journal of Human Resources}} \bibinfo{volume}{50}, \bibinfo{number}{2} (\bibinfo{year}{2015}), \bibinfo{pages}{420--445}.
\newblock
\bibitem[Wu et~al\mbox{.}(2023)]%
{wu-tkdd23}
\bibfield{author}{\bibinfo{person}{Yao Wu}, \bibinfo{person}{Jian Cao}, {and} \bibinfo{person}{Guandong Xu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Fairness in recommender systems: evaluation approaches and assurance strategies}.
\newblock \bibinfo{journal}{\emph{ACM Transactions on Knowledge Discovery from Data}} \bibinfo{volume}{18}, \bibinfo{number}{1} (\bibinfo{year}{2023}), \bibinfo{pages}{1--37}.
\newblock
\bibitem[Yan et~al\mbox{.}(2022)]%
{yan-sigir22}
\bibfield{author}{\bibinfo{person}{Le Yan}, \bibinfo{person}{Zhen Qin}, \bibinfo{person}{Honglei Zhuang}, \bibinfo{person}{Xuanhui Wang}, \bibinfo{person}{Michael Bendersky}, {and} \bibinfo{person}{Marc Najork}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Revisiting two-tower models for unbiased learning to rank}. In \bibinfo{booktitle}{\emph{Proceedings of the 45th international ACM SIGIR conference on research and development in information retrieval}}. \bibinfo{pages}{2410--2414}.
\newblock
\bibitem[Yuan et~al\mbox{.}(2019)]%
{yuan-cikm19}
\bibfield{author}{\bibinfo{person}{Bowen Yuan}, \bibinfo{person}{Jui-Yang Hsia}, \bibinfo{person}{Meng-Yuan Yang}, \bibinfo{person}{Hong Zhu}, \bibinfo{person}{Chih-Yao Chang}, \bibinfo{person}{Zhenhua Dong}, {and} \bibinfo{person}{Chih-Jen Lin}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Improving ad click prediction by considering non-displayed events}. In \bibinfo{booktitle}{\emph{Proceedings of the 28th ACM International Conference on Information and Knowledge Management}}. \bibinfo{pages}{329--338}.
\newblock
\bibitem[Yue et~al\mbox{.}(2010)]%
{yue-www10}
\bibfield{author}{\bibinfo{person}{Yisong Yue}, \bibinfo{person}{Rajan Patel}, {and} \bibinfo{person}{Hein Roehrig}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{Beyond position bias: Examining result attractiveness as a source of presentation bias in clickthrough data}. In \bibinfo{booktitle}{\emph{Proceedings of the 19th international conference on World wide web}}. \bibinfo{pages}{1011--1018}.
\newblock
\bibitem[Zhao et~al\mbox{.}(2025)]%
{zhao-www25}
\bibfield{author}{\bibinfo{person}{Chu Zhao}, \bibinfo{person}{Enneng Yang}, \bibinfo{person}{Yuliang Liang}, \bibinfo{person}{Pengxiang Lan}, \bibinfo{person}{Yuting Liu}, \bibinfo{person}{Jianzhe Zhao}, \bibinfo{person}{Guibing Guo}, {and} \bibinfo{person}{Xingwei Wang}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Graph Representation Learning via Causal Diffusion for Out-of-Distribution Recommendation}. In \bibinfo{booktitle}{\emph{Proceedings of the ACM on Web Conference 2025}}. \bibinfo{pages}{334--346}.
\newblock
\bibitem[Zhu et~al\mbox{.}(2020)]%
{zhu-recsys20}
\bibfield{author}{\bibinfo{person}{Ziwei Zhu}, \bibinfo{person}{Yun He}, \bibinfo{person}{Yin Zhang}, {and} \bibinfo{person}{James Caverlee}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Unbiased implicit recommendation and propensity estimation via combinational joint learning}. In \bibinfo{booktitle}{\emph{Proceedings of the 14th ACM Conference on Recommender Systems}}. \bibinfo{pages}{551--556}.
\newblock
\bibitem[Zou et~al\mbox{.}(2022)]%
{zou-cikm22}
\bibfield{author}{\bibinfo{person}{Lixin Zou}, \bibinfo{person}{Changying Hao}, \bibinfo{person}{Hengyi Cai}, \bibinfo{person}{Shuaiqiang Wang}, \bibinfo{person}{Suqi Cheng}, \bibinfo{person}{Zhicong Cheng}, \bibinfo{person}{Wenwen Ye}, \bibinfo{person}{Simiu Gu}, {and} \bibinfo{person}{Dawei Yin}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Approximated doubly robust search relevance estimation}. In \bibinfo{booktitle}{\emph{Proceedings of the 31st ACM International Conference on Information \& Knowledge Management}}. \bibinfo{pages}{3756--3765}.
\newblock
\end{thebibliography}