Reducing Disparate Exposure in Ranking: A Learning To Rank Approach
Extracted source text ยท arXiv 1805.08716
\documentclass[sigconf]{acmart}
\settopmatter{printacmref=false} % Removes citation information below abstract
\renewcommand\footnotetextcopyrightpermission[1]{} % removes footnote with conference information in first column
\pagestyle{plain} % removes running headers
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% BEFORE SUBMITTING
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Before submitting for review
% - Spellcheck
% - Check anonymity and anonymize authors (SIGIR 2019 IS DOUBLE-BLIND)
% - Check page limits (SIGIR 2019 LIMIT IS 9 PAGES plus references)
% - Change [show] to [hide] in "\usepackage[show]{chato-notes}"
\usepackage[hide]{chato-notes} % \note, \todo, \inote, \citemissing
% - Remove page numbers by setting printfolios=false next
\settopmatter{printfolios=true}
% Before submitting camera ready
% - Include ccsxml, ccsdesc, and keywords in METADATA and set printcss=true
\settopmatter{printccs=true}
% - Include bibliographic info in METADATA and set printacmref=true below
\settopmatter{printacmref=false}
% - Set colorlinks=false in hypersetup below
% - Include links to code and data releases
% - Include acknowledgments if they were removed
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Packages
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\usepackage{booktabs} % For formal tables
\usepackage{tikz, pgfplots}
\pgfplotsset{compat=1.14} % Prevent compatibility warning
\usepackage{graphicx}
\usepackage{dblfloatfix}
\usepackage{breqn}
\usepackage{paralist}
%\usepackage{wasysym}
\usepackage{balance} % Removed -- causes problems with pages of floats
\usepackage{mathtools}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage[captionskip=1pt,farskip=2pt]{subfig}
\usepackage{wrapfig}
\usepackage[justification=justified]{caption}
\usepackage{placeins}
\usepackage{array}
\hypersetup{
colorlinks,
linkcolor={red!20!black},
citecolor={green!20!black},
urlcolor={blue!20!black}
}
% Reduce overfull \hbox warnings
\hfuzz=10pt
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% ACM TEMPLATE OPTIONS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Copyright
\setcopyright{none}
%\setcopyright{rightsretained}
%\acmDOI{TBA}
%\acmISBN{TBA}
%\acmConference[CIKM'19]{CIKM}{November 2019}{Beijing, China}
%\acmYear{2019}
%\copyrightyear{2019}
%\acmArticle{TBA}
%\acmPrice{}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NEW COMMANDS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newcommand{\doc}[1]{\ensuremath{d^{(#1)}}}
\newcommand{\generaljudg}[1]{\ensuremath{s^{(#1)}}}
\newcommand{\trainjudg}[1]{\ensuremath{y^{(#1)}}}
\newcommand{\predictjudg}[1]{\ensuremath{\hat{y}^{(#1)}}}
\newcommand{\feat}[1]{\ensuremath{x^{(#1)}}}
\newcommand{\lfair}[0]{\ensuremath{L_{\mathit{DELTR}}}}
\newcommand{\twopartdef}[4]
{
\left\{
\begin{array}{ll}
#1 & \mbox{if } #2 \\
#3 & \mbox{if } #4
\end{array}
\right.
}
\newcommand{\barplotsize}{0.33}
\newcommand{\barplotheight}{0.8in}
\newcommand{\scattersize}{0.33}
\newcommand{\scatterheight}{1.8in}
\newcommand{\spara}[1]{\smallskip\noindent{\bf #1}}
\newcommand{\mpara}[1]{\medskip\noindent{\bf #1}}
\newcommand{\methodname}{\textsc{DELTR}\xspace}
\newcommand{\methodnameexpanded}{Disparate Exposure in Learning To Rank}
\newcommand{\baseline}{\textsc{FA*IR}\xspace}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% TITLE, AUTHORS, AND METADATA
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Use \mbox{} for the parts you want to keep together
\title{\mbox{Reducing Disparate Exposure in Ranking:} \mbox{A Learning to Rank Approach}}
% Author information
\author{Meike Zehlike}
\affiliation{%
\institution{Humboldt Universit\"at zu Berlin}
\institution{Max Planck Inst. for Software Systems}
}
\email{meikezehlike@mpi-sws.org}
\author{Gina-Theresa Diehn}
\affiliation{%
\institution{Technische Universit\"at Berlin}
}
\email{gina.diehn@googlemail.com}
\author{Carlos Castillo}
\affiliation{%
\institution{Universitat Pompeu Fabra}
}
\email{chato@acm.org}
% The default list of authors is too long for headers}
\renewcommand{\shortauthors}{Zehlike et. al}
%%% METADATA
%
% Generate using: http://dl.acm.org/ccs.cfm
%\begin{CCSXML}
%<ccs2012>
%<concept>
%<concept_id>10002951.10003317.10003338.10003343</concept_id>
%<concept_desc>Information systems~Learning to rank</concept_desc>
%<concept_significance>500</concept_significance>
%</concept>
%<concept>
%<concept_id>10002951.10003317.10003338.10003346</concept_id>
%<concept_desc>Information systems~Top-k retrieval in databases</concept_desc>
%<concept_significance>500</concept_significance>
%</concept>
%<concept>
%<concept_id>10010405.10010455</concept_id>
%<concept_desc>Applied computing~Law, social and behavioral sciences</concept_desc>
%<concept_significance>300</concept_significance>
%</concept>
%</ccs2012>
%\end{CCSXML}
%\ccsdesc[500]{Information systems~Learning to rank}
%\ccsdesc[500]{Information systems~Top-k retrieval in databases}
%\ccsdesc[300]{Applied computing~Law, social and behavioral sciences}
%
%\keywords{Ranking, Algorithmic Fairness, Disparate Impact}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% MAIN BODY
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
\linespread{0.9}
%\graphicspath{{pics/}}
%\input{plots}
\input{01-introduction}
\input{02-related-work}
\input{03-fundamentals}
\input{04-deltr}
\input{05-experiments}
\input{06-conclusions}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% BIBLIOGRAPHY
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\balance % Removed -- causes problems with pages of floats
\balance
\bibliographystyle{ACM-Reference-Format}
\bibliography{main}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\appendix
%\input{appendix}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% REMOVED MATERIAL
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\clearpage
%\section*{Removed Material}
%\input{removed}
\end{document}
%!TEX root=main.tex
\begin{abstract}
In this paper we introduce \methodname, a learning-to-rank framework that addresses potential issues of discrimination and unequal opportunity in rankings.
%
Following long-standing empirical observations showing that users of information retrieval systems rarely look past the first few results, we measure these problems in terms of discrepancies in the \emph{average group exposure}.
%
Specifically, we define our notion of group exposure as the average probability of items from a \emph{legally protected} social group to be ranked at the top position.
%
With this we design a ranker that optimizes search results in terms of relevance, while at the same time reducing potential discrimination or inequality of opportunity.
We describe this objective formally, how to optimize it efficiently, and how to implement it.
%
We perform an extensive experimental study showing that being ``colorblind,'' i.e. ignoring protected attributes such as race or gender, can be among the best choices or the worst choices from the perspective of relevance and exposure, depending on how much and which kind of bias is present in the training set.
%
As baselines for benchmarking our in-processing method we use pre-processing and post-processing methods based on \baseline, a state-of-the-art algorithm to re-rank search results according to predefined fairness constraints.
%
%We then demonstrate how it can be applied in a pre-processing manner to obtain presumably fairer training data.
%
We show that our in-processing method performs better in terms of relevance and equality of exposure than pre-processing and post-processing across all tested scenarios. %in both scenarios, as it considers both item relevance and exposure during the construction of rankings. % has distinct drawbacks in both scenarios, because it does not consider item relevance.
Our proposed method neither makes assumptions about biases in the training data, nor does it ignore relevance scores of items and thus can reduce discrimination and inequality of opportunity without having to introduce large distortions in ranking relevance.
\end{abstract}
\maketitle
\section{Introduction}
\label{sec:introduction}
Ranked search results, news feeds, and recommendations, have become the main mechanism by which we find content, products, places, and people online.
%
These rankings are typically constructed to provide a maximum utility to searchers, for instance, %through a framework such as the Probability Ranking Principle (PRP), which states that a ranking system should
by ordering items by decreasing probability of being relevant~\cite{robertson1977probability}.
%
However, when the items to be ranked represent people, businesses or places, ranking algorithms have consequences that go beyond immediate utility for searchers.
%
With hiring, selecting, purchasing, and dating being increasingly mediated by algorithms, rankings may determine career and business opportunities, educational placement, access to benefits, and even social and reproductive success.
Over the past decade, data mining researchers became increasingly concerned with various systematic biases \cite{friedman1996bias} against any specific group (i.e.,~\emph{group discrimination}), caused by historic and current discriminatory patterns in society making their way into data-driven models.
%
A common element in this line of research is the presence of a historically and currently disadvantaged \emph{protected group}, and the concern of \emph{disparate impact}, i.e., loss of opportunity for the protected group independently of whether they are (intentionally) treated differently.
%
In the case of rankings, a natural way of understanding disparate impact is by considering differences in exposure~\cite{SINGH2018FAIRNESS} or inequality of attention~\cite{biega2018equity}, which translate into systematic differences in access to economic or social opportunities. % for different social groups. %benefits such as business opportunities or employment.
%
%For example, a large online-shopping platform was developing a machine-learning based hiring tool that was found to discriminate systematically against women due to biased training data; the tool was ultimately abandoned~\cite{reuters2018}.
\spara{Disparate exposure in rankings.}
%
There are a number of issues, sometimes appearing simultaneously, that call for reducing disparate exposure in information retrieval systems.
%
First, there can be a situation in which minimal differences in relevance translate into large differences in exposure for different groups~\cite{SINGH2018FAIRNESS,biega2018equity}, because of the large skew in the distribution of exposure brought by positional bias~\cite{joachims2017accurately}.
%
Second, there can be a legal requirement, policy, or voluntary commitment that requires that elements in the protected group are given sufficient visibility among the top positions in a ranking~\cite{zehlike2017fa,celis2017ranking}.
%
Third, there can be systematic differences in the way in which documents are constructed, as in the case of different sections in online resumes, which are completed differently by men and women~\cite{altenburger2017there}; these differences may systematically affect ranking algorithms.
%
Fourth, there can be systematic differences in the way ground truth rankings have been generated due to historical discrimination and/or annotator bias. %, such as a particular process for merging lists that uses a different probability for picking elements from each group~\cite{yang2017measuring}.
These situations point to two conceptually different goals:
\emph{reducing inequality of opportunities} and \emph{reducing discrimination} (as defined by \citet{roemer1998equality}, chapter 12).
%
%Both goals can be addressed using the same tools.
%
Equality of opportunity seeks to correct a historical or present disadvantage for a group of society.
%and is the main driving force behind positive action programs,
Non-discrimination seeks to allocate resources in a way that does not consider irrelevant attributes, and is a matter of efficiency.
%
%Equality of opportunity directly benefits individuals that experience a situation of disadvantage and indirectly benefits society at large; non-discrimination directly benefits both.
\methodname can be applied in both cases, as our experiments show.
\spara{Post-processing methods for fair rankings.}
%
Fairness-aware data-driven methods can be classified into \emph{pre-, in-} and \emph{post-processing} approaches, in which pre-processing methods seek to mitigate bias in training data, in-processing methods learn a bias-free model, and post-processing methods re-rank output items~\cite{hajian2016algorithmic}. % according to pre-defined fairness criteria~\cite{hajian2016algorithmic}.
For rankings, several post-processing methods have been presented in the scientific literature \cite{biega2018equity, celis2017ranking, SINGH2018FAIRNESS, zehlike2017fa}, but these methods are problematic for various reasons.
%
First, the post-processing idea suggests that there is \emph{always} a trade-off between an optimally \emph{fair} and an optimally \emph{relevant} ranking, because a presumably ``exact'' model produces a ``relevant'' ranking that is then reordered to meet fairness constraints.
%
We show in our experiments that this assumption is wrong, as reducing bias against a protected group can increase relevance (experiment~\ref{subsec:exp-chile-highschool}).
%
Second, it is likely that post-processing algorithms are legally open to criticism as the Ricci v. DeStefano ruling (2009) in the US Supreme Court illustrates.
%
At the center of this case was a test administered by the firefighter department of New Haven, in which white firefighters scored significantly better than black ones.
%
The result was then rejected by the department.
%
The US Supreme Court ruled that the white firefighters were subjected to race discrimination, because the department could not demonstrate a ``strong basis in evidence that using the results would cause them to lose a disparate-impact suit.''
%
However, ``[e]mployers may consider potential racial impact during the test-design stage''.~\cite{liptak2009supreme}
%
This decision states that it is as much unlawful to score people solely on the basis of race, as it is to reject scores produced from lawful exams and procedures solely on that very same basis, which is exactly what post-processing ranking algorithms do.
%
One crucial advantage of in-processing approaches such as \methodname is that they satisfy the condition of ``considering potential racial impact during the test-design stage''.
%
Pre-processing methods also satisfy this condition. However, they suggest that if we only had unbiased training data, we could use standard ranking tools without having to worry about biased models.
%
We show that creating an unbiased training set is not trivial and may easily lead to reverse discrimination.
\spara{Our contribution.}
%
We address the problem of mitigating discrimination and inequality of opportunity in rankings by reducing disparate exposure under a learning-to-rank framework.
%
We design \methodname as a list-wise learning-to-rank approach that
% incorporates a measure of relevance as well as a measure of fairness into its objective function.
%
%The result is
provides an \emph{in-processing} approach to fairness-aware rankings.
We perform extensive experiments in two different ranking tasks: expert search in a document retrieval setting, and ranking students by predicted performance.
%
Our experiments comprise three real-world datasets, of which two are newly introduced (section~\ref{subsec:dataset-trec} and \ref{subsec:dataset-chile}).
%
We further show that being ``colorblind'' on discriminatory training data, i.e., simply ignoring protected attributes, which is a naive attempt to overcome discriminatory models, can yield the best results in some cases, while in others it is among the worst results both in terms of performance and fairness.
%
This makes it difficult to identify when or when not to include the protected attribute in the training process.
%
We describe the reasons for this somewhat counterintuitive behavior in detail and show that \methodname performs well in terms of fairness and relevance in all tested scenarios.
As one baseline we demonstrate a pre-processing approach for fairness in rankings by applying a post-processing method, \baseline~\cite{zehlike2017fa} to our training data before the learning routine starts.
%
These experiments show two interesting insights:
\begin{inparaenum}[(i)]
\item it is not easy to produce fair training data, because discrimination is baked into all attributes, and a truly bias-free dataset is just wishful thinking, as argued earlier in this section; and
\item re-ordering items in a ``fair'' way can lead to significant performance decline and even to reverse discrimination.
\end{inparaenum}
%
We show that \methodname does not suffer from the aforementioned problems of post- or pre-processing methods, as it makes no assumptions on whether or not bias against a protected group is present in the training data. It performs well in both cases.
Additionally, we show that the current understanding of a necessary trade-off between relevance and fairness can be sometimes misleading.
%
Only if we seek to enhance equality of opportunity, we have to trade performance against fairness (experiments~\ref{subsec:exp-chile-gender}, \ref{subsec:exp-law-gender} and \ref{subsec:exp-law-black}).
%
In a case of non-discrimination, optimizing for fairness as equal exposure will increase relevance (experiments~\ref{subsec:exp-trec} and \ref{subsec:exp-chile-highschool}).
%!TEX root = main.tex
\section{Related Work}\label{sec:related-work}
\spara{Fairness in data-driven modeling.}
%
In recent years the data mining and machine learning communities have been increasingly concerned with \emph{algorithmic bias}, particularly with the fact that sensitive attributes
%such as gender, race, or disability status,
have been found to have an observable impact on machine learning outcomes~\cite{hajian2016algorithmic}.
%
Membership in socially salient groups defines a protected characteristic, while merely removing sensitive attributes from training data may have little or no effect on a data-driven model~\cite{hardt2014big}.
%
Algorithmic fairness as defined by~\citet{zliobaite2015survey} seeks that:
\begin{inparaenum}[(1)]
\item people that are similar in terms of non-protected characteristics should receive
similar predictions, and
\item differences in predictions across groups of people can only be as large as justified by non-protected characteristics.
\end{inparaenum}
%
The study of algorithmic discrimination and fairness is connected to open debates in moral philosophy including, among other topics, distributive justice and egalitarianism (see, e.g., \citet{binns2018fairness}).
There are various approaches to algorithmic fairness.
%
A basic method for algorithmic fairness seeks to reduce \emph{disparate impact} by achieving \emph{statistical parity} between the outcomes for protected and non-protected elements; however, this can be inadequate if the outcomes also depend on non-protected, legitimate attributes~\cite{dwork2012fairness}.
%
Other methods define algorithmic fairness in terms of predicted and actual outcomes, reducing differences in false positive or true positive rates that have been called \emph{disparate mistreatment}~\cite{zafar2017fairness} or unequal opportunity~\cite{hardt2016equality}.
%Our research is inspired by these works, in the sense that the increase in exposure of a protected group that we seek, is comparable to the increase in the rate of positive outcomes that fairness-aware machine learning researchers seek.
%%
%However, our goal is ranking, not classification.
\spara{Fairness in rankings.}
%
%In comparison with research on fair classification, much less work has been done on fair rankings.
%
Fairness in ranking is concerned with a sufficient presence, a consistent treatment, and a proper representation of different groups across all ranking positions~\cite{castillo2018fairness}.
%
At a high level, this research is motivated by producing rankings based on relevant characteristics of items, in which items belonging to the protected group are not under-represented or relegated systematically to lower ranking positions~\cite{yang2018nutritional}.
%
This requires new evaluation metrics that extend relevance-based ones~\cite{culpepper2018swirl}.
\citet{yang2017measuring} introduce a generative model for fair rankings with two groups (protected and non-protected).
%This model is based on ranking the two groups separately, and then merging these rankings by drawing independently at random the next element with probability $p$ from the protected group, and with probability $1-p$ from the non-protected group.
%
They also introduce a series of ranking-aware measures of disparity, such as averaging differences in NDCG (Normalized Discounted Cumulative Gain) at different cut-off points across both groups.
\citet{zehlike2017fa} construct a statistical test for the generative model of \citet{yang2017measuring}. Given a parameter $p$ and a statistical significance $\alpha$, reject a ranking that has probably not been generated according to this process, based on an adjusted binomial test.
%
%The test is based on a minimal number of protected elements at each position.
%
They also provide a method for generating a ranking that passes this statistical test, given two separate rankings for the two groups.
%
\citet{celis2017ranking} consider a situation in which several protected groups exist and hence several vectors containing the exact number of protected elements (one per group) at each position are given as input.
\citet{singh2018equality} introduce the concept of \emph{exposure} of a group based on empirical observations that show that the probability that a user examines an item ranked at a certain position, decreases rapidly with the position.
%
%Given a vector of relevance scores or utilities for each item, a vector indicating group membership, and a vector of exposures associated to each ranking position, they construct a probabilistic ranking bounding the difference of the summation of the exposure of the two groups.
%
\citet{biega2018equity}, in work parallel to \citet{singh2018equality}, introduce an integer linear programming formulation that receives a vector of relevance scores and produces a ranking that places high-scoring elements first, and at the same time minimizes the accumulated attention received by elements in both groups.
%
%They incorporate the idea of optimizing for amortized attention across several queries in an online manner.
%\citet{zehlike2017fa}, \citet{singh2018equality} and \citet{biega2018equity} take as input scores representing the relevance or utility of items.
%
Previous works in fair rankings~\cite{yang2017measuring,zehlike2017fa,celis2017ranking,singh2018equality,biega2018equity} have been concerned with creating a fairness-aware ranking given a set of scores, and can be considered \emph{post-processing} approaches---they are given a ranking and re-rank elements to achieve a desired objective.
%
In contrast, our approach \methodname is \emph{learning-based} and the first \emph{in-processing} approach to reduce discrimination and inequality of opportunity in rankings, because it learns a ranking function with an additional objective that reduces disparate exposure.
%
In the experiments on this paper, we additionally describe how to use a post-processing method on training data to implement a pre-processing approach.
\spara{Diversity in rankings.}
%
%Comparatively, \emph{diversity} in ranking has been explored more in depth in information retrieval than fairness.
%
A classical definition of diversity in ranking is related to the marginal relevance of a document for a user, considering the documents ranked above it, that the user has already seen~\cite{carbonell1998use}.
%
Another often-used definition is that diversity should be understood as a way of incorporating uncertainty over user intents, in the sense that all queries have some degree of ambiguity~\cite{agrawal2009diversifying}.
%
Both interpretations (``seeking variety'' and ``hedging bets'') are present in contemporary accounts of diversity in data-driven methods~\cite{drosou2017diversity}.
In contrast with diversification approaches, \emph{we are not only concerned with the utility that search system users receive}, but also with the exposure of the items being ranked, which can represent individual, organizations, or places.
%
In other words, we also consider their utility.
%
Another key difference is that diversification is usually symmetric so groups are interchangeable, while \emph{fairness-aware algorithms are usually asymmetric}, as they focus on increasing the overall benefit received by a disadvantaged or protected group.
%!TEX root = main.tex
\section{Preliminaries: ListNet}\label{sec:fundamentals}
In this section we describe ListNet~\cite{cao2007learning},
a well-known list-wise learning to rank framework.
%
Given that lists present a natural way to measure disparate exposure across groups for an entire ranking, we describe \methodname using ListNet as a base algorithm.
%
\begin{table}[t]
\caption{Summary of Notation}
\centering\small
\begin{tabular}{r p{7cm} }
\toprule
$ Q $ & set of queries $q$ with $ |Q| = m $\\
$ D $ & set of documents \\
$ \doc{q}_i $ & a document associated to query $ q $\\
$\generaljudg{q}_i$ & a general judgment on document $ \doc{q}_i $ for query $q$\\
$\feat{q}_i$ & feature vector for document $\doc{q}_i$\\
$\trainjudg{q}$ & list of training judgments \\
$f$ & ranking model\\
$\predictjudg{q}$ & list of predicted judgments \\
$ L $ & error between the training judgments and those predicted by model $ f $ \\
$ P_s(i) $ & probability for document $ i $ to be ranked at the top position\\
$ G_k $ & groups of documents identifiable by the presence or absence of sensitive attributes\\
$ v_j $ & position bias\\
$ U $ & disparate exposure metric\\
$ \lfair $ & loss function that incorporates $ L $ and $ U $ at the same time\\
$ \gamma $ & tuning parameter\\
\bottomrule
\end{tabular}
\vspace*{-\baselineskip}
\label{tbl:notation}
\end{table}
%
Readers familiar with ListNet can skip this section.
%
The notation we use is summarized on Table~\ref{tbl:notation}.
\spara{List-wise learning to rank.}\label{subsec:Notation}
%
We consider a set of queries $ Q $ with $|Q|=m$ and a set of documents $D$ with $|D|=n$. %= \left\{d_1, d_2, \ldots, d_{n(q)}\right\}$ a set of documents, where
%
Each query $q$ is associated with a list of candidate documents $\doc{q} \subseteq D$. %$\doc{q} = ( \doc{q}_1, \dots, \doc{q}_{n(q)} )$ with $n(q) \le n$.
%
We denote by $ \generaljudg{q}_i \in \mathbb{R}_0^+$ for $q \in Q; i = 1, 2, \dots, |\doc{q}|$ a judgment on document $ \doc{q}_i $ for query $q$, that indicates the extent to which document candidate $\doc{q}_i$ is relevant for $ q $.
%It can be a label in a classification task or a real number in a scoring task, as at this point of the discussion it is not yet necessary to define, whether these judgments relate to a point-, pair- or list-wise learning to rank approach.
For each query $ q $ the list of candidate documents is associated with a list of judgments: $ \doc{q} \rightarrow \generaljudg{q} = \left( \generaljudg{q}_1, \generaljudg{q}_2, \ldots,\generaljudg{q}_{n^{(q)}} \right)$.
%
%The judgment $ \generaljudg{q}_i $ represents, if the associated document is relevant to query $ q $ or not, if it should be above another document from $ \doc{q} $, or represent the relevance degree of $ \doc{q}_i $ to $ q $.
%
For a clearer distinction between different judgment sets we call $ \trainjudg{q} $ the judgments of the training data and $ \predictjudg{q} $ the judgments predicted by the model.
From each document $ \doc{q}_i $ we can derive a feature vector $\feat{q}_i$. % = \Psi \left(q, \doc{q}_i\right)$.
%
Each list of feature vectors $ \feat{q} $ $=$ $ \left( \feat{q}_1, \feat{q}_2, \ldots, \feat{q}_{n^{(q)}} \right) $ and the corresponding list of judgments $ \trainjudg{q} = \left( \trainjudg{q}_1, \trainjudg{q}_2, \ldots, \trainjudg{q}_{n^{(q)}} \right) $ form an instance of the training set $\mathcal{T} = \left\{(\feat{q}, \trainjudg{q})\right\}_{q \in Q} $.
%
The standard learning-to-rank objective then is to learn a ranking function $ f $ that outputs a new judgment $ \predictjudg{q}_i $ for each feature vector $ \feat{q}_i $ which forms a second list of judgments $ \predictjudg{q} = \left( f(\feat{q}_1), f(\feat{q}_2), \ldots, f(\feat{q}_{n^{(q)}}) \right) $.
Ideally, the function $ f $ should be such that the sum of the differences (or losses) $ L $ between the training judgments $ \trainjudg{q} $ and the predicted judgments $ \predictjudg{q} $ is minimized:
\[\min\left(\sum_{q \in Q} L \left( \trainjudg{q}, \predictjudg{q} \right)\right)~.\]
In list-wise learning to rank, training elements are processed as lists of elements (not as individual elements having scores, which corresponds to point-wise learning to rank, or as pairs of elements, which corresponds to pair-wise learning to rank).
\spara{Probability models.}
%
As rankings are combinatorial objects the naive approach to find an optimal solution for $ L $ leads to exponential execution time in the number of documents. Hence instead of considering an actual permutation of documents we will reuse Theorem 6 and Lemma 7 from \citet{cao2007learning}, which focuses on the probability for a document $ d_i^{(q)} $ to be ranked onto the top position:
\begin{equation}
\label{eq:topOneProbability}
P_{s^{(q)}} \left( d_i^{(q)} \right) = \frac{\phi\left(s^{(q)}_i\right)}{\sum_{j=1}^{n}\phi\left(s^{(q)}_j\right)}
\end{equation}
with $ \phi : \mathbb{R}^+_0 \longrightarrow \mathbb{R}^+ $ being an increasing strictly positive function and in which $ s^{(q)}_j $ denotes a relevance score/judgment for document $ j $. %(as previously explained).
%
The top-one-probabilities $ P_{s^{(q)}}(d_i^{(q)}) $ form a probability distribution $ P_{s^{(q)}} $ over $ d^{(q)} $.
In a general list-wise learning-to-rank setting the document judgments $ \generaljudg{q} $ are given as lists of scores that represent the respective relevance degree of document $ \doc{q}_i $ to query $ q $.
%
%Following~\citet{cao2007learning},
%
Documents are sorted by decreasing top-one probabilities as predicted by the algorithm.
\spara{Loss function in list-wise learning to rank.}\label{concept:loss-standard}
%
Setting $ P_{s^{(q)}} $ to $P_{\trainjudg{q}}(\feat{q}_i)$ or $P_{\predictjudg{q}}(\feat{q}_i)$ respectively, leads to a way of measuring the distance between the judgments provided in the training set $ \trainjudg{q} $ and the judgments $ \predictjudg{q} $ produced by our function $ f $.
%
Following \citet{cao2007learning} we use the Cross Entropy metric for the loss function:
\begin{equation}
L \left( \trainjudg{q}, \predictjudg{q} \right) = - \sum_{i=1}^{|\doc{q}|}P_{\trainjudg{q}}(\feat{q}_i)\log\left(P_{\predictjudg{q}}(\feat{q}_i)\right)
\end{equation}
%!TEX root = main.tex
\section{Our Method: \methodname}\label{sec:deltr}
In this section we describe our method, \methodname (\methodnameexpanded).
%
We assume that the retrieved items represent people belonging to two distinct socially salient~\cite{lippert2014born} groups (such as men and women, or majority and minority ethnicity).
%
Using terminology from non-discriminatory data mining, we assume one of these groups is \emph{protected}~\cite{pedreshi2008discrimination}.
%
\methodname is a supervised learning-to-rank algorithm that simultaneously seeks to minimize ranking errors with respect to training data, and to reduce disadvantages experienced by the protected group in terms of exposure.
%Without loss of generality, we use document retrieval as the running example in this paper, and
%We describe the algorithm in terms of the two phases in supervised learning: training and testing.
%
At training time, we are given an annotated set consisting of queries and ordered lists of items for each query. The algorithm learns from training data by minimizing a loss function. % Our modified optimization problem shall integrate a measure of accuracy as well as a measure of disparate exposure and find the optimal solution for both constraints.
%
At testing time, we provide a query and a document collection, and expect as output a set of top-$k$ items from the collection that should be relevant for the query, and additionally should not exhibit disparate exposure.
%
%We demonstrate this general approach using ListNet~\cite{cao2007learning}, which is a classic list-wise learning to rank method. In contrast to other types of learning-to-rank such as pair-wise and point-wise, list-wise learning to rank provides a natural way of expressing disparate exposure.
%
%We first introduce a measure of disparate exposure (Section~\ref{subsec:disparate-exposure}).
%
%Next, we combine the loss function from the base method with the measure of disparate exposure (Section~\ref{subsec:problem-statement}).
%
%Finally, we show how this optimization can be implemented using Gradient Descent (Section~\ref{subsec:optimization}).
\subsection{Disparate Exposure} % Across Groups}
\label{subsec:disparate-exposure}
We assume that items in $D$ belong to two different groups, which we denote by $G_0$ for the non-protected group, and $G_1$ for the protected groups. Items in the protected group have a certain protected attribute, such as belonging to an underprivileged group.
%
As argued in Section~\ref{sec:introduction}, the protected group may, due to various causes including historic discrimination or erratic data collection procedures, have a significant disadvantage in the training dataset.
%
This is likely to cause a model to predict rankings with a large discrepancy in exposure, and hence not only to incorporate but to reproduce discrimination and unequal opportunities for already disadvantaged groups.
%
As a remedy we design a learning-to-rank objective to optimize the results not only for accuracy with respect to the training data, but also with respect to the unfairness of the predictions.
To define a measure of unfairness we borrow~\cite{SINGH2018FAIRNESS}'s definition of exposure of a document $d$ in a ranked list generated by a probabilistic ranking $P$ as:
\begin{equation}
\label{eq:documentExposure}
\operatorname{Exposure} \left( d | P\right) = \sum_{j=1}^{n}P_{d, j} \cdot v_j
\end{equation}
where $P_{d, j}$ is the probability that document $d$ will be ranked in position $j$, and $ v_j $ is the \emph{position bias} of position $j$, indicating its relative importance for users of a ranking system.
%
We use a logarithmic discount function $ v_j = \frac{1}{\log(1+j)} $ which is commonly used~\cite{jarvelin2002cumulated}.
%
In our current implementation of this framework we deal only with top-one probabilities, i.e., we adapt equation~\ref{eq:documentExposure} such that the exposure of document $\doc{q}_i$ represented by features $\feat{q}_i$ is its probability of achieving the top position:
\begin{equation}
\operatorname{Exposure}\left(\feat{q}_i|P_{\predictjudg{q}}\right) = P_{\predictjudg{q}}\left(\feat{q}_i\right) \cdot v_1
\end{equation}
Hence, the average exposure of documents in group $ G_p $ with $ p \in \left\{0, 1\right\} $ is
\begin{equation}
\operatorname{Exposure}(G_p|P_{\predictjudg{q}}) = \frac{1}{|G_p|} \sum_{\feat{q}_i \in G_p} \operatorname{Exposure}(\feat{q}_i|P_{\predictjudg{q}})
\end{equation}
%
Finally, we adapt the first definition of equal exposure in~\citet{SINGH2018FAIRNESS}, \emph{demographic parity}, which compares the average exposure across items from all groups is equal.
%
With this we can now introduce an unfairness criterion measured in terms of disparate exposure:
\begin{equation}%\label{eq:diffexp-criterion}
U(\predictjudg{q}) = \max \left(0, \operatorname{Exposure}(G_0|P_{\predictjudg{q}}) - \operatorname{Exposure}(G_1|P_{\predictjudg{q}})\right)^2
\label{eq:exposure}
\end{equation}
%
Note that using the squared hinge loss allows us to have a differentiable loss function that prefers rankings in which the exposure of the protected group is not less than the exposure of the non-protected group \emph{but not vice versa}.
%
This means that our definition will optimize only relevance in cases where the protected group already receives as much exposure as the nonprotected group.
We note that other definitions of disparate exposure can be used as long as they can be optimized efficiently (e.g., differentiable), and that the definition in equation~\ref{eq:exposure} can be trivially extended to multiple protected groups by considering average or maximum difference of exposure between a protected group and the nonprotected one.
%We remark that other definitions of disparate exposure are possible and can be adopted within our framework.
\subsection{Formal Problem Statement}
\label{subsec:problem-statement}
Learning to rank obtains a ranking function $f$ that is learned by solving a minimization problem with respect to a loss function $L$, as described in Section~\ref{concept:loss-standard}.
%
In our case, we learn $f$ by solving a minimization problem with respect to loss function $\lfair$, which incorporates a measure of accuracy with respect to the training data $L$, as well as a measure of unfairness $U$ in terms of exposure with respect to the generated rankings.
%
Specifically, we seek to minimize a weighted summation of the two elements, controlled by a parameter $\gamma \in \mathbb{R}^+_0$:
\begin{equation}
\lfair \left( \trainjudg{q}, \predictjudg{q} \right) = L \left( \trainjudg{q}, \predictjudg{q} \right) + \gamma U \left( \predictjudg{q} \right)
\end{equation}
with larger $\gamma $ expressing preference for solutions that reduce disparate exposure for the protected group, and smaller $\gamma$ expressing preference for solutions that reduce the differences between the training data and the output of the ranking algorithm.
%
The parameter $ \gamma $ depends on desired trade-offs between ranking utility and disparate exposure that are application-dependent.
%
In our experiments, we consider two settings: $\gamma_{\operatorname{large}}$ in which $\gamma$ is comparable to the value of the standard loss $L$, and $\gamma_{\operatorname{small}}$ in which it is an order of magnitude smaller.
We remark that $ U $ only depends on $ \hat{y} $ and is hence not directly affected by biases in the training data, which is a great advantage compared to the naive ``colorblind'' approach.
%
Our new objective can handle both cases, the one in which it is desirable to exclude the protected attribute during training as well as the case in which it is desirable to include it.
%
%We further remark that $ \lfair $ is a general formalism and can be adopted to any kind of learning-to-rank task that seeks to reduce disparate exposure.
\subsection{Optimization}
\label{subsec:optimization}
For the ranking function to infer the document judgments we use a linear function $ f_\omega(\feat{q}_i) = \langle \omega \cdot \feat{q}_i \rangle $, and Gradient Descent to find an optimal solution for $\lfair$.
%
%Reusing the ListNet algorithm of \cite{cao2007learning} we combine a linear model $ \omega $ with Gradient Descent as an optimization algorithm to find an optimal solution for $ \lfair $.
%
We can now rewrite the top-one-probability for a document (Equation~\ref{eq:topOneProbability}) and set $ \phi $ to an exponential function, which is strictly positive and increasing and convenient to derive:
\begin{equation}\label{eq:topOneProbabilityExp}
P_{\predictjudg{q}(f_\omega)}(\feat{q}_i) = \frac{\operatorname{exp}(f_\omega(\feat{q}_i))}{\sum_{k=1}^{n}\operatorname{exp}(f_\omega(\feat{q}_k))}
\end{equation}
%
To use Gradient Descent we need the derivative of $ \lfair(\trainjudg{q}, \predictjudg{q}) $ which in turn consists of the derivatives of the disparate exposure and accuracy metric respectively.
%
\begin{align}
\frac{\partial \lfair \left( \trainjudg{q}, \predictjudg{q} \right)}{\partial \omega} =
\frac{\partial L(\trainjudg{q}, \predictjudg{q})}{\partial \omega} +
\gamma \cdot \frac{\partial U(\predictjudg{q})}{\partial \omega}
\end{align}
%
The derivative of $ L $ can be found in~\cite{cao2007learning}.
%
For brevity we write $ e_{i,q} $ instead of $ \operatorname{exp}(f_\omega(\feat{q}_i)) $ in equation~\ref{eq:topOneProbabilityExp}:
\begin{equation*}
P_{\predictjudg{q}(f_\omega)}(\feat{q}_i) =
\frac{e_{i,q}}{\sum_{k=1}^{n}
e_{k,q}}
\end{equation*}
%
Hence, the inner derivative of the top-one probability with respect to coordinate $ \omega_j $ is:
%
\begin{multline}
\frac{\partial P_{\predictjudg{q}(f_\omega)}(\feat{q}_i)}{\partial \omega_j} = \frac{ e_{i,q} \feat{q}_{i,j} \cdot \sum_{k=1}^{n} e_{k,q} - e_{i,q} \sum_{k=1}^{n} e_{k,q} \cdot \feat{q}_{k,j}
} {
\left( \sum_{k=1}^{n} e_{k,q} \right)^2}
\end{multline}
%
We summarize these equations into a vector $ \omega $ such that we can write a single equation:
%
\begin{equation}
\frac{\partial P_{\predictjudg{q}(f_\omega)}(\feat{q}_i)}{\partial \omega} = \frac{ e_{i,q} \feat{q}_{i} \cdot \sum_{k=1}^{n} e_{k,q} - e_{i,q} \sum_{k=1}^{n} e_{k,q} \cdot \feat{q}_{k}
} {
\left( \sum_{k=1}^{n} e_{k,q} \right)^2}
\end{equation}
%
Finally the gradient becomes,
%
\begin{multline}
\frac{\partial U(\predictjudg{q})}{\partial \omega} = \\
2 \left(\frac{1}{|G_0|}\sum_{\feat{q}_i \in G_0}P_{\predictjudg{q}(f_\omega)}(\feat{q}_i) \cdot v_1 - \frac{1}{|G_1|}\sum_{\feat{q}_i \in G_1}P_{\predictjudg{q}(f_\omega)}(\feat{q}_i) \cdot v_1\right) \\
\cdot \left(\frac{1}{|G_0|}\sum_{\feat{q}_i \in G_0}\frac{\partial P_{\predictjudg{q}(f_\omega)}(\feat{q}_i)}{\partial \omega} \cdot v_1 - \frac{1}{|G_1|}\sum_{\feat{q}_i \in G_1}\frac{\partial P_{\predictjudg{q}(f_\omega)}(\feat{q}_i)}{\partial \omega} \cdot v_1\right)
\end{multline}
%!TEX root = main.tex
%\subsection{Demonstration on Synthetic Data}
%\label{sec:experiments-synthetic}
%
%To show how \methodname works by ensuring the protected group has sufficient exposure, we first demonstrate it on synthetic data.
%%
%The synthetic dataset has a single input query and each item $d_i^{(q)}$ is represented by two features: their protection status $ p $ and a score between 0 and 1: $x_i^{(q)} = ( x_{i, 1}^{(q)}, x_{i, 2}^{(q)} )$.
%%
%The attribute $x_{i, 1}$ is 0 if the item belongs to the non-protected group $G_0$, or 1 if it belongs to the protected group $G_1$.
%%
%The scores $x_{i, 2}$ are distributed uniformly at random over two non-overlapping intervals: $[A_0, B_0]$ for the non-protected group, and $[A_1, B_1]$ for the protected group.
%
%\begin{figure}[h]
% \centering
%
% \subfloat[Case where all non-protected elements appear first in the training set\label{fig:synthethic-nonprotected-first}]{\includegraphics[width=0.48\textwidth]{uniform_male_top_rankings_group_property_only}}
%
% \subfloat[Case where all protected elements appear first in the training set\label{fig:synthetic-protected-first}]{\includegraphics[width=0.48\textwidth]{uniform_female_top_rankings_group_property_only}}
%
% \caption{Depiction of test results using synthetic data.
% %
% Top: \methodname reduces disparate exposure.
% %
% Bottom: asymmetry in \methodname, which does not change rankings if protected elements already appear in the first positions (in these experiments, $\gamma_{\operatorname{small}} = 75$; $\gamma_{\operatorname{large}} = 150$).}
%\end{figure}
%
%Training documents are ordered by decreasing scores, hence the top element is the one that has the highest score.
%%
%We first consider a scenario in which $B_1 < A_0$ and hence all protected elements have strictly smaller scores than all non-protected elements. This means that the training list has the non-protected elements at the top.
%%
%Figure~\ref{fig:synthethic-nonprotected-first} depicts this case for a synthetic workload of 50 documents.
%%
%A standard learning to rank algorithm in this case places all non-protected elements above all protected elements, giving them a larger exposure.
%%
%Instead, \methodname with increasing values of $\gamma$ reduces the disparate exposure, while still considering the discrepancy in the score values.
%%
%Figure~\ref{fig:synthetic-protected-first} shows the asymmetry of the method, in the sense that if the protected elements already receive an exposure that is larger than the one of non-protected elements (by having $B_0 < A_1$), there is no change introduced by \methodname with respect to the standard learning to rank approach.
\section{Experiments}\label{sec:experiments-datasets}
In our experiments, we consider three real-world datasets
%which are summarized on Table~\ref{tbl:datasets} and further visualized in Appendix~\ref{app:dataset-insights-engineering} and \ref{app:dataset-insights-law}.
%
that help us study \emph{non-discrimination}, through experiments that seek to reduce exposure disparities due to biases that are unrelated to utility (experiments \ref{subsec:exp-trec} and \ref{subsec:exp-chile-highschool}),
%
and \emph{equality of opportunity}, through experiments that seek to reduce exposure disparities due to utility differences that pre-exist (experiments \ref{subsec:exp-chile-gender}, \ref{subsec:exp-law-gender} and \ref{subsec:exp-law-black}).
%
As explained in the introduction, these are the two prototypical cases for applying this kind of method~\cite{roemer1998equality}.
%
The datasets are presented in subsections \ref{subsec:dataset-trec}, \ref{subsec:dataset-chile} and \ref{subsec:dataset-law}.
%
\emph{All of the data and code required, plus instructions to reproduce all the experiments, will be available with the camera-ready version of this paper.}
We apply \methodname with two different values of $\gamma$ to each dataset, and compare the results against several baselines:
\begin{inparaenum}[(i)]
\item a ``colorblind'' learning-to-rank approach, which excludes protected attributes during training;
\item a standard learning-to-rank method, which considers them during training;
\item a post-processing approach that applies learning to rank and then re-ranks the output; and
\item a pre-processing approach that modifies the training data.
\end{inparaenum}
%
%For the pre- and post-processing baselines we use \baseline~\cite{zehlike2017fa}.
%
Baselines are described in subsection~\ref{subsec:baselines}.
\subsection{W3C experts (TREC Enterprise) Dataset}
\label{subsec:dataset-trec}
%
This dataset originates from the expert search task at the TREC 2005 Enterprise Track~\cite{craswell2005overview}, where an algorithm has to retrieve a sorted list of experts for a given topic, given a corpus of e-mails written by possible candidates. It contains 198,395 mail messages in mailing lists of the World Wide Web Consortium (W3C).
%
A series of 60 topics and a list of hand-picked experts for each topic are provided; each list contains between 7 and 20 experts, where the available expert relevance judgments are binary (expert or non-expert), hence all experts are considered equally expert.
%
The number of candidates (people who authored at least one e-mail) is 1092.
We computed a series of text retrieval features for each query-document pair, such as word count and normalized tf-idf scores by usage of the Learning to Rank ElasticSearch Plug-in~\cite{elasticsearch_l2r}.
%
Furthermore we created a set of query expansion terms for each topic manually to improve the quality of retrieval.
We consider a scenario in which women are the protected group. % as they represent the minority of experts.
%
We manually attributed gender to each candidate on the basis of their given names.
%
Women comprise 10.5\% of e-mail authors and on average 13.93\% of experts across queries.
%However we denote that the number of female experts per query varies strongly by query.
%
%For example query EXTRAIN02 (``Timed Text'') has 14 men and 0 women experts whereas query EXTRAIN06 (``Communications Team'') has 3 men and 4 women (57.1\%).
%
To create training data, for each query, we created a list of 200 people with all experts at the top, followed by random non-experts sampled using the same male/female proportion as for the entire set of candidates.
Given that, to the best of our knowledge, no real-world dataset proven to contain discriminatory patterns is readily available for document retrieval tasks, we injected a discriminatory pattern in this dataset.
%
This was done by sorting the training list for each training query in the following order:
\begin{inparaenum}[1.]
\item all male experts,
\item all female experts,
\item all male non-experts, and
\item all female non-experts.
\end{inparaenum}
%
This simulates a scenario where expertise has been judged correctly, but training lists have been ordered with a bias against women, placing them systematically below men having the same level of expertise.
%
For our training data we created a six-fold cross-validation dataset, each containing 50 training queries and 10 testing queries.
%
We will release the dataset with the camera-ready version of this paper.
\subsection{Engineering Students Dataset}
\label{subsec:dataset-chile}
%
This dataset corresponds to the task of sorting a list of applicants to an engineering school by predicted academic performance, for instance, to give a scholarship to the $k$ most promising candidates.
%
Academic performance is measured by grades after the first year in university.
%
The dataset contains anonymized historical information from first-year students at a large school in a
%Chilean university.
university (name withheld for double-blind review).
%
It covers 5 years and on average 675 students per year.
%
Most of them (94.17\%) are admitted to the university based on a standardized, country-wide university admission test (details withheld for double-blind review)
%named
%PSU (\emph{Prueba de Selecci\'{o}n Universitaria}, University Selection Test)
%PSU (University Selection Test)
and their high-school grades.
%
Other students are admitted through positive action programs aimed at
outstanding students from public schools (4.06\%), % "5% municipalizados, Sistema de Ingreso Prioritario de Equidad Educativa"
women right below the cut-off score (1.04\%), % "Equidad genero"
and students who excel in sports (0.74\%). % "Deportistas"
For each student, the following features are available:
\begin{inparaenum}[(i)]
\item their average high-school grades, computed using an official standardization formula;
\item their scores in math, language, and science in a standardized test;
\item the number of university credits taken, passed, and failed during the first year; and
\item their average grades at the end of their first year.
\end{inparaenum}
%
Additionally, we are given the gender for each student, and whether they come from a public high school or from a private one.
%
We considered two scenarios, one in which women are the protected group (they comprise less than 21\% of students), and one in which students from public high schools are the protected group (measures of educational quality have consistently shown public high schools lag behind private ones
%in Chile~\cite{cornejo2006experimento}).
in this country [reference withheld for double-blind review]).
We created a five-fold cross-validation setup in which each fold contains four classes (years) of students for training and one class for testing.
%
The ground truth is created by sorting students by decreasing grades upon finishing the first year, in which grades are weighted by the credits of each course they passed (and divided by the total number of credits they took).
%
%This places first the students with the highest grades who passed all the courses.
We will provide instructions for access to this dataset and a cleaning script with the camera-ready version of this paper.
\subsection{Law Students Dataset}
\label{subsec:dataset-law}
%
This dataset originates from a study by \citet{wightman1998lsac} that examined whether the LSAT (Law Students Admission Test in the US) is biased against ethnic minorities.
%
It contains anonymized historical information from first-year students at different law schools and consists of 21,792 students in total.
%
We use a uniform sample of 10\% of this dataset, while maintaining the distribution of gender and ethnicity respectively.
%
Our training data corresponds to 80\% of all candidates in this sample, and the ground truth is created by sorting students by decreasing grades upon finishing the first year.
%
For the different experiments we divided the candidates into protected and non-protected groups.
%
When dividing by gender, we consider women as the protected group.
%
They comprise 44\% of the students in the dataset, compared to 21\% in the engineering students dataset; there is also a much smaller difference in terms of academic performance between men and women than in the engineering students dataset.
%
When dividing by ethnicity, we consider minority `Black' (the term used in the US census for African American) as the protected group, while the non-protected group is the majority (`White'). `Blacks' comprise less than 6\% of the students.
%
For each student, the following features are available:
\begin{inparaenum}[(i)]
\item average high-school grades;
\item scores in the LSAT, a law-specific scholastic assessment test; and
\item average grades at the end of the first year.
\end{inparaenum}
%
Additionally, we are given the gender and ethnicity for each student.
The experimental setting is very similar to the engineering students and also simulates a scenario in which one seeks to anticipate the top-k students, e.g., to give a benefit such as a scholarship.
%
We predict academic performance after the first year in university on the basis of high school grades and the LSAT score.
\subsection{Baselines}
\label{subsec:baselines}
We compare \methodname with a small and a large value for $ \gamma $ to the following \emph{pre-, in-} and \emph{post-processing} approaches:
%
since we are still lacking another in-processing fair learning-to-rank approach, our \emph{in-processing baselines} constitute
\begin{inparaenum}[(i)]
\item a ``colorblind'' LTR approach in which a standard learning to rank algorithm (\citet{cao2007learning}, i.e., \methodname with $\gamma=0$) is applied over all the non-sensitive attributes; and
\item the same learning to rank approach in which all attributes are used (including the protected attribute).
\end{inparaenum}
%
Note that in real life, a ``true'' colorblind result is likely to be unachievable, because of dependencies between protected and non-protected attributes (red lining effect \cite{calders2010three}).
%
These dependencies become particularly problematic in cases of different baselines across groups, in the sense that e.g. the same result in a test corresponds to different levels of performance or expertise, because the test's design happens to favor the non-protected group.
%
We will investigate on this in section \ref{subsec:exp-chile-highschool}.
%
Therefore a method is needed that neither takes any assumptions on correlations of features to sensitive characteristics such as standard LTR, nor is blind to biased non-sensitive features such as colorblind LTR.
%
\methodname constitutes such a method.
In the \emph{pre-} and \emph{post-processing} baselines we apply the algorithm~\baseline \cite{zehlike2017fa}
\begin{inparaenum}[(i)]
\item to the training data in order to reduce bias before model training; and respectively
\item to the rankings predicted by a standard LTR method to increase exposure of the protected group if necessary.
\end{inparaenum}
%
\baseline is a top-$k$ ranking algorithm that ensures a minimum proportion of a protected group at every position in a ranking based on a statistical significance test.
%
With a given minimum target proportion $ p $ of protected candidates, \baseline reorders a given predicted ranking and places protected items at higher positions, if its statistical test would fail otherwise.
%
If a top-k ranking passes the test, it just orders items according to their predicted scores.
In our \emph{pre-processing} baseline experiments we process a given training dataset with \baseline to free the data from potential bias and create fair training data.
%
We use three different values of $ p $, $p^{*} = $ the ratio of protected candidates in the dataset, $ p^{+} = p^{*} + 0.1 $ and $ p^{-} = p^{*} - 0.1 $, to show how crucial the right choice of $ p $ is, especially in a pre-processing setting.
%
Afterwards we use standard LTR \cite{cao2007learning} to train a ranker over all features, both sensitive and non-sensitive.
%
The \emph{post-processing} baseline uses the same learning to rank algorithm \cite{cao2007learning} and trains a ranker over all available features, including the protected one.
%
Afterwards \baseline is applied to the predicted rankings, potentially resulting in a reordering of the items.
%
We use the same parameters $ p^{*}, p^{+} $ and $ p^{-} $ as in the pre-processing experiments.
\section{Experimental Results}\label{sec:experiments-results}
In this section we present the results of each experimental setting.
%
Figure~\ref{fig:scatter} depicts the relations between exposure of the protected group and overall relevance.
%
Figure~\ref{fig:trec} illustrates the distribution of men and women across ranking positions for experiment~\ref{subsec:exp-trec}.
\input{results-scatterplots}
\input{results-barplots-trec}
%\input{results-table}
\subsection{W3C experts (gender)}
\label{subsec:exp-trec}
Experimental results are shown on Figures \ref{fig:scatter-trec} and \ref{fig:trec}, averaged over all folds, using $\gamma_{\texttt{small}}=20K$, $\gamma_{\texttt{large}}=200K$ and $p^{*}=0.105$, which is the proportion of women in the dataset.
In this experiment we expect the ``colorblind'' approach to achieve the best results, because we injected a strong bias against women \emph{that was completely unrelated to their expertise}.
%
This setting corresponds to a \emph{non-discrimination case}, where we want to exclude the protected feature from training for relevance reasons and we expect to see no trade-off between relevance and exposure by usage of \methodname.
%
Instead we expect that a larger value for $ \gamma $ corresponds to better exposure \emph{and} better relevance.
%Note that we also did not change any of their non-sensitive features, which means that no correlation between the protected and the non-protected features is present in the data and hence we do not expect a red-lining effect in the colorblind results.
Figures~\ref{fig:scatter-trec} and \ref{fig:trec} confirm our expectations.
%
Note that for this experiment we measure utility in terms of precision in the top ten positions instead of Kendall's tau, because we want to know which algorithm finds most of the true experts and ranks them accordingly.
%
Colorblind L2R performs best in terms of relevance and achieves almost equal exposure for men and women, by distributing women evenly across rankings (Figure \ref{fig:trec-colorblind}).
%
Standard L2R (including the biased protected feature) performs worse in terms of relevance and exposure than most of the other approaches.
%
Indeed, it exaggerates the bias against women, placing all women at the bottom of the ranking, as shown in Figure~\ref{fig:trec-ltr}, even those that were considered experts in the ground truth.
Our goal for \methodname in this experiment is therefore to get closest to the colorblind results and away from the standard L2R results.
%
We achieve this goal, as shown in Figure~\ref{fig:scatter-trec}, where \methodname with a large $ \gamma $ and post-processing \baseline with a large $ p $ produce the best results.
%
\methodname reduces the gap in exposure between men and women, and scores best in terms of relevance compared to all other fair algorithms.
%
Post-processing with $ p^{+} $ achieves better exposure, but leads to a slight over-representation of women at the top-positions (Figure~\ref{fig:trec-postproc-p+}), which causes the drop in relevance (Figure~\ref{fig:scatter-trec}).
A few more things are worth noticing:
First, we see that the choice of $ p $ is crucial for the success of \baseline and that this choice is not trivial.
%
Particularly as a pre-processing approach (orange ``$\overline{F}$'' symbols), using the intuitive $ p^* $, which corresponds to the proportion of women in the dataset, does not help to de-bias the L2R model.
%
Figure \ref{fig:trec-preprocessed-p*} shows that this setting produces exactly the same biased distribution as standard L2R.
%
Also in all other cases of \baseline, post- \emph{and} pre-processing, a too small $ p $ does not show \emph{any} effect on the exposure of women in the rankings (Figure~\ref{fig:trec-preprocessed-p-} and Figure~\ref{fig:trec-postproc-p-}).
%
In contrast, \methodname by design \emph{always} results in better exposure, even if $ \gamma $ is set low. The change may be little but it is never zero, unless $ \gamma=0 $.
%
Second, a too large $ p $ can result in an over-representation of protected elements at the top positions (Figure~\ref{fig:trec-preprocessed-p+} and Figure~\ref{fig:trec-postproc-p+}).
%
Especially in the pre-processing case (Figure~\ref{fig:trec-preprocessed-p+}) this may result into inverting the bias, such that non-protected items are now ranked lower by a standard LTR method because of their group membership.
%
In addition this leads into a profound decline of result relevance, as shown in Figure~\ref{fig:scatter-trec}, where pre-processing using $ p^{+} $ not only increases disparate exposure to the detriment of the non-protected group, but also performs significantly worse than all other approaches in terms of relevance.
We note that, in contrast to \baseline, \methodname by design excludes the risk of reverse discrimination.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Engineering students (gender)}
\label{subsec:exp-chile-gender}
Figure~\ref{fig:scatter-chile-gender} summarizes the averaged results obtained by applying each ranker on all five cross-validation folds ($\gamma_{\texttt{small}}=3K$, $\gamma_{\texttt{large}}=50K$, $ p^{*}=0.202$, which is the proportion of women in this dataset).
%
We know that women score worse than men in the university entrance test and also worse in terms of academic success.
%
We therefore expect a trade-off between utility and exposure, if we optimize for more exposure than the protected group should receive based on their true performance.
%
This is desirable if one wants to achieve equality of opportunity.
In terms of performance the best approaches are standard L2R together with \methodname with small $ \gamma $ and \baseline with small values of $p$.
%
Among these, \methodname achieves the highest exposure for the protected group.
%
In contrast, pre- and post-processing \baseline with small $ p $s do not have any effect on the rankings, which means that they produce the same results as a standard L2R setting.
%
This is confirmed by the fact that their markers are very close to the standard L2R one (blue cross).
Compared to standard L2R, colorblind scores slightly less in terms of relevance, but a lot better in terms of exposure.
%
Because women tend to have lower scores in the standardized test and also lower grades after the first university year, we expect neither colorblind nor standard L2R to be close to the line of total equality of exposure.
%
The standard learning-to-rank however, by usage of the protected feature emphasizes the disadvantage of women and puts them even lower then they deserve.
%
This problem can be mitigated using both \methodname and post-processing \baseline.
%
If one seeks to increase the exposure of women beyond the colorblind result, due to, e.g., requirements on equal opportunities or affirmative action policies, Figure~\ref{fig:scatter-chile-gender} shows that this comes with a penalty in terms of relevance.
%
Post-processing with $p^{+}$ and \methodname with a large $\gamma$ come closest to equal exposure across groups (a ratio of 1 means total equality), but the relevance decrease is smaller for \methodname.
%
This means when using \methodname we trade less relevance for the same fairness achievement in a search result than when using \baseline.
As in the previous experiment, pre-processing with $p^{+}$ inverts the disparate exposure problem by ranking female students to the top positions just because they are female, which also causes a decline in relevance.
%
This shows again that trying to de-bias a dataset before with a ``fair'' algorithm, can lead to reverse discrimination.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Engineering students (high school type)}
\label{subsec:exp-chile-highschool}
In this experiment, we consider students coming from public high schools as the protected group and those from private high schools as the non-protected.
%
Results appear in Figure~\ref{fig:scatter-chile-highschool} ($\gamma_{\texttt{small}}=100K$, $\gamma_{\texttt{large}}=5M$ and $ p^{*}=0.348$, which is the proportion of students from public high schools).
We know that students from public schools perform worse on average in the entrance test, but tend to have higher grades in university than students from private high schools with the same scores in the standardized test.
%
One explanation is that public schools tend to provide an education of inferior quality compared to private schools in the country under study.
%
This means that for achieving the same test scores, students from public schools need to have better academic aptitudes and/or more grit than students from private ones (similarly to observations by \cite{glynn2019community} with respect to community colleges in the US).
%
Under these circumstances, which also correspond to a case of non-discrimination, \emph{including} the protected attribute will lead to better performance in terms of relevance \emph{and} exposure.
%
We therefore expect the colorblind to be among the worst approaches and standard L2R to be among the best.
Given that, our goal for this experiment is to achieve results close to standard L2R and away from the colorblind results.
%
The results in Figure~\ref{fig:scatter-chile-highschool} confirm our expectations.
%
We can see that the colorblind method performs significantly worse than most approaches both in terms of exposure and in terms of relevance.
%
\methodname, given that students from the protected group already receive more exposure than the students in the non-protected group in learning to rank, does not further increase their exposure, preserving the quality of the ranking result, both for small and large values of $ \gamma $ (due to the asymmetry of the method from the hinge loss function that we use).
%
The same is true for \baseline in pre- and post-processing with small values of $ p $.
%
As in the previous experiments, they \emph{always} behave like standard L2R and do not change the rankings.
%
\methodname only behaves like standard L2R when the exposure of the protected group exceeds the one of the non-protected group.
%
In the post-processing setting \baseline with $ p^{+} $ achieves equal exposure ratios as \methodname, but less relevance.
%
In the pre-processing experiment, if $ p $ becomes too large ($ p^{*} $ and $ p^{+} $), more candidates than necessary are pushed towards the top positions in the training data, which in turn leads the LTR algorithm to place too much weight on the protected feature, resulting in a decline of relevance.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Law students (gender)}
\label{subsec:exp-law-gender}
%
Figure~\ref{fig:scatter-law-gender} summarizes the results obtained by the different methods with $\gamma_{\texttt{small}}=3K$, $\gamma_{\texttt{large}}=50K$ and $ p^{*}=0.437$ (which is the proportion of women in this dataset).
%
We observe that the standard L2R approach performs marginally better than colorblind in terms of relevance, which suggests that women's LSAT results lag slightly behind men's in the ground truth.
%
However, the standard LTR exacerbates this small difference and the loss of exposure for women is large.
%
\methodname corrects this mistake.
%
While the experiment with a small $ \gamma $ only shows a marginal decrease in performance but a comparatively large increase in mitigating disparate exposure, \methodname with large $ \gamma $ achieves basically the same relevance as the colorblind approach.
Interestingly the post-processing setting for \baseline did not perform well in this experiment, which suggests that its success depends on the underlying true score distribution of the training dataset.
%
With all values $ p^{*}, p^{-} $ and $ p^{+} $ we observe a substantial decrease of relevance, for the same achievements in exposure for women in comparison to \methodname.
%
We interpret this as ``too many women that performed poorly being pushed to the higher positions'' by \baseline, as it does not take relevance into consideration, while \methodname will balance exposure and relevance of a candidate.
Again the pre-processing approach with $ p^{*} $ and $ p^{-} $ show the same results as standard LTR, while in case of $ p^{+} $ the results exhibit a too high exposure and a huge decline in relevance (the point lies far outside of the plot boundaries).
%
This means that using \baseline to de-bias the training data did not produce any meaningful fair results in this experiments.
\subsection{Law students (race)}
\label{subsec:exp-law-black}
In this experiment African American students, the largest minority group in this dataset (6.4\%), are considered the protected group.
%
Results appear in Figure~\ref{fig:scatter-law-black} ($\gamma_{\texttt{small}}=1M$, $\gamma_{\texttt{large}}=50M$ and $p^{*}=0.064$).
%
We did not use $p^{-}$ because it would have been a negative number.
%
Colorblind learning to rank places black students a little lower than white students, which corresponds to their lower high school and admission test performance found during our data analysis.
%
In contrast, the standard learning to rank approach weights the sensitive feature `race' to such extent that all black students are relegated to the lowest positions (not shown in the figures), even those students who scored well in terms of grades and standardized tests.
%
This experiment corresponds to a scenario in which both objectives, non-discrimination and equal opportunity, are relevant simultaneously.
%
Non-discrimination is required because standard L2R places \emph{all} black students to the bottom of the ranking, even those who performed well in LSAT and have good results in university.
%
\methodname corrects this bias, with a penalty in terms of relevance.
%
Non-discrimination explains also, why colorblind achieves better relevance than the other methods.
%
If we want to additionally achieve equal opportunity for blacks and increase their overall exposure beyond the colorblind level, this comes with a penalty in terms of relevance.
%
\methodname allows us to balance between exposure and performance of the students, depending on how we choose $ \gamma $.
The pre-processing method with $ p^{*} $ achieves approximately the same exposure as \methodname with a small $ \gamma $, but less relevance.
%
The post-processing approach with $ p^{*} $ performs slightly better in terms of exposure than \methodname with small $\gamma$, however from these three, it shows the least relevance.
%
In case of $ p^{+} $ both pre- and post-processing results show poor relevance, because they overcompensate for the protected feature.
Note again that the design of \methodname prevents the model from overrating protected candidates at the top positions, which avoids a severe decline of precision.
%
\methodname optimizes for exposure until equality is achieved and not further. Then, only non-sensitive features are considered.
%\subsection{Running time}
%All experiments were performed in a commodity desktop PC (Intel i5, 16GB RAM) running Ubuntu Linux. The implementation was done in Octave. The time to complete experiments, dominated by training time, was about 45 hours for the W3C experts dataset, about 8 hours for the engineering students dataset, and about 3 hours for the law students dataset sample.
\begin{figure*}[t]
\centering
\captionsetup[subfigure]{justification=centering}
%\subfigure[Distribution of protected and non-protected elements in groundtruth. \label{fig:trec-groundtruth}]{\includegraphics[height=1in,width=\barplotsize\textwidth]{pics/protNonprotDistribution_groundtruth_trec.png}}
%
\subfloat[W3C experts (gender)\label{fig:scatter-trec}]{\includegraphics[height=\scatterheight,width=\scattersize\textwidth]{scatter_precision-top10-exposure-prot-predtrec.png}}\hfill
%
\subfloat[Engineering Students (gender)][Engineering Students \\ (gender)\label{fig:scatter-chile-gender}]{\includegraphics[height=\scatterheight,width=\scattersize\textwidth]{scatter_kendall-tau-exposure-prot-predengineering-gender-withoutSemiPrivate.png}}\hfill
%
\subfloat[Engineering Students (high school type)][Engineering Students \\ (high school type)\label{fig:scatter-chile-highschool}]{\includegraphics[height=\scatterheight,width=\scattersize\textwidth]{scatter_kendall-tau-exposure-prot-predengineering-highschool-withoutSemiPrivate.png}}\hfill
%
\subfloat[Law Students (gender)\label{fig:scatter-law-gender}]{\includegraphics[height=\scatterheight,width=\scattersize\textwidth]{scatter_kendall-tau-exposure-prot-predlaw-gender.png}}\hfill
%
\subfloat[Law Students (ethnicity)\label{fig:scatter-law-black}]{\includegraphics[height=\scatterheight,width=\scattersize\textwidth]{scatter_kendall-tau-exposure-prot-predlaw-black.png}}%\hfill
%
\subfloat[Legend\label{fig:scatter-legend}]{\includegraphics[trim={-15mm 0 -3mm 0}, clip,height=\scatterheight,width=\scattersize\textwidth]{scatter_legend.png}}
%
\caption{(Best seen in color.) Comparison of relevance and exposure achieved by each approach. Each plot (a)-(e) is one experimental setting consisting of a dataset and a protected attribute; (f) contains the legend.
%
In all plots, relevance is in the x axis, while the exposure of the protected group relative to the nonprotected group is in the y axis.
%
We see that a trade-off between exposure and relevance is not a law of nature. Instead its presence or absence depends on the concrete underlying bias in the training data (plot (b) vs plot (c)).
%
In case we observe a trade-off between performance and exposure (plot (b), (d) and (e)), \methodname mostly outperforms the pre- and post-processing approaches.
%
The plots focus on high-relevance results, hence settings that obtain substantially lower relevance are omitted, their approximate position can be inferred from the respective lines joining settings using the same approach with different parameters.
%
We give the respective value of $ \gamma $ and $ p^{*} $ in each evaluation subsection.
%
%Note that for experiment~\ref{fig:scatter-law-black} we exclude \baseline with $ p^{-} $ because its value would have been negative.
}
\label{fig:scatter}
\end{figure*}
\begin{figure*}[t]
\centering
%\subfigure[Distribution of protected and non-protected elements in groundtruth. \label{fig:trec-groundtruth}]{\includegraphics[height=1in,width=\barplotsize\textwidth]{pics/protNonprotDistribution_groundtruth_trec.png}}
%
\subfloat[Colorblind LTR\label{fig:trec-colorblind}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=colorblind_trec.png}}
%
\subfloat[LTR\label{fig:trec-ltr}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=0_trec.png}}\hfill
\subfloat[\baseline as pre-processing $p=0.005$ \label{fig:trec-preprocessed-p-}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=PREPROCESSED_PMinus_trec.png}}
%
\subfloat[\baseline as pre-processing $p=0.105$ \label{fig:trec-preprocessed-p*}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=PREPROCESSED_trec.png}}
%
\subfloat[\baseline as pre-processing $p=0.205$ \label{fig:trec-preprocessed-p+}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=PREPROCESSED_PPlus_trec.png}}
\subfloat[\baseline as post-processing $p=0.005$ \label{fig:trec-postproc-p-}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_FAIR_p-trec.png}}
%
\subfloat[\baseline as post-processing $p=0.105$ \label{fig:trec-postproc-p*}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_FAIR__p_trec.png}}
%
\subfloat[\baseline as post-processing $p=0.205$ \label{fig:trec-postproc-p+}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_FAIR_p+trec.png}}
\subfloat[\methodname, small $ \gamma $ \label{fig:trec-ours-small}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=small_trec.png}}
%
\subfloat[\methodname, large $ \gamma $ \label{fig:trec-ours-large}]{\includegraphics[height=\barplotheight,width=\barplotsize\textwidth]{protNonprotDistribution_Gamma=large_trec.png}}\hfill
\caption{(Best seen in color.) Distribution of men (blue) and women (orange) along rankings with \methodname and different baselines in the W3C experts dataset, using gender as the protected attribute.
%
The training data has been biased against women by breaking ties in expertise always in favor of men.
%
At the top we see that the colorblind learning-to-rank approach distributes women evenly (a), while the inclusion of gender leads learning-to-rank to place all women in the bottom positions (b).
%
Plots (c), (d), (e) show the results of pre-processing: depending on the parameter used, this can quickly go from no change at all in exposure for women to the other extreme, in which women are placed in top positions because of their protected feature.
%
Plots (f), (g), (h) show the results of post-processing: re-ranking the output of standard learning-to-rank algorithm may increase the share of women in the top positions. However if the expected proportion parameter $ p $ is not set well, it can lead to an \emph{under- and over-representation} of women.
%
Plots (i) and (j) show the results of \methodname, which reduces the impact of the biased training set to the model.
%
When $ \gamma $ is too small, \methodname may behave similar to standard learning to rank and cause an under-representation of women in the top-positions. Contrary to the pre- and post-processing methods, \emph{over-representation of the protected group is not possible} due to the design of the algorithm, even if $\gamma$ were set to a very large value.
}
\label{fig:trec}
\end{figure*}
%!TEX root = main.tex
\balance
\section{Conclusions}\label{sec:conclusions}
Rankings obtained using learning to rank can reproduce and exaggerate differences in exposure between groups that can be present in training data.
%
In this paper we have presented \methodname, which extends a list-wise learning to rank method with an objective that reduces the extent to which non-protected elements receive less exposure than protected elements.
Our experiments showed that optimizing for fairer results does not necessarily come with a trade-off in relevance.
%
On the contrary, when the training data is strongly biased against the protected group, without any relationship with utility, aiming for fair search results will \emph{increase} relevance.
%
We also showed that this objective can be achieved by \emph{explicitly excluding} or \emph{explicitly including} the protected feature.
%
As it is hard to understand a-priori, what kind of underlying bias is present in the training data, and whether to include or exclude the protected feature, \methodname provides a convenient approach, which can handle both situations.
%
At the same time it maintains high relevance compared to other state-of-the-art fair ranking approaches, and critically, it cannot ``overcompensate'' due to its asymmetry.
%In the worst case, it behaves as standard learning to rank.
\spara{Limitations and future work.}
%
The parameter $\gamma$ provides great flexibility for combining in the objective relevance with respect to the training data and avoiding large differences in exposure.
%
To set it, we looked at the scale of $L$ and $U$, which depend on many factors including the number of items to be ranked, and then started with a value of $\gamma$ that reflected the ratio of these scales.
%
However, more work is required to provide a systematic way of setting this parameter, and to understand the implications of different values.
Our differential exposure criterion of Equation~\ref{eq:diffexp-criterion} can be easily extended to multiple protected groups, for instance, by considering the maximum difference in exposure between the non-protected and any protected group.
%
However, this needs to be experimentally validated.
%\[
%U(\predictjudg{q}) =
%\max \left( 0, \max_{p \ge 1}
% \left(
% \operatorname{Exposure}(G_0|P_{\predictjudg{q}}) - \operatorname{Exposure}(G_p|P_{\predictjudg{q}})
% \right)
%\right)^2
%\]
%
%Gradients can be easily computed. However, additional experimental validation, which we intend to undertake as future work, would be needed in this case.
%A interesting next step in the future development of the method is to extend the idea of optimizing simultaneously for performance and exposure to other types of learning to rank methods (pair- and point-wise).
%
%However, pair- and point-wise optimization functions present a challenge in expressing disparate exposure for an entire group of documents,
%in a way that can be optimized efficiently,
%as in these cases at most two documents are seen at a time, instead of complete lists.
%
%Further research is needed to understand how to express group exposure under these settings.
\spara{Reproducibility.} All of the data and code required, as well as instructions for reproducing all the experiments we have presented, including code implementing \methodname, will be made publicly available with the camera-ready version of this paper.
% Exclude acknowledgments to satisfy double-blind review constraints
%\spara{Acknowledgments.}
%
% We thank Gina XXX and Pere Urbon for their support in data preparation for the experimental section.
%
% We thank Sara Hajian for insightful discussions leading to this problem formulation.
%
%C. Castillo is partially funded by La Caixa project LCF/PR/PR16/11110009.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{35}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[\protect\citeauthoryear{Agrawal, Gollapudi, Halverson, and
Ieong}{Agrawal et~al\mbox{.}}{2009}]%
{agrawal2009diversifying}
\bibfield{author}{\bibinfo{person}{Rakesh Agrawal}, \bibinfo{person}{Sreenivas
Gollapudi}, \bibinfo{person}{Alan Halverson}, {and} \bibinfo{person}{Samuel
Ieong}.} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{Diversifying search results}. In
\bibinfo{booktitle}{\emph{Proceedings of the second ACM international
conference on web search and data mining}}. ACM, \bibinfo{pages}{5--14}.
\newblock
\bibitem[\protect\citeauthoryear{Altenburger, De, Frazier, Avteniev, and
Hamilton}{Altenburger et~al\mbox{.}}{2017}]%
{altenburger2017there}
\bibfield{author}{\bibinfo{person}{Kristen~M Altenburger},
\bibinfo{person}{Rajlakshmi De}, \bibinfo{person}{Kaylyn Frazier},
\bibinfo{person}{Nikolai Avteniev}, {and} \bibinfo{person}{Jim Hamilton}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Are There Gender Differences in Professional
Self-Promotion? An Empirical Case Study of LinkedIn Profiles Among Recent MBA
Graduates}. In \bibinfo{booktitle}{\emph{ICWSM}}. \bibinfo{pages}{460--463}.
\newblock
\bibitem[\protect\citeauthoryear{Biega, Gummadi, and Weikum}{Biega
et~al\mbox{.}}{2018}]%
{biega2018equity}
\bibfield{author}{\bibinfo{person}{Asia~J Biega}, \bibinfo{person}{Krishna~P
Gummadi}, {and} \bibinfo{person}{Gerhard Weikum}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Equity of Attention: Amortizing Individual Fairness
in Rankings}.
\newblock \bibinfo{journal}{\emph{arXiv:1805.01788 (pre-print)}}
(\bibinfo{year}{2018}).
\newblock
\bibitem[\protect\citeauthoryear{Binns}{Binns}{2018}]%
{binns2018fairness}
\bibfield{author}{\bibinfo{person}{Reuben Binns}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Fairness in Machine Learning: Lessons from
Political Philosophy}. In \bibinfo{booktitle}{\emph{Proceedings of the 1st
Conference on Fairness, Accountability and Transparency}}
\emph{(\bibinfo{series}{Proceedings of Machine Learning Research})},
\bibfield{editor}{\bibinfo{person}{Sorelle~A. Friedler} {and}
\bibinfo{person}{Christo Wilson}} (Eds.), Vol.~\bibinfo{volume}{81}.
\bibinfo{publisher}{PMLR}, \bibinfo{address}{New York, NY, USA},
\bibinfo{pages}{149--159}.
\newblock
\urldef\tempurl%
\url{http://proceedings.mlr.press/v81/binns18a.html}
\showURL{%
\tempurl}
\bibitem[\protect\citeauthoryear{Calders and Verwer}{Calders and
Verwer}{2010}]%
{calders2010three}
\bibfield{author}{\bibinfo{person}{Toon Calders} {and} \bibinfo{person}{Sicco
Verwer}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{Three naive Bayes approaches for
discrimination-free classification}.
\newblock \bibinfo{journal}{\emph{Data Mining and Knowledge Discovery}}
\bibinfo{volume}{21}, \bibinfo{number}{2} (\bibinfo{year}{2010}),
\bibinfo{pages}{277--292}.
\newblock
\bibitem[\protect\citeauthoryear{Cao, Qin, Liu, Tsai, and Li}{Cao
et~al\mbox{.}}{2007}]%
{cao2007learning}
\bibfield{author}{\bibinfo{person}{Zhe Cao}, \bibinfo{person}{Tao Qin},
\bibinfo{person}{Tie-Yan Liu}, \bibinfo{person}{Ming-Feng Tsai}, {and}
\bibinfo{person}{Hang Li}.} \bibinfo{year}{2007}\natexlab{}.
\newblock \showarticletitle{Learning to rank: from pairwise approach to
listwise approach}. In \bibinfo{booktitle}{\emph{Proceedings of the 24th
international conference on Machine learning}}. ACM,
\bibinfo{pages}{129--136}.
\newblock
\bibitem[\protect\citeauthoryear{Carbonell and Goldstein}{Carbonell and
Goldstein}{1998}]%
{carbonell1998use}
\bibfield{author}{\bibinfo{person}{Jaime Carbonell} {and} \bibinfo{person}{Jade
Goldstein}.} \bibinfo{year}{1998}\natexlab{}.
\newblock \showarticletitle{The use of MMR, diversity-based reranking for
reordering documents and producing summaries}. In
\bibinfo{booktitle}{\emph{Proceedings of the 21st annual international ACM
SIGIR conference on Research and development in information retrieval}}. ACM,
\bibinfo{pages}{335--336}.
\newblock
\bibitem[\protect\citeauthoryear{Castillo}{Castillo}{2018}]%
{castillo2018fairness}
\bibfield{author}{\bibinfo{person}{Carlos Castillo}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Fairness and Transparency in Ranking}.
\newblock \bibinfo{journal}{\emph{{SIGIR} Forum}} \bibinfo{volume}{52}
(\bibinfo{date}{12} \bibinfo{year}{2018}).
\newblock
Issue 2.
\bibitem[\protect\citeauthoryear{Celis, Straszak, and Vishnoi}{Celis
et~al\mbox{.}}{2017}]%
{celis2017ranking}
\bibfield{author}{\bibinfo{person}{L~Elisa Celis}, \bibinfo{person}{Damian
Straszak}, {and} \bibinfo{person}{Nisheeth~K Vishnoi}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Ranking with fairness constraints}.
\newblock \bibinfo{journal}{\emph{arXiv:1704.06840 (pre-print)}}
(\bibinfo{year}{2017}).
\newblock
\bibitem[\protect\citeauthoryear{Craswell, de~Vries, and Soboroff}{Craswell
et~al\mbox{.}}{2005}]%
{craswell2005overview}
\bibfield{author}{\bibinfo{person}{Nick Craswell}, \bibinfo{person}{Arjen~P de
Vries}, {and} \bibinfo{person}{Ian Soboroff}.}
\bibinfo{year}{2005}\natexlab{}.
\newblock \showarticletitle{Overview of the TREC 2005 Enterprise Track.}. In
\bibinfo{booktitle}{\emph{Trec}}, Vol.~\bibinfo{volume}{5}.
\bibinfo{pages}{199--205}.
\newblock
\bibitem[\protect\citeauthoryear{Culpepper, Diaz, , and (editors)}{Culpepper
et~al\mbox{.}}{2018}]%
{culpepper2018swirl}
\bibfield{author}{\bibinfo{person}{J.~Shane Culpepper},
\bibinfo{person}{Fernando Diaz}, \bibinfo{person}{}, {and}
\bibinfo{person}{Mark D.~Smucker (editors)}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Report from the Third Strategic Workshop on
Information Retrieval (SWIRL)}.
\newblock (\bibinfo{year}{2018}).
\newblock
\bibitem[\protect\citeauthoryear{Dastin}{Dastin}{2018}]%
{reuters2018}
\bibfield{author}{\bibinfo{person}{Jeffrey Dastin}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Amazon scraps secret AI recruiting tool that showed
bias against women}.
\newblock \bibinfo{journal}{\emph{Reuters}} (\bibinfo{date}{10}
\bibinfo{year}{2018}).
\newblock
\bibitem[\protect\citeauthoryear{Drosou, Jagadish, Pitoura, and
Stoyanovich}{Drosou et~al\mbox{.}}{2017}]%
{drosou2017diversity}
\bibfield{author}{\bibinfo{person}{Marina Drosou}, \bibinfo{person}{HV
Jagadish}, \bibinfo{person}{Evaggelia Pitoura}, {and} \bibinfo{person}{Julia
Stoyanovich}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Diversity in Big Data: A Review}.
\newblock \bibinfo{journal}{\emph{Big Data}} \bibinfo{volume}{5},
\bibinfo{number}{2} (\bibinfo{year}{2017}), \bibinfo{pages}{73--84}.
\newblock
\bibitem[\protect\citeauthoryear{Dwork, Hardt, Pitassi, Reingold, and
Zemel}{Dwork et~al\mbox{.}}{2012}]%
{dwork2012fairness}
\bibfield{author}{\bibinfo{person}{Cynthia Dwork}, \bibinfo{person}{Moritz
Hardt}, \bibinfo{person}{Toniann Pitassi}, \bibinfo{person}{Omer Reingold},
{and} \bibinfo{person}{Richard Zemel}.} \bibinfo{year}{2012}\natexlab{}.
\newblock \showarticletitle{Fairness through awareness}. In
\bibinfo{booktitle}{\emph{Proc. of Innovations in Theoretical Computer
Science (ITCS)}}. ACM, \bibinfo{pages}{214--226}.
\newblock
\bibitem[\protect\citeauthoryear{Friedman and Nissenbaum}{Friedman and
Nissenbaum}{1996}]%
{friedman1996bias}
\bibfield{author}{\bibinfo{person}{Batya Friedman} {and} \bibinfo{person}{Helen
Nissenbaum}.} \bibinfo{year}{1996}\natexlab{}.
\newblock \showarticletitle{Bias in computer systems}.
\newblock \bibinfo{journal}{\emph{ACM Transactions on Information Systems}}
\bibinfo{volume}{14}, \bibinfo{number}{3} (\bibinfo{year}{1996}),
\bibinfo{pages}{330--347}.
\newblock
\bibitem[\protect\citeauthoryear{Glynn}{Glynn}{2019}]%
{glynn2019community}
\bibfield{author}{\bibinfo{person}{Jennifer Glynn}.}
\bibinfo{year}{2019}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Persistence: The Success of Students Who
Transfer from Community Colleges to Selective Four-Year Institutions}}.
\newblock \bibinfo{type}{{T}echnical {R}eport}.
\newblock
\bibitem[\protect\citeauthoryear{Hajian, Bonchi, and Castillo}{Hajian
et~al\mbox{.}}{2016}]%
{hajian2016algorithmic}
\bibfield{author}{\bibinfo{person}{Sara Hajian}, \bibinfo{person}{Francesco
Bonchi}, {and} \bibinfo{person}{Carlos Castillo}.}
\bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Algorithmic bias: From discrimination discovery to
fairness-aware data mining}. In \bibinfo{booktitle}{\emph{Proceedings of the
22nd ACM SIGKDD international conference on knowledge discovery and data
mining}}. ACM, \bibinfo{pages}{2125--2126}.
\newblock
\bibitem[\protect\citeauthoryear{Hardt}{Hardt}{2014}]%
{hardt2014big}
\bibfield{author}{\bibinfo{person}{Moritz Hardt}.}
\bibinfo{year}{2014}\natexlab{}.
\newblock \showarticletitle{How big data is unfair: Understanding sources of
unfairness in data driven decision making}.
\newblock \bibinfo{journal}{\emph{Medium}} (\bibinfo{year}{2014}).
\newblock
\bibitem[\protect\citeauthoryear{Hardt, Price, Srebro, et~al\mbox{.}}{Hardt
et~al\mbox{.}}{2016}]%
{hardt2016equality}
\bibfield{author}{\bibinfo{person}{Moritz Hardt}, \bibinfo{person}{Eric Price},
\bibinfo{person}{Nati Srebro}, {et~al\mbox{.}}}
\bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Equality of opportunity in supervised learning}. In
\bibinfo{booktitle}{\emph{Advances in Neural Information Processing Systems
(NIPS)}}. \bibinfo{pages}{3315--3323}.
\newblock
\bibitem[\protect\citeauthoryear{J{\"a}rvelin and
Kek{\"a}l{\"a}inen}{J{\"a}rvelin and Kek{\"a}l{\"a}inen}{2002}]%
{jarvelin2002cumulated}
\bibfield{author}{\bibinfo{person}{Kalervo J{\"a}rvelin} {and}
\bibinfo{person}{Jaana Kek{\"a}l{\"a}inen}.} \bibinfo{year}{2002}\natexlab{}.
\newblock \showarticletitle{Cumulated gain-based evaluation of IR techniques}.
\newblock \bibinfo{journal}{\emph{ACM Transactions on Information Systems
(TOIS)}} \bibinfo{volume}{20}, \bibinfo{number}{4} (\bibinfo{year}{2002}),
\bibinfo{pages}{422--446}.
\newblock
\bibitem[\protect\citeauthoryear{Joachims, Granka, Pan, Hembrooke, and
Gay}{Joachims et~al\mbox{.}}{2017}]%
{joachims2017accurately}
\bibfield{author}{\bibinfo{person}{Thorsten Joachims}, \bibinfo{person}{Laura
Granka}, \bibinfo{person}{Bing Pan}, \bibinfo{person}{Helene Hembrooke},
{and} \bibinfo{person}{Geri Gay}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Accurately interpreting clickthrough data as
implicit feedback}. In \bibinfo{booktitle}{\emph{ACM SIGIR Forum}},
Vol.~\bibinfo{volume}{51}. Acm, \bibinfo{pages}{4--11}.
\newblock
\bibitem[\protect\citeauthoryear{Lippert-Rasmussen}{Lippert-Rasmussen}{2014}]%
{lippert2014born}
\bibfield{author}{\bibinfo{person}{Kasper Lippert-Rasmussen}.}
\bibinfo{year}{2014}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Born free and equal?: a philosophical
inquiry into the nature of discrimination}}.
\newblock \bibinfo{publisher}{Oxford University Press}.
\newblock
\bibitem[\protect\citeauthoryear{Liptak}{Liptak}{2009}]%
{liptak2009supreme}
\bibfield{author}{\bibinfo{person}{Adam Liptak}.}
\bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{Supreme Court finds bias against white
firefighters}.
\newblock \bibinfo{journal}{\emph{The New York Times A}} \bibinfo{volume}{1}
(\bibinfo{year}{2009}), \bibinfo{pages}{A13}.
\newblock
\bibitem[\protect\citeauthoryear{{OpenSource Connections}}{{OpenSource
Connections}}{2017}]%
{elasticsearch_l2r}
\bibfield{author}{\bibinfo{person}{{OpenSource Connections}}.}
\bibinfo{year}{2017}\natexlab{}.
\newblock \bibinfo{title}{We're Bringing Learning to Rank to Elasticsearch}.
\newblock
\bibinfo{howpublished}{https://opensourceconnections.com/blog/2017/02/14/elasticsearch-learning-to-rank/}.
\newblock
\bibitem[\protect\citeauthoryear{Pedreshi, Ruggieri, and Turini}{Pedreshi
et~al\mbox{.}}{2008}]%
{pedreshi2008discrimination}
\bibfield{author}{\bibinfo{person}{Dino Pedreshi}, \bibinfo{person}{Salvatore
Ruggieri}, {and} \bibinfo{person}{Franco Turini}.}
\bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Discrimination-aware data mining}. In
\bibinfo{booktitle}{\emph{Proceedings of the 14th ACM SIGKDD international
conference on Knowledge discovery and data mining}}. ACM,
\bibinfo{pages}{560--568}.
\newblock
\bibitem[\protect\citeauthoryear{Robertson}{Robertson}{1977}]%
{robertson1977probability}
\bibfield{author}{\bibinfo{person}{Stephen~E Robertson}.}
\bibinfo{year}{1977}\natexlab{}.
\newblock \showarticletitle{The probability ranking principle in IR}.
\newblock \bibinfo{journal}{\emph{Journal of documentation}}
\bibinfo{volume}{33}, \bibinfo{number}{4} (\bibinfo{year}{1977}),
\bibinfo{pages}{294--304}.
\newblock
\bibitem[\protect\citeauthoryear{Roemer}{Roemer}{1998}]%
{roemer1998equality}
\bibfield{author}{\bibinfo{person}{John~E Roemer}.}
\bibinfo{year}{1998}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Equality of opportunity}}.
\newblock \bibinfo{publisher}{Harvard University Press}.
\newblock
\bibitem[\protect\citeauthoryear{Singh and Joachims}{Singh and
Joachims}{2017}]%
{singh2018equality}
\bibfield{author}{\bibinfo{person}{Ashudeep Singh} {and}
\bibinfo{person}{Thorsten Joachims}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Equality of Opportunity in Rankings}. In
\bibinfo{booktitle}{\emph{Workshop on Prioritizing Online Content (WPOC) at
NIPS}}.
\newblock
\bibitem[\protect\citeauthoryear{Singh and Joachims}{Singh and
Joachims}{2018}]%
{SINGH2018FAIRNESS}
\bibfield{author}{\bibinfo{person}{Ashudeep Singh} {and}
\bibinfo{person}{Thorsten Joachims}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Fairness of Exposure in Rankings}.
\newblock \bibinfo{journal}{\emph{arXiv:1802.07281 (pre-print)}}
(\bibinfo{year}{2018}).
\newblock
\bibitem[\protect\citeauthoryear{\v{Z}liobait\.{e}}{\v{Z}liobait\.{e}}{2015}]%
{zliobaite2015survey}
\bibfield{author}{\bibinfo{person}{Indr\.{e} \v{Z}liobait\.{e}}.}
\bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{A survey on measuring indirect discrimination in
machine learning}.
\newblock \bibinfo{journal}{\emph{arXiv:1511.00148 (pre-print)}}
(\bibinfo{year}{2015}).
\newblock
\bibitem[\protect\citeauthoryear{Wightman}{Wightman}{1998}]%
{wightman1998lsac}
\bibfield{author}{\bibinfo{person}{Linda~F Wightman}.}
\bibinfo{year}{1998}\natexlab{}.
\newblock \showarticletitle{LSAC National Longitudinal Bar Passage Study. LSAC
Research Report Series.}
\newblock (\bibinfo{year}{1998}).
\newblock
\bibitem[\protect\citeauthoryear{Yang and Stoyanovich}{Yang and
Stoyanovich}{2017}]%
{yang2017measuring}
\bibfield{author}{\bibinfo{person}{Ke Yang} {and} \bibinfo{person}{Julia
Stoyanovich}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Measuring Fairness in Ranked Outputs}. In
\bibinfo{booktitle}{\emph{Proc. of International Conference on Scientific and
Statistical Database Management (SSDB)}}. \bibinfo{publisher}{ACM},
\bibinfo{pages}{22}.
\newblock
\bibitem[\protect\citeauthoryear{Yang, Stoyanovich, Asudeh, Howe, Jagadish, and
Miklau}{Yang et~al\mbox{.}}{2018}]%
{yang2018nutritional}
\bibfield{author}{\bibinfo{person}{Ke Yang}, \bibinfo{person}{Julia
Stoyanovich}, \bibinfo{person}{Abolfazl Asudeh}, \bibinfo{person}{Bill Howe},
\bibinfo{person}{HV Jagadish}, {and} \bibinfo{person}{Gerome Miklau}.}
\bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{A Nutritional Label for Rankings}.
\newblock \bibinfo{journal}{\emph{arXiv:1804.07890 (pre-print)}}
(\bibinfo{year}{2018}).
\newblock
\bibitem[\protect\citeauthoryear{Zafar, Valera, Gomez~Rodriguez, and
Gummadi}{Zafar et~al\mbox{.}}{2017}]%
{zafar2017fairness}
\bibfield{author}{\bibinfo{person}{Muhammad~Bilal Zafar},
\bibinfo{person}{Isabel Valera}, \bibinfo{person}{Manuel Gomez~Rodriguez},
{and} \bibinfo{person}{Krishna~P Gummadi}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Fairness beyond disparate treatment \& disparate
impact: Learning classification without disparate mistreatment}. In
\bibinfo{booktitle}{\emph{Proceedings of the 26th International Conference on
World Wide Web}}. International World Wide Web Conferences Steering
Committee, \bibinfo{pages}{1171--1180}.
\newblock
\bibitem[\protect\citeauthoryear{Zehlike, Bonchi, Castillo, Hajian, Megahed,
and Baeza-Yates}{Zehlike et~al\mbox{.}}{2017}]%
{zehlike2017fa}
\bibfield{author}{\bibinfo{person}{Meike Zehlike}, \bibinfo{person}{Francesco
Bonchi}, \bibinfo{person}{Carlos Castillo}, \bibinfo{person}{Sara Hajian},
\bibinfo{person}{Mohamed Megahed}, {and} \bibinfo{person}{Ricardo
Baeza-Yates}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{{FA*IR}: A fair top-k ranking algorithm}. In
\bibinfo{booktitle}{\emph{Proc. of the 2017 ACM on Conference on Information
and Knowledge Management}}. ACM, \bibinfo{pages}{1569--1578}.
\newblock
\end{thebibliography}