Evidence of What, for Whom? The Socially Contested Role of Algorithmic Bias in a Predictive Policing Tool
Extracted source text · arXiv 2405.07715
%%
%% This is file `sample-manuscript.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx (with options: `manuscript')
%%
%% IMPORTANT NOTICE:
%%
%% For the copyright see the source file.
%%
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-manuscript.tex.
%%
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%%
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%%
%% Commands for TeXCount
%TC:macro \cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%%
%% The first command in your LaTeX source must be the \documentclass
%% command.
%%
%% For submission and review of your manuscript please change the
%% command to \documentclass[manuscript, screen, review]{acmart}.
%%
%% When submitting camera ready or to TAPS, please change the command
%% to \documentclass[sigconf]{acmart} or whichever template is required
%% for your publication.
%%
%%
%\documentclass[manuscript,screen,review]{acmart}
%\documentclass[manuscript,screen,review,anonymous]{acmart}
%\documentclass[manuscript,screen,review,anonymous,nonacm]{acmart}
\documentclass[sigconf]{acmart}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage{supertabular}
\usepackage{enumitem}
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
Bib\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
%\setcopyright{acmcopyright}
\setcopyright{none}
% \copyrightyear{2018}
% \acmYear{2024}
% \acmDOI{XXXXXXX.XXXXXXX}
\usepackage{url}
\usepackage{hyperref}
\usepackage[hyphenbreaks]{breakurl}
%% These commands are for a PROCEEDINGS abstract or paper.
%\acmYear{2024}\copyrightyear{2024}
%\acmConference[ACM FAccT '24]{ACM Conference on Fairness, Accountability, and Transparency}{June 3--6, 2024}{Rio de Janeiro, Brazil}
%\acmBooktitle{ACM Conference on Fairness, Accountability, and Transparency (ACM FAccT '24), June 3--%\acmDOI{10.1145/3630106.3658991}
%\acmISBN{979-8-4007-0450-5/24/06}
\copyrightyear{2024}
\acmYear{2024}
\setcopyright{rightsretained}
\acmConference[FAccT '24]{The 2024 ACM Conference on Fairness, Accountability, and Transparency}{June 3--6, 2024}{Rio de Janeiro, Brazil}
\acmBooktitle{The 2024 ACM Conference on Fairness, Accountability, and Transparency (FAccT '24), June 3--6, 2024, Rio de Janeiro, Brazil}\acmDOI{10.1145/3630106.3658991}
\acmISBN{979-8-4007-0450-5/24/06}
%\acmConference[FAccT '24]{June 3--6, 2024}{Rio de Janeiro, Brazil}%{Make sure to enter the correct
% conference title from your rights confirmation emai}{June 03--05,
% 2018}{Woodstock, NY}
%%
%% Uncomment \acmBooktitle if the title of the proceedings is different
%% from ``Proceedings of ...''!
%%
%%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
%% June 03--05, 2018, Woodstock, NY}
% \acmPrice{15.00}
% \acmISBN{978-1-4503-XXXX-X/18/06}
\newcommand{\tabitem}{~~\llap{\textbullet}~~}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{\textbf{Evidence of What, for Whom? The Socially Contested Role of Algorithmic Bias in a Predictive Policing Tool}}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
\author{Marta Ziosi}
\authornote{Both authors contributed equally to this work.}
\affiliation{%
\institution{Oxford University}
\streetaddress{Oxford}
\city{Oxford}
%\state{Pennsylvania}
\country{UK}
\postcode{12345}
}
\email{marta.ziosi@sant.ox.ac.uk}
\author{Dasha Pruss}
\authornotemark[1]
%\orcid{0000-0002-8124-5266}
\affiliation{%
\institution{Harvard University}
\streetaddress{Harvard}
\city{Cambridge}
%\state{Pennsylvania}
\country{USA}
\postcode{02138}
}
\email{dpruss@fas.harvard.edu}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
This paper presents a critical, qualitative study of the social role of algorithmic bias in the context of the Chicago crime prediction algorithm, a predictive policing tool that forecasts when and where in the city crime is most likely to occur. Through interviews with 18 Chicago-area community organizations, academic researchers, and public sector actors, we show that stakeholders from different groups articulate diverse problem diagnoses of the tool’s algorithmic bias, strategically using it as evidence to advance criminal justice interventions that align with stakeholders' positionality and political ends. Drawing inspiration from Catherine D'Ignazio’s taxonomy of ``refusing and using'' data, we find that stakeholders use evidence of algorithmic bias to reform the policies around police patrol allocation; reject algorithm-based policing interventions; reframe crime as a structural rather than interpersonal problem; reveal data on authority figures in an effort to subvert their power; repair and heal families and communities; and, in the case of more powerful actors, to reaffirm their own authority or existing power structures. We identify the implicit assumptions and scope of these varied uses of algorithmic bias as evidence, showing that they require different (and sometimes conflicting) values about policing and AI. This divergence reflects long-standing tensions in the criminal justice reform landscape between the values of liberation and healing often centered by system-impacted communities and the values of surveillance and deterrence often instantiated in data-driven reform measures. We advocate for centering the interests and experiential knowledge of communities impacted by incarceration to ensure that evidence of algorithmic bias can serve as a device to challenge the status quo.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
% \begin{CCSXML}
% <ccs2012>
% <concept>
% <concept_id>10010520.10010553.10010562</concept_id>
% <concept_desc>Computer systems organization~Embedded systems</concept_desc>
% <concept_significance>500</concept_significance>
% </concept>
% <concept>
% <concept_id>10010520.10010575.10010755</concept_id>
% <concept_desc>Computer systems organization~Redundancy</concept_desc>
% <concept_significance>300</concept_significance>
% </concept>
% <concept>
% <concept_id>10010520.10010553.10010554</concept_id>
% <concept_desc>Computer systems organization~Robotics</concept_desc>
% <concept_significance>100</concept_significance>
% </concept>
% <concept>
% <concept_id>10003033.10003083.10003095</concept_id>
% <concept_desc>Networks~Network reliability</concept_desc>
% <concept_significance>100</concept_significance>
% </concept>
% </ccs2012>
% \end{CCSXML}
%
% \ccsdesc[500]{Computer systems organization~Embedded systems}
% \ccsdesc[300]{Computer systems organization~Redundancy}
% \ccsdesc{Computer systems organization~Robotics}
% \ccsdesc[100]{Networks~Network reliability}
% \begin{CCSXML}
% <ccs2012>
% <concept>
% <concept_id>10003120.10003121.10011748</concept_id>
% <concept_desc>Human-centered computing~Empirical studies in HCI</concept_desc>
% <concept_significance>500</concept_significance>
% </concept>
% <concept>
% <concept_id>10010405.10010455</concept_id>
%% <concept_desc>Applied computing~Law, social and behavioral sciences</concept_desc>
%% <concept_significance>500</concept_significance>
%% </concept>
%% </ccs2012>
% \end{CCSXML}
% \ccsdesc[500]{Human-centered computing~Empirical studies in HCI}
% \ccsdesc[500]{Applied computing~Law, social and behavioral sciences}
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10003120.10003121.10011748</concept_id>
<concept_desc>Human-centered computing~Empirical studies in HCI</concept_desc>
<concept_significance>500</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Human-centered computing~Empirical studies in HCI}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{criminal justice; predictive policing; algorithmic bias; qualitative}
%\received{20 February 2007}
% \received[revised]{12 March 2009}
% \received[accepted]{5 June 2009}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
Over the past two decades, public scrutiny and outrage toward policing scandals has led some of the largest police departments in the US to turn to ``data-driven'' policing as a reform and public relations measure \cite{brayne_predict_2020}. Place-based predictive policing tools, which forecast crime ``hotspots'' based on past crime incidence data, have become an increasingly popular and controversial strategy for addressing a range of issues in policing, from resource limitations to unconscious biases in decision-making \cite{selbst2017disparate}.
Proponents laud predictive policing as an evidence-based policing reform that promises more objective decision-making \cite{kennedy_risk-based_2018, pearsall2010predictive, porter_trust_1995, espeland_accountability_2007}, while critics argue that it exacerbates the very issues it is meant to address \cite{harcourt_against_2008, shapiro_predictive_2019, richardson_dirty_2019}. Algorithmic bias -- the disparate treatment of or impact on protected groups by an algorithm -- has been at the center of the discourse about predictive policing, including concerns about the algorithms' reliance on ``dirty data'' and the possibility of feedback loops in officers' arresting behavior \cite{lum_predict_2016, richardson_dirty_2019, ensign_runaway_2018, selbst2017disparate}.
%Algorithm-assisted decision-making is an increasingly popular strategy for addressing a range of issues in the public sector, from resource limitations to undesirable inconsistencies in discretionary decision-making. Criminal risk prediction algorithms, such as recidivism risk assessment instruments and predictive policing, are a particularly contentious instance of this broader trend, with proponents lauding the tools as an evidence-based reform \cite{kennedy_risk-based_2018} and critics arguing that the algorithms exacerbate the very issues they are meant to address \cite{starr_evidence-based_2014, harcourt_against_2008, shapiro_predictive_2019}. Algorithmic bias -- the disparate treatment of or impact on different protected groups by an algorithm -- has been at the center of the discourse about criminal risk prediction instruments \cite{angwin_machine_2016}.
%"More recently, predictive policing systems have come under scrutiny due to their lack of transparency [53 ] and concerns that they may lead to further over-policing of minority communities by virtue of being trained on biased or ``dirty'' data [19 , 33 , 45 ]. Critics commonly point to the possibility that such systems may produce dangerous feedback loops, vicious cycles wherein data on recent arrests is used to deploy police in still greater numbers to neighbourhoods where they zealously seek out suspicious activity and conduct even more arrests. Recent work by Lum and Isaac [33] and Ensign et al . [19] has demonstrated both empirically and theoretically how such feedback loops can arise."
%Formal methods of measuring and minimizing algorithmic bias have become a standard part of the audit and validation process of predictive policing tools, but
Standard algorithmic fairness approaches have focused on identifying, removing, or managing statistical biases \cite{fazelpour_algorithmic_2021}, with strategies ranging from pre-, during and post-deployment strategies to preventing models from learning or outputting biases \cite{ensign_runaway_2018, fahse_managing_2021, yeung_identifying_2021, zhang_mitigating_2018}. However, many critics have pointed out that verifying that an algorithm is correctly `calibrated' is not the right question to ask when underlying data reflect structural injustices \cite{balayn2021beyond, hoffmann2019fairness}. Indeed, the bias exhibited by predictive policing algorithms is considered by many scholars to be an inevitable artifact of higher police presence in historically marginalized communities \cite{lum_predict_2016, zilka_progression_2023}, a reflection of the structural race- and class-biased patterns of the criminal legal system and an unequal society more broadly, rather than a technical ``glitch'' \cite{benjamin_race_2019}. In response, some researchers have called for more substantive notions of algorithmic fairness that take algorithms' objectives and sociotechnical contexts into account \cite{barabas_beyond_2019, green_algorithmic_2020, green_escaping_2022}. Others have focused on providing affected communities with alternative tools to respond to the deeper societal issues indicated by algorithmic bias, such as the use of civilian dashcam %\footnote{A video camera mounted on the dashboard or windscreen of a vehicle and used to continuously record the view of the road, traffic, etc. through the windscreen.}
data to detect disparities in police deployments \cite{franchi_detecting_2023}. In short, there has been widespread disagreement about how to respond to evidence of algorithmic bias.
%distinguishing between use based harms and implementation based harms: if you believe that algorithm use in criminal justice is fine but the problem is that they're biased, this is a problem of implementation if your problem is with the use of algorithms in criminal justice system at all, and how algorithms are used to reify notions of criminality, this is a use based harm
%Critical scholars have respond to the deeper societal issues reflected in bias by calling for more substantive notions of algorithmic fairness that take algorithms' objectives and sociotechnical contexts into account, with the possibility that algorithmic decision-making systems should be removed from the process entirely \cite{eubanks_automating_2018, barabas_beyond_2019, barabas_studying_2020, green_algorithmic_2020, green_escaping_2022}.
%"We argue that the core ethical debate surrounding the use of regression in risk assessments is not one of bias or accuracy. Rather, it’s one of purpose. If machine learning is operationalized merely in the service of predicting future crime, then it becomes difficult to break cycles of criminalization that are driven by the iatrogenic effects of the criminal justice system itself. We posit that machine learning should not be used for prediction, rather it should be used to surface covariates that are fed into a causal model for understanding the social, structural and psychological drivers of crime. We propose an alternative application of machine learning and causal inference away from predicting risk scores to risk mitigation." barabas_interventions_2018
%Other scholars argue that though these proposals are useful for minimizing bias locally, they are unable to account for the system-level origins of bias and can result in `fairwashing', the legitimation of algorithms that have serious shortcomings in practice \cite{aivodji_fairwashing:_2019}.
%Critical scholars have respond to the deeper societal issues reflected in bias by calling for more substantive notions of algorithmic fairness that take algorithms' objectives and sociotechnical contexts into account, with the possibility that algorithmic decision-making systems should be removed from the process entirely \cite{eubanks_automating_2018, barabas_beyond_2019, barabas_studying_2020, green_algorithmic_2020, green_escaping_2022}.
This paper presents a critical, qualitative study on the socially contested role of algorithmic bias in the context of the Chicago crime prediction algorithm \cite{rotaru_event-level_2022}, a predictive policing tool that predicts the geographic regions in Chicago in which crime is most likely to occur. The algorithm's developers demonstrate that it exhibits ``enforcement bias'' -- that is, the uneven or disproportionate patrolling, arrests, or other actions by police toward certain neighborhoods.
Through interviews with Chicago-area academic researchers, %including developers of the tool and criminal justice researchers;
community organizations,
%including Chicago-area reentry and human rights community organizations;
and public sector employees, %including a county commissioner and police officers,
we find that evidence of algorithmic bias has the potential to challenge as much as to affirm the status quo. We show that stakeholders from different groups articulate different problem diagnoses of the algorithm's bias, strategically using these diagnoses to advance criminal justice interventions that align with individuals' political ends. These strategies include using data to reform the police that generated the biased data in the first place, such as by allocating police patrols more equitably; %advocating for greater transparency, accountability, and education around the development or use of predictive policing;
reaffirm or strengthen existing models of policing; reject the use of predictive policing tools; %incorporating diverse stakeholders in the tools' development and assessment;
and reframe the narrative away from reforms like data-driven policing toward abolitionist aims, such as repairing communities.
We identify the implicit assumptions and scope of these varied uses of algorithmic bias as evidence, showing that they require different (and sometimes conflicting) values about policing and AI. Notably, using enforcement bias to inform more efficient allocation of police patrols diverges from the use and interpretation of the same evidence by stakeholders with lived experience in the criminal legal system, who diagnose the source of the bias as systemic. This divergence reflects long-standing tensions in the criminal justice reform landscape between the abolitionist values of liberation and healing often centered by system-impacted communities and the values of surveillance and deterrence often instantiated in data-driven reform measures. We discuss the implications of these differences in relation to participants' social positions, and we advocate for centering the interests and experiential knowledge of communities impacted by incarceration through participatory methods to ensure that algorithmic bias evidence can function as a device to challenge the status quo.
%We find that algorithmic bias has the potential to challenge as much as to affirm existing power structures. If we want to place algorithmic bias in the service of the affected communities, thus, we need to make explicit how this power plays through the different diagnoses and the solutions identified by differently-positioned stakeholders. Recognizing and characterizing each position in relation to the power it exerts is a first step towards revealing which solutions truly work in support of affected communities.
\section{Background and Related Work}
%\subsection{Algorithmic bias as ``evidence''}
%The fact that algorithmic bias reflects and reproduces existing systemic issues has become increasingly established in the literature on algorithmic fairness, especially in the context of criminal risk prediction. Bias in algorithms can reflect human bias, in terms of normative assumptions and technical decisions embedded in algorithms \cite{abebe_roles_2020}, \todo{Add Malik 2020} as well as demographic disparities in society \cite{abebe_roles_2020, benjamin_race_2019, eubanks_automating_2018}\todo{add mayson 2018, zajko 2021}, such as disparate arrest practices \cite{barabas_interventions_2018}\todo{cite Mayson 2018} in the use of arrests to infer rates of crime \cite{zilka_progression_2023}. The recognition of systemic problems reflected in bias has brought the field to recognize it as an object of social critique. Different strands consider it as a problematic object to technically resolve or manage, one to critically assess and address, and one to respond to with alternative tools. Solutions, thus, mostly focus on how to respond to this bias. However, they do not typically focus on how to use this bias to prevent or intervene on the deeper problems that it represents; namely, as a device for social change. %Repeating stuff in the intro
%Notwithstanding their recognition of the societal issues embedded in algorithmic bias, many approaches still consider identifying, removing, or managing statistical biases as the main solution, rather than directly addressing its underlying causes \cite{fazelpour_algorithmic_2021}. These solutions typically vary from pre-, during and post-deployment strategies, to prevent models from learning or outputting biases (e.g., \cite{ensign_runaway_2018, fahse_managing_2021, yeung_identifying_2021, zhang_mitigating_2018}). These include techniques to tackle the overrepresentation of minority subjects in datasets about violence and crime \cite{pastaltzidis_data_2022} and methods to quantify unobserved offenses and take into account arrest disparities \cite{zilka_progression_2023}.
%Other scholars argue that though these proposals are useful for minimizing bias locally, they are unable to account for the system-level origins of bias and can result in `fairwashing', the legitimation of algorithms that have serious shortcomings in practice \cite{aivodji_fairwashing:_2019}. They respond to the deeper societal issues reflected in bias by calling for more substantive notions of algorithmic fairness that take algorithms' objectives and sociotechnical contexts into account, with the possibility that algorithmic decision-making systems should be removed from the process entirely \cite{eubanks_automating_2018, barabas_beyond_2019, barabas_studying_2020, green_algorithmic_2020, green_escaping_2022}.
%Alternatively, several researchers have also focused on providing affected communities with additional or alternative tools to resist and respond to the deeper societal issues made manifest by algorithmic bias. An increasing number of policing data have been made available to the public (34, 53, 65) to promote police accountability. This allows policymakers, researchers and activists to scrutinize policing behavior such as stops, searchers and arrests. Statistical analyses of police data have been used to document disparities in police stops, searches, use of force and respect (2, 25, 27, 28, 29,52, 53, 64, 73).\todo{add citations} For example, \cite{franchi_detecting_2023} have proposed to use dashcam data to detect disparities in police deployments.
\subsection{Algorithmic bias: from an object of social critique to a device for social change}
Notwithstanding the widespread claim that algorithmic bias is a mirror of existing disparities \cite{mayson_bias_nodate}, it is rarer for algorithmic bias to be taken seriously as a source of evidence to emancipate and support marginalized communities. Bias has helpfully been identified as a boundary object \cite{grill_bias_2023}, an entity which, despite disagreement about its nature, provides a lens for critique and confrontation of different stakeholders' views. In this paper, we take this characterization of bias a step further; namely, we want to move from bias as an object of and for critique to a pragmatic device to address the disparities it mirrors. %Specifically, this article aims to reconcile the diverse diagnoses or interpretations that can be made of bias with the need to formulate pragmatic solutions that support affected communities.
Skepticism about applying a computational lens to societal problems \cite{broussard_artificial_2019, dignazio_counting_2024, eubanks_automating_2018, gebru_oxford_2019, green_data_2021, benjamin_race_2019} has led to a reconsideration of the roles of computational tools from optimization and individual predictions toward systemic interventions to address deeper issues of justice and equity \cite{barabas_interventions_2018, green_data_2021, barabas_refusal_2022}. Recent work in critical data studies \cite{abebe_roles_2020, dignazio_counting_2024, passi2019problem}, as well as past work at the intersection of sociology and statistics \cite{bruno_statactivism_2014, desrosieres1998politics} has mapped out ways in which practices around the use of computational tools, from the design to the deployment of these tools, can be used to address existing disparities or injustices in the status quo.
Computational tools can help unveil or criticize specific aspects of society. Through their reliance on historical and empirical data, tools both in statistics \cite{bruno_statactivism_2014} and in machine learning \cite{abebe_roles_2020} can help disclose the wrongdoings of an institution by appropriating or questioning its own numbers. The categorization and labeling needed to make data legible can indirectly lead to the creation or resurfacing of neglected or unrecognized social categories, %. The categorization and labeling needed to make data legible can help prove the existence of a neglected or unrecognized category, attempt to
redefine an existing category, or provide a means to defend a category and its rights (e.g., make it ``legible'' to the system) \cite{bruno_statactivism_2014, dignazio_counting_2024}. %\todo{(add present-day example fro D'Ignazio's work?).}
The use of computational tools can also reformulate social problems and reshape public objectives. The exercise of problem formulation and formalization the tools entail can help reason about and reassign an institution's priorities \cite{bruno_statactivism_2014}, articulate and reason about how societal problems are formulated \cite{passi2019problem}, and highlight the limitations presented by computational techniques \cite{abebe_roles_2020}.
%to denounce the ability of an institution to shape (e.g., counterfeit) reality, and to denote the possibility of a reality other than the official by, for example, using alternative data. Referring to machine learning algorithms, \cite{abebe_roles_2020} suggest that given their use or reliance on historical data, they can help diagnose emerging injustices and let historical ones resurface. For example, \cite{caliskan_gender_2022} showed how regularities in language corpora encode well-known social biases, especially gender biases, into word embeddings.
%Similarly, the emergence of the category of a ``high risk'' person or neighborhood becomes specifically relevant, especially when this categorization aligns with existing patterns of injustice and segregation. Its emergence can become a site for contestation or a target for advocacy \cite{abebe_roles_2020}. It can draw attention to a series of judgements or decisions made on the basis of historical data and specific rules.
%Statistics could also be used to point to other relevant yet overshadowed aspects of reality, and propose alternative indicators \cite{bruno_statactivism_2014}. \cite{passi2019problem} suggest that problem formulation in computer science should already be considered a site of ethical deliberation and reasoning around societal problems. The need for clear and measurable definitions in machine learning algorithms can indirectly help articulate and reason on how societal problems are formulated and highlight the limitations presented by computational techniques \cite{abebe_roles_2020}. Reasoning about the formulation of ``risk'' in criminal justice, for example, prompted \cite{meyer_flipping_2022} to suggest reformulating the use of risk assessment tools to measure the risk that the system poses to the individual rather than vice versa. Similarly, this formalization can trigger questions around which goal or task we want these tools to achieve, and whether they should be developed at all.
The power of numerical tools, however, can work in both directions. It can expose as much as it can affirm existing power structures \cite{bruno_statactivism_2014, barabas_refusal_2022}. The same numbers that can be used by communities to expose their oppression can, for example, be used by police to justify their intervention. %Even Chelsea Barabas \cite{barabas_refusal_2022} stresses how, notwithstanding a focus toward ethical approaches to computation, key stakeholders are often not equipped with the conceptual tools necessary to interrogate, resist, and re-imagine the relationships through which power shapes their work.
Starting from these considerations, this article explores whether algorithmic bias can be used as a device to disclose or address the societal disparities that it mirrors. At the same time, we explore how evidence of bias enables diverse problem diagnoses and supports the identification of solutions contingent upon individual worldviews. Given its potential to disclose as much as affirm power, we explore how -- if at all -- this evidence can serve the interests of affected communities.
We connect these questions to a parallel tension in criminal justice between abolitionist and reformist responses to the crisis of mass incarceration. Abolitionist philosophy calls for an end to the reliance on imprisonment and policing and proposes alternatives that engage holistically with the structural conditions underlying violence and suffering %``holistic engagement with the structural conditions that give rise to suffering, as well as the interpersonal dynamics involved in violence''
\cite{mcleod2018envisioning}. In this vein, abolitionists often advocate for `non-reformist reform', change that is not cordoned by what is possible within the current system but rather change that is premised on what should be made possible long-term given human needs %``conceived not in terms of what is possible within the framework of a given system and administration, but in view of what should be made possible in terms of human needs and demands''
\cite{gorz1968strategy}. Non-reformist reform prioritizes decarceration and reinvestment of resources back into communities. This is in contrast to `reformist reform', which limits its objectives to the maintenance and practicality of the current system. Most data science applications in the context of criminal justice are reformist reforms \cite{green2019good}; as we see in the interview results, the predictive policing tool itself is seen as a reformist reform, drawing a sharp line with reform advocates in favor and abolitionists opposed. %although we did encounter some exceptions, like a police officer that is opposed. % (this includes the reforms discussed in \S\ref{sec:allocation}).
%\todo{Integrate; cite Ben Green.}
%Applying the considerations above to the context of crime risk prediction, evidence of police enforcement bias could be used to expose or denounce policing in specific communities, defend or produce the social category of ``policed communities,'' and spark a reevaluation of the very purpose of crime prediction algorithms.\todo{Consider moving to discussion} However, at the same time, this evidence could help affirm the absence of insufficient patrols in a neighborhood and justify their increased presence, help sediment or justify an emerging social category of the ``criminal'' individual or the ``high risk'' neighborhood, and help reaffirm the re-establishment of the status quo as the ``only solution''. Starting from these considerations, this article explores whether algorithmic bias can be used as a device to disclose or address the societal disparities that it mirrors. At the same time, however, it does so by exploring how the evidence of bias enables diverse problem diagnoses and supports the identification of solutions contingent upon individual worldviews.
\subsection{The Chicago Crime Prediction Tool}
%\todo{cite; George Floyd murder and BLM; data-driven reformist reform} %These place-based predictive policing tools use past crime incidence data to predict the likelihood of future crime across time and geography.
%Data-driven decision-making is a standard reform measure, particularly for institutions experiencing legitimacy problems and a corresponding loss of trust in human decision-makers \cite{porter_trust_1995}.
Since the early 2000s, police departments around the US have begun adopting new technologies to forecast crime, often in partnership with private companies like PredPol and Palantir \cite{brayne_predict_2020}. The Chicago Police Department (CPD) first began using CompStat, an early data-driven geographic forecasting policing program, in 2003 \cite{verma_heat_2021}. In 2012, the CPD started using the Strategic Subject List, a person-based predictive policing tool made by the Illinois Institute of Technology, to generate a ``heat list'' of likely perpetrators or victims of violent crime \cite{stroud_automated_2021}. The program was terminated in 2019 following investigations showing that the program was inaccurate, ineffective, and racially biased, among other problems \cite{kunichoff_contradictions_2017, sweeney_for_2020}. In 2018, the CPD started using ShotSpotter, a controversial gunshot audio detection technology, to dispatch police to areas with suspected recent gunfire, which led to high-intensity investigatory stops and fatal interactions between the CPD and city residents \cite{simkin_looking_2023}. In March 2021, ShotSpotter was involved in an incident in which police killed 13-year-old Adam Toledo, intensifying community opposition to the technology. After investigations found that Shotspotter was inaccurate, exacerbated the over-policing of neighborhoods of color, and increased unconstitutional stop and frisk by police \cite{ferguson2021chicago, feathers_gunshot-detecting_2021}, city officials announced that they would end their contract with the company by September 2024 \cite{mohtasham_chicago_2024}. %The Chicago mayor recently announced that the program would be terminated in September 2024 has been criticized for inaccuracy, racial bias and law enforcement misuse. Chicago’s contract with SoundThinking, a public safety technology company that says its ShotSpotter tool is used in roughly 150 cities, expires on Friday. The city plans to wind down use of ShotSpotter technology by late September, according to city officials. Since 2018, the city has spent \$49m on ShotSpotter.
%
These events all happened in the backdrop of the 2017 consent decree, a court order mandating broad police reform in the city of Chicago. %\todo{add citations and more context}
%From Akpinar et al: "More modern incarnations of predictive policing date back to 2008, when the Los Angeles Police Department (LAPD) began its explorations of these systems, followed shortly thereafter by efforts such as the New York Police Department's use of tools developed by firms including Azavea, KeyStats and PredPol (2012+). Far from being a US-centric phenomenon, such systems are widely used throughout Europe, the UK, and China"
%"Focusing specifically on geospatial predictive policing systems, I argue that ``predictive policing for reform'' should be seen as a flawed attempt to rationalize police patrols through an algorithmic remediation of patrol geographies. The attempt is flawed because predictive systems operate on the sociotechnical practices of police patrols, which are themselves contradictory enactments of the state's power to distribute safety and harm. The ambiguities and contradictions of the patrol are not resolved through algorithmic remediation. Instead, they lead to new indeterminacies, trade-offs, and experimentations based on unfalsifiable claims. " Shapiro "Predictive Policing for Reform? Indeterminacy and Intervention in Big Data Policing"
A recent article on event-level prediction of urban crime in Chicago (herein referred to as the Chicago crime prediction algorithm) \cite{rotaru_event-level_2022} is the latest development in this spotted history, though the algorithm is not yet deployed by any police department. Using violent and property crime incidence data from the city of Chicago, Rotaru et al.\ built a spatiotemporal network that inferred patterns from past event occurrences. This network, known as the Granger network, comprised local estimators forming a communicating network to predict future infractions. Using publicly recorded historical event logs, Rotaru et al.\ extended their analysis beyond the training sample to events in the subsequent year. See \autoref{figure:model_card} for a model card \cite{mitchell_model_2019} outlining additional details of the algorithm. %The study focused on two overarching categories of reported criminal infractions: violent crimes (e.g., homicide, assault, and battery) and property crimes (e.g., burglary, theft, and motor vehicle theft).
In contrast to the CPD's wave of failed predictive policing experiments, Rotaru et al. present their novel algorithm as taking a different approach to predictive policing. Their stated aim is not simply to predict hotspots of crime with greater accuracy than commercially available tools but also to ``garner deep insight into the nature of the dynamical processes through which policing and crime co-evolve in urban spaces,'' including how policing interacts with, modulates and reinforces crime. They emphasize that %``Rather than simply increasing the power of states by predicting the when and where of anticipated crime, our tools allow us to audit them for enforcement biases, and garner deep insight into the nature of the dynamical processes through which policing and crime co-evolve in urban spaces.''
they see their algorithm as serving the dual role of enhancing state power through criminal surveillance while also providing a means to scrutinize states by allowing researchers to ``audit them for enforcement biases'' \cite{rotaru_event-level_2022}. Indeed, %n addition to assessing the algorithm's accuracy in predicting future crime,
%Rotaru et al.\ argued that their algorithm
Rotaru et al.\ write that the Chicago crime prediction algorithm indicates the presence of socioeconomic bias in law enforcement. Modeling the number of individuals arrested during each recorded event allowed the researchers to %probe the possibility and pattern of this bias, that is, to
contrast crime rates with arrest rates, revealing that increases in crime rates correlated with increased arrests in wealthier areas, whereas in low socioeconomic status %disadvantaged
neighborhoods, arrest rates dropped without a corresponding decrease in crime. This pattern persisted across all the years under analysis. The researchers attributed this bias to resource constraints on police, compounded by discretionary prioritization of wealthier neighborhoods, corroborating suspected enforcement bias toward US suburbs with high socioeconomic status \cite{meyer_suburban_2016, lipton_why_1977}. The authors used this evidence of socioeconomic enforcement bias to suggest a need-based reallocation of police patrols across neighborhoods in Chicago.
%, one with reduced impact depending on the socioeconomic status of the involved communities. [Commented out because wans't sure what it meant]
\begin{figure}
\caption[Model Card]{Chicago Crime Prediction Algorithm Model Card}
\fbox{\parbox{8.3cm}{
\noindent \begin{itemize}[leftmargin=*]
\small
\item \textbf{Model Details}
\begin{itemize}
\item Spatio-temporal network inference algorithm: infers patterns of past event occurrences and constructs a communicating network (Granger network) of local estimators to predict future infractions.
\item Published by academic researchers Rotaru et al.\ at the University of Chicago in \textit{Nature Human Behavior} in 2022 \cite{rotaru_event-level_2022}.
\end{itemize}
\item \textbf{Intended Use}
\begin{itemize}
\item To accurately predict hotspots of crime in Chicago, producing insights on the processes of crime and its interaction with policing.
\item The authors recommend policy changes for ``more equitable, need-based resource allocation'' of police and suggest that their tool can ``be used to track the extent to which such policies approach this trace of equitable enforcement allocation.''
%\tabitem To model how police enforcement interacts with, modulates and reinforces crime.
\end{itemize}
\item \textbf{Training Data}
\begin{itemize}
\item Spatiotemporal logs of crime incidence data in Chicago, as well as the nature, category and description of each incident.
\begin{itemize}
\item Public domain criminal event logs were also analyzed for seven additional cities: Detroit, Philadelphia, Atlanta, Austin, San Francisco, Los Angeles and Portland.
\end{itemize}
\item Socioeconomic variables corresponding to regions of Chicago.
\item The number of individuals arrested during each recorded event is modelled separately, allowing the investigation of enforcement bias.
\end{itemize}
% \item \textbf{Factors}
% \begin{itemize}
% \item Location and time of crime incidence data, as well as the nature, category and a brief description of each incident.
% \item Two categories of reported criminal infractions are considered: violent crimes consisting of homicide, assault and battery, and property crimes consisting of burglary, theft and motor vehicle theft.
% \item The number of individuals arrested during each recorded event is modelled separately, allowing the investigation of enforcement bias.
% \end{itemize}
\item \textbf{Metrics and Evaluation}
\begin{itemize}
\item Area Under Curve (AUC)%Out-of-sample AUC for predicting violent and property crimes. Distribution of AUC on average, individually for violent and property crimes. The mean AUC is close to 90\%.
, Predictive Accuracy Index (PAI) (normalized event rate in identified hotspots) %normalized event rate in identified city hotspots.
, Prediction Efficiency Index (PEI). %the ratio of the PAI bounded between 0 and 1.
\item PAI and PEI results were compared against the best-performing teams in a crime forecast challenge hosted by the National Institute of Justice in 2017. %(https://nij.ojp.gov/funding/real-time-crime-forecasting-challenge).
\item Enforcement bias: ``In wealthier neighbourhoods, the response to elevated crime rates is increased arrests, while arrest rates in disadvantaged neighbourhoods drop but the converse does not occur.''
\end{itemize}
\item \textbf{Ethical Considerations}
\begin{itemize}
\item Potential misuse: the authors acknowledge that these results may be falsely interpreted to justify increased police intervention in higher-crime (often more racially and ethnically diverse) neighborhoods, which could lead to increased surveillance and fatal police interactions in marginalized communitiees.
\item Bias in crime reporting data: Rotaru et al.\ also note that incident logs likely contain ``biases arising from disproportionate crime reportage and surveillance [of communities of color],'' which ``no amount of scrubbing or clever statistical controls can reliably erase.''
%\item Rotaru et al.\ acknowledge the danger that powerful predictive tools place in the hands of powerful states in the name of civilian protection, but they emphasize the ability to audit enforcement biases and hold states accountable in ways inconceivable in the past.
% \item
\end{itemize}
% \item \textbf{Evaluation Data}
% \begin{itemize}
% \end{itemize}
% \item \textbf{Caveats and Recommendations}
% \begin{itemize}
% %\item To the best of our knowledge, the tool has not been acquired or deployed by any police department. However, we are aware that it has been presented to a significant commission on community safety in Chicago and about plans for its adoption in at least two major US police departments.
% \item Potential misuse. These results may be used to justify increased police intervention in higher-crime (often more racially and ethnically diverse) neighborhoods.
% \item Bias in crime reporting data. ``We doubt that any amount of scrubbing or clever statistical controls can reliably erase such ecological patterning of apparent crime. Any policy informed by our results must keep this caveat in mind'' \cite{rotaru_event-level_2022}
% \end{itemize}
\end{itemize}}}
\label{figure:model_card}
\end{figure}
% \begin{table}[t!]
% \small
% \centering
% % \singlespacing
% \begin{tabular}{l}
% % \hline
% \textbf{Model Card: Chicago Crime Prediction Algorithm} \\
% \hline
% \textbf{Model Details}\\
% % \hline
% \hspace{1.5em} \parbox{8 cm}{\tabitem Spatio-temporal network inference algorithm, which infers patterns of past event occurrences and constructs a communicating network (the Granger network) of local estimators to predict future infractions.}\vspace{.3em}\\
% \hline
% \textbf{Intended Use}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem To accurately predict hotspots of crime in Chicago, producing insights on the system of crime and its rules of organization. \vspace{.3em}
% \tabitem To model how police enforcement interacts with, modulates and reinforces crime.}\vspace{.3em}\\
% \hline
% \textbf{Factors}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem Two broad categories of reported criminal infractions are considered: violent crimes consisting of homicide, assault and battery, and property crimes consisting of burglary, theft and motor vehicle theft. \vspace{.3em}
% \tabitem The number of individuals arrested during each recorded event is modelled separately, allowing the investigation of the possibility and pattern of enforcement bias.} \vspace{.3em}\\
% \hline
% \textbf{Metrics}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem Area Under Curve (AUC): Out-of-sample AUC for predicting violent and property crimes. Distribution of AUC on average, individually for violent and property crimes. The mean AUC is close to 90\%. \vspace{.3em}
% \tabitem Predictive accuracy index (PAI): normalized event rate in identified city hotspots.\vspace{.3em}
% \tabitem Prediction efficiency index (PEI): the ratio of the PAI bounded between 0 and 1.\vspace{.3em}
% \tabitem Enforcement bias: In wealthier neighbourhoods, elevated crime rates correlate with increased arrests. Conversely, in disadvantaged neighbourhoods, arrest rates dropped without a corresponding decrease in crime.} \vspace{.3em}\\
% \hline
% \textbf{Ethical Considerations}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem Rotaru et al.\ acknowledge the danger that powerful predictive tools place in the hands of powerful states in the name of civilian protection, but they particularly stress their unprecedented ability to audit enforcement biases and hold states accountable in ways inconceivable in the past. \vspace{.3em}
% \tabitem \textbf{Potential misuse.} These results may be used to justify increased police intervention in higher-crime (often more racially and ethnically diverse) neighborhoods.} \vspace{.3em}\\
% %`` our results may be falsely interpreted to mean that there is ‘too much’ policing in low-crime (often predominantly White) communities, and too little policing in higher-crime (often more racially and ethnically diverse) neighbourhoods''
% ``We doubt that any amount of scrubbing or clever statistical controls can reliably erase such ecological patterning of apparent crime. Any policy informed by our results must keep this caveat in mind.''
% \hline
% \textbf{Training Data}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem Historical geolocated crime incidence data, number of arrests made for each ‘event’, socio-economic variables from the city of Chicago
% \tabitem They also analyse criminal event logs available in the public domain for seven additional major US cities: Detroit, Philadelphia, Atlanta, Austin, San Francisco, Los Angeles and Portland.}\vspace{.3em}\\
% \hline
% \textbf{Evaluation Data}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem They compared results of the PAI and PEI against the best-performing teams in a crime forecast challenge hosted by the NIJ in 2017 (https://nij.ojp.gov/funding/real-time-crime-forecasting-challenge).}\vspace{.3em}\\
% \hline
% \textbf{Caveats and Recommendations}\\
% \hspace{1.5em} \parbox{8 cm}{\tabitem To the best of our knowledge, the tool has not been acquired or deployed by any police department. However, we are aware that it has been presented to a significant commission on community safety in Chicago and about concrete plans for its utilization in policing applications in Boston and Philadelphia.}\vspace{.3em}\\
% % \textbf{Community Organizations (C)}\\
% % \hspace{1.5em}Re-entry associations&4&22.2\%\\
% % \hspace{1.5em} Safety \& Community Initiatives&3&16.6\%\\
% % \hline
% % \textbf{Researchers (R)}& 6&33.4\%\\
% % \hspace{1.5em}Tool Developers&1&5.6\%\\
% % \hspace{1.5em}Academic Researchers&5&27.8\%\\
% % \hline
% % \textbf{Public Sector (P)}&5&27.8\%\\
% % \hspace{1.5em}Public Officials&1&5.6\%\\
% % \hspace{1.5em}Police&2&11.1\%\\
% % \hspace{1.5em}Contractors \& Oversight bodies\footnote{This refers to people such as contractors and consultants working with or for criminal legal system institutions and independent bodies working as a ``broker'' or as a link between communities and the criminal legal system.}&2&11.1\%\\
% % Total&18&100\%\\
% % \hline
% \hline
% \end{tabular}
% %\vspace{.8em}
% \caption[Model Card]{Chicago Crime Prediction Algorithm Model Card.}
% %\centering \textbf{Table 1:} Features of interviewed judge population based on demographic survey (15 judges)
% \label{table:demographics}
% \end{table}
% \textbf{Model Card}
% \textit{Model Details:}
% \begin{itemize}
% \item Spatio-temporal network inference algorithm
% \end{itemize}
% \begin{itemize}
% \item It infers patterns of past event occurrences and constructs a communicating network (the Granger network) of local estimators to predict future infractions.
% \end{itemize}
% \textit{Intended use:}
% \begin{itemize}
% \item To accurately predict hotspots of crime in Chicago, producing insights on the system of crime and its rules of organization
% \end{itemize}
% \begin{itemize}
% \item To model how police enforcement interacts with, modulates and reinforces crime
% \end{itemize}
% \textit{Factors:}
% \begin{itemize}
% \item Considers two broad categories of reported criminal infractions: violent crimes consisting of homicide, assault and battery, and property crimes consisting of burglary, theft and motor vehicle theft.
% \end{itemize}
% \begin{itemize}
% \item The number of individuals arrested during each recorded event is modelled separately, allowing to investigate the possibility and pattern of enforcement bias.
% \end{itemize}
% \textit{Metrics:}
% \begin{itemize}
% \item AUC: Out-of-sample AUC for predicting violent and property crimes. Distribution of AUC on average, individually for violent and property crimes. The mean AUC is close to 90%
% \end{itemize}
% \begin{itemize}
% \item They also compute the predictive accuracy index (PAI) - normalized event rate in identified city hotspots - and the prediction efficiency index (PEI) - the ration of the PAI bounded between 0 and 1
% \end{itemize}
% \begin{itemize}
% \item Enforcement bias: In wealthier neighbourhoods, elevated crime rates correlate with increased arrests. Conversely, in disadvantaged neighbourhoods, arrest rates dropped without a corresponding decrease in crime.
% \end{itemize}
% \textit{Ethical Considerations:}
% \begin{itemize}
% \item They acknowledge the danger that powerful predictive tools place in the hands of powerful states in the name of civilian protection, but they particularly stress their unprecedented ability to audit enforcement biases and hold states accountable in ways inconceivable in the past
% \end{itemize}
% \begin{itemize}
% \item Potential misuse of the tool: their results may be falsely interpreted to mean that there is too little policing in higher-crime (often more racially and ethnically diverse) neighbourhoods and justify police intervention
% \end{itemize}
% \textit{Training Data: }
% \begin{itemize}
% \item Historical geolocated crime incidence data, number of arrests made for each ‘event’, socio-economic variables from the city of Chicago
% \end{itemize}
% \begin{itemize}
% \item They also analyse criminal event logs available in the public domain for seven additional major US cities: Detroit, Philadelphia, Atlanta, Austin, San Francisco, Los Angeles and Portland.
% \end{itemize}
% \textit{Evaluation data:}
% \begin{itemize}
% \item They compared results of the PAI and PEI against the best-performing teams in a crime forecast challenge hosted by the NIJ in 2017 (https://nij.ojp.gov/funding/real-time-crime-forecasting-challenge).
% \end{itemize}
% \textit{Caveats and Recommendations:}
% \begin{itemize}
% \item To our best knowledge, the tool has not been deployed yet. However, we are aware that it has been presented to a significant commission on community safety in Chicago and about concrete plans for its utilization in policing applications in Boston and Philadelphia.
% \end{itemize}
\vspace{-.4em}
\section{Methods}
To interrogate the ways in which the enforcement bias captured by the Chicago crime prediction algorithm can serve as evidence, we conducted semi-structured interviews with 18 stakeholders, primarily from the Chicago metropolitan area.
\noindent \textbf{Recruitment and Demographics.} We conducted interviews with stakeholders from various backgrounds, including academia, community organizations, and the public sector. After an initial mapping of relevant actors through desk research, we selected the rest through referrals from initial respondents. At the time of this writing, the Chicago crime prediction algorithm has not been publicly deployed; snowball sampling allowed us to trace a latent network of relevant actors that we otherwise would not have identified \cite{parker_snowball_2019}. We continued recruiting and interviewing participants from each background until no new themes or information directly relevant to the project emerged from the interviews \cite{small_how_2009} or our timeframe for data collection passed. In total, we attempted to recruit 62 participants, achieving a response rate of 29\%.
%
We made an effort to select participants with variation \cite{weiss_learning_1994} across roles (see \autoref{table:demographics}) and level of personal exposure to the criminal legal system (see \autoref{table:cls_exposure}), ranging from no prior involvement to formerly incarcerated. This variation enabled us to consider the role of individuals' positionality in the results; we indicate each quoted participant's background (`C' for community organization, `R' for researcher, or `P' for public sector). While eight participants had direct experience or knowledge of the algorithm, several interviewees had no prior familiarity with it; these latter interviews helped inform our understanding of Chicago-area policing more broadly.
Due to confidentiality concerns, recruiting participants from the public sector was challenging. This is a common issue in interviews with participants in senior or official roles -- so-called ``elite'' interviews \cite{richards_elite_1996}. We accounted for this by recruiting two additional participants, one who is a contractor for the criminal legal system and another who works for an oversight body. %(who asked to remain anonymous).
\begin{table}[t!]
\small
\centering
% \singlespacing
\begin{tabular}{ l c r }
\textbf{Role} & \textbf{Frequency} & \textbf{\%} \\
\hline
\textbf{Community Organizations (C)}&7&38.8\%\\
\hspace{1.5em}Re-entry Associations&4&22.2\%\\
\hspace{1.5em}Safety \& Community Initiatives&3&16.6\%\\
\hline
\textbf{Researchers (R)}& 6&33.4\%\\
\hspace{1.5em}Tool Developers&1&5.6\%\\
\hspace{1.5em}Academic Researchers&5&27.8\%\\
\hline
\textbf{Public Sector (P)}&5&27.8\%\\
\hspace{1.5em}Public Officials&1&5.6\%\\
\hspace{1.5em}Police&2&11.1\%\\
\hspace{1.5em}Contractors \& Oversight Bodies\footnote{This refers to people such as contractors and consultants working with or for criminal legal system institutions and independent bodies working as a ``broker'' or as a link between communities and the criminal legal system.}&2&11.1\%\\
Total&18&100\%\\
\hline
\end{tabular}
%\vspace{.8em}
\caption[Roles of participant population]{Roles of participant population.}
%\centering \textbf{Table 1:} Features of interviewed judge population based on demographic survey (15 judges)
\label{table:demographics}
\end{table}
\begin{table}[t!]
\small
\centering
% \singlespacing
\begin{tabular}{ l c r }
\textbf{Exposure to the criminal legal system} & \textbf{Frequency} & \textbf{\%} \\
\hline
System-impacted individuals&2&11.1\%\\
Working directly with system-impacted individuals&7&38.9\%\\
Employed by the criminal legal system (e.g., police)&2&11.1\%\\
Working with the criminal legal system&3&16.7\%\\
No direct exposure (e.g., researchers) &4&22.2\%\\
Total & 18 & 100\%\\
\hline
\end{tabular}
%\vspace{.8em}
\caption[Participant exposure to the criminal legal system]{Participant exposure to the criminal legal system}
%\centering \textbf{Table 1:} Features of interviewed judge population based on demographic survey (15 judges)
\label{table:cls_exposure}
\end{table}
\noindent \textbf{Interview Process.} Interviews ranged from 30 minutes to 1 hour, with an average length of 39 minutes. The first author conducted all interviews; two interviews were carried out in person and the rest via video call. Interviews were semi-structured \cite{edwards_what_2013}, allowing us to focus and dig deeply into an initial set of topics, while also letting participants' expertise guide the interview and allowing new themes to emerge \cite{roulston_sage_2018}. Specifically, we asked about participants' understanding of and attitudes toward algorithmic bias generally and police enforcement bias specifically; the challenges and problems they believe to be associated with these biases; and the interventions available or those they envision in the future (see \autoref{sec:interview_guide} for a full list of interview questions). We adjusted the content and wording of the questions slightly depending on the participants' particular knowledge of or experience with the algorithm and the criminal legal system. For example, for participants with intimate knowledge of the algorithm, such as its developers, more technically detailed questions were posed, while for participants with local knowledge of the criminal legal system, such as members of community organizations, a stronger emphasis was placed on questions about the Chicago context and community safety.
\noindent \textbf{Qualitative Analysis.} We analyzed interview transcripts using the qualitative data analysis software Nvivo. Both authors analyzed interviews iteratively, identifying recurring themes, reviewing transcriptions 2-3 times, and switching between inductive coding and data collection to clarify and strengthen themes and uncover opposing evidence \cite{miles_qualitative_2014}. We converged on seven high-level themes (in order of most to least prevalence in the interview data): downsides of AI; upsides of AI; structural bias; lived experience and positionality; abolition; reformism; and controversy around policing (see \autoref{sec:code_table} for a code table). We complemented the results from the interviews with a document analysis \cite{karppinen_what_2011} of policy proposals and programs produced by the interviewed stakeholders, as well as other active participants in the domain of policing and justice reform in the Chicago area (e.g., policymakers, regulators, community organizations, and think tanks). Through triangulation with findings from our interview data, we were able to elucidate, refute, or expand on the findings from our interview data \cite{frey_sage_2018}. %Specifically, we were able to gain a better understanding of the role that bias may play for each stakeholder group.
\section{Results}
%\todo[inline]{Results section should just report what was in interviews -- our interpretation and analysis should come in the discussion}
%Participants articulated diverse problem diagnoses of the tool's algorithmic bias, ranging from systemic racism to resource allocation problems. These diagnoses in turn stemmed from diverse understandings of crime -- whether crime is an ``event'' to be tabulated and predicted, a ``disease'' to be treated and controlled, or a ``punishment'' for poverty and lack of opportunity.
%Participants themselves recognize that there are multiple ways of interpreting data or categories like community safety. TODO Check out codes: multiple possible uses of data, multiple ways of achieving same goal, multiple ways of interpreting community safety
Across the board, interviewed stakeholders strategically used algorithmic bias as evidence to advance interventions that aligned with their own positionality and values about the criminal legal system. Drawing inspiration from Catherine D'Ignazio's taxonomy of feminicide activists ``refusing and using'' data \cite{dignazio_counting_2024}, we see that stakeholders from different backgrounds use evidence of algorithmic bias to \textbf{reform} the policies around police patrol allocation; in the case of more powerful actors, \textbf{reaffirm} their own authority or existing power structures; \textbf{reject} algorithm-based policing interventions; \textbf{reveal} counterdata on authority figures in an effort to subvert their power; \textbf{reframe} crime as a structural rather than interpersonal problem; and \textbf{repair} and heal families and communities. %We elaborate on each of these terms entails will be made explicit in the following paragraphs. [I took this out because it reads weird, I feel it's unnecessary despite the annoyed reviewer!]
%Whether and how algorithmic bias was seen as useful for advancing each of these goals varied according to each stakeholder's broader understanding of the source of police enforcement disparities -- structural inequality versus interpersonal bias -- and the set of interventions aligned with each respective problem diagnosis -- abolitionist/non-reformist reforms versus reformist reforms. This difference reflects tensions between the assumptions underlying interventions grounded in reform versus abolition, but we also see areas of agreement across stakeholder groups, such as on the importance of community safety.
%We also saw substantial overlap among interviewees from all backgrounds regarding potential downsides to the use of AI and data in the context of policing, such as AI's possibility to miss the nuance of human discretion, and notable differences relating to AI's upsides, specifically its potential to `disrupt' the criminal legal system in a beneficial way. Techno-optimism mapped onto the extent to which a stakeholder understood the problems as primarily interpersonal rather than structural, though proponents of the former still saw value in the use of grassroots data to counter narratives put forward by police. \todo{don't want to delete this paragraph because I think will be important for discussion -- think about shortening/incorporating elsewhere}
\subsection{Reform: Towards ``a legitimate, ethical, fair police force.'' [P013]}
\label{sec:reform}
\subsubsection{Policing as an optimization problem.}
\label{sec:allocation} Making policing more `fair' is a standard goal of criminal justice reform \cite{van2017achieving}. Stakeholders from academic and public sector contexts tended to focus on `over-' or `under-policing' as the source of the Chicago crime prediction algorithm's bias, %, with corresponding attention to the goal of achieving an optimal distribution of policing across all communities.
emphasizing the benefits of allocating policing more optimally, such as increased public safety and police legitimacy. However, these participants had mixed views about whether predictive policing was a viable strategy to achieve this goal. %Some academic and public sector participants supported predictive policing as a viable strategy to achieve this goal, while others expressed hesitance and caution.
Three of the academic researchers we interviewed, including one of the tool's developers and members of a Chicago-area research group focused on criminal justice research, framed the problem revealed by algorithmic bias as a police allocation issue. One senior member of the research group [R006] explained the need for improving police presence in marginalized neighborhoods: %pointed out the high crime rates faced by people in marginalized neighborhoods in Chicago and the corresponding need for increased police presence in those neighborhoods:
``People who live in those communities ask for more police because they're facing high levels [of violent crime] ... %, like particularly in Chicago. It's not, you know, high levels of property crime. It's high levels of, like, serious violence and murder. Yeah. And, you know, there's a problem with under-policing, and, and so,
I mean, the reality is like somehow we are both, like, those communities are both over- and under-policed at the same time.'' Another member of the research group [R007] posited that ``police presence alone deters crime,'' even if police make no arrests.
These attitudes aligned with perspectives voiced by two police officers. Echoing the senior academic's perspective, one officer [P003] argued that ``the residents in those [Black and Brown] neighborhoods, that's the silent majority. They want us there, they need us.'' Another officer [P013] distinguished ``feeling safe'' and ``being safe,'' emphasizing the significance of the former and highlighting that police presence contributes to a sense of security among citizens. %This is starkly different from the perspective of some interviewees from community organizations for which police presence is an ``enforcement presence, not a not a deterrent, but an enforcement'' [C001].
On this shared view, there is intrinsic value to policing, and distributing it optimally is a primary reform goal. %, and fairness requires a consistent distribution of enforcement efforts according to crime incidence.
However, some stakeholders who were otherwise proponents of reforming police allocation expressed reservations about the use of predictive policing specifically to achieve those goals. The officer who argued that increased police presence promotes the feeling of safety [P013] warned that the use of predictive policing and other technological systems could be ``preventing cops from doing their job, which is a social job'' by making them too ``comfortable coming into an office and sitting behind a computer all day,'' thereby diminishing the physical presence of police in communities and compromising the feeling of safety. Likewise, one of the tool's developers [R002] did not want the tool to be directly used to inform police allocation decisions.%\todo{improve wording last sentence}
%[P013]: ``Now, technology is really useful in some ways because you can highlight things like hotspots of crime and when things are happening and predictive analysis and all that sort of stuff. But actually, sometimes it's preventing cops from doing their job, which is a social job. It's actually being out in the community, speaking to folks. ... A lot of cops are quite comfortable coming into an office and sitting behind a computer all day. But actually, I don't want them there. I want them in the community.''
\subsubsection{Reducing interpersonal bias.} \label{sec:412} Several public sector actors saw the tool's enforcement bias as a potential result of another kind of unfairness -- interpersonal bias, including implicit bias, the bad judgment of a ``few bad apples,'' or simply limitations in human cognition. Reducing interpersonal bias among police officers through implicit bias training, data-driven policing, or more diverse police forces is another standard route of reform \cite{spencer2016implicit, van2017achieving}. %Improving that is a strategy to remedy enforcement bias.
%Interpersonal bias is understood to be a central problem by multiple public sector actors, including [P015],
One county commissioner [P015] in a majority Latino district of Chicago called implicit bias ``very damaging'' to communities and explained that police officers, judges, and ``all of our criminal justice system'' must attend implicit bias training. %``law enforcement, but also like judges, right, all of our criminal justice system -- our state attorneys, our public defenders -- like, they all need to undergo that, and to my knowledge, most of these offices do go to that type of training.'' [P015]
A police officer [P013] also described what he called declining ``standards of professional behavior'' among police officers, underscoring the need for strong leadership programs, adding that this would improve community trust and legitimacy of police.
Arguing for the benefits of ``research-driven'' policing, another police officer [P003] admitted that ``we're not the brightest bulbs'' but that ``there are smarter minds analyzing all the different mathematical models in the back end of the AI, so they'll be able to inform us and they'll keep all of the AI models accountable.'' Using the example of people's tendency to overcount the significance of a stale criminal record, one academic researcher [R006] argued that an algorithm could ``have a sort of more built-in decay mechanism than like a human could,'' thus improving on limitations in human cognition.
%[R006]: ``You know, you're looking at somebody who was arrested in 2023, and their most recent conviction was 2005. You should really discount that heavily, and that would come up, that would come out in the data. It's not obvious that humans sort of do that, you know, because like a lot of times, like, 2005 doesn't seem that long ago, but, you know, it actually was, and that would be an instance of which I think it's, it's sort of possible for an algorithm to have a sort of more built-in decay mechanism than like a human could.'' %[P003]. %``We need strong leadership that insists on proper standards of professional behavior. Yeah, we need proper accountability. We need a proper performance management framework in place so we can measure success or failure.'' [P013]
%In some cases, eliminating interpersonal bias and improving leadership is also discussed as a way of building trust with communities. This is reflected in the value placed on two respective training programs aimed at police and community violence intervention ``leaders'' by some of the researchers interviewed and by the police officers who took part in it. These programs provide hands-on training in data-driven management, violence reduction, and community trust. One of them is being run as a randomized controlled trial to measure their impact on police departments and the communities.
Participants across all roles also proposed diversity, equity and inclusion (DEI) as a strategy for minimizing interpersonal bias, especially integrating diverse perspective or other stakeholders in the creation of data and algorithms. %``DEI values'' is a common reformist reform (e.g., ``we need more Black female cops'') but at the same time, when it's done well (e.g., participatory action research) it can also come in support of abolitionist goals. \todo{Dasha I think you know this better. Could you elaborate?}
%
As one interviewee from a reentry association [C001] put it, ``whose voice should be in the room for the algorithms that are being built, right? And I don't just mean a little bit of, yeah, engaging in participatory research where you sit down with a group of folks and this and that and you go off in your room and you build this thing. No, I mean you actually have to make them or the folks from these communities where these tools are going to be deployed an active creator of the system, right?'' [C001] Several researchers working with system-impacted individuals and victims of violence also stressed the importance of incorporating those groups' voices in research. %, and others expressed the same for victims of violence.
Another participant from a reentry association [C004], however, highlighted that incorporating diverse stakeholders on its own would not solve the problem: ``You could take a Black person or a Brown person, or some other disenfranchised individual, and they could be the author of, you know, some new algorithm. But unless they have a specific training and knowledge around systems of bias -- how they're constructed, how they operate and how they can be deconstructed -- then I don't think that these individuals that are constructing these programs can, can effectively create solutions for them.'' %\todo{maybe move to reject?}
%predictive policing was proposed as a strategy for minimizing human bias. For instance, the senior academic researcher [R006] used the example of a stale criminal record to illustrate how an algorithm could improve on biased (inaccurate) human judgment:
%[R006]: ``You know, you're looking at somebody who was arrested in 2023, and their most recent conviction was 2005. You should really discount that heavily, and that would come up, that would come out in the data. It's not obvious that humans sort of do that, you know, because like a lot of times, like, 2005 doesn't seem that long ago, but, you know, it actually was, and that would be an instance of which I think it's, it's sort of possible for an algorithm to have a sort of more built-in decay mechanism than like a human could.''
%[Could say something about reducing interpersonal bias by incorporating more diversity in the police force -- this is also a very common reform discourse; did any participants talk about this?]
\subsubsection{Increasing transparency and accountability.} \label{sec:transparency}
Beyond algorithmic bias, some participants associated another reform goal with the predictive policing tool, namely, accountability and transparency. %A third reform goal participants associated with predictive policing is transparency and accountability.
This is a common byline in police reform rhetoric \cite{white2021can} and is often presented as a justification for data-driven policing and surveillance through body-worn cameras.%\footnote{Police are trained to manipulate and withhold body-worn camera footage to their own benefit \cite{umansky_failed_2023}.}
Referring to the Chicago crime prediction tool, one police officer [P003] said that ``it being open source is a winner'' because of what he pejoratively referred to as the ``two big buzzwords'' around policing following George Floyd's murder in 2020, namely, accountability and transparency. %the predictive policing tool promotes transparency and accountability because it is open-source: %nature promotes transparency and accountability because it is open-source:
%``Why I think [the tool] is going to, is going to be successful is because it's open source. ... %He's got nothing to hide. Here's, here's the open source software.
``Here's the open source data, you know.
You look at it then I'm not hiding anything. There's no secret sauce to this. There's no ulterior motives.'' %He explains that ``it being open source is a winner'' because of what he pejoratively refers to as the ``two big buzzwords'' around policing following George Floyd's murder in 2020, accountability and transparency. %values centered by police oversight groups are pejoratively referred to as post-Floyd ``buzzwords,'' superficially satisfied by the technical availability of code.
%[interviewer]: ``Yeah, kind of like transparency, accountability.''
%
%[P003]: ``Yes, and
%those are the two big buzzwords that they talk about lately in policing in America. Ever since, you know, 2020 BLM post-Floyd, it's all of this accountability, transparency. So it being open source is a winner.% So the ACLU or any digital privacy organizations that are concerned, they-- we welcome them to look at it.
However, other participants pointed out that the relationship between open-source and accountability is far from a given. One reentry community organizer [C001] argued that ``the people that build them [algorithms] should have some accountability for how they're used ... even if you made it open source, right? Even more then, in some ways, right? You have an obligation to have thought about how it could be used for better or for worse, right?'' One of the tool's developers [R002] acknowledged the same issue with open-source, noting that ``you have to also be careful when you're giving it away'' %. They're trying to, like, make it free, that it actually stays free. If you just give it away without any rights, somebody can take it and monetize it.'' [R002] N
because nothing prevents private companies from forking open-source software into a private (i.e., non-transparent) repository. The fact that police officers explicitly reported wanting to acquire the predictive policing tool underscores that even open-source software, once developed, may be used in ways contrary to the desires of their developers, particularly once it becomes commercialized \cite{widder2022limits, widder2023dislocated}.
%``... it's not about building them [algorithms] per se, but making sure that, having some-- I think the people that build them should have some accountability for how they're used, you know, a lot of people would be like, well, `We built it. We disagree with the way they used it'. Well, you sold it to them. Or you licensed it. Yeah. And that's one of my main concerns with these algorithms. Or even if you made it open source, right? Even more then in some ways, right? You have an obligation to have thought about how it could be used for better or for worse, right? And [...] if you aren't constantly interrogating yourself about these tools and the use of this, then you haven't finished the job, right? You haven't finished the job.'' [C001]
%However, some stakeholders who are otherwise proponents of police reform expressed reservations about the use of predictive policing specifically to achieve those goals. The officer who argued that increased police presence promotes the feeling of safety [P013] warned that the use of predictive policing and other technological systems could potentially diminish the physical presence of police in communities, thereby compromising the feeling of safety:
%[P013]: ``Now, technology is really useful in some ways because you can highlight things like hotspots of crime and when things are happening and predictive analysis and all that sort of stuff. But actually, sometimes it's preventing cops from doing their job, which is a social job. It's actually being out in the community, speaking to folks. ... A lot of cops are quite comfortable coming into an office and sitting behind a computer all day. But actually, I don't want them there. I want them in the community.''
%Developers of the tool also did not want it to be used for police patrol allocation decisions.
\subsection{Reaffirm: ``How can there be a bias if in that triangle it's all the same race?'' [P003]}
\label{sec:reaffirm}
\subsubsection{Following the data.} Participants in public sector positions denied the presence of bias or used evidence of algorithmic bias to reaffirm their own authority and the legitimacy of existing policing practices. This is exemplified well by one officer [P003], who candidly explained why there is no over-policing problem: ``Where the crime is happening, the concentration is 99\% Black or Brown and the victims are 99\% Black or Brown, so that's our baseline. ... % And let's work from there like, you know, in policing we've been knocked around that we're over-policing certain communities, right? But can we explain that the reason we're in this geography is because the numbers, the data is telling us that this is where the homicides and shootings happen? Do you want us to go someplace else? The residents in those neighborhoods, that's the silent majority. They want us there, they need us, and it's not, there's no bias to us being there.
We're there because the data says that's where we need to be.'' He added that ``% So, the AI is the same way, right? That the AI is only functioning off of data. So the data is telling the model 'this is what you have to work with.' Give me an answer and the answer is you need to be here. So the AI --
it's not that the AI is biased. The AI is just working with the data, and the data says that this geography, this demographic is, you know, these individuals.'' %[P003] %\todo{shorten quote}
This officer provided a problem diagnosis of the algorithm's behavior that essentialized the policed community -- ``these individuals'' -- as inherently exhibiting more criminal behavior, thereby using the pattern to justify or take as given the status quo. Disparities in crime, in other words, are the factor that drives disparities in policing and in turn the algorithmic non-bias, and the officer lamented that ``in policing we've been knocked around'' for simply responding to what ``the data is telling us.''
% and referring to current crime prediction approaches as ``setting people up for failure.'' [C017].\todo{I don't understand how this relates to reaffirming power, can you explain a bit? --> I added this more to highlight the fact that also the affected communities feel the existence of an intrinsic suspicion towards them, the one coming from the police/the system. It was just a little additional point so feel free to delete!}
\subsubsection{New and improved technology.}
\label{sec:422}
Another response to algorithmic bias that reaffirms police power is the call to invest in more technology. Some public sector participants saw the algorithm as an opportunity to do their work better, and others as the only way forward. For instance, speaking about the motivation behind the development of the crime prediction tool, one developer explained that ``all of them [previous predictive policing algorithms] were doing it wrong'' and the need to ``show what is the right way to do it.'' %``I thought that so many different AI's predictive policing algorithms have been suggested and deployed. Commercialized I kind of felt all of them were doing it wrong. So if you feel like that about anything, I think what you should do as well, you think everyone is wrong? Then show what is the right way to do it'' [R002].
One police officer [P003] %Similarly, a police officer suggested that the problem is not with the AI, but with the police not being research-driven enough [P003]. He cited
called for the need for policing to be informed by the crime prediction algorithm and to ``try a different approach on the AI.''
\subsubsection{More policing.} As we discuss in \S\ref{sec:allocation}, one response to enforcement bias is the call to increase police presence in areas with higher crime incidence, which implicitly affirms the legitimacy of the current policing system. For instance, one police officer reported being interested in ``using this technology to break down the cycles that have been perpetuated for years'' [P003]. However, it became clear that what they meant by this was sending more police to Black and Brown neighbourhoods in order to deter crime, rather than to change the structure of the current system or ``break down'' any cycles.
An interviewee from a community association [C011] voiced concern about affected communities similarly endorsing approaches that do not change the existing system and go against their own interests, such as ``more policing,'' and attributed this to a lack of awareness of alternative options. Referring to the race for a major election in a large US city, they said, ``... older Black residents who are concerned about public safety, and many of them, their response is `we need more police'. And the reason is because it's the only tool they know, right?'' %He explains how this can bring communities that are most impacted by crime prediction technologies to also endorse their increased use against their own interests.
%I moved this here from the AI Hype spectrum section, so that we can use it as a point of discussion there but we introduce it before
\subsection{Reject: ``I don't think it should be used in any way, shape or form.'' [C005]}
%[C005, reentry community organizer]
%What the problems with data and AI are depends on how stakeholders understood the source of the problem, but s
\subsubsection{Garbage in, garbage out.}
\label{sec:garbage}
Stakeholders across all groups shared some inherent reservations about AI in the policing context, particularly concerns about its underlying data. In particular, the theme of algorithmic bias being a ``result of the system'' [C001] -- that is, reflecting structural bias and systemic racism -- recurred frequently. %One reentry organization member referred to bias as a ``result of the system.''
One system-impacted participant [C005] argued that the system should not be used at all because it does not account for systemic and individual bias: %``it refuses to account for systemic racism and individual bias
%``The system for me I think is not a good system. I don't think it should be used in any way, shape, or form because it refuses to account for systemic racism and individual bias, and because it refuses to do those things, it cannot be neutral.
``It just cannot be neutral, and therefore it should not be used because people that look like me are the ones who are going to be impacted.''
Another concern voiced by researchers working with system-impacted individuals was a general distrust of low-quality or manipulated police data. ``Their [police] incentives are wrapped up in what is reported. You know what crimes are in the news and what people are saying. It's like they're kind of incentivized to obscure data.'' [R008] Researcher participants described that documentation of police misconduct is likely to be reduced to the most minor violation; for instance, an illegal search will only be reported if it also includes sexual assault. %They also expressed that policed individuals are more likely to be incarcerated through the creation of justifications or excuses to fit the police mistrust of these populations. They brought up the example of a 16-year old boy in Chicago who was incarcerated for a murder he did not commit.
%
Another researcher working with system-impacted individuals [R009] pointed to several cases where police officers intentionally misreported the race of individuals in order to conceal bias and discrimination in their practices. He mentioned that in Louisiana, for example, several law enforcement agencies targeted Latino individuals but identified them as white on tickets in order to conceal discrimination \cite{webster2021if}, and similar incidents have also taken place in Connecticut \cite{nierenberg_over_2023}, Los Angeles and San Francisco \cite{barba_are_2023}.
Finally, several participants from commmunity organizations had reservations about the inherently interpretative component of AI, which they said biases its results. A member from a reentry community organization [C001] argued that AI is an %... what differs to me about data science, artificial intelligence [rather] then a gun which is a mechanical, this
%really isn't mechanical any anymore, right? You really are using it as an
``interpretive instrument'' rather than a mechanical one. ``In an ideal sense, let's say the [AI] system had no politics, right? But the people that will use the results and interpret the information do, and you know, as they, you know, they always say, you know, people can make statistics tell any story they want it to tell.''
\subsubsection{Unneeded, unconvincing, and resource-intensive.}
\label{sec:unneeded}
Other respondents were opposed to the predictive policing tool for more pragmatic reasons. For instance, several participants felt that the algorithm's enforcement bias did not show anything that was not already known. One academic researcher [R006] commented that developing an algorithm is ``a pretty inefficient way'' to inform police reallocation, arguing that ``%I don't think you need an algorithm here to look at-- yeah,
you wouldn't need that, to develop an algorithm to reach these same conclusions.'' An interviewee working for a reentry association [C017] expressed similar skepticism about the need for a tool that predicts crime. ``He [system-impacted individual] is always going to be discriminated against in one form or fashion, maybe an employment house. He's always going to be there, and that's a predictor of recidivism or returning to be a criminal. Well, yeah. We don't need an algorithm to say that.''
The academic researcher [R006] also said that the algorithm was ``not particularly convincing'' for other stakeholders: ``I barely understand exactly what they're doing and how they reach their conclusion, ... And like, I have a PhD and I do this for a living. So I think, you know, like trying to convince like, other stakeholders is very difficult.'' Other participants, including another academic researcher and several community organizations, also reported not understanding the paper or how the tool worked, though in the case of one police officer [P003], that was a positive feature because it indicated that the tool's developers were ``smarter minds.''
The same police officer [P003] also commented on the resource-intensiveness, both in terms of computational power and departmental funding, of acquiring and using a predictive policing tool in practice.
%
%[P003]: ``So [R002] and I talked about computational power. He's able to do it because he has the resources. ... Yes, so, resource allocation. The financial investment, you know, will, will the police department--
``Will we have a budget for this? Or do I need to go out and fundraise for it and convince foundations that, you know, this is the way we need to use the limited resources that we have?'' % You know, are we able to implement it? And, and that's, that's going to be-- so resource allocation is a big hurdle that I have to overcome.'' \todo{Technical sweetness, data is a good problem to solve, Kashmir Hill}
Understanding the underlying motivation for developing the tool helps contextualize the above criticisms. One of the tool's developers [R002] explained that the team had been working on a general algorithm and were looking for a real-world test case to demonstrate how the algorithm could be instantiated. ``Why crime? No particular reason. We were looking for a problem that would be an impactful thing.'' An advantage of working on crime, they add, is that there is ``lots of data on it.'' The research team had access to compute power that a typical community organization or police force does not, and it is clear that the team did not have the goal of convincing stakeholders or revealing surprising patterns. Rather, the research team saw predicting crime as an ``interesting'' challenge, seeking the `technical sweetness' \cite{hill2023your} that engineers feel when they encounter the solution to a challenging technical puzzle. %of solving a challenging technical problem Scientists and engineers use this phrase when a puzzle’s solution presents itself, when all the pieces fit beautifully and functionally together, when success at the particular endeavor presents itself in a neat package..
\subsection{Reveal: ``Data that helps us understand how the police see our communities.'' [R009]}
\label{sec:reveal}
%\subsubsection{Power and sousveillance}
Enforcement bias was one type of evidence community organizations used to engage in sousveillance, or `watching from below', in which the traditional framing of panoptic state surveillance is inverted, allowing citizens to observe powerful state actors \cite{mann2003sousveillance, newell_introduction_2020}. Researchers and community organizers described how they co-opted surveillance methods to watch the watchers. Alternate data sources, or counterdata \cite{dignazio_counting_2024}, reveal issues with policing, remediate problems with how police and victim data are collected and reported, and allow alternate narratives to emerge. % \S\ref{sec:reframe}.
%Burke (2020: 75) has recently framed “digital sousveillance” as a research method, involving “the co-optation of digital data and the use of computational methods and techniques (e.g., network analysis) to resituate technologies of control and surveillance of individuals to instead observe the observer—as a method for studying surveillance” (Burke 2020: 77).
One member of a community safety initiative [C101] lamented that %I mean because historically if you look at crime, you also could look at public health data. You could look at educational data, right? Anytime you map communities that are struggling everything overlaps ... but I think the difference is
``a lot of times what criminologists will tend to do is they'll just look at the data. And I think what's missing is that there's a story behind it.'' To provide this missing context, one human rights researcher [R009] described working with grassroots groups that do on-the-ground documentation through cop watching and court watching, gathering counterdata on police officers, judges, and prosecutors. They argue that ``you need to record different things than what the courts are'' in order to include context normally absent in court data. The same researcher also described using data about human rights violations to understand patterns of violence, and even to make statistical estimates of events that are not represented in the data. %``like like people who are survivors of domestic violence or stuff like this anyways, but like a lot of that is the type of context, that is not present in court data at all. And so the only way that you can find out about like who is this population around which you've built your organization is is like in a systematic way is through court watching and that kind of data collection because you need to record different things than what the courts are.''f
%we work with like groups who are doing kind of on the ground documentation, for example through cop watching and cop watching is like a canonical example because it looks a very particular way. But like I think one of the more common ways that this happens in the US is through court watching and this is like a fairly common tactic for a lot of like community and tactical troops in local jurisdictions in different places, and I like, it is worth seeing the data that is collected by groups who do court watching.”
%“The organization Survived and Punished, one of their strategies for organizing is also based on court watching and so Survived and Punished as an organization kind of dedicated to kind of advocating for the rights of women who have been criminalized and or incarcerated for actions that they took in context of survival so often,
%``like like people who are survivors of domestic violence or stuff like this anyways, but like a lot of that is the type of context, that is not present in court data at all. And so the only way that you can find out about like who is this population around which you've built your organization is is like in a systematic way is through court watching and that kind of data collection because you need to record different things than what the courts are.''
As we discuss in \S\ref{sec:garbage}, several researchers working with system-impacted individuals also reported the existence of low-quality, unreliable or manipulated police data. % [e.g., R008, R009].
By collecting data from the Chicago Police Department, the city of Chicago, and the Civilian Office of Police Accountability through Freedom of Information requests, one research group [R008] described how they published an alternative dataset about police misconduct in Chicago. They also recounted collaborating with system-impacted individuals and victims of police violence to build a basic classifier to predict police misconduct using complaint data generated by community members. This alternate algorithm revealed a culture of abuse within policing: ``%very basic classifier using training data generated by community members to better parse through some of this narrative text that is associated with these, you know, top level complaints. And again we were able to bring in the context of police misconduct. Right? So
instead of something like operation and personnel violation, you know, we're able to see clusters of, you know, families who are like ... `this is how police treated me'. And what was really fascinating about that project is that we were able to, you know, move away from this idea of like, individual officers, like, back up to what is maybe the culture of policing that exists, what are the kind of tactics that, that people are reporting that connect them across space and time?''
%\subsubsection{Resistance and counterdata}
%Others have suggested ``Understanding public data not as data that helps us understand our own communities, but rather data that helps us understand how the police see our communities'' [R009].
% \todo{insert sousveillance summary from Monahan + Newell}
\subsection{Reframe and Repair: ``Deconstructing the status quo and recreating a new status quo.'' [C004]}
\label{sec:reframe}
Countering mainstream understandings of crime among respondents from the public sector and some academic researchers, many participants from community organizations presented alternate data epistemologies \cite{ricaurte2019data} focused on abolitionist understandings of the ``root of the problem'' [C016]. %do not fundamentally change the power structure of the criminal legal system and do not center abolitionist values of liberation and community.
In this vein, participants responded to enforcement bias by calling for the need to recognize structural racism, reframe the narrative toward an understanding of crime as structural rather than interpersonal, and repair and build healthy communities.
%Stakeholders from community organizations emphasized the importance of using data for what one community worker [C005] referred to as ``narrative change work.'' Participants called for reframing the narrative toward an understanding of crime as structural rather than interpersonal, generating alternate data epistemologies \cite{ricaurte2019data} focused on the ``root of the problem'' [C016] and alternatives to the traditional policing model.
\subsubsection{Recognition of systemic racism.}
\label{sec:recognition}
A common thread in the interview data was the ``tale of two cities,'' [C001; P015] or Chicago's history of segregation and systematic disinvestment in its Black and Brown communities. Participants emphasized the significance of Chicago's history of segregation, citing it as a root cause of not only crime but also numerous other social problems, such as disparities in life expectancy -- ``I want to say 20-30 year difference in life span between those that live in the poor neighbour community'' [C001] -- and in wealth --``You can go to the different parts of the north side and you can just see the wealth, right? ... and then you can go to the south side or the southwest side and see completely different-- you, you can almost think that you're in a different city'' [P015].
Some system-impacted individuals and researchers working with them also remarked on the intrinsic suspicion they experience toward policed communities, accompanied by a tendency to trust or believe those doing the policing. One of them [C017] argued that criminal legal policies are ``setting people up for failure,'' using Nixon's 1970s policy on potential delinquents as an example of how `criminals' are created out of assumptions rather than facts.
Multiple participants stressed that the first step to resolving the problem presented by enforcement bias is having a clear understanding of what the problem is. One member of a community safety initiative in Chicago [C011] noted that ``before anybody can heal, there needs to be a reckoning, there needs to be some kind of identification that, that the systems that were created actually have caused harm.'' Two more participants [C004; C005] echoed this sentiment almost verbatim, arguing that ``if we're not going to address a system of white supremacy, then we're not going to get to any real hard solutions.'' [C004] %``We have systems rooted in racism and white supremacy and patriarchy, and we have to acknowledge those things. We will not be able to come up with solutions if we don't start with that.'' [C005] % We understand these things exist now.''
\subsubsection{Narrative change.}
Beyond reckoning, participants emphasized reframing the narrative around system-impacted individuals, police, and community safety. As one community worker [C005] put it, ``The delusion that somehow we can scare people out of crimes of poverty is ridiculous. We have been so indoctrinated by current systems to accept the claims, regardless of the the reality, right?'' They argued that ``One of the biggest things that we do is narrative change work'' in order to push back on dominant narratives around crime and policing. % ... because we have to really push back on the norms that have been around and sold to society for more than my lifetime.'' %Another community organizer [C001] suggested, ``... instead of thinking about how do we reduce violence and crime, what if we pose the question of how this information helps us build healthy communities?'' %Yes, right. So now you've opened up a whole different line of data utilization.''
In particular, participants call for a reorientation toward a structural understanding of the ``root cause'' [C014] of crime, focusing on treating the cause rather than the symptom. Two participants criticized interventions that focus on the interpersonal level, with one participant [C011] comparing the current criminal legal system to a hospital that only contains an emergency room and another [R104] lamenting that interventions like community violence and prevention programs ``stop at the interpersonal level, but they don't also account for the structural systemic impediments at play.'' % that are creating the conditions that a community violence intervention program is trying, it's basically responding to, right?''
\subsubsection{``Building healthy communities rather than reducing crime'' [C001]}
%Another community organizer [C001] suggested, ``... instead of thinking about how do we reduce violence and crime, what if we pose the question of how this information helps us build healthy communities?'' %Yes, right. So now you've opened up a whole different line of data utilization.''
Grounded in an alternate understanding of crime as structural, participants responded to enforcement bias by advocating for non-reformist reforms such as alternatives to police and rethinking how to use data to promote community safety.
One community organizer [C001] asked, ``instead of thinking about how we reduce violence and crime, what if we pose the question of how this information helps us build healthy communities?'' We saw in \S\ref{sec:reveal} how researchers and activists gather counterdata on police and make up for limitations in data. This strategy centers the needs of affected communities, which community organization participants agreed got little benefit from predicting crime. Another researcher [R008] explained how AI could instead be used to reduce crime in the sense of supporting survivors of violence, especially police violence: ``When we pivot our approach to, you know, engaging with survivors, then we're better understanding the implications of policing and the ways that we can use artificial intelligence to support survivors of violence and reduce crime, like, period.''
%\subsubsection{Structural problems require structural solutions.}
Beyond the use of data, several participants also advocated for non-reformist reforms. Drawing on an abolitionist critique of police, one researcher [R008] working with system-impacted individuals challenged the assumptions underlying policing:
``Policing should not be about `we gotta get the bad guys', right? It should be about `how do we keep people safe'.'' Taking this view a step further, one community organizer [C101] questioned the belief that ``we need more [police],'' adding that ``if we bring more tools to the community, there's-- the solutions are here. We just haven't given you all the equipment to do so.'' Notably, the police and academic position on police promoting safety was starkly different from the perspective of some interviewees from community organizations for which police presence was an ``enforcement presence, not a-- not a deterrent, but an enforcement'' [C001].%\todo{integrate this point here}
%``in a lot of our communities we have this contradiction that when they think about public safety, their only response is we need more [police], and actually I think there's a strategy that says no, if we bring more tools to the community, there's-- the solutions are here. We just haven't given you all the equipment to do so.'' [101]
%``But in my view, a lot of the Community violence and subvention programs stop at the interpersonal level, but they don't also account for the structural systemic impediments at play that are creating the conditions that a community violence intervention program is trying, it's basically responding to, right? They just, they stop at the individual level, but they don't address the systematic issues or the systemic barriers.'' [C014]
Even when there was agreement about the need to strengthen communities, however, respondents differed in their views on which kinds of interventions to put in place to do so. Some reentry organizations advocated for giving the same rights to system-impacted individuals or clearing their past criminal record after some time [C017], while others fought for more police and state accountability [R008]. One individual working as a contractor for the public sector [P016] recognized the structural nature of the problem and yet focused on ``changing the individual'' as a solution, namely, moving individuals out of a high-crime environment or, alternatively, strategically reclaiming high-crime spaces by organizing community events (e.g., sports competitions, community barbecues, etc.). Another member of a community organization [C011] focused on ``changing the conditions'' -- namely, focusing interventions on the socioeconomic determinants of crime such as wealth and health. In short, even within the non-reformist reform branch of responses to enforcement bias, there were a diversity of perspectives on the appropriate kinds of interventions.
\section{Discussion}
Our results shed light on the socially contested nature of algorithmic bias within the realm of predictive policing, illustrating that bias is not understood as a singular challenge but rather evidence of a multifaceted package of problems with a corresponding broad range of possible interventions. We grouped the interview findings according to whether stakeholders from different backgrounds used evidence of algorithmic bias to reform, reaffirm, reject, reveal, reframe or repair, showing the ability of enforcement bias to be used both to support and challenge the status quo.
Whether and how algorithmic bias was seen as useful for advancing each of these goals varied according to each stakeholder's broader understanding of the source of police enforcement disparities -- structural inequality versus interpersonal bias -- and the set of interventions aligned with each respective problem diagnosis. Stakeholders from different professional roles and positionalities tended to articulate problem diagnoses of the bias aligned with their own relationship to the criminal legal system, though we saw a diversity of attitudes and responses within each subgroup and points of convergence between individuals with different positionalities and roles. For instance, we saw substantial overlap among interviewees from all backgrounds regarding the importance of community safety and potential downsides to the use of AI and data in the context of policing, such as AI's possibility to diminish the role of nuanced human discretion. On the other hand, we saw disagreement between groups about whether police presence promotes community safety, and there were differences within groups relating to whether predictive policing tools would improve policing and whether incorporating affected stakeholders in the development of predictive policing could improve the tools' viability. %, specifically about its potential to `disrupt' the criminal legal system in a beneficial way.
%by looking "through" power; namely, by taking into consideration the power imbalances and what role power plays . It helps us to disclose their politics in order to help affected communities to challenge the seeming veil of "objectivity" behind politically-motivated solutions.
%I tried to incorporate this but I just wasn't sure what you meant :( power is so vague. I think we need to revisit this in the future with more theory, maybe Foucault?
%Contested categories were policed communities; community safety; whether communities want more police; whether police make communities safer.
Many of the inter-group differences we saw reflect tensions between the assumptions underlying interventions grounded in reform versus abolition -- reformist reform that operates within the logic of the current system and non-reformist reform that aims for longer-term systemic change, respectively. Along these lines, we saw other contested concepts emerge, such as the notion of community safety and `overpoliced' communities, with the connotation and valence of `overpoliced' changing depending on the socioeconomic status of the overpoliced community. For example, in their paper, developers of the Chicago crime prediction algorithm suggest that police are significantly more present in wealthier neighborhoods because those areas have more resources. %, rather than in poor neighborhoods.
`Overpoliced' here has a positive connotation -- the privilege and safety of well-resourced communities. At the same time, interviewees from community organizations and from the public sector tended to refer to overpoliced communities as poorer communities with higher rates of crime.
While inter-group differences reflect historical tensions between reform and abolition, one additional significant binary emerged that correlated with participants' attitudes toward the criminal legal system -- `AI hype' rhetoric. We examine this finding below and we end by discussing the relationship between participant positionality and research on algorithmic bias in predictive policing.
\subsection{AI hype rhetoric}
With stakeholders from all groups, except the academic researchers most closely familiar with the tool's development, attitudes toward predictive policing tended to fall into one of two AI hype narratives: optimism about AI's potential to `disrupt' the criminal legal system in a beneficial way, and inherent opposition to AI on the basis that it is racist and fundamentally flawed. Techno-optimism mapped onto the extent to which a stakeholder understood the problems the algorithm was capturing as primarily interpersonal rather than structural, which correlated with positionalities in higher positions of power. However, some endorsers of structural inequality narratives still saw value in certain applications of AI, such as grassroots classifiers to counter narratives put forward by police or to reveal patterns of police violence.
The clearest example of disruption rhetoric came from officer P003, who praised the %who repeatedly emphasized his optimism about the
``smarter minds'' developing the predictive policing tool (\S\ref{sec:412}) and argued that more technology -- or trying ``a different approach on the AI'' [P003] -- could help tackle the bias in the algorithm (\S\ref{sec:422}). They also %Initially, he appears to be endorsing AI as a solution to a structural problem: ``%I'm hoping that I can educate this group that this AI is going to help break those cycles of economic hardships of crime, hardships of domestic violence hardships.
%We're using technology to break down the cycles that have been being perpetuated for years.'' However, it soon becomes clear that what he means by this is to send more police to crime-afflicted Black and Brown neighborhoods in order to deter crime, rather than put forth any changes to the structure of the current system or ``break down'' any cycles. The officer also --> As this quote had not been introduced before, I moved it to the Reveal section in 4.4 to provide a counterpoint to the community organizations' view.
remarked repeatedly that AI could help the police force's ``marketing ability to brand us as open as transparent and as accountable,'' what they called the two salient ``buzzwords'' of the ``2020, BLM, post-Floyd'' era (\S\ref{sec:transparency}).
AI may yet become a successful marketing tool for police forces, similarly to past cases in which police acquisitions of technologies like body-worn cameras strategically co-opted community calls for accountability to justify larger police budgets \cite{umansky_failed_2023, karakatsanis2019usual}.\footnote{Police are trained to manipulate and withhold body-worn camera footage to their own benefit \cite{umansky_failed_2023}.} However, the underlying claim that predictive policing promotes transparency and accountability in policing in based in fiction. Sociologist Sarah Brayne provides a compelling argument that predictive policing simply displaces discretion to other, less publicly visible stages of the policing process, thereby reducing both transparency and accountability \cite{brayne_predict_2020}.
On the other end of the AI hype spectrum, we saw community organizers describe AI as inherently racist. As we discussed in \S\ref{sec:garbage}, %This is exemplified by C005,
a system-impacted leader of a reentry organization [C005] argued that predictive algorithms should not be used at all in the context of the criminal legal system because ``it's just racist and it needs to be stopped.'' Those in positions of power, they argued, are incentivized to use algorithmic systems because ``it keeps the status quo in place,'' which is what gives those individuals power. An interviewee running a reentry association [C017] echoed that these systems ``set people up for failure'' (\S\ref{sec:recognition}) and that people with a criminal record are ``always going to be discriminated against in one form or fashion'' (\S\ref{sec:unneeded}). % Anything that threatens the systems that empower them. %``If I, if I start to challenge the systems that empower me, right, that, that and that, that give me this power and this privilege. I start to challenge those systems, then I may not have that privilege.
Like P003, C005's and C017's perspective paints a broad and uniform brush across all uses of AI in the context of the criminal legal system. Both groups at the extreme ends of the spectrum had direct experience with the police but from starkly different positions of power; from being a police officer to having been previously incarcerated.
Not all the participants we interviewed fell into this AI hype dichotomy, but the polarity was widespread enough that we believe it is important to explicitly take into account in future work that aims to use algorithmic bias evidence to challenge the status quo. In practice, existing empirical research of evidence-based reforms shows that, despite their hype and high price tag, they often have minimal effects that wash out over time, suggesting that these reforms are indeed harmful but not in the ways that critics may expect \cite{pruss2023ghosting, stevenson_assessing_2018}. As we explain in the following section, participatory research is crucial to develop responses to algorithmic bias that are attentive to the interests and knowledge of communities impacted by incarceration; at the same time, such work would be made more robust by helping participants resist one-dimensional narratives about the harms of technology in the criminal legal system.
\subsection{``Bias as defined by who, right? So ultimately, who's the judge?'' [C001]}
The range of problem diagnoses and uses of algorithmic bias might be interpreted as a `relativistic' stance toward the issue, where the understanding of algorithmic bias is contingent upon an individual's subjective world view. Rather than dismissing these differences, we believe that they provide a valuable lens to understand the conditions under which algorithmic bias becomes a device to reinforce or challenge power dynamics. Drawing on standpoint theory \cite{harding_rethinking_1992}, we argue that unraveling the relationship between bias and positionality serves as a critical tool to evaluate shortcomings and mistaken assumptions in potential responses to algorithmic bias coming from actors in relatively high positions of power, such as police officers and even academic researchers such as ourselves.
Standpoint theory posits that individuals' social positions shape their access to knowledge, influencing how they interpret and respond to phenomena such as bias. Social locations `at the margins' that have histories of repression and resistance have better awareness of injustices and can more easily question the `naturalness' of existing power structures \cite{hooks2000feminist, d2023data, noauthor_feminist_nodate}. In the context of algorithmic bias and predictive policing, this means that those in positions of power, such as public sector actors and academics, may inadvertently reaffirm fundamentally unjust systems due to their distance from the lived experience of incarceration.
Our findings emphasize the importance of centering the needs and experiential understanding of system-impacted communities through community-based participatory research in designing and evaluating responses to algorithmic bias. %By doing so, responses to enforcement bias can be designed and evaluated with a more critical, nuanced and contextually informed approach. %That is, our results emphasize for the importance of centering the needs and experiential understanding of system-impacted communities through community-based participatory research in designing and evaluating responses to enforcement bias.
%
A critical consideration of positionality and bias may promote interventions that can serve affected communities' stated goals of community investment and healing, contributing to the development of interventions that address the root cause of algorithmic bias in predictive policing. Our results suggest that such a community-centered response to bias would in fact leave no room for the existence of crime prediction tools like predictive policing, which take the current system as a given, and would prompt a reimagination of ethical data science grounded in abolitionist philosophy and a refusal of oppressive structures \cite{benjamin_informed_2016, barabas_refusal_2022}. %Asking whether and how technology may help serve these liberatory goals is a promising question for community-based researchers to consider.\todo{cite nikki stevens' work if they have something} %We believe there is much-needed future work to be done on community-driven technology grounded. %It calls for a recalibration of responses to algorithmic bias, urging a shift towards community-based participatory research that actively involves and prioritizes the voices of those directly impacted by biased algorithms. Such an approach not only acknowledges the limitations and biases inherent in responses from positions of power but also contributes to the development of more equitable and just interventions that address the root causes of algorithmic bias in predictive policing.
%\subsection{Affirming the status quo, questioning the status quo}
%How can algorithmic bias evidence serve affected communities' stated (abolitionist) goals of community investment? Conversely, how can it reaffirm power and the status quo?
%How can this come in support of affected communities?
%Linking the findings to larger societal issues, the study highlights the parallel tension between abolitionist and reformist responses to mass incarceration. Algorithmic bias becomes a focal point in the broader debate on justice, equity, and systemic interventions. Understanding its implications is crucial for addressing not only bias in predictive policing but also the fundamental issues within the criminal justice system.
% 2. Strategic Uses of Evidence: Stakeholders' Perspectives
% 1. Algorithmic Bias in Predictive Policing: A Multifaceted Challenge
% The study sheds light on the intricate nature of algorithmic bias within the realm of predictive policing, emphasizing that it is not a singular challenge but rather a complex, multifaceted issue. Stakeholders from diverse backgrounds articulate distinct problem diagnoses of the bias, reflecting the nuanced perspectives within the criminal justice landscape.
% 2. Strategic Uses of Evidence: Stakeholders' Perspectives
% The evidence of algorithmic bias becomes a strategic tool for stakeholders with varying political ends. Drawing inspiration from Catherine D'Ignazio's taxonomy of data usage, stakeholders employ different strategies such as reforming police patrol allocation, reaffirming existing power structures, rejecting algorithm-based policing interventions, revealing counterdata, reframing crime as a structural problem, and promoting repair and healing in communities.
% 3. Implicit Assumptions and Conflicting Values
% Implicit assumptions and conflicting values underpin the varied uses of algorithmic bias as evidence. The divergence in interpretations reflects the enduring tensions in the criminal justice reform landscape, pitting abolitionist values against reformist responses to mass incarceration. These conflicting values underscore the complex nature of algorithmic bias and its embeddedness in broader societal debates.
% 4. Algorithmic Bias as a Tool for Empowerment or Affirmation of Power
% The study illuminates the dual role of algorithmic bias as both a challenge to and an affirmation of existing power structures. While it has the potential to challenge oppressive systems, it also runs the risk of reinforcing them. Recognizing and characterizing each stakeholder's position in relation to power is crucial in understanding how algorithmic bias can either empower or further marginalize affected communities.
% 5. Reconciling Diagnoses and Formulating Pragmatic Solutions
% Reconciling diverse diagnoses of algorithmic bias poses a significant challenge. The study calls attention to the need for pragmatic solutions that support affected communities, acknowledging the role of algorithmic bias as a boundary object. It serves as a lens for critique while also acting as a pragmatic device to address the disparities it mirrors.
% 6. Connect to Larger Societal Issues
% Linking the findings to larger societal issues, the study highlights the parallel tension between abolitionist and reformist responses to mass incarceration. Algorithmic bias becomes a focal point in the broader debate on justice, equity, and systemic interventions. Understanding its implications is crucial for addressing not only bias in predictive policing but also the fundamental issues within the criminal justice system.
% 7. Implications for Future Research and Action
% In conclusion, the study's findings have implications for future research and action. It calls for a deeper exploration of algorithmic fairness that considers sociotechnical contexts, as well as a reevaluation of current reform measures. Empowering communities and promoting just practices require a nuanced understanding of algorithmic bias and its potential to either challenge or affirm existing power structures.
%Initially, we had considered how evidence of police enforcement bias could be used to expose or denounce policing in specific communities, defend or produce the social category of ``policed communities,'' and spark a reevaluation of the very purpose of crime prediction algorithms. At the same time, we had anticipated that this evidence could help affirm the absence of insufficient patrols in a neighborhood and justify their increased presence, help sediment or justify an emerging social category of the ``criminal'' individual or the ``high risk'' neighborhood, and help reaffirm the re-establishment of the status quo as the ``only solution''.
%How does the disruption rhetoric around AI manifest on both sides? AI developers are so smart, they'll solve all the problems, vs. it's so complicated even people with PhDs don't understand how it works.
%Quintessential AI ``disruption'' rhetoric: ``I'm hoping that I can educate this group that this AI is going to help break those cycles of economic hardships of crime, hardships of domestic violence hardships. We're using technology to break down the cycles that have been being perpetuated for years.'' [P003, police officer]
%Distrust of human discretion has been diagnosed as the motivation for institutions embracing automated decision-making \cite{porter_trust_1995}
%``As social philosopher André Gorz proposes, we must distinguish between “reformist reforms” and “non-reformist reforms” \cite{gorz1968strategy}. “A reformist reform,” explains Gorz, “is one which subordinates its objectives to the criteria of rationality and practicability of a given system and policy.” A non-reformist reform, on the other hand, “is conceived not in terms of what is possible within the framework of a given system and administration, but in view of what should be made possible in terms of human needs and demands.” Because of the distinct ways that these two types of reforms are conceived, pursuing one or the other can lead to widely divergent social and political outcomes.'' Ben Green \cite{green2019good}
%However, sociologist Sarah Brayne argues that predictive policing simply displaces discretion to other, less publicly visible stages of the policing process, thereby reducing both transparency and accountability \cite{brayne_predict_2020}.
%How does the disruption rhetoric around AI manifest on both sides? AI developers are so smart, they'll solve all the problems, vs. it's so complicated even people with PhDs don't understand how it works.
%Quintessential AI ``disruption'' rhetoric: ``I'm hoping that I can educate this group that this AI is going to help break those cycles of economic hardships of crime, hardships of domestic violence hardships. We're using technology to break down the cycles that have been being perpetuated for years.'' [P003, police officer]
%Also techno-optimism about the `smarter minds' of AI: ``I mean the technologies of technology in policing. We're not the brightest bulbs; we, we have, we have our strong suits, but math and you know, technology is not our strong suit. We're, we're, and we're analogue guys trying to function in a digital world. There are, there are smarter minds out there than us and there are smarter minds analyzing all the different mathematical models in the back end of the AI, so they'll be able to inform us and they'll keep all of the AI models accountable.''
%Overall, we learn that bias can a) help us denounce or expose mistreatment, b) expose existing disparities and c) question the use of crime prediction tools in the first place. These are ways in which bias can disclose injustices or help marginalized communities. However, as we have seen, it simultaneously can affirm power. In this paper, we learnt that... (both political and pragmatic aspect. Framework to unite them).
%Significance
%Emphasizing the importance of leveraging algorithmic bias as evidence for anti-discriminatory policy-making.
%Lessons Learned/Open Questions:
%``Affected communities as active creators'' (Harry): How do you make affected communities a significant part of the creation? Beyond ``participatory research''.
%Future Research
%Suggesting potential avenues for further research in the field.
%Community Involvement
%Reiterating the commitment to engage with affected communities throughout the research process.
%Study Limitations
%Limits on who we were able to recruit, limits of qualitative study and researcher subjectivity...
%Generalizability
%Acknowledgment that the study's results may not be fully generalizable.
%Incomplete Solutions
%Recognition that the study may not provide comprehensive solutions but rather insights.
%Resource Limitations and time constraints
%Mention of limited funding for the study that's why several interviews are online
%Reply to the research questions
%Framework
\section{Conclusion}
%Summary
%Recap of the research project's objectives, methodology, and findings.
%\todo[inline]{Reword this -- right now it's identical to the abstract --> I reworded it! we can always add one last sentence out of the discussion section once we are done with it :)}
In this article, we presented a critical, qualitative study on evidence of enforcement bias within the Chicago crime prediction algorithm \cite{rotaru_event-level_2022}. Our research, based on interviews with various stakeholders, including community organizations, academic researchers, and public sector figures in the Chicago area, reveals that stakeholders offer diverse diagnoses of this bias, strategically leveraging these diagnoses to promote interventions that align with their respective positionality.
These interventions encompass a range of approaches, such as using data to reform police practices that contributed to biased data collection; advocating for increased transparency and accountability in the development and utilization of predictive policing tools; discontinuing the use of such tools altogether; involving a diverse set of stakeholders in the creation and evaluation of these tools; and proposing a shift away from investing in data-driven policing reforms toward more abolitionist goals, such as community investment. We examined the implicit assumptions and the scope underlying these diverse responses to algorithmic bias, demonstrating that they are grounded in reformist and abolitionist thinking. We concluded by reflecting on the divergent AI hype views reported by participants and its relationship to lived experience with the criminal legal system. We advocate for centering the interests and experiential knowledge of system-impacted communities to ensure that evidence of algorithmic bias can be mobilized as a device to challenge the status quo. %This diversity mirrors longstanding tensions in the criminal justice reform landscape, reflecting the contrast between the values of liberation and healing often emphasized by communities affected by incarceration, and the values of surveillance and deterrence, frequently embedded in data-driven reform measures.
\section{Ethical Considerations}
%"We strongly encourage authors to include in their papers 1) a description of the ethical concerns the authors mitigated while conducting the work (as part of an ethical considerations statement), 2) reflections on how their background and experiences inform or shape the work (as part of a researcher positionality statement), 3) reflection on the adverse, unintended impact the work might have once published (as part of an adverse impact statement)."
\subsection{Researcher Positionality}
As white women academic researchers from Europe and the US, we acknowledge that our personal and professional backgrounds shaped how we selected research questions and analyzed data, as well as the manner in which we interacted with study participants from backgrounds similar and dissimilar to ours. Both authors have a privileged academic affiliation, race, and lack of personal contact with the US criminal legal system, which undoubtedly shaped the information that grassroots community organizations and system-impacted interviewees were comfortable sharing with us. In fact, the theme of exploitation of marginalized communities by academic researchers came up explicitly in one interview, which we discuss in \S\ref{sec:ethical_concerns}. On the other hand, our privileged position as white academic researchers from well-recognized institutions helped build rapport with other academic researchers in the study, who had similarly received training at elite academic institutions. %Being female researchers conducting elite interviews likely placed us in inverted situations of power, \cite{puwar_reflections_1997, abels_interviewing_2009}, expressed in repeated interruptions, lengthy monologues and not being taken seriously \cite{boucher_power_2017}. At the same time, however, we recognize that this may have resulted in heightened trustworthiness being placed on us \cite{deem_researching_1994, ortbals_fieldwork_2009, marshall_elites_1984}.
\subsection{Ethical Concerns}
\label{sec:ethical_concerns}
%An IRB approval was obtained, along with a consent form for each interview participant. %Brainstorming possible ethical issues we should mention: anonymity of subjects; power differentials/exploiting the community; forcing individuals to relive traumatic experiences for the sake of research; posing questions that are biased;
%Privacy and anonymity of subjects
\textbf{Privacy and anonymity.} In line with ACM Ethics Guidelines and the requirements of our IRB approval, participants' identities were protected by pseudonymizing interview transcripts. The document matching pseudonym to identifiable information was stored separately from the research data. %, and participants were identified in quotes only if they explicitly consented to this. [Do we identify any participants? If not, remove] \todo{For smaller groups, like the tool's developers, it is harder to protect identity.} %Transcripts were analyzed in NVivo, a fully GDPR-compliant software used for qualitative and mixed-methods research. Signed consent forms and the document matching pseudonyms to identifiable information were stored in a hard drive within two separate folders, separately from the research folders. All electronic data stored within the hard drive will be permanently deleted after three years, apart from the consent forms. Contact details will only be kept for research purposes. They will also be stored in a separate folder in the lead researcher's hard drive which is password protected and encrypted.
\noindent \textbf{Engaging non-extractively with affected communities.} The primary ethical challenge we encountered had to do with how to involve stakeholders impacted by the criminal legal system in our research in a non-extractive and mutually beneficial manner. Due to our own resource constraints, we did not compensate interviewees for their time, which meant there was no direct benefit for study participation, and our positionality meant that affected stakeholders had little incentive or reason to place their trust in us. This became particularly salient at the end of an interview with a system-impacted participant, who works with returning citizens in Chicago. In response to a question about referring us to other study participants, they responded: ``I'm going to be completely transparent with you. Our community is exploited, often by academia. And, and so I don't feel comfortable because of that constant exploitation to put other people in that position. ... %I mean, I can, I can put myself in that position, but I can't put anybody else in that position because they
[Academic researchers] take our expertise, they subject us to trauma by asking us to tell our stories, and then they ... %don't, they don't, they don't even accept us in institutions of higher learning like that, and then, and then they don't, and then they, they take our information, and they raise off of it and do all of these wonderful things and
never come back and connect it to us.''
We acknowledge the respondent's concern and plan to share all research outputs with research participants, crediting them when desired, with the understanding that we cannot give back as much as we have received from their participation. The experience of this power differential demonstrated to us the importance of building longer-lasting community relationships outside the scope of academic projects prior to engaging in research, and increasing communities' ownership and participation of the research, with the gold standard in mind of participatory action research.
%As hooks also reminds us, the margins are not always a comfortable place to operate from—and efforts from computational practitioners to engage at the margins might be met with skepticism and distrust. Benjamin (2016b) explains that such distrust often stems from a long history of asymmetrical social relationships, which are reproduced through an uneven distribution of material resources and symbolic power.\todo{Fix}
\subsection{Adverse Impacts}
By focusing on the Chicago crime prediction algorithm, our research may inadvertently publicize the availability of the tool or promote more intensive policing. Beyond recognizing this limitation, we wish to make explicit that we do not support the development of this (or similar tools) in any form. We hope that, by shining light on the tool and the different concerns it raises, we can show how and why solutions addressed at tackling bias often do not end up benefiting the affected communities.
%At the same time, we recognize that we are putting forward research that may impede short term gains in safety for the sake of longer term structural improvements. The need for safety, however, is also often the result of structural problems and, given the systemic history of racism in the criminal legal system in the US, we think it paramount to focus on and improve the latter.
\begin{acks}
We thank the respondents from community organizations in Chicago and all other interview participants for their time, trust and invaluable knowledge. The research was supported by the first author's affiliation with the Oxford Internet Institute’s Programme on AI, Government and Policy, funded by the Dieter Schwarz Stiftung gGmbH, and the second author's fellowship at the Berkman Klein Center for Internet \& Society at Harvard University. We thank the Embedded EthiCS group at Harvard University for their feedback on this work.
\end{acks}
\bibliographystyle{ACM-Reference-Format}
\bibliography{Law+ML}
\appendix
\newpage
\section{Interview Guide}
\label{sec:interview_guide}
\noindent \begin{enumerate}
\item \textbf{Context and Positionality}
% Probes:
\begin{itemize}
\item Can you introduce yourself?
\item Can you tell us more about your work?
\item How are you related to the predictive policing project? How have you heard of it, if at all?
\item How are you involved with other initiatives around community and public safety in Chicago?
\end{itemize}
\item \textbf{Bias definition and utility}
% Probes:
\begin{itemize}
\item What is algorithmic bias to you in general?
\item What is the problem with it? Why do you think it exists?
\item What is enforcement bias to you specifically?
\item What is the problem with it? Why do you think it exists?
\item What is its specific relevance to your work? And the context of Chicago?
\item What is your role in tackling bias and problems faced in doing so (if applicable) at work?
\end{itemize}
\item \textbf{Perceived feasibility and desirability of using evidence of bias to tackle social disparities}
% Probes:
\begin{itemize}
\item How should bias be addressed?
\item What would an ideal solution look like to you (especially with relevance to your role/work)?
\item What are the barriers to resolving bias? How can it be realistically tackled, in your opinion?
\end{itemize}
\item \textbf{Snowball sampling}
\begin{itemize}
\item Do you know other people it might be relevant for us to interview?
\end{itemize}
\end{enumerate}
%\noindent (Start by introducing yourself and asking how their day is going.)
%\newpage
\section{Code Table}
\label{sec:code_table}
%\begin{table}[h]
\small
%\centering
% \singlespacing
\begin{supertabular}{ l }
\textbf{Abolition} \\
\hline
Alternatives to current system/police\\
Building healthy communities\\
Calls for changing the narrative/status quo\\
The need to acknowledge harms of current system\\
Importance of history and education\\
Importance of hope and belief in change\vspace{.2em}\\
\textbf{Co-option and exploitation} \\ \hline
Academic exploitation\\
Co-opting evidence-based label\\
Commercial interest in predicting crime\\
Disruption rhetoric \\
Open source as a marketing strategy\\
Responsibilities of developers\vspace{.2em}\\
\textbf{Downsides of AI} \\
\hline
AI is racist/not objective\\
Inherent reservations about AI\\
Garbage in garbage out/distrust of data\\
Human discretion is important\\
Nuance not captured by algorithm\\
Technology isn't going to fix policing\vspace{.2em}\\
\textbf{Lived experience and positionality} \\ \hline
Affected people are best suited to solve problems\\
Barriers to entering research\\
Impacted groups lack resources\\
Different stakeholders care about different forms of bias\\
People not affected don't have the same understanding\\
Who generated the data/designed the tool matters\\
Importance of trust\vspace{.2em}\\
\textbf{Reformism}\\ \hline
Interpersonal bias rather than structural\\
Resource allocation\\
Safety is the priority\\
Police promotes safety\vspace{.2em}\\
\textbf{Structural bias}\\
\hline
Importance of addressing root cause of crime\\
Racial profiling and segregation\\
Structural racism requires a structural solution\\
The system is biased\vspace{.2em}\\
\textbf{Upsides of AI}\\
\hline
Algorithm can be more objective/humans are biased\\
Algorithms increase efficiency \\
Open source\\
Using data to understand authority figures (sousveillance)\\
Information has multiple uses\vspace{.2em}\\
\hline
\end{supertabular}
%\end{table}
\end{document}
\endinput
%%
%% End of file `sample-manuscript.tex'.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{81}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Abebe et~al\mbox{.}(2020)]%
{abebe_roles_2020}
\bibfield{author}{\bibinfo{person}{Rediet Abebe}, \bibinfo{person}{Solon Barocas}, \bibinfo{person}{Jon Kleinberg}, \bibinfo{person}{Karen Levy}, \bibinfo{person}{Manish Raghavan}, {and} \bibinfo{person}{David~G. Robinson}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Roles for {Computing} in {Social} {Change}}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020 {Conference} on {Fairness}, {Accountability}, and {Transparency}}}. \bibinfo{pages}{252--260}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.1145/3351095.3372871}
\showDOI{\tempurl}
\newblock
\shownote{arXiv:1912.04883 [cs]}.
\bibitem[Balayn and G{\"u}rses(2021)]%
{balayn2021beyond}
\bibfield{author}{\bibinfo{person}{Agathe Balayn} {and} \bibinfo{person}{Seda G{\"u}rses}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Beyond Debiasing: Regulating AI and its inequalities}.
\newblock \bibinfo{journal}{\emph{EDRi Report. https://edri. org/wp-content/uploads/2021/09/EDRi\_Beyond-Debiasing-Report\_Online. pdf}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Barabas(2019)]%
{barabas_beyond_2019}
\bibfield{author}{\bibinfo{person}{Chelsea Barabas}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Beyond {Bias}: {Re}-{Imagining} the {Terms} of ‘{Ethical} {AI}’ in {Criminal} {Law}}}.
\newblock \bibinfo{type}{{SSRN} {Scholarly} {Paper}} ID 3377921. \bibinfo{institution}{Social Science Research Network}, \bibinfo{address}{Rochester, NY}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.2139/ssrn.3377921}
\showDOI{\tempurl}
\bibitem[Barabas(2022)]%
{barabas_refusal_2022}
\bibfield{author}{\bibinfo{person}{Chelsea Barabas}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Refusal in {Data} {Ethics}: {Re}-{Imagining} the {Code} {Beneath} the {Code} of {Computation} in the {Carceral} {State}}.
\newblock \bibinfo{journal}{\emph{Engaging Science, Technology, and Society}} \bibinfo{volume}{8}, \bibinfo{number}{2} (\bibinfo{date}{Sept.} \bibinfo{year}{2022}), \bibinfo{pages}{35--57}.
\newblock
\showISSN{2413-8053}
\urldef\tempurl%
\url{https://doi.org/10.17351/ests2022.1233}
\showDOI{\tempurl}
\newblock
\shownote{Number: 2}.
\bibitem[Barabas et~al\mbox{.}(2018)]%
{barabas_interventions_2018}
\bibfield{author}{\bibinfo{person}{Chelsea Barabas}, \bibinfo{person}{Madars Virza}, \bibinfo{person}{Karthik Dinakar}, \bibinfo{person}{Joichi Ito}, {and} \bibinfo{person}{Jonathan Zittrain}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Interventions over {Predictions}: {Reframing} the {Ethical} {Debate} for {Actuarial} {Risk} {Assessment}}. In \bibinfo{booktitle}{\emph{Conference on {Fairness}, {Accountability} and {Transparency}}}. \bibinfo{pages}{62--76}.
\newblock
\urldef\tempurl%
\url{http://proceedings.mlr.press/v81/barabas18a.html}
\showURL{%
\tempurl}
\bibitem[Barba and Baustin(2023)]%
{barba_are_2023}
\bibfield{author}{\bibinfo{person}{Michael Barba} {and} \bibinfo{person}{Noah Baustin}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Are {San} {Francisco} {Police} {Officers} {Misreporting} the {Races} of {People} {They} {Stop}?}
\newblock \bibinfo{journal}{\emph{The San Francisco Standard}} (\bibinfo{date}{Sept.} \bibinfo{year}{2023}).
\newblock
\urldef\tempurl%
\url{https://sfstandard.com/2023/09/13/san-francisco-police-officer-misrepresented-race-bias-investigation/}
\showURL{%
\tempurl}
\bibitem[Benjamin(2016)]%
{benjamin_informed_2016}
\bibfield{author}{\bibinfo{person}{Ruha Benjamin}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Informed {Refusal}: {Toward} a {Justice}-based {Bioethics}}.
\newblock \bibinfo{journal}{\emph{Science, Technology, \& Human Values}} \bibinfo{volume}{41}, \bibinfo{number}{6} (\bibinfo{date}{Nov.} \bibinfo{year}{2016}), \bibinfo{pages}{967--990}.
\newblock
\showISSN{0162-2439}
\urldef\tempurl%
\url{https://doi.org/10.1177/0162243916656059}
\showDOI{\tempurl}
\newblock
\shownote{Publisher: SAGE Publications Inc}.
\bibitem[Benjamin(2019)]%
{benjamin_race_2019}
\bibfield{author}{\bibinfo{person}{Ruha Benjamin}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Race {After} {Technology}: {Abolitionist} {Tools} for the {New} {Jim} {Code}}}.
\newblock \bibinfo{publisher}{John Wiley \& Sons}.
\newblock
\showISBNx{978-1-5095-2643-7}
\bibitem[Bowell({[n.\,d.]})]%
{noauthor_feminist_nodate}
\bibfield{author}{\bibinfo{person}{T Bowell}.} \bibinfo{year}{[n.\,d.]}\natexlab{}.
\newblock \bibinfo{title}{Feminist {Standpoint} {Theory} {\textbar} {Internet} {Encyclopedia} of {Philosophy}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://iep.utm.edu/fem-stan/}
\showURL{%
\tempurl}
\bibitem[Brayne(2020)]%
{brayne_predict_2020}
\bibfield{author}{\bibinfo{person}{Sarah Brayne}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Predict and {Surveil}: {Data}, {Discretion}, and the {Future} of {Policing}}}.
\newblock \bibinfo{publisher}{Oxford University Press}.
\newblock
\showISBNx{978-0-19-068409-9}
\bibitem[Broussard et~al\mbox{.}(2019)]%
{broussard_artificial_2019}
\bibfield{author}{\bibinfo{person}{Meredith Broussard}, \bibinfo{person}{Nicholas Diakopoulos}, \bibinfo{person}{Andrea~L. Guzman}, \bibinfo{person}{Rediet Abebe}, \bibinfo{person}{Michel Dupagne}, {and} \bibinfo{person}{Ching-Hua Chuan}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Artificial {Intelligence} and {Journalism}}.
\newblock \bibinfo{journal}{\emph{Journalism \& Mass Communication Quarterly}} \bibinfo{volume}{96}, \bibinfo{number}{3} (\bibinfo{date}{Sept.} \bibinfo{year}{2019}), \bibinfo{pages}{673--695}.
\newblock
\showISSN{1077-6990}
\urldef\tempurl%
\url{https://doi.org/10.1177/1077699019859901}
\showDOI{\tempurl}
\newblock
\shownote{Publisher: SAGE Publications Inc}.
\bibitem[Bruno et~al\mbox{.}(2014)]%
{bruno_statactivism_2014}
\bibfield{author}{\bibinfo{person}{Isabelle Bruno}, \bibinfo{person}{Emmanuel Didier}, {and} \bibinfo{person}{Tommaso Vitale}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \bibinfo{title}{Statactivism: {Forms} of {Action} between {Disclosure} and {Affirmation}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://papers.ssrn.com/abstract=2466882}
\showURL{%
\tempurl}
\bibitem[Desrosi{\`e}res(1998)]%
{desrosieres1998politics}
\bibfield{author}{\bibinfo{person}{Alain Desrosi{\`e}res}.} \bibinfo{year}{1998}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{The politics of large numbers: A history of statistical reasoning}}.
\newblock \bibinfo{publisher}{Harvard University Press}.
\newblock
\bibitem[D'Ignazio(2024)]%
{dignazio_counting_2024}
\bibfield{author}{\bibinfo{person}{Catherine D'Ignazio}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Counting {Feminicide}: {Data} {Feminism} in {Action}}}.
\newblock \bibinfo{publisher}{The MIT Press}, \bibinfo{address}{Cambridge, Massachusetts}.
\newblock
\bibitem[D'ignazio and Klein(2023)]%
{d2023data}
\bibfield{author}{\bibinfo{person}{Catherine D'ignazio} {and} \bibinfo{person}{Lauren~F Klein}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Data feminism}}.
\newblock \bibinfo{publisher}{MIT press}.
\newblock
\bibitem[Edwards and Holland(2013)]%
{edwards_what_2013}
\bibfield{author}{\bibinfo{person}{Rosalind Edwards} {and} \bibinfo{person}{Janet Holland}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{What is {Qualitative} {Interviewing}?}}
\newblock \bibinfo{publisher}{Bloomsbury Academic}.
\newblock
\showISBNx{978-1-84966-802-6 978-1-84966-801-9}
\urldef\tempurl%
\url{https://doi.org/10.5040/9781472545244}
\showDOI{\tempurl}
\newblock
\shownote{Accepted: 2022-10-14T14:53:13Z}.
\bibitem[Ensign et~al\mbox{.}(2018)]%
{ensign_runaway_2018}
\bibfield{author}{\bibinfo{person}{Danielle Ensign}, \bibinfo{person}{Sorelle Friedler}, \bibinfo{person}{Scott Neville}, \bibinfo{person}{Carlos Scheidegger}, {and} \bibinfo{person}{Suresh Venkatasubramanian}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Runaway {Feedback} {Loops} in {Predictive} {Policing}}.
\newblock \bibinfo{journal}{\emph{Proceedings of Machine Learning Research}} \bibinfo{volume}{81} (\bibinfo{date}{Jan.} \bibinfo{year}{2018}).
\newblock
\urldef\tempurl%
\url{https://scholarship.haverford.edu/compsci_facpubs/113}
\showURL{%
\tempurl}
\bibitem[Espeland and Vannebo(2007)]%
{espeland_accountability_2007}
\bibfield{author}{\bibinfo{person}{Wendy~Nelson Espeland} {and} \bibinfo{person}{Berit~Irene Vannebo}.} \bibinfo{year}{2007}\natexlab{}.
\newblock \showarticletitle{Accountability, quantification, and law}.
\newblock In \bibinfo{booktitle}{\emph{Annual {Review} of {Law} and {Social} {Science}}}, \bibfield{editor}{\bibinfo{person}{John Hagan}, \bibinfo{person}{Kim~Lane Scheppele}, {and} \bibinfo{person}{Tom Tyler}} (Eds.). \bibinfo{pages}{21--43}.
\newblock
\showISBNx{978-0-8243-4103-9}
\urldef\tempurl%
\url{https://doi.org/10.1146/annurev.lawsocsci.2.081805.105908}
\showDOI{\tempurl}
\bibitem[Eubanks(2018)]%
{eubanks_automating_2018}
\bibfield{author}{\bibinfo{person}{Virginia Eubanks}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Automating {Inequality}: {How} {High}-{Tech} {Tools} {Profile}, {Police}, and {Punish} the {Poor}}}.
\newblock \bibinfo{publisher}{St. Martin's Publishing Group}.
\newblock
\showISBNx{978-1-4668-8596-7}
\bibitem[Fahse et~al\mbox{.}(2021)]%
{fahse_managing_2021}
\bibfield{author}{\bibinfo{person}{Tobias Fahse}, \bibinfo{person}{Viktoria Huber}, {and} \bibinfo{person}{Benjamin van Giffen}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Managing {Bias} in {Machine} {Learning} {Projects}}. In \bibinfo{booktitle}{\emph{Innovation {Through} {Information} {Systems}}} \emph{(\bibinfo{series}{Lecture {Notes} in {Information} {Systems} and {Organisation}})}, \bibfield{editor}{\bibinfo{person}{Frederik Ahlemann}, \bibinfo{person}{Reinhard Schütte}, {and} \bibinfo{person}{Stefan Stieglitz}} (Eds.). \bibinfo{publisher}{Springer International Publishing}, \bibinfo{address}{Cham}, \bibinfo{pages}{94--109}.
\newblock
\showISBNx{978-3-030-86797-3}
\urldef\tempurl%
\url{https://doi.org/10.1007/978-3-030-86797-3_7}
\showDOI{\tempurl}
\bibitem[Fazelpour and Danks(2021)]%
{fazelpour_algorithmic_2021}
\bibfield{author}{\bibinfo{person}{Sina Fazelpour} {and} \bibinfo{person}{David Danks}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Algorithmic bias: {Senses}, sources, solutions}.
\newblock \bibinfo{journal}{\emph{Philosophy Compass}} \bibinfo{volume}{16}, \bibinfo{number}{8} (\bibinfo{year}{2021}), \bibinfo{pages}{e12760}.
\newblock
\showISSN{1747-9991}
\urldef\tempurl%
\url{https://doi.org/10.1111/phc3.12760}
\showDOI{\tempurl}
\newblock
\shownote{\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/phc3.12760}.
\bibitem[Feathers(2021)]%
{feathers_gunshot-detecting_2021}
\bibfield{author}{\bibinfo{person}{Todd Feathers}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \bibinfo{title}{Gunshot-{Detecting} {Tech} {Is} {Summoning} {Armed} {Police} to {Black} {Neighborhoods}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.vice.com/en/article/88nd3z/gunshot-detecting-tech-is-summoning-armed-police-to-black-neighborhoods}
\showURL{%
\tempurl}
\bibitem[Ferguson and Witzburg(2021)]%
{ferguson2021chicago}
\bibfield{author}{\bibinfo{person}{J Ferguson} {and} \bibinfo{person}{Deborah Witzburg}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{The Chicago Police Department’s Use of Shotspotter Technology}.
\newblock \bibinfo{journal}{\emph{Chicago, IL: City of Chicago Office of Inspector General}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Franchi et~al\mbox{.}(2023)]%
{franchi_detecting_2023}
\bibfield{author}{\bibinfo{person}{Matt Franchi}, \bibinfo{person}{J.D. Zamfirescu-Pereira}, \bibinfo{person}{Wendy Ju}, {and} \bibinfo{person}{Emma Pierson}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Detecting disparities in police deployments using dashcam data}. In \bibinfo{booktitle}{\emph{Proceedings of the 2023 {ACM} {Conference} on {Fairness}, {Accountability}, and {Transparency}}} \emph{(\bibinfo{series}{{FAccT} '23})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{534--544}.
\newblock
\showISBNx{9798400701924}
\urldef\tempurl%
\url{https://doi.org/10.1145/3593013.3594020}
\showDOI{\tempurl}
\bibitem[Frey(2018)]%
{frey_sage_2018}
\bibfield{author}{\bibinfo{person}{Bruce~B. Frey}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{The {SAGE} {Encyclopedia} of {Educational} {Research}, {Measurement}, and {Evaluation}}}.
\newblock \bibinfo{publisher}{SAGE Publications, Inc.}
\newblock
\showISBNx{978-1-5063-2613-9}
\urldef\tempurl%
\url{https://doi.org/10.4135/9781506326139}
\showDOI{\tempurl}
\bibitem[Gebru(2019)]%
{gebru_oxford_2019}
\bibfield{author}{\bibinfo{person}{Timnit Gebru}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \bibinfo{title}{Oxford {Handbook} on {AI} {Ethics} {Book} {Chapter} on {Race} and {Gender}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://doi.org/10.48550/arXiv.1908.06165}
\showDOI{\tempurl}
\newblock
\shownote{arXiv:1908.06165 [cs]}.
\bibitem[Gorz(1968)]%
{gorz1968strategy}
\bibfield{author}{\bibinfo{person}{Andre Gorz}.} \bibinfo{year}{1968}\natexlab{}.
\newblock \showarticletitle{Strategy for labor. A radical proposal}.
\newblock \bibinfo{journal}{\emph{Science and Society}} \bibinfo{volume}{32}, \bibinfo{number}{4} (\bibinfo{year}{1968}).
\newblock
\bibitem[Green(2019)]%
{green2019good}
\bibfield{author}{\bibinfo{person}{Ben Green}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Good” isn’t good enough}. In \bibinfo{booktitle}{\emph{Proceedings of the AI for Social Good workshop at NeurIPS}}, Vol.~\bibinfo{volume}{17}.
\newblock
\bibitem[Green(2021)]%
{green_data_2021}
\bibfield{author}{\bibinfo{person}{Ben Green}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Data {Science} as {Political} {Action}: {Grounding} {Data} {Science} in a {Politics} of {Justice}}.
\newblock \bibinfo{journal}{\emph{Journal of Social Computing}} \bibinfo{volume}{2}, \bibinfo{number}{3} (\bibinfo{date}{Sept.} \bibinfo{year}{2021}), \bibinfo{pages}{249--265}.
\newblock
\showISSN{2688-5255}
\urldef\tempurl%
\url{https://doi.org/10.23919/JSC.2021.0029}
\showDOI{\tempurl}
\newblock
\shownote{arXiv:1811.03435 [cs]}.
\bibitem[Green(2022)]%
{green_escaping_2022}
\bibfield{author}{\bibinfo{person}{Ben Green}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Escaping the {Impossibility} of {Fairness}: {From} {Formal} to {Substantive} {Algorithmic} {Fairness}}.
\newblock \bibinfo{journal}{\emph{Philosophy \& Technology}} \bibinfo{volume}{35}, \bibinfo{number}{4} (\bibinfo{date}{Oct.} \bibinfo{year}{2022}), \bibinfo{pages}{90}.
\newblock
\showISSN{2210-5441}
\urldef\tempurl%
\url{https://doi.org/10.1007/s13347-022-00584-6}
\showDOI{\tempurl}
\bibitem[Green and Viljoen(2020)]%
{green_algorithmic_2020}
\bibfield{author}{\bibinfo{person}{Ben Green} {and} \bibinfo{person}{Salomé Viljoen}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Algorithmic realism: expanding the boundaries of algorithmic thought}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020 {Conference} on {Fairness}, {Accountability}, and {Transparency}}} \emph{(\bibinfo{series}{{FAT}* '20})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{Barcelona, Spain}, \bibinfo{pages}{19--31}.
\newblock
\showISBNx{978-1-4503-6936-7}
\urldef\tempurl%
\url{https://doi.org/10.1145/3351095.3372840}
\showDOI{\tempurl}
\bibitem[Grill et~al\mbox{.}(2023)]%
{grill_bias_2023}
\bibfield{author}{\bibinfo{person}{Gabriel Grill}, \bibinfo{person}{Fabian Fischer}, {and} \bibinfo{person}{Florian Cech}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Bias as {Boundary} {Object}: {Unpacking} {The} {Politics} {Of} {An} {Austerity} {Algorithm} {Using} {Bias} {Frameworks}}. In \bibinfo{booktitle}{\emph{Proceedings of the 2023 {ACM} {Conference} on {Fairness}, {Accountability}, and {Transparency}}} \emph{(\bibinfo{series}{{FAccT} '23})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{1838--1849}.
\newblock
\showISBNx{9798400701924}
\urldef\tempurl%
\url{https://doi.org/10.1145/3593013.3594120}
\showDOI{\tempurl}
\bibitem[Harcourt(2008)]%
{harcourt_against_2008}
\bibfield{author}{\bibinfo{person}{Bernard~E. Harcourt}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Against {Prediction}: {Profiling}, {Policing}, and {Punishing} in an {Actuarial} {Age}}}.
\newblock \bibinfo{publisher}{University of Chicago Press}.
\newblock
\showISBNx{978-0-226-31599-7}
\bibitem[Harding(1992)]%
{harding_rethinking_1992}
\bibfield{author}{\bibinfo{person}{Sandra Harding}.} \bibinfo{year}{1992}\natexlab{}.
\newblock \showarticletitle{Rethinking {Standpoint} {Epistemology}: {What} is ``{Strong} {Objectivity}?"}.
\newblock \bibinfo{journal}{\emph{The Centennial Review}} \bibinfo{volume}{36}, \bibinfo{number}{3} (\bibinfo{year}{1992}), \bibinfo{pages}{437--470}.
\newblock
\showISSN{0162-0177}
\urldef\tempurl%
\url{https://www.jstor.org/stable/23739232}
\showURL{%
\tempurl}
\bibitem[Hill(2023)]%
{hill2023your}
\bibfield{author}{\bibinfo{person}{Kashmir Hill}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Your Face Belongs to Us: The Secretive Startup Dismantling Your Privacy}}.
\newblock \bibinfo{publisher}{Simon and Schuster}.
\newblock
\bibitem[Hoffmann(2019)]%
{hoffmann2019fairness}
\bibfield{author}{\bibinfo{person}{Anna~Lauren Hoffmann}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Where fairness fails: data, algorithms, and the limits of antidiscrimination discourse}.
\newblock \bibinfo{journal}{\emph{Information, Communication \& Society}} \bibinfo{volume}{22}, \bibinfo{number}{7} (\bibinfo{year}{2019}), \bibinfo{pages}{900--915}.
\newblock
\bibitem[Hooks(2000)]%
{hooks2000feminist}
\bibfield{author}{\bibinfo{person}{Bell Hooks}.} \bibinfo{year}{2000}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Feminist theory: From margin to center}}.
\newblock \bibinfo{publisher}{Pluto Press}.
\newblock
\bibitem[Karakatsanis(2019)]%
{karakatsanis2019usual}
\bibfield{author}{\bibinfo{person}{Alec Karakatsanis}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Usual cruelty: The complicity of lawyers in the criminal injustice system}}.
\newblock \bibinfo{publisher}{The New Press}.
\newblock
\bibitem[Karppinen and Moe(2011)]%
{karppinen_what_2011}
\bibfield{author}{\bibinfo{person}{Kari Karppinen} {and} \bibinfo{person}{Hallvard Moe}.} \bibinfo{year}{2011}\natexlab{}.
\newblock \showarticletitle{What we talk about when we talk about document analysis}.
\newblock In \bibinfo{booktitle}{\emph{Trends in {Communication} {Policy} {Research}}}. \bibinfo{publisher}{Intellect}, \bibinfo{address}{Bristol}.
\newblock
\showISBNx{978-1-84150-467-4}
\bibitem[Kennedy et~al\mbox{.}(2018)]%
{kennedy_risk-based_2018}
\bibfield{author}{\bibinfo{person}{Leslie~W. Kennedy}, \bibinfo{person}{Joel~M. Caplan}, {and} \bibinfo{person}{Eric~L. Piza}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Risk-{Based} {Policing}: {Evidence}-{Based} {Crime} {Prevention} with {Big} {Data} and {Spatial} {Analytics}}}.
\newblock \bibinfo{publisher}{Univ of California Press}.
\newblock
\showISBNx{978-0-520-29563-6}
\bibitem[Kunichoff and Sier(2017)]%
{kunichoff_contradictions_2017}
\bibfield{author}{\bibinfo{person}{Yana Kunichoff} {and} \bibinfo{person}{Patrick Sier}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \bibinfo{title}{The {Contradictions} of {Chicago} {Police}'s {Secretive} {List}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.chicagomag.com/city-life/August-2017/Chicago-Police-Strategic-Subject-List/}
\showURL{%
\tempurl}
\bibitem[Lipton(1977)]%
{lipton_why_1977}
\bibfield{author}{\bibinfo{person}{Michael Lipton}.} \bibinfo{year}{1977}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Why poor people stay poor : a study of urban bias in world development}}.
\newblock \bibinfo{publisher}{Temple Smith ; Australian National University Press}.
\newblock
\urldef\tempurl%
\url{https://openresearch-repository.anu.edu.au/handle/1885/114902}
\showURL{%
\tempurl}
\newblock
\shownote{Accepted: 2017-04-18T05:59:25Z Last Modified: 2022-11-17}.
\bibitem[Lum and Isaac(2016)]%
{lum_predict_2016}
\bibfield{author}{\bibinfo{person}{Kristian Lum} {and} \bibinfo{person}{William Isaac}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{To predict and serve?}
\newblock \bibinfo{journal}{\emph{Significance}} \bibinfo{volume}{13}, \bibinfo{number}{5} (\bibinfo{year}{2016}), \bibinfo{pages}{14--19}.
\newblock
\showISSN{1740-9713}
\urldef\tempurl%
\url{https://doi.org/10.1111/j.1740-9713.2016.00960.x}
\showDOI{\tempurl}
\newblock
\shownote{\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1740-9713.2016.00960.x}.
\bibitem[Mann et~al\mbox{.}(2003)]%
{mann2003sousveillance}
\bibfield{author}{\bibinfo{person}{Steve Mann}, \bibinfo{person}{Jason Nolan}, {and} \bibinfo{person}{Barry Wellman}.} \bibinfo{year}{2003}\natexlab{}.
\newblock \showarticletitle{Sousveillance: Inventing and using wearable computing devices for data collection in surveillance environments.}
\newblock \bibinfo{journal}{\emph{Surveillance \& society}} \bibinfo{volume}{1}, \bibinfo{number}{3} (\bibinfo{year}{2003}), \bibinfo{pages}{331--355}.
\newblock
\bibitem[Mayson({[n.\,d.]})]%
{mayson_bias_nodate}
\bibfield{author}{\bibinfo{person}{Sandra~G. Mayson}.} \bibinfo{year}{[n.\,d.]}\natexlab{}.
\newblock \bibinfo{title}{Bias {In}, {Bias} {Out}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.yalelawjournal.org/article/bias-in-bias-out}
\showURL{%
\tempurl}
\bibitem[McLeod(2018)]%
{mcleod2018envisioning}
\bibfield{author}{\bibinfo{person}{Allegra~M McLeod}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Envisioning abolition democracy}.
\newblock \bibinfo{journal}{\emph{Harv. L. Rev.}} \bibinfo{volume}{132} (\bibinfo{year}{2018}), \bibinfo{pages}{1613}.
\newblock
\bibitem[Meyer and Graybill(2016)]%
{meyer_suburban_2016}
\bibfield{author}{\bibinfo{person}{William~B. Meyer} {and} \bibinfo{person}{Jessica~K. Graybill}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{The suburban bias of {American} society?}
\newblock \bibinfo{journal}{\emph{Urban Geography}} \bibinfo{volume}{37}, \bibinfo{number}{6} (\bibinfo{date}{Aug.} \bibinfo{year}{2016}), \bibinfo{pages}{863--882}.
\newblock
\showISSN{0272-3638}
\urldef\tempurl%
\url{https://doi.org/10.1080/02723638.2015.1118990}
\showDOI{\tempurl}
\newblock
\shownote{Publisher: Routledge \_eprint: https://doi.org/10.1080/02723638.2015.1118990}.
\bibitem[Miles et~al\mbox{.}(2014)]%
{miles_qualitative_2014}
\bibfield{author}{\bibinfo{person}{Matthew~B. Miles}, \bibinfo{person}{A.~Michael Huberman}, {and} \bibinfo{person}{Johnny Saldana}.} \bibinfo{year}{2014}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Qualitative {Data} {Analysis}}}.
\newblock \bibinfo{publisher}{SAGE}.
\newblock
\showISBNx{978-1-4522-5787-7}
\bibitem[Mitchell et~al\mbox{.}(2019)]%
{mitchell_model_2019}
\bibfield{author}{\bibinfo{person}{Margaret Mitchell}, \bibinfo{person}{Simone Wu}, \bibinfo{person}{Andrew Zaldivar}, \bibinfo{person}{Parker Barnes}, \bibinfo{person}{Lucy Vasserman}, \bibinfo{person}{Ben Hutchinson}, \bibinfo{person}{Elena Spitzer}, \bibinfo{person}{Inioluwa~Deborah Raji}, {and} \bibinfo{person}{Timnit Gebru}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Model {Cards} for {Model} {Reporting}}. In \bibinfo{booktitle}{\emph{Proceedings of the {Conference} on {Fairness}, {Accountability}, and {Transparency}}} \emph{(\bibinfo{series}{{FAT}* '19})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{220--229}.
\newblock
\showISBNx{978-1-4503-6125-5}
\urldef\tempurl%
\url{https://doi.org/10.1145/3287560.3287596}
\showDOI{\tempurl}
\bibitem[Mohtasham(2024)]%
{mohtasham_chicago_2024}
\bibfield{author}{\bibinfo{person}{Diba Mohtasham}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Chicago will drop controversial {ShotSpotter} gunfire detection system}.
\newblock \bibinfo{journal}{\emph{NPR}} (\bibinfo{date}{Feb.} \bibinfo{year}{2024}).
\newblock
\urldef\tempurl%
\url{https://www.npr.org/2024/02/15/1231394334/shotspotter-gunfire-detection-chicago-mayor-dropping}
\showURL{%
\tempurl}
\bibitem[Newell(2020)]%
{newell_introduction_2020}
\bibfield{author}{\bibinfo{person}{Bryce Newell}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Introduction: {The} {State} of {Sousveillance}}.
\newblock \bibinfo{journal}{\emph{Surveillance \& Society}} \bibinfo{volume}{18}, \bibinfo{number}{2} (\bibinfo{date}{June} \bibinfo{year}{2020}), \bibinfo{pages}{257--261}.
\newblock
\showISSN{1477-7487}
\urldef\tempurl%
\url{https://doi.org/10.24908/ss.v18i2.14013}
\showDOI{\tempurl}
\bibitem[Nierenberg(2023)]%
{nierenberg_over_2023}
\bibfield{author}{\bibinfo{person}{Amelia Nierenberg}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Over 100 {Connecticut} {State} {Troopers} {Accused} of {Faking} {Traffic} {Stops}}.
\newblock \bibinfo{journal}{\emph{The New York Times}} (\bibinfo{date}{Sept.} \bibinfo{year}{2023}).
\newblock
\showISSN{0362-4331}
\urldef\tempurl%
\url{https://www.nytimes.com/2023/09/04/nyregion/connecticut-false-tickets.html}
\showURL{%
\tempurl}
\bibitem[Parker et~al\mbox{.}(2019)]%
{parker_snowball_2019}
\bibfield{author}{\bibinfo{person}{C. Parker}, \bibinfo{person}{S. Scott}, {and} \bibinfo{person}{A. Geddes}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Snowball {Sampling}}.
\newblock \bibinfo{journal}{\emph{SAGE Research Methods Foundations}} (\bibinfo{date}{Sept.} \bibinfo{year}{2019}).
\newblock
\urldef\tempurl%
\url{http://methods.sagepub.com/foundations/snowball-sampling}
\showURL{%
\tempurl}
\newblock
\shownote{Publisher: SAGE}.
\bibitem[Passi and Barocas(2019)]%
{passi2019problem}
\bibfield{author}{\bibinfo{person}{Samir Passi} {and} \bibinfo{person}{Solon Barocas}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Problem formulation and fairness}. In \bibinfo{booktitle}{\emph{Proceedings of the conference on fairness, accountability, and transparency}}. \bibinfo{pages}{39--48}.
\newblock
\bibitem[Pearsall(2010)]%
{pearsall2010predictive}
\bibfield{author}{\bibinfo{person}{Beth Pearsall}.} \bibinfo{year}{2010}\natexlab{}.
\newblock \showarticletitle{Predictive policing: The future of law enforcement}.
\newblock \bibinfo{journal}{\emph{National Institute of Justice Journal}} \bibinfo{volume}{266}, \bibinfo{number}{1} (\bibinfo{year}{2010}), \bibinfo{pages}{16--19}.
\newblock
\bibitem[Porter(1995)]%
{porter_trust_1995}
\bibfield{author}{\bibinfo{person}{Theodore~M. Porter}.} \bibinfo{year}{1995}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Trust in numbers: the pursuit of objectivity in science and public life}}.
\newblock \bibinfo{publisher}{Princeton University Press}, \bibinfo{address}{Princeton, N.J}.
\newblock
\showISBNx{978-0-691-03776-9}
\bibitem[Pruss(2023)]%
{pruss2023ghosting}
\bibfield{author}{\bibinfo{person}{Dasha Pruss}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Ghosting the Machine: Judicial Resistance to a Recidivism Risk Assessment Instrument}. In \bibinfo{booktitle}{\emph{Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency}}. \bibinfo{pages}{312--323}.
\newblock
\bibitem[Ricaurte(2019)]%
{ricaurte2019data}
\bibfield{author}{\bibinfo{person}{Paola Ricaurte}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Data epistemologies, the coloniality of power, and resistance}.
\newblock \bibinfo{journal}{\emph{Television \& New Media}} \bibinfo{volume}{20}, \bibinfo{number}{4} (\bibinfo{year}{2019}), \bibinfo{pages}{350--365}.
\newblock
\bibitem[Richards(1996)]%
{richards_elite_1996}
\bibfield{author}{\bibinfo{person}{David Richards}.} \bibinfo{year}{1996}\natexlab{}.
\newblock \showarticletitle{Elite {Interviewing}: {Approaches} and {Pitfalls}}.
\newblock \bibinfo{journal}{\emph{Politics}} \bibinfo{volume}{16}, \bibinfo{number}{3} (\bibinfo{year}{1996}), \bibinfo{pages}{199--204}.
\newblock
\showISSN{1467-9256}
\urldef\tempurl%
\url{https://doi.org/10.1111/j.1467-9256.1996.tb00039.x}
\showDOI{\tempurl}
\newblock
\shownote{\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/j.1467-9256.1996.tb00039.x}.
\bibitem[Richardson et~al\mbox{.}(2019)]%
{richardson_dirty_2019}
\bibfield{author}{\bibinfo{person}{Rashida Richardson}, \bibinfo{person}{Jason Schultz}, {and} \bibinfo{person}{Kate Crawford}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Dirty {Data}, {Bad} {Predictions}: {How} {Civil} {Rights} {Violations} {Impact} {Police} {Data}, {Predictive} {Policing} {Systems}, and {Justice}}}.
\newblock \bibinfo{type}{{SSRN} {Scholarly} {Paper}} ID 3333423. \bibinfo{institution}{Social Science Research Network}, \bibinfo{address}{Rochester, NY}.
\newblock
\urldef\tempurl%
\url{https://papers.ssrn.com/abstract=3333423}
\showURL{%
\tempurl}
\bibitem[Rotaru et~al\mbox{.}(2022)]%
{rotaru_event-level_2022}
\bibfield{author}{\bibinfo{person}{Victor Rotaru}, \bibinfo{person}{Yi Huang}, \bibinfo{person}{Timmy Li}, \bibinfo{person}{James Evans}, {and} \bibinfo{person}{Ishanu Chattopadhyay}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Event-level prediction of urban crime reveals a signature of enforcement bias in {US} cities}.
\newblock \bibinfo{journal}{\emph{Nature Human Behaviour}} \bibinfo{volume}{6}, \bibinfo{number}{8} (\bibinfo{date}{Aug.} \bibinfo{year}{2022}), \bibinfo{pages}{1056--1068}.
\newblock
\showISSN{2397-3374}
\urldef\tempurl%
\url{https://doi.org/10.1038/s41562-022-01372-0}
\showDOI{\tempurl}
\newblock
\shownote{Number: 8 Publisher: Nature Publishing Group}.
\bibitem[Roulston and Choi(2018)]%
{roulston_sage_2018}
\bibfield{author}{\bibinfo{person}{Kathryn Roulston} {and} \bibinfo{person}{Myungweon Choi}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{The {SAGE} {Handbook} of {Qualitative} {Data} {Collection}}}.
\newblock \bibinfo{publisher}{SAGE Publications Ltd}.
\newblock
\showISBNx{978-1-5264-1607-0}
\urldef\tempurl%
\url{https://doi.org/10.4135/9781526416070}
\showDOI{\tempurl}
\bibitem[Selbst(2017)]%
{selbst2017disparate}
\bibfield{author}{\bibinfo{person}{Andrew~D Selbst}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Disparate impact in big data policing}.
\newblock \bibinfo{journal}{\emph{Ga. L. Rev.}} \bibinfo{volume}{52} (\bibinfo{year}{2017}), \bibinfo{pages}{109}.
\newblock
\bibitem[Shapiro(2019)]%
{shapiro_predictive_2019}
\bibfield{author}{\bibinfo{person}{Aaron Shapiro}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Predictive {Policing} for {Reform}? {Indeterminacy} and {Intervention} in {Big} {Data} {Policing}}.
\newblock \bibinfo{journal}{\emph{Surveillance \& Society}} \bibinfo{volume}{17}, \bibinfo{number}{3/4} (\bibinfo{date}{Sept.} \bibinfo{year}{2019}), \bibinfo{pages}{456--472}.
\newblock
\showISSN{1477-7487}
\urldef\tempurl%
\url{https://doi.org/10.24908/ss.v17i3/4.10410}
\showDOI{\tempurl}
\bibitem[Simkin(2023)]%
{simkin_looking_2023}
\bibfield{author}{\bibinfo{person}{Maya Simkin}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Looking {Forward} to the {End} of {Chicago}'s {Contract} with {ShotSpotter}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.chicagoappleseed.org/2023/05/10/ending-chicago-contract-with-shotspotter/}
\showURL{%
\tempurl}
\bibitem[Small(2009)]%
{small_how_2009}
\bibfield{author}{\bibinfo{person}{Mario~Luis Small}.} \bibinfo{year}{2009}\natexlab{}.
\newblock \showarticletitle{`{How} many cases do {I} need?': {On} science and the logic of case selection in field-based research}.
\newblock \bibinfo{journal}{\emph{Ethnography}} \bibinfo{volume}{10}, \bibinfo{number}{1} (\bibinfo{date}{March} \bibinfo{year}{2009}), \bibinfo{pages}{5--38}.
\newblock
\showISSN{1466-1381}
\urldef\tempurl%
\url{https://doi.org/10.1177/1466138108099586}
\showDOI{\tempurl}
\newblock
\shownote{Publisher: SAGE Publications}.
\bibitem[Spencer et~al\mbox{.}(2016)]%
{spencer2016implicit}
\bibfield{author}{\bibinfo{person}{Katherine~B Spencer}, \bibinfo{person}{Amanda~K Charbonneau}, {and} \bibinfo{person}{Jack Glaser}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Implicit bias and policing}.
\newblock \bibinfo{journal}{\emph{Social and Personality Psychology Compass}} \bibinfo{volume}{10}, \bibinfo{number}{1} (\bibinfo{year}{2016}), \bibinfo{pages}{50--63}.
\newblock
\bibitem[Stevenson(2018)]%
{stevenson_assessing_2018}
\bibfield{author}{\bibinfo{person}{Megan~T. Stevenson}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Assessing {Risk} {Assessment} in {Action}}.
\newblock \bibinfo{journal}{\emph{Minnesota Law Review}} \bibinfo{volume}{103} (\bibinfo{year}{2018}), \bibinfo{pages}{303}.
\newblock
\urldef\tempurl%
\url{https://papers.ssrn.com/abstract=3016088}
\showURL{%
\tempurl}
\bibitem[Stroud(2021)]%
{stroud_automated_2021}
\bibfield{author}{\bibinfo{person}{Matt Stroud}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \bibinfo{title}{An automated policing program got this man shot twice}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.theverge.com/c/22444020/heat-listed-csk-entry}
\showURL{%
\tempurl}
\bibitem[Sweeney(2020)]%
{sweeney_for_2020}
\bibfield{author}{\bibinfo{person}{Annie Sweeney}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \bibinfo{title}{For years {Chicago} police rated the risk of tens of thousands being caught up in violence. {That} controversial effort has quietly been ended.}
\newblock
\newblock
\urldef\tempurl%
\url{https://www.chicagotribune.com/2020/01/24/for-years-chicago-police-rated-the-risk-of-tens-of-thousands-being-caught-up-in-violence-that-controversial-effort-has-quietly-been-ended/}
\showURL{%
\tempurl}
\bibitem[Umansky(2023)]%
{umansky_failed_2023}
\bibfield{author}{\bibinfo{person}{Eric Umansky}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{The {Failed} {Promise} of {Police} {Body} {Cameras}}.
\newblock \bibinfo{journal}{\emph{The New York Times}} (\bibinfo{date}{Dec.} \bibinfo{year}{2023}).
\newblock
\showISSN{0362-4331}
\urldef\tempurl%
\url{https://www.nytimes.com/2023/12/13/magazine/police-body-cameras-miguel-richards.html}
\showURL{%
\tempurl}
\bibitem[Van~Craen and Skogan(2017)]%
{van2017achieving}
\bibfield{author}{\bibinfo{person}{Maarten Van~Craen} {and} \bibinfo{person}{Wesley~G Skogan}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Achieving fairness in policing: The link between internal and external procedural justice}.
\newblock \bibinfo{journal}{\emph{Police quarterly}} \bibinfo{volume}{20}, \bibinfo{number}{1} (\bibinfo{year}{2017}), \bibinfo{pages}{3--23}.
\newblock
\bibitem[Verma(2023)]%
{verma_heat_2021}
\bibfield{author}{\bibinfo{person}{Pransha Verma}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Heat {Listed}}.
\newblock \bibinfo{journal}{\emph{The Washington Post}} (\bibinfo{date}{Dec.} \bibinfo{year}{2023}).
\newblock
\showISSN{0362-4331}
\urldef\tempurl%
\url{https://www.washingtonpost.com/technology/2022/07/15/predictive-policing-algorithms-fail/}
\showURL{%
\tempurl}
\bibitem[Webster(2021)]%
{webster2021if}
\bibfield{author}{\bibinfo{person}{Richard~A Webster}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{“If everybody’s white, there can’t be any racial bias”: The disappearance of hispanic drivers from traffic records}.
\newblock \bibinfo{journal}{\emph{ProPublica--Journalism in the Public Interest. https://www. propublica. org/article/if-everybodys-white-there-cant-be-any-racial-bias-the-disappearance-of-hispanic-drivers-from-traffic-records}} (\bibinfo{year}{2021}).
\newblock
\bibitem[Weiss(1994)]%
{weiss_learning_1994}
\bibfield{author}{\bibinfo{person}{Robert~Stuart Weiss}.} \bibinfo{year}{1994}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Learning from {Strangers}: {The} {Art} and {Method} of {Qualitative} {Interview} {Studies}}}.
\newblock \bibinfo{publisher}{Free Press}.
\newblock
\showISBNx{978-0-02-934625-9}
\newblock
\shownote{Google-Books-ID: SLm6AAAAIAAJ}.
\bibitem[White et~al\mbox{.}(2021)]%
{white2021can}
\bibfield{author}{\bibinfo{person}{Michael~D White}, \bibinfo{person}{Henry~F Fradella}, {and} \bibinfo{person}{Michaela Flippin}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{How can we achieve accountability in policing? The (not-so-secret) ingredients to effective police reform}.
\newblock \bibinfo{journal}{\emph{Lewis \& Clark L. Rev.}} \bibinfo{volume}{25} (\bibinfo{year}{2021}), \bibinfo{pages}{405}.
\newblock
\bibitem[Widder and Nafus(2023)]%
{widder2023dislocated}
\bibfield{author}{\bibinfo{person}{David~Gray Widder} {and} \bibinfo{person}{Dawn Nafus}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Dislocated accountabilities in the “AI supply chain”: Modularity and developers’ notions of responsibility}.
\newblock \bibinfo{journal}{\emph{Big Data \& Society}} \bibinfo{volume}{10}, \bibinfo{number}{1} (\bibinfo{year}{2023}), \bibinfo{pages}{20539517231177620}.
\newblock
\bibitem[Widder et~al\mbox{.}(2022)]%
{widder2022limits}
\bibfield{author}{\bibinfo{person}{David~Gray Widder}, \bibinfo{person}{Dawn Nafus}, \bibinfo{person}{Laura Dabbish}, {and} \bibinfo{person}{James Herbsleb}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Limits and possibilities for “Ethical AI” in open source: A study of deepfakes}. In \bibinfo{booktitle}{\emph{Proceedings of the 2022 ACM Conference on Fairness, Accountability, and Transparency}}. \bibinfo{pages}{2035--2046}.
\newblock
\bibitem[Yeung et~al\mbox{.}(2021)]%
{yeung_identifying_2021}
\bibfield{author}{\bibinfo{person}{Douglas Yeung}, \bibinfo{person}{Inez Khan}, \bibinfo{person}{Nidhi Kalra}, {and} \bibinfo{person}{Osonde~A. Osoba}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{Identifying {Systemic} {Bias} in the {Acquisition} of {Machine} {Learning} {Decision} {Aids} for {Law} {Enforcement} {Applications}}}.
\newblock \bibinfo{type}{{T}echnical {R}eport}. \bibinfo{institution}{RAND Corporation}.
\newblock
\urldef\tempurl%
\url{https://www.rand.org/pubs/perspectives/PEA862-1.html}
\showURL{%
\tempurl}
\bibitem[Zhang et~al\mbox{.}(2018)]%
{zhang_mitigating_2018}
\bibfield{author}{\bibinfo{person}{Brian~Hu Zhang}, \bibinfo{person}{Blake Lemoine}, {and} \bibinfo{person}{Margaret Mitchell}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \bibinfo{title}{Mitigating {Unwanted} {Biases} with {Adversarial} {Learning}}.
\newblock
\newblock
\urldef\tempurl%
\url{https://doi.org/10.48550/arXiv.1801.07593}
\showDOI{\tempurl}
\newblock
\shownote{arXiv:1801.07593 [cs]}.
\bibitem[Zilka et~al\mbox{.}(2023)]%
{zilka_progression_2023}
\bibfield{author}{\bibinfo{person}{Miri Zilka}, \bibinfo{person}{Riccardo Fogliato}, \bibinfo{person}{Jiri Hron}, \bibinfo{person}{Bradley Butcher}, \bibinfo{person}{Carolyn Ashurst}, {and} \bibinfo{person}{Adrian Weller}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{The {Progression} of {Disparities} within the {Criminal} {Justice} {System}: {Differential} {Enforcement} and {Risk} {Assessment} {Instruments}}. In \bibinfo{booktitle}{\emph{Proceedings of the 2023 {ACM} {Conference} on {Fairness}, {Accountability}, and {Transparency}}} \emph{(\bibinfo{series}{{FAccT} '23})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{1553--1569}.
\newblock
\showISBNx{9798400701924}
\urldef\tempurl%
\url{https://doi.org/10.1145/3593013.3594099}
\showDOI{\tempurl}
\end{thebibliography}