Commenotes: Synthesizing Organic Comments to Support Community-Based Fact-Checking
Extracted source text · arXiv 2509.11052
%% The first command in your LaTeX source must be the \documentclass command.
\documentclass[manuscript,nonacm]{acmart}
% manuscript,anonymous,review,nonacm
\newcommand{{\proj}}[0]{Commenotes}
\usepackage{graphicx}
\usepackage{enumitem}
% \usepackage{amssymb}
\usepackage{tikz}
\usepackage{tabularx}
\usepackage{multirow}
\usepackage{multicol}
\usepackage[xcdraw,table]{xcolor}
\usetikzlibrary{trees}
% \usepackage{subfig}
\usepackage{graphicx}
\usepackage{caption}
\captionsetup[figure]{skip=2pt}
\captionsetup[table]{skip=2pt}
\usepackage[subrefformat=parens,skip=0pt]{subcaption}
\usepackage[bb=boondox,bbscaled=.95,cal=boondoxo]{mathalfa}
\newcommand{\X}{$\mathbb{X}$\xspace}
\newcommand{\TODO}[1]{{\color{red}#1}}
\newcommand{\REV}[1]{{\color{blue}#1}}
%%
%% \BibTeX command to typeset BibTeX logo in the docs
\AtBeginDocument{%
\providecommand\BibTeX{{%
\normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
%% Rights management information. This information is sent to you
%% when you complete the rights form. These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{acmcopyright}
\copyrightyear{2018}
\acmYear{2018}
\acmDOI{XXXXXXX.XXXXXXX}
%% These commands are for a PROCEEDINGS abstract or paper.
\acmConference[Conference acronym 'XX]{Make sure to enter the correct
conference title from your rights confirmation email}{June 03--05,
2018}{Woodstock, NY}
\acmPrice{15.00}
\acmISBN{978-1-4503-XXXX-X/18/06}
%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}
%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.\input{UIST-v1}
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%
%%
%% The majority of ACM publications use numbered citations and
%% references. The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}
%%
%% end of the preamble, start of the body of the document source.
\begin{document}
%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title[Commenotes: Synthesizing Organic Comments to Support Community-Based Fact-Checking]{Commenotes: Synthesizing Organic Comments to Support Community-Based Fact-Checking}
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
% \author{Ben Trovato}
% \authornote{Both authors contributed equally to this research.}
% \email{trovato@corporation.com}
% \orcid{1234-5678-9012}
% \author{G.K.M. Tobin}
% \authornotemark[1]
% \email{webmaster@marysville-ohio.com}
% \affiliation{%
% \institution{Institute for Clarity in Documentation}
% \streetaddress{P.O. Box 1212}
% \city{Dublin}
% \state{Ohio}
% \country{USA}
% \postcode{43017-6221}
% }
% \author{Lars Th{\o}rv{\"a}ld}
% \affiliation{%
% \institution{The Th{\o}rv{\"a}ld Group}
% \streetaddress{1 Th{\o}rv{\"a}ld Circle}
% \city{Hekla}
% \country{Iceland}}
% \email{larst@affiliation.org}
% \author{Valerie B\'eranger}
% \affiliation{%
% \institution{Inria Paris-Rocquencourt}
% \city{Rocquencourt}
% \country{France}
% }
\author{Shuning Zhang}
\orcid{0000-0002-4145-117X}
\affiliation{%
\institution{Tsinghua University}
\city{Beijing}
\country{China}
}
\email{zsn23@mails.tsinghua.edu.cn}
\author{Linzhi Wang}
\authornotemark[1]
\affiliation{%
\institution{Tsinghua University}
\city{Beijing}
\country{China}
}
\email{wang-lz22@mails.tsinghua.edu.cn}
\author{Dai Shi}
\authornote{Equal contribution.}
\affiliation{
\institution{College of Design and Innovation, Tongji University}
\city{Shanghai}
\country{China}
}
\email{shidai@tongji.edu.cn}
\author{Yuwei Chuai}
\affiliation{
\institution{University of Luxembourg}
\country{Luxembourg}}
\email{yuwei.chuai@uni.lu}
\author{Jingruo Chen}
\orcid{0009-0007-1606-5780}
\affiliation{%
\institution{Information Science, Cornell University}
\city{Ithaca}
\state{New York}
\country{USA}
}
\email{jc3564@cornell.edu}
\author{Yunyi Chen}
\affiliation{%
\institution{Tsinghua University}
\city{Beijing}
\country{China}
}
\email{chenyuny24@mails.tsinghua.edu.cn}
\author{Yifan Wang}
\affiliation{
\institution{Communication University of China}
\city{Beijing}
\country{China}
}
\email{3634641624@qq.com}
\author{Yating Wang}
\affiliation{
\institution{Tsinghua University}
\city{Beijing}
\country{China}
}
\email{wang-yt24@mails.tsinghua.edu.cn}
\author{Xin Yi}
\orcid{0000-0001-8041-7962}
\authornote{Corresponding author.}
\affiliation{
\institution{Tsinghua University}
\city{Beijing}
\country{China}
}
\email{yixin@tsinghua.edu.cn}
\author{Hewu Li}
\orcid{0000-0002-6331-6542}
\affiliation{
\institution{Tsinghua University}
\city{Beijing}
\country{China}
}
\email{lihewu@cernet.edu.cn}
% \author{Huifen Chan}
% \affiliation{%
% \institution{Tsinghua University}
% \streetaddress{30 Shuangqing Rd}
% \city{Haidian Qu}
% \state{Beijing Shi}
% \country{China}}
% \author{Charles Palmer}
% \affiliation{%
% \institution{Palmer Research Laboratories}
% \streetaddress{8600 Datapoint Drive}
% \city{San Antonio}
% \state{Texas}
% \country{USA}
% \postcode{78229}}
% \email{cpalmer@prl.com}
% \author{John Smith}
% \affiliation{%
% \institution{The Th{\o}rv{\"a}ld Group}
% \streetaddress{1 Th{\o}rv{\"a}ld Circle}
% \city{Hekla}
% \country{Iceland}}
% \email{jsmith@affiliation.org}
% \author{Julius P. Kumquat}
% \affiliation{%
% \institution{The Kumquat Consortium}
% \city{New York}
% \country{USA}}
% \email{jpkumquat@consortium.net}
%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{Zhang et al.}
%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
Community-based fact-checking is promising to reduce the spread of misleading posts at scale. However, its effectiveness can be undermined by the delays in fact-check delivery. Notably, user-initiated organic comments often contain debunking information and have the potential to help mitigate this limitation. Here, we investigate the feasibility of synthesizing comments to generate timely high-quality fact-checks. To this end, we analyze over 2.2 million replies on X and introduce Commenotes, a two-phase framework that filters and synthesizes comments to facilitate fact-check delivery. Our framework reveals that fact-checking comments appear early and sufficiently: 99.3\% of misleading posts receive debunking comments within the initial two hours since post publication, with synthesized \textit{commenotes} successfully earning user trust for 85.8\% of those posts. Additionally, a user study (N=144) found that the synthesized commenotes were often preferred, with the best-performing model achieving a 70.1\% win rate over human notes and being rated as significantly more helpful.
\end{abstract}
%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
\begin{CCSXML}
<ccs2012>
<concept>
<concept_id>10003120.10003130.10003131.10011761</concept_id>
<concept_desc>Human-centered computing~Social media</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10002951.10003260.10003282.10003296</concept_id>
<concept_desc>Information systems~Crowdsourcing</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10002978.10003029</concept_id>
<concept_desc>Security and privacy~Human and societal aspects of security and privacy</concept_desc>
<concept_significance>500</concept_significance>
</concept>
<concept>
<concept_id>10003120.10003130.10011762</concept_id>
<concept_desc>Human-centered computing~Empirical studies in collaborative and social computing</concept_desc>
<concept_significance>500</concept_significance>
</concept>
</ccs2012>
\end{CCSXML}
\ccsdesc[500]{Human-centered computing~Social media}
\ccsdesc[500]{Information systems~Crowdsourcing}
\ccsdesc[500]{Human-centered computing~Empirical studies in collaborative and social computing}
\ccsdesc[500]{Security and privacy~Human and societal aspects of security and privacy}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{Social media, Misinformation, Crowdsourced fact-checking, Community notes, Large Language Model, Comments}
% \begin{teaserfigure}
% \centering
% \includegraphics[width=0.9\textwidth]{}
% \caption{The illustration of \proj{}.}
% \label{fig:teaser}
% \end{teaserfigure}
\begin{teaserfigure}
\centering
\includegraphics[width=1\textwidth]{Figure/comment.jpg}
\caption{The algorithm flow of \proj{}, which could be decomposed into two stages: filtering and synthesizing. During filtering, we prompted fine-tuned Language Models (LMs) to identify fact-checking comments. During synthesizing, we used LLMs to generate commenotes for these posts.}
\label{fig:synthesize_flow}
\end{teaserfigure}
% \received{20 February 2007}
% \received[revised]{12 March 2009}
% \received[accepted]{5 June 2009}
%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle
\section{Introduction}
The spread of online misinformation remains a critical challenge, with far-reaching and oftentimes negative effects on societies~\cite{xu2023combating,yu2024fake}. Social media platforms are under pressure to innovate in their content moderation strategies for effectively fact-check misleading information at scale. Among emerging strategies, community-based fact-checking has gained prominence~\cite{prollochs2022community,drolsbach2024community,chuai2024community}. For instance, $\mathbb{X}$ has rolled out Community Notes system to enable users collaboratively add context annotations to potentially misleading posts~\cite{prollochs2022community}. Community-based fact-checking is valued for its effectiveness, transparency, and ability to incorporate diverse perspectives, thereby fostering a degree of trust~\cite{drolsbach2024community,gao2024can}.
Despite these strengths, a key limitation of community-based fact-checking, such as current implementation of Community Notes on $\mathbb{X}$, is the speed to generate fact-checks at the early stage of misinformation diffusion~\cite{chuai2024community}. In-depth investigation of claims can be time-consuming, often taking many hours or even days~\cite{guo2022survey}. This challenge is compounded by the limited number of available fact-checkers, making it infeasible to scrutinize every piece of content. Consequently, fact-checkers are often forced to prioritize overtly false or intentionally misleading claims over material that is more intricate or nuanced~\cite{robertson2020uses}. These leads to a significant time lag between post creation and note creation, with many community notes coming after the half-life of posts' spreading~\cite{chuai2024community,pfeffer2023half}. This delay allows misinformation to spread widely before corrective notes appear, severely limiting the effectiveness of the intervention~\cite{chuai2024did}.
Recent advances in Large Language Models (LLMs) show promise for synthesizing user-initiated debunking materials into timely fact-checking notes~\cite{de2025supernotes}. We posit that the vast volume of user comments, often generated moments after a post appears, contains valuable, early-emerging debunking information. To systematically assess the feasibility of harnessing this information with LLMs, we first analyze the informational landscape of user comments, then introduce our framework, and finally evaluate its effectiveness. This leads to our first research question (RQ1):\\
\noindent \textbf{RQ1.} \textit{Do comments on misleading posts contain debunking information, and are these debunking comments created earlier than the display of community notes?}\\
If comments to misleading posts do contain factual corrections or fact-checking content, we proceed to our second research question (RQ2):\\
\noindent \textbf{RQ2.} \textit{Can large language models (LLMs) effectively summarize the fact-checking information in comments and generate community-note–style fact-checks that are helpful to users?}
Based on a large-scale dataset of N=2,225,260 comments across 1,841 source posts from $\mathbb{X}$'s Community Notes platform~\cite{chuai2025community}, we analyzed whether the comments contain debunking information, and their validity in generating high-quality fact-checks. For RQ1, we found that 73.1\% of posts received fact-checking comments within the first hour, and a median of 36.5 such comments appeared before the corresponding community note was displayed. We further found a strong positive correlation between post popularity and the number of fact-checking comments, indicating the potential of comment-based fact-checking for viral posts.
Motivated by the high volume and rapid emergence of these user-generated fact-checks, we developed Commenotes, a two-stage framework designed to harness these comments. The framework first employs a filtering stage, using fine-tuned LMs to automatically filter relevant fact-checking comments of a post. Subsequently, a synthesis stage prompts LLMs to consolidate the information from the filtered comments into a concise, synthesized \textit{\textbf{commenote}}\footnote{Note that we used the upper-cased \textit{Commenotes} to denote the two-stage framework, and lower-cased \textit{commenotes} to denote the synthesized notes, similar to Community Notes vs. community notes.}. The framework was carefully designed to ensure accurate information extraction, minimize bias, and improve efficiency in note generation.
Towards RQ2, we conducted extensive technical evaluations and a user study. Because the choice of LLM in the synthesis stage may substantially influence output quality and biases, we treated model choice as a key factor, comparing four distinct models: gpt-4o, claude-3.7-sonnet, grok-4, and qwen2.5-72b-instruct. We found that using the comments within just two hours, 85.8\% of the commenotes are already useful, and 73.9\% of the commenotes are useful using the first 60 comments of a post for synthesizing. A subsequent robustness evaluation revealed that, on average, over 90\% of all commenotes were rated as either ``helpful'' or ``somewhat helpful'' across all experimental conditions. Notably, commenotes showed high helpfulness even for posts without existing community notes: 92.0\% of notes created for posts lacking a displayed note and 91.2\% for posts with no notes at all were rated as at least ``somewhat helpful.'' A detailed analysis of note characteristics further confirmed a strong overall performance, with the average score combining all models exceeding 3.70 out of 5 on most quality dimensions. These results demonstrated the feasibility of \proj{} to timely and reliably synthesize comments into commenotes.
In the subsequent user study (N=144), we compared commenotes against human-written community notes~\cite{de2025supernotes}, and found that most models produced notes of comparable or superior quality. Notably, grok-4 achieved a 70.1\% win rate over human notes and was significantly preferred in terms of clarity, coverage, and impariality. Furthermore, the effectiveness of the notes such as helpfulness showed no systematic correlation with post topics or user demographics like political stance, indicating the robustness of our approach across varied contexts. In summary, this paper makes three contributions:
\begin{itemize}[leftmargin=*]
\item The first empirical investigation into the feasibility of using user-generated comments for community-based fact-checking, showing that debunking information emerges early in the lifecycle of misleading posts.
\item The design and instantiation of Commenotes, a two-stage framework that filters fact-checking comments and synthesizes them into commenotes using LLMs.
\item Comprehensive technical evaluations and a large-scale user study (N=144), demonstrating the superior helpfulness of synthesized notes, with grok-4 achieving win rates as high as 70.1\%.
\end{itemize}
% Those popular comments are centralized towards political contents (XX\% in
% Towards RQ1, we found XX\% of the before fact-checking contained fact-check evidence, averaging XX comments per post. The linguistic features of the comments~\cite{} also showed a strong correlation ($r = .XX$) with fact-checking, with XX\% of the comments being fact-based claims rather than opinions, suggesting the usefulness of these comments. The top-ranked linguistic features are XX, XX and XX, with the top-ranked topics distributed across XX, XX and XX. We conducted an analysis between the comments and the original post and found the fact-checking comments highly correlated with the original post semantically ($r = .XX$).
\section{Background and Related Works}
Our work is situated at the intersection of three aspects. First, we build on foundational work in analyzing user comments, which leverages classification~\cite{madden2013classification,kristo2021computer} and semantic analysis~\cite{cheng2019airbnb,bonifazi2024representation} to structure and interpret user perspectives. Second, we draw from research that used comments for misinformation detection, either in conjunction with original posts~\cite{albahar2021hybrid,yanagi2020fake} or as a standalone signal~\cite{zhang2018fauxbuster,jiang2018linguistic}. Finally, we address the pressing need for scaling up fact-checking, where crowdsourced~\cite{xcommunitynotes,allen2021scaling} and automated approaches~\cite{quelle2024perils,zhou2024correcting} emerged as key alternative to expert-based methods.
% As social media platforms such as $\mathbb{X}$ and Facebook have become central to online communication~\cite{chu2025artificial,angelini2024differences}, user-generated comments provide a rich source of information about public opinion and collective discourse. Prior research on comments can be broadly divided into two directions: (1) analyzing the semantics and implications of comments—for example, how users express stance~\cite{mccambridge2022describing}, convey emotions~\cite{wickramaarachchi2017approach}, and influence perceptions of trustworthiness and credibility~\cite{gierth2020attacking}, and (2) leveraging comment data for practical applications such as personalized content recommendation~\cite{li2010user,saraswathi2023deep}, news filtering~\cite{momeni2013identification}, professional development feedback~\cite{luo2020like}, and misinformation detection or intervention~\cite{jiang2018linguistic}.
\subsection{Analyzing Users' Comments on Social Media Platforms}
Research on analyzing comments focuses on two areas: \textit{classification and filtering}, and \textit{semantic analysis}. Classification and filtering methods enable the creation of structured datasets from raw comments, which can be used to extract specific insights. For example, Madden et al. \cite{madden2013classification} developed a classification scheme for YouTube comments, while Kristo et al. \cite{kristo2021computer} refined this by first filtering out spam. Structured comment data not only improves the understanding of user perspectives \cite{madden2013classification} but also helps isolate topic-relevant segments for online governance \cite{georgakopoulos2018convolutional}. Our research builds on this by focusing on extracting misinformation cues from factually grounded comments, analyzing semantic patterns for signals of authenticity.
In addition to classification, semantic analysis of comments reveals deeper meaning. Studies have used text mining and semantic annotation to uncover user concerns in Airbnb reviews \cite{cheng2019airbnb, collins2015can} and to provide more robust analysis of user feedback \cite{yadav2020semantic}. Recent work underscores the value of these semantic approaches in gaining deeper insights into user perspectives \cite{bonifazi2024representation}. Our work extends these efforts by exploring the feasibility of user comments for user-centered fact-checking, moving beyond simply extracting viewpoints to assessing their utility for identifying misinformation.
\subsection{User-Centered Evidence in Misinformation Detection}
Research on user comments showed that they can be valuable for misinformation detection by revealing collective perspective and emotional trends. These works focuses on two main approaches: combining comments with original content, and using comments alone.
Some studies integrate user comments with the associated post to improve detection. For instance, \citet{hamed2023fake} used emotional features from both comments and news content to train a model to identify misinformation. \citet{albahar2021hybrid} and \citet{yanagi2020fake} combined news content with user comments, achieving higher recall than methods that used articles alone. In contrast, our study focuses exclusively on user-generated comments to detect misinformation, without relying on the analysis of original post.
There are also research that focused solely on comments for misinformation detection. For image-based misinformation, \citet{zhang2018fauxbuster} and \citet{shang2020fauxward} developed detectors using the topological, semantic, and linguistic features of comments to distinguish between authentic and manipulated images. While effective, these methods have limited applicability to other types of misinformation. For textual misinformation, Jiang et al.~\cite{jiang2018linguistic} explored linguistic indicators in comments, noting differences in emotional tone and topic focus between comments on factual and misleading content. However, they found only a weak correlation between these features and the presence of misinformation~\cite{jiang2018linguistic}. Our work goes beyond simple linguistic feature analysis, proposing a two-stage framework for synthesizing comments to aid in fact-checking.
\subsection{Scaling Up Fact-Checking}
Scaling up traditional fact-checking by human experts is challenging due to issues of scalability and user trust \cite{pennycook2019fighting, straub2022americans}. In response, researchers have explored crowdsourced and automated fact-checking as scalable alternatives \cite{kim2020leveraging, quelle2024perils}. Crowdsourced fact-checking, exemplified by Community Notes \cite{xcommunitynotes}, has been shown to align with expert judgments \cite{allen2021scaling}, resist motivated reasoning \cite{epstein2020will}, and reach broader online communities \cite{micallef2020role}.
Automated fact-checking, leveraging AI models like LLMs, has been studied for its potential to complement or replace various fact-checking steps, including identifying misleading claims \cite{qi2024sniffer}, verifying information \cite{wang2023explainable}, and generating explanations \cite{yue2024evidence, he2023reinforcement, zeng2024justilm}. Some research proposes fully automating the process by using LLMs to handle multimodal data and query external sources \cite{zhou2024correcting}. While some work, such as that by De et al.~\cite{de2025supernotes}, focuses on synthesizing community notes, our work differentiates itself by using LLM to synthesize user-generated organic comments to support community-based fact-checking.
Other research has demonstrated the potential of LLMs to enhance collective decision-making and democratic processes \cite{yang2024llm}. LLMs have been shown to effectively aggregate diverse perspectives \cite{burton2024large} and map opinions to consensus statements \cite{bakker2022fine}. Fish et al. \cite{fish2024generative} combined LLMs with social choice theory to generate multiple summary statements instead of a single consensus. Although their approaches inspired our work, they did not specifically test the LLMs' feasibility on aggregating the vast volume of comments, which is this paper's focus.
\section{Feasibility of Comments as Fact-Check Evidence}\label{sec:comments-feasibility}
\subsection{Dataset}\label{sec:dataset}
In this study, we examined whether user comments can serve as a source of fact-checking evidence, potentially contributing to the Community Notes system. We used the dataset collected by Chuai et al~\cite{chuai2025community}, which contains 2,225,260 direct replies (i.e., comments) across 1,841 source posts. The dataset combines three different sources: (i) the note dataset, (ii) the post dataset, and (iii) the reply dataset. For each post, we acquired the time of the note display and the timestamps of replies.
Before a note is displayed, users replying to the post can only see the original content. After a note is displayed, they can see both the post and the community note.
Because replies written after the note display may reference the note itself, we primarily focus on replies created before note publication to avoid bias.
% dataset
\subsection{Methodology}\label{sec:methodology}
% discern which post is fact-check post
\textbf{Classifying fact-check comments.}
We define fact-check comments as those that provide information to correct inaccuracies in a post. Such comments must meet two criteria: (1) they explicitly or implicitly point out an inaccuracy, and (2) they provide reasoning or relevant information to support this judgment. We prompted GPT-4o to identify fact-check comments, which received the original post and its direct reply, and was tasked with determining whether the reply contained fact-checking information relevant to the post.
The detailed prompt is provided in Appendix~\ref{sec:prompt_filter}. To validate model performance, we manually annotated a subset of comments, comparing human and model labels (see Appendix~\ref{app:annot} for rating criteria).
GPT-4o achieved an accuracy of 0.82 on this validation set ($N = 150$), indicating fair accuracy.
%We defining fact-check comments as comments that provide fact-checking information to correct inaccuracies in the post. Fact-check comments must meet the following two characteristics: (1) the comment points out inaccuracies in the post, and (2) the comment provides reason or relevant information for its judgment. As LLMs have become powerful tools in the field of NLP, we employ LLMs to discern whether a comment is a fact-checking comment. LLM was provided of the original post and a direct reply (comment) to this post and was required to estimate whether the comment provide fact-checking information and the whether information is related to the post. The detailed prompt can be found in Appendix~\ref{sec:prompt_filter}. We use GPT-4o to complete this task. To ensure the judgment of LLM to be accurate, we manually labeled a sample of comments and compared LLM labels and human labels (see Appendix~\ref{app:annot} for a detailed rating criteria). Based on the results of manual annotation, the accuracy of LLM reached 0.82 (N = 150).
%We employ LLMs to synthesize comments in the following time periods: (1) within one hour after the post is created; (2) within two hours after the post is created; (3) within five hours after the post is created; (4) before the community note is displayed. Comments in those periods are provided to LLMs and synthesized to AI-synthesized Notes. Due to the limit of context length of LLM, we use at most 500 comments, and when the number of comments exceeds 500, we randomly select 500 of them, though this may cause loss of fact-check information in comments. The AI-synthesized Notes are manually compared with notes displayed on X to see whether AI-synthesized Notes contain useful fact-check information. We use the following criteria to determine whether an AI-synthesized Note is useful: AI-synthesized Note should correctly point out the inaccuracies in the post and provide relevant information (see Appendix~\ref{app:annot} for a detailed guide). Since users tended to use abbreviated expressions on social media platforms~\cite{alrumaih2020sentiment}, we did not require the information in the AI-synthesized Note to be comprehensive and detailed. We record the earliest time period in which LLM can synthesize comments into useful fact-check note.
% analyze temporal distribution
\textbf{Analyze temporal distribution.} To analyze the temporal distribution of fact-check comments, we segmented the timeline following each post's creation into 15-minute intervals. For each interval, we calculated the mean and median number of fact-check comments across all posts. We also computed the cumulative proportion of fact-check comments at each time point relative to the total number of fact-check comments. This approach allowed us to model the emergence and accumulation rate of corrective comments provided by the community. % rate at which corrective comment information emerges over time.
\textbf{Analyze correlation with features of posts.} We investigated how the volume of fact-checking comments correlated with three key post features known to influence content propagation and potential for misinformation: popularity, author status, and topic~\cite{chuai2024did,xcommunitynotes}. Following prior work~\cite{wu2017sequential}, we operationalized post popularity as: $$ s = log_2\frac{c}{h} + 1 $$ where $s$ is the normalized popularity score, $c$ is the number of comments, and $h$ is the number of hours since the post was created. To prevent late comments from skewing the popularity, we considered only comments posted within the earliest 80\% of each post's timeline. Author status (i.e., verified or unverified) and post topics were extracted directly from the dataset, adhering to the methodology of Chuai et al.~\cite{chuai2025community}. We then analyzed the relationship between these features and both the absolute number and relative proportion of fact-checking comments.
% We plot the results of the LLM to explore the temporal distribution of fact-check comments. We counted the number of fact-check comments every 15 minutes and calculated the mean and median number of fact-check comments. We also calculated the proportion of fact-check comments before each time point relative to all fact check comments. This approach reflects the temporal distribution of comment counts across posts.
% analyze linguistic features
% analyze topic
% \textbf{Analyze whether it is opinion or claims.} We use pre-trained model\footnote{https://huggingface.co/lighteternal/fact-or-opinion-xlmr-el} to determine whether a comment is opinion or claims, which takes a sentence as a input and outputs a label which indicate whether the comment is opinion or claims. The model is reported to have a F1 score of 0.952.
% analyze whether it is related to topics
% \textbf{Temporal Feature of Fact-Check Information}
\subsection{Results}
\subsubsection{Distribution of Fact-Check Comments By Topic}\label{sec:Comments Topic}
To investigate the influence of post topic on user-generated fact-checking, we analyzed the distribution of these comments across four predominant topics: Finance \& Business (30.49\%), Politics (28.30\%), Entertainment (23.53\%), and Science \& Technology (10.53\%). We note that individual posts could be classified under multiple topics. % Note that a single post can be associated with multiple topics.
% As shown in Figure~\ref{fig:fact_check_variation_by_time}, posts about \textit{Politics} elicited the highest absolute number of fact-checking comments. However, the differences across topics were not statistically significant (Kruskal–Wallis test, $H = 4.250, p = .2357$). Similarly, Figure~\ref{fig:fact_check_variation_by_count} shows no significant differences in the proportion of fact-checking comments across the four categories (Kruskal–Wallis test, $H = 1.651, p = 0.6479$). These results suggest that users provide fact-checking comments consistently across diverse topical domains.
As illustrated in Figure~\ref{fig:fact_check_variation_by_time}, while posts concerning \textit{Politics} attracted the highest absolute number of fact-checking comments, a Kruskal-Wallis test revealed no statistically significant differences in comment counts among the topics ($H=4.250$, $p=.2357$). Similarly, the proportion of fact-checking comments relative to total comments did not significantly vary across these categories ($H=1.651$, $p=.6479$), as shown in Figure~\ref{fig:fact_check_variation_by_count}. These results indicate that the practice of user-initiated fact-checking is a robust phenomenon that manifests with comparable frequency across diverse topical domains.
%As shown in Figure~\ref{fig:category}(a), posts concerning Politics elicited the highest absolute count of fact-checking comments, but the result does not show a significant difference (Kruskal-Wallis test, H = 4.250, p = 0.2357).And as shown in Figure~\ref{fig:category}(b) the proportion of fact-checking comments of posts across the four topics does not show significant differences (Kruskal-Wallis, H = 1.651, p = 0.6479). This finding suggests that users are generous to provide fact-checking comments in posts on various topics.
% and OngoingNewsStory, accounting for 30.49\%, 28.30\%, 23.53\%, 10.53\% and 7.92\% respectively (noting that a post may have more than one topic). The result is shown in Figure~\ref{fig:category}. Posts about politics contain the highest amount as well as percentage of fact-checking comments.
\begin{figure}
\centering
\begin{subfigure}{0.49\textwidth}
\includegraphics[width=\textwidth]{Figure/topic-count.pdf}
\caption{}
\label{fig:fact_check_variation_by_time}
\end{subfigure}
\hfill
\begin{subfigure}{0.49\textwidth}
\includegraphics[width=\textwidth]{Figure/topic-percentage.pdf}
\caption{}
\label{fig:fact_check_variation_by_count}
\end{subfigure}
\caption{(a) Average count of fact-checking comments, and (b) percentage of fact-check comments among all comments across different topics. \textit{FinanceBusiness}, \textit{Politics}, \textit{Entertainment}, and \textit{SciTech} denoted \textit{Finance \& Business}, \textit{Politics}, \textit{Entertainment} and \textit{Science \& Technology} classes respectively.}
\label{fig:category}
\end{figure}
\subsubsection{Temporal Feature of Fact-Check Information}
%The analysis of fact-check comment distribution, as presented in Figure~\ref{fig:images-all-percentage}, reveals that those community-driven fact-checking efforts occur very early in a post's lifecycle. Figure~\ref{fig:images-all-percentage}, which plots the accumulated percentage of fact-check comments, visually confirms that the vast majority of comments are created well before the median time of community note display. Specifically, the mean curve (orange) and the median curve (green) both surpass the 80\% accumulation threshold before the median note display time, which is visually marked at approximately 22 hours from the post's creation.
The distribution of fact-checking comments, shown in Figure~\ref{fig:images-all-percentage}, indicates that community-driven corrections emerge early in a post's lifecycle. 99.3\% of the posts received debunking comments within the first two hours. The accumulated percentage curve confirms that more than half of fact-checking comments appear well before the median community note creation time. Specifically, both the mean (blue) and median (red) curves surpass the 50\% threshold prior to the median creation time, which occurs roughly 9.7 hours after post creation.
\begin{figure}
\centering
\begin{subfigure}{0.49\textwidth}
\includegraphics[width=\textwidth]{images-all-percentage.pdf}
\caption{}
\label{fig:images-all-percentage}
\end{subfigure}
\hfill
\begin{subfigure}{0.49\textwidth}
\includegraphics[width=\textwidth]{images-all-count.pdf}
\caption{}
\label{fig:images-all-count}
\end{subfigure}
\caption{The change of fact-check comments’ number and their proportion relative to the cumulative total of fact-check comments observed up to the collection of the dataset. (a)~Accumulated percentage of fact-check comments. A median of more than 50\% fact-checking comments emerged before the creation of community notes. (b)~Accumulated count of fact-check comments over time. A median of more than 20 fact-checking comments emerged before the creation of community notes.}
\label{fig:ts_comments}
\end{figure}
Further corroboration is provided by Figure~\ref{fig:images-all-count}, which illustrates the accumulated counts of these comments. On average, a significant number of fact-check comments emerge before the note's creation. At the median note creation time, the mean accumulated count of fact-check comments converges around 70, with a median of 15.5. These results indicate that a substantial amount of user-initiated fact-checking occurs before the official note becomes visible, providing a strong basis for our algorithm to synthesize timely fact-checks. %This indicates that a substantial volume of user-initiated fact-checking has already been completed, on average, by the time the community note becomes visible. This suggests an important foundation for our algorithm to synthesize these comments for rapid fact-checking.
% \subsubsection{Linguistic Features and Topics Distribution}
\subsubsection{Relatedness}
% Following Wu et al.~\cite{wu2017sequential}, we defined post popularity as: $$ s = log_2\frac{c}{h} + 1 $$ where $s$ is the normalized value of popularity, $c$ is the number of comments, and $h$ is the number of hours since the post was created. To prevent very late comments from distorting popularity, we considered only comments posted within the earliest 80\% of each timeline.
To examine factors associated with the number of fact-checking comments, we analyzed their relationship with post popularity and author status. We found a strong correlation between popularity and the number of fact-checking comments within two hours ($\rho = .6169$, $p < .001$), but there are no correlation between popularity and the proportion of fact-checking comments ($\rho = .0068$, $p = .9338$). Popular posts attracted more fact-checking comments within the same period compared to non-popular posts, likely because higher visibility triggered faster corrective responses from users who perceived greater stakes in correcting the record.
We further analyzed the relatedness between authors and fact-checking comments within two hours, with results shown in Table~\ref{tab:comparison_author}.
A Welch's t-test confirmed that posts by verified authors attracted a significantly higher number of fact-checking comments (M=27.76) compared to those by unverified authors (M=15.95) ($t=1.390$, $p=.0217 < .05$). In contrast, the proportion of fact-checking comments relative to the total comment volume was statistically similar for both verified (9.69\%) and unverified (12.35\%) authors ($t=-0.948$, $p=.3464$). This suggests that verified accounts attract more corrections primarily because they receive greater engagement overall, rather than being disproportionately targeted. These validate the potential of prioritizing popular posts to efficiently harvest a larger volume of corrective content where the potential for misinformation spread is highest~\cite{chuai2024did}.
\begin{table}[htbp]
\centering
\caption{Comparison between fact-check comments within two hours for verified and unverified author.}
\label{tab:comparison_author}
\begin{tabular}{c|c|c}
\toprule
& average number & proportion\\ \midrule
verified author & 144.95 & 6.75\% \\
unverified author & 66.85 & 7.42\% \\
\bottomrule
\end{tabular}
\end{table}
\section{\proj{}: Design and Implementation}\label{sec:ai-synth}
We first introduced the design of \proj{}, and then detailed the two phases. We finally introduced the implementation settings for this framework.
\subsection{Algorithm Flow}
Our algorithm consists of a filtering stage and a synthesis stage (Figure~\ref{fig:synthesize_flow}). In the filtering stage, we identify comments that contribute to fact-checking. In the synthesis stage, these selected comments are consolidated into \textit{\textbf{commenotes}}. This two-stage architecture is essential for addressing the problem that fact-check comments is abundant yet sparse. For example, posts in our dataset receive an average of 277.43 comments within two hours, yet only 23.99 of these are fact-checks. The filtering stage is therefore critical for isolating high-quality input, which improves the accuracy and reliability of the final synthesis while reducing computational overhead.
%Our algorithm has two phases, filter phase and synthesize phase, as shown in Figure~\ref{fig:synthesize_flow}. The filter phase selected the comments which could contribute to fact-checking, and the synthesize phase used these comments which contributed to fact-checking to synthesize consensus as generated community notes. This decision is because usually we have a vast amount of comments on social media within a short range of time in today's lives, and only a small percentage of comments on social media contributes to fact-checking, therefore we needed the filter process to first select ``high-quality'' comments, both to increase the accuracy and reliability of the synthesizing, and also to reduce cost.
\subsection{Filter Stage}
This first stage filters comments to identify those containing fact-checks, thereby managing the large volume (see Figure~\ref{fig:category}) and variable quality of user-generated comments~\cite{park2016supporting}. For this binary classification task, we fine-tuned small LMs like E5 and BERT. This approach was chosen over prompting LLMs as it yielded comparable accuracy with potentially lower computational cost (see Section~\ref{sec:technical_feasibility}).
To perform the classification, we provided the model with the original post's text alongside each comment to preserve context. A comment was classified as a fact-check if it satisfied two criteria (detailed in Section~\ref{sec:methodology}): (1) it points out a factual error in the corresponding post, and (2) it substantiates its correlation with reasons or evidence for correction. This is similar to the prior classification~\cite{jiang2018linguistic} but differed from studies that require external links as evidence, as not all fact-check comments provided external URLs~\cite{bond2023engagement}. Furthermore, to aid in our later analysis, we excluded comments whose factual content was irrelevant to the post's claims. The model's final output was a binary label for each comment, and the detailed prompt and criteria are available in Appendix~\ref{sec:prompt_filter}.
% The aim of the filter phase is to screen out fact-checking comments, as all comments may be exceedingly vast in number (see Figure~\ref{fig:category}), and may be of low quality~\cite{park2016supporting}. We feed the information into an LLM via a prompt that describes (1) the task that the LLMs need to filter out the fact-check comments, (2) the criterion of fact-checking comments, and (3) the output format. The definition of fact-check comments is as follows. Fact-check comments must meet the following two characteristics: (1) the comment points out inaccuracies in the post, and (2) the comment provides reason or relevant information for its judgment, as described in Section~\cite{sec:methodology}. We added an extra rule to facilitate the latter synthesize process: comments containing only fact-checking information that is irrelevant to the accuracy of the post are not seen as fact-check comments (see Appendix~\ref{sec:prompt_filter} for detailed prompts). This definition followed XX practices. Additionally, to discern the correlation of comments and posts, thereby ensuring the comments are fact-checks for posts, we provided the post text to the LMs. The LMs would output binary labels, classifying between whether the comment is a fact-checking comment.
% We used language models such as E5, BERT rather than LLMs for the sake of accuracy and cost, where we found E5 and BERT, when fine-tuned could reach an satisfying accuracy, and would reduce the cost of LLMs (see Section~\ref{sec:technical_feasibility})
% In this phase, we classify comments into two classes: comments with fact-check information and comments without fact-check information.
\subsection{Synthesize Stage}
%In the synthesize phase, our goal was to generate a concise, factual note using the filtered fact-checking comments. To manage the LLM's context window, we set a maximum threshold for the number of comment. If more were available, a random subset was selected. These comments were first preprocessed to remove irrelevant user mentions, such as ``@username''. We then used a prompt, informed by De et al.~\cite{de2025supernotes} that framed the task for the LLM as a Community Notes contributor. The prompt (1) introduce Community Notes to LLMs to help them better understand the task, (2) ask LLMs to find fact-check information in comments and point out inaccuracies in the post, and (3) emphasize the exclusion of biased or argumentative language and emphasize not mentioning ``comments''. We let LLMs to only output the note to avoid interference of other reasoning process or information.
In the synthesis stage, our goal was to generate a concise, factual commenote from the filtered fact-checking comments. To manage the LLM's context window, we set a maximum threshold for the number of comments. If more were available, we randomly sampled a subset. Before synthesis, comments were preprocessed to remove irrelevant user mentions (e.g., @username). We then applied a prompt, adapted from De et al.~\cite{de2025supernotes}, that framed the task as if the LLM was a Community Notes contributor. The prompt (1) introduced Community Notes to provide task context, (2) instructed the LLM to identify fact-check information in the comments and highlight inaccuracies in the post, and (3) emphasized avoiding biased or argumentative language and forbade mentioning the word ``comments''. Finally, we instructed the LLM to output only the note itself to prevent extraneous reasoning or explanations.
To align with platform standards~\cite{x_community_notes_quickstart}, we enforced a strict character limit and implemented a post-generation check to automatically regenerate any note that exceeded the limit. Furthermore, to promote diversity of perspective and avoid the potential biases of automated ranking~\cite{ganiuly2024ethics}, we did not use scoring or aggregation mechanisms to select a single best output.
% For the synthesize phase, we set a threshold of comments to avoid exceeding the context window limit of LLMs and use as many fact-checking comments as possible within this threshold. Worth noting, if there are more comments than the threshold from the filtering phase, we randomly select the comments.
% We provided the following prompt to LLMs for synthesizing. We (1) introduced Community Notes to LLMs to help them better understand the task, (2) ask LLMs to find fact-check information in comments and point out inaccuracies in the post, and (3) emphasize the exclusion of biased or argumentative language and emphasize not mentioning ``comments''. We preprocessed the comments by removing the ``@\{replied person\}'' appearing at the beginning of the comments. These information are unrelated to our task. We only let LLMs output the note to avoid interference of other reasoning process or information. To ensure the notes' quality, we wrote in prompt that the note should not exceed the maximum length of community notes, aligning with community notes' requests~\cite{x_community_notes_quickstart}. Besides, we also automatically checked the length requirements and regenerated if it did not adhere to the requirement. We did not specifically opt for aggregation or scoring mechanisms to rank the notes to ensure diversity and justice, thereby avoiding bias~\cite{ganiuly2024ethics}.
% In this phase, comments with fact-check information will be used to generate notes. We use a prompt to instruct LLM to synthesize the original post and comments filtered in the former phase and point out inaccuracies in the post. Our prompt is informed by De et al.~\cite{de2025supernotes}. The final version of our prompt can be separated into several parts: (1) an introduction to Community Notes, (2)instruct that asks LLM to generate note, (3) the textual content of the original post and comments filtered in the previous stage, (4) additional instructions, including emphasizing bias reduction, avoiding mention of ``comments,'' and other instructions intended to make the notes more similar to a Community Note.
\subsection{Implementation Details}
% For the filtering phase, we empirically set the temperature of LLMs to 0.6 and top-p to 1. We kept other hyperparameters as default. For the synthesizing phase, we used default settings of the hyperparameters, as we empirically tested several other settings, but no significant differences are observed. We set the threshold of synthesizing the comments as 300. The average length of fact-checking comments is 126.3 characters (32.2 tokens). Context window of LLMs typically varied from 100K tokens\footnote{\url{https://www.anthropic.com/news/100k-context-windows}} to 1M tokens\footnote{\url{https://www.anthropic.com/news/1m-context}}, where 300 comments would not exceed the typically token length. We empirically tested embedding more comments and found some examples that 500 comments exceed the context window of gpt-4o. Notably, 83.3\% posts' comment number within the first two hours did not exceed 300, therefore a threshold of 300 comments is enough and appropriate. For the prompt optimization, we empirically and randomly sampled about 50 posts for each prompt optimization iteration and use manually labeling to compare different versions, with ``usefulness'' as the metric (see Section~\ref{sec:study_setup} for the definition of ``usefulness'').
For our implementation, we configured the model hyperparameters and data input parameters as follows. In the filtering phase, we empirically set the LLM's temperature to 0.6 and top-p to 1.0, while all other hyperparameters were kept at their default settings. For the synthesis phase, we used the default hyperparameter configuration, as our empirical tests with alternative settings yielded no significant difference in output quality. We established a data input threshold of 300 comments for the synthesis stage. This number was chosen to balance two factors: it is well within the typical context window of modern LLMs (100K\footnote{\url{https://www.anthropic.com/news/100k-context-windows}}-1M\footnote{\url{https://www.anthropic.com/news/1m-context}} tokens), as 300 comments correspond to approximately 9,700 tokens based on an average length of 32.2 tokens per comment. It is also ecologically valid, as our analysis showed that 83.3\% of posts receive fewer than 300 comments within the first two hours. Finally, our prompts underwent an iterative optimization process where we randomly sampled and manually evaluated approximately 50 posts per iteration, using ``usefulness'' as the primary metric for comparison (see Section~\ref{sec:study_setup} for definition).
\section{Technical Evaluation}
\subsection{Study Setup}\label{sec:study_setup}
The evaluation focused on two aspects: the feasibility of filtering and synthesizing comments, and the robustness of LLMs to different types of posts, especially those uncovered by community notes.
\textbf{Feasibility evaluation.} To assess the viability of our approach, we first evaluated automated identification of fact-checking comments, and then evaluated the synthesis of these comments into commenotes.
% the capability of models in judging whether the comments are fact-check comments or not, corresponding to the filter phase, and then evaluated the capability of models in synthesizing comments.
In the filter stage, we benchmarked the classification performance of four commonly investigated models~\cite{wang2022text,zhang2023twhin}. These include two commonly used models, BERT-base-uncased\footnote{https://huggingface.co/google-bert/bert-base-uncased}, xlm-roberta-base\footnote{https://huggingface.co/FacebookAI/xlm-roberta-base}, one model expert at classification tasks, E5-base~\cite{wang2022text} and one reported to be trained on $\mathbb{X}$'s data, twhin-bert-based~\cite{zhang2023twhin}. We fine-tuned each model for five epochs on the dataset described in Section~\ref{sec:methodology}, which was split into training (80\%), validation (10\%), and testing (10\%) sets. All models were trained using an AdamW optimizer with a learning rate of 5e-5, a weight decay of 0.01, a batch size of 16, and a Cross-Entropy Loss function. We report performance on the test set.
% We use an AdamW optimizer with a learning rate of 5e-5 and a weight decay of 0.01. We fine-tuned for 5 epochs for each model, noticing that this would provide optimized results. We use Cross-Entropy Loss Function to calculate loss. The batch size is set as 16. We used the annotated dataset described in Section~\ref{sec:methodology} and split the dataset into 8:1:1 for cross-validation. We reported the results on the test set.
In the synthesis stage, we evaluated the capability of LLMs to generate useful commenotes from the filtered comments across two conditions: \textbf{temporal availability} and \textbf{information volume}. For the temporal availability, we generated commenotes across four distinct time windows: within one hour, two hours, and five hours of the original post's creation, and in the period before the official community note appeared. For the information volume, we generated notes from the first 15, 30, 60 and 500 comments. The 500-comment group was included specifically to test the impact of a much larger input on synthesis quality.\footnote{Note that in this evaluation, to test the capability of the synthesis phase, we did not restrict the number of comments' input, and showed the statistics of fact-check comments in Section~\ref{sec:technical_feasibility}.}
Experimenters manually annotated the usefulness of each generated notes. Consistent with our user study (Section~\ref{sec:userstudy}) and prior work~\cite{de2025supernotes}, the evaluation was guided by a primary criterion: \textit{Does the note meaningfully provide fact-check evidence for the post?} A note was considered \textbf{useful} if it accurately identified the post's inaccuracy and provided relevant context. Acknowledging the prevalence of abbreviated expressions on social media~\cite{alrumaih2020sentiment}, we did not require notes to be fully comprehensive. Detailed annotation guidelines are available in Appendix~\ref{app:useful_notes}.
% We next evaluated whether LLMs could synthesize comments into useful fact-checking notes within different time windows: (1) within one hour of a post’s creation, (2) within two hours, (3) within five hours, and (4) before the community note appeared. Comments from each period were provided to the LLM to generate synthetic notes. Because of context length limits, we used at most 500 comments per synthesis; when more were available, we randomly sampled 500, which may omit some fact-check information. We manually compared LLM-generated notes with the official Community Notes to assess whether they contained useful fact-checking content. A note was considered useful if it accurately identified the post’s inaccuracy and provided relevant context (see Appendix~\ref{app:annot}).
% Given the prevalence of abbreviated expressions on social media~\cite{alrumaih2020sentiment}, we did not require notes to be fully comprehensive. For each post, we recorded the earliest time window in which the LLM could generate a useful note.
% Experimenters manually annotated the generated notes to assess whether they meaningfully reflected the truthfulness of the posts and functioned as valid notes. The evaluation criteria were consistent with the user study (Section~\ref{sec:userstudy}) and prior work~\cite{de2025supernotes}, and were based on the following definition: \textit{``Does the note meaningfully provide fact-check evidence for the post?''} Detailed annotation guidance is provided in Appendix~\ref{app:useful_notes}.
\textbf{Robustness evaluation.} We evaluated the robustness of our approach on recent posts not covered by existing community notes. This experiment also tested \proj{}'s ability to synthesize helpful notes for novel posts, thereby mitigating the risk of data contamination where models might rely on memorized knowledge from their training sets.
We curated the dataset by sampling recent posts from $\mathbb{X}$ via its official API. We targeted posts that received user requests for Community Notes, but for which no note was publicly visible at the time of sampling. This approach ensured that the content was relevant and addressed a genuine gap in the existing Community Note ecosystem. We collected posts published after Sep 1st, 2025, and categorized them based on their final Community Note status, resulting in three distinct experimental conditions. The final dataset comprised 124 posts and 24,237 corresponding comments (M=194.5 comments per post). The conditions were: (1) \textit{with note} (N=27), where a Community Note became publicly visible at the time of sampling, (2) \textit{without display} (N=48), where a note was written but did not meet the criteria for public display, and (3) \textit{without note} (N=49), where no Community Notes was ever created.
For each of the 124 posts, we used \proj{} to synthesize notes with four LLMs, consistent with Section~\ref{sec:design}. We then conducted a between-subjects user study with 96 participants (50 male, 46 female, with a mean age of 23.3, SD=5.6) recruited via distributing posters on different social media. Participants were randomly assigned to one of the three experimental conditions (N=32 per condition). Within each condition, participants were further assigned to evaluate the notes generated by only one of the four models. This design ensured that for any given post, the note generated by each model was evaluated by eight distinct participants. Adopting the methodology from De et al.~\cite{de2025supernotes}, we evaluated the notes based on their helpfulness and characteristics. Participants were each compensated \$4.20, and the study protocol received approval from our institution's Institutional Review Board (IRB). Since the study was conducted primarily with Chinese participants, all posts and generated notes were provided in both their original language and a translated Chinese version. We used GPT-4o for translation as past work proved even smaller LLMs could produce translation with good quality~\cite{sizov2024analysing}, and two experimenters manually checked the accuracy to ensure the translation did not alter the original meaning.
% We targeted posts that had received user requests for Community Notes\footnote{https://communitynotes.x.com/guide/en/under-the-hood/note-requests} but had not yet had Community Notes written. This sampling ensured the content was both (1) relevant and likely in need of fact-checking, and (2) represented a genuine gap in the current Community Note system. Operational-wise, we restricted the requests to be published not earlier than Sep 1st, 2025, and randomly selected the posts with requests thereafter. This process resulted in a final dataset of 124 posts and 24,237 corresponding comments (mean=194.5 comments per post), including 27 posts with displayed notes, 48 posts without displayed notes, and 49 posts without notes.
% For these selected posts, we employed \proj{} to synthesize notes using four different candidate models, and adopted Du et al.'s method~\cite{de2025supernotes} for evaluation, focusing on helpfulness and note characteristics. These designs were consistent with Section~\ref{sec:design} except that these notes have no real Community Notes to compare with.
% We recruited 96 participants (50 male, 46 female, mean age=23.3, SD=5.6) through distributing posters on different social media platforms. We employed a between-subjects study design using a custom-built online platform. Participants were randomly divided into three groups, with each group having 32 participants. Participant for each group was randomly assigned to evaluate the notes generated by only one of the four models for the corresponding condition, with notes displayed, without notes displayed but with notes, and without notes at all. With this strategy, the posts generated with each model was being assessed by eight distinct participants. Participants were compensated \$4.20 for their time, and the study protocol received approval from our university's Institutional Review Board (IRB). The study was conducted primarily with Chinese participants; therefore, we provided both English notes and translated the notes and original posts into Chinese without altering their meaning.
\subsection{Results}
\subsubsection{Technical Feasibility}\label{sec:technical_feasibility}
\textbf{Feasibility of the filtering stage across models}. The performance of the models were shown in Table~\ref{tab:performance}. The accuracy of all models exceeded 0.85, while E5-base has the highest accuracy and recall, exceeding or reaching 0.90. The results showed that those embedding models are capable to accurately complete the filter stage's task. We finally chose the fine-tuned E5 model~\cite{wang2022text} to complete the classification task.
\begin{table}[htbp]
\centering
\caption{Performance of different models for the filtering phase, where the highest performance per metric is highlighted in bold.}
\label{tab:performance}
\begin{tabular}{c|c|c|c|c}
\toprule
model & accuracy & recall & precision & F1 score\\ \midrule
twhin-bert-base~\cite{zhang2023twhin} & 0.8588 & 0.8796 & 0.7743 & 0.8236 \\
BERT-base-uncased & 0.8785 & 0.8880 & \textbf{0.8608} & \textbf{0.8742} \\
E5-base & \textbf{0.8952} & \textbf{0.9068} & 0.8294 & 0.8664 \\
xlm-roberta-base & 0.8668 & 0.8893 & 0.7843 & 0.8335\\ \bottomrule
\end{tabular}
\end{table}
\textbf{Feasibility of the synthesizing stage across time.} Our analysis first examined the impact of temporal availability on synthesis quality. As shown in Figure~\ref{fig:usefulness_variation_by_time}, the proportion of useful commenotes increased significantly as the comment collection window expanded. Synthesizing comments from the first hour yielded useful notes in 73.1\% of cases, which rose to 85.8\% when using comments from the first two hours. Performance began to plateau thereafter. The proportion of useful notes at five hours (90.3\%) was not statistically different from the two-hour condition ($p=.2584$). This saturation suggests that the most crucial fact-checking information is typically contributed within the first two hours of a post's lifecycle, with further delays providing only marginal gains in synthesis quality.
A corresponding trend was observed for information volume, as depicted in Figure~\ref{fig:usefulness_variation_by_count}. To contextualize this finding, we note that on average, the first 15, 30, and 60 comments contained 2.09 (SD=2.15), 4.06 (SD=3.54), and 7.92 (SD=6.60) fact-checking comments, respectively. The LLM's synthesis performance scaled directly with the number of these comments provided, and plateaued thereafter. This indicates that while performance improves with more input, a moderate volume of community discussion is sufficient for the model to synthesize useful notes. Based on these findings, we selected a two-hour window for generating commenotes. This timeframe strikes an effective balance between timeliness and accuracy. It allows the system to capture the vast majority of essential corrective information to produce a highly reliable note (85.8\% accuracy) without the latency of waiting longer.
% We first showed the statistics of fact-check comments within comments. Out of 15, 30 and 60 comments, there are 2.09 (SD=2.15), 4.06 (SD=3.54), 7.92 (SD=6.60) identified fact-check comments respectively. As shown in Figure~\ref{fig:usefulness_variation_by_time}, the percentage of useful commenotes improves significantly with comments from a longer period. While using comments from the first hour yields a 73.1\% of useful notes, this percentage rises to 85.8\% within two hours. Notably, performance begins to plateau thereafter. The accuracy at five hours (90.3\%) is not statistically different from the two-hour condition ($p=.2584$). This saturation suggests that the most crucial fact-checking comments is typically contributed within the first two hours, and extending the data collection window further provides only marginal gains. A similar pattern, depicted in Figure~\ref{fig:usefulness_variation_by_count}, emerges when considering comment volume. The LLM's performance scales with the number of comments available, indicating that a critical mass of community discussion is a prerequisite for reliably distilling correct information.
% Comments with fact-check information will be gathered and used in the next phase. As is shown in Table~\ref{tab:performance}, 85.8\% posts has been able to extract useful fact-checking information from comments within two hours. To ensure the algorithm flow to be executed quickly and the result of algorithm flow to be with high quality, we choose to use comments made within the first two hours after a post was created.
\begin{figure}
\centering
\begin{subfigure}{0.49\textwidth}
\includegraphics[width=\textwidth]{Figure/variation_by_time.pdf}
\caption{}
\label{fig:usefulness_variation_by_time}
\end{subfigure}
\hfill
\begin{subfigure}{0.49\textwidth}
\includegraphics[width=\textwidth]{Figure/variation_by_count.pdf}
\caption{}
\label{fig:usefulness_variation_by_count}
\end{subfigure}
\caption{(a) Performance based on the time window from which comments were collected, measured since the post's creation. (b) Performance based on the number of initial comments used for synthesis (e.g., the first 15, 30, or 60 comments).}
\label{fig:usefulness}
\end{figure}
\subsubsection{Robustness of \proj{}}
% Our evaluation demonstrates the robustness of \proj{} in generating high-quality, helpful notes. \proj{} correctly identified 3 notes from the \textit{with note} group, 11 from the \textit{without display} group, and 26 from the \textit{without note} group as lacking a sufficient number of fact-checking comments to proceed, all with less than 25 comments, and 2, 7, 15 with even no comments. These were subsequently excluded from the user evaluation.
Our evaluation demonstrates the robustness of \proj{}, where \proj{} correctly identified that it could not proceed on 3 notes from the \textit{with note} group, 11 from the \textit{without display} group, and 26 from the \textit{without note} group, as all had fewer than 25 fact-checking comments (with 24 of these having no fact-checking comments at all). This crucial quality-control step ensured that only posts with a viable basis for synthesis were included in the subsequent user evaluation.
For those notes generated, user ratings indicated a high degree of helpfulness across all conditions, as illustrated in Figure~\ref{fig:helpfulness_stacked}. On aggregate, 89.2\% of commenotes in \textit{with notes} group, 91.2\% of the commenotes in \textit{without notes} group, and 92.0\% of the commenotes in \textit{without display} group received a rating of either \textit{helpful} or \textit{somewhat helpful}. The aggregated helpfulness scores reveal that the model performed equally well for those subject to community fact-checking (M=0.713) compared to the posts that were not (M=0.716), though the score was lower for those \textit{without display} (M=0.647), which perhaps was because those posts were inherently difficult for both note writers and AI. This suggests the robustness and generalizability of \proj{} to posts without community notes' coverage.
Among the tested models, grok-4 and claude-3.7-sonnet demonstrated superior performance. Grok-4 achieved the highest scores in the group \textit{without notes} (0.794) and \textit{without display} (0.694) conditions, while claude-3.7-sonnet was the top performer in the group \textit{with notes} (0.768). These results highlight the advanced capabilities of grok-4 in fact-checking.
\begin{figure}[!htbp]
\centering
\subfloat[Helpfulness scores.]{
\includegraphics[width=0.8\textwidth]{Figure/summary_helpfulness_grouped_stacked.png}
\label{fig:helpfulness_stacked}
}
\subfloat[Characteristics for notes in the with note group.]{
\includegraphics[width=0.8\textwidth]{Figure/dimension_score_with_note.png}
\label{fig:robust_displayed_note}
}
\subfloat[Characteristics for notes in the without display group.]{
\includegraphics[width=0.8\textwidth]{Figure/dimension_score_without_display.png}
\label{fig:robust_non_display}
}
\subfloat[Characteristics for notes in the without note group.]{
\includegraphics[width=0.8\textwidth]{Figure/dimension_score_without_note.png}
\label{fig:robust_no_note}
}
\caption{Technical evaluation of commenotes' quality across four LLMs. (a) Distribution of helpfulness ratings for notes generated by each model and in aggregate. (b-d) Boxplots showing scores for notes' characteristics, evaluated on the groups of (b) posts with (displayed) notes, (c) posts without displayed notes, and (d) posts without notes. The aggregated averaged results is shown in the group ``all models''.}
\end{figure}
A granular analysis of note characteristics across five dimensions reveals consistently high performance, demonstrating \proj{}'s robustness irrespective of whether it was covered by community notes. As shown across the group \textit{with notes}, (Figure~\ref{fig:robust_displayed_note}), \textit{without notes} (Figure~\ref{fig:robust_no_note}), or \textit{without display} (Figure~\ref{fig:robust_non_display}) conditions, the aggregated mean scores for all models surpassed 3.70 on most dimensions, affirming the effectiveness of synthesizing notes from comments.
While overall quality was high and stable across the three groups, the performance of individual models showed nuanced, context-dependent differences. In the group with notes, claude-3.7-sonnet emerged as the clear top performer, achieving the highest mean scores in four of the five dimensions, most notably in \textit{Clarity} (M=4.23, SD=0.84) and \textit{Quality} (M=4.13, SD=0.96). Conversely, in the \textit{without display} condition where no prior community notes existed, gpt-4o demonstrated exceptional performance, leading in dimensions such as \textit{Coverage} (M=4.14, SD=0.94). Interestingly, some models exhibited variable performance. For example, grok-4's score for \textit{Coverage} was notably lower in the \textit{without notes} condition (M=2.94, SD=1.42) compared to other models. These findings indicate that while the overall framework is effective, the optimal choice of an LLM may depend on the specific context of the content being evaluated.
% For a granular analysis of note characteristics, we assessed user scores across five quality dimensions, as shown in Figure~\ref{fig:grouped_boxplot}. The results indicate a strong overall performance, with the mean score for the aggregate of all models exceeding 3.60 on every dimension. Claude-3.7-Sonnet consistently achieved the highest average scores across all five dimensions, particularly excelling in providing context (M=4.20,SD=0.98) and maintaining source quality (M=4.19,SD=0.94). Grok-4 also performed well, especially in terms of clarity (M=3.92,SD=0.75). However, its score for impartiality (M=3.24,SD=1.10) was notably the lowest among all models, suggesting a potential area for improvement. The performance of gpt-4o and qwen-2.5 was largely comparable, with their mean scores for all dimensions falling between 3.25 and 3.79. This suggests that despite high general helpfulness, different models exhibit nuanced performance profiles. For example, grok-4's lower impartiality score indicates that models can vary on specific criteria, suggesting that the optimal model choice may be context-dependent.
\section{User Study: Evaluating \proj{} with $\mathbb{X}$'s Data} \label{sec:userstudy}
We conducted a study to evaluate whether the commenotes are useful to end users, especially compared with human-written notes (i.e., the original community notes).
% This dataset combines note data, post data, and reply data with temporal information, enabling analysis of comments before and after note display.
\subsection{Experiment Material Selection}
We utilized the dataset in Section~\ref{sec:dataset}. From this dataset, we selected 60 posts meeting four criteria: (1) having a displayed community note to serve as a comparative baseline (with 79.37\% of posts fulfilling this criterion), (2) accumulating sufficient comments ($\ge$ 30) within 2 hours for meaningful synthesis (with 67.24\% of posts fulfilling this criterion), (3) being in English to ensure consistent evaluation, and (4) representing diverse topics across Finance \& Business, Politics, Entertainment, and Science \& Technology categories (see Section~\ref{sec:Comments Topic}), to ensure broad coverage. Selected posts were processed through our \proj{} framework using the 2-hour comment window, based on our finding that 99.3\% of the posts contain debunking comments, and 85.8\% of the commenotes synthesized are rated useful. Two researchers independently verified that materials met the selection criteria and that commenotes are successfully synthesized, with failed synthesis attempts excluded to maintain sample integrity.
\subsection{Participants and Apparatus}
We conducted an a priori power analysis before the experiment. The analysis targeted the detection of the within-subjects effect, with a medium effect size (f=0.25), statistical power (1-$\beta$) of .80, and a significance level ($\alpha$) of .05. The result indicated that a minimum of 35 participants per each of the four between-subjects conditions was required, totaling 140 participants. We therefore recruited 144 participants (67 males, 77 females) with a mean age of 26.2 (SD=6.6) through distributing posters on different social media platforms. Participants were required to have an age of at least 18 years old, and were not required to have used $\mathbb{X}$ before. We crafted an experimental website using our institution's server, mimicking the interface of De et al.~\cite{de2025supernotes} (with the interface shown in Figure~\ref{fig:interface} of Appendix~\ref{app:interface}). This study got the approval of our institution's IRB, and each participant was compensated \$8.4 for their time. The study was conducted primarily with Chinese participants, therefore we provided both the original language and translated version of posts and notes, consistent with the settings in Section~\ref{sec:study_setup}.
\subsection{Study Design}\label{sec:design}
%We adopted a between-subjects design, with model architecture as a between-subjects factor. For each model architecture, we used a within-subjects design to compare AI notes and human-written notes. We chose four representative models with different architectures, sizes, and brands: gpt-4o, claude-3.7-sonnet, grok-4, and qwen2.5-72b-instruct. Grok-4 was specifically chosen as it may represent the foundational model for those agents used on $\mathbb{X}$.
We adopted a mixed factorial $2\times4$ design, \emph{Model architecture} (gpt-4o, claude-3.7-sonnet, grok-4, qwen2.5-72b-instruct) was \textbf{between-subjects} factor, where each participant was assigned to notes generated with one model. \emph{Note source} (AI-synthesized vs.\ human-written Community Note) was \textbf{within-subjects} factor. We selected models to span architectures, sizes, and brands, where grok-4 was specifically included for its relevance to agents on $\mathbb{X}$.
%For each group, we synthesized notes via the aforementioned method, and presented users with AI-generated and human-written notes in conjunction after displaying the original post. The two notes' orders are randomized across posts for each participant to balance the order effect. We assessed the following aspects according to prior practice~\cite{de2025supernotes} and guidelines\footnote{https://help.x.com/en/using-x/community-notes}:
Each participant needed to rate the notes for 20 posts. For each post, participants first read the original post, then viewed a \emph{pair} of notes, one commenote (Section~\ref{sec:ai-synth}) and one human-written community note (denoted human note in the results section), presented in randomized order. Note order was counterbalanced across posts within the participant. We then collected the following measures per post according to prior work~\cite{de2025supernotes} and official guidelines.\footnote{\url{https://help.x.com/en/using-x/community-notes}}
\textbf{Helpfulness~\cite{de2025supernotes}}: The helpfulness ratings adopted the original rating entry in $\mathbb{X}$, where participants could rate ``not helpful'', ``somewhat helpful'' or ``helpful''.
\textbf{Win Rate~\cite{de2025supernotes}}: Besides rating helpfulness, we also let participants rate whether an commenote or a human note is more helpful via a binary choice question, to further compare commenotes directly with human notes.
\textbf{Characteristics~\cite{de2025supernotes}}: We let participants to rate characteristics along five dimensions, which $\mathbb{X}$ originally adopted to as feedback to community notes. The five dimensions are (1) ``Content on note are high-quality and relevant'' (\textit{Quality})\footnote{We modified the original scale from ``Source on note is high-quality and relevant'' to ``Content on note is high-quality and relevant'', as commenotes could directly synthesize organic comments which did not feature links. This preserves the same construct, perceived information quality and relevance, without penalizing the content without links.}, (2) ``Note is written in clear language'' (\textit{Clarity}), (3) ``Note addresses all key claims in the post'' (\textit{Coverage}), (4) ``Note provides important context'' (\textit{Context}), (5) ``Note is NOT argumentative, speculative or biased'' (\textit{Impartiality}). All entries are rated on five-point scales from 1=Strongly disagree to 5=Strongly agree.
We also measured participants' demographics and included covariates for cross-partisan agreement on note effectiveness. Operational-wise, we used an ANES-style 7-point ideology item (1=very progressive/liberal, 4=moderate/neutral, 7=very conservative/traditional) \cite{wang2024lower} and two 0--100 feeling thermometer ratings toward the progressive/liberal and conservative/traditional camps to derive an affective-bridging score \(AP=|FT_{\mathrm{view1}}-FT_{\mathrm{view2}}|\) \cite{Civic_Broadcast_chi2017}. These variables were selected to test whether commenotes achieve cross-group helpfulness, an important validation criterion in $\mathbb{X}$'s Community Notes system, alongside the \textit{Helpfulness}, \textit{Win Rate}, \textit{Characteristics} measures.
\subsection{Procedure}
%Before the study, we briefed participants on the study's aim and content, including the potential risks. We let them signed the consent before proceeding. For the study, each participant is first instructed on the meaning of dimensions and about the context that there would be a number of posts and corresponding AI-generated or human-written notes. They needed to view each post, the corresponding notes and rated the helpfulness of these pairs. There are 20 posts in total for each participant, randomly selected from 60 items we selected, totaling 1440 rates per group, resulting in 12 rates per note. The order of presenting AI notes and human-written notes are randomized to avoid order effect, and participants were not told about some notes are written by AI, and some notes are written by human.
Before the study, participants were briefed on the study's aims, content, and potential risks. They then signed informed consent before proceeding. Each participant was instructed on the meaning of the evaluation dimensions and informed that they would be shown a set of posts with corresponding debunking information. We did not tell participants that some are synthesized and some are human-written notes to avoid bias. Participants viewed each post, read the paired notes, and rated corresponding entries per post. Each participant evaluated 20 posts, randomly selected from a pool of 60, yielding 1,440 ratings per group and 12 ratings per note. The selection algorithm ensured each note received the same number of ratings. The order of commenotes and human notes was randomized to avoid order effects. We debriefed participants after the study about the study's content.
\subsection{Data Analysis}
%We adopted quantitative analysis on the rating data. For the helpfulness ratings and note characteristics, we adopted Wilcoxon signed-rank test, and further run a linear mixed-effects regression to account for the fact that each participant rated notes on multiple posts and each post's notes were rated by multiple participants. For the win rate, we first used paired T test and then similarly adopted a linear mixed-effects regression. For further comparing argumentativeness of different groups, we adopted Mann Whitney U test for comparison. For the correlation analysis, we similarly adopted linear regression model to account for their effect. All post-hoc tests were performed with appropriate adjustments.
We conducted quantitative analyses of the rating data. Following common practices~\cite{de2025supernotes,xcommunitynotes}, ``not helpful'', ``somewhat helpful'' and ``helpful'' was mapped to 0, 0.5 and 1 to calculate helpfulness scores. For helpfulness ratings and note characteristics, we used the Wilcoxon signed-rank test, followed by linear mixed-effects regression to account for the nested structure of participants and posts. For win rate analysis, we first applied paired $t$-tests and then linear mixed-effects regression. To compare argumentativeness across groups, we used the Mann–Whitney U test. For analyses on the topic and user demographic factors' influence, we similarly employed Wilcoxon test, paired samples t-tests and linear regression models. All post-hoc tests were performed with appropriate adjustments.
\subsection{Results}
We first presented the helpfulness of the synthesized notes and the characteristics. We then presented an in-depth analysis of these notes in correlation with users' demographics, such as stance and polarization.
\subsubsection{Helpfulness Score}
Our analysis of helpfulness scores, as illustrated in Figure~\ref{fig:helpful}, reveals that notes synthesized by all evaluated LLMs were perceived as helpful, with mean scores consistently exceeding the neutral 0.5 threshold. In particular, the notes synthesized by grok-4 (M=0.78) and claude-3.7-sonnet (M=0.72) were rated most favorably, indicating a high degree of perceived utility. In comparison, \textit{human notes} achieved a lower average helpfulness score (M = 0.61). These results suggest that \textit{commenotes} are not only a viable alternative but also frequently outperform their human-written counterparts in terms of helpfulness.
% for claude-3.7-sonnet and grok, the helpfulness score even reached 0.719 and 0.782 respectively, suggesting that these AI notes are highly feasible. In comparison, the helpfulness score of human-written notes were around 0.6, where in many cases AI notes were comparable with human-written notes.
A model-specific breakdown confirms that notes synthesized through most models were rated as statistically significantly more helpful than \textit{human notes}. Grok-4 exhibited the most substantial advantage (M=0.78 vs. M=0.58, Wilcoxon signed-rank test $W = 30,217$, $p = .0000 < .001$, $r =.884$). Similarly significant preferences were observed for gpt-4o (M=0.70 vs. M=0.61, $W = 52,623$, $p < .001$, $r = .797$) and claude-3.7-sonnet (M=0.72 vs. M=0.64, $W = 52,982$, $p < .001$, $r = .796$). In contrast, notes synthesized by the qwen2.5-72b-instruct model (M=0.62) were rated comparably to \textit{human notes} (M=0.61), and this difference was not statistically significant ($W=60,521$, $p=.299$).
The distribution of user ratings further highlights a strong preference for \textit{commenotes}. An overwhelming majority of participants found \textit{commenotes} useful. For example, 96\% of ratings for notes from both grok-4 and claude-3.7-sonnet were ``at least somewhat helpful.'' Overall, more than 60\% of participants rated those \textit{commenotes} as ``helpful", which stands in stark contrast to the 26.9\% who rated \textit{human notes} at the same level. Even for the qwen2.5-72b-instruct model, approximately 90\% of its notes were deemed ``at least somewhat helpful'', a proportion comparable to that of \textit{human notes}. This widespread positive reception indicates that LLMs are highly capable of synthesizing notes that users overwhelmingly find valuable.
% Dissecting models, we found grok is the most prominent models, with all models outperforming AI notes regarding the average helpfulness score. Specifically, for gpt-4o, grok and claude, AI notes are rated significantly better than human-written notes, while for qwen model, AI notes are rated comparable against human-written notes.
% When dissecting into distributions, we found a large distribution of participants found AI notes helpful. This is most prominent for grok model, where 96\% of participants at least rated AI notes somewhat helpful. Among all participants, over 60\% participants rated AI notes helpful, which directly contrasts with the cases of AI notes, where only 26.9\% of the AI notes are rated helpful. Similarly, over 96\% of participants rated the notes generated by claude-3.7-sonnet as at least somewhat helpful, among which 47.6\% participants thought the notes as helpful. Even for those worst case, where qwen2.5 also has about 90\% participants rating the AI notes as at least somewhat helpful, which is comparable with the cases of human-written notes. This suggested that AI notes is highly capable of providing helpful information to participants, and was overwhelmingly preferred.
% For claude model, the analysis revealed a statistically significant difference in helpfulness scores, with AI-synthesized Notes being rated as more helpful than Community Notes ($W=52,982$,$p=.000019<.001$). The magnitude of this difference was large, as indicated by a rank-biserial correlation of $r=.796$. On average, AI-synthesized Notes received higher helpfulness ratings ($M=0.72$) compared to Community Notes ($M=0.64$). These results provide strong evidence that users in our study found the LLM-generated notes to be significantly more helpful than those created by the community.
% For gpt-4o model, the analysis revealed a statistically significant difference in helpfulness scores, with AI-synthesized Notes being rated as more helpful than Community Notes ($W=52,624$,$p=.000005<.001$). The magnitude of this difference was large, as indicated by a rank-biserial correlation of $r=.797$. On average, AI-synthesized Notes received higher helpfulness ratings ($M=0.70$) compared to Community Notes ($M=0.61$). These results provide strong evidence that users in our study found the LLM-generated notes to be significantly more helpful than those created by the community.
% For grok model, the analysis revealed a statistically significant difference in helpfulness scores, with AI-synthesized Notes being rated as more helpful than Community Notes ($W=30,218$,$p=.000000<.001$). The magnitude of this difference was large, as indicated by a rank-biserial correlation of $r=.884$. On average, AI-synthesized Notes received higher helpfulness ratings ($M=0.78$) compared to Community Notes ($M=0.58$). These results provide strong evidence that users in our study found the LLM-generated notes to be significantly more helpful than those created by the community.
% For qwen model, the analysis revealed a statistically significant difference in helpfulness scores, with AI-synthesized Notes being rated as more helpful than Community Notes ($W=60,522$,$p=.299242$). The magnitude of this difference was large, as indicated by a rank-biserial correlation of $r=.767$. On average, AI-synthesized Notes received higher helpfulness ratings ($M=0.62$) compared to Community Notes ($M=0.61$). These results provide strong evidence that users in our study found the LLM-generated notes to be significantly more helpful than those created by the community.
\begin{figure*}[!htbp]
\centering
\subfloat[gpt-4o.]{
\includegraphics[width=0.49\textwidth]{Figure/helpfulness_comparison_chart_gpt4o.png}
\label{fig:gpt4o_helpfulness_score}
}
\subfloat[grok-4.]{
\includegraphics[width=0.49\textwidth]{Figure/helpfulness_comparison_chart_grok.png}
\label{fig:grok_helpfulness_score}
}
\subfloat[gpt-4o.]{
\includegraphics[width=0.49\textwidth]{Figure/numerical_helpfulness_comparison_gpt4o.png}
\label{fig:gpt4o_helpfulness_distribution}
}
\subfloat[grok-4.]{
\includegraphics[width=0.49\textwidth]{Figure/numerical_helpfulness_comparison_grok.png}
\label{fig:grok_helpfulness_distribution}
}
\subfloat[claude-3.7-sonnet.]{
\includegraphics[width=0.49\textwidth]{Figure/helpfulness_comparison_chart_claude.png}
\label{fig:claude_helpfulness_score}
}
\subfloat[qwen2.5-72b-instruct.]{
\includegraphics[width=0.49\textwidth]{Figure/helpfulness_comparison_chart_qwen.png}
\label{fig:qwen_helpfulness_score}
}
\subfloat[claude-3.7-sonnet.]{
\includegraphics[width=0.49\textwidth]{Figure/numerical_helpfulness_comparison_claude.png}
\label{fig:claude_helpfulness_distribution}
}
\subfloat[qwen2.5-72b-instruct.]{
\includegraphics[width=0.49\textwidth]{Figure/numerical_helpfulness_comparison_qwen.png}
\label{fig:qwen_helpfulness_distribution}
}
\caption{The helpfulness scores of commenotes across different model settings. (a), (b), (e), (f) showed the helpfulness scores of the notes synthesized with gpt-4o, grok-4, claude-3.7-sonnet, and qwen2.5-72b-instruct, respectively. (c), (d), (g), (h) showed the helpfulness distributions of commenotes synthesized with gpt-4o, grok-4, claude-3.7-sonnet, and qwen2.5-72b-instruct, respectively.}
\label{fig:helpful}
\end{figure*}
\subsubsection{Win Rates}
To evaluate the relative helpfulness of \textit{commenotes}, we conducted head-to-head comparisons against \textit{human notes}, with results summarized in Figure~\ref{fig:win_rate}. Our analysis reveals that notes synthesized by most models were significantly preferred over their corresponding \textit{human notes}. The notes from grok-4 exhibited the highest performance, chosen as more helpful in 70.1\% cases (95\% CI [66.7\%, 73.4\%], $p = .0000 <.001$). Similarly, notes from claude-3.7-sonnet and gpt-4o also achieved statistically significant win rates of 55.7\% (95\% CI [52.0\%, 59.3\%], $p=.0025 < .001$) and 53.9\% (95\% CI [50.2\%, 57.5\%], $p=.0403 < .05$), respectively. In contrast, the notes produced by qwen2.5-72b-instruct were chosen in 49.2\% of cases, a proportion not statistically different from the 50\% chance baseline (95\% CI [45.5\%, 52.8\%], $p=.6819$).
% We analyzed the win rates of different models' AI notes compared with human-written notes, as shown in Figure~\ref{fig:win_rate}. Overall, except those produced by qwen2.5-72b (see Figure~\ref{fig:qwen_winning_rate}), the AI notes produced by other models have a win rate of over 50\%. The AI notes produced by grok has the highest win rate at 70.1\%, while the notes produced by gpt-4o (see Figure~\ref{fig:gpt4o_winning_rate}) and claude-3.7 (see Figure~\ref{fig:claude_winning_rate}) also have win rates at over 53\%. This suggested that the notes produced by grok is capable of replacing AI notes.
% We further dissected and analyzed when would people found AI notes helpful. We specifically analyzed the cases where both AI notes and human-written notes are rated helpful, consistent with prior practices~\cite{de2025supernotes}. We found grok and claude reached a win rate of over 50\%, which suggests that human would prefer AI notes even if they rated both as helpful. Worth noting, the notes produced by other models were less helpful compared with human notes. Overall, the win rate of AI notes when both are rated helpful was lower than the overall win rate, suggesting that for those where AI notes and human notes are not both rated helpful, AI notes may have better usage. This suggested that the AI notes produced by many models could effectively act as human-written notes' complement, especially when human notes are not rated as helpful.
% For claude model, a one-sample proportion test was conducted to analyze the helpfulness win rate of AI note against Community Notes. Across 720 head-to-head comparisons, AI note was selected as the more helpful note in 55.7\% of cases, with a 95\% confidence interval of [52.0\%, 59.3\%]. This proportion was found to be statistically significantly greater than the null hypothesis of 50\% ($p=0.0025<.01$). When the analysis was limited to the 81 comparisons where both notes were rated as ``helpful,'' the AI-synthesized Note win rate was 51.9\% (95\% CI: [41.1\%, 62.4\%]), which was not statistically different from 50\%.
% For gpt-4o model, a one-sample proportion test was conducted to analyze the helpfulness win rate of AI note against Community Notes. Across 720 head-to-head comparisons, AI note was selected as the more helpful note in 53.9\% of cases, with a 95\% confidence interval of [50.2\%, 57.5\%]. This proportion was found to be statistically significantly greater than the null hypothesis of 50\% ($p=0.0403<.05$). When the analysis was limited to the 85 comparisons where both notes were rated as ``helpful,'' the AI-synthesized Note win rate was 42.4\% (95\% CI: [32.4\%, 53.0\%]), which was not statistically different from 50\%.
% For grok model, a one-sample proportion test was conducted to analyze the helpfulness win rate of AI note against Community Notes. Across 720 head-to-head comparisons, AI note was selected as the more helpful note in 70.1\% of cases, with a 95\% confidence interval of [66.7\%, 73.4\%]. This proportion was found to be statistically significantly greater than the null hypothesis of 50\% ($p=0.0000<.001$). When the analysis was limited to the 87 comparisons where both notes were rated as ``helpful,'' the AI-synthesized Note win rate was 62.1\% (95\% CI: [51.6\%, 71.5\%]), which was statistically higher than 50\%.
% For qwen model, a one-sample proportion test was conducted to analyze the helpfulness win rate of AI note against Community Notes. Across 720 head-to-head comparisons, AI note was selected as the more helpful note in 49.2\% of cases, with a 95\% confidence interval of [45.5\%, 52.8\%]. This proportion was found to be statistically significantly greater than the null hypothesis of 50\% ($p=0.6819$). When the analysis was limited to the 56 comparisons where both notes were rated as ``helpful,'' the AI-synthesized Note win rate was 41.1\% (95\% CI: [29.2\%, 54.1\%]), which was statistically lower than 50\%.
\begin{figure*}[!htbp]
\subfloat[gpt-4o.]{
\includegraphics[width=0.49\textwidth]{Figure/winning_rate_analysis_plot_gpt4o.png}
\label{fig:gpt4o_winning_rate}
}
\subfloat[grok-4.]{
\includegraphics[width=0.49\textwidth]{Figure/winning_rate_analysis_plot_grok.png}
\label{fig:grok_winning_rate}
}
\subfloat[claude-3.7-sonnet.]{
\includegraphics[width=0.49\textwidth]{Figure/winning_rate_analysis_plot_claude.png}
\label{fig:claude_winning_rate}
}
\subfloat[qwen2.5-72b-instruct.]{
\includegraphics[width=0.49\textwidth]{Figure/winning_rate_analysis_plot_qwen.png}
\label{fig:qwen_winning_rate}
}
\caption{The winning rate of the \textit{commenotes} compared with \textit{human notes} for different models: (a) gpt-4o, (b) grok-4, (c) claude-3.7-sonnet, (d) qwen2.5-72b-instruct.}
\label{fig:win_rate}
\end{figure*}
We further investigated user preference in scenarios where both the \textit{commenotes} and \textit{human notes} were deemed ``helpful,'' a methodology consistent with prior work~\cite{de2025supernotes}. In this more stringent condition, grok-4's notes continued to be significantly preferred, winning in 62.1\% of such cases (n=87, 95\% CI [51.6\%, 71.5\%], $p<.05$). The preference for claude-3.7-sonnet's notes was 51.9\% (n=81, 95\% CI [41.1\%, 62.4\%]), which was not statistically distinguishable from 50\%. Conversely, the win rates for notes from gpt-4o (42.4\%, n=85) and qwen2.5-72b-instruct (41.1\%, n=56) fell below 50\%, although neither was statistically different from the baseline. The general decrease in \textit{commenotes}' win rates in this subset analysis further suggests that the primary advantage of \textit{commenotes} may lie in their ability to serve as a high-quality complement, particularly in instances where \textit{human notes} are absent or rated as unhelpful.
\subsubsection{Note Characteristics}
A detailed analysis of users' ratings reveals that \textit{commenotes} frequently surpass or perform comparably to \textit{human notes} across several key dimensions, as shown in Figure~\ref{fig:characteristic}. The superiority of \textit{commenotes} was most pronounced in two dimensions, including whether ``Note addresses all key claims in the post'' (\textit{Coverage}) and whether ``Note is written in clear language'' (\textit{Clarity}). For these dimensions, notes from all tested models were rated significantly higher than their human-written counterparts. For example, ratings for \textit{Coverage} were consistently superior from grok-4 ($W=27,510$, $p < .001$, $r = .092$) to gpt-4o ($W = 51,256$, $p < .001$, $p = .171$). We attribute this to the inherent capability of LLMs to synthesize a broad spectrum of information from user comments, enabling comprehensive coverage of a post's key claims compared to human authors who may be less exhaustive. Furthermore, our prompting methodology, which imposes structural and linguistic constraints, likely contributed to the significantly higher clarity ratings observed across all models.
Among the models, grok-4 was consistently high-performing, with its notes rated significantly better than \textit{human notes} across all five evaluated dimensions: \textit{Quality} ($W=22,908$, $p=.0000<.001$, $r=.076$), \textit{Clarity} ($W=30,070$, $p=.0001<.001$, $r=.100$), \textit{Coverage} ($W=27,510$, $p=.0000<.001$, $r=.092$), \textit{Context} ($W=24,149$, $p=.0000<.001$, $r=.081$), and \textit{Impartiality} ($W=41,936$, $p=.0001<.001$, $r=.140$) (see Figure~\ref{fig:grok_feature_rating}). This superior performance, particularly in providing non-argumentative, unbiased, and contextually rich content, may stem from grok-4's potential specialization and fine-tuning for the $\mathbb{X}$ platform. Similarly, notes from claude-3.7-sonnet also demonstrated statistically significant improvements over human notes on all five dimensions, including \textit{Quality} ($W = 38,664$, $p = .0000 < .001$, $r = .129$, and \textit{Impartiality} ($W = 43,386$, $p = .0095 < .01$, $r = .145$).
% We found for many models and on many key dimensions for community notes, AI notes are rated higher than community notes (see Figure~\ref{fig:characteristic}). For other dimensions AI notes are also rated at least comparably with human-written notes. The most outstanding dimensions are ``Note addresses all key claims in the post'', and ``Note is written in clear language'', where for all models, the ratings of the AI notes were significantly higher than those for human-written notes. These dimensions are linked to the capability of LLM and comments, which suggested that those notes written by LLM through synthesizing comments are easier to cover broader aspects and broader coverage, which potentially could addresses all key claims in the post easier, as compared to their human counterparts where human needed to think out which point to address by themselves, therefore may suffer from incomprehensiveness. Besides, as we prompted LLMs to synthesize comments with restrictions, LLMs have a higher probability to write the notes in an easy-to-understand manner, using clear language and illustrations. Therefore, in these two dimensions, AI notes are rated higher than human notes.
% Surprisingly, for most of the models except qwen-2.5, participants rated the content of note as high-quality and relevant, suggesting that even for those synthesizing from comments, our algorithm could produce high-quality and relevant AI notes, guaranteeing the quality of governance.
% Besides, grok is still the most outperforming model (see Figure~\ref{fig:grok_feature_rating}), where for all dimensions, the AI notes are rated significantly higher than human-written notes. This included that the notes generated by grok is not argumentative, speculative or biased. It provides important context, addresses all key claims in the post, is written in clear language and the content on notes are also of high-quality and relevant. This is potentially because grok may officially act as the assistant for X, therefore may be specifically tailored to X's context, which is expert at synthesizing for fact-checking.
% Notably, although for some models such as gpt-4o (see Figure~\ref{fig:gpt4o_feature_rating}) and qwen (see Figure~\ref{fig:qwen_feature_rating}), the average ratings of the dimension ``note is NOT argumentative, speculative or biased'' is lower than human-written notes, there is no significant difference. These results suggest that in our algorithm, LLMs could be prompted to produce unbiased notes, which suggests its feasibility.
% For the claude model (see Figure~\ref{fig:claude_feature_rating}), a Wilcoxon signed-rank test was conducted to compare the ratings of LLM-generated notes (AI notes) and human-written community notes across five qualitative dimensions. The results indicate that AI notes were rated significantly higher than Community Notes on all five dimensions: Source Quality ($W=38,664$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.129$); Clarity ($W=30,666$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.102$); Scope ($W=37,718$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.126$); Context ($W=50,614$, $p=.0004<0.001$, $n=720$, Cohen's $r=0.169$); and Neutrality ($W=43,386$, $p=.0095<0.01$, $n=720$, Cohen's $r=0.145$). While the effect sizes are small across all dimensions, these findings suggest that LLM-generated notes are perceived as being of higher quality overall.
% For the gpt-4o model, a Wilcoxon signed-rank test was conducted to compare the ratings of LLM-generated notes (AI notes) and human-written community notes across five qualitative dimensions. The results indicate that AI notes were rated significantly higher than Community Notes on all five dimensions: Source Quality ($W=44,218$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.148$); Clarity ($W=37,066$, $p=.0001<0.001$, $n=720$, Cohen's $r=0.124$); Scope ($W=51,256$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.171$); Context ($W=61,664$, $p=.0600$, $n=720$, Cohen's $r=0.206$); and Neutrality ($W=54,034$, $p=.8956$, $n=720$, Cohen's $r=0.180$). While the effect sizes are small across all dimensions, these findings suggest that LLM-generated notes are perceived as being of higher quality overall.
% For the grok model, a Wilcoxon signed-rank test was conducted to compare the ratings of LLM-generated notes (AI notes) and human-written community notes across five qualitative dimensions. The results indicate that AI notes were rated significantly higher than Community Notes on all five dimensions: Source Quality ($W=22,908$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.076$); Clarity ($W=30,070$, $p=.0001<0.001$, $n=720$, Cohen's $r=0.100$); Scope ($W=27,510$, $p=.0000<0.001$, $n=720$, Cohen's $r=0.092$); Context ($W=24,149$, $p=.0000<.001$, $n=720$, Cohen's $r=0.081$); and Neutrality ($W=41,936$, $p=.0001<.001$, $n=720$, Cohen's $r=0.140$). While the effect sizes are small across all dimensions, these findings suggest that LLM-generated notes are perceived as being of higher quality overall.
% For the qwen model, a Wilcoxon signed-rank test was conducted to compare the ratings of LLM-generated notes (AI notes) and human-written community notes across five qualitative dimensions. The results indicate that AI notes were rated significantly higher than Community Notes on all five dimensions: Source Quality ($W=61,122$, $p=.8081$, $n=720$, Cohen's $r=0.204$); Clarity ($W=39,429$, $p=.0015<0.01$, $n=720$, Cohen's $r=0.132$); Scope ($W=51,018$, $p=.0003<0.001$, $n=720$, Cohen's $r=0.170$); Context ($W=59,514$, $p=.7743$, $n=720$, Cohen's $r=0.199$); and Neutrality ($W=45,753$, $p=.4122$, $n=720$, Cohen's $r=0.153$). While the effect sizes are small across all dimensions, these findings suggest that LLM-generated notes are perceived as being of higher quality overall.
\begin{figure*}[!htbp]
\subfloat[gpt-4o]{
\includegraphics[width=0.49\textwidth]{Figure/feature_rating_diverging_chart_gpt4o.png}
\label{fig:gpt4o_feature_rating}
}
\subfloat[grok-4.]{
\includegraphics[width=0.49\textwidth]{Figure/feature_rating_diverging_chart_grok.png}
\label{fig:grok_feature_rating}
}
\subfloat[claude-3.7-sonnet.]{
\includegraphics[width=0.49\textwidth]{Figure/feature_rating_diverging_chart_claude.png}
\label{fig:claude_feature_rating}
}
\subfloat[qwen2.5-72b-instruct.]{
\includegraphics[width=0.49\textwidth]{Figure/feature_rating_diverging_chart_qwen.png}
\label{fig:qwen_feature_rating}
}
\caption{The \textit{commenotes}' characteristics for different models: (a) gpt-4o, (b) grok-4, (c) claude-3.7-sonnet, (d) qwen2.5-72b-instruct.}
\label{fig:characteristic}
\end{figure*}
The performance of other models was more varied. While gpt-4o's notes were rated significantly higher for \textit{Quality} ($W=44,218$, $p=.0000<0.001$, $r=0.148$), \textit{Clarity} ($W=37,066$, $p=.0001<0.001$, $r=0.124$) and \textit{Coverage} ($W=51,256$, $p=.0000<0.001$, $r=0.171$), they were not rated significantly different from \textit{human notes} on \textit{Context} ($p = .0600$) and \textit{Impartiality} ($p = .8956$). A similar pattern was observed for qwen2.5-72b-instruct, which excelled in \textit{Clarity} ($W=39,429$, $p=.0015<0.01$, $r=0.132$) and \textit{Coverage} ($W=51,018$, $p=.0003<0.001$, $r=0.170$) but showed no significant difference in ratings for \textit{Quality} ($p = .8081$), \textit{Context} ($p = .7743$) or \textit{Impartiality} ($p = .4122$), Notably, even when average ratings for \textit{Impartiality} were lower than \textit{human notes} for some models, the lack of a statistically significant difference suggests that our algorithm's prompting strategies were effective in guiding the LLMs to produce content that is perceived as comparably unbiased, affirming the feasibility of this approach.
\subsubsection{Correlation with Posts' Topics and User Demographics}
A key finding from our analysis is that posts' topic did not have a statistically significant effect on user perceptions (see Figure~\ref{fig:grok_effect} and Appendix~\ref{app:effect_factors}). Across all evaluated dimensions, the topic of the post (\textit{Finance \& Business}, \textit{Politics}, \textit{Entertainment}, or \textit{Science \& Technology}) did not have significant effects on the ratings, suggesting the robustness of \proj{}, across different models.
Specifically, for the helpfulness dimension, no topic showed a significant influence, with results ranging from \textit{Science \& Technology} ($\beta=-0.002$, $z = -0.062$, $p = .950$) to \textit{Finance \& Business} ($\beta = 0.028$, $z = 1.517$, $p = .129$). A similar pattern of non-significance was observed for ratings of \textit{Clarity}, where the coefficients for all topics were close to zero and p-values were non-significant (e.g., \textit{Politics}: $\beta = 0.006$, $z = 0.143$, $p = .887$, \textit{Entertainment}: $\beta = -0.064$, $z = -1.075$, $p = .282$).
\begin{figure}[!htbp]
\includegraphics[width=1\textwidth]{Figure/consolidated_plot_GROK.png}
\caption{The effect of different topics on different aspects of ratings for the commenotes synthesized by grok-4, with error bar showing 95\% confidence interval (the other models' results were shown in Appendix~\ref{app:effect_factors}).}
\label{fig:grok_effect}
\end{figure}
This trend continued across the other four metrics. For \textit{Context}, \textit{Coverage}, \textit{Quality}, and \textit{Impartiality}, none of the topic categories reached statistical significance. For instance, in the assessment of \textit{Impartiality}, the p-values for topics ranged from 0.190 (\textit{Finance \& Business}) to 0.873 (\textit{Science \& Technology}). Likewise, for \textit{Quality}, the p-values were consistently non-significant, ranging from 0.117 (\textit{Politics}) to 0.804 (\textit{Entertainment}). The absence of a main effect for topic indicates that the factors influencing the perceived quality of notes generalize across subject domains, as their effectiveness was not contingent on the specific subject matter.
% . This suggests that participants' evaluations were not contingent on the specific subject matter being discussed.
We further analyzed whether the perceived helpfulness of \textit{commenotes} varied across user demographics. While we found a consistent preference for \textit{commenotes}, the magnitude of this preference was significantly moderated by users' political stance, though not by political polarization. Furthermore, the influence of these factors varied across LLM settings. For political stance, overall we found a consistent and statistically significant preference for \textit{commenotes}. Paired samples t-tests showed this effect was most pronounced for participants identifying as \textit{Left/Progressive} (mean difference = 0.114, $p = .0000 < .001$, Cohen's d = .232), followed by those identifying as \textit{Neutral} (mean difference = 0.089, $p = .0006 < .001$, Cohen's d = .207). The preference was weakest, though still significant, among \textit{Right/Conversative} participants (mean difference = 0.043, $p = .031 < .05$, Cohen's d = .091). A Kruskal-Wallis test confirmed that the distribution of these preference scores differed significantly across the three groups ($H = 9.213$, $p = .010 < .05$).
To account for participant- and item-level variance, we adopted a linear mixed-effects model, which confirmed a significant interaction between note type (\textit{human notes} vs. \textit{commenotes}) and political stance. Using the \textit{Left/Progressive} group as the baseline, the model revealed a significant negative interaction for the \textit{Right/Conservative} group ($\beta = -0.072$, $p = .0005 < .001$), indicating the helpfulness advantage of \textit{commenotes} was significantly diminished for right-leaning participants. The interaction for the \textit{Neutral} group was not significant ($\beta = -0.016$, $p = .555$). At each model's level, this sensitivity was most apparent for gpt-4o and grok-4. \textit{Left/Progressive} participants rated gpt-4o significantly higher on \textit{Quality} (coef. = 0.635, $p < .05$) and \textit{Clarity} (coef. = 0.600, $p < .05$). In contrast, both \textit{Left/Progressive} and \textit{Right/Conservative} participants rated grok-4's \textit{Quality} significantly lower (coef. = -0.682, $p < .001$ and coef. = -0.673, $p < .01$ respectively).
For political polarization, we found through paired samples t-tests that participants with both low-to-medium polarization (mean difference = 0.088, $p = .0000 < .001$, Cohen's d = .180) and high polarization (mean difference = 0.111, $p = .0000 < .001$, Cohen's d = .230) rated \textit{commenotes} as significantly more helpful. A more robust linear mixed-effects model revealed that the interaction between note type and polarization level was not statistically significant ($\beta = 0.023$, $p = .171$). This suggests that political polarization does not significantly moderate the overall perceived helpfulness of synthesized notes compared to human notes. Nonetheless, polarization did influence perceptions of specific models. For claude-3.7-sonnet, highly polarized user gave significantly higher \textit{Clarity} ratings (coef. = 0.380, $p < .01$). Conversely, for qwen2.5-72b-instruct, highly polarized participants rated the notes as providing significantly less \textit{Context} (coef. = -0.371, $p < .01$). For both of these models, a user's specific political stance (Left/Right/Neutral) was not a significant factor.
% We first analyzed the moderating effect of political stance. Overall, we found through a paired samples t-test, a consistent and statistically significant preference for AI notes over human-written community notes across all political groups. However, the magnitude of this preference varied. The effect was most pronounced for participants identifying as Left/Progressive (mean difference = 0.114, $p < .0001$, Cohen's d = .232), followed by those who identified as Neutral (mean difference = .098, $p = .0006 < .001$, Cohen's d = 0.207). The preference was weakest, though still statistically significant, among participants identifying as Right/Conservative (mean difference = 0.043, $p = .031 < .05$, Cohen's d = .091). A Kruskal-Wallis test confirmed that the distribution of these preference scores differed significantly across the three political groups ($H = 9.213$, $p = .010 < .05$).
% To further investigate this relationship while accounting for participant- and item-level variance, we employed a linear mixed-effects model with interaction terms. The model confirmed a significant interaction between the note type (AI vs. human) and political stance. Using the Left/Progressive group as the baseline, the model revealed a significant negative interaction for the Right/Conservative group ($\beta=-0.072$, $p = .0005 < .001$). This indicates that the helpfulness advantage of AI notes over human-written notes was significantly diminished for right-leaning participants compared to their left-leaning counterparts. The interaction effect for the Neutral group was not statistically significant ($\beta=-0.016$, $p = .555$).
% We also examined whether the degree of a user's political polarization influences their ratings. We showed through paired samples t-tests that participants with both low and medium polarization (mean difference = 0.088, $p < .0001$, Cohen's d = .180) and high polarization (mean difference = 0.111, $p < .0001$, Cohen's d = .230) rated AI notes as significantly more helpful.
% However, when this was tested in a more robust linear mixed-effects model, the interaction between note type and polarization level was not statistically significant ($\beta = 0.023$, $p = .171$). This suggests that while there may be a trend towards a strong preference for AI notes among highly polarized individuals, we did not find conclusive statistical evidence that polarization significantly moderates the perceived helpfulness of AI notes in our study.
\begin{figure}[!htbp]
\includegraphics[width=\textwidth]{Figure/plot_GROK.png}
\caption{The effect of user demographics on different aspects of ratings for the notes synthesized by the grok-4 model, with errorbar showing 95\% confidence interval (the other models' results were shown in Appendix~\ref{app:demographic}).}
\label{fig:grok_demographic}
\end{figure}
% Our analysis reveals that, while an aggregated analysis across all models shows no statistically significant effect, gpt-4o and grok-4 were more sensitive to users' political stances, while claude-3.7-sonnet and qwen2.5-72b-instruct were more influenced by users' polarization. The influence of political stance was most evident in the evaluation of gpt-4o and grok-4, though in opposite directions. For gpt-4o, participant with a \textit{Left/Progressive} stance provided significantly higher ratings for \textit{source quality} (Coef. = 0.635, $p < .05$) and \textit{clarity} (Coef. = 0.600, $p < .05$) compared to the neutral baseline. In contrast, for grok, both \textit{Left/Progressive} and \textit{Right/Conservative} participants rated its source quality significantly lower (Coef. = -0.682, $p < .001$ and Coef. = -0.673, $p < .01$ respectively). This negative perception from \textit{Left/Progressive} users also extended to helpfulness, which was rated significantly lower (Coef. = -0.145, $p < .05$).
% Conversely, user polarization was the key factor for claude-3.7-sonnet and qwen2.5-72b-instruct. For claude-3.7-sonnet, high polarization was associated with more favorable ratings, with highly polarized users rating notes as significantly higher in clarity (Coef. = 0.380, $p < .01$). For qwen2.5-72b-instruct, however, the effect was negative. Highly polarized participants rated the notes as providing significantly less context (Coef. = -0.371, $p < .01$). In both of these models, a users' specific political stance did not emerge as a significant factor.
\section{Discussions}
\subsection{Feasibility of Commenotes}
\textbf{Timeliness of organic debunking comments.} The current implementation of Community Notes system on $\mathbb{X}$ faces significant delays in generating helpful annotations to reduce the spread of misleading posts at the early stage~\cite{chuai2024did,chuai2024community}. For instance, it takes, on average 24.29 hours (median=9.69 hours) for contributors to propose community notes based on our collected data. Additionally, community notes are updated on an hourly cycle and typically become visible only after a median delay of 22 hours~\cite{chuai2024community}. That is, the delays originate from the system's two inherited design mechanisms: (i) the delay from post creation to note creation, which is driven by the selection and responsiveness of contributors; and (ii) the delay from note creation to note display, i.e., the evaluation process of community notes. It has been explored to synthesize existing human-written community notes and generate ``Supernotes''~\cite{de2025supernotes}, which can foster consensus among the contributors and reduce the delay from note creation to note display. Complementary to this research, our studies focus on reducing the delay from post creation to note creation and shows that fact-checking comments appear rapidly after a post's publication, with 73.1\% of posts receiving such comments within the first hour. Our findings highlight that organic user comments can provide correction information much earlier than the platform's current Community Notes mechanisms, thereby contributing to community-based fact-checking.
\textbf{Quality and accuracy of synthesized notes.} Our evaluations demonstrate the feasibility of $\proj{}$ (Figures~\ref{fig:helpful} and ~\ref{fig:characteristic}). Specifically, grok-4 emerged as the most effective and robust model across different post topics and user demographics (Figures~\ref{fig:grok_effect} and ~\ref{fig:grok_demographic}). Despite the overall effectiveness, we observed nuanced variance in capability across models. For instance, the lower performance of qwen2.5-72b-instruct may be attributable to its smaller size, a finding that aligns with established scaling laws~\cite{kaplan2020scaling}. Furthermore, models exhibited different degrees of bias. Consistent with prior work indicating the bias of gpt-4o~\cite{hu2025generative}, we found both gpt-4o and qwen2.5-72b-instruct were less impartial than grok-4 and claude-3.7-sonnet. Overall, our findings extend previous research~\cite{de2025supernotes} by showing that LLMs can synthesize unbiased content from diverse comments.
Besides helpfulness, \proj{} must also account for false positives and false negatives. With enough posts, we did not identify false negatives for \proj{}, where misleading posts are erroneously identified not needing to fact-check. Regarding false positives, through manual examination, our algorithm would reject to synthesize by output ``could not synthesize'' for those posts that might not need fact-checking. However, we did not test the probability of false positives, which could be evident if deployed in a large scale. Research undertaken to date suggests that even in the event of high overall accuracy, false positives have the potential to significantly reduce user trust in authentic news sources~\cite{horne2025people}. Regarding these risks, potential mitigation mechanisms could include human-assisted verification~\cite{li2025scaling}, uncertainty prompts~\cite{zhou2023navigating}, or cross-validation of evidence from multiple sources~\cite{li2025scaling}.
\textbf{Low cost of synthesizing processes.} From an economic perspective, community notes depend on collective labor and sustained contributor effort, while commenotes scale primarily with computational resources such as inference time and API calls, which exhibit much lower marginal cost~\cite{li2025scaling}. For instance, by processing comments posted within the first two hours, \proj{} costs a mere average cost of \$0.0065 with gpt-4.1 for each post (input 2858.5 tokens, output 94.5 tokens on average), and \$0.01 with grok-4, underscoring its financial feasibility for large-scale deployment. Notably, this approach is especially well-suited for platforms that already deploy LLM, such as $\mathbb{X}$ with its grok-4.
\textbf{Bias and diversity of the comments and synthesized notes.} Another challenge lies in the potential amplification of bias during synthesis. Prior research showed user contributions are often unevenly distributed, risking that algorithmic summarization may privilege dominant perspectives and neglect minority or dissenting voices~\cite{zhang2017wikum,dash2019summarizing}, leading to the overlook of important alternative viewpoints. This risk is compounded by the fact that AI models themselves can introduce and perpetuate biases from their training data~\cite{ntoutsi2020bias}.
Our algorithm was designed to directly address this by (1) randomly selecting a representative sample of notes for synthesis, and (2) explicitly prompted the LLMs to produce unbiased, neutral results, guiding the model to synthesize content that accurately reflects the full spectrum of comments. Our empirical analysis provides strong support for this design. Participants consistently rated commenotes as comparable or even superior to community notes in the \textit{Impartiality} dimension (Figure~\ref{fig:characteristic}), indicating our method successfully guide LLMs to produce unbiased notes. Furthermore, our analysis of note effectiveness and characteristics found no significant interaction effect between the topic or user demographics (Figures~\ref{fig:grok_effect} and ~\ref{fig:grok_demographic}), indicating that \proj{} is robust to user's background or the specific post topic.
% Prior work on online deliberation and algorithmic mediation has shown that user contributions are often unevenly distributed across demographics or viewpoints, and algorithmic summarization may privilege majority perspectives while overlooking minority voices~\cite{zhang2017wikum}. LLM-based synthesis could exacerbate this imbalance by overemphasizing dominant narratives and obscuring dissenting or less frequent perspectives~\cite{dash2018fairsumm}, risking an artificial ``consensus'' that neglects important alternatives~\cite{zhang2024fairabstractive} .
% Our algorithm meticulously balanced the viewpoints by randomly selecting notes, and prompted the LLMs to care about producing unbiased results. Our empirical analysis also supported this point, where participants consistently rated synthesized notes as comparable or even superior than human notes in the \textit{Neurality} dimension (Figure~\ref{fig:characteristic}). Our analysis also found no interaction effect between topic and user demographics, and note effectiveness, indicating our algorithm's robustness (Figures~\ref{fig:grok_effect} and~\ref{fig:grok_demographic}).
% However, our empirical analyses suggest that this risk did not materialize in our setting. As shown in the user study, participants consistently rated synthesized notes as comparable or superior to human-written notes across all five qualitative dimensions, with no significant disadvantage on neutrality (Figure~\ref{fig:characteristic}). Further, our analysis (Figures~\ref{fig:grok_effect} and~\ref{fig:grok_demographic}) found no interaction effect between topics, popularity or user demographics, and note effectiveness, indicating that synthesized notes maintained robustness across different contexts. Together, these results suggest that although bias remains a theoretical concern, in practice, the synthesis process produced notes that were broadly balanced and trustworthy.
\subsection{Generalizability}
\proj{}'s generalizability is rooted in its method of synthesizing user-generated evidence, which is user-behavior-centric rather than platform-dependent. First, this approach allows for generalizability in its deployment model. The demonstrated helpfulness of commenotes (Figure~\ref{fig:helpful}), coupled with their automated generation process, facilitates a flexible hybrid model that complements rather than replaces Community Notes~\cite{li2025scaling}. In this model, synthesized notes can be strategically deployed to expand fact-checking coverage to posts that currently lack human-written contributions. For high-engagement or viral content, they can also augment human efforts by generating multiple notes to capture diverse perspectives, mirroring the functionality of the existing Community Notes system~\cite{xcommunitynotes}. However, such scalability necessitates robust governance mechanisms to mitigate information overload and avoid amplifying particular narratives~\cite{zhang2024fairabstractive}.
% At the same time, we underscore the potential to use synthesized notes to complement human-written notes but not to replace them. This strategy allows for flexible deployment: synthesized notes can be prioritized for posts that lack human-contributed notes, thereby expanding fact-checking coverage. For high-traffic or viral posts, they can also supplement human moderators by generating multiple notes to capture diverse perspectives, mirroring the functionality of the existing Community Notes system~\cite{xcommunitynotes}. However, such an approach necessitates robust governance to prevent information overload and avoid privileging certain narratives~\cite{zhang2024fairabstractive}. Ultimately, this hybrid model balances the efficiency of AI with the nuance of human judgment, offering a sustainable and cost-effective path to scaling community-based fact-checking~\cite{techxplore2025_hybrid}.
Besides, \proj{} could be adapted to platforms that feature analogous, structured contribution systems, including complementing Meta's~\cite{meta_community_notes} and YouTube Shorts'~\cite{youtube_crowd_notes} Community Notes, and TikTok's footnotes~\cite{tiktok_footnotes}. This transferability is supported by prior research demonstrating that user comments can serve as effective features for misinformation detection~\cite{bu2023combating}, and fact-checking across platforms like Facebook, Twitter, and YouTube~\cite{jiang2018linguistic}. The methodology could also potentially be extended to short video platforms with similar comment structures, such as Bilibili~\cite{bilibili_wiki} or Instagram~\cite{instagram_meta_notes}. However, such an adaptation would first require a thorough analysis to determine if comments on those platforms consistently contain sufficient high-quality information for fact-checking.
Another key dimension for generalization involves extending our text-focused system to handle multimodal content. Applying \proj{} to media-rich posts and comments that include images or videos may necessitate significant architectural enhancements. Specifically, this may require integrating Vision-Language Models to effectively process and synthesize evidence from these varied formats~\cite{qi2024sniffer, yue2024evidence}.
% The generalizability of \proj{} stems from the methodology of synthesizing user-generated evidence for fact-checking, which is tied to user behaviors, but independent of platforms, as prior work also adopted comments as features for misinformation videos detection~\cite{bu2023combating}, and Jiang et al.~\cite{jiang2018linguistic} found linguistic features may be useful for fact-checking across Facebook, Twitter and YouTube platforms.
% Therefore, \proj{} could be transferred to platforms with analogous structured contribution systems, such as Meta~\cite{meta_community_notes} and YouTube Short's~\cite{youtube_crowd_notes} Community Notes, and Tiktok's footnotes~\cite{tiktok_footnotes}. The algorithm may could also be adapted for platforms with unstructured comment sections such as Bilibili~\cite{bilibili_wiki} or Instagram~\cite{instagram_meta_notes}, however more analysis is needed on whether their comments contain enough fact-check information.
% Second, a critical issue is to transfer to media posts and comments on $\mathbb{X}$, where we currently take in primarily text-based posts and comments. This may require the integration of Vision-Language Models~\cite{qi2024sniffer,yue2024evidence} for understanding the comments and posts, and also may require the transcription of these comments and posts to fit in \proj{}'s architecture.
% through synthesizing user-centric comments and demonstrating this additional context as a form of user-centric intervention~\cite{de2025supernotes}, because the underlying comment structure and ecosystem are similar.
% This paper focused on $\mathbb{X}$ as it is the earliest platform that feature community notes, and has open-sourced the dataset of community notes which could be used publicly by researchers~\cite{chuai2024communitynotes}. This provided the foundation for us to analyze, synthesize, and compare, similar to prior work~\cite{allen2024}. However, the rationale of synthesizing user-contributed comments into reliable fact-check evidence~\cite{de2025supernotes} could far extend $\mathbb{X}$ to other platforms featuring community notes, such as Meta~\cite{meta_community_notes}, TikTok~\cite {tiktok_footnotes}, and YouTube Shorts~\cite{youtube_crowd_notes}. They were also expected to be effective even if the platform currently did not feature community notes, such as Bilibili~\cite{bilibili_wiki} or Instagram~\cite{instagram_meta_notes}, through synthesizing user-centric comments and demonstrating this additional context as a form of user-centric intervention~\cite{de2025supernotes}.
% \subsection{Robustness Against Harmful Actions}
% A key concern for comment-based synthesis is its robustness against harmful comments, which may results in both intentional attacks and unintentional consequences. We discuss these two main types of risks and envision corresponding potential mitigation strategies.
% \textbf{Intentional attacks.} Malicious actors may attempt to bias or derail synthesis through adversarial tactics such as prompt injection~\cite{liu2024_prompt_injection} or coordinated flooding of misleading comments~\cite{tucker2023_online_manipulation}. Although the rapid emergence of fact-check comments may make manipulation more difficult in practice, the vulnerability of synthesis to adversarial coordination remains an open concern. Mitigation could involve integrating filtering modules that detect abnormal comment patterns~\cite{goldblum2020_data_poisoning}, balancing strategies that prevent domination by a single viewpoint~\cite{pelekis2025_adversarial_review}, and hybrid workflows where AI-generated notes serve only as drafts later refined by human contributors~\cite{li2025scaling}.
% \textbf{Unintentional harms.} Even without malicious intent, synthesis can be compromised if inaccurate but confidently stated comments dominate early discussions, inadvertently poisoning the generated notes. This risk highlights the need for mechanisms that weigh comments not only by volume or timing but also by credibility. Possible safeguards include prioritizing comments from historically reliable contributors, requiring cross-verification across diverse users as Community Notes system do, and embedding AI synthesis within broader collaborative fact-checking ecosystems rather than using it as a standalone replacement~\cite{de2025supernotes}.
% Overall, robustness should be understood not only as a technical property but also as an ecosystem issue. Future work will need to examine these risks more systematically and explore how combined technical filters, governance mechanisms, and hybrid human–AI designs can strengthen resilience against both intentional and unintentional distortions.
\subsection{Broader Applications and Implications}
Our findings suggest that \proj{} have several broader implications for platform governance and the design of socio-technical systems. We structure these implications around three interconnected themes: augmenting existing fact-checking workflows, generalizing the synthesis framework to other user-generated signals, and rethinking the role of community signals in real-time content moderation.
\textbf{Augmenting existing fact-checking flows:} Beyond serving directly as Community Notes, the notes synthesized by \proj{} can be integrated into community-based fact-checking pipelines, such as claim detection, evidence retrieval, verdict decision and communication~\cite{warren2025show}. Synthesized notes, generated rapidly from early user comments, could serve as preliminary drafts or sources of evidence for human note writers~\cite{li2025scaling}, thereby accelerating their note creation process.
\textbf{Generalizing the synthesis framework to other user-generated signals:} The two-stage framework of \proj{} could be adapted to harness other forms of implicit and explicit user-generated corrective signals, such as danmu~\cite{he2017exploring} (``bullet screen'') on Bilibili platform.
\textbf{Rethinking the role of community signals in real-time content moderation:} The finding that fact-checking information emerges within the first two hours provide a powerful temporal signal for intervention. Platform could integrate this signal into their governance models~\cite{shim2024incorporating}, complementing existing user-flagging mechanisms~\cite{zhang2023_flagging}.
% An even broader opportunity lies in extending synthesis beyond textual comments. On short-video platforms such as TikTok or YouTube Shorts, users often respond with screenshots, stitched videos, or image-based memes that reference external evidence. Instagram and Bilibili similarly allow image replies that serve as visual proof or contextual clarification. Incorporating these multimodal signals could enrich fact-checking notes. Images can illustrate miscaptioned scenes, screenshots may point to authoritative sources, and video reactions can provide context that text alone cannot. The key challenge is how to synthesize heterogeneous evidence at scale. Currently \proj{} is optimized for text, and extending it to multimodal settings would require fusion models~\cite{qi2024sniffer} or cross-modal retrieval pipelines~\cite{yue2024evidence}, which remain an open gap. Our two-stage approach, which first filters relevant evidence and then synthesizes it, could in principle be adapted to multimodal data, but it would require integration with visual–language models and domain-specific annotation standards.
% Beyond user-facing interventions, platforms may also use comments to support decisions on harmful content, similar to existing flagging mechanisms~\cite{zhang2023_flagging}. This approach could provide a natural and informative layer of platform governance~\cite{shim2024_procedural_fairness}. However, such signals may also be vulnerable to manipulation if adversarial actors strategically distort comment patterns. We return to this issue in the next section on robustness.
% \subsection{Limitations}
% We acknowledge several limitations in this study. First, our dataset was primarily drawn from community notes in 2023–2024, a period when contributors were relatively few and the average delay in note publication was longer than in the current ecosystem~\cite{de2025supernotes}. This may limit representativeness, but also provide a useful baseline for testing AI synthesis in low-resource settings. Second, our evaluation was based on English-language textual comments and a limited participant pool~\cite{chuai2025community}, meaning that linguistic, cultural, and demographic diversity was not fully captured. Extending to multilingual and cross-cultural contexts, as well as to multimodal inputs such as images, video replies, or transcribed speech, would better reflect real-world misinformation environments~\cite{shaar2023multilingual}. Third, our pipeline capped the number of comments at 500 per post and remained sensitive to prompt design and context length~\cite{ribeiro2020checklist,brown2020gpt3}, leaving deeper threads underexplored. In addition, we did not systematically benchmark against the full range of alternative interventions such as warning labels, credibility scores, or linked fact-checks~\cite{tucker2023misinfointerventions,pennycook2020labels}.
% Finally, our study held constant the interface through which AI-generated notes were presented, whereas prior work suggests that note placement, framing, and multiplicity can significantly affect trust and comprehension~\cite{amershi2019trust,scheurer2024communitynotes}. Exploring alternative presentation and governance strategies, therefore, remains an important direction. Together, these constraints highlight that while our findings demonstrate the promise of synthesizing user-initiated comments into AI-generated notes, further work is needed to validate their robustness, inclusivity, and impact across platforms, modalities, and deployment settings.
\section{Limitations and Future Work}
This paper has three limitations. First, our evaluations relied mainly on English-language comments and a participant pool with limited demographic diversity~\cite{chuai2025community}. While the robustness study incorporated some non-English comments, multilingual and cross-cultural contexts remain underexplored~\cite{noman2024designing}. Moreover, our dataset was limited to $\mathbb{X}$, the only platform that currently provides large-scale community notes. Although $\mathbb{X}$ includes a number of Chinese users and posts written in Chinese~\cite{zhang2023sinophobia}, it cannot be considered representative of the broader Chinese social media ecosystem, where no equivalent mechanism currently exists. The applicability of our findings to domestic Chinese platforms should therefore be interpreted with caution.
Second, our synthesis pipeline capped the number of comments at 300 per post to fit within LLMs' context windows, which may excluded threads containing additional fact-checking information. The pipeline also remained sensitive to prompt design and context length~\cite{ribeiro2020checklist,brown2020language}, indicating that replication with different parameter settings could produce variation.
% Finally, our user study was conducted on a controlled experimental platform rather than directly on $\mathbb{X}$. Although the interface design followed prior practice~\cite{de2025supernotes}, the absence of real-world deployment means that the study may not fully capture platform dynamics, governance mechanisms, and user interaction patterns.
Third, our user study was conducted in a controlled environment, not as a live deployment on $\mathbb{X}$. This approach, mimicking prior practice~\cite{de2025supernotes}, while ensuring experimental control, does not capture the complex dynamics of a real-world ecosystem. Another critical aspect not addressed is the system's robustness against adversarial manipulation. Our work focused on the technical feasibility and usefulness of synthesizing comments, therefore did not test for vulnerabilities to intentional attacks, such as prompt injection~\cite{liu2024automatic} or coordinated comment flooding~\cite{walter2021evaluating}, nor for unintentional harms like the amplification of early, inaccurate information. A viable path is to integrate synthesis tools into Community Note's system, and let the notes to be rated by existing note helpfulness rating algorithm~\cite{xcommunitynotes}. Future work could also explore hybrid human-AI workflows~\cite{li2025scaling}, or balancing strategies that prevent domination by a single viewpoint~\cite{pelekis2025adversarial}.
% The potential influence of alternative presentation strategies, such as note placement, framing, or multiplicity, also remains unexplored~\cite{amershi2019trust,scheurer2024communitynotes}.
\section{Conclusions}
This paper introduced Commenotes, a framework designed to augment community-based fact-checking by leveraging LLMs to synthesize user-initiated comments. Our large-scale analysis of over 2.2 million replies on $\mathbb{X}$ demonstrated that fact-checking information is not only abundant in user comments but also emerges rapidly, with a majority of misleading posts receiving corrective feedback well before community notes are generated. We then proposed and evaluated \proj{}, a two-stage algorithm that first filters for relevant fact-checking comments and subsequently synthesizes them into concise, high-quality notes. Technical evaluations confirmed the framework's efficacy and robustness, showing it can produce useful notes for a vast majority of posts within two hours, and even for those posts not covered by community notes. Furthermore, a comparative user study revealed that \textit{\textbf{commenotes}} were often perceived as superior, with the best-performing model's notes being rated as significantly more helpful than their human-written counterparts, with a 70.1\% win rate. These findings underscore the significant potential of \proj{} to address the critical issue of latency in community fact-checking, offering an effective pathway to providing timely and impactful context to online misinformation.
% \subsection{Dataset}
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}
\vspace{10em}
\textcolor{red}{Disclaimer: Note that some of the examples we showed may contain harmful or misleading content.}
\newpage
\appendix
\section{Annotation Guidance for `Fact-check' Comments}\label{app:annot}
The objective is to reliably identify comments that contain substantive, corrective information suitable for synthesizing into a fact-checking note. A comment is classified into one of two categories: fact-check comment or not a fact-check comment.
\subsection{Criteria For A `Fact-Check' Comment}
To be classified as a `Fact-check' comment, the comment must fulfill all three of the following criteria.
\noindent \textbf{1. Direct and Specific Refutation}: The comment must directly and explicitly address a specific claim made in the original post. It should not merely provide general context or an alternative viewpoint but must articulate how and why the post's claim is inaccurate or misleading.
\noindent \textbf{2. Verifiable Evidence or Source}: The comment must provide direct, verifiable evidence or a credible source to support its refutation. This goes beyond a ``good-faith attempt'' to provide a basis. The evidence must be readily confirmable.
$\bullet$ \textbf{External Links}: Links to reputable news organizations, academic journals, official government reports, or established fact-checking websites (e.g., Snopes, PolitiFact).
$\bullet$ \textbf{Specific Data Points}: Citation of specific statistics, dates, or names that can be independently verified.
$\bullet$ \textbf{Causality}: The comment must clearly explain the causal link between the provided evidence and the refutation of the original claim.
\noindent \textbf{3. Synthesized Conclusion}: The comment must provide a clear, concise summary of the corrective information. The comment itself should function as a mini-fact-check, presenting a conclusion that is easily understood by a lay audience without requiring them to visit the external link to grasp the core point.
\subsection{Criteria for `Not a Fact-Check' Comment}
A comment is classified as ``Not a Fact-check'' if it fails to meet any of the three criteria above. This category is for comments that, while potentially `Useful' as defined by the other framework, do not represent a complete or methodologically sound fact-check.
$\bullet$ \textbf{Generalized or Vague Challenges}: Comments that express disagreement without identifying the specific inaccuracies.
$\bullet$ \textbf{Unverifiable or Anecdotal Evidence}: The comment provides a basis for its claim (e.g., ``my friend said...'') but the information is not from a verifiable public source.
$\bullet$ \textbf{Missing or Broken Links}: Links to sources are either missing, broken, or lead to a page that does not directly support the claim.
$\bullet$ \textbf{Incomplete Information}: The comment provides a link but fails to summarize the content, leaving the user to perform the fact-checking themselves.
\subsection{Illustrative Examples}
\textbf{Example 1: A `Fact-check' Comment}
Post: ``The U.S. national debt has doubled in the last five years, proof of reckless spending. \#Economy''
Comment: ``This is incorrect. According to data from the Treasury Department, the national debt was approximately \$22 trillion five years ago and is now around \$34.6 trillion. This represents an increase of about 57\%, not a doubling.''
Label: Fact-check
Rationale: The comment directly refutes the ``doubled'' claim (Criterion 1), provides a specific, verifiable source and data points (Criterion 2), and offers a clear, synthesized conclusion (a 57\% increase) that is immediately understandable (Criterion 3).
\textbf{Example 2: A `Not a Fact-check' Comment}
Post: ``There's an unusual cloud formation over the city today. It must be a new military experiment.''
Comment: ``That's a beautiful photo! It reminds me of the sky on my trip to Arizona last year. The colors are amazing.''
Label: Not a Fact-check Comment
Rationale: The comment is entirely focused on a personal, subjective reaction to the image. It doesn't acknowledge the claim about a ``military experiment'' at all. It provides no counter-evidence, alternative explanation, or any information that could be used for a fact-checking note. Its purpose is purely to express personal sentiment and share an anecdote, placing it firmly outside the scope of any fact-checking or corrective framework.
\section{Annotation Guidance for `Useful' Notes}\label{app:useful_notes}
For a note to be useful, it needs to provide important, meaningful context that complements the corresponding post. We detailed the criterias and examples below.
\subsection{Criteria for `Useful' Notes}
To be classified as Useful, a note must satisfy two primary characteristics, as defined in the methodology of the \proj{} framework:
\noindent \textbf{1. Identifies Inaccuracy}: The note must explicitly or implicitly point out a factual inaccuracy, misleading claim, or lack of context in the original post.
\noindent \textbf{2. Provides Substantiation}: The note must substantiate its challenge with reasons, evidence, or relevant information. This includes, but is not limited to:
$\bullet$ \textbf{Offering Counter-Evidence}: Providing specific facts, data, or links to sources that contradict the post.
$\bullet$ \textbf{Presenting Alternative Explanations}: Proposing a different, plausible cause for the phenomena described in the post.
$\bullet$ \textbf{Supplying Missing Context}: Adding crucial information that reframes the claims made in the post.
\textbf{Important Note for Annotators}: The validity or absolute truthfulness of the evidence provided in a note is not assessed in this phase. The guideline is to determine if the user has provided a basis for their claim, not whether that basis is itself flawlessly accurate. The presence of a good-faith attempt to offer a reason or evidence is sufficient for a ``Useful'' classification.
\subsection{Criteria for `Not Useful' Notes}
A note is classified as Not Useful if it fails to meet the criteria above. Common examples include:
\textbf{1. Mere Disagreement or Emotional Reaction}: Notes that express simple agreement or disagreement (e.g., ``This is fake'', ``True'', ``Propaganda'') without any accompanying reasoning or evidence.
\textbf{2. Irrelevant Information}: Notes containing factual statements that are not relevant to the accuracy of the central claims in the post.
\textbf{3. Unsubstantiated Counter-Claims}: Notes that make a different claim without providing any support.
\textbf{4. Questions, Sarcasm, or General Discussion}: Notes that do not contribute fact-checking information.
\subsection{Illustrative Examples}
The following examples demonstrate the application of these guidelines:
\textbf{Example 1: Useful Note}
Post: ``The earthquake in Turkey looks like a punitive operation (HAARP) by NATO or the US against Turkey. The video shows lightning strikes, which are not normal in earthquakes, but always happen in harp operations.''
Note: ``There was a thunderstorm in the area before the earthquake started. Plus, many electrical faults occur causing bright flashes. Not some crazy US attack....''
Label: Useful
Rationale: The note meets both criteria. It identifies the inaccuracy (the claim of a HAARP attack) and provides substantiation by offering specific, plausible alternative explanations (a thunderstorm and electrical faults) for the evidence presented in the post (lightning flashes).
\textbf{Example 2: Not Useful Note}
Post: `Allegedly, looting has started in Memphis. \#TyreNicholsVideo [video attached]''
Note: ``Fake af!!! Stop with the propaganda!!! \#MemphisStrong''
Label: Not Useful
Rationale: While the note identifies a potential inaccuracy (``Fake af!!!''), it fails the second criterion. It is a mere dismissal and provides no substantiation, evidence, or reasoning to support its claim. It is an emotional reaction rather than an informative contribution.
\section{Prompt to filter comments}\label{sec:prompt_filter}
We wrote this prompt to test the performance of LLMs on filtering fact-check comments and obtain data to analyze the features of fact-check comments and finetune models. We excluded comments containing only fact-checking information that is irrelevant to the accuracy of the post. The full prompt is as follows:
(a) You will receive a post from $\mathbb{X}$ along with a comment to the post. Please check whether there is any fact-checking information in the comment that points out inaccuracies in the post. Your answer should be no if fact-checking information is irrelevant to the accuracy of the post. You should answer with ``1'' if there is, or ``0'' if there is not. Do not include any other words in your answer.
(b) POST: <post text>
(c) COMMENT: <comment text>
\section{Prompt to synthesize comments}\label{sec:prompt_synthesize}
We designed this prompt to synthesize filtered comments into a concise paragraph highlighting inaccuracies in a post. To help the LLM adopt an appropriate style, we introduced the concept of Community Notes in the prompt and instructed it to write in the neutral, factual tone expected of such notes. We explicitly emphasized avoiding biased or argumentative language and prohibited any mention of the word ``comments''. The prompt adopts the following structure:
(a) $\mathbb{X}$ has a crowd-sourced fact-checking program called Community Notes. Here, users can write `notes' on potentially misleading content. Community Notes will be shown publicly alongside the piece of content.
You will be provided with a post from $\mathbb{X}$ and some comments. Your job is to find fact-check information in those comments that indicates that there is something inaccurate in the post and use that information to write a piece of Community Note yourself. It should be in unbiased language, not argumentative. It should also be within 280 characters.
(b) POST: <post text>
(c) COMMENTS: <filter comments texts separated by line break>
(d) Do not use the word ``comments'' in your answer. Regard the comments as your own thoughts and opinions. Answer in English.
``Answer in English.'' is added because models in China, like the Qwen series, have probabilities of answering in Chinese.
\section{The Illustration of The Interfaces in The User Evaluation Study}\label{app:interface}
Figure~\ref{fig:post_note} and~\ref{fig:ratings} showed the interfaces in the user evaluation study, where Figure~\ref{fig:post_note} showed the posts and notes to be showed, and Figure~\ref{fig:ratings} showed the ratings items. Participants needed to first viewed the post and notes, and then needed to fill in the rating items before proceeding.
\begin{figure}[!htbp]
\subfloat[Posts and notes to be showed.]{
\includegraphics[width=0.49\textwidth]{Figure/interface1.pdf}
\label{fig:post_note}
}
\subfloat[Rating items.]{
\includegraphics[width=0.49\textwidth]{Figure/interface2.pdf}
\label{fig:ratings}
}
\caption{The interfaces used in the user evaluation study.}
\label{fig:interface}
\end{figure}
\section{Effect of Topic Factors on Users' Ratings}\label{app:effect_factors}
This section details our analysis of the factors influencing user ratings across various quality dimensions. We present the results graphically to illustrate the magnitude and direction of these effects. Figure~\ref{fig:combined} shows the coefficient plot from our comprehensive model, which assesses the overall impact of predictors when data from all language models are aggregated. For a more granular, model-specific examination, Figure~\ref{fig:claude}, ~\ref{fig:gpt4o}, and ~\ref{fig:qwen} depict these effects individually for the claude-3.7-sonnet, gpt-4o, and qwen2.5-72b-instruct models, respectively. Collectively, these visualizations highlighted that topic-related factors did not have a significant effect on users' ratings across different models.
\begin{figure}[!htbp]
\subfloat[Overall.]{
\includegraphics[width=0.78\textwidth]{Figure/consolidated_plot_COMBINED.png}
\label{fig:combined}
}
% \caption{The effect of different factors on users' ratings for different models.}
\subfloat[claude-3.7-sonnet.]{
\includegraphics[width=0.78\textwidth]{Figure/consolidated_plot_CLAUDE.png}
\label{fig:claude}
}
\subfloat[gpt-4o.]{
\includegraphics[width=0.78\textwidth]{Figure/consolidated_plot_GPT4O.png}
\label{fig:gpt4o}
}
\subfloat[qwen2.5-72b-instruct.]{
\includegraphics[width=0.78\textwidth]{Figure/consolidated_plot_QWEN.png}
\label{fig:qwen}
}
\caption{The effect of topic factors on users' ratings for (a) overall, (b) claude-3.7-sonnet, (c) gpt-4o, and (d) qwen2.5-72b-instruct models.}
\end{figure}
% \begin{figure}[!htbp]
% \caption{The effect of different factors on users' ratings for the claude model.}
% \end{figure}
% \begin{figure}[!htbp]
% \end{figure}
% \begin{figure}[!htbp]
% \end{figure}
\section{Effect of Demographic Factors on Users' Ratings}\label{app:demographic}
Figure~\ref{fig:overall_demographic}, ~\ref{fig:claude_demographic}, ~\ref{fig:gpt4o_demographic}, and ~\ref{fig:qwen_demographic} separately showed the effect of demographic factors, including political stance and polarization, on users' ratings. We found a limited and non-significant effect of these demographic factors on users' ratings.
\begin{figure}[!htbp]
\centering
\subfloat[Overall.]{
\includegraphics[width=0.78\textwidth]{Figure/consolidated_plot_OVERALL.png}
\label{fig:overall_demographic}
}
\subfloat[claude-3.7-sonnet.]{
\includegraphics[width=0.78\textwidth]{Figure/plot_CLAUDE.png}
\label{fig:claude_demographic}
}
\subfloat[gpt-4o.]{
\includegraphics[width=0.78\textwidth]{Figure/plot_GPT4O.png}
\label{fig:gpt4o_demographic}
}
\subfloat[qwen2.5-72b-instruct.]{
\includegraphics[width=0.78\textwidth]{Figure/plot_QWEN.png}
\label{fig:qwen_demographic}
}
\caption{The effect of user demographics (i.e., political stance, and polarization) on different rating metrics for (a) Overall, (b) claude-3.7-sonnet, (c) gpt-4o, (d) qwen2.5-72b-instruct models. }
\label{fig:demographic_appendix}
\end{figure}
\end{document}
%%
%% End of file `sample-manuscript.tex'.
%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].
\begin{thebibliography}{80}
%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command. Each of them MUST provide its own final punctuation,
%%% except for \shownote{}, \showDOI{}, and \showURL{}. The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showDOI}[1]{\unskip} % LaTeX syntax
%%%
%%% \def \showDOI #1{\unskip} % plain TeX syntax
%%%
%%% ====================================================================
\ifx \showCODEN \undefined \def \showCODEN #1{\unskip} \fi
\ifx \showDOI \undefined \def \showDOI #1{#1}\fi
\ifx \showISBNx \undefined \def \showISBNx #1{\unskip} \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii #1{\unskip} \fi
\ifx \showISSN \undefined \def \showISSN #1{\unskip} \fi
\ifx \showLCCN \undefined \def \showLCCN #1{\unskip} \fi
\ifx \shownote \undefined \def \shownote #1{#1} \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1} \fi
\ifx \showURL \undefined \def \showURL {\relax} \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}
\bibitem[Albahar(2021)]%
{albahar2021hybrid}
\bibfield{author}{\bibinfo{person}{Marwan Albahar}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{A hybrid model for fake news detection: Leveraging news content and user comments in fake news}.
\newblock \bibinfo{journal}{\emph{IET Information Security}} \bibinfo{volume}{15}, \bibinfo{number}{2} (\bibinfo{year}{2021}), \bibinfo{pages}{169--177}.
\newblock
\bibitem[Allen et~al\mbox{.}(2021)]%
{allen2021scaling}
\bibfield{author}{\bibinfo{person}{Jennifer Allen}, \bibinfo{person}{Antonio~A Arechar}, \bibinfo{person}{Gordon Pennycook}, {and} \bibinfo{person}{David~G Rand}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Scaling up fact-checking using the wisdom of crowds}.
\newblock \bibinfo{journal}{\emph{Science Advances}} \bibinfo{volume}{7}, \bibinfo{number}{36} (\bibinfo{year}{2021}), \bibinfo{pages}{eabf4393}.
\newblock
\bibitem[Alrumaih et~al\mbox{.}(2020)]%
{alrumaih2020sentiment}
\bibfield{author}{\bibinfo{person}{Abdulrahman Alrumaih}, \bibinfo{person}{Ali Al-Sabbagh}, \bibinfo{person}{Ruaa Alsabah}, \bibinfo{person}{Harith Kharrufa}, {and} \bibinfo{person}{James Baldwin}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Sentiment analysis of comments in social media.}
\newblock \bibinfo{journal}{\emph{International Journal of Electrical \& Computer Engineering (2088-8708)}} \bibinfo{volume}{10}, \bibinfo{number}{6} (\bibinfo{year}{2020}).
\newblock
\bibitem[Bakker et~al\mbox{.}(2022)]%
{bakker2022fine}
\bibfield{author}{\bibinfo{person}{Michiel Bakker}, \bibinfo{person}{Martin Chadwick}, \bibinfo{person}{Hannah Sheahan}, \bibinfo{person}{Michael Tessler}, \bibinfo{person}{Lucy Campbell-Gillingham}, \bibinfo{person}{Jan Balaguer}, \bibinfo{person}{Nat McAleese}, \bibinfo{person}{Amelia Glaese}, \bibinfo{person}{John Aslanides}, \bibinfo{person}{Matt Botvinick}, {et~al\mbox{.}}} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Fine-tuning language models to find agreement among humans with diverse preferences}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} \bibinfo{volume}{35} (\bibinfo{year}{2022}), \bibinfo{pages}{38176--38189}.
\newblock
\bibitem[Bond and Garrett(2023)]%
{bond2023engagement}
\bibfield{author}{\bibinfo{person}{Robert~M Bond} {and} \bibinfo{person}{R~Kelly Garrett}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Engagement with fact-checked posts on Reddit}.
\newblock \bibinfo{journal}{\emph{PNAS Nexus}} \bibinfo{volume}{2}, \bibinfo{number}{3} (\bibinfo{year}{2023}), \bibinfo{pages}{pgad018}.
\newblock
\bibitem[Bonifazi et~al\mbox{.}(2024)]%
{bonifazi2024representation}
\bibfield{author}{\bibinfo{person}{Gianluca Bonifazi}, \bibinfo{person}{Francesco Cauteruccio}, \bibinfo{person}{Enrico Corradini}, \bibinfo{person}{Michele Marchetti}, \bibinfo{person}{Giorgio Terracina}, \bibinfo{person}{Domenico Ursino}, {and} \bibinfo{person}{Luca Virgili}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Representation, detection and usage of the content semantics of comments in a social platform}.
\newblock \bibinfo{journal}{\emph{Journal of Information Science}} \bibinfo{volume}{50}, \bibinfo{number}{2} (\bibinfo{year}{2024}), \bibinfo{pages}{317--341}.
\newblock
\bibitem[Brown et~al\mbox{.}(2020)]%
{brown2020language}
\bibfield{author}{\bibinfo{person}{Tom Brown}, \bibinfo{person}{Benjamin Mann}, \bibinfo{person}{Nick Ryder}, \bibinfo{person}{Melanie Subbiah}, \bibinfo{person}{Jared~D Kaplan}, \bibinfo{person}{Prafulla Dhariwal}, \bibinfo{person}{Arvind Neelakantan}, \bibinfo{person}{Pranav Shyam}, \bibinfo{person}{Girish Sastry}, \bibinfo{person}{Amanda Askell}, {et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Language models are few-shot learners}.
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}} \bibinfo{volume}{33} (\bibinfo{year}{2020}), \bibinfo{pages}{1877--1901}.
\newblock
\bibitem[Bu et~al\mbox{.}(2023)]%
{bu2023combating}
\bibfield{author}{\bibinfo{person}{Yuyan Bu}, \bibinfo{person}{Qiang Sheng}, \bibinfo{person}{Juan Cao}, \bibinfo{person}{Peng Qi}, \bibinfo{person}{Danding Wang}, {and} \bibinfo{person}{Jintao Li}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Combating online misinformation videos: Characterization, detection, and future directions}. In \bibinfo{booktitle}{\emph{Proceedings of the 31st ACM International Conference on Multimedia}}. \bibinfo{pages}{8770--8780}.
\newblock
\bibitem[Burton et~al\mbox{.}(2024)]%
{burton2024large}
\bibfield{author}{\bibinfo{person}{Jason~W Burton}, \bibinfo{person}{Ezequiel Lopez-Lopez}, \bibinfo{person}{Shahar Hechtlinger}, \bibinfo{person}{Zoe Rahwan}, \bibinfo{person}{Samuel Aeschbach}, \bibinfo{person}{Michiel~A Bakker}, \bibinfo{person}{Joshua~A Becker}, \bibinfo{person}{Aleks Berditchevskaia}, \bibinfo{person}{Julian Berger}, \bibinfo{person}{Levin Brinkmann}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{How large language models can reshape collective intelligence}.
\newblock \bibinfo{journal}{\emph{Nature Human Behaviour}} \bibinfo{volume}{8}, \bibinfo{number}{9} (\bibinfo{year}{2024}), \bibinfo{pages}{1643--1655}.
\newblock
\bibitem[{CBS News}(2025)]%
{meta_community_notes}
\bibfield{author}{\bibinfo{person}{{CBS News}}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{Meta starts testing community-sourced fact-checking based on X's Community Notes}.
\newblock
\newblock
\urldef\tempurl%
\url{https://www.cbsnews.com/sanfrancisco/news/meta-community-notes-crowd-sourced-fact-checking-based-on-x-twitter-testing/}
\showURL{%
\tempurl}
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Cheng and Jin(2019)]%
{cheng2019airbnb}
\bibfield{author}{\bibinfo{person}{Mingming Cheng} {and} \bibinfo{person}{Xin Jin}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{What do Airbnb users care about? An analysis of online review comments}.
\newblock \bibinfo{journal}{\emph{International Journal of Hospitality Management}} \bibinfo{volume}{76} (\bibinfo{year}{2019}), \bibinfo{pages}{58--70}.
\newblock
\bibitem[Chuai et~al\mbox{.}(2024a)]%
{chuai2024community}
\bibfield{author}{\bibinfo{person}{Yuwei Chuai}, \bibinfo{person}{Moritz Pilarski}, \bibinfo{person}{Thomas Renault}, \bibinfo{person}{David Restrepo-Amariles}, \bibinfo{person}{Aurore Troussel-Cl{\'e}ment}, \bibinfo{person}{Gabriele Lenzini}, {and} \bibinfo{person}{Nicolas Pr{\"o}llochs}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Community-based fact-checking reduces the spread of misleading posts on social media}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2409.08781}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Chuai et~al\mbox{.}(2025)]%
{chuai2025community}
\bibfield{author}{\bibinfo{person}{Yuwei Chuai}, \bibinfo{person}{Anastasia Sergeeva}, \bibinfo{person}{Gabriele Lenzini}, {and} \bibinfo{person}{Nicolas Pr{\"o}llochs}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Community fact-checks trigger moral outrage in replies to misleading posts on social media}. In \bibinfo{booktitle}{\emph{Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems}}. \bibinfo{pages}{1--23}.
\newblock
\bibitem[Chuai et~al\mbox{.}(2024b)]%
{chuai2024did}
\bibfield{author}{\bibinfo{person}{Yuwei Chuai}, \bibinfo{person}{Haoye Tian}, \bibinfo{person}{Nicolas Pr{\"o}llochs}, {and} \bibinfo{person}{Gabriele Lenzini}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Did the roll-out of community notes reduce engagement with misinformation on X/Twitter?}
\newblock \bibinfo{journal}{\emph{Proceedings of the ACM on Human-Computer Interaction}} \bibinfo{volume}{8}, \bibinfo{number}{CSCW2} (\bibinfo{year}{2024}), \bibinfo{pages}{1--52}.
\newblock
\bibitem[Collins(2015)]%
{collins2015can}
\bibfield{author}{\bibinfo{person}{Luke Collins}.} \bibinfo{year}{2015}\natexlab{}.
\newblock \showarticletitle{How can semantic annotation help us to analyse the discourse of climate change in online user comments?}
\newblock \bibinfo{journal}{\emph{Linguistik Online}} \bibinfo{volume}{70}, \bibinfo{number}{1} (\bibinfo{year}{2015}), \bibinfo{pages}{43--60}.
\newblock
\bibitem[Corp.(2025)]%
{xcommunitynotes}
\bibfield{author}{\bibinfo{person}{X Corp.}} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{About Community Notes on X}.
\newblock \bibinfo{howpublished}{\url{https://help.x.com/en/using-x/community-notes}}.
\newblock
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Dash et~al\mbox{.}(2019)]%
{dash2019summarizing}
\bibfield{author}{\bibinfo{person}{Abhisek Dash}, \bibinfo{person}{Anurag Shandilya}, \bibinfo{person}{Arindam Biswas}, \bibinfo{person}{Kripabandhu Ghosh}, \bibinfo{person}{Saptarshi Ghosh}, {and} \bibinfo{person}{Abhijnan Chakraborty}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Summarizing user-generated textual content: Motivation and methods for fairness in algorithmic summaries}.
\newblock \bibinfo{journal}{\emph{Proceedings of the ACM on Human-Computer Interaction}} \bibinfo{volume}{3}, \bibinfo{number}{CSCW} (\bibinfo{year}{2019}), \bibinfo{pages}{1--28}.
\newblock
\bibitem[De et~al\mbox{.}(2025)]%
{de2025supernotes}
\bibfield{author}{\bibinfo{person}{Soham De}, \bibinfo{person}{Michiel~A. Bakker}, \bibinfo{person}{Jay Baxter}, {and} \bibinfo{person}{Martin Saveski}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Supernotes: Driving consensus in crowd-sourced fact-checking}. In \bibinfo{booktitle}{\emph{Proceedings of the ACM Web Conference 2025}}. \bibinfo{pages}{3751--3761}.
\newblock
\bibitem[Drolsbach et~al\mbox{.}(2024)]%
{drolsbach2024community}
\bibfield{author}{\bibinfo{person}{Chiara~Patricia Drolsbach}, \bibinfo{person}{Kirill Solovev}, {and} \bibinfo{person}{Nicolas Pr{\"o}llochs}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Community notes increase trust in fact-checking on social media}.
\newblock \bibinfo{journal}{\emph{PNAS Nexus}} \bibinfo{volume}{3}, \bibinfo{number}{7} (\bibinfo{year}{2024}), \bibinfo{pages}{pgae217}.
\newblock
\bibitem[Epstein et~al\mbox{.}(2020)]%
{epstein2020will}
\bibfield{author}{\bibinfo{person}{Ziv Epstein}, \bibinfo{person}{Gordon Pennycook}, {and} \bibinfo{person}{David Rand}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Will the crowd game the algorithm? Using layperson judgments to combat misinformation on social media by downranking distrusted sources}. In \bibinfo{booktitle}{\emph{Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems}}. \bibinfo{pages}{1--11}.
\newblock
\bibitem[Fish et~al\mbox{.}(2024)]%
{fish2024generative}
\bibfield{author}{\bibinfo{person}{Sara Fish}, \bibinfo{person}{Paul G{\"o}lz}, \bibinfo{person}{David~C Parkes}, \bibinfo{person}{Ariel~D Procaccia}, \bibinfo{person}{Gili Rusak}, \bibinfo{person}{Itai Shapira}, {and} \bibinfo{person}{Manuel W{\"u}thrich}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Generative social choice}. In \bibinfo{booktitle}{\emph{Proceedings of the 25th ACM Conference on Economics and Computation}}. \bibinfo{pages}{985--985}.
\newblock
\bibitem[Ganiuly and Smaiyl(2024)]%
{ganiuly2024ethics}
\bibfield{author}{\bibinfo{person}{Daniyal Ganiuly} {and} \bibinfo{person}{Assel Smaiyl}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Ethics of using llms in content moderation on twitter}.
\newblock \bibinfo{journal}{\emph{International Journal of Innovative Science and Research Technology (IJISRT)}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Gao et~al\mbox{.}(2024)]%
{gao2024can}
\bibfield{author}{\bibinfo{person}{Yang Gao}, \bibinfo{person}{Maggie~Mengqing Zhang}, {and} \bibinfo{person}{Huaxia Rui}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Can crowdchecking curb misinformation? Evidence from community notes}.
\newblock \bibinfo{journal}{\emph{Evidence from Community Notes (October 17, 2024)}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Georgakopoulos et~al\mbox{.}(2018)]%
{georgakopoulos2018convolutional}
\bibfield{author}{\bibinfo{person}{Spiros~V Georgakopoulos}, \bibinfo{person}{Sotiris~K Tasoulis}, \bibinfo{person}{Aristidis~G Vrahatis}, {and} \bibinfo{person}{Vassilis~P Plagianakos}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Convolutional neural networks for toxic comment classification}. In \bibinfo{booktitle}{\emph{Proceedings of the 10th Hellenic Conference on Artificial Intelligence}}. \bibinfo{pages}{1--6}.
\newblock
\bibitem[Guo et~al\mbox{.}(2022)]%
{guo2022survey}
\bibfield{author}{\bibinfo{person}{Zhijiang Guo}, \bibinfo{person}{Michael Schlichtkrull}, {and} \bibinfo{person}{Andreas Vlachos}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{A survey on automated fact-checking}.
\newblock \bibinfo{journal}{\emph{Transactions of the Association for Computational Linguistics}} \bibinfo{volume}{10} (\bibinfo{year}{2022}), \bibinfo{pages}{178--206}.
\newblock
\bibitem[Hamed et~al\mbox{.}(2023)]%
{hamed2023fake}
\bibfield{author}{\bibinfo{person}{Suhaib~Kh Hamed}, \bibinfo{person}{Mohd~Juzaiddin Ab~Aziz}, {and} \bibinfo{person}{Mohd~Ridzwan Yaakub}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Fake news detection model on social media by leveraging sentiment analysis of news content and emotion analysis of users’ comments}.
\newblock \bibinfo{journal}{\emph{Sensors}} \bibinfo{volume}{23}, \bibinfo{number}{4} (\bibinfo{year}{2023}), \bibinfo{pages}{1748}.
\newblock
\bibitem[He et~al\mbox{.}(2023)]%
{he2023reinforcement}
\bibfield{author}{\bibinfo{person}{Bing He}, \bibinfo{person}{Mustaque Ahamad}, {and} \bibinfo{person}{Srijan Kumar}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Reinforcement learning-based counter-misinformation response generation: a case study of COVID-19 vaccine misinformation}. In \bibinfo{booktitle}{\emph{Proceedings of the ACM Web Conference 2023}}. \bibinfo{pages}{2698--2709}.
\newblock
\bibitem[He et~al\mbox{.}(2017)]%
{he2017exploring}
\bibfield{author}{\bibinfo{person}{Ming He}, \bibinfo{person}{Yong Ge}, \bibinfo{person}{Enhong Chen}, \bibinfo{person}{Qi Liu}, {and} \bibinfo{person}{Xuesong Wang}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Exploring the emerging type of comment for online videos: Danmu}.
\newblock \bibinfo{journal}{\emph{ACM Transactions on the Web (TWEB)}} \bibinfo{volume}{12}, \bibinfo{number}{1} (\bibinfo{year}{2017}), \bibinfo{pages}{1--33}.
\newblock
\bibitem[Horne and Nevo(2025)]%
{horne2025people}
\bibfield{author}{\bibinfo{person}{Benjamin~D Horne} {and} \bibinfo{person}{Dorit Nevo}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{People adhere to content warning labels even when they are wrong due to ecologically rational adaptations}.
\newblock \bibinfo{journal}{\emph{Scientific Reports}} \bibinfo{volume}{15}, \bibinfo{number}{1} (\bibinfo{year}{2025}), \bibinfo{pages}{13896}.
\newblock
\bibitem[Hu et~al\mbox{.}(2025)]%
{hu2025generative}
\bibfield{author}{\bibinfo{person}{Tiancheng Hu}, \bibinfo{person}{Yara Kyrychenko}, \bibinfo{person}{Steve Rathje}, \bibinfo{person}{Nigel Collier}, \bibinfo{person}{Sander van~der Linden}, {and} \bibinfo{person}{Jon Roozenbeek}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Generative language models exhibit social identity biases}.
\newblock \bibinfo{journal}{\emph{Nature Computational Science}} \bibinfo{volume}{5}, \bibinfo{number}{1} (\bibinfo{year}{2025}), \bibinfo{pages}{65--75}.
\newblock
\bibitem[Jiang and Wilson(2018)]%
{jiang2018linguistic}
\bibfield{author}{\bibinfo{person}{Shan Jiang} {and} \bibinfo{person}{Christo Wilson}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Linguistic signals under misinformation and fact-checking: Evidence from user comments on social media}.
\newblock \bibinfo{journal}{\emph{Proceedings of the ACM on Human-Computer Interaction}} \bibinfo{volume}{2}, \bibinfo{number}{CSCW} (\bibinfo{year}{2018}), \bibinfo{pages}{1--23}.
\newblock
\bibitem[Kaplan et~al\mbox{.}(2020)]%
{kaplan2020scaling}
\bibfield{author}{\bibinfo{person}{Jared Kaplan}, \bibinfo{person}{Sam McCandlish}, \bibinfo{person}{Tom Henighan}, \bibinfo{person}{Tom~B Brown}, \bibinfo{person}{Benjamin Chess}, \bibinfo{person}{Rewon Child}, \bibinfo{person}{Scott Gray}, \bibinfo{person}{Alec Radford}, \bibinfo{person}{Jeffrey Wu}, {and} \bibinfo{person}{Dario Amodei}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Scaling laws for neural language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2001.08361}} (\bibinfo{year}{2020}).
\newblock
\bibitem[Kim and Walker(2020)]%
{kim2020leveraging}
\bibfield{author}{\bibinfo{person}{Hyunuk Kim} {and} \bibinfo{person}{Dylan Walker}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Leveraging volunteer fact checking to identify misinformation about COVID-19 in social media}.
\newblock \bibinfo{journal}{\emph{Harvard Kennedy School Misinformation Review}} \bibinfo{volume}{1}, \bibinfo{number}{3} (\bibinfo{year}{2020}).
\newblock
\bibitem[Kristo(2021)]%
{kristo2021computer}
\bibfield{author}{\bibinfo{person}{Eklekta Kristo}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Computer-aided analysis of video comments for requirements analysis}.
\newblock (\bibinfo{year}{2021}).
\newblock
\bibitem[Li et~al\mbox{.}(2025)]%
{li2025scaling}
\bibfield{author}{\bibinfo{person}{Haiwen Li}, \bibinfo{person}{Soham De}, \bibinfo{person}{Manon Revel}, \bibinfo{person}{Andreas Haupt}, \bibinfo{person}{Brad Miller}, \bibinfo{person}{Keith Coleman}, \bibinfo{person}{Jay Baxter}, \bibinfo{person}{Martin Saveski}, {and} \bibinfo{person}{Michiel~A Bakker}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Scaling human judgment in community notes with LLMs}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2506.24118}} (\bibinfo{year}{2025}).
\newblock
\bibitem[Liu et~al\mbox{.}(2024)]%
{liu2024automatic}
\bibfield{author}{\bibinfo{person}{Xiaogeng Liu}, \bibinfo{person}{Zhiyuan Yu}, \bibinfo{person}{Yizhe Zhang}, \bibinfo{person}{Ning Zhang}, {and} \bibinfo{person}{Chaowei Xiao}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Automatic and universal prompt injection attacks against large language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2403.04957}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Madden et~al\mbox{.}(2013)]%
{madden2013classification}
\bibfield{author}{\bibinfo{person}{Amy Madden}, \bibinfo{person}{Ian Ruthven}, {and} \bibinfo{person}{David McMenemy}.} \bibinfo{year}{2013}\natexlab{}.
\newblock \showarticletitle{A classification scheme for content analyses of YouTube video comments}.
\newblock \bibinfo{journal}{\emph{Journal of Documentation}} \bibinfo{volume}{69}, \bibinfo{number}{5} (\bibinfo{year}{2013}), \bibinfo{pages}{693--714}.
\newblock
\bibitem[Maruyama et~al\mbox{.}(2017)]%
{Civic_Broadcast_chi2017}
\bibfield{author}{\bibinfo{person}{Misa Maruyama}, \bibinfo{person}{Scott~P. Robertson}, \bibinfo{person}{Sara Douglas}, \bibinfo{person}{Roxanne Raine}, {and} \bibinfo{person}{Bryan Semaan}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Social watching a civic broadcast: understanding the effects of positive feedback and other users' opinions}. In \bibinfo{booktitle}{\emph{Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing}} (Portland, Oregon, USA) \emph{(\bibinfo{series}{CSCW '17})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{794–807}.
\newblock
\showISBNx{9781450343350}
\bibitem[{Meta Transparency Center}(2025)]%
{instagram_meta_notes}
\bibfield{author}{\bibinfo{person}{{Meta Transparency Center}}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{Community Notes on Instagram}.
\newblock
\newblock
\urldef\tempurl%
\url{https://transparency.meta.com/features/community-notes}
\showURL{%
\tempurl}
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Micallef et~al\mbox{.}(2020)]%
{micallef2020role}
\bibfield{author}{\bibinfo{person}{Nicholas Micallef}, \bibinfo{person}{Bing He}, \bibinfo{person}{Srijan Kumar}, \bibinfo{person}{Mustaque Ahamad}, {and} \bibinfo{person}{Nasir Memon}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{The role of the crowd in countering misinformation: A case study of the COVID-19 infodemic}. In \bibinfo{booktitle}{\emph{2020 IEEE International Conference on Big Data (big data)}}. IEEE, \bibinfo{pages}{748--757}.
\newblock
\bibitem[Noman et~al\mbox{.}(2024)]%
{noman2024designing}
\bibfield{author}{\bibinfo{person}{Muaadh Noman}, \bibinfo{person}{Selin Gurgun}, \bibinfo{person}{Keith Phalp}, {and} \bibinfo{person}{Raian Ali}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Designing social media to foster user engagement in challenging misinformation: a cross-cultural comparison between the UK and Arab countries}.
\newblock \bibinfo{journal}{\emph{Humanities and Social Sciences Communications}} \bibinfo{volume}{11}, \bibinfo{number}{1} (\bibinfo{year}{2024}), \bibinfo{pages}{1--13}.
\newblock
\bibitem[Ntoutsi et~al\mbox{.}(2020)]%
{ntoutsi2020bias}
\bibfield{author}{\bibinfo{person}{Eirini Ntoutsi}, \bibinfo{person}{Pavlos Fafalios}, \bibinfo{person}{Ujwal Gadiraju}, \bibinfo{person}{Vasileios Iosifidis}, \bibinfo{person}{Wolfgang Nejdl}, \bibinfo{person}{Maria-Esther Vidal}, \bibinfo{person}{Salvatore Ruggieri}, \bibinfo{person}{Franco Turini}, \bibinfo{person}{Symeon Papadopoulos}, \bibinfo{person}{Emmanouil Krasanakis}, {et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Bias in data-driven artificial intelligence systems—An introductory survey}.
\newblock \bibinfo{journal}{\emph{Wiley Interdisciplinary Reviews: Data Mining and Knowledge Discovery}} \bibinfo{volume}{10}, \bibinfo{number}{3} (\bibinfo{year}{2020}), \bibinfo{pages}{e1356}.
\newblock
\bibitem[Park et~al\mbox{.}(2016)]%
{park2016supporting}
\bibfield{author}{\bibinfo{person}{Deokgun Park}, \bibinfo{person}{Simranjit Sachar}, \bibinfo{person}{Nicholas Diakopoulos}, {and} \bibinfo{person}{Niklas Elmqvist}.} \bibinfo{year}{2016}\natexlab{}.
\newblock \showarticletitle{Supporting comment moderators in identifying high quality online news comments}. In \bibinfo{booktitle}{\emph{Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems}}. \bibinfo{pages}{1114--1125}.
\newblock
\bibitem[Pelekis et~al\mbox{.}(2025)]%
{pelekis2025adversarial}
\bibfield{author}{\bibinfo{person}{Sotiris Pelekis}, \bibinfo{person}{Thanos Koutroubas}, \bibinfo{person}{Afroditi Blika}, \bibinfo{person}{Anastasis Berdelis}, \bibinfo{person}{Evangelos Karakolis}, \bibinfo{person}{Christos Ntanos}, \bibinfo{person}{Evangelos Spiliotis}, {and} \bibinfo{person}{Dimitris Askounis}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Adversarial machine learning: a review of methods, tools, and critical industry sectors}.
\newblock \bibinfo{journal}{\emph{Artificial Intelligence Review}} \bibinfo{volume}{58}, \bibinfo{number}{8} (\bibinfo{year}{2025}), \bibinfo{pages}{226}.
\newblock
\bibitem[Pennycook and Rand(2019)]%
{pennycook2019fighting}
\bibfield{author}{\bibinfo{person}{Gordon Pennycook} {and} \bibinfo{person}{David~G Rand}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Fighting misinformation on social media using crowdsourced judgments of news source quality}.
\newblock \bibinfo{journal}{\emph{Proceedings of the National Academy of Sciences}} \bibinfo{volume}{116}, \bibinfo{number}{7} (\bibinfo{year}{2019}), \bibinfo{pages}{2521--2526}.
\newblock
\bibitem[Pfeffer et~al\mbox{.}(2023)]%
{pfeffer2023half}
\bibfield{author}{\bibinfo{person}{J{\"u}rgen Pfeffer}, \bibinfo{person}{Daniel Matter}, {and} \bibinfo{person}{Anahit Sargsyan}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{The half-life of a tweet}. In \bibinfo{booktitle}{\emph{Proceedings of the International AAAI Conference on Web and Social Media}}, Vol.~\bibinfo{volume}{17}. \bibinfo{pages}{1163--1167}.
\newblock
\bibitem[Platform(2025)]%
{x_community_notes_quickstart}
\bibfield{author}{\bibinfo{person}{X~Developer Platform}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{Community Notes Quickstart}.
\newblock \bibinfo{howpublished}{\url{https://docs.x.com/x-api/community-notes/quickstart}}.
\newblock
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Pr{\"o}llochs(2022)]%
{prollochs2022community}
\bibfield{author}{\bibinfo{person}{Nicolas Pr{\"o}llochs}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Community-based fact-checking on Twitter's Birdwatch platform}. In \bibinfo{booktitle}{\emph{Proceedings of the International AAAI Conference on Web and Social Media}}, Vol.~\bibinfo{volume}{16}. \bibinfo{pages}{794--805}.
\newblock
\bibitem[Qi et~al\mbox{.}(2024)]%
{qi2024sniffer}
\bibfield{author}{\bibinfo{person}{Peng Qi}, \bibinfo{person}{Zehong Yan}, \bibinfo{person}{Wynne Hsu}, {and} \bibinfo{person}{Mong~Li Lee}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Sniffer: Multimodal large language model for explainable out-of-context misinformation detection}. In \bibinfo{booktitle}{\emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}}. \bibinfo{pages}{13052--13062}.
\newblock
\bibitem[Quelle and Bovet(2024)]%
{quelle2024perils}
\bibfield{author}{\bibinfo{person}{Dorian Quelle} {and} \bibinfo{person}{Alexandre Bovet}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{The perils and promises of fact-checking with large language models}.
\newblock \bibinfo{journal}{\emph{Frontiers in Artificial Intelligence}} \bibinfo{volume}{7} (\bibinfo{year}{2024}), \bibinfo{pages}{1341697}.
\newblock
\bibitem[Ribeiro et~al\mbox{.}(2020)]%
{ribeiro2020checklist}
\bibfield{author}{\bibinfo{person}{Marco~Tulio Ribeiro}, \bibinfo{person}{Tongshuang Wu}, {and} \bibinfo{person}{Carlos Guestrin}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Beyond accuracy: behavioral testing of NLP models with checkList}. In \bibinfo{booktitle}{\emph{Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}}. \bibinfo{pages}{4902--4912}.
\newblock
\bibitem[Robertson et~al\mbox{.}(2020)]%
{robertson2020uses}
\bibfield{author}{\bibinfo{person}{Craig~T Robertson}, \bibinfo{person}{Rachel~R Mour{\~a}o}, {and} \bibinfo{person}{Esther Thorson}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Who uses fact-checking sites? The impact of demographics, political antecedents, and media use on fact-checking site awareness, attitudes, and behavior}.
\newblock \bibinfo{journal}{\emph{The International Journal of Press/Politics}} \bibinfo{volume}{25}, \bibinfo{number}{2} (\bibinfo{year}{2020}), \bibinfo{pages}{217--237}.
\newblock
\bibitem[Shang et~al\mbox{.}(2020)]%
{shang2020fauxward}
\bibfield{author}{\bibinfo{person}{Lanyu Shang}, \bibinfo{person}{Yang Zhang}, \bibinfo{person}{Daniel Zhang}, {and} \bibinfo{person}{Dong Wang}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Fauxward: a graph neural network approach to fauxtography detection using social media comments}.
\newblock \bibinfo{journal}{\emph{Social Network Analysis and Mining}} \bibinfo{volume}{10}, \bibinfo{number}{1} (\bibinfo{year}{2020}), \bibinfo{pages}{76}.
\newblock
\bibitem[Shim and Jhaver(2024)]%
{shim2024incorporating}
\bibfield{author}{\bibinfo{person}{Yunhee Shim} {and} \bibinfo{person}{Shagun Jhaver}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Incorporating procedural fairness in flag submissions on social media platforms}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2409.08498}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Sizov et~al\mbox{.}(2024)]%
{sizov2024analysing}
\bibfield{author}{\bibinfo{person}{Fedor Sizov}, \bibinfo{person}{Cristina Espa{\~n}a-Bonet}, \bibinfo{person}{Josef van Genabith}, \bibinfo{person}{Roy Xie}, {and} \bibinfo{person}{Koel~Dutta Chowdhury}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Analysing translation artifacts: A comparative study of LLMs, NMTs, and human translations}. In \bibinfo{booktitle}{\emph{Proceedings of the Ninth Conference on Machine Translation}}. \bibinfo{pages}{1183--1199}.
\newblock
\bibitem[Straub and Spradling(2022)]%
{straub2022americans}
\bibfield{author}{\bibinfo{person}{Jeremy Straub} {and} \bibinfo{person}{Matthew Spradling}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Americans' perspectives on online media warning labels}.
\newblock \bibinfo{journal}{\emph{Behavioral Sciences}} \bibinfo{volume}{12}, \bibinfo{number}{3} (\bibinfo{year}{2022}), \bibinfo{pages}{59}.
\newblock
\bibitem[{Tiktok}(2025)]%
{tiktok_footnotes}
\bibfield{author}{\bibinfo{person}{{Tiktok}}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{Testing a new feature to enhance content on TikTok}.
\newblock
\newblock
\urldef\tempurl%
\url{https://newsroom.tiktok.com/en-us/footnotes}
\showURL{%
\tempurl}
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Walter et~al\mbox{.}(2021)]%
{walter2021evaluating}
\bibfield{author}{\bibinfo{person}{Nathan Walter}, \bibinfo{person}{John~J Brooks}, \bibinfo{person}{Camille~J Saucier}, {and} \bibinfo{person}{Sapna Suresh}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Evaluating the impact of attempts to correct health misinformation on social media: A meta-analysis}.
\newblock \bibinfo{journal}{\emph{Health Communication}} \bibinfo{volume}{36}, \bibinfo{number}{13} (\bibinfo{year}{2021}), \bibinfo{pages}{1776--1784}.
\newblock
\bibitem[Wang and Shu(2023)]%
{wang2023explainable}
\bibfield{author}{\bibinfo{person}{Haoran Wang} {and} \bibinfo{person}{Kai Shu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Explainable claim verification via knowledge-grounded reasoning with large language models}. In \bibinfo{booktitle}{\emph{Findings of the Association for Computational Linguistics: EMNLP 2023}}. \bibinfo{pages}{6288--6304}.
\newblock
\bibitem[Wang et~al\mbox{.}(2022)]%
{wang2022text}
\bibfield{author}{\bibinfo{person}{Liang Wang}, \bibinfo{person}{Nan Yang}, \bibinfo{person}{Xiaolong Huang}, \bibinfo{person}{Binxing Jiao}, \bibinfo{person}{Linjun Yang}, \bibinfo{person}{Daxin Jiang}, \bibinfo{person}{Rangan Majumder}, {and} \bibinfo{person}{Furu Wei}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Text embeddings by weakly-supervised contrastive pre-training}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2212.03533}} (\bibinfo{year}{2022}).
\newblock
\bibitem[Wang et~al\mbox{.}(2024)]%
{wang2024lower}
\bibfield{author}{\bibinfo{person}{Stephanie Wang}, \bibinfo{person}{Shengchun Huang}, \bibinfo{person}{Alvin Zhou}, {and} \bibinfo{person}{Dana{\"e} Metaxa}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Lower quantity, higher quality: Auditing news content and user perceptions on Twitter/X algorithmic versus chronological timelines}.
\newblock \bibinfo{journal}{\emph{Proceedings of the ACM on Human-Computer Interaction}} \bibinfo{volume}{8}, \bibinfo{number}{CSCW2} (\bibinfo{year}{2024}), \bibinfo{pages}{1--25}.
\newblock
\bibitem[Warren et~al\mbox{.}(2025)]%
{warren2025show}
\bibfield{author}{\bibinfo{person}{Greta Warren}, \bibinfo{person}{Irina Shklovski}, {and} \bibinfo{person}{Isabelle Augenstein}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Show me the work: Fact-checkers' requirements for explainable automated fact-Checking}. In \bibinfo{booktitle}{\emph{Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems}}. \bibinfo{pages}{1--21}.
\newblock
\bibitem[{Wikipedia contributors}(2025)]%
{bilibili_wiki}
\bibfield{author}{\bibinfo{person}{{Wikipedia contributors}}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{Bilibili}.
\newblock
\newblock
\urldef\tempurl%
\url{https://en.wikipedia.org/wiki/Bilibili}
\showURL{%
\tempurl}
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Wu et~al\mbox{.}(2017)]%
{wu2017sequential}
\bibfield{author}{\bibinfo{person}{Bo Wu}, \bibinfo{person}{Wen-Huang Cheng}, \bibinfo{person}{Yongdong Zhang}, \bibinfo{person}{Qiushi Huang}, \bibinfo{person}{Jintao Li}, {and} \bibinfo{person}{Tao Mei}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Sequential prediction of social media popularity with deep temporal context networks}. In \bibinfo{booktitle}{\emph{Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence (IJCAI)}}. \bibinfo{pages}{3062--3068}.
\newblock
\urldef\tempurl%
\url{https://doi.org/10.24963/ijcai.2017/427}
\showDOI{\tempurl}
\bibitem[Xu et~al\mbox{.}(2023)]%
{xu2023combating}
\bibfield{author}{\bibinfo{person}{Danni Xu}, \bibinfo{person}{Shaojing Fan}, {and} \bibinfo{person}{Mohan Kankanhalli}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Combating misinformation in the era of generative AI models}. In \bibinfo{booktitle}{\emph{Proceedings of the 31st ACM International Conference on Multimedia}}. \bibinfo{pages}{9291--9298}.
\newblock
\bibitem[Yadav et~al\mbox{.}(2020)]%
{yadav2020semantic}
\bibfield{author}{\bibinfo{person}{Aman Yadav}, \bibinfo{person}{Rishab Sharma}, {and} \bibinfo{person}{Fatemeh~H Fard}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{A semantic-based framework for analyzing app users' feedback}. In \bibinfo{booktitle}{\emph{2020 IEEE 27th International Conference on Software Analysis, Evolution and Reengineering (SANER)}}. IEEE, \bibinfo{pages}{572--576}.
\newblock
\bibitem[Yanagi et~al\mbox{.}(2020)]%
{yanagi2020fake}
\bibfield{author}{\bibinfo{person}{Yuta Yanagi}, \bibinfo{person}{Ryohei Orihara}, \bibinfo{person}{Yuichi Sei}, \bibinfo{person}{Yasuyuki Tahara}, {and} \bibinfo{person}{Akihiko Ohsuga}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Fake news detection with generated comments for news articles}. In \bibinfo{booktitle}{\emph{2020 IEEE 24th International Conference on Intelligent Engineering Systems (INES)}}. IEEE, \bibinfo{pages}{85--90}.
\newblock
\bibitem[Yang et~al\mbox{.}(2024)]%
{yang2024llm}
\bibfield{author}{\bibinfo{person}{Joshua~C Yang}, \bibinfo{person}{Damian Dalisan}, \bibinfo{person}{Marcin Korecki}, \bibinfo{person}{Carina~I Hausladen}, {and} \bibinfo{person}{Dirk Helbing}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Llm voting: Human choices and ai collective decision-making}. In \bibinfo{booktitle}{\emph{Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society}}, Vol.~\bibinfo{volume}{7}. \bibinfo{pages}{1696--1708}.
\newblock
\bibitem[{Youtube Official Blog}(2024)]%
{youtube_crowd_notes}
\bibfield{author}{\bibinfo{person}{{Youtube Official Blog}}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{title}{Testing new ways to offer viewers more context and information on videos}.
\newblock
\newblock
\urldef\tempurl%
\url{https://blog.youtube/news-and-events/new-ways-to-offer-viewers-more-context/}
\showURL{%
\tempurl}
\newblock
\shownote{[Accessed: 2025-09-01]}.
\bibitem[Yu et~al\mbox{.}(2024)]%
{yu2024fake}
\bibfield{author}{\bibinfo{person}{Xiaomin Yu}, \bibinfo{person}{Yezhaohui Wang}, \bibinfo{person}{Yanfang Chen}, \bibinfo{person}{Zhen Tao}, \bibinfo{person}{Dinghao Xi}, \bibinfo{person}{Shichao Song}, \bibinfo{person}{Simin Niu}, {and} \bibinfo{person}{Zhiyu Li}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Fake artificial intelligence generated contents (faigc): A survey of theories, detection methods, and opportunities}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2405.00711}} (\bibinfo{year}{2024}).
\newblock
\bibitem[Yue et~al\mbox{.}(2024)]%
{yue2024evidence}
\bibfield{author}{\bibinfo{person}{Zhenrui Yue}, \bibinfo{person}{Huimin Zeng}, \bibinfo{person}{Yimeng Lu}, \bibinfo{person}{Lanyu Shang}, \bibinfo{person}{Yang Zhang}, {and} \bibinfo{person}{Dong Wang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Evidence-driven retrieval augmented response generation for online misinformation}. In \bibinfo{booktitle}{\emph{Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)}}. \bibinfo{pages}{5628--5643}.
\newblock
\bibitem[Zeng and Gao(2024)]%
{zeng2024justilm}
\bibfield{author}{\bibinfo{person}{Fengzhu Zeng} {and} \bibinfo{person}{Wei Gao}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Justilm: Few-shot justification generation for explainable fact-checking of real-world claims}.
\newblock \bibinfo{journal}{\emph{Transactions of the Association for Computational Linguistics}} \bibinfo{volume}{12} (\bibinfo{year}{2024}), \bibinfo{pages}{334--354}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2023c)]%
{zhang2023_flagging}
\bibfield{author}{\bibinfo{person}{Alice~Qian Zhang}, \bibinfo{person}{Kaitlin Montague}, {and} \bibinfo{person}{Shagun Jhaver}.} \bibinfo{year}{2023}\natexlab{c}.
\newblock \showarticletitle{Cleaning up the streets: Understanding motivations, mental models, and concerns of users flagging social media content}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2309.06688}} (\bibinfo{year}{2023}).
\newblock
\bibitem[Zhang et~al\mbox{.}(2017)]%
{zhang2017wikum}
\bibfield{author}{\bibinfo{person}{Amy~X. Zhang}, \bibinfo{person}{Lea Verou}, {and} \bibinfo{person}{David Karger}.} \bibinfo{year}{2017}\natexlab{}.
\newblock \showarticletitle{Wikum: Bridging discussion forums and wikis using recursive summarization}. In \bibinfo{booktitle}{\emph{Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing}}. \bibinfo{pages}{2082--2096}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2018)]%
{zhang2018fauxbuster}
\bibfield{author}{\bibinfo{person}{Daniel~Yue Zhang}, \bibinfo{person}{Lanyu Shang}, \bibinfo{person}{Biao Geng}, \bibinfo{person}{Shuyue Lai}, \bibinfo{person}{Ke Li}, \bibinfo{person}{Hongmin Zhu}, \bibinfo{person}{Md~Tanvir Amin}, {and} \bibinfo{person}{Dong Wang}.} \bibinfo{year}{2018}\natexlab{}.
\newblock \showarticletitle{Fauxbuster: A content-free fauxtography detector using social media comments}. In \bibinfo{booktitle}{\emph{2018 IEEE International Conference on Big Data (big data)}}. IEEE, \bibinfo{pages}{891--900}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2023b)]%
{zhang2023twhin}
\bibfield{author}{\bibinfo{person}{Xinyang Zhang}, \bibinfo{person}{Yury Malkov}, \bibinfo{person}{Omar Florez}, \bibinfo{person}{Serim Park}, \bibinfo{person}{Brian McWilliams}, \bibinfo{person}{Jiawei Han}, {and} \bibinfo{person}{Ahmed El-Kishky}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{Twhin-bert: A socially-enriched pre-trained language model for multilingual tweet representations at twitter}. In \bibinfo{booktitle}{\emph{Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}}. \bibinfo{pages}{5597--5607}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2023a)]%
{zhang2023sinophobia}
\bibfield{author}{\bibinfo{person}{Yongjun Zhang}, \bibinfo{person}{Hao Lin}, \bibinfo{person}{Yi Wang}, {and} \bibinfo{person}{Xinguang Fan}.} \bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{Sinophobia was popular in Chinese language communities on Twitter during the early COVID-19 pandemic}.
\newblock \bibinfo{journal}{\emph{Humanities and Social Sciences Communications}} \bibinfo{volume}{10}, \bibinfo{number}{1} (\bibinfo{year}{2023}), \bibinfo{pages}{1--12}.
\newblock
\bibitem[Zhang et~al\mbox{.}(2024)]%
{zhang2024fairabstractive}
\bibfield{author}{\bibinfo{person}{Yusen Zhang}, \bibinfo{person}{Li Zhou}, \bibinfo{person}{Harini Suresh}, \bibinfo{person}{Imed Zitouni}, {and} \bibinfo{person}{Milad Shokouhi}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Fair abstractive summarization of diverse perspectives}. In \bibinfo{booktitle}{\emph{Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}}. \bibinfo{pages}{3356--3372}.
\newblock
\bibitem[Zhou et~al\mbox{.}(2023)]%
{zhou2023navigating}
\bibfield{author}{\bibinfo{person}{Kaitlyn Zhou}, \bibinfo{person}{Dan Jurafsky}, {and} \bibinfo{person}{Tatsunori~B Hashimoto}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \showarticletitle{Navigating the grey area: How expressions of uncertainty and overconfidence affect language models}. In \bibinfo{booktitle}{\emph{Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}}. \bibinfo{pages}{5506--5524}.
\newblock
\bibitem[Zhou et~al\mbox{.}(2024)]%
{zhou2024correcting}
\bibfield{author}{\bibinfo{person}{Xinyi Zhou}, \bibinfo{person}{Ashish Sharma}, \bibinfo{person}{Amy~X Zhang}, {and} \bibinfo{person}{Tim Althoff}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Correcting misinformation on social media with a large language model}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2403.11169}} (\bibinfo{year}{2024}).
\newblock
\end{thebibliography}