AI Cartography: Mapping the Latent Landscape of AI Benchmark Ecosystems
Extracted source text · arXiv 2605.25272
%%%%%%%% ICML 2026 EXAMPLE LATEX SUBMISSION FILE %%%%%%%%%%%%%%%%%
\documentclass{article}
% Recommended, but optional, packages for figures and better typesetting:
\usepackage{microtype}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs} % for professional tables
\usepackage{mathrsfs} % for script
% hyperref makes hyperlinks in the resulting PDF.
% If your build breaks (sometimes temporarily if a hyperlink spans a page)
% please comment out the following usepackage line and replace
% \usepackage{icml2026} with \usepackage[nohyperref]{icml2026} above.
\usepackage{hyperref}
% Attempt to make hyperref and algorithmic work together better:
\newcommand{\theHalgorithm}{\arabic{algorithm}}
% Use the following line for the initial blind version submitted for review:
% \usepackage{icml2026}
% For preprint, use
% \usepackage[preprint]{icml2026}
% If accepted, instead use the following line for the camera-ready submission:
\usepackage[accepted]{icml2026}
\usepackage{amsmath,amssymb,amsfonts}%
\usepackage{inconsolata}
\usepackage[T1]{fontenc} % Crucial for properly rendering quotes and symbols
\usepackage{textcomp} % Enables proper straight quotes
% \usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
\usepackage{tabularx,booktabs,adjustbox,makecell}
\newcolumntype{Y}{>{\raggedright\arraybackslash}X} % wrapped, left-aligned X column
\usepackage{enumitem}
% if you use cleveref..
\usepackage[capitalize,noabbrev]{cleveref}
% \usepackage{minted}
\usepackage[english]{babel}
% \newenvironment{longlisting}{\captionsetup{type=listing}}{}
\usepackage{longtable}
% \usepackage{rotating}
\usepackage{lineno}
\usepackage{caption}
\usepackage{listings}
\usepackage{xcolor}
\newenvironment{longlisting}{\captionsetup{type=listing}}{}
\newcommand{\cmtt}[1]{{\fontfamily{cmtt}\selectfont #1\normalfont}}
\usepackage{algorithm}
\usepackage{tabularray}
\usepackage{float}
\usepackage{graphicx}
% \usepackage{titletoc}
% \usepackage{minitoc}
% \doparttoc
% \usepackage[title,toc,page]{appendix}
% \usepackage{tocloft}
\usepackage{etoc}
\usepackage{codehigh}
\usepackage[normalem]{ulem}
\UseTblrLibrary{booktabs}
\usepackage{setspace}
\usepackage{threeparttable} % For better table notes
\UseTblrLibrary{siunitx}
\newcommand{\tinytableTabularrayUnderline}[1]{\underline{#1}}
\newcommand{\tinytableTabularrayStrikeout}[1]{\sout{#1}}
\NewTableCommand{\tinytableDefineColor}[3]{\definecolor{#1}{#2}{#3}}
% \usepackage{hyperref}
\usepackage{makecell} % Required package
\renewcommand\theadfont{\normalsize} % Optional: Adjust font size if needed
\lstset{
basicstyle=\ttfamily\small,
breaklines=true, % Enable line wrapping
frame=single, % Adds a frame around the code
% numbers=left, % Line numbers on the left
breakatwhitespace=false, % Allow breaks at any character, not just spaces
breakindent=20pt, % Indent the wrapped portion of the line
postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space}, % Visual marker for wrapped lines
% numberstyle=\tiny,
keywordstyle=\color{blue},
commentstyle=\color{gray},
stringstyle=\color{red},
showstringspaces=false
}
\makeatletter
\let\tableofcontents\relax
\makeatother
\usepackage{tocloft}
\usepackage{titletoc}
% Custom appendix TOC
% \newcommand{\appendixtoc}{%
% \begingroup
% \setcounter{tocdepth}{3}
% \startcontents[appendix]
% \printcontents[appendix]{}{1}{}
% \endgroup
% }
\newcommand{\appendixtoc}{%
\printcontents[appendix]{}{1}{\setcounter{tocdepth}{3}}%
}
\newcommand{\appsection}[1]{%
\section{#1}
\addcontentsline{toc}{section}{Appendix \thesection\quad #1}
}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% THEOREMS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
% Todonotes is useful during development; simply uncomment the next line
% and comment out the line below the next line to turn off comments
%\usepackage[disable,textsize=tiny]{todonotes}
\usepackage[textsize=tiny]{todonotes}
% The \icmltitle you define below is probably too long as a header.
% Therefore, a short form for the running title is supplied here:
\icmltitlerunning{AI Cartography: Mapping the Latent Landscape of AI Benchmark Ecosystems}
\begin{document}
\twocolumn[
\icmltitle{AI Cartography: Mapping the Latent Landscape of AI Benchmark Ecosystems}
% \icmltitle{Noise Tectonics: Measuring the Stability of AI Benchmark Ecosystems}
% It is OKAY to include author information, even for blind submissions: the
% style file will automatically remove it for you unless you've provided
% the [accepted] option to the icml2026 package.
% List of affiliations: The first argument should be a (short) identifier you
% will use later to specify author affiliations Academic affiliations
% should list Department, University, City, Region, Country Industry
% affiliations should list Company, City, Region, Country
% You can specify symbols, otherwise they are numbered in order. Ideally, you
% should not use this facility. Affiliations will be numbered in order of
% appearance and this is the preferred way.
\icmlsetsymbol{equal}{*}
\begin{icmlauthorlist}
\icmlauthor{Michael Hardy}{equal,yyy}
\icmlauthor{Anka Reuel}{yyy}
\icmlauthor{Lijin Zhang}{yyy}
\icmlauthor{Jodi M. Casabianca}{comp}
\icmlauthor{Sang Truong}{yyy}
\icmlauthor{Yash Dave}{yyy}
\icmlauthor{Hansol Lee}{yyy}
\icmlauthor{Benjamin Domingue}{yyy}
\icmlauthor{Sanmi Koyejo}{yyy}
\end{icmlauthorlist}
\icmlaffiliation{yyy}{Stanford University, Stanford, CA, USA}
\icmlaffiliation{comp}{BroadMetrics, Edison, NJ, USA}
% \icmlaffiliation{sch}{School of ZZZ, Institute of WWW, Location, Country}
\icmlcorrespondingauthor{Michael Hardy}{\texttt{hardym[}$\alpha\tau$\texttt{]stanford[}$\circ$\texttt{]edu}}
% You may provide any keywords that you find helpful for describing your
% paper; these are used to populate the "keywords" metadata in the PDF but
% will not be shown in the document
\icmlkeywords{Machine Learning, ICML, benchmark, leaderboard, reliability, generalizability, factor analysis, evaluation, latent regression}
\vskip 0.3in
]
% this must go after the closing bracket ] following \twocolumn[ ...
% This command actually creates the footnote in the first column listing the
% affiliations and the copyright notice. The command takes one argument, which
% is text to display at the start of the footnote. The \icmlEqualContribution
% command is standard text for equal contribution. Remove it (just {}) if you
% do not need this facility.
% Use ONE of the following lines. DO NOT remove the command.
% If you have no special notice, KEEP empty braces:
\printAffiliationsAndNotice{} % no special notice (required even if empty)
% Or, if applicable, use the standard equal contribution text:
% \printAffiliationsAndNotice{\icmlEqualContribution}
\input{sections/0abstract}
\input{sections/1intro_reduce_draft}
% \input{sections/1intro}
\input{sections/new_methods}
% \input{sections/2back_related_comb_draft}
% \input{sections/2relatedwork}
% \input{sections/3background}
% \input{sections/4methods}
\input{sections/6exp}
\input{sections/7discussion}
\input{sections/8conclusion}
\input{sections/9impact}
\section*{Acknowledgments} We thank Mark Wilson for his feedback and support. We are also grateful for the members of Berkeley's BEAR Center, Stanford STAIR Lab, Stanford Psychometrics Lab, and reviewers for constructive feedback.
\bibliography{references}
\bibliographystyle{icml2026}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
\newpage
\appendix
\onecolumn
\input{appendix/addl_background_related}
\input{appendix/dataset_details}
\input{appendix/latent_structure_descriptions}
\input{appendix/local_dependence_via_LM_MI}
\input{appendix/bootstrap_details}
\input{appendix/CFA_fit_stats}
\input{appendix/bifactor_stat_tests}
\input{appendix/var_decomp}
\input{appendix/latent_regressions}
\input{appendix/code_listings}
\input{appendix/extended_discussion_results}
\input{appendix/notation_table}
% \printAffiliationsAndNotice{}
\end{document}
\begin{abstract}
While aggregate leaderboard scores drive AI development, they contain substantial measurement noise whose sources and magnitudes remain unquantified, making it unclear when rankings reflect genuine capability differences versus evaluation artifacts. We introduce a framework for measuring the latent landscape in AI benchmark ecosystems. Applying Confirmatory Factor Analysis (CFA) and Generalizability Theory to 4,000+ models from the Open LLM Leaderboard, we decompose sources of ranking variance and establish:
(1) structures assumed in current reporting practice underestimate the strength of relationships between benchmarks;
(2) evidence of local dependence among leaderboard items, undermining uses of benchmarks as measurement instruments under current scoring systems;
(3) contributor metadata explains more rank-relevant variance ($\approx9\%$) than architecture or deployment categories in this context;
(4) a manifest-score ``scaling law'' slope has low reliability ($\mathcal{R}_{\beta}=0.53$); by contrast, the latent general-factor size slope is highly stable across ecosystem controls ($\mathcal{R}_g=0.97$).
We are able to provide unique insights into benchmark dynamics, such as which benchmarks are a function of LLM size and which can be oppositely impacted by post-training practices. We provide actionable diagnostics to determine how benchmark rankings can be trusted and how benchmark design can be improved.
\end{abstract}
% (1) we can more reliably rank LLMs by their human contributor information than by the architecture or deployment type, accounting for $\sim9\%$ and 4.8\%, respectively, of variance in scores,
% human contributors account for more variance (9\%) than model architecture (4.8\%), revealing that nonrandom benchmark noise is identifiable as much from submission practices as from model characteristics;
% (2) model size accounts for 40\% of observed variance, dominating aggregate rankings without explicit control; and (3) scaling laws exhibit context-dependent reliability (Benchmark PSI = 30\%). We introduce novel metrics, including Slope Signal-to-Noise Ratio (SNR$_\beta$) and Proportion of Slope Instability (PSI), and provide actionable diagnostics for when benchmark rankings can be trusted and how benchmark design can be improved.
% (1) Current reporting practices do not reflect the most likely organization of LLM capabilities.
% that one can more reliably rank LLM performance by contributor information than by either architecture or post-training regimes;
% a general ``scaling law'' of model size for leaderboard ranking has a reliability of 0.53, but after controlling for ecosystem influences, the reliability of general capability scaling increases to 0.97.
% We measure human influences on AI performance from the data.
\section{Introduction}
\label{sec:introduction}
AI benchmark ecosystems have become the measurement instruments of modern machine learning.
Open leaderboards such as HELM~\citep{liang_holistic_2023} and the Open LLM Leaderboard~\citep{fourrier_open_2024} summarize model behavior on diverse tasks (e.g., MMLU, \citealt{hendrycks_measuring_2020}; BBH, \citealt{suzgun_challenging_2022}) into a small set of per-benchmark scores and then a single composite.
This compression is operationally convenient, but it hides strong and rarely validated assumptions about what the scores \emph{mean}~\citep{salaudeen_measurement_2025,reuel_betterbench_2024,truong_fantastic_2025,casabianca_psychometrics_2025}. A benchmark score is typically treated as a scalar measure of a \textit{latent} intended construct (e.g., ``reasoning'').
Formally, this assumes a latent variable exists and that aggregation is valid: item responses reflect a low-dimensional capability vector plus measurement error.
% When these assumptions fail, aggregation does not merely add variance--it produces \emph{structural confounding}:
% benchmark totals can mix general capability, benchmark overlap, redundant items, and evaluation artifacts into a single number
When these assumptions fail, aggregation can produce \textit{structural confounding}: the same total score may combine general capability, benchmark-specific residual structure, item redundancy, and evaluation artifacts in unknown proportions.
(see \S~\ref{apx:related_work}).
This matters because the ecosystem is central in AI development, referenced for funding, and used to infer scaling laws and compare training interventions.
% AI leaderboard rankings can conflate size with developer innovation.
% Benchmarks are intended to measure \emph{latent constructs} that are not directly observable, but do so through imperfect instruments.
We seek to improve practitioner decisions by disentangling the various signals within benchmark scores.
\begin{figure}[th]
\centering
\includegraphics[width=1\linewidth]{figures/cartography_overview.pdf}
\caption{AI Latent Landscape Cartography: Study Overview}
\label{fig:tectonics}
\end{figure}
\paragraph{Estimating instead of assuming.} A leaderboard ranking is implicitly a measurement-theoretic claim: that a scalar composite of item-level responses reflects one or more latent capabilities of interest, and that variation in the composite score corresponds to variation in the latent capability under measurement. Making this claim explicit requires:
\begin{enumerate}[leftmargin=*,nosep]
\item \textbf{Exploring and confirming internal structure.} Which latent structure best explains the observed dependence among benchmark items? Current reporting assumes each benchmark measures a single independent construct; violating this assumption causes structural confounding (\S\ref{sec:m1}, Fig. \ref{fig:cfa_diagram}).
\item \textbf{Ecosystem noise decomposition.} How much of the observed score variance is attributable to construct irrelevant ecosystem features--benchmark, contributor practices, architecture, deployment? Without quantifying these, rankings can conflate signal with artifact (\S\ref{sec:m2}).
\item \textbf{Covariate-controlled inference.} Can model size be disentangled from training innovation once latent structure and ecosystem variance are modeled jointly? Na\"ive regressions on manifest scores confound all three (\S\ref{sec:m3}).
\end{enumerate}
In response to those needs, we present a framework for measuring AI capabilities and scaling laws controlling for many sources of variance in AI benchmark ecosystems. We measure the score variation found within each LLM (\S~\ref{sec:m1}) and contributed by their ecosystem (\S~\ref{sec:m2}) before combining these tools to map de-noised scaling laws (\S~\ref{sec:m3}). In brief, we measure the leaderboard ecosystem as a noisy ``landscape'' of structural, ecosystem, and statistical variation. This framework has three progressive methods (Fig. \ref{fig:tectonics}):
\begin{enumerate}[leftmargin=*,nosep]
\item \textbf{Method 1: Structural discovery (CFA/structural equation modeling [SEM]):} we statistically test hypothetical latent internal structures
% (single-$g$, correlated factors, hierarchical, bifactor)
% meta-analytically
across bootstrapped sets of items
% with permutation controls, and we conduct
% including comprehensive
% % 1-degree of freedom (1-df)
% tests for measuring local dependence
~\citep{bollen_structural_1989,browne_alternative_1992,brown_confirmatory_2015,muthen_goodness_1993,saris_testing_2009}. Because the variables of interest are not observed, we take rigorous approaches to determine the latent structure that best explains AI capabilities, avoiding overfit propensities %, using multiple tests to interrogate these structures' propensities to overfit
(\S~\ref{sec:m1}, \ref{apx:models}-\ref{apx:math}). \textbf{Preview}: \textit{regardless of the latent structure tested, significant nonrandom error persists }(unrelated to the AI capability of interest) that could threaten the validity of leaderboard score comparisons.
\item \textbf{Method 2: Ecosystem variance decomposition (G-theory):} we use a fully crossed mixed-effects variance decomposition model (architecture, benchmark, contributor/provenance, deployment type, \S~\ref{sec:m2}, \ref{apx:var_decomp}) to quantify sources of leaderboard variability and ranking reliability~\citep{brennan_generalizability_2001,cronbach_my_2004,hardy_all_2025}.
% Thus, we decompose the ecosystem noise separating sources of variation and their interactions (\S~\ref{sec:m2}), observing the same generalizable distribution of variance across estimations.
% We show that
\textbf{Preview}: \textit{we can more reliably rank-order AI performance by contributor metadata than we can based on architecture or deployment type}.
\item \textbf{Method 3: Mixed-effects latent regression:} we regress AI behaviors on the bifactor structure from Method 1 above using noise structures from Method 2 in a latent regression to estimate scaling laws specific to observable practices in ML
%, while correcting pseudo-replication and controlling deployment/provenance
~\citep{de_boeck_explanatory_2011,chalmers_extended_2015}.
Using latent regression (\S~\ref{sec:m3}, \ref{apx:latreg}), we estimate ecosystem effects on specific AI benchmark behaviors so we can infer what tuning practices help underlying capabilities as they scale. \textbf{Preview}: \textit{while scaling improves performance, this is not uniformly true}, as scaling progress in one benchmark can be negatively associated with performance in another.
% With explicit controls, \textit{we can disentangle scale vs innovation} in benchmark rankings, showing that scaling progress in one benchmark can be negatively associated with performance in another.
\end{enumerate}
% \paragraph{Estimating instead of assuming.}
% Let $y_{ij}\in\{0,1\}$ denote the response of model submission $i$ to item $j$ pooled across benchmarks.
% Latent measurement models posit abilities $\boldsymbol{\eta}_i\in\mathbb{R}^m$ such that the dependence structure of item responses is explained by a low-rank component plus item-specific noise.
% Under a standard confirmatory factor analysis (CFA; \citeauthor{brown_confirmatory_2015}, \citeyear{brown_confirmatory_2015}) latent-response formulation,
% \begin{equation}
% \label{eq:intro_cfa}
% \mathbf{y}^\star_i=\boldsymbol{\Lambda}\boldsymbol{\eta}_i+\boldsymbol{\epsilon}_i,
% \qquad
% \boldsymbol{\Sigma}(\boldsymbol{\theta})=\boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top+\boldsymbol{\Theta},
% \end{equation}
% so the inferential target is the observed (tetrachoric/robust) correlation matrix, not just mean accuracy.
% Different scientific hypotheses about ``what benchmarks measure'' correspond to different constraints on
% $(\boldsymbol{\Lambda},\boldsymbol{\Phi},\boldsymbol{\Theta})$ and can be tested.
% \begin{proposition}[Why ecosystem totals can be misleading]
% \label{prop:intro_nonsuff}
% If the ecosystem is multidimensional ($m>1$) and items load on multiple factors, then benchmark totals are not sufficient statistics for capability profiles: there exist $i\neq i'$ with identical benchmark totals but $\boldsymbol{\eta}_i\neq\boldsymbol{\eta}_{i'}$.
% \end{proposition}
% \paragraph{Ecosystem complication: pseudo-replication.}
% Unlike typical ML datasets, leaderboard submissions are not independent and identically distributed (i.i.d.)\ draws: many entries are fine-tunes, merges, or quantized variants of a few base families, inducing strong dependence across ``models''.
% Ignoring this structure inflates effective sample size and can bias latent structure estimates and scaling conclusions.
% ============================================================
\paragraph{How the three methods interlock}\label{sec:interlock}
% ============================================================
% Each question motivates one of the three methods below.
% CFA tests which underlying capacities most likely give rise to the patterns of observed model responses among benchmark items; G-theory quantifies the proportion of observed variation in scores attributable to components within the ecosystem (benchmark, contributor, architecture, deployment); and, building on those, latent mixed regression asks how do model descriptors like size and fine-tuning approaches affect benchmark performance after controlling for sources of ecosystem variation.
The framework's sequential logic is deliberate: Method~1 establishes \emph{what} is being measured, Method~2 quantifies \emph{how noisy} measurements are, and Method~3 estimates \emph{what drives} variation in the latent quantities after controlling for both structure and noise.
% The three methods form a progressive refinement, each building on insights from its predecessor:
CFA identifies a bifactor latent structure, motivating an investigation into general scaling laws and reveals persistent residual dependencies, motivating generalizability studies of ecosystem noise sources that latent structures alone cannot absorb. Then, variance decomposition measures the facets of the noise landscape, such as contributor variance ($\approx$9\%, Table \ref{tab:varcomp_slope}) and model size-linked variance ($\approx$40\%, \S~\ref{sec:size_linked_variance_corrected}), but this estimation cannot disentangle these sources across \emph{latent} levels.
This motivates latent regression, which regresses capacities directly on size and post-training while controlling for ecosystem variance, completing the decontaminated picture of how model capabilities are shaped. % via
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{Method 1} $\to$ \textbf{Method 2}: CFA establishes that the latent structure is bifactor-family (general + benchmark specific capacities), motivating the choice of the general factor as a target of scaling-law analysis. The persistent residual dependencies identified %by MI/SEPC diagnostics
% motivate measuring ecosystem noise sources that the latent structure alone cannot absorb.
% \item \textbf{Method 2} $\to$ \textbf{Method 3}: G-theory reveals that contributor variance ($\approx$9\%) and model size ($\approx$40\%)
% contribute substantially to the noise landscape, but this estimation cannot disentangle these sources across \emph{latent} levels. This motivates the latent regression, which regresses capacities
% % $\boldsymbol{\theta}_i$
% directly on size and deployment while controlling for other ecosystem variance, and completing the decontaminated picture of how model capabilities are shaped. % via $\boldsymbol{\zeta}$.
% % \item \textbf{Method 3} $\to$ \textbf{Synthesis}: The latent regression yields the \emph{scaling vector} $\boldsymbol{\beta}$, provenance variance $\boldsymbol{\Sigma}_u$, and deployment effects $\{\hat{\gamma}_d\}$---the complete decontaminated picture of how model characteristics shape latent abilities under the ecosystem's dependence structure.
% \end{itemize}
% \paragraph{Main findings (preview).}
% On the Open LLM Leaderboard, a bifactor-family structure best predicts the underlying relationship giving rise to model behaviors confirming the spirit of scaling laws: a strong general factor, strongly associated with model capacity, explains most cross-benchmark covariance, while benchmark-specific factors capture residual grouping. However, scaling laws and abilities are not 1:1. Variance decomposition quantifies how much scale, benchmark idiosyncrasies, provenance effects, and architecture/tuning contribute contribute to observed score variation, implying that naive rankings conflate capability with ecosystem artifacts.
% Mixed latent regression yields \emph{noise-controlled scaling vectors}:
% general capability scales robustly with size, while several benchmark-specific abilities exhibit weak, negligible, or even negative size effects once provenance and deployment are controlled. This study answers: \textit{so, what are we actually measuring?}
\paragraph{Contributions.}
% This study answers: \textit{so, what are we actually measuring?}
To the best of our knowledge, this is the first study to use these advanced methods in AI benchmarking and to measure and control for ecosystem factors. By so doing, we show how current reporting misrepresents the underlying constructs measured (\S~\ref{sec:disc_latent_landscape}). We provide de-noised scaling laws for general AI capability and for benchmark-specific constructs (Figs. \ref{fig:scaling_law}-\ref{fig:scaling_law_deploy}, \S~\ref{sec:disc_scaling_vector}). We disentangle post-training practices from model size effects (Table \ref{tab:lreg_reliab}, \S~\ref{sec:disc_scaling_vector}), and measure the share of AI score variation attributable to each source of variation (\S~\ref{sec:disc_noise_layers}, Table \ref{tab:varcomp_slope}). We demonstrate how, after noise controls, some benchmarks have markedly less usable signal (\S~\ref{sec:specific_bench_analyses}). We offer novel methodological solutions that make advanced measurement theoretic tools usable in benchmarking ecosystems (\S~\ref{sec:roadmap}). By offering clarity around ecosystem and AI factors influencing benchmark rankings (\S~\ref{sec:rank_changes}), we provide a map of the latent landscape for a popular AI benchmark suite.
% We provide (i) the first systematic statistical testing of the de facto structures assumed by leaderboard ecosystems,
% (ii) an interpretation of where the best latent structures are inadequate, (iii) %under several rigorous statistical methods including Lagrangian multiplier tests, item perturbations and permutation controls, and prediction of both out-of-sample behaviors and of the hypothesized covariance structure; (ii)
% a generalizability-theoretic
% measurement of benchmark ecosystem noise sources
% , (iv) conditional ranking reliability to measure sources within the noise
% that influence AI benchmark behaviors, and
% (iv) a rigorous measure of the strength and relationship of current practices to changes in leaderboards. %across benchmarks. %measurement of what is capability-specific, provenance-adjusted scaling estimates via mixed latent regression.
% We offer methodological solutions that make advanced psychometric tools usable in benchmarking practice, resulting in valuable insights, such as those mentioned in the abstract.
% Together, these results turn benchmark ecosystems from scoreboards and floorboards into interpretable
% % and operationalizable
% measurement systems.
% \section{Introduction}
% \label{sec:introduction}
% AI evaluation is the science of separating signal from noise. Open leaderboards like the Open LLM Leaderboard~\citep{fourrier_open_2024} and HELM~\citep{liang_holistic_2023} have become the de facto instruments for measuring progress, guiding research priorities based on aggregate scores from benchmarks like MMLU~\citep{hendrycks_measuring_2020} and BBH~\citep{suzgun_challenging_2022}. The predominant practice is to distill a model's rich, high-dimensional response profile into a single number per benchmark. While convenient, this compression rests on unvalidated and often incorrect measurement assumptions, systematically distorting what these scores represent by conflating genuine capability (signal) with a host of statistical and procedural artifacts (noise).
% This paper introduces a psychometric framework to move beyond assumption-based evaluation. We provide a systematic methodology to identify, quantify, and control for the multiple sources of noise that contaminate AI benchmark ecosystems, enabling more rigorous and interpretable assessments of model capabilities. The central challenge we address is not merely that leaderboard scores are noisy, but that there are multiple, structurally distinct forms of noise that require different statistical tools to disentangle.
% First, there is \textbf{structural noise}, arising from the misspecification of what benchmarks measure. Reporting a single score per benchmark assumes \textit{unidimensionality}—that all its items measure a single, coherent latent construct \citep{truong_fantastic_2025}. If a benchmark is actually multidimensional, its total score becomes an uninterpretable composite, allowing two models with identical scores to possess entirely different underlying ability profiles \citep{casabianca_psychometrics_2025}.
% Second, there is \textbf{ecosystem noise}, stemming from systematic variation in measurement conditions that are unrelated to a model's intrinsic capabilities. A model’s rank is a function not only of its architecture but also of its size, its specific deployment choices (e.g., quantization, fine-tuning), and its provenance (e.g., the unstated training recipes and data mixtures of the contributor who submitted it). Without explicitly modeling these factors, their effects are confounded with true architectural signal.
% Third, there is \textbf{statistical noise}, generated by the data structure itself. Leaderboard submissions are not independent draws; they are dominated by fine-tunes and minor variants of a few base architectures, creating massive pseudo-replication. Ignoring this hierarchical dependence leads to artificially inflated precision and biased conclusions.
% To decontaminate AI evaluation, we must model all noise sources. Our approach is a three-stage analysis that maps directly onto this tripartite problem.
% \begin{enumerate}
% \item \textbf{To map the latent structure (Method 1)}, we use Confirmatory Factor Analysis (CFA) within a meta-analytic bootstrap design. Unlike methods that predict labels, CFA tests hypotheses about the \emph{dependence structure} of item responses. It decomposes the covariance matrix into signal (shared variance from latent abilities) and noise (item-specific error), allowing us to formally compare competing theories of capability organization. The core CFA equation, $\boldsymbol{\Sigma}(\boldsymbol{\theta}) = \boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top + \boldsymbol{\Theta}$, makes this separation explicit.
% \item \textbf{To quantify ecosystem noise (Method 2)}, we use Generalizability Theory to decompose the variance of manifest scores into fully crossed facets: architecture, benchmark, contributor, and deployment. This allows us to estimate the proportion of variance attributable to each ecosystem factor.
% \item \textbf{To synthesize these insights and control for statistical noise (Method 3)}, we develop a Hierarchical Mixed-Effects Latent Regression. This model uses the latent structure identified in Method 1 as a measurement backbone but estimates its parameters within a mixed-effects framework that accounts for the pseudo-replication and covariates identified in Method 2. This yields "noise-controlled" estimates of how factors like model size affect true latent abilities.
% \end{enumerate}
% Applying this framework to the Open LLM Leaderboard ecosystem, we make the following contributions:
% \begin{enumerate}
% \item \textbf{We provide the first rigorous map of an AI benchmark ecosystem's latent structure,} empirically rejecting the assumption that benchmarks are independent modules. Instead, a strong general factor ($g$) explains most cross-item covariance, implying that per-benchmark scores are noisy measures that conflate general ability with benchmark-specific artifacts.
% \item \textbf{We quantify the magnitude of multiple, interacting noise sources.} We find that benchmark difficulty accounts for 44\% of score variance. Critically, after controlling for benchmark effects, variance from unobserved contributor practices (provenance noise) is nearly double that of model architecture. We show that scaling laws estimated from manifest scores are heavily confounded, with 36\% of explainable rank-order variance attributable to model size.
% \item \textbf{We deliver a framework and actionable diagnostics for building more reliable evaluations.} We introduce novel metrics (e.g., Slope SNR, Proportion of Slope Instability) to assess scaling-law reliability and demonstrate how our methods produce "noise-controlled" scaling laws that differ dramatically by latent ability. For instance, after controls, general knowledge scales robustly with size, but complex reasoning and instruction-following do not, revealing a potential trade-off between alignment and flexible reasoning.
% \end{enumerate}
% Ultimately, this paper provides both a diagnosis and a remedy. We demonstrate that current evaluation practices are structurally flawed but show how a principled, psychometric approach can separate signal from noise to yield more trustworthy, interpretable, and scientifically valid assessments of AI progress.
\section{Methods: from Benchmarking to Measuring}\label{sec:roadmap}
The central challenge in AI evaluation is separating signal (genuine capability) from noise (measurement artifacts, see Appendix \S~\ref{apx:related_work}). Leaderboards compress a high-dimensional response profile into per-benchmark accuracies and then a single composite. This compression is convenient but implicitly assumes that (i) each benchmark is an approximately unidimensional scale, (ii) benchmark totals are commensurate indicators of a smaller set of underlying capabilities, and (iii) local independence--items are no longer correlated after controlling for the latent construct. When these assumptions fail, aggregation makes scores \emph{structurally uninterpretable} and confounds signal with multiple forms of ecosystem noise (\citealt{salaudeen_measurement_2025,casabianca_psychometrics_2025}; \S~\ref{apx:related_work}).
\paragraph{Pseudo-replication.}
In particular, leaderboard submissions are not i.i.d.\@ draws from a population of independent models. Many entries are fine-tunes, merges, or quantizations of a small number of base architectures, creating hierarchical dependence that inflates precision and biases structure toward dominant families. Methods~2 and~3 control for such contribution phenomena explicitly. %: G-theory models contributor/provenance as a crossed random-effect facet, and the latent regression incorporates contributor-level random effects $\boldsymbol{\zeta}$ on latent abilities.
\paragraph{High-dimensional item pools.}
Each benchmark contains hundreds to thousands of items (Table~\ref{tab:bench_counts}), so fitting a CFA (Method 1, \S~\ref{sec:m1}) or IRT (Method 3, \S~\ref{sec:m3}) model to the full item set is computationally infeasible and invites overfitting to item-specific artifacts. We address this via \emph{meta-analytic item set bootstrapping}: for each of $B$ replications, we sample a subset of $r_k$ items per benchmark $k$, fit the target model, and aggregate results across replications % using mixed-effects meta-regression (Eq.~\ref{eq:meta_reg})
(see \ref{apx:boots}). The results are \emph{population quantities}, the expectations under the
% expected fit under the item
ecosystem-induced distribution. When meta-aggregating model statistics, Method 1 uses mixed-effects meta-regression for reported fit statistics (see \S \ref{apx:meta_regress}) and Method 3 uses inverse-error weighted methods drawn from the posterior (see \S \ref{apx:boots_inverr}).
%, rather than a point estimate conditional on a sample item draw. % induced by the ecosystem,
% \subsection{Why measurement modeling for AI benchmarks?}\label{sec:why}
% A leaderboard ranking is, implicitly, a psychometric claim: that a scalar composite of item-level responses reflects one or more latent capabilities of interest. Making this claim explicit requires three things:
% \begin{enumerate}[leftmargin=*,nosep]
% \item \textbf{Structural testing.} Which latent structure best explains the observed dependence among benchmark items? Current reporting assumes each benchmark measures a single independent construct; violating this assumption produces \emph{structural confounding} (\S\ref{sec:m1}).
% \item \textbf{Ecosystem noise decomposition.} How much of the observed score variance is attributable to the ecosystem---benchmark identity, contributor practices, architecture, deployment? Without quantifying these sources, rankings conflate signal with artifact (\S\ref{sec:m2}).
% \item \textbf{Covariate-controlled inference.} To what extent can model size be disentangled from training innovation once latent structure and ecosystem variance are modeled jointly? Na\"ive regressions on manifest scores confound all three (\S\ref{sec:m3}).
% \end{enumerate}
% Each question motivates one of the three methods below. Their sequential logic is deliberate: Method~1 establishes \emph{what} is being measured, Method~2 quantifies \emph{how noisy} those measurements are, and Method~3 estimates \emph{what drives} variation in the latent quantities after controlling for both structure and noise.
% \subsubsection{The core obstacle: $n \ll p$ and pseudo-replication}\label{sec:obstacles}
% Two features of the AI benchmark ecosystem make standard latent-variable estimation impractical and motivate the shared bootstrap-and-aggregation infrastructure underlying all three methods.
% ============================================================
\subsection{Method 1: CFA for latent structure discovery}\label{sec:m1}
% ============================================================
\begin{figure*}[htb]
\centering
\includegraphics[width=0.725\textwidth]{figures/latent_structures.pdf}
\caption{\textbf{Possible Manifest and Latent Scoring Structures} used or assumed in AI Benchmark Ecosystem. \textbf{Left (i, ii, iii)}: common manifest scores structures. \textbf{Middle (a, b, c)}: analogous latent structures, where pairs (a,i), (b,ii), and (c,iii) differ in the causal direction implied by their estimations. \textbf{Right (d, e, f)}: other potential latent structures which may be explanatory of observed scores. *Structure for a LMArena leaderboard (e.g., Text) under Bradley-Terry-style pairwise comparison unidimensionality assumptions \citep{chiang_chatbot_2024}.}
\label{fig:cfa_diagram}
\end{figure*}
% \paragraph{Goal}
CFA can statistically test which latent factor structure best predicts the observed dependence (tetrachoric/robust correlations) among benchmark items and LLM responses. Because the variables of interest--latent AI capabilities--are unobserved, we compare six competing structural hypotheses (Table~\ref{tab:structural_models}, Fig.~\ref{fig:cfa_diagram}) and let the data adjudicate among them. By so doing, we create a coarse map of LLMs' ``vectors of mind'' \citep{thurstone_vectors_1935} statistically testing each as a ``nomological network'' \citep{cronbach_construct_1955}.
\subsubsection{The General CFA measurement model}\label{sec:cfa_eqs}
For LLM $i$ and item $j$ ($j$=1, ...$p$),\footnote{A comprehensive notation guide can be found in \S~\ref{apx:roadmap:notation}} a general\footnote{The general form of the present study uses binary response data. Let $y_{ij}\in\{0,1\}$ be LLM submission $i$'s response to item $j$, and let $\boldsymbol{\eta}_i\in\mathbb{R}^m$ be latent abilities. Under a general latent-response formulation for \textit{binary} items:
% \begin{equation}
% \label{eq:found_latent_response}
$y^\star_{ij} = \nu_j + \boldsymbol{\lambda}_j^\top \boldsymbol{\eta}_i + \epsilon_{ij},\,
\epsilon_{ij}\sim\mathcal{N}(0,\varepsilon_j),\,
y_{ij}=\mathbb{1}[y^\star_{ij}>\tau_j].$
% % \end{equation}
% where $\boldsymbol{\Lambda}$ is sparse (which items are linked which factors as shown in Fig. \ref{fig:cfa_diagram}), $\boldsymbol{\Phi}=\mathrm{Cov}(\boldsymbol{\eta})$, and $\boldsymbol{E}$ is typically diagonal.
} CFA model is
\begin{equation}\label{eq:cfa}
\mathbf{y}_i = \boldsymbol{\nu} + \boldsymbol{\Lambda}\boldsymbol{\eta}_i + \boldsymbol{\epsilon}_i, \quad
\boldsymbol{\eta}_i \sim \mathcal{N}(\mathbf{0},\boldsymbol{\Phi}),\;
\boldsymbol{\epsilon}_i \sim \mathcal{N}(\mathbf{0},\boldsymbol{E}),
\end{equation}
where $\boldsymbol{\Lambda}$ is a $p \times m$ matrix of factor loadings encoding which items load on which latent factors (as in Fig. \ref{fig:cfa_diagram}), $\boldsymbol{\Phi}$ is the $m \times m$ factor covariance, and $\boldsymbol{E}$ is the (typically diagonal) residual covariance.\footnote{Traditionally, $\boldsymbol{\Theta}$ is used for $\mathrm{Cov}(\boldsymbol{\epsilon})$; we use $\boldsymbol{E}$ here to distinguish from $\boldsymbol{\Theta}$ in Method 3.} The model-implied covariance is
% \begin{equation}\label{eq:sigma}
$\boldsymbol{\Sigma}(\boldsymbol{\theta}) = \boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top + \boldsymbol{E}$. Each candidate structure $s \in \mathcal{S}$ specifies a distinct set of constraints on $(\boldsymbol{\Lambda}, \boldsymbol{\Phi}, \boldsymbol{E})$,
% \end{equation}
and estimation proceeds by minimizing a discrepancy $F\bigl(\boldsymbol{\Sigma}(\boldsymbol{\theta}),\,\mathbf{S}\bigr)$ between the model-implied and observed sample covariance $\mathbf{S}$. CFA %does \emph{not}
primarily predicts the implied \emph{dependence structure} (correlations), rather than labels; %it predicts the \emph{dependence structure} (tetrachoric/robust correlations) implied and
this enables hypothesis testing %about benchmark structures
by providing evidence confirming the internal structure of the leaderboard \cite{casabianca_psychometrics_2025}.
% \begin{remark}[CFA is not PCA]\label{rem:cfa_pca}
% Unlike PCA, CFA does not merely identify dominant directions of variance. It tests falsifiable hypotheses about \emph{dependence structure} by encoding benchmark theories as constraints, modeling measurement error explicitly, and testing whether residual covariance remains after conditioning on the latent construct. This is precisely the distinction needed to evaluate whether benchmark composites are valid measures.
% \end{remark}
\subsubsection{CFA methodological solutions}\label{sec:cfa_solutions}
% \paragraph{Meta-analytic bootstrapping across fitted item sets.} For each replication $b \in \{1,\ldots,B\}$, sample $r_k$ items per benchmark $k$ without replacement, fit all six structures $s \in \mathcal{S}$ to the pooled tetrachoric correlation matrix, and record fit statistics $t_{b,s}$ and parameter estimates $(\hat{\boldsymbol{\Lambda}}, \hat{\boldsymbol{\Phi}})$%, and local-fit diagnostics
% . Aggregate via a mixed-effects meta-regression:
% \begin{equation}\label{eq:meta_reg}
% t_{b,s,r} = \alpha_s + \beta_s r + u_b + v_b r + \varepsilon_{b,s,r},
% \end{equation}
% % where $r \in \{0,1\}$ indicates true versus permuted assignment and $(u_b, v_b)$ capture item-sample heterogeneity. This yields estimates (i)~an estimate $\alpha_s$
% of each structure's expected fit under correct assignment ($\alpha_s$) and of %, (ii)~an estimate %$\beta_s$
% of how much fit is attributable to true structure versus model flexibility ($\beta_s$), while correctly propagating item-sampling variability (\S~\ref{apx:boots}). %This targets a \emph{population quantity}, the expected fit of the item distribution induced by the ecosystem, rather than a point estimate conditional on one draw.
\paragraph{Addressing overfitting tendencies of flexible latent structures.}
Higher-capacity models (e.g., correlated bifactor) can absorb covariance artifacts unrelated to true latent organization, inflating apparent fit. \textbf{Solution 1:} \textbf{{Non-traditional out-of-sample prediction.}} For each bootstrap, we compute held-out AUC and MAE on within-sample latent factor scores, providing a direct overfitting diagnostic beyond traditional indices (Table~\ref{tab:cfa_fitstats_medians}, \S~\ref{apx:comparison_fit_stats}). \textbf{Solution 2:} \textbf{{Within-replication permutation controls.}} For each bootstrap $b$, we additionally construct a randomized item-to-benchmark mapping $\pi^{(b)}$ preserving per-benchmark counts $\{r_k\}$ and refit. The contrast $\Delta T_s^{(b)} = T(s \mid \pi^{(b)}) - T(s \mid \text{id})$ quantifies how much of $s$'s advantage depends on true benchmark grouping rather than model flexibility (\S~\ref{apx:comparison_fit_stats}, Table \ref{tab:trad_fit_perm}). \textbf{Solution 3:} \textbf{{Multiple estimation methods.}} We fit with both Diagonally Weighted Least Squares (DWLS) and Metropolis-Hastings Robbins-Monro (MH-RM) estimations.
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution~2a:] \emph{Out-of-sample prediction.} For each bootstrap, we compute held-out AUC and MAE on within-sample latent factor scores, providing a direct overfitting diagnostic beyond traditional in-sample indices (Table~\ref{tab:cfa_fitstats_medians}).
% \item[Solution~2b:] \emph{Within-replication permutation controls.} For each bootstrap $b$, we additionally construct a randomized item-to-benchmark mapping $\pi^{(b)}$ preserving per-benchmark counts $\{r_k\}$ and refit. The contrast $\Delta T_s^{(b)} = T(s \mid \pi^{(b)}) - T(s \mid \text{id})$ quantifies how much of $s$'s advantage depends on true benchmark grouping rather than generic flexibility (Appendix \ref{apx:comparison_fit_stats}).
% \end{description}
\paragraph{Identifying residual misfit in each structure by aggregating score-test trends.}
Global fit indices (e.g., RMSEA, CFI) do not localize \emph{where} a latent structure fails.
% \textbf{Solution:} \textbf{{Comprehensive score-test (modification index, MI) trend aggregation.}}
We compute the locally most powerful Lagrange-multiplier or modification-index (MI) statistics,\footnote{%Such single 1-df score tests are typically limited to hyperlocalized diagnostics and often
Typically used to identify item redundancy, bootstrapped 1-df score tests can quantify residual covariances for each structure and identify violations of local dependence that threaten the use of benchmarks as a measurement tool: %gain insight into the latent fit of thousands of 1-df tests. Often used in practice to identify redundancy, we use these to test the edges of the fits of our models and to quantify violations of local dependence that threaten the use of benchmarks as a measurement tool.
% \paragraph{Localized diagnostics (MI/SEPC).}
% To identify systematic local dependence and redundancy,
% We compute modification indices ($\mathrm{MI}$) and SEPC for constrained residual covariances.
\begin{proposition}[Modification indices as Lagrange-multiplier tests]\label{prop:mi_lm}
Let $\ell(\tilde{\boldsymbol{\theta}})$ be twice continuously differentiable at the constrained optimum $\hat{\boldsymbol{\theta}}$, with $\psi$ constrained to zero. Under standard regularity conditions, the score statistic
% \[
$\mathrm{MI}(\psi) = U_\psi(\hat{\boldsymbol{\theta}})^\top\, \mathcal{I}_{\psi\psi\cdot\theta}(\hat{\boldsymbol{\theta}})^{-1}\, U_\psi(\hat{\boldsymbol{\theta}}) \;\xrightarrow{d}\; \chi^2_1$,
% \]
where $U_\psi(\hat{\boldsymbol{\theta}}) = \partial \ell / \partial \psi \big|_{\psi=0}$ is the score and $\mathcal{I}_{\psi\psi\cdot\theta}$ is the Schur complement of the partitioned Fisher information. The SEPC $\widehat{\Delta\psi} \approx \mathcal{I}_{\psi\psi\cdot\theta}^{-1} U_\psi(\hat{\boldsymbol{\theta}})$ estimates the magnitude and sign of the residual association on a correlation scale. \emph{(Proof: \S~\ref{sec:mi_score_test}, see Appendix \S~\ref{apx:math})}
\end{proposition}} $\text{MI}(\psi)$, for every candidate residual covariance and cross-loading $\psi = \Theta_{ab}$, then aggregate Standardized Expected Parameter Changes (SEPCs) across bootstraps and structures. This maps
%\emph{where} the benchmark ecosystem violates local independence, identifying
item pairs whose residual associations exceed what the latent structure can explain, and whether such violations are stable structural properties or item-idiosyncratic (\S~\ref{apx:math}; Figs. \ref{fig:bench_rel_misfit}, \ref{fig:bench_item_misfit}).
% \paragraph{Exploiting locally most powerful statistical tests with bootstrapping.}
% \footnote{For a constrained parameter $\psi$, $
% \mathrm{MI}(\psi)=U_\psi(\hat{\boldsymbol{\theta}})^\top
% \mathcal{I}_{\psi\psi\cdot\theta}(\hat{\boldsymbol{\theta}})^{-1}
% U_\psi(\hat{\boldsymbol{\theta}})
% \;\overset{H_0}{\approx}\;\chi^2_1,
% $ with standardized expected parameter change (SEPC) used as an effect size (derivation and conditions in Appendix~\ref{apx:math}). The formal statistical foundation for MIs as LM tests is provided in Appendix \ref{sec:mi_lm_proof}. % Persistent large $|\mathrm{SEPC}|$ flags ecosystem-level redundancy.
% }
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution:] \emph{Comprehensive score-test (modification index) trend aggregation.} We compute the Lagrange-multiplier statistic $\text{MI}(\psi)$ for every candidate residual covariance $\psi = \Theta_{ab}$ and every candidate cross-loading, then aggregate Standardized Expected Parameter Changes (SEPCs) across bootstraps and structures. This maps \emph{where} the benchmark ecosystem violates local independence---identifying item pairs whose residual association exceeds what the latent structure can explain---and whether these violations are stable structural properties or item-idiosyncratic (Appendix~\ref{apx:math}; Figs. ~\ref{fig:bench_rel_misfit},~\ref{fig:bench_item_misfit}).
% \end{description}
% ============================================================
\input{tables/CFA_structs3}
% ============================================================
% ============================================================
\subsection{Method 2: Fully crossed generalizability study}\label{sec:m2}
% ============================================================
% \paragraph{Goal}
Variance decomposition can quantify how much observed score variation is attributable to each ecosystem facet and interaction and assess the stability of observed rankings.
%under different decision rules
% While Method~1 asks ``what is the structure of ability?'', Method~2 asks ``how much of the \emph{observed variance} is signal versus noise, and noise from \emph{which source}?''
\subsubsection{The Generalizability study model}
We use leaderboard metadata to identify characteristics belonging to a set of benchmark scores: (a) architecture, (b) benchmark uniqueness, (c) contributors\footnote{Contributors represent provenance: the facet is an operational metadata grouping. It should not be interpreted as a person-level causal effect; it aggregates submission source, availability status, documentation practices, and unobserved training/provenance differences. This includes variation due to missing model card and subsequent removal \citep{clementine_fourrier__is_off_till_dec_2026_hiking_clefourrier_open_2024}.} to the leaderboard, and (d) deployment, post-training, and tuning decisions. (More information in \S~\ref{apx:var_decomp}). Let $s_i$ be the score of submission $i$ on benchmark $b_i$ with architecture $a_i$, contributor $c_i$, and deployment $d_i$. This crossed random-effects model is
\begin{equation}\label{eq:gstudy}
s_i = \mu + \!\!\!\!\!\!\!\!\! \sum_{S \in \mathcal{S}:\mathcal{P}^*(\{a,b,c,d\})} \!\!\!\!\!\!\!\!\! u_{S(i)} + \varepsilon_i,
\end{equation}
where $\mathcal{P}^*$ denotes the non-empty Power Set defining the facets (i.e., all main effects and their interactions $A, B, AB, AC, \ldots, ABCD$), each $u_{S(i)} \sim \mathcal{N}(0, \sigma^2_S)$, and $\varepsilon_i$ absorbs the residual and highest-order interaction ($ABCD$). Variance shares $p_S = \sigma^2_S / (\sum_{S'} \sigma^2_{S'} + \sigma^2_\varepsilon)$ quantify the noise sources, with additional details in
% Table \ref{tab:varcomp_slope} and in
\S~\ref{apx:var_decomp}.
\subsubsection{G-study methodological solutions}
\paragraph{Controlling for influence of model size with random slopes.}
Without controlling for size, variance attributable to %architecture or contributor
each facet may simply reflect scale differences rather than genuine ecosystem effects. Extend (\ref{eq:gstudy}) with the log-size covariate $x_i=\log_{10}(\#\mathrm{params_i (B)})$ %$x_i = \log_{10}(\texttt{\#params}_i)$
and random slopes:
\begin{equation}\label{eq:gstudy_slopes}
s_i = \mu + \beta x_i + \sum_{S \in \mathcal{S}} \bigl(u^S_{g_S} + v^S_{g_S} x_i\bigr) + \varepsilon_i,
\end{equation}
where $u^S_{g_S} \sim \mathcal{N}(0,\sigma^2_{S,0})$ are random intercepts and $v^S_{g_S} \sim \mathcal{N}(0,\sigma^2_{S,1})$ are random slopes, constrained uncorrelated within each term. This decomposes total variance into intercept components (baseline performance) and slope components (scaling-relationship heterogeneity), enabling the novel reliability diagnostics $\mathrm{SNR}_\beta$ and $\mathrm{PSI}_S$ defined below.
\paragraph{Robustness checks} Because the four-facet crossing is sparse, we interpret higher-order components descriptively and emphasize robustness of broad variance-ordering patterns rather than individual high-order interaction estimates. We perform robustness checks on decomposition sensitivity, including adjusting category granularity and Bayesian estimation. We verify that the observed order of $B \gg C > A > D$ in our main finding is stable.\footnote{We also estimate \eqref{eq:gstudy_slopes}${}^\star$, with correlations are estimated freely. \eqref{eq:gstudy_slopes}${}^\star$ resulted in a larger contributor-linked variance share, $p_C=0.12$. The fit was singular, so we only report it in Table \ref{tab:vcov_gstudies}. \S~\ref{apx:robust_var_decomp} has details.}
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution:] \emph{Random-slopes extension.} Extend (\ref{eq:gstudy}) with the log-size covariate $x_i = \log_{10}(\texttt{\#params}_i)$ and random slopes:
% \begin{equation}\label{eq:gstudy_slopes}
% s_i = \mu + \beta x_i + \sum_{S \in \mathcal{S}} \bigl(u^S_{g_S} + v^S_{g_S} x_i\bigr) + \varepsilon_i,
% \end{equation}
% where $u^S_{g_S} \sim \mathcal{N}(0,\sigma^2_{S,0})$ are random intercepts and $v^S_{g_S} \sim \mathcal{N}(0,\sigma^2_{S,1})$ are random slopes, constrained uncorrelated within each term. This decomposes total variance into intercept components (baseline performance) and slope components (scaling-relationship heterogeneity), enabling the novel reliability diagnostics $\mathrm{SNR}_\beta$ and $\mathrm{PSI}_S$ defined below.
% \end{description}
\begin{definition}[Slope Signal-to-Noise Ratio and Generalizability]\label{def:snr}
The SNR of the fixed-effect slope $\beta$ is the ratio of its square to the total variance of the random slopes:
% \[
$\mathrm{SNR}_\beta \;\equiv\; \frac{\beta^2}{\sum_S \sigma^2_{S,1}}\; \text{and}\; \mathcal{R}_\beta \equiv \frac{\beta^2}{\beta^2+\sum_S \sigma^2_{S,1}}$,
% \]
where $\mathcal{R}_\beta$ is the reliability of a general scaling law. $\mathrm{SNR}_\beta \gg 1$ implies a robust, universal scaling law; $\mathrm{SNR}_\beta \approx 1$ or less indicates that the average scaling law is misleading because context-specific variation rivals the fixed effect (see \S~\ref{sec:disc_scaling_vector}).
\end{definition}
% We summarize reliability of the fixed size effect with (i) slope SNR, $\mathrm{SNR}_\beta=\beta^2/\sum_S\sigma^2_{S,1}$ and (ii) instability decomposition, $\mathrm{PSI}_S=\sigma^2_{S,1}/\sum_{S'}\sigma^2_{S',1}$ (details in Appendix~\ref{apx:scaling_reliability}).
\begin{definition}[Proportion of Slope Instability]\label{def:psi}
For each facet or interaction term $S$:
% \[
$\mathrm{PSI}_S \;\equiv\; \frac{\sigma^2_{S,1}}{\sum_{S'} \sigma^2_{S',1}}$.
% \]
$\mathrm{PSI}_S$ attributes scaling-law instability to its ecosystem sources: a large $\mathrm{PSI}$ implies that different sources of variation respond to size in fundamentally different ways (see Table \ref{tab:varcomp_slope}).
\end{definition}
We use the \textit{bifactor} downstream because it balances covariance fit, predictive performance, parsimony, and interpretability. The \textit{correlated bifactor} acts as an upper-bound flexibility check rather than the preferred scientific structure.
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution~2a:] \emph{Multi-granularity robustness checks.} We re-estimate variance shares at three levels of categorical resolution (finest, fine, coarse; Table~\ref{tab:granularity_sensitivity}) to verify that the ordering $B \gg C > A > D$ is not an artifact of a particular coding scheme.
% \item[Solution~2b:] \emph{Bayesian estimation.} We complement REML with a Bayesian MCMC framework (4~chains, weak priors) to obtain full posterior distributions over variance shares, providing 95\% HDI intervals for each component (Table~\ref{tab:bayesfit}).
% \end{description}
% ============================================================
\subsection{Method 3: Mixed-effects latent regression}\label{sec:m3}
% ============================================================
To estimate \emph{ecosystem-controlled scaling laws} on \emph{latent abilities}, both general and benchmark-specific, %, while respecting pseudo-replication via random effects%Where Methods~1 and~2 operate on observed scores or covariance structure,
Method 3 operates directly in the latent space, unifying measurement and regression into a single coherent model.
\subsubsection{Hierarchical two-component model}
The hierarchical mixed-effects latent regression model comprises a \textbf{measurement layer} (items $\to$ latent traits) and a \textbf{structural layer} (latent traits $\to$ covariates).
\paragraph{Measurement layer (bifactor IRT).}
Let $\boldsymbol{\theta}_i = (\theta_{i0}, \theta_{i1}, \ldots, \theta_{iK})^\top \in \mathbb{R}^{K+1}$ be the latent ability vector for LLM $i$, with general factor $g=\theta_{i0}$ and benchmark-specific factors $\theta_{ik}$. For item $j$ in benchmark $k(j)$:
\begin{equation}\label{eq:irt}
\Pr(y_{ij} = 1 \mid \boldsymbol{\theta}_i) = \sigma\bigl(a_{j0}\,\theta_{i0} + a_{j,k(j)}\,\theta_{i,k(j)} - b_j\bigr),
\end{equation}
where $\sigma(t) = (1+e^{-t})^{-1}$, $a_{j0}$ and $a_{j,k(j)}$ are discrimination parameters for the general and specific factors (compare with $\lambda$ in Table \ref{tab:structural_models}, bifactor), and $b_j$ is item difficulty. Each item loads on the general factor \emph{and} one specific factor.%--the bifactor constraint.
\paragraph{Structural layer (mixed-effects regression on latent traits).}
The matrix of latent traits $\boldsymbol{\Theta} \in \mathbb{R}^{N \times (K+1)}$ is decomposed into fixed and random components:
\begin{equation}\label{eq:structural}
\boldsymbol{\Theta} = \mathbf{V}\boldsymbol{\Gamma} + \mathbf{W}\boldsymbol{\zeta} + \mathbf{E},
\end{equation}
where $\mathbf{V}$ is the $N \times F$ fixed-effect design matrix (intercept, $\log_{10}$(\#params), deployment indicators), $\boldsymbol{\Gamma}$ is the $F \times (K+1)$ %matrix of
fixed-effect coefficients (each column $\boldsymbol{\gamma}_k$ has covariate effects on latent dimension $k$), $\mathbf{W}$ is the random-effect design matrix for contributor group, $\boldsymbol{\zeta} \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Sigma}_\zeta)$ captures contributor-level heterogeneity, and $\mathbf{E}$ is residual latent variation with rows $\mathbf{e}_i \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Sigma}_E)$, accounting for pseudo-replication. Because regressing manifest benchmark scores on size conflates $g$, benchmark specifics, and benchmark-dependent measurement error, a key output is the \emph{scaling vector} $\boldsymbol{\beta} = (\beta_0, \beta_1, \ldots, \beta_K)$, where $\beta_k = \Gamma_{x,k}$ is the effect of log-size on latent ability $k$. This replaces the scalar ``scaling law'' with a dimension-specific profile.
% \begin{proposition}[Latent regression corrects size-effect confounding]\label{prop:atten}
% Assume that the observed benchmark score $\bar{y}_{ik}$ is a noisy proxy for $\theta_{ik}$ with benchmark-dependent attenuation: $\bar{y}_{ik} = \lambda_k \theta_{ik} + e_{ik}$, $\mathbb{E}[e_{ik} \mid x_i] = 0$, $\mathrm{Var}(e_{ik})$ varies with $k$. Then OLS regression of $\bar{y}_{ik}$ on $x_i$ yields benchmark-dependent slope attenuation proportional to $\lambda_k$, while regression of $\theta_{ik}$ on $x_i$ (as in Eq.~\ref{eq:structural}) recovers the latent slope $\beta_k$ up to the mixed-model dependence structure. \emph{(Proof: Appendix~H, Proposition~3.1.)}
% \end{proposition}
\begin{proposition}[Latent regression corrects size effects confounded by measurement error]
\label{prop:latent_reg_advantage}
Assume the observed benchmark score $\bar{y}_{ik}$ is a noisy proxy for $\theta_{ik}$ with benchmark-dependent attenuation:
$
\bar{y}_{ik} = \lambda_k \theta_{ik} + e_{ik},\, \mathbb{E}[e_{ik}\mid x_i]=0,\, \mathrm{Var}(e_{ik}) \text{ varies with } k.
$
Then OLS regression of $\bar{y}_{ik}$ on $x_i$ yields benchmark-dependent slope attenuation proportional to $\lambda_k$, while regression of $\theta_{ik}$ on $x_i$ (as in~\ref{eq:structural}) recovers the latent slope $\beta_k$ up to the mixed-model dependence structure.
\end{proposition}
\begin{proof}
By linearity, $\mathbb{E}[\bar{y}_{ik}\mid x_i]=\lambda_k\mathbb{E}[\theta_{ik}\mid x_i]$ and the population regression coefficient is $\lambda_k\beta_k$ when $\theta_{ik}=\alpha_k+\beta_k x_i+\text{noise}$, whereas regressing $\theta_{ik}$ yields $\beta_k$. Mixed effects preserve unbiasedness under correct specification by accounting for cluster dependence in the likelihood.
\end{proof}
% \paragraph{Challenge: Size vs Post-training and Deployment.}
% Raw scaling regressions on manifest scores confound model size with deployment choices (chat templating, RLHF, precision settings) and benchmark-specific measurement error. Thus, %the design matrix $\mathbf{V}$ in (\ref{eq:structural}) includes both $\log_{10}$(\#params) and deployment and tuning decisions (e.g., types, templates, merges). % \texttt{type}, \texttt{chat\_template}, \texttt{mo\_e}, \texttt{merged\_precision}).
% Fixed effects $\boldsymbol{\Gamma}$ estimate the association of each covariate with each latent dimension \emph{simultaneously}, removing confounding at the latent level.
\subsubsection{Latent Regression Method Estimation}
% to address challenges of infeasibility. Importantly,
The term $\mathbf{W}\boldsymbol{\zeta}$ in (\ref{eq:structural}) models contributor-level random effects on each latent dimension, absorbing the cluster structure induced by pseudo-replication and preventing inflated precision. %The residual contributor-specific variance $\boldsymbol{\Sigma}_\zeta$ provides a direct estimate of \emph{provenance variance}--the share of latent ability variation attributable to unobserved training practices.
% \paragraph{Challenge: Intractability of Traditional CFA fit approaches.}
% The composite model--bifactor IRT with mixed-effects structural layer--has thousands of parameters, making standard MLE infeasible. \textbf{Solution:} \textbf{\emph{Meta-analytic item bootstrapping}} (as in Method~1). For each of $B=500$ replications, we sample a fixed number of items per benchmark, fit both the baseline bifactor and the full regression model, and aggregate fixed-effect estimates %$\{\hat{\beta}_d\}$
% via inverse-variance weighting. % Outputs are (i) $\{\hat{\beta}_d\}$ (size effects per latent ability), (ii) deployment effects $\{\hat{\gamma}_{d}\}$, and (iii) provenance variance $\boldsymbol{\Sigma}_u$.
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution:] \emph{Meta-analytic item bootstrapping} (as in Method~1). For each of $B=500$ replications, we sample a fixed number of items per benchmark, fit both the baseline bifactor and the full regression model, and aggregate fixed-effect estimates $\{\hat{\beta}_d\}$ via inverse-variance weighting.
% \end{description}
% \paragraph{Challenge: Identifiability in a complex latent landscape.}
The bifactor measurement structure coupled with fixed and random effects on $K+1$ latent dimensions creates a high-dimensional, multimodal likelihood surface. %\textbf{Solution:} \textbf{\emph{Metropolis-Hastings Robbins-Monro (MH-RM) estimation.}} % We use the stochastic EM algorithm of \citet{cai_metropolis-hastings_2010}, which
Thus, we fit~\eqref{eq:irt}--\eqref{eq:structural} with Metropolis-Hastings Robbins-Monro (MH-RM) \citep{cai_metropolis-hastings_2010,chalmers_extended_2015}.\footnote{MH-RM is a strong choice in high-dimensional IRT as it alternates between (i)~a MH step to sample from the conditional posterior of latent traits given current parameters, and (ii)~a RM step to update parameters along the stochastic gradient of the marginal log-likelihood.} Similar to Method 1 (\S~\ref{sec:cfa_solutions}), item set bootstrapping is used. %MH-RM is specifically designed for high-dimensional IRT models and scales to the item-set sizes and random-effect structures required here (implemented via \texttt{mirt}; \citealt{chalmers_mirt_2012}).
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution:] \emph{Metropolis-Hastings Robbins-Monro (MH-RM) estimation.} We use the stochastic EM algorithm of \citet{cai_metropolis-hastings_2010}, which alternates between (i)~a Metropolis-Hastings step to sample from the conditional posterior of latent traits given current parameters, and (ii)~a Robbins-Monro step to update parameters along the stochastic gradient of the marginal log-likelihood. MH-RM is specifically designed for high-dimensional IRT models and scales to the item-set sizes and random-effect structures required here (implemented via \texttt{mirt}; \citealt{chalmers_mirt_2012}).
% \end{description}
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution:] \emph{Latent fixed effects on factors.} The design matrix $\mathbf{V}$ in (\ref{eq:structural}) includes both $\log_{10}$(\#params) and deployment covariates (e.g., \texttt{type}, \texttt{chat\_template}, \texttt{mo\_e}, \texttt{merged\_precision}). Fixed effects $\boldsymbol{\Gamma}$ estimate the association of each covariate with each latent dimension \emph{simultaneously}, removing confounding at the latent level.
% \end{description}
% \paragraph{Challenge: Ecosystem confounders (pseudo-replication).}
% Fine-tunes, merges, and quantizations of the same base model violate the independence assumption. \textbf{Solution:} \textbf{\emph{Decomposition of the ability matrix to include mixed effects.}} The term $\mathbf{W}\boldsymbol{\zeta}$ in (\ref{eq:structural}) models contributor-level random effects on each latent dimension, absorbing the cluster structure induced by pseudo-replication and preventing inflated precision. The residual contributor-specific variance $\boldsymbol{\Sigma}_u$ provides a direct estimate of \emph{provenance variance}---the share of latent ability variation attributable to unobserved training practices (Appendix~H.3.2).
% \begin{description}[leftmargin=1.5em,nosep]
% \item[Solution:] \emph{Decomposition of the ability matrix to include mixed effects.} The term $\mathbf{W}\boldsymbol{\zeta}$ in (\ref{eq:structural}) models contributor-level random effects on each latent dimension, absorbing the cluster structure induced by pseudo-replication and preventing inflated precision. The residual contributor-specific variance $\boldsymbol{\Sigma}_u$ provides a direct estimate of \emph{provenance variance}---the share of latent ability variation attributable to unobserved training practices (Appendix~H.3.2).
% \end{description}
\subsubsection{Slope reliability in latent space}
For each latent dimension $d \in \{0,1,\ldots,K\}$, we define the \emph{slope reliability index}:
% \begin{equation}\label{eq:rel}
$\mathcal{R}_d \;\equiv\; \frac{\beta_d^2}{\beta_d^2 + \tau_d^2}$,
% \end{equation}
where $\tau_d^2 = \mathrm{Var}(v_{\cdot,d})$ is the random-slope variance for dimension $d$ across bootstraps (see \S~\ref{apx:boots}). High $\mathcal{R}_d$ indicates that the scaling law for latent ability $d$ is stable across contributors; low $\mathcal{R}_d$ signals context-dependent scaling (Table~\ref{tab:lreg_reliab}).
% \paragraph{Estimation and robustness.}
% We fit~\eqref{eq:m3_2pl_bifact}--\eqref{eq:theta_decomp} with MH-RM \citep{cai_metropolis-hastings_2010,chalmers_extended_2015}. To ensure conclusions are not item-idiosyncratic, we repeat estimation across $B$ item bootstraps and aggregate coefficients via inverse-variance weighting; we compare the regression model to the measurement-only bifactor using likelihood-based tests within each bootstrap (Appendix~\ref{apx:boots}). Outputs are (i) $\{\hat{\beta}_d\}$ (size effects per latent ability), (ii) deployment effects $\{\hat{\gamma}_{d}\}$, and (iii) provenance variance $\boldsymbol{\Sigma}_u$.
% \paragraph{Identifying capability-specific scaling.}
% Regressing manifest benchmark scores on size conflates $g$, benchmark specifics, and benchmark-dependent measurement error. Method~3 instead estimates scaling on $\theta_{i0}$ and $\theta_{ik}$ directly, i.e., a \emph{scaling vector} $\boldsymbol{\beta}=(\beta_0,\dots,\beta_K)$, disentangling general capability scaling from benchmark-specific scaling under the ecosystem’s dependence structure.
\noindent
\begin{table*}[ht]
\centering
% \label{tab:structural_models}
\scriptsize
\caption{Latent Structural Factor Models for AI Benchmark Ecosystem Analysis and Basic Results}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{@{}p{0.105\textwidth}p{0.15\textwidth}p{0.27\textwidth}p{0.21\textwidth}p{0.0425\textwidth}p{0.03\textwidth}p{0.025\textwidth}@{}l}
\toprule
\textbf{Structure} & \textbf{Formulae:} $y_{ikj} = $ & \textbf{Covariance} %$\text{Cov}(\mathbf{y})=$
& \textbf{Key Idea} & \textbf{RMSEA} & \textbf{${\text{AUC}}$} & \textbf{${\text{MAE}}$}\\
\midrule
% \textbf{M2:}
\makecell[lt]{Independent Factors \\ (Fig \ref{fig:cfa_diagram}.a, \S~\ref{apx:indepfact})} & \makecell[t]{$\lambda_{kj} f_{ik} + \epsilon_{ikj}$ \\ \ $ \boldsymbol{\Lambda}\mathbf{I}\boldsymbol{\Lambda}^\top + \boldsymbol{E}$} &
Block-diagonal across benchmarks (no inter-benchmark covariance) $\text{Cov}(f_{ik}, f_{ik'}) = 0$%, \; \forall k \neq k'$
& No meaningful cross-domain transfer. One factor per benchmark. (6 factors) & 0.09 &\textbf{0.95} & 0.22\\ % , k \neq k'
% \addlinespace[0.15ex]
% \textbf{M1:}
\makecell[lt]{One General Factor/ \\ Unidimensional \\(Fig \ref{fig:cfa_diagram}.b, \S~\ref{apx:gfact})} & \makecell[t]{$\lambda_{kj} g_i + \epsilon_{ikj}$ \\ $\boldsymbol{\Lambda}\boldsymbol{\Lambda}^\top + \boldsymbol{E}$} & All covariance explained by one source (+ diagonal residuals): $\mathrm{Var}(g_i)=1$ \ & Single, general factor ($g$) explaining all shared variance. (1 factor) &0.06 &0.90 & 0.23 \\
% \addlinespace[0.15ex]
% \textbf{M4:}
\makecell[lt]{Hierarchical \\ Structure/2nd-Order \\(Fig \ref{fig:cfa_diagram}.c, \S~\ref{apx:hier2ord})} & \makecell[t]{$\lambda_{kj} f_{ik} + \epsilon_{ikj}$,\\ $f_{ik} = \gamma_k g_i + \delta_{ik}$ \\ $\boldsymbol{\gamma}\boldsymbol{\gamma}^\top + \boldsymbol{\Delta}$ }& Inter-benchmark covariance implied via $g$ through $f_k$ (hierarchically structured)
$\text{Cov}(f_{ik}, f_{ik'}) = \gamma_k \gamma_{k'}$
& $g$ acts as the cause of correlations among the first-order factors (7 factors) & 0.05 &0.94 & 0.24 \\
% \addlinespace[0.15ex]
% \textbf{M3:}
\makecell[lt]{Correlated Factors/ \\ Multidimensional \\(Fig \ref{fig:cfa_diagram}.d, \S~\ref{apx:corrfact})} &\makecell[t]{ $\lambda_{kj} f_{ik} + \epsilon_{ikj}$ \\ $\boldsymbol{\Lambda}\boldsymbol{\Psi}\boldsymbol{\Lambda}^\top + \boldsymbol{E}$ } &
Cross-benchmark covariance via $\mathbf\Psi$ (dense factor correlation matrix) \ $\text{Cov}(\mathbf{f}_i) = \mathbf{\Psi}$
& Benchmark abilities are distinct but interrelated (6 factors) &0.05 &0.73 & 0.35\\
% \addlinespace[0.15ex]
% \textbf{M5:}
\makecell[lt]{Bifactor/ Orthogonal \\General Factor) \\(Fig \ref{fig:cfa_diagram}.e, \S~\ref{apx:bifact}, \ref{apx:code})} & \makecell[tc]{\vspace{-1.5pt}$\lambda^{(g)}_{kj} g_i + \lambda^{(f)}_{kj} f_{ik} + \epsilon_{ikj}$\\$\boldsymbol{\Lambda}_g\boldsymbol{\Lambda}_g^\top + \boldsymbol{\Lambda}_f\boldsymbol{\Lambda}_f^\top + \boldsymbol{E}$}%$\text{Cov}(g, \mathbf{s}) = \mathbf{0}$
& Covariance decomposes into general + benchmark-specific; no correlations among factors:
$g_i \perp f_{ik}, f_{ik} \perp f_{ik'}$
& Assumes benchmark-specific ($f_k$) variance is orthogonal to general ($g$) sources. (7 factors) & 0.05 & \textbf{0.95} & \textbf{0.21}\\
% \addlinespace[0.15ex]
% \textbf{M6:}
\makecell[lt]{Correlated Bifactor \\ (Fig. \ref{fig:cfa_diagram}.f, \S~\ref{apx:corrbifact})} & \makecell[t]{\vspace{-1.5pt}$\lambda^{(g)}_{kj} g_i + \lambda^{(f)}_{kj} f_{ik} + \epsilon_{ikj}$ \\ $\boldsymbol{\Lambda}_g\boldsymbol{\Lambda}_g^\top + \boldsymbol{\Lambda}_f\boldsymbol{\Psi}\boldsymbol{\Lambda}_f^\top + \boldsymbol{E}$} & Residual cross-benchmark covariance after removing $g$ captured by $\mathbf\Psi$ over specifics: $g_i \perp \mathbf{f}_i, \; \text{Cov}(\mathbf{f}_i) = \mathbf{\Psi}$
% $\text{Cov}(\mathbf{s}) = \boldsymbol{\Psi}$
& Tests for residual latent covariance after partialing $g$ (7 factors) & \textbf{0.04} & 0.77 & 0.32\\
\bottomrule
\end{tabular} \label{tab:structural_models}
\vspace{0.25ex}
\caption*{\small Note: $y_{ikj}$ = response of LLM $i$ to item $j$ in benchmark $k$; $g_i$ = general factor; $f_{ik}$ = benchmark-specific factors; $\lambda$ = loadings; $\epsilon_{ikj} \sim \mathcal{N}(0, \theta_{kj})$ = residuals; $\boldsymbol{\Psi}$ = correlation matrix; $\boldsymbol{E}$ = diagonal residual covariance matrix. Model details are in \ref{apx:models} and Tables \ref{tab:oos_agg_mae_auc}--\ref{tab:trad_fit_perm}. Study-wide notation is in \S~\ref{apx:roadmap:notation} and Bifactor code is found in \ref{apx:code} (all code and model specifics are in the online repository).
The final columns report the aggregated sample-weighted mean AUC and MAE across bootstraps for held out observations. RMSEA is a traditional CFA metric aggregated across estimations. %Details can be found in Tables \ref{tab:oos_agg_mae_auc}--\ref{tab:trad_fit_perm}.
RMSEA values reported in an earlier version were the result of a transcription error.
}
\end{table*}
\begin{figure*}[H]
\centering
\includegraphics[width=0.8\linewidth]{figures/scaling_law_estimates_robust.pdf}
\caption*{\footnotesize Noise-controlled Scaling Laws: Log-log of Performance by Number of Parameters. Shaded regions indicate the 95\% confidence
bands for the linear relationship between x and y estimated by robust regression using an M estimator with Tukey's biweight \cite{venables_modern_2002}. Prior to log transformation of the y-axis, each estimated latent score $\Theta_k$ is converted from its assumed distribution to positive values via the inverse cumulative normal distribution function, $S_k = \Phi^{-1}(\Theta_k)$ before all scores are min-max scaled to the range $s \in S_k:[1 . . . 100]$.}
\label{fig:scaling_robust}
\end{figure*}
\section{Experiment and Data}\label{sec:data}
% \subsection{Data}
We analyze public submissions to the Hugging Face Open LLM Leaderboard ~\cite{fourrier_open_2024},\footnote{\url{https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard}} a comprehensive and diverse collection of AI benchmark data, encompassing over 4,000 unique Large Language Model (LLM) instances. This large-scale dataset provides an ideal testbed for noise decomposition: models vary across architectures, contributors, deployment types, and scales, enabling systematic quantification of noise sources. The public dataset contains model metadata, benchmark scores, and availability of item-level responses. The Open LLM Leaderboard is a large-scale initiative to transparently track, rank, and evaluate open-source LLMs. The analyzed benchmarks are six widely used tasks with relatively objective metrics: IFEval~\cite{zhou_instruction-following_2023}, BBH~\cite{suzgun_challenging_2022}, MATH Level 5~\cite{hendrycks_measuring_2021}, GPQA~\cite{rein_gpqa_2023}, MuSR~\cite{sprague_musr_2024}, and MMLU-Pro~\cite{wang_mmlu-pro_2024}. Appendix \S~\ref{apx:data_detail} has additional data details.
The CFA and latent regression analyses use item-level responses, while the variance decomposition analyses focus on %the benchmark scores. % The primary dataset is sourced from the Open LLM Leaderboard, augmented with several other widely recognized benchmarks to ensure a broad representation of model capabilities. % Critically, our analyses utilize the raw, item-level response data for each LLM, rather than pre-aggregated summary scores, enabling a fine-grained examination of the latent structure underlying performance.
% Our analysis draws upon a comprehensive and diverse collection of AI benchmark data, encompassing over 4,000 unique Large Language Model (LLM) instances.
% \subsubsection{Open LLM Leaderboard}
% \subsubsection{Benchmark Rankings}\label{subsec:data_vardecomp}
% For variance decomposition, the unit of analysis is
a \emph{model--benchmark} score: each observation corresponds to one evaluated model (as represented by a specific leaderboard submission) on one benchmark, which is essential for identifying % (i) how much variance is attributable to benchmark difficulty and (ii)
how reliably one can generalize across benchmarks rather than merely fit the leaderboard’s composite.
%We reshape the dataset to long format with one row per benchmark score:
% \[
% y_{i b} \equiv \texttt{val} \quad \text{for model submission } i \text{ and benchmark } b\in\mathcal{B}.
% \]
% \paragraph{Score normalization and comparability}
% The Open LLM Leaderboard aggregates benchmark scores by normalizing raw scores relative to a random baseline and a maximum possible score, then averaging normalized scores.
%Our variance decomposition is conducted at the \emph{per-benchmark score} level which is essential for identifying (i) how much variance is attributable to benchmark difficulty and (ii) how reliably one can generalize across benchmarks rather than merely fit the leaderboard’s composite.
\begin{table}
\centering
\caption{Method 2 Variance Component Estimates for \eqref{eq:gstudy}-\eqref{eq:gstudy_slopes}}
\label{tab:varcomp_slope}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lrrr}
\toprule
\textbf{Variation Source} & \textbf{$p_S$ \eqref{eq:gstudy}} & \textbf{$p_S$ \eqref{eq:gstudy_slopes}} &\textbf{$\text{PSI}_S$} \\
\midrule
A (Architecture) & 0.042& 0.063& 0.13 \\
B (Benchmark) & 0.431& 0.461& 0.30 \\
C (Contributor) & 0.086& 0.078& 0.17 \\
D (Deployment) & 0.019& 0.011& 0.03 \\
A$\times$B & 0.029& 0.037& 0.05 \\
A$\times$C & 0.035& 0.041& 0.06 \\
A$\times$D & 0.002& 0.003& 0.00 \\
B$\times$C & 0.037& 0.027& 0.03 \\
B$\times$D & 0.049& 0.046& 0.02 \\
C$\times$D & 0.008& 0.011& 0.01 \\
A$\times$B$\times$C & 0.023& 0.038& 0.07 \\
A$\times$B$\times$D & 0.004& 0.006& 0.01 \\
A$\times$C$\times$D & 0.080& 0.047& 0.07 \\
B$\times$C$\times$D & 0.006& 0.018& 0.05 \\
$\epsilon_{\text{resid}}$ (+ A$\times$B$\times$C$\times$D) & 0.150& 0.113& \\
\bottomrule
\end{tabular}}
\begin{tablenotes}
\item \small Proportion of variance explained $p_S$ represents the total variance associated with (both slope and) intercept. The raw variances for the base ~\eqref{eq:gstudy} and slope \eqref{eq:gstudy_slopes} models can be found in Table \ref{tab:varcomp_base} and \ref{tab:varcomp_slope_apx}, respectively. Robustness checks and Bayesian posterior draws of $p_S$ ~\eqref{eq:gstudy} are in Tables \ref{tab:granularity_sensitivity} and \ref{tab:bayesfit}, respectively.
\end{tablenotes}
\end{table}
\section{Results and Discussion}
\label{sec:discussion}
\begin{figure*}[th]
\centering
\includegraphics[width=0.93 \linewidth]{figures/scaling_law_estimates_robust.pdf}
\caption{Noise-controlled Scaling Laws: Log-log of Performance by Number of Parameters. Shaded regions indicate the 95\% confidence bands for the linear relationship estimated by robust regression using an M estimator with Tukey's biweight \cite{venables_modern_2002}. Prior to log transformation of the y-axis, each estimated latent score $\Theta_k$ is converted from its assumed distribution to positive values via the cumulative normal distribution function, $S_k= \mathbf{\Phi}(\Theta_k)$ before all scores are min-max scaled to the range $s \in S_k : [1\dots100]$.}
\label{fig:scaling_law}
\end{figure*}
\begin{figure*}[th]
\centering
\includegraphics[width=0.97\linewidth]{figures/scale_law_deploy_rlm_allchat.pdf}
\caption{Noise-controlled Scaling Laws by Deployment Type: Log-log of Performance by Number of Parameters. Shaded regions indicate the 95\% confidence bands for the linear relationship between x and y estimated by robust regression using an M estimator with Tukey's biweight \cite{venables_modern_2002}. In this plot, the HF metadata categorizations of ``chat model'' and ``chat template'' are combined ($cm \vee ct$) to better highlight the effects of common SFT practices. Categories are mutually exclusive. Prior to log transformation of the y-axis, each estimated latent score $\Theta_k$ is converted from its assumed distribution to positive values via the cumulative normal distribution function, $S_k= \mathbf{\Phi}(\Theta_k)$ before all scores are min-max scaled to the range $s \in S_k : [1\dots100]$.}
\label{fig:scaling_law_deploy}
\end{figure*}
Benchmark ecosystems are now treated as scientific instruments, used to rank models, infer scaling laws, and justify training interventions. Our results show that, without explicit measurement modeling, these instruments mix signal with multiple layers of noise. We summarize the implications as a \emph{Latent Cartography}: we map %observed
leaderboard topographies defined by latent ability structure, ecosystem facets of variation, %that induce pseudo-replication and systematic heterogeneity,
LLM covariates, and residual item-level artifacts. % that violate local independence. %These findings inform actionable recommendations for benchmark designers, leaderboard operators, and researchers (see \S~\ref{apx:recs}).
\subsection{What the ecosystem is actually measuring}
\label{sec:disc_latent_landscape}
\paragraph{A size-related general factor dominates; benchmark factors are largely residual.}
Across item-resampled CFA comparisons, bifactor-family models best predict the dependence structure (see \S~\ref{apx:cfa_fit}), implying that much of the cross-benchmark covariance is captured by a single general factor $g$ with benchmark-specific residual structure. This supports a \emph{two-level} decomposition: $
\text{performance} \approx g \;+\; \{s_k\}_{k=1}^K \;+\; \text{item noise}$,
where $s_k$ captures what remains common within benchmark $k$ after partialing out $g$. $g$ is strongly correlated with number of parameters (Fig. \ref{fig:scaling_law}).
\begin{proposition}[Benchmark scores are not construct-preserving summaries]
\label{prop:disc_totals}
If the ecosystem is better approximated by a bifactor model than by per-benchmark unidimensional models, then benchmark total scores are not sufficient statistics for latent capability profiles: there exist submissions $i\neq i'$ such that
$
\mathrm{Score}_{ik}=\mathrm{Score}_{i'k}\;\; \forall k
\;\text{but}\;
(g_i,s_{i1},\dots,s_{iK})\neq(g_{i'},s_{i'1},\dots,s_{i'K}).$ %Models can be rank-equivalent on totals yet differ on $(g,s_1,\dots,s_K)$, inducing different generalization behavior across items and benchmarks.
\end{proposition}
% \begin{proposition}[Non-equivalence of benchmark totals under multidimensional measurement]
% If the true item dependence is better approximated by a bifactor model than by a benchmark-wise unidimensional model, then benchmark total scores are not sufficient statistics for latent capability profiles; models can be rank-equivalent on totals yet differ on $(g,s_1,\dots,s_K)$, inducing different generalization behavior across items and benchmarks.
% \end{proposition}
\input{tables/slope_reliability}
\paragraph{Benchmark labels do not fully organize residual dependence.}
Permutation controls show that flexible SEMs
% (notably correlated bifactor)
can retain strong fit even under randomized item-to-benchmark assignment. The implication %is not that benchmarks are useless; it
is that benchmark membership is an incomplete index of the residual covariance structure once $g$ is removed. %Item types,
Shared evaluation artifacts, and latent overlaps cut across benchmark labels. Consequently, benchmark ecosystems should be evaluated as \emph{measurement systems}, not as a set of independent test suites. Importantly, all latent structures tested still underestimate the strength of the interbenchmark item relationships (see also \S~\ref{apx:math}, \S~\ref{apx:cfa_fit} and Fig. \ref{fig:bench_item_misfit}). Taken together, the meta-analytic bootstrap, permutation control, and MI-based local diagnostics provide convergent evidence that (i) unidimensional and independent-benchmark assumptions are untenable, (ii) a strong general factor is real and stable across item perturbations, and (iii) benchmark labels only partially explain residual structure. %—motivating the subsequent methods that explicitly decompose ecosystem variance and connect latent structure to sources of variation beyond the benchmarks themselves.
\subsection{Actionable cartographic layers}
\label{sec:disc_noise_layers}
\paragraph{Predictable benchmark difficulty (a controllable nuisance)}
Variance decomposition shows that a large share of leaderboard score variance is attributable to the benchmark main effect (difficulty). This is ``nuisance'' variation in the sense that it is predictable from benchmark identity and can be controlled by (i) stratified reporting, (ii) difficulty-normalized scoring, or (iii) explicit modeling (our G-study). %The high generalizability of benchmark difficulty ordering ($G_B\approx 0.73$) implies that this component is stable enough to be corrected rather than averaged away.
\paragraph{Contributor variance (signal about training; noise for architecture)}
The contributor/provenance facet is consistently larger than the combined architecture and deployment main effects plus their interaction, across estimations and robustness checks. In this case, contributor variance is \emph{not} ``random noise''---it is structured variation associated with unobserved implementation factors (data mixtures, compute, tuning recipes). It is noise only relative to the question ``what does architecture explain?'' This motivates two distinct evaluation modes: \textbf{Architecture-centric evaluation:} treat provenance as a nuisance facet and report provenance-adjusted uncertainty; and \textbf{Method-centric evaluation:} treat provenance as the object of measurement and analyze what practices drive it. Mixed latent regression makes this separation explicit by estimating provenance variance \emph{in latent space}, after controlling for size and deployment.
% and redundancy (inflated reliability without information)
\paragraph{Violations of Local dependence}
Bootstrapped SEPC maps reveal persistent residual couplings between item pairs even under the best-fitting structures (see \S~\ref{apx:cfa_fit}): shared LLM behaviors unrelated to reported constructs persisted even after accounting for various latent structures, which can threaten the use of benchmarks as measurement instruments.
%This means that observed % by freeing their covariance as a parameter and then aggregating across models and benchmarks
% : things done that improve IFEval may have an inverse relationship with MuSR.
% This is the operational meaning of \emph{violations of local independence}: the model underpredicts specific item associations, indicating duplicated content, shared templates, or method effects. Redundancy is harmful because it increases apparent stability of benchmark totals without increasing construct coverage. A practical rule is to treat persistent large $|\mathrm{SEPC}(\Theta_{ab})|$ as a candidate for item revision/removal; doing so increases information per evaluation token.
\subsection{Scaling laws are a vector, not a scalar}
\label{sec:disc_scaling_vector}
One motivation for leaderboards is to infer scaling effects, and our results indicate that ``scaling'' is not a single ``law'' of the ecosystem but a \emph{vector of slopes} over latent abilities. From the mixed-effects covariance decomposition~\eqref{eq:gstudy_slopes}, we calculate the signal-to-noise ratio of the effects of size: %. Using ~\eqref{eq:gstudy_slopes}, we estimate that
$\mathrm{SNR}_\beta = \frac{\beta^2}{\sum_{S} \sigma^2_{S,1}} = 1.12$ and $\mathcal{R}_\beta = 0.53$, suggesting that an average scaling law would be misleading and that scaling is context-dependent (see \S~\ref{apx:scaling_metrics}).
% : $ \boldsymbol{\beta}=(\beta_g,\beta_{s_1},\dots,\beta_{s_K}),$ estimated most cleanly in Method~3
Using \eqref{eq:irt}-\eqref{eq:structural}, $g$ exhibits the strongest and most stable scaling with $\log$-parameters ($\mathcal{R}_g = 0.97$, Table \ref{tab:lreg_reliab}), consistent with the observation that leaderboard trends track $g$.\footnote{The $\mathcal{R}_\beta$ from Method 2 and $\mathcal{R}_d$ from Method 3 are conceptually similar but not directly exchangeable: the former describes the stability of a manifest-score scaling slope across ecosystem facets and the latter describes the stability of latent-dimension slopes after noise controls.} In contrast, several benchmark-specific factors have negligible size-slopes with deployment and provenance controls. This resolves a common ambiguity in raw-score regressions: apparent scaling of a benchmark can be mediated primarily through $g$ rather than through the specific skill the benchmark purports to isolate. Without size control, signal is buried in confounded noise (see \S~\ref{apx:size_control_implications}).
% \paragraph{A capability that does not scale is a measurement signal.}
% Two findings are especially diagnostic: (i) instruction-following (IF-Eval-specific) shows little size dependence after controls, and (ii) soft reasoning (MuSR-specific) can be mildly negative under full controls. We interpret these as ecosystem-level evidence that current optimization practices (often correlated with scale and deployment) do not uniformly improve all latent dimensions, such as explicit ``instruction-following'' or implicit ``soft reasoning''. For benchmarking, the key point is methodological: latent mixed regression can identify where leaderboard gains are driven by $g$ versus where they reflect targeted improvements (or regressions) in specific abilities.
\label{sec:latent_regression_results}
\paragraph{Disentangling Scaling Laws from Ecosystem Noise}
By regressing latent abilities on model size, we can isolate a ``denoised'' scaling relationship from confounding ecosystem factors. Figure \ref{fig:scaling_law} shows the estimated scaling laws for the general factor and specific abilities under three views: (i) the raw leaderboard scores, (ii) the basic bifactor latent scores, and (iii) the ``denoised'' latent scores from the mixed-effects regression. For the general factor, which closely tracks the overall leaderboard average, the bifactor $g$ reveals a scaling relationship that is as strong as the na\"ive leaderboard trend. After controlling for deployment choices and contributor effects, increasing model size robustly improves general cross-domain capability. The bifactor-only estimate, in contrast, shows a weaker relationship, demonstrating that failing to control for ecosystem noise attenuates the apparent effect of scale on latent ability.
\subsection{Insights into specific capabilities and benchmarks}\label{sec:specific_bench_analyses}
\paragraph{Does Knowledge Scale and Reasoning Stagnate?}
%The power of our approach becomes evident when examining the specific factors.
We gain insights into specific factors:
\begin{itemize} [left=0pt,nosep]
\item \textbf{General Knowledge (MMLU-Pro):} The denoised scaling effect for the MMLU-specific factor remains strong, adding to the trend for the general factor. This provides rigorous, model-based evidence that suggests that gains on knowledge-intensive benchmarks are heavily driven by scale, even beyond the enhancement of general capability. %This provides rigorous, model-based evidence for the hypothesis that scaling disproportionately benefits capabilities related to knowledge recall or synthesis.
\item \textbf{Complex Reasoning (GPQA and BBH):} In stark contrast, the denoised scaling laws for the reasoning-focused GPQA and BBH factors have trivial effect sizes%. While a slight, statistically significant positive trend remains, the effect size is trivial. This is a critical finding
: after controlling for general capability and ecosystem factors, simply increasing model size yields diminishing returns for these complex reasoning skills.
\end{itemize}
\paragraph{Specific Benchmarks: Focus improves usable signal.}
Our approach has insights for specific benchmarks.
\begin{itemize} [left=0pt,nosep]
\item \textbf{Heterogeneous Reasoning (BBH):} BBH is composed of a diverse set of the 23 most difficult tasks
% selected as the most difficult
from BIG-bench \citep{srivastava_beyond_2023} and fails to show a relationship with any of the post-training methods captured. % by deployment type.
Improvements on BBH in this ecosystem are driven by $g$, not a unique benchmark signal, \emph{challenging the practice of hill climbing on benchmarks primarily designed to be difficult}.
\item \textbf{Clear Targets (IF-Eval and MATH)} By contrast, IF-Eval and Math Level 5 are benchmarks with more clearly defined tasks (verifiable instruction following and math problem solving, respectively), and their effects exhibit high reliability with deployment choices. Whether positive or negative, focused benchmarks give clearer signal.
% Figure \ref{fig:scaling_law} shows the difference in controlling for ecosystem for MATH.
\end{itemize}
\paragraph{Instruction Following and ``Soft Reasoning'': Evidence of a Cognitive Trade-off?}
Striking results emerge from IF-Eval (instruction following) and MuSR (soft reasoning).
\begin{itemize}[left=0pt,nosep]
\item For the \textbf{IF-Eval-specific factor}, we find no significant scaling effect attributable to model size after controls. Unique gains on this benchmark appear to be driven almost entirely by specific tuning choices not raw scale.
\item For the \textbf{MuSR-specific factor}, we observe a significant, albeit small, \emph{negative} relationship with model size in the fully controlled model. Furthermore, several covariates associated with improved instruction-following %(e.g., \texttt{chat\_template})
show a negative effect on the MuSR factor. This suggests a potential ``alignment tax'' or cognitive trade-off: the very techniques used to make models better instruction-followers may subtly impair their capacity for the kind of flexible, ``soft'' reasoning required by MuSR tasks.
\end{itemize}
% This relationship was notably sensitive: in bootstrap replications with fewer items, the correlation between the IFEval and MuSR factors was highly unstable. This instability suggests a delicate and complex interplay between these constructs, where item selection can dramatically alter the perceived relationship.
Interestingly, the most significant SEPC \textit{overfit} across latent structures occurred between MuSR--IF-Eval items, suggesting a possible inversion of effects of training practices. We interpret these findings as ecosystem-level evidence that current optimization practices (often correlated with scale and deployment) do not uniformly improve all latent dimensions, such as explicit ``instruction-following'' or implicit ``soft reasoning''. For benchmarking, the key point is methodological: latent mixed regression can identify where leaderboard gains are driven by $g$ versus where they reflect targeted improvements (or regressions) in specific abilities.
% \paragraph{Quantifying Provenance Variance: The Persistent Signal of Training Methodology.}
% Even after explicitly modeling item-level effects, model size, and deployment strategies, the random effect for contributor remains a significant predictor, accounting for 4\% of the residual variance in latent ability. We term this \textbf{provenance variance}. It is crucial to interpret this component correctly: from the perspective of comparing base model architectures, it is a source of noise, reflecting unobserved differences in training data, compute resources, and fine-tuning recipes that are confounded with the contributor. However, from the perspective of understanding what makes models effective, this variance is pure signal. It quantifies the substantial impact of proprietary or unstated implementation choices on model capability, highlighting that \textit{how} a model is trained remains as important as its architecture or scale. Our model allows us to disentangle these effects, providing a clearer view of architectural merits while simultaneously quantifying the magnitude of the ``secret sauce'' in model development.
\subsection{Ranking Changes under $g$ and Ceiling Sensitivity}\label{sec:rank_changes}
We compare reported leaderboard percent ranks with percent ranks derived from the precision-weighted means for $g$ \eqref{eq:irt}-\eqref{eq:structural} of posterior draws, $\boldsymbol{\widetilde{\theta}_{i0}}$ (see \S~\ref{apx:rank_agg}).
% Old long table
% \begin{table}[htbp]
% \centering
% \caption{Association between leaderboard rankings and general-factor rankings after controlling for ecosystem noise.}
% \label{tab:ranking_changes}
% \begin{tabular}{lcc}
% \toprule
% Metric & $r$ & 95\% CI \\
% \midrule
% Kendall & 0.66 & [0.65, 0.68] \\
% Spearman & 0.86 & [0.85, 0.86] \\
% Distance Correlation (dCor) & 0.82 & [0.81, 0.83] \\
% \bottomrule
% \end{tabular}
% \end{table}
%% Current wide table
% \begin{table}[htbp]
% \centering
% \caption{Association between leaderboard rankings and general-factor rankings after controlling for ecosystem noise.}
% \label{tab:ranking_changes}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{ccc}
% \toprule
% Spearman $\rho$ & Distance Correlation (dCor) &Kendall $\tau$ \\
% \midrule
% 0.86 [0.85, 0.86] & 0.82 [0.81, 0.83] &0.66 [0.65, 0.68] \\
% \bottomrule
% \end{tabular}}
% \end{table}
Overall rankings (Spearman's $\rho = 0.86$) and their broader distribution (Distance Correlation $\mathrm{dCor} = 0.82$) are largely preserved after noise control, but with meaningful localized disruptions (Kendall's $\tau = 0.66$). In particular, only one model in the top 1\% of the original overall leaderboard remains in the top 1\% after conditioning on $g$, whereas approximately 90\% of models remain within the top decile. This is contrasted with the bottom 1\% of models: the majority (64\%) stayed in the bottom 1\%. These findings suggest that top leaderboard positions are substantially sensitive to ecosystem noise, even when the broader ordering of models remains comparatively stable. This directly supports the central claim of the paper: na\"ive leaderboard rankings conflate underlying capability with ecosystem artifacts, with the largest practical consequences occurring at the decision-relevant margins of the ranking distribution.
% \subsection{Implications for benchmark design}
% \label{sec:disc_recommendations}
% \paragraph{Recommendation 1: report bifactor scores, not only totals.}
% Leaderboards should report (i) a general factor score $\hat{g}$ and (ii) benchmark-specific residual scores $\{\hat{s}_k\}$ (or a principled low-dimensional alternative), with uncertainty. This preserves construct structure and prevents benchmark totals from being overinterpreted as distinct skills.
% \paragraph{Recommendation: publish reliability context for rankings.}
% Rankings should be accompanied by conditional reliability: at minimum, variance shares (G-study) and a size-adjusted comparison mode. Without size control, a substantial fraction of explainable rank-order variance is size-linked, and architectural claims are easily confounded.
% \begin{table}
% \centering
% \label{tab:sloperel}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{lrrrrrrr}
% \toprule
% \multicolumn{1}{c}{ } & \multicolumn{7}{c}{Slope Reliability Estimate from Latent Regression by Factor and Training} \\
% \cmidrule(l{3pt}r{3pt}){2-8}
% Variable & G & bbh & gpqa & ifeval & mmlu & musr & openllm\\
% \midrule
% chat models (RLHF, DPO, IFT) & 0.99 & 0.01 & 0.09 & 0.98 & 0.61 & 0.47 & 0.93\\
% chat\_template & 0.86 & 0.05 & 0.44 & 0.99 & 0.19 & 0.47 & 0.86\\
% log\_nbpars & 0.97 & 0.05 & 0.20 & 0.39 & 0.57 & 0.07 & 0.18\\
% mo\_e & 0.97 & 0.10 & 0.19 & 0.00 & 0.35 & 0.27 & 0.00\\
% chat models (RLHF, DPO, IFT, ...) & 0.98 & 0.01 & 0.42 & 0.96 & 0.17 & 0.53 & 0.90\\
% \addlinespace
% domain-specific fine-tuned & 0.98 & 0.18 & 0.31 & 0.95 & 0.56 & 0.45 & 0.86\\
% base merges and moerges & 0.89 & 0.05 & 0.35 & 0.95 & 0.65 & 0.31 & 0.80\\
% pretrained & 0.76 & 0.00 & 0.21 & 0.91 & 0.35 & 0.80 & 0.87\\
% continuously pretrained & 0.99 & 0.32 & 0.07 & 0.86 & 0.26 & 0.77 & 0.79\\
% \bottomrule
% \end{tabular}}
% \caption*{\small Results from bootstrapped latent Regression analysis }
% \end{table}
% \begin{table}
% \centering
% \label{tab:lreg_reliab}
% \caption{Reliabilities of Slopes in Latent Regression}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{lrrrrrrr}
% \toprule
% % \multicolumn{1}{c}{ } & \multicolumn{7}{c}{Slope Reliability Estimates} \\
% \cmidrule(l{3pt}r{3pt}){2-8}
% Covariate & G & BBH & GPQA & IFEVAL & MMLU & MUSR & Math5 \\
% \midrule
% chat (RLHF, DPO, IFT,...) & 0.64 & 0.01 & 0.37 & 0.72 & 0.14 & 0.36 & 0.79\\
% domain-specific fine-tuned & 0.98 & 0.18 & 0.31 & 0.95 & 0.56 & 0.45 & 0.86\\
% base merges and MoErges & 0.89 & 0.05 & 0.35 & 0.95 & 0.65 & 0.31 & 0.80\\
% pretrained & 0.76 & 0.00 & 0.21 & 0.91 & 0.35 & 0.80 & 0.87\\
% continuously pretrained & 0.99 & 0.32 & 0.07 & 0.86 & 0.26 & 0.77 & 0.79\\
% ``chat\_template'' & 0.86 & 0.05 & 0.44 & 0.99 & 0.19 & 0.47 & 0.86\\
% log(Model Size) & 0.97 & 0.05 & 0.20 & 0.39 & 0.57 & 0.07 & 0.18\\
% Mixture of Experts & 0.97 & 0.10 & 0.19 & 0.00 & 0.35 & 0.27 & 0.00\\
% \bottomrule
% \end{tabular}}
% \caption*{\small Results from bootstrapped latent Regression analysis }
% \end{table}
\begin{table}
\centering
\caption{Reliabilities of Slopes in Latent Regression}\label{tab:lreg_reliab}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{llrrrrrrr}
\toprule
Coefficient & stat & $g$ & bbh & gpqa & ifeval & mmlu & musr & math5\\
\midrule
$\log_{10}$(\#params) & $\boldsymbol{\beta}$ & 1.05 & -0.05 & 0.07 & -0.08 & 0.52 & 0.04 & 0.13\\
& $\mathcal{R}_d$ & 0.97 & 0.05 & 0.20 & 0.39 & 0.57 & 0.07 & 0.18\\
`Chat template'/IFT & $\boldsymbol{\gamma}$ & -0.22 & -0.03 & -0.66 & 1.82 & 0.08 & -1.03 & 0.29\\
& $\mathcal{R}_d$ & 0.86 & 0.05 & 0.44 & 0.99 & 0.19 & 0.47 & 0.86\\
Chat (RLHF, DPO, etc) & $\boldsymbol{\gamma}$ & -1.05 & 0.02 & 0.42 & -0.84 & -0.31 & 0.52 & -0.92\\
& $\mathcal{R}_d$ & 0.64 & 0.01 & 0.37 & 0.72 & 0.14 & 0.36 & 0.79\\
Domain-specific tune & $\boldsymbol{\gamma}$ & -1.16 & -0.07 & 0.20 & -0.80 & -0.46 & 0.39 & -0.43\\
& $\mathcal{R}_d$ & 0.98 & 0.18 & 0.31 & 0.95 & 0.56 & 0.45 & 0.86\\
Merges \& MoErges & $\boldsymbol{\gamma}$ & -0.65 & 0.09 & 0.24 & -0.79 & -0.64 & 0.41 & -0.58\\
& $\mathcal{R}_d$ & 0.89 & 0.05 & 0.35 & 0.95 & 0.65 & 0.31 & 0.80\\
Pretrained only & $\boldsymbol{\gamma}$ & -1.19 & 0.01 & 0.14 & -0.86 & -0.40 & 0.70 & -0.61\\
& $\mathcal{R}_d$ & 0.76 & 0.00 & 0.21 & 0.91 & 0.35 & 0.80 & 0.87\\
Continuous pretrain & $\boldsymbol{\gamma}$ & -1.51 & -0.17 & 0.06 & -1.05 & -0.34 & 0.72 & -0.74\\
& $\mathcal{R}_d$ & 0.99 & 0.32 & 0.07 & 0.86 & 0.26 & 0.77 & 0.79\\
Mixture of Experts & $\boldsymbol{\gamma}$ & -0.84 & -0.04 & 0.11 & 0.01 & -0.29 & -0.08 & 0.00\\
& $\mathcal{R}_d$ & 0.97 & 0.10 & 0.19 & 0.00 & 0.35 & 0.27 & 0.00\\
\bottomrule
\end{tabular}}
\caption*{\small Results from bootstrapped latent regression analysis. Variables are taken directly from the Hugging Face Open LLM Leaderboard. Only $\log_{10}$(\#params) is continuous; all others are binary. $\boldsymbol{\beta}$ and $\boldsymbol{\gamma}$ represent the per-factor estimates associated with that coefficient and $\mathcal{R}_d$ represents the reliability of the slope defined by that estimate. Chat and `Chat template' are not mutually exclusive.}
\end{table}
\section{Conclusion}
We introduced an advanced framework for AI benchmark ecosystems that systematically identifies, quantifies, and controls for measurement noise. Combining CFA, G-Theory, and latent regression, our analyses demonstrate that major sources of benchmark noise can be identified, quantified, and adjusted for. Collectively, the evidence points toward the strength of human decisions in impacting benchmark performance, whether good or bad: contributor information comprises a significant share of observed variation; instruction-tuning practices work but can also be negatively associated with ``soft reasoning''; and benchmark construction approaches distinguish usable signal. By measuring the entire latent landscape, the research community can make benchmark rankings more trustworthy and evaluation claims more credible.
% \paragraph{Recommendation 3: treat redundancy as an ecosystem optimization problem.}
% Use MI/SEPC maps to remove duplicated items and to diversify item templates. This reduces evaluation cost while increasing construct coverage. In ecosystems that evaluate thousands of models, even small redundancy reductions yield large compute savings.
% \subsection{Actionable recommendations for a noise-aware evaluation science}
% Our findings are not merely critical; they are constructive. The methods we introduce provide a toolkit for building more reliable and interpretable evaluation systems.
% \paragraph{For benchmark designers and leaderboard operators:}
% \begin{enumerate}[nosep,leftmargin=*]
% \item \textbf{Adopt bifactor scoring.} Report a general factor score ($g$) and benchmark-specific residual scores ($s_k$). This disentangles "globally strong" from "uniquely strong on benchmark $k$."
% \item \textbf{Decompose and display variance sources.} Use G-theory (Method~2) to show users what proportion of ranking variance comes from architecture vs.\ provenance vs.\ scale. This transparency makes rankings interpretable.
% \item \textbf{Implement redundancy checks.} Use modification indices (MI/SEPC) as an automated diagnostic to flag redundant items ($|\mathrm{SEPC}|>0.40$) during benchmark development, enabling targeted "benchmark slimming."
% \item \textbf{Report conditional reliability.} Instead of a single ranking, provide context-dependent views: rankings at a fixed scale, rankings within a contributor group, and rankings adjusted for provenance effects.
% \end{enumerate}
% \paragraph{For researchers and practitioners:}
% \begin{enumerate}[nosep,leftmargin=*]
% \item \textbf{Disaggregate scaling claims.} When claiming scaling effects, specify which capability is scaling (general vs. specific) by using latent variable models (Method~3).
% \item \textbf{Control for provenance.} In comparative analyses, treat "contributor" as a random effect to account for unobserved training differences. Our results show this is a larger source of variance than architecture.
% \item \textbf{Question unidimensionality.} Be skeptical of single benchmark scores as measures of a single, pure construct. Use the methods herein to test these assumptions before drawing strong conclusions.
% \end{enumerate}
\section*{Impact Statement}
A potential positive impact is more transparent and responsible interpretation of model comparisons, scaling claims, and architectural improvements, which may reduce overconfident conclusions drawn from small or unstable leaderboard differences. By providing tools to report uncertainty and disentangle general from benchmark-specific gains, this framework could support better research prioritization and evaluation standards. However, increasing the precision of benchmark analysis may also reinforce leaderboard-centric development or enable more targeted optimization for dominant latent factors while neglecting under-measured abilities such as safety or real-world robustness. Additionally, latent factors are statistical abstractions and could be misinterpreted as definitive constructs of intelligence. Overall, the work seeks to strengthen evaluation rigor, but its benefits depend on careful use and continued attention to broader societal and safety considerations beyond benchmark scores.
\paragraph{Limitations and scope}
% \label{sec:disc_limitations}
Our empirical results are based on a particular leaderboard snapshot and six benchmarks; variance proportions and factor strengths are ecosystem-dependent. The estimates are measured via available observational data and metadata and should not be interpreted causally. The metadata itself is of unknown quality, making contributor/deployment labels noisy proxies. Further, models submitted to Open LLM Leaderboard are not representative of all LLMs. Leaderboards themselves can exhibit temporal instability, as their ecosystems change with time.
Latent structural equation model selection is not ontology: bifactor fit supports a parsimonious decomposition of covariance, but the semantics of $g$ and $s_k$ depend on the item pool and evaluation protocol. Bifactor is known to over-extract general factors and absorb local dependence on human data, but it has not been studied whether that is true for LLMs. We leave these open questions and lines of work for future studies.
\section{Background and Related Work}
\label{apx:related_work}
AI benchmark ecosystems now serve as de facto scientific instruments: they are used to compare models, to validate training interventions, and to infer scaling laws. Yet the dominant reporting primitive remains a \emph{sum score} (accuracy or normalized accuracy) per benchmark and then an average across benchmarks. This practice implicitly assumes (i) that each benchmark is approximately unidimensional and internally coherent, and (ii) that benchmark totals are commensurate indicators of a smaller number of underlying capabilities. When these assumptions fail, ecosystem conclusions become unstable: model rankings conflate general capability with benchmark-specific artifacts; benchmark overlap inflates perceived evidence; and scaling relationships can be driven by item-selection effects rather than transferable competence.
\subsection{Measurement critiques of leaderboards and benchmark aggregation}
Recent work has argued that leaderboard scores embed unvalidated measurement assumptions and can distort comparisons when measurement error and construct mismatch are ignored \citep{salaudeen_measurement_2025,reuel_betterbench_2024, hardy_knowledge_2026}. The unidimensionality assumption has been highlighted as particularly consequential: when violated, benchmark totals become confounded composites \citep{truong_fantastic_2025}, allowing structural misspecification to masquerade as signal \citep{casabianca_psychometrics_2025}. Our contribution is to make these critiques testable at the ecosystem level by explicitly comparing CFA/SEM structures and quantifying residual dependence. In our setting, local independence corresponds to a diagonal $\boldsymbol{E}$: conditional on $\boldsymbol{\eta}_i$, item residuals should be independent.
\begin{proposition}[Benchmark totals are not sufficient under multidimensional measurement]
\label{prop:found_nonsuff}
If item responses are generated by~\eqref{eq:cfa} with $m>1$ and nontrivial loadings on multiple dimensions, then there exist submissions $i,i'$ such that (i) they have identical benchmark total scores, yet (ii) $\boldsymbol{\eta}_i\neq \boldsymbol{\eta}_{i'}$, implying different capability profiles.
\end{proposition}
\begin{proof}
Let two items load on different factors (non-collinear $\boldsymbol{\lambda}_j$). Construct $\boldsymbol{\eta}_i,\boldsymbol{\eta}_{i'}$ that swap strength across dimensions while preserving marginal correctness probabilities on the benchmark's mixture of items; totals can match while latent vectors differ. Thus totals do not uniquely identify $\boldsymbol{\eta}$.
\end{proof}
% \paragraph{Critiques of Aggregation and Unidimensionality.} The measurement assumptions underlying benchmark aggregations have received growing scrutiny. \citeauthor{salaudeen_measurement_2025} (\citeyear{salaudeen_measurement_2025}) demonstrate that ignoring measurement error systematically distorts performance comparisons, while \citeauthor{reuel_betterbench_2024} (\citeyear{reuel_betterbench_2024}) show how aggregating non-commensurate items produces misleading composites. The implicit assumption of unidimensionality has been a particular focus; \citeauthor{truong_fantastic_2025} (\citeyear{truong_fantastic_2025}) argue that its violation renders single summary scores uninterpretable. Our work provides the first large-scale, confirmatory test of these assumptions across an entire benchmark ecosystem.
\subsection{Psychometric methods for dataset and benchmark evaluation}
IRT has been used to estimate item difficulty/discrimination, detect saturated datasets, and improve leaderboard rankings \citep{lalor_understanding_2018,vania_comparing_2021,rodriguez_evaluation_2021}. Adaptive testing has been proposed to reduce evaluation cost \citep{maia_polo_tinybenchmarks_2024}. These efforts demonstrate the value of item-level modeling but typically assume unidimensionality or focus on within-benchmark properties. Our focus is complementary: we test \emph{ecosystem latent structure} (e.g., bifactor vs.\ correlated factors) and use SEM diagnostics to localize redundancy and method effects (See \S \ref{sec:mi_local_dependence}) \citep{muthen_goodness_1993,saris_testing_2009}.
A key limitation of prior exploratory work (e.g., PCA on LLM response matrices) is that it identifies dominant directions of variance without specifying \emph{which} constructs are being measured. In contrast, Confirmatory Factor Analysis (CFA) and Structural Equation Modeling (SEM) are hypothesis-driven: they predict the \emph{dependence structure} among item responses and yield interpretable latent constructs, with explicit constraints corresponding to competing scientific theories of capability organization.
\begin{remark}[CFA is not PCA]\label{rem:cfa_pca}
Unlike PCA, CFA does not merely identify dominant directions of variance. It tests falsifiable hypotheses about dependence structure by encoding benchmark theories as constraints, modeling measurement error explicitly, and testing whether residual covariance remains after conditioning on the latent construct. This is precisely the distinction needed to evaluate whether benchmark composites are valid measures.
\end{remark}
\paragraph{Identification of latent constructs in LLM benchmarking}
Several works attempt to infer latent constructs but can produce unstable or poorly aligned factors when i.i.d.\ assumptions and ecosystem structure are ignored \citep{kipnis_textttmetabench_2024,burnell_revealing_2023}. Misalignment between inferred constructs and benchmark intent has also been observed, e.g., \citeauthor{polo_sloth_2025} find that a construct of general knowledge is ``needed'' 8x more for Winogrande (commonsense pronoun resolution, \cite{sakaguchi_winogrande_2019}) than it is for MMLU-Pro (general knowledge). Our approach addresses these failure modes by (i) testing explicit SEM hypotheses with robustness controls, and (ii) embedding latent measurement within mixed-effects models to correct pseudo-replication.
% \paragraph{Psychometric Applications in AI Evaluation.} Item Response Theory (IRT), a close relative of CFA, has been used to analyze dataset difficulty \citep{lalor_understanding_2018}, detect benchmark saturation \citep{vania_comparing_2021}, and improve the efficiency of evaluation \citep{maia_polo_tinybenchmarks_2024}. These works typically employ unidimensional models focused on item-level properties. Our work differs by using multidimensional CFA to investigate the \emph{latent structure of abilities} and the relationships between them, moving from item analysis to ecosystem analysis. Exploratory Factor Analysis (EFA) has also been used \citep{polo_sloth_2025,burnell_revealing_2023}, but EFA lacks the hypothesis-testing framework of CFA, making it difficult to formally compare competing structural theories or obtain statistically unique solutions.
% \paragraph{Ecosystem complication: pseudo-replication.}
% Unlike typical ML datasets, leaderboard submissions are not independent and identically distributed (i.i.d.)\ draws: many entries are fine-tunes, merges, or quantized variants of a few base families, inducing strong dependence across ``models''.
% Ignoring this structure inflates effective sample size and can bias latent structure estimates and scaling conclusions.
% \subsection{Measurement, noise, and scaling in AI evaluation}
% \label{sec:related_work}
% \subsection{Measurement critiques of leaderboards and benchmark aggregation}
% Recent work has argued that leaderboard scores embed unvalidated measurement assumptions and can distort comparisons when measurement error and construct mismatch are ignored \citep{salaudeen_measurement_2025,reuel_betterbench_2024}. The unidimensionality assumption has been highlighted as particularly consequential: when violated, benchmark totals become confounded composites \citep{truong_fantastic_2025}, allowing structural misspecification to masquerade as signal \citep{casabianca_psychometrics_2025}. Our contribution is to make these critiques testable at the ecosystem level by explicitly comparing CFA/SEM structures and quantifying residual dependence.
% \subsection{Psychometric methods for dataset and benchmark evaluation}
% IRT has been used to estimate item difficulty/discrimination, detect saturated datasets, and improve leaderboard rankings \citep{lalor_understanding_2018,vania_comparing_2021,rodriguez_evaluation_2021}. Adaptive testing has been proposed to reduce evaluation cost \citep{maia_polo_tinybenchmarks_2024}. These efforts demonstrate the value of item-level modeling but typically assume unidimensionality or focus on within-benchmark properties. Our focus is complementary: we test \emph{ecosystem latent structure} (e.g., bifactor vs.\ correlated factors) and use SEM diagnostics to localize redundancy and method effects (See Apx. \ref{sec:mi_local_dependence}) \citep{muthen_goodness_1993,saris_testing_2009}.
% \paragraph{Identification of latent constructs in LLM benchmarking}
% Several works attempt to infer latent constructs but can produce unstable or poorly aligned factors when i.i.d.\ assumptions and ecosystem structure are ignored \citep{kipnis_textttmetabench_2024,burnell_revealing_2023}. Misalignment between inferred constructs and benchmark intent has also been observed, e.g., \citeauthor{polo_sloth_2025} find that a construct of general knowledge is ``needed'' 8x more for Winogrande (commonsense pronoun resolution, \cite{sakaguchi_winogrande_2019}) than it is for MMLU-Pro (general knowledge). Our approach addresses these failure modes by (i) testing explicit SEM hypotheses with robustness controls, and (ii) embedding latent measurement within mixed-effects models to correct pseudo-replication.
\subsection{Noise sources and generalizability in benchmark ecosystems}
Structured noise in benchmarks has been discussed, but the relative magnitudes of ecosystem noise sources are rarely quantified. Generalizability theory provides a principled decomposition of score variance into crossed facets and their interactions for AI evaluation \citep{brennan_generalizability_2001,cronbach_my_2004,hardy_all_2025}. We adapt this to benchmark ecosystems to estimate conditional ranking reliability under different generalization decisions (e.g., ranking architectures while treating contributors/deployments as nuisance).
% \subsubsection{The Dependent Noise of Ecosystem Structures}
% \label{subsec:ecosystem_challenge}
% Standard CFA assumes observations are i.i.d. This is violated in AI benchmark ecosystems, which are dominated by fine-tunes, merges, and quantized versions of a few base architectures—a phenomenon known as \emph{pseudo-replication}. This hierarchical dependence structure, if ignored, can severely bias parameter estimates and artificially inflate statistical power or misinterpret signals \citep{kipnis_textttmetabench_2024}. Furthermore, other ecosystem facets—such as contributor practices and deployment choices—introduce systematic variance that is conflated with model capability in both manifest scores and simple latent models. We employ methods here which can address these context challenges above and beyond typical latent model difficulties described herein: %\footnote{and the additional challenges found in Appendix \ref{apx:related_work}} %A comprehensive analysis must therefore not only model the latent measurement structure but also account for the hierarchical, multi-faceted nature of the data. This motivates our multi-method approach: we employ them here illustrate first
% we establish the most plausible latent structure using a robust bootstrap-and-permutation framework (Method 1, \S \ref{sec:m1}), then quantify the sources of ecosystem variance using mixed-effects generalizability theory (Method 2, \S \ref{sec:m2}), and finally integrate these insights into a hierarchical latent mixed-effects model that simultaneously estimates the effect of scaling on our understanding of abilities to be scaled while controlling for ecosystem noise (Method 3, \S \ref{sec:m3}). By determining which latent structure best gives rise to the observed data (Method 1, \S \ref{sec:m1}, Table \ref{tab:structural_models}, and Fig. \ref{fig:cfa_diagram}), we can produce an evidence-based map of the ecosystem's latent landscape. Detailed specifications for all models are provided in \S \ref{apx:models}.
\subsection{Scaling laws under confounding}
Scaling laws are typically studied under controlled training runs \citep{kaplan_scaling_2020,hoffmann_training_2022,schaeffer_pretraining_2025,isik_scaling_2025,sun_breaking_2026}. In open ecosystems, size is confounded with deployment and provenance (training recipes, data mixtures, compute budgets). Our contribution is methodological: we introduce reliability metrics for scaling slopes in a fully crossed mixed-effects setting and, crucially, estimate \emph{capability-specific} scaling in latent space via mixed latent regression \citep{de_boeck_explanatory_2011,chalmers_extended_2015}.
% \paragraph{Noise Sources and Scaling Laws.} The existence of "scaling laws" is well-established \citep{kaplan_scaling_2020,hoffmann_training_2022}, but analyses typically use controlled settings. In real-world ecosystems, size effects are confounded with other noise sources. Our variance decomposition approach (Method 2) extends Generalizability Theory \citep{brennan_generalizability_2001,cronbach_my_2004,hardy_all_2025} to the AI ecosystem context, allowing us to quantify the relative magnitude of noise from benchmark difficulty, contributor practices, and deployment choices. This enables us to introduce novel metrics (Slope SNR, PSI) that assess the \emph{reliability} of scaling laws in the presence of this noise.
% \subsection{A Confirmatory Framework for Latent Structure}
% \label{subsec:cfa_framework}
% Exploratory methods like PCA can reveal a dominant variance component in item-response data, but they are descriptive, not inferential. PCA cannot test competing theories of how capabilities are structured, nor does it explicitly model measurement error. To overcome these limitations, we employ CFA, a hypothesis-driven method from the family of SEMs \citep{joreskog_general_1969}.
% Where most machine learning models predict outcomes, the objective of CFA is to predict the \emph{covariance structure} of the observed data. This represents a fundamental shift in perspective: instead of assuming benchmark items "cause" a score, we posit that unobservable latent capabilities \emph{cause} the observed pattern of item responses.
% \paragraph{The CFA Measurement Model.} Let $\mathbf{y}_i \in \{0,1\}^p$ be the vector of binary responses from model submission $i$ to $p$ items. CFA postulates a vector of $m$ latent abilities (factors) $\boldsymbol{\eta}_i \in \mathbb{R}^m$ and a loading matrix $\boldsymbol{\Lambda} \in \mathbb{R}^{p \times m}$ that specifies which items measure which factors. For binary data, this is typically formulated via a latent continuous response $\mathbf{y}_i^\star$:
% \begin{equation}
% \label{eq:cfa_model}
% \mathbf{y}_i^\star = \boldsymbol{\Lambda}\boldsymbol{\eta}_i + \boldsymbol{\epsilon}_i, \quad \text{with} \quad y_{ij} = \mathbb{1}[y^\star_{ij} > \tau_j],
% \end{equation}
% where $\boldsymbol{\epsilon}_i \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Theta})$ is a vector of item-specific errors with a diagonal covariance matrix $\boldsymbol{\Theta}$, and $\tau_j$ are item-specific thresholds. The model-implied covariance of the latent responses, $\boldsymbol{\Sigma}(\boldsymbol{\theta})$, is decomposed into signal and noise:
% \begin{equation}
% \label{eq:cfa_covariance}
% \boldsymbol{\Sigma}(\boldsymbol{\theta})
% = \underbrace{\boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top}_{\text{Signal: Shared variance from latent abilities}} + \underbrace{\boldsymbol{\Theta}}_{\text{Noise: Item-specific variance + error}},
% \end{equation}
% where $\boldsymbol{\Phi}=\mathrm{Cov}(\boldsymbol{\eta})$ is the covariance matrix of the latent factors. The model is fit by minimizing the discrepancy between $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ and the observed item dependence structure (e.g., the tetrachoric correlation matrix).
% \subsection{Competing Models as Signal-Noise Hypotheses}
% \label{subsec:competing_models}
% The power of CFA lies in its ability to formally compare competing theories about how abilities are structured. We test six primary hypotheses (visualized in Figure \ref{fig:cfa_diagram}), each representing a different strategy for separating signal from noise.
% % :
% \begin{itemize}
% \item \textbf{Single General Factor ($g$):} All shared variance (signal) is due to one monolithic ability.
% \item \textbf{Correlated Factors:} Signal is partitioned into distinct but related benchmark-level abilities.
% \item \textbf{Hierarchical:} A higher-order general factor explains the correlations among first-order factors.
% \item \textbf{Bifactor:} Signal is additively decomposed into a general factor ($g$) and orthogonal, benchmark-specific factors. Here, benchmark-specific variance is what remains \emph{after} accounting for general ability.
% \end{itemize}
% \subsection{Related Work on Measurement in AI}
% \subsection{Ecosystem-specific complications}
% \paragraph{Pseudo-replication}
% Unlike most psychometric datasets, benchmark ecosystems violate i.i.d.\ assumptions. The Open LLM Leaderboard contains many fine-tunes, merges, and quantized variants derived from a small number of base families (e.g., Llama/Mistral), inducing strong dependence across submissions \citep{fourrier_open_2024}. Ignoring this pseudo-replication can inflate precision and bias structural conclusions toward dominant lineages. This motivates our combined design in \S~\ref{sec:m3}.% of robust structural discovery via item bootstraps, variance decomposition, and mixed latent regression that models item responses while attributing variance to ecosystem facets.
% \paragraph{Redundancy and Benchmark Design.} While benchmark overlap is known to be an issue \citep{rein_gpqa_2023}, systematic methods for detecting it are rare. Our use of modification indices (MI) and standardized expected parameter changes (SEPC) provides a statistically grounded approach rooted in SEM literature \citep{muthen_goodness_1993,saris_testing_2009} to flag trends in shared residual correlation across items and benchmarks, offering an actionable tool for benchmark refinement.
% Let $y_{ikj}\in\{0,1\}$ denote the response of evaluated model/submission $i$ to item $j$ in benchmark $k$. The central object of inference is not the mean score but the correlation/covariance structure induced by the joint distribution of $\mathbf{y}_i=\{y_{ikj}\}_{k,j}$. CFA specifies a latent vector $\boldsymbol{\eta}_i\in\mathbb{R}^m$ and loadings $\boldsymbol{\Lambda}$ such that the model-implied covariance
% % \[
% $\boldsymbol{\Sigma}(\boldsymbol{\theta})
% =
% \boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top + \boldsymbol{\Theta}$
% % \]
% matches the observed tetrachoric dependence (or a robust analog). Different structural hypotheses correspond to different sparsity patterns in $\boldsymbol{\Lambda}$ and different constraints on $\boldsymbol{\Phi}$ (e.g., a single general factor, correlated benchmark factors, hierarchical $g$, bifactor decompositions).
% However, benchmark ecosystems introduce a second, rarely-modeled complication: \emph{pseudo-replication}. Leaderboards contain thousands of submissions, but they are not i.i.d.\ draws from a population of independent models: many submissions are fine-tunes, merges, quantizations, or minor variants of a few base architectures. Treating all submissions as independent artificially increases effective sample size, inflates precision, and can bias factor structure toward idiosyncrasies of dominant families. This motivates our multi-method design:
% (1) a \emph{meta-analytic bootstrap} over CFA structures to estimate latent dimensionality robustly under item perturbations;
% (2) a \emph{fully crossed generalizability} decomposition to quantify non-item sources of variation in leaderboard scores; and
% (3) \emph{mixed latent regressions} that integrate (1) and (2) by modeling item responses with an explicit latent structure while attributing variance to ecosystem facets (e.g., contributor, deployment) and to scaling covariates (e.g., log-parameters).
% In this section we present Method~1, which establishes the latent dimensionality map of the ecosystem and provides localized diagnostics for redundancy and misfit. These results then motivate Methods~2--3, which address pseudo-replication and isolate reliable scaling signals from ecosystem-specific noise.
% \subsection{Method 1: Meta-analytic bootstrapping across CFA/SEM structures}
% \label{sec:method1_meta_cfa}
% \subsubsection{Candidate structural models as hypotheses about ecosystem constructs}
% \label{sec:method1_models}
% We compare six CFA structures (Table~\ref{tab:structural_models}) that correspond to competing hypotheses about what benchmark ecosystems measure: (i) a single general factor $g$, (ii) independent benchmark factors, (iii) correlated benchmark factors, (iv) a second-order hierarchical model, (v) an orthogonal bifactor model, and (vi) a correlated bifactor model.
% Let $\tilde{p}$ be the number of sampled items in a replication (defined below), and let $\boldsymbol{\Sigma}(\boldsymbol{\theta}; s)$ denote the implied covariance under structure $s$. The estimation target is the dependence structure of item responses (tetrachoric or robust correlation), not prediction of responses.
% \subsubsection{Bootstrap-by-item design (robustness to item selection and computational constraints)}
% \label{sec:method1_bootstrap}
% Fitting SEMs to the full item bank is often computationally prohibitive and can yield conclusions that are sensitive to the specific subset of items included. To estimate \emph{ecosystem-level} latent structure robustly, we implement a bootstrap over items within benchmarks.
% Let benchmarks be indexed by $k\in\{1,\dots,K\}$ with item sets $\mathcal{J}_k$ and sizes $|\mathcal{J}_k|=p_k$. For each bootstrap replication $b\in\{1,\dots,B\}$:
% \begin{enumerate}
% \item Sample $r_k$ items without replacement from each benchmark: $\tilde{\mathcal{J}}^{(b)}_k\subset\mathcal{J}_k$, $|\tilde{\mathcal{J}}^{(b)}_k|=r_k$.
% \item Form the pooled item set $\tilde{\mathcal{J}}^{(b)}=\bigcup_k \tilde{\mathcal{J}}^{(b)}_k$ with $\tilde{p}=\sum_k r_k$.
% \item Fit each structure $s\in\mathcal{S}$ to the corresponding response submatrix and store robust fit indices and parameter summaries.
% \end{enumerate}
% This produces an empirical distribution over fit statistics and latent parameters induced by item variation, allowing stable inferences about which structural hypotheses are supported on average.
% \subsubsection{Permutation control: structural signal vs.\ flexibility}
% \label{sec:method1_permutation}
% Flexible SEMs can fit well even when benchmark grouping is arbitrary. We therefore augment each bootstrap replication with a benchmark-label permutation that preserves group sizes but destroys benchmark semantics.
% Within replication $b$, let $\pi^{(b)}$ be a random reassignment of the sampled items to benchmarks such that $|\tilde{\mathcal{J}}^{(b)}_k|=r_k$ is preserved. For each structure $s$ and any fit statistic $T$, define the within-replication permutation contrast
% \[
% \Delta T_{s}^{(b)} \;=\; T\!\left(s \mid \pi^{(b)}\right) - T\!\left(s \mid \mathrm{id}\right),
% \]
% where $\mathrm{id}$ denotes the true assignment. When $\Delta T_{s}^{(b)}$ is consistently large (in the direction of worse fit under permutation), the model's advantage is attributable to meaningful ecosystem structure rather than generic flexibility.
% \subsubsection{Meta-analytic aggregation of fit via mixed-effects regression}
% \label{sec:method1_meta}
% Let $t_{b,s,r}$ denote a robust fit statistic (e.g., scaled RMSEA) for replication $b$, structure $s$, and randomization condition $r\in\{0,1\}$ (true vs.\ permuted). We aggregate across replications using a mixed-effects meta-regression:
% \begin{equation}
% t_{b,s,r} \;=\; \alpha_s + \beta_s r + u_b + v_b r + \varepsilon_{b,s,r},
% \label{eq:meta_fit}
% \end{equation}
% where $u_b$ captures item-sample heterogeneity and $v_b$ allows the permutation penalty to vary across replications. We weight each cell by its convergence proportion to avoid instability-driven bias.
% \subsubsection{Localized diagnostics: modification indices as Lagrange-multiplier tests}
% \label{sec:method1_mi}
% Global fit indices indicate which structure best predicts the covariance matrix; they do not reveal \emph{where} misfit arises or which items are redundant. We therefore compute modification indices (MIs) and standardized expected parameter changes (SEPCs) for residual covariances.
% For a constrained parameter $\psi$ (e.g., $\Theta_{ab}=0$ for residual covariance between items $a,b$), the MI is the Lagrange-multiplier (score) test:
% \[
% \mathrm{MI}(\psi)
% \;=\;
% U_\psi(\hat{\boldsymbol{\theta}})^\top
% \mathcal{I}_{\psi\psi\cdot\theta}(\hat{\boldsymbol{\theta}})^{-1}
% U_\psi(\hat{\boldsymbol{\theta}})
% \;\overset{H_0}{\approx}\;\chi^2_1,
% \]
% where $U_\psi$ is the score at the constrained optimum and $\mathcal{I}_{\psi\psi\cdot\theta}$ is the Schur complement of the information matrix. SEPC provides an effect-size approximation of the freed parameter, enabling a redundancy heuristic: persistent $|\mathrm{SEPC}(\Theta_{ab})|$ above a threshold indicates stable local dependence and potential item duplication.
% \subsection{Results and discussion (Method 1): bifactor-family structures dominate but benchmark boundaries are not the organizing principle}
% \label{sec:results_method1}
% \subsubsection{Global structure: a strong general factor with residual group structure}
% Across item-resampled replications, bifactor-family models consistently achieve the best average robust fit to the observed tetrachoric dependence. In the meta-analytic mixed-effects model~\eqref{eq:meta_fit}, the correlated bifactor has the lowest expected scaled RMSEA, followed closely by the orthogonal bifactor; correlated factors and second-order models fit worse, and independent benchmark factors fit worst. This rejects the implicit modularity assumption that benchmarks are independent measures of distinct capabilities.
% \paragraph{Scientific interpretation.}
% The ecosystem behaves as a two-level measurement system:
% \begin{enumerate}
% \item a dominant cross-item general capability $g$ explaining the majority of shared covariance across benchmarks, and
% \item within-benchmark dependence that remains after conditioning on $g$, captured by specific/group factors.
% \end{enumerate}
% This clarifies prior PCA findings: the leading component corresponds to an interpretable latent general factor in the sense of covariance prediction under constraints, not merely variance maximization.
% \subsubsection{Permutation analysis: correlated bifactor partially overfits and weakens benchmark semantics}
% A critical nuance emerges under permutation control. The correlated bifactor retains strong fit even when item-to-benchmark assignments are randomized, sometimes outperforming simpler models fit to the correctly assigned data. This indicates that part of its fit advantage comes from flexibility in capturing residual covariance via correlated specifics rather than from benchmark semantics. In contrast, the orthogonal bifactor’s fit degrades more under permutation, suggesting that its advantage is more tightly tied to genuine structure.
% \paragraph{Practical implication for benchmark design.}
% After partialing out a general factor, the residual dependence structure is only weakly aligned with benchmark labels: items appear to cluster by properties not captured by benchmark membership alone (shared templates, instruction format, domain overlap, or evaluation artifacts). Consequently, reporting a benchmark total score as if it were a single construct can be misleading: benchmark membership is not a reliable proxy for a coherent latent dimension.
% \subsubsection{Localized misfit: MIs and SEPCs reveal redundancy and method effects}
% Aggregated MI/SEPC diagnostics show that bifactor-family models reduce extreme residual dependencies compared to correlated-factors and hierarchical structures. Remaining high-SEPC residual covariances tend to be stable across bootstrap replications, indicating systematic local dependence (redundant items, shared prompts, or method effects). These diagnostics yield an actionable procedure for ecosystem refinement: remove or rewrite items that repeatedly exhibit high residual coupling, improving construct coverage per item and reducing artificially inflated reliability due to duplication.
% \begin{proposition}[Non-sufficiency of benchmark totals under bifactor structure]
% If the ecosystem’s item dependence is better approximated by a bifactor model than by a per-benchmark unidimensional model, then benchmark totals are not sufficient statistics for latent capability profiles: there exist model pairs $(i,i')$ with equal benchmark totals but different latent vectors $(g_i,s_{i1},\dots,s_{iK})\neq(g_{i'},s_{i'1},\dots,s_{i'K})$.
% \end{proposition}
% \paragraph{Implication for ecosystem conclusions.}
% Method~1 establishes that (i) unidimensional and independent-benchmark assumptions are untenable, (ii) a robust general factor exists across item perturbations, and (iii) benchmark labels only partially explain the remaining dependence once $g$ is removed. This directly motivates Methods~2--3: when pseudo-replication and ecosystem facets (contributors, deployment choices) are modeled explicitly, we can test whether the general factor and benchmark-specific factors remain stable and, crucially, whether observed scaling laws reflect true latent capability growth or confounding ecosystem variation.
% \subsection{From benchmark scores to \emph{reliable} scaling laws under latent structure}
% Leaderboard “scaling laws” are typically estimated by regressing an aggregate score (often a benchmark average) on $\log$-parameters. This is statistically fragile in benchmark ecosystems for two reasons. First, benchmark scores are \emph{measurement outcomes}: they are noisy functions of heterogeneous items that partially overlap in construct coverage. Second, leaderboard submissions are \emph{not i.i.d.}---they are dominated by fine-tunes, merges, quantizations, and minor variants of a small number of base model families. This pseudo-replication inflates precision, confounds “size effects” with deployment and provenance effects, and can bias conclusions about which capabilities actually scale.
% Methods~1--2 address these issues separately: Method~1 identifies a stable latent measurement structure (bifactor-family) and diagnoses local dependence; Method~2 decomposes ecosystem variance into fully crossed facets (architecture, benchmark, contributor/provenance, deployment). Method~3 unifies them: we estimate scaling laws \emph{in the latent space} while explicitly modeling (i) item parameters, (ii) general vs.\ benchmark-specific latent abilities, and (iii) multi-facet clustering that generates pseudo-replication. This produces \emph{noise-controlled scaling laws}: effects of size and deployment on latent abilities, with uncertainty that respects the ecosystem’s dependence structure.
% \paragraph{Interpretation.}
% Equations ~\eqref{eq:structural} and ~\eqref{eq:latent_mixed_reg} are the latent analogue of a fully crossed mixed model, except that the response is not the observed benchmark score but the \emph{latent capability} driving item responses under~\eqref{eq:irt}. Consequently, fixed effects quantify how model size and deployment shift the general factor and each benchmark-specific factor, while $\boldsymbol{\Sigma}_u$ estimates the magnitude of provenance-dependent variation after controlling for those covariates.
\section{Dataset information}\label{apx:data_detail}
The Hugging Face Open LLM Leaderboard \cite{fourrier_open_2024} is a public, large-scale initiative to transparently track, rank, and evaluate open-source LLMs, with 87,939,588 unique response observations. The version we use includes LLM submissions recorded on or before 2025-03-13 $(N=$4,287), according to their submission date. It employs the Eleuther AI Language Model Evaluation Harness to automate standardized scoring across a suite of benchmarks, providing a reproducible and continuously updated snapshot of the field. We employ all six prominent benchmarks in this leaderboard representing a range of cognitive tasks:
\begin{itemize}[leftmargin=*,nosep]
\item \textbf{MMLU-Pro}~\cite{wang_mmlu-pro_2024} : A challenging extension of the Massive Multitask Language Understanding benchmark ~\cite{hendrycks_measuring_2020}, designed to measure general knowledge and problem-solving skills across a vast array of subjects.
\item \textbf{IF-Eval} ~\cite{zhou_instruction-following_2023}: An instruction-following benchmark that assesses a model's ability to adhere to complex and nuanced user directives.
\item \textbf{BBH} ~\cite{suzgun_challenging_2022}: The Big-Bench Hard benchmark, a collection of tasks identified as being particularly difficult for contemporary LLMs, focusing on multi-step and advanced reasoning.
\item \textbf{MuSR}~\cite{sprague_musr_2024} : The Multi-Step Soft Reasoning benchmark, which evaluates a model's capacity for reasoning that requires integrating multiple pieces of information without strict logical formality.
\item \textbf{GPQA} ~\cite{rein_gpqa_2023}: A graduate-level question-answering dataset composed of difficult questions written by domain experts, designed to probe deep, specialized knowledge.
\item \textbf{Open LLM Math Level 5}/``MATH'' ~\cite{hendrycks_measuring_2021}: A benchmark testing mathematical problem-solving ability, aligned with high school mathematics curricula (abbreviated in some figures and tables as `openllm').
\end{itemize}
Counts of unique items/tasks and LLM models per benchmark are in Table \ref{tab:bench_counts}.
\paragraph{Leaderboard Scores} The Open LLM Leaderboard aggregates benchmark scores, used in Method 2, by normalizing raw scores relative to a random baseline and a maximum possible score, then averaging normalized scores. Scores are the leaderboard-reported task metrics (e.g., accuracy, normalized accuracy, exact match), expressed on the leaderboard scale.
\paragraph{Data Exclusion} We exclude from our analyses in Methods 2 and 3 the six observations with no positive model size (\# params$\le0$), leaving 4,281 total LLMs. For item set bootstraps in DWLS estimation method of Method 1, items with missingness are not included in the item set. For item set bootstraps in MH-RM estimation Methods 1 and 3, item-LLM-level items are not included if they have no variance across all LLMs (i.e., all LLM responses are equal; for item j, $\sigma^2(x_j)=0$).
\begin{table}[!h]
\centering
\caption{Per-benchmark item and model counts after dropping missing values}\label{tab:bench_counts}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lrrrr}
\toprule
Benchmark & N(items/tasks) & N(models tested) & Avg. item score & SD item score \\
\midrule
BBH & 5758 & 4239 & 0.491 & 0.500\\
GPQA & 1192 & 4237 & 0.301 & 0.459\\
IF-Eval & 531 & 4240 & 0.418 & 0.493\\
MMLU-Pro & 11864 & 4240 & 0.340 & 0.474\\
MuSR & 712 & 4239 & 0.428 & 0.495\\
Open LLM Math Level 5 & 686 & 4239 & 0.124 & 0.329\\
\bottomrule
\end{tabular}}
\end{table}
\section{Structural Model Relationships}
\label{apx:models}
To determine the latent dimensional structure of the AI benchmark ecosystem, we employ a meta-analytic hypothesis-testing framework using Confirmatory Factor Analysis (CFA). This approach allows us to formally specify and compare a series of competing theoretical models about how different capabilities relate to one another. Given the vast number of items, we use a random-item subsampling procedure to ensure our findings are robust and not artifacts of a specific item selection.
\subsection{The Confirmatory Factor Analysis Model}
CFA is a measurement model within the broader SEM framework that tests hypotheses about the relationships between observed variables (item responses) and unobserved latent variables (factors or constructs). The general CFA model links a $p \times 1$ vector of observed item responses $\mathbf{y}_i$ for a given LLM $i$ to an $m \times 1$ vector of its latent factor scores $\boldsymbol{\eta}_i$ via \eqref{eq:cfa}:
\begin{equation*}
\mathbf{y}_i = \boldsymbol{\nu} + \boldsymbol{\Lambda} \boldsymbol{\eta}_i + \boldsymbol{\epsilon}_i
\end{equation*}
where $\boldsymbol{\nu}$ is a vector of item intercepts, $\boldsymbol{\Lambda}$ is a $p \times m$ matrix of factor loadings specifying the relationship between items and factors, and $\boldsymbol{\epsilon}_i$ is a $p \times 1$ vector of item-specific residuals. The model-implied covariance matrix of the observed variables, $\boldsymbol{\Sigma}(\boldsymbol{\theta})$, is the target of prediction as shown in Sec. \ref{sec:cfa_eqs}:
\begin{equation}
\boldsymbol{\Sigma}(\boldsymbol{\theta}) = \boldsymbol{\Lambda} \boldsymbol{\Psi} \boldsymbol{\Lambda}^\top + \boldsymbol{\Theta}
\label{eq:cfa_cov}
\end{equation}
Here, $\boldsymbol{\Psi} = \text{Cov}(\boldsymbol{\eta})$ is the $m \times m$ covariance matrix of the latent factors, and $\boldsymbol{\Theta} = \text{Cov}(\boldsymbol{\epsilon})$ is the $p \times p$ (typically diagonal) covariance matrix of the residuals.\footnote{To support readers navigating other literature, for the remainder of this appendix, we will use the traditional notation: $\boldsymbol{\Theta}=\mathrm{Cov}(\boldsymbol{\epsilon})$.} The parameters $\boldsymbol{\theta} = \{\boldsymbol{\Lambda}, \boldsymbol{\Psi}, \boldsymbol{\Theta}\}$ are estimated to minimize the discrepancy between $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ and the observed sample covariance matrix $\mathbf{S}$.
For our binary item response data, we use two estimation algorithms to increase the robustness of the findings: one parameterization from the Factor Analysis tradition and another from the Item Response Theory tradition. In the dichotomous cases shown in this study, these parameterization were proved to have mathematical equivalence by \cite{takane_relationship_1987}. For the former, we use the tetrachoric correlation matrix as $\mathbf{S}$ and the Diagonally Weighted Least Squares (DWLS) estimator \citep{rosseel_lavaan_2012}. For the latter, we use a Metropolis-Hastings Robbins-Monro algorithm based on full information likelihood \citep{chalmers_mirt_2012}.
% \subsection{A Hierarchy of Competing Latent Structures}
% We test six increasingly complex models, depicted in Figure \ref{fig:cfa_diagram} and detailed in Table \ref{tab:structural_models}. Each represents a distinct hypothesis about the organization of LLM abilities. These range from a simple \textbf{Unidimensional} model, where all variance is explained by a single general factor ($g$), to a fully independent \textbf{Multi-unidimensional} model where each benchmark measures an orthogonal ability. More plausible are \textbf{Correlated Factors} and \textbf{Hierarchical} models, which allow abilities to be related. The central comparison is against two bifactor structures. The \textbf{Bifactor} model posits that item variance is partitioned between a general factor ($g$) and a set of orthogonal specific factors ($s_k$), one for each benchmark. This directly tests if performance is driven by a common ability plus benchmark-specific skills. The \textbf{Correlated Bifactor} model relaxes the orthogonality constraint, allowing specific factors to correlate, testing for residual covariance among benchmarks after accounting for the general factor.
\paragraph{Separating “structural signal” from flexibility with statistical comparisons and randomized permutation controls}
More flexible structural equation models can fit better even when benchmark assignment contains no information. In addition to the tests described next, we resolve the statistical question of these models by analyzing traditional covariance prediction measures, likelihood information criteria, out-of-sample prediction (which in CFA and structural equation modeling is unusual due to the statistical objective), and out-of-sample latent score reliabilities. A bootstrap iteration-wise average percent rank overview of these is in Table \ref{tab:cfa_fitstats_wide}. These far exceed typical reporting requirements, and we report all of these to demonstrate that, while imperfect and prone to overfit, the bifactor model is the most statistically justified latent structure. To quantify further the extent to which “benchmark structure” drives fit improvements, we add a \emph{within-replication permutation} condition. For each bootstrap replication $b$, we additionally construct a randomized mapping $\pi^{(b)}:\tilde{\mathcal{J}}^{(b)}\to\{1,\dots,K\}$ that preserves the per-benchmark counts $\{r_k\}$ but shuffles which items are labeled as belonging to which benchmark. We then refit each model under the permuted mapping, with randomization acting as a ``treatment''. Let $T(\cdot)$ be any fit statistic (e.g., scaled RMSEA). For a given structure $s$ and replication $b$, define $\Delta T_{s}^{(b)} \;=\; T\!\left(s\mid \pi^{(b)}\right) \;-\; T\!\left(s\mid \mathrm{id}\right)$,
where $\mathrm{id}$ denotes the true benchmark assignment. Large positive $\Delta T$ indicates that the true assignment carries structural information exploited by $s$ beyond generic flexibility.
% \subsection{Testing Signal-Noise Separation with CFA}
% \subsubsection{Detailed Overview of Competing Structural Models}
% \label{apx:models_overview}
% Below are conceptual descriptions of the six structural models evaluated in this study (see Table \ref{tab:structural_models} for formal specifications). Each model represents a distinct hypothesis about the nature of signal and noise in the benchmark ecosystem.
% \begin{enumerate}
% \item \textbf{Independent Factors:} This model reflects the naive assumption of many leaderboards. It hypothesizes that each benchmark measures a completely distinct latent ability, and these abilities are uncorrelated. All cross-benchmark correlation is assumed to be zero. See Appendix \ref{apx:indepfact}.
% \item \textbf{Single General Factor ($g$):} The most parsimonious model. It posits that all systematic shared variance among all items across all benchmarks is explained by a single, monolithic latent ability. Any remaining variance is treated as item-specific measurement error (noise). See Appendix \ref{apx:gfact}.
% \item \textbf{Hierarchical (Second-Order):} This model introduces a more structured explanation for the correlations among factors. It proposes that a higher-order general factor ($g$) gives rise to the first-order benchmark factors. Here, $g$ does not influence items directly but acts through the specific benchmark abilities. See Appendix \ref{apx:hier2ord}.
% \item \textbf{Correlated Factors:} A more realistic version of the independent factors model. It posits distinct latent abilities for each benchmark but allows these abilities to be freely correlated. For example, it can model the idea that "math ability" and "reasoning ability" are separate but related constructs. See Appendix \ref{apx:corrfact}.
% \item \textbf{Orthogonal Bifactor:} A fundamentally different structure. It posits that every item's response is influenced by two independent sources of signal: a general factor ($g$) that affects all items, and a benchmark-specific factor that affects only items within that benchmark. The key assumption is that $g$ and the specific factors are uncorrelated (orthogonal). This model is ideal for testing if benchmark-specific variance is merely a "residual grouping" effect after accounting for general ability. See Appendix \ref{apx:bifact}.
% \item \textbf{Correlated Bifactor:} A relaxation of the orthogonal bifactor model. It maintains the direct influence of $g$ on all items but allows the specific factors to be correlated with each other. This tests whether there is systematic residual covariance among benchmarks even after partialing out the main general factor. See Appendix \ref{apx:corrbifact}.
% \end{enumerate}
\subsection{Detailed Overview of Competing Structural Models}
\label{apx:models_detailed}
% \paragraph{Candidate structural hypotheses}
We test six increasingly complex models, depicted in Figure \ref{fig:cfa_diagram} and detailed in Table \ref{tab:structural_models}. Each represents a distinct hypothesis about the organization of LLM abilities (Table~\ref{tab:structural_models}), spanning dominant hypotheses implicit in current benchmark practice. %:
% (i) a single general factor ($g$), (ii) independent benchmark factors, (iii) correlated benchmark factors, (iv) a second-order (hierarchical) model, (v) an orthogonal bifactor model, and (vi) a correlated bifactor model.
% \begin{enumerate}
% \item a simple Unidimensional model (Apx \ref{apx:gfact}), where all variance is explained by a single general factor ($g$),
% \item a fully independent Multi-unidimensional model (Apx \ref{apx:indepfact}) where each benchmark measures an orthogonal ability
% \item Correlated Factors (Apx \ref{apx:corrfact})
% \item Hierarchical (Apx \ref{apx:hier2ord}) models, which allow abilities to be related.
% \end{enumerate}
% A central comparison is against two bifactor structures. The Bifactor (Apx \ref{apx:bifact}) model posits that item variance is partitioned between a general factor ($g$) and a set of orthogonal specific factors ($s_k$), one for each benchmark. This directly tests if performance is driven by a common ability plus benchmark-specific skills. The Correlated Bifactor (Apx \ref{apx:corrbifact}) model relaxes the orthogonality constraint, allowing specific factors to correlate, testing for residual covariance among benchmarks after accounting for the general factor.
Each model is a constraint set on $(\boldsymbol{\Lambda},\boldsymbol{\Phi},\boldsymbol{\Theta})$; importantly, models differ not only in parameter count but in whether benchmark identity is treated as a substantive construct, a residual grouping, or an artifact after partialing out $g$. Critically, these models represent different strategies for partitioning measurement variance into signal (latent factors) and noise (error terms $\boldsymbol{\Theta}$).
Below are conceptual descriptions of the six structural models evaluated in this study (see Table \ref{tab:structural_models} for formal specifications). Each model represents a distinct hypothesis about the nature of signal and noise in the benchmark ecosystem.
\subsubsection{Model 1: Multi-unidimensional Independent Benchmark Factors Model}\label{apx:indepfact}
This model reflects the naive assumption of many leaderboards. It hypothesizes that each benchmark measures a completely distinct latent ability, and these abilities are uncorrelated. All cross-benchmark correlation is assumed to be zero. This model (see (a) Figure \ref{fig:cfa_diagram}%, Code \ref{code:lavaan_indep}
) assumes that each benchmark $k$ measures a distinct latent factor $f_k$ that is uncorrelated with all other benchmark factors. It serves as a baseline for assessing the presence of inter-benchmark correlations.
\begin{equation}
y_{ikj} = \lambda_{kj} f_{ik} + \epsilon_{ikj}, \quad \text{with} \quad
% \end{equation}
% \begin{equation}
\text{Cov}(f_{ik}, f_{ik'}) = 0 \text{ for } k \neq k'
\end{equation}
This represents a modular view of AI capabilities, where proficiency in one domain (e.g., math) is entirely independent of proficiency in another (e.g., instruction following).
\subsubsection{Model 2: Unidimensional, Single General Factor Model}\label{apx:gfact}
The most parsimonious model. It posits that all systematic shared variance among all items across all benchmarks is explained by a single, monolithic latent ability. Any remaining variance is treated as item-specific measurement error (noise). This model (see (b) Figure \ref{fig:cfa_diagram}%, Code \ref{code:lavaan_genfact}
) tests the hypothesis that a single latent factor, $g$, accounts for all non-error covariance among all items across all benchmarks. It represents the most parsimonious explanation of performance.
\begin{equation}
y_{ikj} = \lambda_{kj} g_i + \epsilon_{ikj}
\end{equation}
where $\lambda_{kj} \in \mathbb{R}$ is the loading of item $j$ from benchmark $k$ on the general factor $g_i \sim \mathcal{N}(0,1)$, and $\epsilon_{ikj} \sim \mathcal{N}(0, \theta_{kj})$ is the item-specific residual variance. This model posits that all benchmarks are, in effect, measuring the same underlying construct.
\subsubsection{Model 3: Hierarchical Second-Order Factor Model}\label{apx:hier2ord}
This model introduces a more structured explanation for the correlations among factors. It proposes that a higher-order general factor ($g$) gives rise to the first-order benchmark factors. Here, $g$ does not influence items directly but acts through the specific benchmark abilities. This model (see (c) Figure \ref{fig:cfa_diagram}%, Code \ref{code:lavaan_hier}
) proposes a hierarchical structure where a higher-order general factor, $g$, explains the correlations among the first-order benchmark factors from Model 3.
\begin{align}
y_{ikj} &= \lambda_{kj} f_{ik} + \epsilon_{ikj} \\
f_{ik} &= \gamma_k g_i + \delta_{ik}
\end{align}
Here, the first-order factor $f_{ik}$ is regressed on the second-order factor $g_i$, with $\gamma_k$ being the loading and $\delta_{ik}$ the first-order factor disturbance. This structure implies that the general factor's influence on items is indirect, mediated entirely through the benchmark-specific factors.
\subsubsection{Model 4: Multidimensional Correlated Benchmark Factors Model}\label{apx:corrfact}
A more realistic version of the independent factors model. It posits distinct latent abilities for each benchmark but allows these abilities to be freely correlated. For example, it can model the idea that math ability and reasoning ability are separate but related constructs. This is a standard CFA model (see (d) Figure \ref{fig:cfa_diagram}%, Code \ref{code:lavaan_corrfact}
) that relaxes the strict independence assumption of Model 2. Each item loads onto its designated benchmark factor, but the benchmark factors are allowed to covary.
\begin{equation}
y_{ikj} = \lambda_{kj} f_{ik} + \epsilon_{ikj}, \quad \text{with} \quad \text{Cov}(\mathbf{f}_i) = \mathbf{\Psi}
\end{equation}
where $\mathbf{f}_i$ is the vector of latent factor scores for LLM $i$, and $\mathbf{\Psi}$ is a symmetric correlation matrix with 1s on the diagonal. The off-diagonal elements $\phi_{kk'}$ quantify the degree of relationship between distinct benchmark abilities.
\subsubsection{Model 5: The Bifactor Model}\label{apx:bifact}
A fundamentally different structure. It posits that every item's response is influenced by two independent sources of signal: a general factor ($g$) that affects all items, and a benchmark-specific factor that affects only items within that benchmark. The key assumption is that $g$ and the specific factors are uncorrelated (orthogonal). This model is ideal for testing if benchmark-specific variance is merely a residual grouping effect after accounting for general ability. The bifactor model (see (e) Figure \ref{fig:cfa_diagram}, Code \ref{code:lavaan_bifact}) \citep{dunn_place_2020} provides an alternative explanation for shared variance. It posits that each item's variance is simultaneously explained by a general factor $g$ (common to all items) and a benchmark-specific factor $s_k$ (common only to items within that benchmark). The factors are constrained to be orthogonal.
\begin{equation}
y_{ikj} = \lambda^{(g)}_{kj} g_i + \lambda^{(s)}_{kj} s_{ik} + \epsilon_{ikj}
\end{equation}
where $\text{Cov}(g_i, s_{ik}) = 0$ and $\text{Cov}(s_{ik}, s_{ik'}) = 0$ for $k \neq k'$. This model cleanly partitions item variance into a general component and a specific component, allowing for a direct assessment of how much unique information each benchmark provides after accounting for general ability. While it is not likely that the bifactor model truly represents human latent factor organization (as capabilities are not likely orthogonal), this structure is not yet tested on the latent capability organization for AI models. The code for the model specifications for the bifactor model, as represented in Methods 1 and 3, are found in Appendix \ref{apx:code}.
\subsubsection{Model 6: Multidimensional/Correlated Bifactor Model}\label{apx:corrbifact}
A relaxation of the orthogonal bifactor model. It maintains the direct influence of $g$ on all items but allows the specific factors to be correlated with each other. This tests whether there is systematic residual covariance among benchmarks even after partialing out the main general factor.
This model (see (f) Figure \ref{fig:cfa_diagram}%, Code \ref{code:lavaan_corrbifact}
) is a relaxation of the bifactor model, allowing the specific factors $s_k$ to correlate with each other while remaining orthogonal to the general factor $g$.
\begin{equation}
y_{ikj} = \lambda^{(g)}_{kj} g_i + \lambda^{(s)}_{kj} s_{ik} + \epsilon_{ikj}, \quad \text{with} \quad
% \quad \text{Cov}(g_i, \mathbf{s}_i) = \mathbf{0} \text{ and } \text{Cov}(\mathbf{s}_i) = \mathbf{\Psi}
% \end{equation}
% \begin{equation}
\text{Cov}(g_i, \mathbf{s}_i) = \mathbf{0} \text{ and } \text{Cov}(\mathbf{s}_i) = \mathbf{\Psi}
\end{equation}
where $\mathbf{\Psi}$ is the correlation matrix of the specific factors. This model is useful for testing whether any residual covariance between benchmark domains remains after partialing out a broad general factor, which may be attributable to shared methods or sub-domain content.
In actuality, the more complicated model may be the best representation. If bootstrapped item set sizes were increased (e.g., 500 items per benchmark sampled), the more complex and expressive model would likely fit better. However, this does not mean that this is a more likely ``structural truth''. This would be a statistical artifact of a high-noise multidimensional system being explained by a higher capacity model. A key idea in comparing latent structural models is harmonizing the fit with explainable parsimony.
\subsection{Traditional Model Evaluation and Comprehensive Comparison Framework}\label{apx:comparison_fit_stats}
We evaluate and compare the six models using a combination of global fit indices, formal hypothesis tests, and parameter-level diagnostics to build a convergent body of evidence.
% \subsubsection{Global Fit Indices}
% To assess how well a model's implied covariance matrix $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ reproduces the observed sample covariance matrix $\mathbf{S}$, we use a standard set of fit indices.
\subsubsection{Traditional Measures of Fit}
In CFA, traditional measures of fit assess how well a model's implied covariance matrix $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ reproduces the observed sample covariance matrix $\mathbf{S}$, we use a standard set of fit indices in addition to our more robust metrics and methods.
\begin{itemize}[leftmargin=*,nosep]
\item \textbf{Comparative Fit Index (CFI)} and \textbf{Tucker-Lewis Index (TLI)}: These incremental fit indices measure the improvement in fit of a target model over a null (independence) model. Values $\ge .95$ are typically considered to indicate good fit \citep{medsker_review_1994}.
\item \textbf{Standardized Root Mean Square Residual (SRMR)}: This index represents the average standardized difference between the observed and predicted correlations. Values $\le .08$ are considered indicative of good fit.
\item \textbf{Root Mean Square Error of Approximation (RMSEA)}: This index estimates the model misfit per degree of freedom, with smaller values indicating better fit. Values $\le .06$ have traditionally been used to classify a close fit to the data \citep{medsker_review_1994, kenny_performance_2015}.
\end{itemize}
\paragraph{Weaknesses of Traditional Measures}
Traditional measures of fit are expansive, but share weakeness. For example, RMSEA, while arguably one of the best options for fit, is sensitive to factors such as sample size, number of indicators, and data characteristics \cite{kenny_performance_2015,groskurth_why_2024}. Fixed cutoff values like $0.06$ derived from simulation studies may not be universally applicable across diverse real-world settings \citep{mcneish_dynamic_2023}. With $n \approx 4000, \; p \ge 30$, as in the present study, RMSEA variants can become sensitive to trivially small covariance residuals. Traditional cutoffs were calibrated on $n \in [200,500]$ with $p < 30$ (Remark \ref{remark:finitesamp}). At our scale, minor misspecification can inflate RMSEA while leaving substantive conclusions intact.
Thankfully, our model choice does not rest on covariance RMSEA; we establish practical and statistical fit with our robustness checks, such as out-of-sample AUC and MAE (Tables \ref{tab:cfa_fitstats_medians} and \ref{tab:cfa_fitstats_wide}), Lagrangian Multiplier tests (\S\ref{apx:math}), permutation controls, \S\ref{apx:cfa_fit}, etc.
% \S ~\ref{apx:math}, ~\ref{apx:cfa_fit}
% Apdx C, E, Fig. 5-7, Tab. 5, etc.).
Table \ref{tab:cfa_fitstats_wide} shows bifactor-family models dominate across traditional criteria (e.g., CFI, SRMR, BIC, log-likelihood) in within-bootstrap percent-rank comparisons estimated with DWLS. Tab \ref{tab:lrt_boots} confirms that the bifactor significantly improves over every simpler structure in 100\% of bootstraps via LRTs.
% \subsection{Latent Structure Fit Statistics}
\input{tables/cfa_fit_statistics}
% \begin{enumerate}
% \item \textbf{Single General Factor ($g$):} The most parsimonious model. It posits that all systematic shared variance among all items across all benchmarks is explained by a single, monolithic latent ability. Any remaining variance is treated as item-specific measurement error (noise).
% \item \textbf{Independent Factors:} This model reflects the naive assumption of many leaderboards. It hypothesizes that each benchmark measures a completely distinct latent ability, and these abilities are uncorrelated. All cross-benchmark correlation is assumed to be zero.
% \item \textbf{Correlated Factors:} A more realistic version of the independent factors model. It posits distinct latent abilities for each benchmark but allows these abilities to be freely correlated. For example, it can model the idea that "math ability" and "reasoning ability" are separate but related constructs.
% \item \textbf{Hierarchical (Second-Order):} This model introduces a more structured explanation for the correlations among factors. It proposes that a higher-order general factor ($g$) gives rise to the first-order benchmark factors. Here, $g$ does not influence items directly but acts through the specific benchmark abilities.
% \item \textbf{Orthogonal Bifactor:} A fundamentally different structure. It posits that every item's response is influenced by two independent sources of signal: a general factor ($g$) that affects all items, and a benchmark-specific factor that affects only items within that benchmark. The key assumption is that $g$ and the specific factors are uncorrelated (orthogonal). This model is ideal for testing if benchmark-specific variance is merely a "residual grouping" effect after accounting for general ability.
% \item \textbf{Correlated Bifactor:} A relaxation of the orthogonal bifactor model. It maintains the direct influence of $g$ on all items but allows the specific factors to be correlated with each other. This tests whether there is systematic residual covariance among benchmarks even after partialing out the main general factor.
% \end{enumerate}
\begin{table*}[h]
\centering
\caption{\textbf{Latent Structure Measurement Comparison Overview:} Iteration-wise Median value across Bootstraps by Metric}
% \label{tab:cfa_fitstats_wide}
\label{tab:cfa_fitstats_medians}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lrrrrrrrrrrrrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Fit Indices / Covariance Fit} & \multicolumn{2}{c}{Test Accuracy} & \multicolumn{7}{c}{Latent Factor Test Set Reliability Estimates} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-7} \cmidrule(l{3pt}r{3pt}){8-14} %\cmidrule(l{3pt}r{3pt}){11-17}
Structure & RMSEA & CFI & TLI & SRMR & AUC & MAE & G & IF-EVAL & Math Lvl. 5 & MMLU-Pro & BBH & GPQA & MUSR\\
\midrule
indepfact & 0.09 & 0.31 & 0.28 & 0.22 & 0.90 & 0.25 & & \textit{0.84} & \textbf{0.83} & 0.69 & 0.62 & 0.29 & 0.32\\
gfact & 0.06 & 0.70 & 0.69 & 0.14 & 0.87 & 0.28 & \textbf{0.89} & & & & & & \\
hier2ord & 0.05 & 0.76 & 0.75 & 0.13 & 0.89 & 0.25 & \textit{0.88} & \textit{0.84 }& 0.77 & 0.59 & 0.48 & \textbf{0.30} & \textit{0.29}\\
corrfact & 0.05 & 0.76 & 0.75 & 0.13 & 0.55 & 0.25 & & \textbf{0.85} & \textbf{0.83} & \textbf{0.72} & \textbf{0.65} & 0.24 & 0.19\\
bifact & \textit{0.05} & \textit{0.80} & \textit{0.78} & \textit{0.12} & \textbf{0.91} & \textbf{0.24} & 0.85 & 0.68 & 0.62 & 0.29 & 0.25 & \textit{0.29} & \textit{0.29}\\
corrbifact & \textbf{0.04} & \textbf{0.86} & \textbf{0.84} & \textbf{0.10} & 0.58 & 0.24 & 0.82 & 0.70 & 0.63 & 0.40 & 0.33 & 0.23 & 0.18\\
\bottomrule
\end{tabular}}
\vspace{0.25ex}
\caption*{\small \textbf{Fit Statistics for Hypothesized Latent Structures: aggregated by median across bootstraps}. All metrics were ordered so higher percent ranks correspond with more desirable structural fit. RMSEA, CFI, and TLI all use their scaled and standardized versions which better support comparisons in bootstraps. The RMSEA values differ from those in Table \ref{tab:structural_models} to be comparable with the other traditional metrics (e.g., TLI, SRMR), because these were only estimated with DWLS.}
\end{table*}
\begin{table*}[h]
\centering
\caption{\textbf{Latent Structure Measurement Comparison Overview:} Average Iteration-wise Percent Rank across Bootstraps by Metric}
\label{tab:cfa_fitstats_wide}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lrrrrrrrrrrrrrrrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Fit Indices / Covariance Fit} & \multicolumn{3}{c}{Information Criteria} & \multicolumn{2}{c}{Test Accuracy} & \multicolumn{7}{c}{Latent Factor Test Set Reliability Estimates} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-8} \cmidrule(l{3pt}r{3pt}){9-10} \cmidrule(l{3pt}r{3pt}){11-17}
Structure & RMSEA & CFI & TLI & SRMR & AIC & BIC & log Lik. & AUC & MAE & G & IF-EVAL & Math Lvl. 5 & MMLU-Pro & BBH & GPQA & MUSR\\
\midrule
indepfact & 0.7 & 0.7 & 0.7 & 0.7 & 8.8 & 8.8 & 8.8 & 76.5 & 41.6 & & \textit{74.0 }& \textbf{80.2} & \textbf{74.3} & \textit{66.8} & \textbf{56.5} & \textbf{60.2}\\
gfact & 25.6 & 25.7 & 25.9 & 24.6 & 10.9 & 10.9 & 10.9 & 38.2 & 0.0 & \textbf{86.0} & & & & & & \\
hier2ord & 50.4 & 45.0 & 50.3 & 40.3 & 41.5 & 41.5 & 41.5 & 59.5 & 38.7 & 54.9 & 51.3 & 53.0 & 56.8 & 52.4 & 35.3 & 37.9\\
corrfact & 53.9 & 55.7 & 54.1 & 54.0 & 56.8 & 56.8 & 56.8 & 19.1 & 37.9 & & \textbf{83.8} & \textit{78.6} & \textit{69.6} & \textbf{76.3} & \textit{54.9} & \textit{55.3}\\
bifact & \textit{75.7} & \textit{75.7} & \textit{75.6} & \textit{75.8} & \textit{79.4} & \textit{79.4} & \textit{79.4} & \textbf{94.5} & \textit{86.5} & 40.5 & 18.9 & 14.2 & 24.0 & 21.4 & 50.7 & 48.3\\
corrbifact & \textbf{99.1} & \textbf{98.8} & \textbf{98.8} & \textbf{99.2} & \textbf{100.0} & \textbf{100.0} & \textbf{100.0} & 44.9 & \textbf{93.1} & 20.5 & 22.4 & 25.1 & 28.1 & 34.1 & 46.4 & 43.3\\
\bottomrule
\end{tabular}}
\vspace{0.25ex}
\caption*{\small \textbf{Fit Statistics for Hypothesized Latent Structures: aggregated across bootstraps by within bootstrap percent rank}. All metrics were ordered so higher percent ranks correspond with more desirable structural fit. RMSEA, CFI, and TLI all use their scaled and standardized versions which better support comparisons in bootstraps.}
\end{table*}
% \begin{table*}[h]
% \centering
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{lrrrrrrrrrrrrrrrr}
% \toprule
% \multicolumn{1}{c}{ } & \multicolumn{4}{c}{Fit Indices / Covariance Fit} & \multicolumn{3}{c}{Information Criteria} & \multicolumn{2}{c}{Test Accuracy} & \multicolumn{7}{c}{Latent Factor Test Set Reliability Estimates} \\
% \cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-8} \cmidrule(l{3pt}r{3pt}){9-10} \cmidrule(l{3pt}r{3pt}){11-17}
% Structure & RMSEA & CFI & TLI & SRMR & AIC & BIC & log Lik. & AUC & MAE & G & IF-EVAL & Math Lvl. 5 & MMLU-Pro & BBH & GPQA & MUSR\\
% \midrule
% indepfact & 0.09 & 0.31 & 0.28 & 0.22 & 195380 & 196253 & -97546 & 0.90 & 0.25 & & 0.84 & 0.83 & 0.69 & 0.62 & 0.29 & 0.32\\
% gfact & 0.06 & 0.70 & 0.69 & 0.14 & \textbf{169478} & \textbf{170205} & \textbf{-84619} & 0.87 & 0.28 & \textbf{0.89} & & & & & & \\
% hier2ord & 0.05 & 0.76 & 0.75 & 0.13 & 191261 & 192207 & -95475 & 0.89 & 0.25 & \textit{0.88} & 0.84 & 0.77 & 0.59 & 0.48 & 0.30 & 0.29\\
% corrfact & 0.05 & 0.76 & 0.75 & 0.13 & 189838 & 190802 & -94760 & 0.55 & 0.25 & & 0.85 & 0.83 & 0.72 & 0.65 & 0.24 & 0.19\\
% bifact & \textit{0.05} & \textit{0.80} & \textit{0.78} & \textit{0.12} & 187461 & 188770 & -93514 & \textbf{0.91} & \textbf{0.24} & 0.85 & 0.68 & 0.62 & 0.29 & 0.25 & 0.29 & 0.29\\
% corrbifact & \textbf{0.04} & \textbf{0.86} & \textbf{0.84} & \textbf{0.10} & \textit{185599} & \textit{186999} & \textit{-92568} & 0.58 & 0.24 & 0.82 & 0.70 & 0.63 & 0.40 & 0.33 & 0.23 & 0.18\\
% \bottomrule
% \end{tabular}}
% \vspace{0.25ex}
% \caption*{\small \textbf{Fit Statistics for Hypothesized Latent Structures: aggregated by median across bootstraps}. All metrics were ordered so higher percent ranks correspond with more desirable structural fit. RMSEA, CFI, and TLI all use their scaled and standardized versions which better support comparisons in bootstraps. }
% \end{table*}
\section{Mathematical Set-up for Testing Local Dependence}\label{apx:math}
\subsection{Mathematical Foundations for Score Tests and Local Dependence}
\subsubsection{Modification indices as Lagrange-multiplier diagnostics for local dependence}
\label{sec:mi_lm_proof}
We formalize the common structural equation modeling “modification index” (MI) as a score/Lagrange-multiplier test and show how it quantifies (i) \emph{violations of local dependence} (unmodeled residual dependence between items) and, more generally, (ii) \emph{subject-dependent error} (unmodeled heterogeneity that induces residual correlations after conditioning on the latent variables).
\paragraph{Setup (latent-response CFA).}
Let $\mathbf{y}_i\in\{0,1\}^p$ be item responses for model/subject $i\in\{1,\dots,n\}$ based on Eq. \eqref{eq:cfa}, generated via an underlying continuous response $\mathbf{y}^\star_i\in\mathbb{R}^p$:
\[
\mathbf{y}^\star_i = \boldsymbol{\nu} + \boldsymbol{\Lambda}\boldsymbol{\eta}_i + \boldsymbol{\epsilon}_i,
\;
\boldsymbol{\eta}_i\sim\mathcal{N}(\mathbf{0},\boldsymbol{\Phi}),
\;
\boldsymbol{\epsilon}_i\sim\mathcal{N}(\mathbf{0},\boldsymbol{\Theta}),
\]
with $y_{ij}=\mathbb{1}[y^\star_{ij}>\tau_j]$. In the standard CFA identification, $\boldsymbol{\Theta}$ is constrained to be diagonal. The implied covariance, defined in \S~\ref{sec:cfa_eqs}, of $\mathbf{y}^\star$ is
% \[
$\boldsymbol{\Sigma}(\boldsymbol{\theta}) \;=\; \boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top + \boldsymbol{\Theta},
\;
\boldsymbol{\theta} \text{ collects all free parameters}$.
% \]
% $\boldsymbol{\theta}$ collects all free parameters
Let $\mathbf{S}$ be an estimate of the population correlation/covariance of $\mathbf{y}^\star$ (e.g., tetrachoric correlation matrix). Estimation proceeds by minimizing a discrepancy $F(\boldsymbol{\Sigma}(\boldsymbol{\theta}),\mathbf{S})$ (ML, WLSMV, etc.), equivalently maximizing a log-likelihood $\ell(\boldsymbol{\theta})$.
\paragraph{Local dependence as a constrained parameter.}
Pick a candidate residual covariance between items $(a,b)$, denoted
\[
\psi \;\equiv\; \mathrm{Cov}(\epsilon_a,\epsilon_b) \;=\; \Theta_{ab}.
\]
The standard model imposes the constraint $H_0:\psi=0$. Define the augmented parameter vector $\tilde{\boldsymbol{\theta}}=(\boldsymbol{\theta},\psi)$ and the constrained optimum
\[
\hat{\boldsymbol{\theta}} \;\in\; \arg\max_{\boldsymbol{\theta}} \ell(\boldsymbol{\theta},\psi=0).
\]
\subsubsection{MI as a Lagrange-multiplier / score test}
\label{sec:mi_score_test}
\begin{theorem}[Modification index as an LM statistic]
\label{thm:mi_lm}
Assume $\ell(\tilde{\boldsymbol{\theta}})$ is twice continuously differentiable in a neighborhood of the true parameter and that standard regularity conditions for score tests hold (listed below). Let the score for $\psi$ at the constrained optimum be
\[
U_\psi(\hat{\boldsymbol{\theta}}) \;\equiv\; \left.\frac{\partial \ell(\boldsymbol{\theta},\psi)}{\partial \psi}\right|_{(\boldsymbol{\theta},\psi)=(\hat{\boldsymbol{\theta}},0)}.
\]
Let $\mathcal{I}(\tilde{\boldsymbol{\theta}})$ be the expected Fisher information for $\tilde{\boldsymbol{\theta}}$ and write its block decomposition
\[
\mathcal{I} \;=\;
\begin{bmatrix}
\mathcal{I}_{\theta\theta} & \mathcal{I}_{\theta\psi}\\
\mathcal{I}_{\psi\theta} & \mathcal{I}_{\psi\psi}
\end{bmatrix}.
\]
Define the \emph{partial} information (Schur complement)
\[
\mathcal{I}_{\psi\psi\cdot \theta} \;\equiv\; \mathcal{I}_{\psi\psi} - \mathcal{I}_{\psi\theta}\mathcal{I}_{\theta\theta}^{-1}\mathcal{I}_{\theta\psi}.
\]
Then the Lagrange-multiplier (score) statistic for testing $H_0:\psi=0$ is
\[
\mathrm{MI}(\psi)
\;=\;
U_\psi(\hat{\boldsymbol{\theta}})^\top \,
\mathcal{I}_{\psi\psi\cdot \theta}(\hat{\boldsymbol{\theta}})^{-1}
\,U_\psi(\hat{\boldsymbol{\theta}})
\;\;\xrightarrow{d}\;\; \chi^2_{1}.
\]
Moreover, $\mathrm{MI}(\psi)$ is locally (asymptotically) most powerful among tests of size $\alpha$ for alternatives $\psi = O(n^{-1/2})$ \citep{buse_goodness_1973,engel_chapter_1984}.
\end{theorem}
\begin{proof}[Proof sketch]
The KKT conditions for constrained maximization yield $\hat{\lambda} = -U_\psi(\hat{\boldsymbol{\theta}})$: the Lagrange multiplier equals (minus) the score for the constrained parameter. Normalizing by the Schur complement $\mathcal{I}_{\psi\psi\cdot\theta}$ yields the stated quadratic form. Under $H_0$ and regularity, $n^{-1/2}U_\psi(\hat{\boldsymbol{\theta}})\Rightarrow \mathcal{N}(0,\mathcal{I}_{\psi\psi\cdot\theta})$, so the quadratic form converges to $\chi^2_1$. Local asymptotic optimality follows from standard LAN arguments \citep{buse_goodness_1973}.
\end{proof}
\paragraph{Statistical conditions (typical SEM/ML assumptions).}
A sufficient set of conditions for Theorem~\ref{thm:mi_lm} (and its robust analogs) includes:
\begin{enumerate}[leftmargin=*,nosep]
\item \textbf{Identification:} the constrained model is locally identified at the true parameter; $\mathcal{I}_{\theta\theta}$ is nonsingular.
\item \textbf{Interior point:} $\psi=0$ lies in the interior of the parameter space for the relevant parametrization (or use boundary-corrected tests if not).
\item \textbf{Regularity/smoothness:} $\ell$ is twice continuously differentiable in a neighborhood; interchange of differentiation and integration is valid.
\item \textbf{Asymptotics:} $n\to\infty$ with $p$ fixed (or $p$ increasing slowly under additional conditions); $\mathbf{S}$ is consistent for the population covariance of $\mathbf{y}^\star$.
% \item \textbf{Robustness:} if non-normality/misspecification is present (common with binary items), use sandwich/robust information (e.g., WLSMV/MLR) to replace $\mathcal{I}$ by a consistent estimator of $\mathrm{Var}(U)$.
\end{enumerate}
\begin{remark}[Finite-sample performance]\label{remark:finitesamp}
Simulation studies show that MI statistics maintain approximately correct Type I error rates for $n\geq 200$ and moderate model complexity ($p\leq 50$), with scaled corrections essential for categorical data \citep{saris_testing_2009,kenny_performance_2015, mcneish_dynamic_2023,groskurth_why_2024}. In our bootstrap analysis, each replication uses $n\approx 3{,}000$ LLM submissions, well above the threshold for asymptotic validity.
\end{remark}
\subsubsection{MI quantifies violations of local dependence}
\label{sec:mi_local_dependence}
We now connect the MI for $\psi=\Theta_{ab}$ to the \emph{local dependence} principle.
\begin{definition}[Local independence]
Items are locally independent given the latent variables $\boldsymbol{\eta}$ if
\[
p(\mathbf{y}\mid \boldsymbol{\eta}) = \prod_{j=1}^p p(y_j\mid \boldsymbol{\eta}).
\]
In the Gaussian latent-response CFA above, local independence is equivalent to $\boldsymbol{\Theta}$ being diagonal.
\end{definition}
\begin{proposition}[Residual covariance as a measure of local dependence]
\label{prop:theta_local_dependence}
Under the latent-response CFA, if $\Theta_{ab}\neq 0$ then $y_a \not\!\perp\!\!\!\perp y_b\mid \boldsymbol{\eta}$ (local dependence fails). Conversely, if $\boldsymbol{\Theta}$ is diagonal, then $y_a \perp\!\!\!\perp y_b\mid \boldsymbol{\eta}$ for all $a\neq b$.
\end{proposition}
\begin{proof}
For multivariate Gaussians, conditional independence holds iff the corresponding covariance is zero. Thus $\Theta_{ab}\neq 0$ implies dependence between $y^\star_a$ and $y^\star_b$ given $\boldsymbol{\eta}$, which transfers to the binary items.
\end{proof}
\paragraph{Interpretation of MI.}
In this setting, $\mathrm{MI}(\Theta_{ab})$ is precisely a test for whether the data demand a nonzero residual dependence between items $a$ and $b$ \emph{after} accounting for the latent structure in $\boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top$. The larger the MI, the stronger the evidence that the current factor structure under-explains (or over-explains) the empirical association between $(a,b)$. Geometrically, it measures the instantaneous rate of improvement in log-likelihood if constraint $(a,b)$ were relaxed by an infinitesimal amount. In information-geometric terms, it quantifies the projection of the unconstrained score onto the constraint manifold.
\subsubsection{MI detects subject-dependent error as induced residual dependence}
\label{sec:mi_subject_dependent_error}
Local dependence is not only “item redundancy.” It can also arise because the error is \emph{subject-dependent}: different subjects/models induce different item difficulties or different effective loadings, producing residual covariance when the model incorrectly assumes homogeneity.
\paragraph{A concrete heterogeneity model.}
Let $z_i\in\mathbb{R}$ be an unmodeled subject-specific variable (e.g., a latent “format adherence” trait, prompting style, or evaluator-specific artifact) independent of $\boldsymbol{\eta}_i$, and suppose the true latent response model is
\[
\mathbf{y}^\star_i
=
\boldsymbol{\nu}+\boldsymbol{\Lambda}\boldsymbol{\eta}_i
+ \mathbf{b}\,z_i
+ \boldsymbol{\xi}_i,
\]
\[
z_i\sim \mathcal{N}(0,\sigma_z^2),\
\boldsymbol{\xi}_i\sim \mathcal{N}(\mathbf{0},\boldsymbol{\Theta}_0) \text{ diagonal}.
\]
Then the \emph{marginal} covariance of $\mathbf{y}^\star$ is
\[
\mathrm{Cov}(\mathbf{y}^\star)
=
\boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top
+
\underbrace{\mathbf{b}\mathbf{b}^\top\sigma_z^2}_{\text{induced residual dependence}}
+
\boldsymbol{\Theta}_0.
\]
If we fit the misspecified CFA that omits $z_i$ and forces residuals diagonal, the term $\mathbf{b}\mathbf{b}^\top\sigma_z^2$ cannot be represented by $\boldsymbol{\Theta}$ and will appear as systematic misfit. In particular,
\[
\Theta_{ab}^{\text{true, marginal}} \;=\; b_a b_b \sigma_z^2 \neq 0
\]
whenever $b_a b_b\neq 0$, inducing residual covariance among the affected items.
\begin{corollary}[MI detects unmodeled subject heterogeneity through residual covariances]
\label{cor:mi_heterogeneity}
Under the heterogeneity model above, for any pair $(a,b)$ with $b_a b_b\sigma_z^2\neq 0$, the score $U_{\Theta_{ab}}(\hat{\boldsymbol{\theta}})$ is generically nonzero in the diagonal-residual CFA, and $\mathrm{MI}(\Theta_{ab})$ diverges at rate $O_p(n)$, yielding asymptotic power $1$ to detect the induced local dependence.
\end{corollary}
\begin{proof}
The fitted model cannot match the population covariance entry $b_a b_b\sigma_z^2$ when forcing $\Theta_{ab}=0$, producing a nonzero population score. By M-estimation theory, $\mathrm{MI}$ grows at rate $O_p(n)$, yielding consistency.
\end{proof}
\begin{remark}
Corollary~\ref{cor:mi_heterogeneity} is the key bridge to “subject-dependent error” in benchmark ecosystems: if some models systematically exploit (or fail on) a shared artifact (formatting, instruction parsing, choice biases), that unmodeled trait acts like $z_i$ and induces a low-rank residual covariance pattern among the affected items. MIs will concentrate on item pairs within that pattern—even if the original factor structure is otherwise correct—flagging a \emph{measurement failure mode} rather than merely “redundant questions.”
\end{remark}
\subsubsection{Effect-size direction via expected parameter change (EPC)}
\label{sec:epc_direction}
Beyond hypothesis testing, practitioners want an approximate \emph{magnitude and sign} of the violation. Under a second-order approximation, freeing $\psi$ yields
\[
\widehat{\Delta \psi}
\;\approx\;
\mathrm{EPC}(\psi)
\;=\;
\mathcal{I}_{\psi\psi\cdot\theta}(\hat{\boldsymbol{\theta}})^{-1}\,U_\psi(\hat{\boldsymbol{\theta}}).
\]
In particular:
\begin{itemize}[leftmargin=*,nosep]
\item $\mathrm{EPC}(\Theta_{ab})>0$ indicates the model underpredicts the residual association of items $(a,b)$ after conditioning on the factors (shared method/content, induced heterogeneity); the items share more dependence than the latent factors $\boldsymbol{\eta}$ can explain.
\item $\mathrm{EPC}(\Theta_{ab})<0$ indicates overprediction (the factor structure implies stronger association than observed), often signaling cross-loading structure, suppressor effects, or miscoded item polarity.
\end{itemize}
The \textbf{Standardized EPC (SEPC)} by implied residual variances places this on a correlation scale, making it interpretable as the estimated residual correlation, as used in our redundancy analyses.
% Standardizing EPC by implied residual variances gives the SEPC used in our redundancy analyses.
\subsubsection{Summary: what MI certifies and what it does not}
\label{sec:mi_summary}
\begin{itemize}[leftmargin=*,nosep]
\item \textbf{What MI is:} a Lagrange-multiplier/score statistic measuring the (curvature-normalized) gradient pressure to relax a constraint. For $\Theta_{ab}=0$, it is a direct diagnostic for conditional dependence remaining after the latent structure.
\item \textbf{What MI detects in specific:} (i) local dependence from item overlap effects, and (ii) residual dependence induced by unmodeled subject heterogeneity or other misspecification that manifests as correlated errors.
\item \textbf{What MI detects across bootstraps:} (i) propensities in local dependence, and (ii) trends in residual dependence induced by unmodeled subject heterogeneity or other misspecification that manifests as correlated errors.
\item \textbf{What MI does not certify:} a causal explanation. A large MI says “this constraint is wrong for the dependence structure,” not “this pair is semantically redundant.” Pairwise disambiguation can require substantive inspection or modeling the alternative mechanism (e.g., adding a method factor versus correlating residuals, bootstrapping for generalization).
\end{itemize}
This formal view justifies our use of MI/SEPC aggregated over bootstrap replications: it yields a statistically principled, computationally efficient, and interpretable map of \emph{where} the benchmark ecosystem violates local independence, and whether these violations are stable enough to be treated as structural (rather than idiosyncratic) properties of the item pool.
% \subsection{Bootstrap Details}\label{apx:boots}
% \subsubsection{Meta-analytic bootstrap aggregation}
% A second way of extracting results across bootstraps is using meta-analytic aggregation techniques. We compute the above item-wise fixed-effect metrics under bootstrap resampling of the observation table to quantify their stability. Let $b\in\{1,\dots,B\}$ index bootstrap samples, and let $Z_{ib}$ denote an aggregated item metric of interest (e.g., $R^2_i$, residual energy, or a composite) computed in bootstrap $b$. Because $Z_{ib}$ can be mechanically affected by design properties (how many models/items/observations appear in the bootstrap), we residualize these effects using:
% \[
% Z_{ib} = \kappa + \mathbf{x}_{ib}^\top \psi + w_b + \epsilon_{ib},
% \quad w_b\sim\mathcal{N}(0,\sigma^2_w),
% \]
% where $\mathbf{x}_{ib}$ includes (i) number of models, (ii) number of items, and (iii) total observations in bootstrap $b$ (and, when relevant, item-specific counts within $b$). The residualized item metric is
% \[
% \widetilde{Z}_{ib} = Z_{ib} - \widehat{\mathbf{x}}_{ib}^\top \hat\psi - \hat w_b,
% \]
% and we report $\widetilde{Z}_i=\frac{1}{B}\sum_{b=1}^B \widetilde{Z}_{ib}+\kappa $ as a design-adjusted measure of item idiosyncrasy. The readdition of the grand mean $\kappa $ has no effect on metric-based ordering, and serves only to recenter each metric. This step ensures that items are compared on an equal footing, separating substantive reliability differences from artifacts of missingness or bootstrap composition.
% \input{tables/oos_auc_mae_meta}
\section{Robustness via Meta-Analytic Bootstrapping}\label{apx:boots}
\subsection{Bootstrapped Methods 1 and 3 and Aggregations of Statistics}
\label{sec:method1_meta_cfa}
Fitting models to the thousands of available items is computationally intractable and risks overfitting to item-specific idiosyncrasies. To ensure our conclusions are stable and generalizable, we implement a meta-analytic bootstrapping procedure. For each of $B=400$ replications, we:
\begin{enumerate}[leftmargin=*,nosep]
\item Randomly sample a small, computationally manageable set of items from each of the 6 benchmarks.
\item Fit all six structural models to the tetrachoric correlation matrix of this item subset.
\item Store key model fit indices (e.g., scaled RMSEA, CFI), parameter estimates ($\boldsymbol{\Lambda}, \boldsymbol{\Psi}$), and local-fit diagnostics (modification indices).
\end{enumerate}
This process yields distributions of fit indices and parameters, allowing us to perform a meta-analysis. We assess the central tendency and stability of each model's performance, effectively integrating out the noise from any single item sample. To formally compare models, we fit a mixed-effects regression to the stacked results from all bootstrap replications, predicting fit indices from model structure while accounting for between-replication variance.
\paragraph{Bootstrap-by-item design (computational tractability and robustness)}
Let benchmarks be indexed by $k\in\{1,\dots,K\}$, with item sets $\mathcal{J}_k$ and $|\mathcal{J}_k|=p_k$. In bootstrap replication $b\in\{1,\dots,B\}$ we sample (without replacement) a subset $\tilde{\mathcal{J}}_{k}^{(b)}\subset\mathcal{J}_k$ of size $r_k\ll p_k$ for each benchmark and pool to $\tilde{\mathcal{J}}^{(b)}=\cup_k \tilde{\mathcal{J}}_{k}^{(b)}$ with $\tilde{p}=\sum_k r_k$. We then fit all candidate models to the same sampled item pool. This design targets a population quantity: the expected comparative fit of each structural hypothesis under the distribution of items induced by the benchmark ecosystem. It also provides a principled stability check: a structural conclusion is credible only if it persists across many item-resampled realizations.
\subsubsection{Motivation for Meta-aggregations}\label{apx:boots:motivation}
Each bootstrap replication $b \in \{1,\ldots,B\}$ samples a different item subset $\tilde{\mathcal{J}}^{(b)} = \cup_k \tilde{\mathcal{J}}_k^{(b)}$, with $|\tilde{\mathcal{J}}_k^{(b)}| = r_k \ll p_k$ items per benchmark. The MH-RM algorithm \citep{cai_high-dimensional_2010} then returns, for each estimand $\xi$ of interest, a point estimate $\hat{\xi}^{(b)}$ together with a standard error $\mathrm{SE}^{(b)}(\hat{\xi})$ derived from the observed information matrix at convergence. Two properties of this design make na\"ive averaging ($\bar{\xi} = B^{-1}\sum_b \hat{\xi}^{(b)}$) statistically inefficient:
\begin{enumerate}[leftmargin=*,itemsep=2pt]
\item \textbf{Heterogeneous precision.} Different item draws yield different effective test lengths, item discrimination profiles, and conditioning on the latent trait. A bootstrap that happens to sample highly discriminating items produces a more precise estimate of $\beta_d$ than one dominated by low-discrimination items. Treating both equally wastes information.
\item \textbf{Occasional near-degeneracy.} Some item subsets may produce near-singular Fisher information for particular parameters (e.g., a specific factor receiving very few high-discrimination items), inflating $\mathrm{SE}^{(b)}$ by orders of magnitude. Na\"ive averaging allows these unstable estimates to contaminate the aggregate.
\end{enumerate}
Inverse-variance weighting addresses both issues: it automatically downweights imprecise estimates in proportion to their squared standard error, yielding the minimum-variance linear unbiased combination under independence.
\subsection{Summary of the aggregation pipeline}\label{apx:boots:summary}
The complete pipeline, applied for Methods 1 and 3, proceeds as follows:
\begin{enumerate}[leftmargin=*,itemsep=2pt]
\item \textbf{Sample.} Draw item subset $\tilde{\mathcal{J}}^{(b)}$ with $r_k$ items per benchmark $k$, without replacement.
\item \textbf{Fit.} Estimate the target model (CFA structures for Method~1 via both MH-RM and DWLS; bifactor IRT with/without latent regression for Method~3 via MH-RM), obtaining estimates $\hat{\xi}^{(b)}$ and standard errors $\mathrm{SE}^{(b)}$.
\item \textbf{Weight and Aggregate via Mixed Effect Meta-regression (Method 1).} Obtain statistical estimates based on \eqref{eq:metaagg}.
\item \textbf{Weight (Method 3).} Compute $w^{(b)} = 1/(\mathrm{SE}^{(b)})^2$, applying fallback rules (\ref{eq:ivw_fallback}) for degenerate cases.
\item \textbf{Aggregate (Method 3).} Form IVW point estimates $\tilde{\xi}$ via (\ref{eq:ivw_mean}) and standard errors via (\ref{eq:ivw_se}).
\item \textbf{Diagnose.} Compute between-bootstrap heterogeneity $\hat{\tau}^2$ and, for scaling slopes, the reliability $\mathcal{R}_d$ via (\ref{eq:rel_from_ivw}).
\end{enumerate}
This design ensures that final estimates reflect the most informative item draws, propagate both within- and between-bootstrap uncertainty, and provide built-in diagnostics for the stability of every reported quantity.
\subsection{Meta-analytic aggregation of bootstrapped estimates via mixed-effects regression}\label{apx:meta_regress}
For more robustness when evaluating statistics and out-of-sample prediction for Method 1, we estimate these relationships using two different estimation techniques: the \textit{full information }stochastic \textit{expectation maximizing} Metropolis-Hastings Robbins-Monro (MH-RM) algorithm \citep{cai_metropolis-hastings_2010,chalmers_extended_2015} and the robust \textit{limited information} \textit{likelihood maximizing} Diagonal Weighted Least Squares \citep{rosseel_lavaan_2012} (See Appendix \ref{apx:code} for further estimation details). Each fitted model yields robust fit indices (we use scaled/robust variants because item responses are non-Gaussian). Let $t_{b,s,r}$ denote a fit statistic from replication $b$, structure $s$, and condition $r\in\{0,1\}$ (true vs.\ randomized assignment). We summarize across replications using a mixed-effects meta-analytic regression:
\begin{equation}\label{eq:metaagg}
t_{b,s,r} \;=\; \alpha_s + \beta_s\,r + u_b + v_b\,r + \varepsilon_{b,s,r},
\end{equation}
where $u_b$ captures bootstrap-to-bootstrap heterogeneity (item-sample difficulty) and $v_b$ allows the randomization penalty to vary by sampled item pool. % We weight observations by the proportion of successful convergences in each cell to avoid overcounting unstable fits.
This regression yields (i) an estimate $\alpha_s$ of the expected fit of structure $s$ under correct item assignment, (ii) an estimate $\beta_s$ of how much of that fit is explainable by structure rather than flexibility, and (iii) uncertainty that correctly propagates item-sampling variability.
\input{tables/oos_auc_mae_meta}
% ============================================================
% Appendix Subsection: Inverse-Error Meta-Analytic Aggregation
% ============================================================
\subsection{Inverse-error meta-analytic aggregation of bootstrapped estimates}\label{apx:boots_inverr}
This subsection describes the statistical machinery that converts the raw output of $B$ bootstrap replications---each producing point estimates with associated standard errors from the MH-RM algorithm---into the final aggregated quantities reported throughout this study. The procedure applies uniformly to fixed-effect coefficients (scaling slopes $\hat{\beta}_d^{(b)}$), latent ability scores (posterior BLUPs $\hat{\theta}_{id}^{(b)}$), and variance components ($\hat{\sigma}^{2\,(b)}_{S}$), though we develop it here primarily for the fixed effects and latent scores of Method~3.
\subsubsection{General framework for inverse error aggregation}\label{apx:boots:framework}
Let $\xi$ denote a generic scalar estimand (a single element of $\boldsymbol{\Gamma}$, a latent score $\theta_{id}$, or a variance component). For each bootstrap $b$, the MH-RM algorithm returns
% \[
$\hat{\xi}^{(b)} \quad \text{with} \quad \mathrm{SE}^{(b)} \equiv \mathrm{SE}\!\bigl(\hat{\xi}^{(b)}\bigr)$.
% \]
The posterior means of the latent trait BLUPs and the asymptotic standard errors from the observed information matrix are direct outputs of MH-RM's stochastic EM convergence diagnostics \citep{cai_metropolis-hastings_2010, chalmers_extended_2015}.
\paragraph{Inverse-variance weights.}
Define bootstrap-specific weights
\begin{equation}\label{eq:ivw_weights}
w^{(b)} \;=\; \frac{1}{\bigl(\mathrm{SE}^{(b)}\bigr)^2},
\end{equation}
so that $w^{(b)}$ is large when the estimate from replication $b$ is precise and small when it is noisy. In the sequel, we write $W = \sum_{b=1}^{B} w^{(b)}$ for the total weight.
\paragraph{Aggregated point estimate.}
The inverse-variance weighted (IVW) mean is
\begin{equation}\label{eq:ivw_mean}
\tilde{\xi} \;=\; \frac{\sum_{b=1}^{B} w^{(b)}\,\hat{\xi}^{(b)}}{\sum_{b=1}^{B} w^{(b)}} \;=\; \frac{1}{W}\sum_{b=1}^{B} w^{(b)}\,\hat{\xi}^{(b)}.
\end{equation}
\paragraph{Standard error of the aggregate.}
Under the assumption that bootstrap replications are conditionally independent given the data (which holds by construction, as each replication draws items independently), the variance of $\tilde{\xi}$ is
\begin{equation}\label{eq:ivw_se}
\mathrm{Var}(\tilde{\xi}) \;=\; \frac{1}{W} \;=\; \frac{1}{\sum_{b=1}^{B} w^{(b)}}, \qquad
\mathrm{SE}(\tilde{\xi}) \;=\; \frac{1}{\sqrt{W}}.
\end{equation}
\begin{proposition}[Optimality of inverse-variance weighting]\label{prop:ivw_optimal}
Among all linear combinations $\hat{\xi}_{\mathbf{a}} = \sum_b a_b\,\hat{\xi}^{(b)}$ with $\sum_b a_b = 1$, the inverse-variance weighted mean $\tilde{\xi}$ uniquely minimizes $\mathrm{Var}(\hat{\xi}_{\mathbf{a}})$ when $\hat{\xi}^{(1)},\ldots,\hat{\xi}^{(B)}$ are uncorrelated with known variances $(\mathrm{SE}^{(b)})^2$.
\end{proposition}
\begin{proof}
By the method of Lagrange multipliers, minimize $\sum_b a_b^2\,(\mathrm{SE}^{(b)})^2$ subject to $\sum_b a_b = 1$. The stationary conditions yield $a_b = w^{(b)}/W$, recovering (\ref{eq:ivw_mean}). The minimized variance is $1/W$.
\end{proof}
\subsubsection{Handling edge cases}\label{apx:boots:edge}
Two practical complications arise in our setting and must be addressed to ensure robustness.
\paragraph{Missing or degenerate standard errors.}
Occasionally, the MH-RM algorithm may fail to converge for a particular parameter in a given bootstrap, or may return $\mathrm{SE}^{(b)} = 0$ (exact boundary solution) or $\mathrm{SE}^{(b)} = \infty$ (non-identified parameter under that item draw). We handle these as follows:
\begin{equation}\label{eq:ivw_fallback}
w^{(b)} = \begin{cases}
1/\bigl(\mathrm{SE}^{(b)}\bigr)^2 & \text{if } 0 < \mathrm{SE}^{(b)} < \infty, \\[4pt]
\displaystyle\frac{1}{2}\min_{b':\,\mathrm{SE}^{(b')}\text{ valid}} w^{(b')} & \text{if } \mathrm{SE}^{(b)} \text{ is missing or degenerate}.
\end{cases}
\end{equation}
That is, degenerate replications receive half the weight of the least precise valid replication. This ensures they contribute minimally without being discarded entirely, preserving the unbiasedness of the mean while bounding their influence.
\paragraph{Infinite or zero weights.}
If $\mathrm{SE}^{(b)} = 0$ exactly (perfect information), we cap the weight at a large but finite value; if $\mathrm{SE}^{(b)}$ is undefined (e.g., non-convergence), the replication is assigned the fallback weight above. In our experiments, fewer than 0.5\% of replication--parameter pairs required fallback treatment.
\subsubsection{Application to fixed-effect coefficients}\label{apx:boots:fixed}
For the scaling slope on latent dimension $d \in \{0,1,\ldots,K\}$, the MH-RM algorithm returns in each bootstrap $b$ a coefficient $\hat{\beta}_d^{(b)} \equiv (\hat{\mathbf{B}}^{(b)})_{d,\,x}$ and its standard error $\mathrm{SE}^{(b)}_d$ from the observed information matrix. The aggregated scaling slope and its uncertainty are
\begin{equation}\label{eq:beta_agg}
\tilde{\beta}_d = \frac{\sum_b w_d^{(b)}\,\hat{\beta}_d^{(b)}}{\sum_b w_d^{(b)}}, \qquad
w_d^{(b)} = \frac{1}{\bigl(\mathrm{SE}_d^{(b)}\bigr)^2}, \qquad
\mathrm{SE}(\tilde{\beta}_d) = \frac{1}{\sqrt{\sum_b w_d^{(b)}}}.
\end{equation}
This procedure is applied independently for each latent dimension, yielding the aggregated scaling vector $\tilde{\boldsymbol{\beta}} = (\tilde{\beta}_0, \tilde{\beta}_1, \ldots, \tilde{\beta}_K)$ reported in Table~\ref{tab:lreg_reliab} and Figures~\ref{fig:scaling_law}--\ref{fig:scaling_law_deploy}.
\begin{remark}[Connection to fixed-effects meta-analysis]\label{rem:meta}
Equation~(\ref{eq:beta_agg}) is the standard fixed-effect meta-analytic estimator. In our context, the ``studies'' are bootstrap replications rather than independent experiments, but the statistical logic is identical: each replication provides an estimate of the \emph{same} population quantity $\beta_d$ (the scaling slope under the item distribution induced by the benchmark ecosystem), and precision varies across replications due to the random item draw. The fixed-effect assumption---that all replications target the same $\beta_d$---is justified because item subsets are drawn from the same benchmark population and models are held fixed across replications.
\end{remark}
\subsubsection{Application to latent ability scores}\label{apx:boots:scores}
Each MH-RM fit produces posterior means (BLUPs) $\hat{\theta}_{id}^{(b)}$ and associated posterior standard deviations $\mathrm{SE}^{(b)}(\hat{\theta}_{id})$ for every model $i$ and latent dimension $d$. The aggregated latent score for model $i$ on dimension $d$ is
\begin{equation}\label{eq:score_agg}
\tilde{\theta}_{id} = \frac{\sum_b w_{id}^{(b)}\,\hat{\theta}_{id}^{(b)}}{\sum_b w_{id}^{(b)}}, \qquad w_{id}^{(b)} = \frac{1}{\bigl(\mathrm{SE}^{(b)}(\hat{\theta}_{id})\bigr)^2}.
\end{equation}
Note that the weights here are \emph{model-and-dimension-specific}: a model that responds to many high-discrimination items in bootstrap $b$ will have a more precisely estimated $\hat{\theta}_{id}^{(b)}$ (smaller posterior SD) and thus receive higher weight for that replication.
\subsubsection{Percentile ranks from aggregated scores.}\label{apx:rank_agg}
% To estimate general factor induced rankings based on the hierarchical latent mixed regression, for each bootstrap, we take 1,000 draws from the posterior and estimate the predicted distribution of the combined fixed and Best Linear Unbiased Prediction (BLUP) distributions for random ability effects for the general factor $g$ for each LLM.
The leaderboard comparisons in \S \ref{sec:rank_changes} use percentile ranks derived from the aggregated general-factor scores $\{\tilde{\theta}_{i0}\}_{i=1}^N$ and from the aggregated latent-regression-adjusted scores. Because both are IVW aggregates, they reflect the full bootstrap distribution of item sampling uncertainty while upweighting the most informative item draws. Percent ranking counts the total number of values less than $\theta_{gi}$, and divides it by the number of observations minus 1: $\frac{1}{n-1}\sum_{x} \mathbb{1}[x< \theta_{gi}]$.
\subsubsection{Decomposing bootstrap variability: within versus between}\label{apx:boots:decomp}
The total variability of $\hat{\xi}^{(b)}$ across bootstraps reflects two distinct sources: \textbf{Within-bootstrap estimation uncertainty} $(\mathrm{SE}^{(b)})^2$: the posterior or asymptotic variance of $\hat{\xi}^{(b)}$ conditional on item set $\tilde{\mathcal{J}}^{(b)}$. This is what the MH-RM information matrix measures. \textbf{Between-bootstrap item-sampling variability} $\tau^2_\xi$: the variance of the true population estimand across item draws, reflecting the sensitivity of the conclusion to the particular items selected. The observed variance of the estimates across bootstraps conflates both:
\begin{equation}\label{eq:total_var_decomp}
\mathrm{Var}_b\bigl(\hat{\xi}^{(b)}\bigr) \;=\; \tau^2_\xi \;+\; \mathbb{E}_b\bigl[(\mathrm{SE}^{(b)})^2\bigr].
\end{equation}
The first term measures genuine item-sampling heterogeneity (how much the estimand depends on which items are drawn); the second measures average estimation noise. Inverse-variance weighting eliminates the second source from the aggregated point estimate, but both contribute to the uncertainty around any single bootstrap's result.
\begin{proposition}[IVW aggregation removes within-bootstrap noise from the point estimate]\label{prop:ivw_noise}
Under the decomposition in (\ref{eq:total_var_decomp}), the variance of the IVW aggregate $\tilde{\xi}$ satisfies
% \[
$\mathrm{Var}(\tilde{\xi}) \;\leq\; \frac{1}{B}\,\mathrm{Var}_b\bigl(\hat{\xi}^{(b)}\bigr)$,
% \]
with equality only when all weights are identical. When weights vary, the IVW estimate achieves strictly lower variance than the unweighted mean, with the improvement proportional to the heterogeneity of $\{(\mathrm{SE}^{(b)})^2\}_b$.
\end{proposition}
\begin{proof}
By Proposition~\ref{prop:ivw_optimal}, $\mathrm{Var}(\tilde{\xi}) = 1/W$. Under identical weights $w^{(b)} = \bar{w}$, we have $W = B\bar{w}$ and $\mathrm{Var}(\tilde{\xi}) = 1/(B\bar{w}) = \mathrm{Var}_b(\hat{\xi}^{(b)})/B$. For heterogeneous weights, the Cauchy-Schwarz inequality applied to $\sum_b a_b^2 (\mathrm{SE}^{(b)})^2$ with optimal $a_b = w^{(b)}/W$ yields strict improvement whenever the $(\mathrm{SE}^{(b)})^2$ are not all equal.
\end{proof}
\subsubsection{Slope reliability from the IVW framework}\label{apx:boots:reliability}
A natural by-product of the IVW aggregation is the \emph{slope reliability index} for each latent dimension $d$. From the bootstrapped estimates $\{\hat{\beta}_d^{(b)}\}_b$, we compute the weighted between-bootstrap variance
\begin{equation}\label{eq:tau_boots}
\hat{\tau}_d^2 \;=\; \frac{\sum_b w_d^{(b)}\bigl(\hat{\beta}_d^{(b)} - \tilde{\beta}_d\bigr)^2}{\sum_b w_d^{(b)}} \;-\; \frac{1}{\bar{w}_d},
\end{equation}
where $\bar{w}_d = W_d / B$ is the average weight, and the subtracted term is a bias correction removing the expected contribution of within-bootstrap noise (analogous to the DerSimonian-Laird estimator in meta-analysis). The slope reliability index is then
\begin{equation}\label{eq:rel_from_ivw}
\mathcal{R}_d \;=\; \frac{\tilde{\beta}_d^2}{\tilde{\beta}_d^2 + \hat{\tau}_d^2}.
\end{equation}
High $\mathcal{R}_d$ (close to 1) indicates that the scaling slope for latent dimension $d$ is stable across item subsets and contributor clusters: the estimated law is a property of the ecosystem, not of a particular item draw. Low $\mathcal{R}_d$ signals that the slope is sensitive to item composition, and any single estimate should be interpreted cautiously. Table~\ref{tab:lreg_reliab} reports $\mathcal{R}_d$ for each latent dimension.
% \subsection{Method 3}
% \subsubsection{Meta-analytic bootstrap aggregation}
% A second way of extracting results across bootstraps is using meta-analytic aggregation techniques. We compute the above item-wise fixed-effect metrics under bootstrap resampling of the observation table to quantify their stability. Let $b\in\{1,\dots,B\}$ index bootstrap samples, and let $Z_{ib}$ denote an aggregated item metric of interest (e.g., $R^2_i$, residual energy, or a composite) computed in bootstrap $b$. Because $Z_{ib}$ can be mechanically affected by design properties (how many models/items/observations appear in the bootstrap), we residualize these effects using:
% % \[
% $Z_{ib} = \kappa + \mathbf{x}_{ib}^\top \psi + w_b + \epsilon_{ib},
% \; w_b\sim\mathcal{N}(0,\sigma^2_w)$,
% % \]
% where $\mathbf{x}_{ib}$ includes (i) number of models, (ii) number of items, and (iii) total observations in bootstrap $b$ (and, when relevant, item-specific counts within $b$). The residualized item metric is
% \[
% \widetilde{Z}_{ib} = Z_{ib} - \widehat{\mathbf{x}}_{ib}^\top \hat\psi - \hat w_b,
% \]
% and we report $\widetilde{Z}_i=\frac{1}{B}\sum_{b=1}^B \widetilde{Z}_{ib}+\kappa $ as a design-adjusted measure of item idiosyncrasy. The readdition of the grand mean $\kappa $ has no effect on metric-based ordering, and serves only to recenter each metric. This step ensures that items are compared on an equal footing, separating substantive reliability differences from artifacts of missingness or bootstrap composition.
% \subsection{Posterior Rank Aggregation}\label{apx:rank_agg}
% To estimate general factor induced rankings based on the hierarchical latent mixed regression, for each bootstrap, we take 1,000 draws from the posterior and estimate the predicted distribution of the combined fixed and Best Linear Unbiased Prediction (BLUP) distributions for random ability effects for the general factor $g$ for each LLM. We estimate means and standard deviations from the distributions of the drawn predictions. The inverse squares of these bootstrap-wise posterior distribution standard deviations are used for the LLM-wise inverse variance weighting across bootstraps. $\boldsymbol{\widehat{\theta}_g}$ is estimated from the precision-weighted mean. The percent ranks of these values are compared with the percent ranks of percent ranks of the original leaderboard score in \ref{sec:rank_changes}. Percent ranking counts the total number of values less than $\theta_{gi}$, and divides it by the number of observations minus 1: $\frac{1}{n-1}\sum_{x \in \boldsymbol{\theta_{g}}< \theta_{gi}}x$.
\begin{table*}
\centering
\caption{Estimated Overall Predictive Effects by latent Structure based on meta-analytic regression}
\label{tab:oos_agg_mae_auc}
\begin{talltblr}[ %% tabularray outer open
entry=none,label=none,
note{}={+ p \num{< 0.1}, * p \num{< 0.05}, ** p \num{< 0.01}, *** p \num{< 0.001}},
] %% tabularray outer close
{ %% tabularray inner open
colspec={Q[]Q[]Q[]Q[]Q[]Q[]Q[]Q[]Q[]},
cell{2}{2}={}{halign=c,},
cell{2}{3}={}{halign=c,},
cell{2}{4}={}{halign=c,},
cell{2}{5}={}{halign=c,},
cell{2}{6}={}{halign=c,},
cell{2}{7}={}{halign=c,},
cell{2}{8}={}{halign=c,},
cell{2}{9}={}{halign=c,},
cell{3}{2}={}{halign=c,},
cell{3}{3}={}{halign=c,},
cell{3}{4}={}{halign=c,},
cell{3}{5}={}{halign=c,},
cell{3}{6}={}{halign=c,},
cell{3}{7}={}{halign=c,},
cell{3}{8}={}{halign=c,},
cell{3}{9}={}{halign=c,},
cell{4}{2}={}{halign=c,},
cell{4}{3}={}{halign=c,},
cell{4}{4}={}{halign=c,},
cell{4}{5}={}{halign=c,},
cell{4}{6}={}{halign=c,},
cell{4}{7}={}{halign=c,},
cell{4}{8}={}{halign=c,},
cell{4}{9}={}{halign=c,},
cell{5}{2}={}{halign=c,},
cell{5}{3}={}{halign=c,},
cell{5}{4}={}{halign=c,},
cell{5}{5}={}{halign=c,},
cell{5}{6}={}{halign=c,},
cell{5}{7}={}{halign=c,},
cell{5}{8}={}{halign=c,},
cell{5}{9}={}{halign=c,},
cell{6}{2}={}{halign=c,},
cell{6}{3}={}{halign=c,},
cell{6}{4}={}{halign=c,},
cell{6}{5}={}{halign=c,},
cell{6}{6}={}{halign=c,},
cell{6}{7}={}{halign=c,},
cell{6}{8}={}{halign=c,},
cell{6}{9}={}{halign=c,},
cell{7}{2}={}{halign=c,},
cell{7}{3}={}{halign=c,},
cell{7}{4}={}{halign=c,},
cell{7}{5}={}{halign=c,},
cell{7}{6}={}{halign=c,},
cell{7}{7}={}{halign=c,},
cell{7}{8}={}{halign=c,},
cell{7}{9}={}{halign=c,},
cell{8}{2}={}{halign=c,},
cell{8}{3}={}{halign=c,},
cell{8}{4}={}{halign=c,},
cell{8}{5}={}{halign=c,},
cell{8}{6}={}{halign=c,},
cell{8}{7}={}{halign=c,},
cell{8}{8}={}{halign=c,},
cell{8}{9}={}{halign=c,},
cell{9}{2}={}{halign=c,},
cell{9}{3}={}{halign=c,},
cell{9}{4}={}{halign=c,},
cell{9}{5}={}{halign=c,},
cell{9}{6}={}{halign=c,},
cell{9}{7}={}{halign=c,},
cell{9}{8}={}{halign=c,},
cell{9}{9}={}{halign=c,},
cell{10}{2}={}{halign=c,},
cell{10}{3}={}{halign=c,},
cell{10}{4}={}{halign=c,},
cell{10}{5}={}{halign=c,},
cell{10}{6}={}{halign=c,},
cell{10}{7}={}{halign=c,},
cell{10}{8}={}{halign=c,},
cell{10}{9}={}{halign=c,},
cell{11}{2}={}{halign=c,},
cell{11}{3}={}{halign=c,},
cell{11}{4}={}{halign=c,},
cell{11}{5}={}{halign=c,},
cell{11}{6}={}{halign=c,},
cell{11}{7}={}{halign=c,},
cell{11}{8}={}{halign=c,},
cell{11}{9}={}{halign=c,},
cell{12}{2}={}{halign=c,},
cell{12}{3}={}{halign=c,},
cell{12}{4}={}{halign=c,},
cell{12}{5}={}{halign=c,},
cell{12}{6}={}{halign=c,},
cell{12}{7}={}{halign=c,},
cell{12}{8}={}{halign=c,},
cell{12}{9}={}{halign=c,},
cell{13}{2}={}{halign=c,},
cell{13}{3}={}{halign=c,},
cell{13}{4}={}{halign=c,},
cell{13}{5}={}{halign=c,},
cell{13}{6}={}{halign=c,},
cell{13}{7}={}{halign=c,},
cell{13}{8}={}{halign=c,},
cell{13}{9}={}{halign=c,},
cell{14}{2}={}{halign=c,},
cell{14}{3}={}{halign=c,},
cell{14}{4}={}{halign=c,},
cell{14}{5}={}{halign=c,},
cell{14}{6}={}{halign=c,},
cell{14}{7}={}{halign=c,},
cell{14}{8}={}{halign=c,},
cell{14}{9}={}{halign=c,},
cell{1}{3}={}{halign=c, halign=c,},
cell{1}{5}={}{halign=c, halign=c,},
cell{1}{7}={}{halign=c, halign=c,},
cell{1}{9}={}{halign=c, halign=c,},
cell{2}{1}={}{halign=l,},
cell{3}{1}={}{halign=l,},
cell{4}{1}={}{halign=l,},
cell{5}{1}={}{halign=l,},
cell{6}{1}={}{halign=l,},
cell{7}{1}={}{halign=l,},
cell{8}{1}={}{halign=l,},
cell{9}{1}={}{halign=l,},
cell{10}{1}={}{halign=l,},
cell{11}{1}={}{halign=l,},
cell{12}{1}={}{halign=l,},
cell{13}{1}={}{halign=l,},
cell{14}{1}={}{halign=l,},
cell{1}{1}={}{halign=l, halign=c,},
cell{1}{2}={c=2,}{halign=c, halign=c, halign=c,},
cell{1}{4}={c=2,}{halign=c, halign=c, halign=c,},
cell{1}{6}={c=2,}{halign=c, halign=c, halign=c,},
cell{1}{8}={c=2,}{halign=c, halign=c, halign=c,},
hline{12}={1,2,3,4,5,6,7,8,9}{solid, black, 0.05em},
} %% tabularray inner close
\toprule
& (AUC) & & (AUC) & & (MAE) & & (MAE) & \\ \cmidrule[lr]{2-3}\cmidrule[lr]{4-5}\cmidrule[lr]{6-7}\cmidrule[lr]{8-9}
& Est. & S.E. & Est. & S.E. & Est. & S.E. & Est. & S.E. \\ \midrule %% TinyTableHeader
bifact Fig. \ref{fig:cfa_diagram}.e, \S \ref{apx:bifact} & \textbf{0.905}*** & 0.010 & \textbf{0.954}*** & 0.011 & \textbf{0.244}*** & 0.007 & \textbf{0.208}*** & 0.007 \\
corrbifact Fig. \ref{fig:cfa_diagram}.f, \S \ref{apx:corrbifact} & 0.718*** & 0.008 & 0.771*** & 0.009 & 0.357*** & 0.005 & 0.321*** & 0.006 \\
corrfact Fig. \ref{fig:cfa_diagram}.d, \S \ref{apx:corrfact} & 0.677*** & 0.008 & 0.731*** & 0.009 & 0.385*** & 0.005 & 0.349*** & 0.006 \\
gfact Fig. \ref{fig:cfa_diagram}.b, \S \ref{apx:gfact} & 0.868*** & 0.006 & 0.915*** & 0.008 & 0.279*** & 0.004 & 0.248*** & 0.005 \\
hier2ordFig. \ref{fig:cfa_diagram}.c, \S \ref{apx:hier2ord} & 0.885*** & 0.010 & 0.938*** & 0.011 & 0.259*** & 0.006 & 0.223*** & 0.007 \\
indepfact Fig. \ref{fig:cfa_diagram}.a, \S \ref{apx:indepfact} & 0.900*** & 0.008 & 0.953*** & 0.010 & 0.250*** & 0.005 & 0.215*** & 0.006 \\
SD (Intercept bootstrap) & 0.039 & & 0.000 & & 0.027 & & 0.000 & \\
SD (Observations) & 0.103 & & 0.102 & & 0.065 & & 0.066 & \\
SD (num. items in estimation) & & & 0.006 & & & & 0.004 & \\
Num.Obs. & 1082 & & 1082 & & 1082 & & 1082 & \\
R2 Marg. & 0.406 & & 0.440 & & 0.368 & & 0.402 & \\
R2 Cond. & 0.482 & & & & 0.462 & & & \\
\bottomrule
\end{talltblr}
\end{table*}
% \begin{table}\label{tab:oos_agg_mae_auc}
% \centering
% \caption{Estimated Overall Predictive Effects by latent Structure}
% \begin{talltblr}[ %% tabularray outer open
% entry=none,label=none,
% note{}={+ p \num{< 0.1}, * p \num{< 0.05}, ** p \num{< 0.01}, *** p \num{< 0.001}},
% ] %% tabularray outer close
% { %% tabularray inner open
% colspec={Q[]Q[]Q[]Q[]Q[]},
% cell{2}{2}={}{halign=c,},
% cell{2}{3}={}{halign=c,},
% cell{2}{4}={}{halign=c,},
% cell{2}{5}={}{halign=c,},
% cell{3}{2}={}{halign=c,},
% cell{3}{3}={}{halign=c,},
% cell{3}{4}={}{halign=c,},
% cell{3}{5}={}{halign=c,},
% cell{4}{2}={}{halign=c,},
% cell{4}{3}={}{halign=c,},
% cell{4}{4}={}{halign=c,},
% cell{4}{5}={}{halign=c,},
% cell{5}{2}={}{halign=c,},
% cell{5}{3}={}{halign=c,},
% cell{5}{4}={}{halign=c,},
% cell{5}{5}={}{halign=c,},
% cell{6}{2}={}{halign=c,},
% cell{6}{3}={}{halign=c,},
% cell{6}{4}={}{halign=c,},
% cell{6}{5}={}{halign=c,},
% cell{7}{2}={}{halign=c,},
% cell{7}{3}={}{halign=c,},
% cell{7}{4}={}{halign=c,},
% cell{7}{5}={}{halign=c,},
% cell{8}{2}={}{halign=c,},
% cell{8}{3}={}{halign=c,},
% cell{8}{4}={}{halign=c,},
% cell{8}{5}={}{halign=c,},
% cell{9}{2}={}{halign=c,},
% cell{9}{3}={}{halign=c,},
% cell{9}{4}={}{halign=c,},
% cell{9}{5}={}{halign=c,},
% cell{10}{2}={}{halign=c,},
% cell{10}{3}={}{halign=c,},
% cell{10}{4}={}{halign=c,},
% cell{10}{5}={}{halign=c,},
% cell{11}{2}={}{halign=c,},
% cell{11}{3}={}{halign=c,},
% cell{11}{4}={}{halign=c,},
% cell{11}{5}={}{halign=c,},
% cell{12}{2}={}{halign=c,},
% cell{12}{3}={}{halign=c,},
% cell{12}{4}={}{halign=c,},
% cell{12}{5}={}{halign=c,},
% cell{1}{3}={}{halign=c, halign=c,},
% cell{1}{5}={}{halign=c, halign=c,},
% cell{2}{1}={}{halign=l,},
% cell{3}{1}={}{halign=l,},
% cell{4}{1}={}{halign=l,},
% cell{5}{1}={}{halign=l,},
% cell{6}{1}={}{halign=l,},
% cell{7}{1}={}{halign=l,},
% cell{8}{1}={}{halign=l,},
% cell{9}{1}={}{halign=l,},
% cell{10}{1}={}{halign=l,},
% cell{11}{1}={}{halign=l,},
% cell{12}{1}={}{halign=l,},
% cell{1}{1}={}{halign=l, halign=c,},
% cell{1}{2}={c=2,}{halign=c, halign=c, halign=c,},
% cell{1}{4}={c=2,}{halign=c, halign=c, halign=c,},
% hline{12}={1,2,3,4,5}{solid, black, 0.05em},
% } %% tabularray inner close
% \toprule
% & (1) & & (2) & \\ \cmidrule[lr]{2-3}\cmidrule[lr]{4-5}
% & Est. & S.E. & Est. & S.E. \\ \midrule %% TinyTableHeader
% indepfact Fig. \ref{fig:cfa_diagram}.a & 0.959*** & 0.010 & 0.210*** & 0.006 \\
% gfact Fig. \ref{fig:cfa_diagram}.b & 0.915*** & 0.008 & 0.248*** & 0.005 \\
% hier2ord Fig. \ref{fig:cfa_diagram}.c & 0.938*** & 0.011 & 0.223*** & 0.007 \\
% corrfact Fig. \ref{fig:cfa_diagram}.d & 0.731*** & 0.009 & 0.349*** & 0.006 \\
% bifact Fig. \ref{fig:cfa_diagram}.e & 0.950*** & 0.011 & 0.208*** & 0.007 \\
% corrbifact Fig. \ref{fig:cfa_diagram}.f & 0.771*** & 0.009 & 0.321*** & 0.006 \\
% SD (Intercept batchiteration) & 0.000 & & 0.000 & \\
% SD (n\_items estimation) & 0.006 & & 0.004 & \\
% SD (Observations) & 0.102 & & 0.066 & \\
% R2 Marg. & 0.440 & & 0.402 & \\
% \bottomrule
% \end{talltblr}
% \end{table}
% \begin{table}
% \centering
% \begin{talltblr}[ %% tabularray outer open
% entry=none,label=none,
% note{}={+ p \num{< 0.1}, * p \num{< 0.05}, ** p \num{< 0.01}, *** p \num{< 0.001}},
% ] %% tabularray outer close
% { %% tabularray inner open
% colspec={Q[]Q[]Q[]},
% column{2,3}={}{halign=c,},
% column{1}={}{halign=l,},
% hline{17}={1,2,3}{solid, black, 0.05em},
% } %% tabularray inner close
% \toprule
% & (1) & (2) \\ \midrule %% TinyTableHeader
% modelbifact & 0.950*** & 0.208*** \\
% & (0.011) & (0.007) \\
% modelcorrbifact & 0.771*** & 0.321*** \\
% & (0.009) & (0.006) \\
% modelcorrfact & 0.731*** & 0.349*** \\
% & (0.009) & (0.006) \\
% modelgfact & 0.915*** & 0.248*** \\
% & (0.008) & (0.005) \\
% modelhier2ord & 0.938*** & 0.223*** \\
% & (0.011) & (0.007) \\
% modelindepfact & 0.959*** & 0.210*** \\
% & (0.010) & (0.006) \\
% SD (Intercept batchiteration) & 0.000 & 0.000 \\
% SD (n\_items estimation) & 0.006 & 0.004 \\
% SD (Observations) & 0.102 & 0.066 \\
% R2 Marg. & 0.440 & 0.402 \\
% \bottomrule
% \end{talltblr}
% \end{table}
\section{Method 1 Fit Statistics}\label{apx:cfa_fit}
\input{appendix/bifactor_stat_tests}
\begin{figure}
\centering
\includegraphics[width=1\linewidth]{figures/fit_stats_cfa_structures.pdf}
\caption{Sensitivity of Fit Estimates to Bootstrap sample size, meta-aggregated across both estimation methods.}
\label{fig:itemsperboot}
\end{figure}
\paragraph{Localized Diagnostics with Modification Indices}
Global fit indices (e.g., CFI, RMSEA) assess overall model-data congruence but do not identify specific sources of misfit or redundancy. For this, we use modification indices (MIs). An MI estimates the expected decrease in the model's $\chi^2$ statistic if a single fixed parameter (e.g., a cross-loading or a residual covariance fixed at zero) were freely estimated \citep{saris_testing_2009}. This provides a powerful, localized diagnostic for detecting systematic measurement noise that could inflates apparent reliability without increasing construct coverage. More importantly, in the bootstrap context, these serve as generalized indicators of where benchmarks are over and underfitting their respective constructs: this is the extent to which items are measuring each other rather than the latent constructs intended by their respective benchmark.
\begin{proposition}[Modification Indices as Score Tests]
Let $L(\boldsymbol{\theta})$ be the log-likelihood function for the model parameters $\boldsymbol{\theta}$. Consider a model with a constraint $c(\boldsymbol{\theta})=0$. The modification index for relaxing this constraint is the score statistic (or Lagrange Multiplier test) for the null hypothesis $H_0: c(\boldsymbol{\theta})=0$. It is calculated as $s(\hat{\boldsymbol{\theta}}_r)^T I(\hat{\boldsymbol{\theta}}_r)^{-1} s(\hat{\boldsymbol{\theta}}_r)$, where $\hat{\boldsymbol{\theta}}_r$ is the maximum likelihood estimate under the constraint, $s(\cdot)$ is the score vector ($\partial L / \partial c$), and $I(\cdot)$ is the Fisher information matrix for the parameter $c$.
\end{proposition}
% Table of Likelihood ratio test significance
\input{tables/lrt_mirt}
\begin{figure}[H]
\centering
\includegraphics[width=0.5\linewidth]{figures/benchmark_relation_misfit.pdf}
\caption{Estimated mean misfit of relationships between benchmark constructs by latent structure using Langrangian Multiplier test standardized expected parameter changes over bootstraps. Higher SEPC values indicate the model underestimates the inter-benchmark relationships, and lower values indicate overestimation. Labels above each plot indicate which two benchmarks and stars indicate levels of significance: ‘***’$< 0.001$; ‘**’$< 0.01$; ‘*’$< 0.05$. }
\label{fig:bench_rel_misfit}
\end{figure}
\begin{figure*}[h!]
\centering
\includegraphics[width=1\linewidth]{figures/mi_test_item_res_cor.pdf}
\caption{Estimated mean misfit of relationships between benchmark constructs by latent structure using Langrangian Multiplier test standardized expected parameter changes after freeing interitem residual variances over bootstraps. Higher SEPC values indicate the hypothesized latent structure underestimates the inter-benchmark relationships, and lower values indicate overestimation. Labels above each plot indicate which two benchmarks and stars indicate levels of significance: ‘***’$< 0.001$; ‘**’$< 0.01$; ‘*’$< 0.05.$}
\label{fig:bench_item_misfit}
\end{figure*}
\begin{figure*}[th]
\centering
\includegraphics[width=1\linewidth]{figures/r2_norm_change_vs_over_under_estimation.pdf}
\caption{Average Effect Size Changes to models based on single-degree Lagrangian Multiplier tests of residual variance. X axis represents the amount of variance explained implied by the score tests; more positive values indicate that the test is capturing more variation. Y axis represents the actual effect sizes; more positive and more negative values mean that the latent structure is underestimating and overestimating, respectively, the relationships. The columns and rows indicate from which benchmarks the residual variance is being freed. }
\label{fig:mod_indices_residual_error}
\end{figure*}
\subsection{Permutation Controls}
Additionally, the traditional fit results from the permutation controls from the DWLS estimation are found in Table \ref{tab:trad_fit_perm}.
\input{tables/permutation_tests}
% \subsection{Results and Discussion: A Dominant General Factor with Specific Abilities}
% Our meta-analytic comparison of latent structures reveals a clear and consistent pattern across hundreds of random item samples from the AI benchmark ecosystem. The results strongly support a bifactor structure, providing crucial insights into the nature of LLM abilities and the limitations of current evaluation practices.
% \subsection{Selected Model: Bifactor Model Provides the Most Parsimonious and Stable Fit}
% Across the 200 bootstrap replications, the bifactor-type models consistently and substantially outperformed all alternatives. A meta-analytic mixed-effects model predicting the scaled Root Mean Square Error of Approximation (RMSEA; lower is better) from model structure confirms this. The \textbf{Correlated Bifactor} model achieved the lowest mean RMSEA (0.043), followed closely by the orthogonal \textbf{Bifactor} model (0.049). Both were statistically superior to the Correlated Factors (0.054), Hierarchical (0.054), General Factor (0.059), and Independent Factors (0.090) models ($p < .001$ for all comparisons).
% While the Correlated Bifactor model provided the numerically best fit, its superiority over the standard Bifactor model was fragile. Permutation tests, where item-to-benchmark assignments were shuffled, revealed that the Correlated Bifactor's complex structure could capitalize on chance correlations, often fitting the randomized data nearly as well as the true data. The orthogonal Bifactor model, in contrast, showed a much larger drop in fit when the item structure was permuted, indicating its fit is more dependent on the genuine, theory-driven grouping of items by benchmark. Given this finding, and for reasons of theoretical parsimony and interpretability, we select the orthogonal Bifactor model as the most robust and meaningful representation of the ecosystem's latent structure.
% \subsection{Interpretation: General Ability is Primary, but Specific Skills are Distinguishable}
% The success of the Bifactor model has profound implications. It posits that an LLM's response to any given benchmark item $y_{ikj}$ is simultaneously influenced by two orthogonal sources of variance:
% % \begin{equation}
% $y_{ikj} = \lambda^{(g)}_{kj} g_i + \lambda^{(s)}_{kj} s_{ik} + \epsilon_{ikj}$
% % \end{equation}
% where $g_i$ is the LLM's score on a \textit{general factor}, $s_{ik}$ is its score on a \textit{specific factor} for benchmark $k$, and the factors are uncorrelated ($\text{Cov}(g_i, s_{ik}) = 0$).
% \begin{itemize}[leftmargin=*,itemsep=0.2em]
% \item \textbf{The General Factor ($g$):} This factor captures a broad capability that is relevant across all benchmarks, regardless of their ostensible content. It is the statistical embodiment of the strong first component observed in PCA but is now modeled as a latent construct. We hypothesize this $g$ factor represents core, foundational LLM capabilities, such as the ability to process natural language instructions, retrieve information from the pre-training corpus, and perform coherent autoregressive generation. Its dominance in explaining variance suggests that a large portion of what current benchmarks measure is a shared, non-specific competence.
% \item \textbf{The Specific Factors ($s_k$):} These factors capture the unique variance shared by items within a particular benchmark \textit{after} accounting for the general factor. They represent the specialized skills required for a given domain, such as `math-specific reasoning` ($s_{\text{OpenLLMMath}}$) or `instruction-following logic` ($s_{\text{IF-eval}}$), that are distinct from the general ability. The fact that these factors are necessary for good model fit proves that benchmarks are not entirely redundant; they do measure unique, separable skills, albeit to a lesser degree than the general factor.
% \end{itemize}
% \subsection{Implications for AI Evaluation}
% The validation of a bifactor structure challenges the foundations of current evaluation practice and offers a clear path forward.
% \paragraph{Rethinking Scoring and Ranking.} The ubiquitous single summary score is fundamentally misspecified. It conflates the general ability $g$ with the specific ability $s_k$, rendering it uninterpretable. A model might achieve a high score on MMLU through a very high $g$ and a moderate $s_{\text{MMLU}}$, while another achieves the same score with a moderate $g$ and a high $s_{\text{MMLU}}$. These models are not equivalent. A more valid approach would be to report at least two scores for each model: an estimate of its general factor score ($g$), and a profile of its specific ability scores ($s_k$). This would allow for a much more nuanced diagnosis of model strengths and weaknesses.
% \paragraph{A Principled Justification for the Bifactor Model in AI.} While the bifactor model is controversial in human psychology due to the difficulty of reifying a general factor ($g$) that is biologically and cognitively orthogonal to specific abilities, its application to LLMs is arguably on much stronger theoretical ground. Unlike the human brain, LLM architectures are known, and their ``experience'' is a direct function of their training data and optimization process. It is plausible that a core set of capabilities derived from internet-scale pre-training could function as a separable general factor, upon which more specialized skills from instruction tuning or reinforcement learning are built.
% \paragraph{Towards More Efficient and Targeted Benchmarks.} Our analysis provides a toolkit for improving the benchmark ecosystem. By examining factor loadings, we can identify which items are the strongest measures of the general factor versus the specific factors. The analysis of modification indices and SEPCs, as discussed in our method, allows for the data-driven identification of redundant items that share variance beyond what is explained by the common factors, flagging them for removal. This points toward a future of shorter, more targeted, and more powerful benchmarks that can efficiently disentangle general competence from specific, high-value skills.
\begin{table*}
\centering
\caption{Likelihood Ratio Tests of Statistical Significance across Bootstraps}
\label{tab:lrt_boots}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lccccc}
\toprule
\multicolumn{1}{c}{Simpler Structure $\rightarrow$} & \multicolumn{5}{c}{Proportion Significant LRT ($\alpha = 0.05$)} \\
\cmidrule(l{3pt}r{3pt}){2-6}
% \makecell{Complex Structure $\downarrow$} & General & Independent & Correlated & Bifactor & Hierarchical \\
% \midrule
% Independent Factors & 0.42 & -- & --& --& --\\
% General Factor & -- & 0.58 & --& --& --\\
% Hierarchical Structure & 1.00 & 1.00 & --& --& 0.00\\
% Correlated Factors & 1.00 & 1.00 & --& --& 0.91\\
% Bifactor & 1.00 & 1.00 & 1.00& --& 1.00\\
% Correlated Bifactor & 1.00 & 1.00 & 1.00& 1.00& 1.00\\
\makecell{Complex Structure $\downarrow$} & Independent & General & Hierarchical & Correlated & Bifactor\\
\midrule
General Factor & $0.58^\dagger$ & -- & -- & -- & --\\
Independent Factors & -- & $0.42^\dagger$ & -- & -- & --\\
Hierarchical Structure & 1.00 & $1.00^\dagger$ & -- & -- & --\\
Correlated Factors & 1.00 & $1.00^\dagger$ & $0.91^\dagger$ & -- & --\\
Bifactor & 1.00 & 1.00 & $1.00^\dagger$ & $1.00^\dagger$ & -- \\
Correlated Bifactor & 1.00 & 1.00 & $1.00^\dagger$ & 1.00 & 1.00 \\
\bottomrule
\end{tabular}}
\caption*{\footnotesize Values represent the proportion of total bootstraps where likelihood ratio tests indicated that the more complex model was statistically significantly a better fit. $\dagger$ indicates pseudo-LRT for non-nested models using evidence ratio and combined information criteria indices.}
\end{table*}
% \begin{table}
% \centering
% \caption{Traditional Fit Statistics under Meta-analytic bootstrap aggregation}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{llllll}
% \toprule
% term & statistic & RMSEA fit & CFI fit & TLI fit & SRMR fit\\
% \midrule
% structbifact & estimate & 0.049*** & 0.843*** & 0.829*** & 0.113***\\
% structbifact & std.error & (0.001) & (0.008) & (0.008) & (0.002)\\
% structcorrbifact & estimate & 0.043*** & 0.886*** & 0.874*** & 0.096***\\
% structcorrbifact & std.error & (0.001) & (0.007) & (0.008) & (0.002)\\
% structcorrfact & estimate & 0.054*** & 0.804*** & 0.793*** & 0.123***\\
% structcorrfact & std.error & (0.001) & (0.007) & (0.008) & (0.002)\\
% structgfact & estimate & 0.059*** & 0.760*** & 0.750*** & 0.131***\\
% structgfact & std.error & (0.001) & (0.007) & (0.008) & (0.002)\\
% structhier2ord & estimate & 0.054*** & 0.803*** & 0.794*** & 0.123***\\
% structhier2ord & std.error & (0.001) & (0.010) & (0.011) & (0.002)\\
% structindepfact & estimate & 0.090*** & 0.344*** & 0.315*** & 0.215***\\
% structindepfact & std.error & (0.001) & (0.007) & (0.008) & (0.002)\\
% structbifact × randTRUE & estimate & 0.009*** & -0.046*** & -0.051*** & 0.012***\\
% structbifact × randTRUE & std.error & (0.001) & (0.005) & (0.005) & (0.001)\\
% structcorrbifact × randTRUE & estimate & 0.006*** & -0.017*** & -0.019*** & 0.008***\\
% structcorrbifact × randTRUE & std.error & (0.001) & (0.004) & (0.005) & (0.001)\\
% structcorrfact × randTRUE & estimate & 0.005*** & -0.025*** & -0.027*** & 0.008***\\
% structcorrfact × randTRUE & std.error & (0.001) & (0.004) & (0.005) & (0.001)\\
% structhier2ord × randTRUE & estimate & 0.005* & -0.025* & -0.026* & 0.007**\\
% structhier2ord × randTRUE & std.error & (0.003) & (0.011) & (0.012) & (0.003)\\
% structindepfact × randTRUE & estimate & 0.015*** & -0.231*** & -0.242*** & 0.022***\\
% structindepfact × randTRUE & std.error & (0.001) & (0.004) & (0.005) & (0.001)\\
% SD (Intercept batchiter batch) & estimate & 0.007 & 0.094 & 0.098 & 0.019\\
% SD (rand iter1 batchiter batch) & estimate & 0.001 & 0.011 & 0.010 & 0.002\\
% Cor (Intercept\textasciitilde{}rand iter1 batchiter batch) & estimate & -1.000 & 1.000 & 1.000 & 1.000\\
% SD (Observations) & estimate & 0.006 & 0.046 & 0.049 & 0.011\\
% Num.Obs. & & 2062 & 2341 & 2341 & 2341\\
% R2 Marg. & & 0.905 & 0.966 & 0.965 & 0.944\\
% AIC & & -14225.2 & -6610.5 & -6333.2 & -13329.5\\
% BIC & & -14140.8 & -6524.1 & -6246.8 & -13243.1\\
% RMSE & & 0.01 & 0.04 & 0.05 & 0.01\\
% \bottomrule
% \end{tabular}}
% \end{table}
\begin{table}
\centering
\caption{Traditional Fit Statistics under Meta-analytic bootstrap aggregation from DWLS Estimator}
\label{tab:trad_fit_perm}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lllllllll}
\toprule
term & RMSEA fit / Est. & RMSEA fit / S.E. & CFI fit / Est. & CFI fit / S.E. & TLI fit / Est. & TLI fit / S.E. & SRMR fit / Est. & SRMR fit / S.E.\\
\midrule
structbifact & 0.049*** & 0.001 & 0.843*** & 0.008 & 0.829*** & 0.008 & 0.113*** & 0.002\\
structcorrbifact & 0.043*** & 0.001 & 0.886*** & 0.007 & 0.874*** & 0.008 & 0.096*** & 0.002\\
structcorrfact & 0.054*** & 0.001 & 0.804*** & 0.007 & 0.793*** & 0.008 & 0.123*** & 0.002\\
structgfact & 0.059*** & 0.001 & 0.760*** & 0.007 & 0.750*** & 0.008 & 0.131*** & 0.002\\
structhier2ord & 0.054*** & 0.001 & 0.803*** & 0.010 & 0.794*** & 0.011 & 0.123*** & 0.002\\
structindepfact & 0.090*** & 0.001 & 0.344*** & 0.007 & 0.315*** & 0.008 & 0.215*** & 0.002\\
\addlinespace
structbifact × randTRUE & 0.009*** & 0.001 & -0.046*** & 0.005 & -0.051*** & 0.005 & 0.012*** & 0.001\\
structcorrbifact × randTRUE & 0.006*** & 0.001 & -0.017*** & 0.004 & -0.019*** & 0.005 & 0.008*** & 0.001\\
structcorrfact × randTRUE & 0.005*** & 0.001 & -0.025*** & 0.004 & -0.027*** & 0.005 & 0.008*** & 0.001\\
structhier2ord × randTRUE & 0.005* & 0.003 & -0.025* & 0.011 & -0.026* & 0.012 & 0.007** & 0.003\\
structindepfact × randTRUE & 0.015*** & 0.001 & -0.231*** & 0.004 & -0.242*** & 0.005 & 0.022*** & 0.001\\
\midrule
SD (intercept item batch) & 0.007 & & 0.094 & & 0.098 & & 0.019 & \\
SD (rand item batch) & 0.001 & & 0.011 & & 0.010 & & 0.002 & \\
Cor (intercept $\sim$ rand item batch) & -1.000 & & 1.000 & & 1.000 & & 1.000 & \\
SD (Observations) & 0.006 & & 0.046 & & 0.049 & & 0.011 & \\
\midrule
Num.Obs. & 2062 & & 2341 & & 2341 & & 2341 & \\
R2 Marg. & 0.905 & & 0.966 & & 0.965 & & 0.944 & \\
AIC & -14225.2 & & -6610.5 & & -6333.2 & & -13329.5 & \\
BIC & -14140.8 & & -6524.1 & & -6246.8 & & -13243.1 & \\
RMSE & 0.01 & & 0.04 & & 0.05 & & 0.01 & \\
\bottomrule
\end{tabular}}
\end{table}
% \section{Variance Decomposition Supplement}\label{apx:var_decomp}
\input{appendix/drafts/var_decomp_rewrite}
\section{Variance Decomposition}\label{apx:var_decomp}
\subsection{Relationships in Variance Decomposition}\label{apx:vardecomp}
\subsubsection{From Variance to Reliability}
Variance components are the building blocks for estimating reliability. The Generalizability Coefficient (G-coefficient) provides a strong measure of reliability for a given measurement context, defined as the ratio of ``true'' variance to expected observed variance:
\begin{equation}
G = \frac{\sigma^2_{\text{true}}}{\sigma^2_{\text{true}} + \sigma^2_{\text{error}}}
\end{equation}
The definitions of ``true'' and ``error'' variance depend on the decision being made. For example, if we wish to assess the reliability of ranking models by their average score across our $n_B=6$ benchmarks, the ``true'' variance is the variance associated with the model's stable characteristics (e.g., $\sigma^2_A, \sigma^2_C, \sigma^2_D$ and their non-benchmark interactions). The ``error'' variance includes all components that interact with the Benchmark facet, as these cause a model's rank to change from one benchmark to another. For a relative decision (ranking), the G-coefficient for a model's average score across $n_B$ benchmarks is:
\begin{equation}
G_{\text{relative}} = \frac{\sigma^2_{\text{Universe Score}}}{\sigma^2_{\text{Universe Score}} + \sigma^2_{\text{Relative Error}}}
\end{equation}
where $\sigma^2_{\text{Universe Score}}$ is the sum of variance components for the object of measurement (e.g., model facets A, C, D) and $\sigma^2_{\text{Relative Error}}$ is the sum of all interaction variances involving the benchmark facet (e.g., $\sigma^2_{AB}/n_B, \sigma^2_{BC}/n_B, \dots$) plus the residual error. This framework allows us to precisely estimate the trustworthiness of leaderboard rankings.
% \paragraph{Reliability of ranking models aggregated across benchmarks.}
% Suppose the object of measurement is an entity $o$ (e.g., an architecture $A$ level, a contributor $C$ level, or a deployment $D$ level) and we score it by averaging across $n_B$ benchmarks. For concreteness, consider ranking architectures by their mean score across benchmarks. The universe-score variance (signal) is $\sigma^2_A$, while the error for an architecture mean is the sum of components that \emph{still vary across benchmarks at fixed architecture}, scaled by $1/n_B$:
% \begin{align}
% \sigma^2_{\Delta(A)}(n_B)
% \;&=\;
% \frac{1}{n_B}\Big(
% \sigma^2_{BA} + \sigma^2_{BAC} + \sigma^2_{BAD} \dots \sigma^2_{BACD} + \sigma^2_{\varepsilon}
% \Big),
% \end{align}
% where additional terms appear depending on whether, in the decision context, contributors/deployments are treated as noise to be generalized over, or held fixed. A corresponding generalizability coefficient is
% \[
% G_A(n)
% \;=\;
% \frac{\sigma^2_A}{\sigma^2_A + \sigma^2_{\Delta(A)}(n_B)}.
% \]
% Analogous coefficients $G_C(n_B)$ and $G_D(n_B)$ quantify reliability of ranking contributors or deployment types across benchmarks.
% \paragraph{Reliability of benchmark difficulty ordering across models.}
% If instead the object is the benchmark ($B$) and we average over $n_M$ models, the signal is $\sigma^2_B$ and the error is driven by model-by-benchmark interactions and residual:
% \[
% G_B(n)
% \;=\;
% \frac{\sigma^2_B}{\sigma^2_B + \frac{1}{n}\big(\sigma^2_{BA}+\sigma^2_{BC}+...+\sigma^2_{BACD}+\sigma^2_\varepsilon\big)}
% \]
% This coefficient answers: \emph{how stably can we rank benchmarks by difficulty using a sample of models?}
% \subsubsection{Reliability as a function of the decision: generalizability coefficients}
% Variance components become actionable when mapped to \emph{reliability of a specific decision}. The key idea is that averaging across $n_B$ benchmarks reduces (divides) all variance terms that vary over benchmarks.
\begin{itemize}
\item \textbf{Architecture facet (A):} groups by base architecture and model generation (e.g., \texttt{architecture:generation}). This captures systematic differences between underlying model families and releases.
\item \textbf{Benchmark facet (B):} groups by benchmark identity \texttt{bench} to capture persistent differences in difficulty and scaling across tasks.
\item \textbf{Contributor facet (C):} groups by the Hugging Face account and submission-related availability metadata. Operationally, we use \texttt{author:removed:not\_avail}, where \texttt{removed} indicates whether the base model in the lineage is removed and \texttt{not\_avail} indicates whether the evaluated model remains available on the hub. This facet measures stable differences attributable to the submission source and associated behaviors (without asserting causal interpretation).
\item \textbf{Deployment facet (D):} groups by tuning approaches/ deployment type \texttt{type:chat\_template:mo\_e: merged:precision}. This captures differences between base vs.\ chat models, use of chat templates, mixture-of-experts, model merging, and precision/quantization.
\end{itemize}
% \subsubsection{Fully-crossed random-effects model}
% Let $A(i)$ denote the architecture group of model submission $i$, $C(i)$ the contributor group, and $D(i)$ the deployment group. Let $b\in\mathcal{B}$ index benchmark (facet $B$). For an observation $y_{i b}$, the fully-crossed random-intercepts model is:
% \begin{align}
% y_{i b}
% &= \mu
% + u^{B}_{b}
% + u^{A}_{A(i)}
% + u^{C}_{C(i)}
% + u^{D}_{D(i)}
% + u^{BA}_{b,A(i)}
% + u^{BC}_{b,C(i)}
% \nonumber\\
% &\quad
% + u^{BD}_{b,D(i)}
% + u^{AC}_{A(i),C(i)}
% + u^{AD}_{A(i),D(i)}
% + u^{CD}_{C(i),D(i)}
% \nonumber\\
% &\quad
% + u^{BAC}_{b,A(i),C(i)}
% + u^{BAD}_{b,A(i),D(i)}
% + u^{BCD}_{b,C(i),D(i)}
% \nonumber\\
% &\quad
% + u^{ACD}_{A(i),C(i),D(i)}
% + u^{BACD}_{b,A(i),C(i),D(i)}
% + \varepsilon_{ib}.
% \label{eq:lmm_full}
% \end{align}
% We estimate these variance components ($\sigma^2_A, \sigma^2_B, \dots, \sigma^2_{\epsilon}$) using Restricted Maximum Likelihood (REML).
\subsection{Reliability of scaling effects and rank ordering under a four-facet crossed mixed model}
\label{apx:scaling_reliability}
To study the \emph{reliability of scaling laws} in benchmark ecosystems, it is not enough to report a fixed-effect slope $\hat\beta$ for model size. We must also quantify (i) how much of cross-model rank ordering is \emph{stably} attributable to size, (ii) how strongly the size--score relationship varies across the ecosystem facets (benchmarks, contributors, deployments, architectures), and (iii) how precisely we can generalize the estimated slope to new benchmarks and ecosystem conditions.
\paragraph{Model.}
Let $y_i$ denote a per-benchmark score observation with associated log-size covariate $x_i=\log_{10}(\#\text{params}_i)$ and facet indices
\[
a(i)\in\mathcal{A},\quad b(i)\in\mathcal{B},\quad c(i)\in\mathcal{C},\quad d(i)\in\mathcal{D}
\]
for Architecture (A), Benchmark (B), Contributor (C), and Deployment (D). We fit a fully crossed random-intercept/random-slope LMM:
\begin{align}
y_i
&= \mu + \beta x_i
+ \sum_{S\in\mathcal{S}} \Big(u^{S}_{g_S(i)} + v^{S}_{g_S(i)} x_i\Big)
+ \varepsilon_i,
\label{eq:lmm_size_full}
\end{align}
where $\mathcal{S}$ is the set of random-effect terms (main effects and interactions) included by the measurement design (e.g., $A,B,C,D,AB,AC,\dots,ABCD$), and $g_S(i)$ maps observation $i$ to the level of term $S$ (e.g., $g_{AB}(i)=(a(i),b(i))$). We assume
% \[
$u^{S}_{g}\sim\mathcal{N}(0,\sigma^2_{S,0}),
\;
v^{S}_{g}\sim\mathcal{N}(0,\sigma^2_{S,1}),
\;
\varepsilon_i\sim\mathcal{N}(0,\sigma^2_\varepsilon)$,
% \]
and, to reduce overfitting in a high-dimensional crossed design, we use the
%``double-bar''
independence parameterization
% \[
$\mathrm{Cov}\!\big(u^{S}_{g},v^{S}_{g}\big)=0
\; \forall S,g$.
% \]
% (If data permit, this can be relaxed; Appendix~X reports sensitivity to allowing intercept--slope covariance for selected terms.)
unconditional variance in $y$ is only valid after clearly defining whether the variance is (i) unconditional over $x$ and facets, (ii) conditional on the realized $x$'s in-sample, and (iii) whether benchmark main effects are included (which otherwise dominate the total variance budget). Below we define inferentially precise quantities aligned with ``reliability of scaling laws'' and ``reliability of size-induced ranking''.
\begin{figure}
\centering
\includegraphics[width=0.5\linewidth]{figures/openllmvardecomp.pdf}
\caption{Variance Decomposition diagram for four fully crossed facets of variation}
\label{fig:full_venn}
\end{figure}
\subsection{Variance Decomposition Robustness Checks for Sensitivity to Categorizations}\label{apx:robust_var_decomp}
\paragraph{Challenge: Sensitivity to categorization.} Variance decompositions can be sensitive to groupings used. \textbf{Solution 1:} \textbf{\emph{Multi-granularity robustness checks.}} We re-estimate main-effects variance shares at three levels of categorical resolution (Table~\ref{tab:granularity_sensitivity}) to verify that the observed ordering of $B \gg C > A > D$ is not an artifact of a particular coding scheme. \textbf{Solution 2:} \textbf{\emph{Bayesian estimation.}} We complement Restricted ML with a Bayesian MCMC framework (4~chains, weak priors) to obtain full posterior distributions over variance shares, providing 95\% HDI intervals for each component (Table~\ref{tab:bayesfit}).
\subsubsection{Granularity Tests}
We conducted robustness analyses across multiple levels of categorical granularity using Bayesian estimation with 95\% credible intervals. Table~\ref{tab:granularity_sensitivity} reports the percentage of variance attributable to each facet under main-effects-only models.
\begin{table}[htbp]
\centering
\caption{\textbf{Categorization Robustness checks:} Variance-share estimates (\%) across alternative granularity specifications. Numbers in parentheses indicate the number of levels within each facet.}
\label{tab:granularity_sensitivity}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lcccc}
\toprule
Facet & Finest & Fine (Table~\ref{tab:varcomp_slope}) & Coarsest & Coarsest Groups \\
\midrule
(A) Architecture
& 11.0 (1120)
& 10.1 (119)
& 9.2 (54)
& LLM ``family'' \\
(B) Benchmark
& 46.4 (6)
& 47.2 (6)
& 46.8 (6)
& Benchmarks \\
(C) Contribution
& 15.8 (1163)
& 13.8 (878)
& 13.8 (714)
& HF Account \\
(D) Deployment
& 2.3 (74)
& 3.8 (61)
& 3.5 (7)
& HF model ``type'' \\
\bottomrule
\end{tabular}}
\end{table}
The primary finding---that contributor variance exceeds architecture variance---is robust across all granularity specifications considered. More broadly, the ordering of variance components remains stable across analyses, with benchmark effects dominating contributor effects, followed by architecture and deployment effects: $B \gg C > A > D.$ This stability suggests that the substantive conclusions of the analysis are not artifacts of any particular categorization scheme.
\subsection{Reliability of the fixed scaling slope: generalizability of $\beta$}
\label{sec:beta_generalizability}
The fixed effect $\beta$ summarizes the average size--score relationship across the ecosystem. Its reliability is naturally expressed by an inferential precision measure for $\hat\beta$ under the fitted mixed model.
% \paragraph{Model-based standard error and robust alternatives.}
% Let $\hat\beta$ be the REML/ML estimate. Under correct model specification, $\hat\beta$ is asymptotically normal:
% % \[
% $\hat\beta \;\dot\sim\; \mathcal{N}\!\big(\beta,\;\mathrm{Var}(\hat\beta)\big),
% % \qquad
% \mathrm{Var}(\hat\beta) = \big(\mathbf{X}^\top \hat{\mathbf{V}}^{-1}\mathbf{X}\big)^{-1}_{\beta\beta}$,
% % \]
% where $\mathbf{V}$ is the marginal covariance induced by all random effects in~\eqref{eq:lmm_size_full}.
% Because ecosystem data are often heteroskedastic and mildly misspecified, we additionally recommend a cluster-robust sandwich estimator with clustering at the finest reliably observed cell (e.g., $ABCD$ if feasible) or at contributor/source:
% \[
% \widehat{\mathrm{Var}}_{\text{CR}}(\hat\beta)
% =
% (\mathbf{X}^\top \hat{\mathbf{V}}^{-1}\mathbf{X})^{-1}
% \Big(\sum_{g\in G}\mathbf{X}_g^\top \hat{\mathbf{V}}_g^{-1}\hat{\mathbf{r}}_g\hat{\mathbf{r}}_g^\top \hat{\mathbf{V}}_g^{-1}\mathbf{X}_g\Big)
% (\mathbf{X}^\top \hat{\mathbf{V}}^{-1}\mathbf{X})^{-1}.
% \]
% We report both model-based and robust SEs when they differ materially.
\subsubsection{Restricted Maximum Likelihood and Bayesian Estimations}
For the base model, we report its Restricted Maximum Likelihood estimation in Table \ref{tab:varcomp_base} and Bayesian estimation in Table \ref{tab:bayesfit}
\begin{table}
\centering
\caption{Variance Component Estimates, Covariate Model \eqref{eq:gstudy_slopes}}
\label{tab:varcomp_slope_apx}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lrrr}
\toprule
\textbf{Variation Source} & \textbf{Var. ($\sigma^2$)} & \textbf{$p_S$ (\%)} &\textbf{$\text{PSI}_S$} \\
\midrule
A (Architecture) & 18.73 & 6.34 & 0.13 \\
B (Benchmark) & 136.30 & 46.13 & 0.30 \\
C (Contributor) & 23.09 & 7.82 & 0.17 \\
D (Deployment) & 3.26 & 1.10 & 0.03 \\
C$\times$D & 3.26 & 1.11 & 0.01 \\
A$\times$C & 12.20 & 4.14 & 0.06 \\
B$\times$A & 10.97 & 3.71 & 0.05 \\
A$\times$D & 0.91 & 0.31 & 0.00 \\
B$\times$D & 13.55 & 4.58 & 0.02 \\
B$\times$C & 7.87 & 2.66 & 0.03 \\
B$\times$C$\times$D & 5.43 & 1.84 & 0.05 \\
B$\times$A$\times$C & 11.12 & 3.77 & 0.07 \\
B$\times$A$\times$D & 1.72 & 0.58 & 0.01 \\
C$\times$A$\times$D & 13.79 & 4.66 & 0.07 \\
$\epsilon_{\text{resid}}$ (+ A$\times$B$\times$C$\times$D) & 33.24 & 11.25 & \\
\bottomrule
\end{tabular}}
\begin{tablenotes}
\item \small Variance component estimates $\sigma^2$ and $p_S$ represent the total variance associated with both slope and intercept. The base model ~\eqref{eq:gstudy} can be found in Table \ref{tab:varcomp_base} and Bayesian posterior draws of $p_S$ can be found in Table \ref{tab:bayesfit}.
\end{tablenotes}
\end{table}
\begin{table}[h!]
\centering
\caption{Variance Component Estimates (No Covariate Model).}
\label{tab:varcomp_base}
\begin{tabular}{@{}llrr@{}}
\toprule
\textbf{Source of Variation} & & \textbf{Var. ($\sigma^2$)} & \textbf{\% of Total} \\ \midrule
\multicolumn{4}{l}{\textit{Main Effects}} \\
(B) Benchmark & & 150.0 & 43.0\% \\
(C) Contributor & & 29.91 & 8.6\% \\
(A) Architecture & & 14.5 & 4.17\% \\
(D) Deployment Type & & 6.64 & 1.91\% \\
\midrule
\multicolumn{4}{l}{\textit{Two-Way Interactions}} \\
B $\times$ C & & 12.7 & 3.66\% \\
A $\times$ C & & 12.3 & 3.5\% \\
B $\times$ A & & 10.21 & 2.94\% \\
B $\times$ D & & 17.2 & 4.9\% \\
C $\times$ D & & 2.64 & 0.76\% \\
A $\times$ D & & 0.77 & 0.22\% \\
\midrule
\multicolumn{4}{l}{\textit{Higher-Order Interactions}} \\
Three-Way Interactions & & 38.9 & 11.2\% \\
% Four-Way Interactions & & 2.08 & 0.6\% \\
$\epsilon_{\text{resid}}$ (+ A$\times$B$\times$C$\times$D) & & 52.0 & 15.0\% \\
\bottomrule
\end{tabular}
\end{table}
\begin{table}[!h]
\centering
\caption{Bayesian Estimation posterior draws for percentage of variance ($p_S$) from base variance decomposition model \eqref{eq:gstudy}}\label{tab:bayesfit}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{lrrrrrrrrrr}
\toprule
Facet & mean & median & SD & MAD & q5 & q95 & $\hat{R}$ & MAP & HDI$_{low}$ & HDI$_{high}$\\
\midrule
A & 0.05 & 0.04 & 0.02 & 0.02 & 0.02 & 0.09 & 1.01 & 0.04 & 0.02 & 0.10\\
A$\times$C & 0.03 & 0.03 & 0.01 & 0.01 & 0.01 & 0.05 & 1.02 & 0.02 & 0.01 & 0.06\\
A$\times$D & 0.00 & 0.00 & 0.00 & 0.00 & 0.00 & 0.01 & 1.02 & 0.00 & 0.00 & 0.01\\
C & 0.08 & 0.08 & 0.03 & 0.03 & 0.04 & 0.13 & 1.01 & 0.08 & 0.03 & 0.13\\
C$\times$A$\times$D & 0.08 & 0.08 & 0.02 & 0.03 & 0.04 & 0.12 & 1.01 & 0.09 & 0.03 & 0.12\\
\addlinespace
C$\times$D & 0.01 & 0.00 & 0.01 & 0.00 & 0.00 & 0.02 & 1.15 & 0.00 & 0.00 & 0.02\\
B & 0.50 & 0.49 & 0.15 & 0.17 & 0.27 & 0.75 & 1.01 & 0.46 & 0.25 & 0.79\\
B$\times$A & 0.03 & 0.03 & 0.01 & 0.01 & 0.01 & 0.04 & 1.01 & 0.03 & 0.01 & 0.04\\
B$\times$A$\times$C & 0.02 & 0.02 & 0.01 & 0.01 & 0.01 & 0.03 & 1.01 & 0.02 & 0.01 & 0.03\\
B$\times$A$\times$D & 0.00 & 0.00 & 0.00 & 0.00 & 0.00 & 0.01 & 1.01 & 0.00 & 0.00 & 0.01\\
\addlinespace
B$\times$C & 0.03 & 0.03 & 0.01 & 0.01 & 0.02 & 0.05 & 1.00 & 0.03 & 0.01 & 0.05\\
B$\times$C$\times$D & 0.01 & 0.01 & 0.00 & 0.00 & 0.00 & 0.01 & 1.01 & 0.01 & 0.00 & 0.01\\
B$\times$D & 0.04 & 0.04 & 0.01 & 0.02 & 0.02 & 0.07 & 1.01 & 0.05 & 0.02 & 0.07\\
D & 0.02 & 0.02 & 0.01 & 0.01 & 0.01 & 0.05 & 1.00 & 0.01 & 0.00 & 0.06\\
sigma & 0.14 & 0.14 & 0.04 & 0.05 & 0.07 & 0.20 & 1.01 & 0.15 & 0.06 & 0.21\\
\bottomrule
\end{tabular}}
\begin{tablenotes}
\small
\item To directly estimate sensitivity of variance shares to categorizations, we re-estimate Table \ref{tab:varcomp_base} base model using a MCMC Bayesian framework with 4 chains and weak priors. The table values represent proportions of variance explained by component, $p_S$, calculated directly from posterior draws. %The 95\% CIs for C and A do not overlap at their central tendencies, supporting the robustness of our headline finding despite dataset limitations.
\end{tablenotes}
\end{table}
\paragraph{A reliability coefficient for slope $\beta$.}
For comparability across datasets, we report a dimensionless reliability-like coefficient for the scaling slope:
\[
R_\beta \;\equiv\; \frac{\hat\beta^2}{\hat\beta^2 + \widehat{\mathrm{Var}}(\hat\beta)} \in [0,1].
\]
This is the fraction of (squared) signal retained after accounting for estimation noise, analogous to a shrinkage factor. High $R_\beta$ indicates that the estimated scaling law is stable under the current measurement design; low $R_\beta$ indicates that the fixed scaling effect is not reliably estimable given crossed ecosystem variability.
\subsection{Reliability of rank ordering attributable to size}
\label{sec:rank_reliability_size}
Scaling laws are often used to justify \emph{rank ordering} models by predicted performance from size alone. We therefore define reliability at the level of the \emph{size-induced predictor} and its stability relative to ecosystem noise.
Let the size-only linear predictor be
\[
\hat{y}^{(\text{size})}_i \;=\; \hat\mu + \hat\beta x_i.
\]
A natural population-level reliability of size-driven ranking is the fraction of marginal variance in $y$ explained by $\hat{y}^{(\text{size})}$:
\begin{equation}
\Omega_x
\;\equiv\;
\frac{\mathrm{Var}(\beta x)}{\mathrm{Var}(y)}
=
\frac{\beta^2\,\mathrm{Var}(x)}{\beta^2\mathrm{Var}(x) + \mathrm{Var}\!\Big(\sum_{S}u^S_{g_S}\Big) + \mathrm{Var}(\varepsilon)},
\label{eq:omega_x_def}
\end{equation}
where $\mathrm{Var}(y)$ is the unconditional marginal variance under~\eqref{eq:lmm_size_full}. In our model,
% \[
$\mathrm{Var}\!\Big(\sum_{S}u^S_{g_S}\Big) \approx \sum_{S}\sigma^2_{S,0}$
(for a randomly drawn observation
% , ignoring duplicate membership effects
),
% \]
and $\mathrm{Var}(\varepsilon)=\sigma^2_\varepsilon$.
We estimate~\eqref{eq:omega_x_def} by plug-in using $\hat\beta$ and variance-component estimates. We interpret $\Omega_x$ as: \emph{how much of overall score variability (and thus potential rank ordering) is attributable to size alone}. For this dataset, $\Omega_x=0.135$, indicating that global rankings cannot be reliably without considering the benchmark's influence.
\paragraph{Benchmark-independent rank ordering.}
Because the benchmark main effect often dominates total variance, $\Omega_x$ can be misleadingly small even when size strongly explains \emph{cross-benchmark} rank order. We therefore also report the benchmark-demeaned reliability:
\begin{equation}
\Omega_x^{(-B)}
\;\equiv\;
\frac{\beta^2\,\mathrm{Var}(x)}{\beta^2\mathrm{Var}(x) + \sum_{S\neq B}\sigma^2_{S,0} + \sigma^2_\varepsilon},
\label{eq:omega_x_minusB}
\end{equation}
i.e., the size-explainable share of variance after removing the benchmark difficulty component (and optionally other nuisance main effects depending on the decision context). This aligns with the practical question: \emph{if we compare models across benchmarks, how much of the benchmark-independent ranking is size-driven?} In this case, $\Omega_x^{(-B)}=0.26$.
% \paragraph{Correlation-of-ranks reliability.}
% Variance shares are scale-dependent; rank reliability is often better summarized by a correlation measure. We therefore calculate the expected correlation between observed scores and size-predicted scores:
% \[
% \rho_x
% \;\equiv\;
% \mathrm{Corr}\!\big(y,\ \mu+\beta x\big)
% =
% \sqrt{\Omega_x},
% \qquad
% \rho_x^{(-B)}=\sqrt{\Omega_x^{(-B)}}.
% \]
% Additionally, for leaderboard-style use, we compute a nonparametric rank version on held-out cells (see below): Spearman $\rho$ between $y$ and $\hat{y}^{(\text{size})}$.
\subsection{Decomposing variance to quantify the reliability of scaling laws}\label{apx:scaling_metrics}
A central claim in modern AI is the existence of so-called ``scaling laws'', where performance improves predictably with model size. In our framework, this corresponds to the fixed effect of log-parameters, $\beta$, in a random-slopes model. However, an estimate of $\beta$ is only meaningful if the scaling relationship is reasonably stable across the ecosystem. A large average effect can mask extreme heterogeneity, where scaling benefits are strong for some benchmarks or architectures but weak or non-existent for others. Our goal is therefore not only to estimate the average scaling effect, but to precisely quantify its \emph{reliability} and diagnose the sources of its instability.
To this end, we extend~\eqref{eq:gstudy} to a full random-slopes model. Let $y_i$ be the score for an observation with associated log-size $x_i = \log_{10}(\text{\# parameters})$. The score is modeled in Eq. ~\eqref{eq:gstudy_slopes}:
\begin{align*}
s_i
&= (\mu + \beta x_i)
+ \sum_{S}\left(u^{S}_{g_S} + v^{S}_{g_S} x_i\right)
+ \varepsilon_{i},
% \label{eq:lmm_size}
\end{align*}
where the sum is over all random-effect terms $S$ (main effects and interactions), and $g_S$ indexes the level of term $S$ for observation $i$. For each term $S$, we estimate a variance component for the random intercepts, $u^{S}_{g_S} \sim \mathcal{N}(0, \sigma^2_{S,0})$, and a variance component for the random slopes, $v^{S}_{g_S} \sim \mathcal{N}(0, \sigma^2_{S,1})$. To ensure model stability and avoid overfitting in the fully crossed design, we constrain the random intercepts and slopes within each term to be uncorrelated.
This model allows us to decompose the total variance of scores into components related to baseline performance (intercepts) and components related to performance scaling (slopes).
\begin{proposition}[Exact variance decomposition under a random-slopes model]
\label{prop:variance_decomp}
Assume the random effects $u^S, v^S$ are independent across facets $S$ and independent of the covariate $x$. The total marginal variance of the outcome $y$ can be decomposed exactly as:
\begin{align}
\mathrm{Var}(y) =
&\underbrace{\sum_{S} \sigma^2_{S,0}}_{\text{Intercept Variance}}
+ \underbrace{\beta^2 \mathrm{Var}(x)}_{\text{Fixed Slope Variance}} % \nonumber \\
+ \underbrace{\left( \mathrm{Var}(x) + \mathrm{E}[x]^2 \right) \sum_{S} \sigma^2_{S,1}}_{\text{Random Slope Variance}}
+ \underbrace{\sigma^2_\varepsilon}_{\text{Residual Variance}}.
\end{align}
\end{proposition}
\begin{proof}
By the law of total variance, $\mathrm{Var}(y) = \mathrm{E}[\mathrm{Var}(y|x)] + \mathrm{Var}(\mathrm{E}[y|x])$.
The conditional expectation is $\mathrm{E}[y_i|x_i] = \mu + \beta x_i$, so its variance is $\mathrm{Var}(\mathrm{E}[y|x]) = \beta^2 \mathrm{Var}(x)$.
The conditional variance is $\mathrm{Var}(y_i|x_i) = \mathrm{Var}\left(\sum_S (u^S_{g_S} + v^S_{g_S}x_i) + \varepsilon_i\right) = \sum_S \sigma^2_{S,0} + x_i^2 \sum_S \sigma^2_{S,1} + \sigma^2_\varepsilon$.
Taking the expectation over $x$ yields $\mathrm{E}[\mathrm{Var}(y|x)] = \sum_S \sigma^2_{S,0} + \mathrm{E}[x^2] \sum_S \sigma^2_{S,1} + \sigma^2_\varepsilon$. Since $\mathrm{E}[x^2] = \mathrm{Var}(x) + \mathrm{E}[x]^2$, the result follows.
\end{proof}
Building on this decomposition, we introduce metrics to provide an inferentially precise assessment of the scaling law's reliability.
% \paragraph{1. Scaling-Related Variance Component (SRVC).}
% This metric quantifies the fraction of total score variance attributable to model size, including both its fixed effect and its variation across facets.
% \begin{equation}
% \mathrm{SRVC} = \frac{\beta^2 \mathrm{Var}(x) + \left( \mathrm{Var}(x) + \mathrm{E}[x]^2 \right) \sum_{S} \sigma^2_{S,1}}{\mathrm{Var}(y)}.
% \end{equation}
% A high SRVC indicates that knowing a model's size is highly informative for predicting its score variance within the ecosystem.
\paragraph{1. Slope Signal-to-Noise Ratio (SNR$_\beta$).}
This novel metric directly quantifies the reliability of the fixed-effect scaling law. It measures the strength of the average scaling trend ($\beta$) relative to its instability across the ecosystem's facets.
\begin{definition}[Slope Signal-to-Noise Ratio]
The SNR of the fixed-effect slope $\beta$ is the ratio of the squared magnitude of the fixed effect to the total variance of the random slopes:
\begin{equation}
\mathrm{SNR}_\beta = \frac{\beta^2}{\sum_{S} \sigma^2_{S,1}}.
\end{equation}
\end{definition}
A high $\mathrm{SNR}_\beta$ (${\gg}1$) implies a robust, universal scaling law: the average effect $\beta$ dominates the context-specific variations. A low $\mathrm{SNR}_\beta$ (${\approx}1$ or less) indicates that the average scaling law is misleading, as the actual slope experienced by a model is highly dependent on context (i.e., which benchmark, which architecture family, etc.).
\paragraph{Result} We find that $\mathrm{SNR}_\beta = 1.12$, suggesting that an average scaling law would be misleading, and that scaling is context-dependent.
\paragraph{2. Proportion of Slope Instability (PSI$_S$).}
While SNR$_\beta$ quantifies the \textbf{total} instability of the scaling law, PSI$_S$ diagnoses its sources by partitioning the total random slope variance. For each facet or interaction term $S$, we define:
\begin{equation}
\mathrm{PSI}_S = \frac{\sigma^2_{S,1}}{\sum_{S'} \sigma^2_{S',1}}.
\end{equation}
PSI$_S$ is the proportion of the total instability in the scaling relationship that is attributable to facet $S$. For example, a large $\mathrm{PSI}_{\text{Benchmark}}$ would imply that a primary reason the scaling law is unreliable is that different benchmarks respond to increases in model size in vastly different ways. This metric provides a clear, quantitative guide to where efforts to create more uniform evaluation standards should be focused.
Together, these metrics transform the variance decomposition from a descriptive exercise into a powerful diagnostic tool for assessing the structural reliability of one of the most fundamental claims in large-scale machine learning. They allow us to move beyond asking ``do models scale?'' to answering the more critical questions: ``how reliably do they scale, and what features of the ecosystem make that scaling relationship untrustworthy?''
% \begin{table*}
% \centering
% \caption{Variance and Proportion Slope Instability}\label{tab:psi}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{lrr}
% \toprule
% Group & Variance Component & Proportion of Slope Variance (PSI)\\
% \midrule
% BxAxC & 8.632 & 0.082\\
% BxCxD & 4.163 & 0.039\\
% BxC & 3.686 & 0.035\\
% BxAxD & 1.267 & 0.012\\
% CxAxD & 8.363 & 0.079\\
% \addlinespace
% AxC & 2.575 & 0.024\\
% CxD & 3.365 & 0.032\\
% BxA & 3.374 & 0.032\\
% C & 19.764 & 0.187\\
% AxD & 0.360 & 0.003\\
% \addlinespace
% BxD & 1.328 & 0.013\\
% A & 7.450 & 0.071\\
% D & 7.097 & 0.067\\
% B & 34.154 & 0.323\\
% \bottomrule
% \end{tabular}}
% \end{table*}
\subsection{Heterogeneity of scaling: where scaling laws break}
\label{sec:slope_heterogeneity}
A single $\beta$ is only meaningful if slope heterogeneity is small relative to the mean slope. Our crossed random slopes make this explicit.
\paragraph{Global slope heterogeneity index.}
Let the effective slope for observation $i$ be
% \[
$\beta_i^{\text{eff}} \;=\; \beta + \sum_{S} v^S_{g_S(i)}$.
% \]
We summarize heterogeneity by the variance of effective slopes:
\begin{equation}
\sigma^2_{\beta,\text{eff}}
\;\equiv\;
\mathrm{Var}(\beta_i^{\text{eff}})
\approx \sum_{S}\sigma^2_{S,1},
\label{eq:slope_var}
\end{equation}
and report the coefficient of variation (dimensionless)
% \[
$\mathrm{CV}_\beta \;\equiv\; \frac{\sqrt{\sigma^2_{\beta,\text{eff}}}}{|\beta|}$.
% \]
Large $\mathrm{CV}_\beta$ indicates that scaling is not a single law but a distribution of laws varying by benchmark and ecosystem facets. In the case of the present study, $\mathrm{CV}_\beta =0.944$, meaning the standard deviation of the slope is 94.4\% of the estimate, suggesting high imprecision for a global scaling law across all the data without appropriate controls.
\paragraph{Facet-specific slope reliability.}
For each facet term $S$, we calculate an intra-class style share of slope variance:
% \[
$H_S \;\equiv\; \frac{\sigma^2_{S,1}}{\sum_{T}\sigma^2_{T,1}}$,
% \]
identifying which ecosystem components (benchmarks vs.\ contributors vs.\ deployments vs.\ architectures and their interactions) most distort scaling behavior. This answers: \emph{where does the scaling law change?}
% \subsection{Predictive generalization: reliability of scaling for new ecosystem cells}
% \label{sec:predictive_reliability_scaling}
% Inference about scaling laws is ultimately about generalization. We therefore complement in-sample variance shares with out-of-sample (OOS) reliability.
% \paragraph{Cell-level cross-validation and rank stability.}
% We perform blocked cross-validation that respects the crossed design by holding out entire levels (or level-combinations) of key facets, e.g.:
% \[
% \text{Hold out } b\in\mathcal{B} \text{ (new benchmark), or } c\in\mathcal{C} \text{ (new contributor)}.
% \]
% On the held-out set, we compute:
% \begin{itemize}[leftmargin=*,nosep]
% \item $\mathrm{RMSE}_{\text{OOS}}$ of $\hat{y}^{(\text{size})}_i=\hat\mu+\hat\beta x_i$ (size-only).
% \item $\Delta\mathrm{RMSE}_{\text{OOS}}$ comparing size-only vs.\ full mixed-model predictions (size + BLUPs).
% \item Spearman $\rho_{\text{OOS}}$ between $y_i$ and $\hat{y}^{(\text{size})}_i$ (rank reliability of size predictions).
% \end{itemize}
% This directly measures whether scaling-based rank order persists under realistic ecosystem shifts.
\subsection{A corrected ``size-linked variance'' decomposition}
\label{sec:size_linked_variance_corrected}
If one wishes to quantify how much of marginal variance is attributable to \emph{all size-related terms} (fixed slope plus random slopes), define the size-linked component conditional on the distribution of $x$:
\[
\mathrm{Var}_{\text{size}}(y)
\;\equiv\;
\mathrm{Var}\!\Big(\beta x + \sum_{S} v^S_{g_S} x\Big).
\]
% Under independence between $x$ and the random slopes and $\mathbb{E}[x]=0$ (centered covariate), this becomes
% \[
% \mathrm{Var}_{\text{size}}(y)
% =
% \beta^2\mathrm{Var}(x) + \sum_{S}\sigma^2_{S,1}\mathrm{Var}(x).
% \]
The corresponding \emph{size-linked share of marginal variance} is then
\begin{equation}
\Omega_{\text{size-all}}
\;\equiv\;
\frac{\beta^2\mathrm{Var}(x) + \left( \mathrm{Var}(x) + \mathrm{E}[x]^2 \right) \sum_{S} \sigma^2_{S,1}}
{\beta^2\mathrm{Var}(x) + \left( \mathrm{Var}(x) + \mathrm{E}[x]^2 \right) \sum_{S} \sigma^2_{S,1} + \sum_{S}\sigma^2_{S,0} + \sigma^2_\varepsilon }.
\label{eq:omega_size_all}
\end{equation}
We emphasize that $\Omega_{\text{size-all}}$ answers a different question than $\Omega_x$: it includes \emph{heterogeneous scaling} effects (random slopes) and therefore reflects both generalizable scaling and ecosystem-specific slope distortions.
\paragraph{Result} For the present study, we calculate \eqref{eq:omega_size_all} to be $\Omega_{\text{size-all}}=0.402$, suggesting that about 40\% of all observed variation is linked to model size, as reported in the introduction.
% \paragraph{Centering and interpretability.}
% All formulas in this subsection assume $x$ is centered (e.g., $x\leftarrow x-\bar{x}$). Centering ensures that $\sigma^2_{S,0}$ truly represents baseline (at mean size) variability rather than absorbing slope effects.
\paragraph{Reporting summary.}
To make scaling-law reliability interpretable, we report:
\begin{enumerate}[leftmargin=*,nosep]
\item $\hat\beta$ with model-based and robust SE, and $R_\beta$.
\item $\Omega_x$ and $\Omega_x^{(-B)}$ (size-driven rank-order potential overall and benchmark-independent).
\item $\sigma^2_{\beta,\text{eff}}$ and $\mathrm{CV}_\beta$ (how much scaling varies across the ecosystem).
% \item $H_S$ for each facet/interactions (where scaling varies).
% \item OOS rank reliability $\rho_{\text{OOS}}$ under blocked CV (generalization under benchmark/contributor/deployment shift).
\end{enumerate}
These metrics separate three phenomena that are conflated in typical leaderboard analyses: (i) whether scale predicts performance on average, (ii) whether scale predicts \emph{rank ordering} robustly across benchmarks, and (iii) whether ``scaling laws'' are stable or conditional on ecosystem facets.
% \subsection{Reliability of scaling effects and rank ordering under a four-facet crossed mixed model}
% \label{sec:scaling_reliability}
% To study the \emph{reliability of scaling laws} in benchmark ecosystems, it is not enough to report a fixed-effect slope $\hat\beta$ for model size. We must also quantify (i) how much of cross-model rank ordering is \emph{stably} attributable to size, (ii) how strongly the size--score relationship varies across the ecosystem facets (benchmarks, contributors, deployments, architectures), and (iii) how precisely we can generalize the estimated slope to new benchmarks and ecosystem conditions.
% \paragraph{Model.}
% Let $y_i$ denote a per-benchmark score observation with associated log-size covariate $x_i=\log_{10}(\#\text{params}_i)$ and facet indices
% \[
% a(i)\in\mathcal{A},\quad b(i)\in\mathcal{B},\quad c(i)\in\mathcal{C},\quad d(i)\in\mathcal{D}
% \]
% for Architecture (A), Benchmark (B), Contributor (C), and Deployment (D). We fit a fully crossed random-intercept/random-slope LMM:
% \begin{align}
% y_i
% &= \mu + \beta x_i
% + \sum_{S\in\mathcal{S}} \Big(u^{S}_{g_S(i)} + v^{S}_{g_S(i)} x_i\Big)
% + \varepsilon_i,
% \label{eq:lmm_size_full}
% \end{align}
% where $\mathcal{S}$ is the set of random-effect terms (main effects and interactions) included by the measurement design (e.g., $A,B,C,D,AB,AC,\dots,ABCD$), and $g_S(i)$ maps observation $i$ to the level of term $S$ (e.g., $g_{AB}(i)=(a(i),b(i))$). We assume
% \[
% u^{S}_{g}\sim\mathcal{N}(0,\sigma^2_{S,0}),
% \qquad
% v^{S}_{g}\sim\mathcal{N}(0,\sigma^2_{S,1}),
% \qquad
% \varepsilon_i\sim\mathcal{N}(0,\sigma^2_\varepsilon),
% \]
% and, to reduce overfitting in a high-dimensional crossed design, we use the independence parameterization
% \[
% \mathrm{Cov}\!\big(u^{S}_{g},v^{S}_{g}\big)=0
% \quad \forall S,g.
% \]
% (If data permit, this can be relaxed; Appendix~X reports sensitivity to allowing intercept--slope covariance for selected terms.)
% % $\mathrm{Var}_{\cdot 1}(x)$ implicitly treats $\beta^2\mathrm{Var}(x)$ and $\sigma^2_{S,1}\mathrm{Var}(x)$ as if they were directly commensurate with unconditional variance in $y$. This is only valid after clearly defining whether the variance is (i) unconditional over $x$ and facets, (ii) conditional on the realized $x$'s in-sample, and (iii) whether benchmark main effects are included (which otherwise dominate the total variance budget).
% \subsection{(1) Reliability of the fixed scaling slope: generalizability of $\beta$}
% \label{sec:beta_generalizability}
% The fixed effect $\beta$ summarizes the \emph{average} size--score relationship across the ecosystem. Its reliability is naturally expressed by an inferential precision measure for $\hat\beta$ under the fitted mixed model.
% \paragraph{Model-based standard error and robust alternatives.}
% Let $\hat\beta$ be the REML/ML estimate. Under correct model specification, $\hat\beta$ is asymptotically normal:
% \[
% \hat\beta \;\dot\sim\; \mathcal{N}\!\big(\beta,\;\mathrm{Var}(\hat\beta)\big),
% \qquad
% \mathrm{Var}(\hat\beta) = \big(\mathbf{X}^\top \hat{\mathbf{V}}^{-1}\mathbf{X}\big)^{-1}_{\beta\beta},
% \]
% where $\mathbf{V}$ is the marginal covariance induced by all random effects in~\eqref{eq:lmm_size_full}.
% Because ecosystem data are often heteroskedastic and mildly misspecified, we additionally recommend a cluster-robust sandwich estimator with clustering at the finest reliably observed cell (e.g., $ABCD$ if feasible) or at contributor/source:
% \[
% \widehat{\mathrm{Var}}_{\text{CR}}(\hat\beta)
% =
% (\mathbf{X}^\top \hat{\mathbf{V}}^{-1}\mathbf{X})^{-1}
% \Big(\sum_{g\in G}\mathbf{X}_g^\top \hat{\mathbf{V}}_g^{-1}\hat{\mathbf{r}}_g\hat{\mathbf{r}}_g^\top \hat{\mathbf{V}}_g^{-1}\mathbf{X}_g\Big)
% (\mathbf{X}^\top \hat{\mathbf{V}}^{-1}\mathbf{X})^{-1}.
% \]
% We report both model-based and robust SEs when they differ materially.
% \paragraph{A reliability coefficient for $\beta$ (signal-to-noise).}
% For comparability across datasets, we report a dimensionless reliability-like coefficient for the scaling slope:
% \[
% R_\beta \;\equiv\; \frac{\hat\beta^2}{\hat\beta^2 + \widehat{\mathrm{Var}}(\hat\beta)} \in [0,1].
% \]
% This is the fraction of (squared) signal retained after accounting for estimation noise, analogous to a shrinkage factor. High $R_\beta$ indicates that the estimated scaling law is stable under the current measurement design; low $R_\beta$ indicates that the fixed scaling effect is not reliably estimable given crossed ecosystem variability.
% \subsection{(2) Reliability of rank ordering attributable to size}
% \label{sec:rank_reliability_size}
% Scaling laws are often used to justify \emph{rank ordering} models by predicted performance from size alone. We therefore define reliability at the level of the \emph{size-induced predictor} and its stability relative to ecosystem noise.
% Let the size-only linear predictor be
% \[
% \hat{y}^{(\text{size})}_i \;=\; \hat\mu + \hat\beta x_i.
% \]
% A natural population-level reliability of size-driven ranking is the fraction of marginal variance in $y$ explained by $\hat{y}^{(\text{size})}$:
% \begin{equation}
% \Omega_x
% \;\equiv\;
% \frac{\mathrm{Var}(\beta x)}{\mathrm{Var}(y)}
% =
% \frac{\beta^2\,\mathrm{Var}(x)}{\beta^2\mathrm{Var}(x) + \mathrm{Var}\!\Big(\sum_{S}u^S_{g_S}\Big) + \mathrm{Var}(\varepsilon)},
% \label{eq:omega_x_def}
% \end{equation}
% where $\mathrm{Var}(y)$ is the unconditional marginal variance under~\eqref{eq:lmm_size_full}. In our model,
% \[
% \mathrm{Var}\!\Big(\sum_{S}u^S_{g_S}\Big) \approx \sum_{S}\sigma^2_{S,0}
% \quad\text{(for a randomly drawn observation, ignoring duplicate membership effects)},
% \]
% and $\mathrm{Var}(\varepsilon)=\sigma^2_\varepsilon$.
% We estimate~\eqref{eq:omega_x_def} by plug-in using $\hat\beta$ and variance-component estimates. We interpret $\Omega_x$ as: \emph{how much of overall score variability (and thus potential rank ordering) is attributable to size alone}.
% \paragraph{Benchmark-independent rank ordering.}
% Because the benchmark main effect often dominates total variance, $\Omega_x$ can be misleadingly small even when size strongly explains \emph{cross-benchmark} rank order. We therefore also report the benchmark-demeaned reliability:
% \begin{equation}
% \Omega_x^{(-B)}
% \;\equiv\;
% \frac{\beta^2\,\mathrm{Var}(x)}{\beta^2\mathrm{Var}(x) + \sum_{S\neq B}\sigma^2_{S,0} + \sigma^2_\varepsilon},
% \label{eq:omega_x_minusB}
% \end{equation}
% i.e., the size-explainable share of variance after removing the benchmark difficulty component (and optionally other nuisance main effects depending on the decision context). This aligns with the practical question: \emph{if we compare models across benchmarks, how much of the benchmark-independent ranking is size-driven?}
% \paragraph{Correlation-of-ranks reliability.}
% Variance shares are scale-dependent; rank reliability is often better summarized by a correlation measure. We therefore report the expected correlation between observed scores and size-predicted scores:
% \[
% \rho_x
% \;\equiv\;
% \mathrm{Corr}\!\big(y,\ \mu+\beta x\big)
% =
% \sqrt{\Omega_x},
% \qquad
% \rho_x^{(-B)}=\sqrt{\Omega_x^{(-B)}}.
% \]
% Additionally, for leaderboard-style use, we compute a nonparametric rank version on held-out cells (see below): Spearman $\rho$ between $y$ and $\hat{y}^{(\text{size})}$.
% \subsubsection{(3) Heterogeneity of scaling: where scaling laws break}
% \label{sec:slope_heterogeneity}
% A single $\beta$ is only meaningful if slope heterogeneity is small relative to the mean slope. Our crossed random slopes make this explicit.
% \paragraph{Global slope heterogeneity index.}
% Let the effective slope for observation $i$ be
% \[
% \beta_i^{\text{eff}} \;=\; \beta + \sum_{S} v^S_{g_S(i)}.
% \]
% We summarize heterogeneity by the variance of effective slopes:
% \begin{equation}
% \sigma^2_{\beta,\text{eff}}
% \;\equiv\;
% \mathrm{Var}(\beta_i^{\text{eff}})
% \approx \sum_{S}\sigma^2_{S,1},
% \label{eq:slope_var}
% \end{equation}
% and report the coefficient of variation (dimensionless)
% \[
% \mathrm{CV}_\beta \;\equiv\; \frac{\sqrt{\sigma^2_{\beta,\text{eff}}}}{|\beta|}.
% \]
% Large $\mathrm{CV}_\beta$ indicates that scaling is not a single law but a distribution of laws varying by benchmark and ecosystem facets.
% \paragraph{Facet-specific slope reliability.}
% For each facet term $S$, we report an intra-class style share of slope variance:
% \[
% H_S \;\equiv\; \frac{\sigma^2_{S,1}}{\sum_{T}\sigma^2_{T,1}},
% \]
% identifying which ecosystem components (benchmarks vs.\ contributors vs.\ deployments vs.\ architectures and their interactions) most distort scaling behavior. This answers: \emph{where does the scaling law change?}
% \subsection{Predictive generalization: reliability of scaling for new ecosystem cells}
% \label{sec:predictive_reliability_scaling}
% Inference about scaling laws is ultimately about generalization. We therefore complement in-sample variance shares with out-of-sample (OOS) reliability.
% \paragraph{Cell-level cross-validation and rank stability.}
% We perform blocked cross-validation that respects the crossed design by holding out entire levels (or level-combinations) of key facets, e.g.:
% \[
% \text{Hold out } b\in\mathcal{B} \text{ (new benchmark), or } c\in\mathcal{C} \text{ (new contributor)}.
% \]
% On the held-out set, we compute:
% \begin{itemize}
% \item $\mathrm{RMSE}_{\text{OOS}}$ of $\hat{y}^{(\text{size})}_i=\hat\mu+\hat\beta x_i$ (size-only).
% \item $\Delta\mathrm{RMSE}_{\text{OOS}}$ comparing size-only vs.\ full mixed-model predictions (size + BLUPs).
% \item Spearman $\rho_{\text{OOS}}$ between $y_i$ and $\hat{y}^{(\text{size})}_i$ (rank reliability of size predictions).
% \end{itemize}
% This directly measures whether scaling-based rank order persists under realistic ecosystem shifts.
% To quantify how much of marginal variance is attributable to \emph{all size-related terms} (fixed slope plus random slopes), define the size-linked component conditional on the distribution of $x$:
% \[
% \mathrm{Var}_{\text{size}}(y)
% \;\equiv\;
% \mathrm{Var}\!\Big(\beta x + \sum_{S} v^S_{g_S} x\Big).
% \]
% Under independence between $x$ and the random slopes and $\mathbb{E}[x]=0$ (centered covariate), this becomes
% \[
% \mathrm{Var}_{\text{size}}(y)
% =
% \beta^2\mathrm{Var}(x) + \sum_{S}\sigma^2_{S,1}\mathrm{Var}(x).
% \]
% The corresponding \emph{size-linked share of marginal variance} is then
% \begin{equation}
% \Omega_{\text{size-all}}
% \;\equiv\;
% \frac{\beta^2\mathrm{Var}(x) + \sum_{S}\sigma^2_{S,1}\mathrm{Var}(x)}
% {\beta^2\mathrm{Var}(x) + \sum_{S}\sigma^2_{S,1}\mathrm{Var}(x) + \sum_{S}\sigma^2_{S,0} + \sigma^2_\varepsilon }.
% \label{eq:omega_size_all}
% \end{equation}
% We emphasize that $\Omega_{\text{size-all}}$ answers a different question than $\Omega_x$: it includes \emph{heterogeneous scaling} effects (random slopes) and therefore reflects both generalizable scaling and ecosystem-specific slope distortions.
% % \paragraph{Centering and interpretability.}
% % All formulas above assume $x$ is centered (e.g., $x\leftarrow x-\bar{x}$). Centering ensures that $\sigma^2_{S,0}$ truly represents baseline (at mean size) variability rather than absorbing slope effects.
% We report:
% \begin{enumerate}
% \item $\hat\beta$ with model-based and robust SE, and $R_\beta$.
% \item $\Omega_x$ and $\Omega_x^{(-B)}$ (size-driven rank-order potential overall and benchmark-independent).
% \item $\sigma^2_{\beta,\text{eff}}$ and $\mathrm{CV}_\beta$ (how much scaling varies across the ecosystem).
% \item $H_S$ for each facet/interactions (where scaling varies).
% \item OOS rank reliability $\rho_{\text{OOS}}$ under blocked CV (generalization under benchmark/contributor/deployment shift).
% \end{enumerate}
% These metrics separate three phenomena that are conflated in typical leaderboard analyses: (i) whether scale predicts performance on average, (ii) whether scale predicts \emph{rank ordering} robustly across benchmarks, and (iii) whether ``scaling laws'' are stable or conditional on ecosystem facets.
\input{tables/var_comp_vcov}
\begin{table}[h]
\centering
\caption{Variance Component Decomposition}
\label{tab:vcov_gstudies}
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}{llrr}
\toprule
Facet & Effect & Main & Free Corrs.\\
\midrule
A & $\sigma^2_{S,0}$ & 4.67 & 2.10\\
A & log\_nbpars & 14.06 & 14.08\\
AxC & $\sigma^2_{S,0}$ & 5.60 & 23.52\\
AxC & log\_nbpars & 6.61 & 24.47\\
AxD & $\sigma^2_{S,0}$ & 0.51 & 2.86\\
\addlinespace
AxD & log\_nbpars & 0.40 & 3.35\\
B & $\sigma^2_{S,0}$ & 104.21 & 105.35\\
B & log\_nbpars & 32.09 & 33.00\\
BxA & $\sigma^2_{S,0}$ & 6.15 & 5.56\\
BxA & log\_nbpars & 4.83 & 4.53\\
\addlinespace
BxAxC & $\sigma^2_{S,0}$ & 3.92 & 0.00\\
BxAxC & log\_nbpars & 7.20 & 10.34\\
BxAxD & $\sigma^2_{S,0}$ & 0.82 & 0.48\\
BxAxD & log\_nbpars & 0.90 & 0.52\\
BxC & $\sigma^2_{S,0}$ & 4.40 & 10.36\\
\addlinespace
BxC & log\_nbpars & 3.47 & 8.39\\
BxCxD & $\sigma^2_{S,0}$ & 0.50 & 0.63\\
BxCxD & log\_nbpars & 4.93 & 2.92\\
BxD & $\sigma^2_{S,0}$ & 11.65 & 8.02\\
BxD & log\_nbpars & 1.90 & 1.34\\
\addlinespace
C & $\sigma^2_{S,0}$ & 5.35 & 10.75\\
C & log\_nbpars & 17.75 & 38.53\\
CxAxD & $\sigma^2_{S,0}$ & 6.36 & 22.98\\
CxAxD & log\_nbpars & 7.42 & 19.06\\
CxD & $\sigma^2_{S,0}$ & 1.88 & 0.98\\
\addlinespace
CxD & log\_nbpars & 1.38 & 1.03\\
D & $\sigma^2_{S,0}$ & 0.00 & 1.86\\
D & log\_nbpars & 3.26 & 0.60\\
Residual & & 33.24 & 32.41\\
\bottomrule
\end{tabular}}
\vspace{0.25ex}
\caption*{\small Note: Variance components are reported as raw estimated random effect variances (Eq. \eqref{eq:gstudy_slopes}.}
\end{table}
% \begin{table}[h]
% \centering
% \caption{Variance Component Decomposition}
% \label{tab:vcov_gstudies}
% \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
% \begin{tabular}{llrrr}
% \toprule
% facet & var1 & base & main & full\\
% \midrule
% A & $\sigma^2_{S,0}$ & 16.78 & 4.67 & 2.10\\
% A & log\_nbpars & & 14.06 & 14.08\\
% AxC & $\sigma^2_{S,0}$ & 11.79 & 5.60 & 23.52\\
% AxC & log\_nbpars & & 6.61 & 24.47\\
% AxD & $\sigma^2_{S,0}$ & 0.44 & 0.51 & 2.86\\
% \addlinespace
% AxD & log\_nbpars & & 0.40 & 3.35\\
% B & $\sigma^2_{S,0}$ & 148.34 & 104.21 & 105.35\\
% B & log\_nbpars & & 32.09 & 33.00\\
% BxA & $\sigma^2_{S,0}$ & 10.11 & 6.15 & 5.56\\
% BxA & log\_nbpars & & 4.83 & 4.53\\
% \addlinespace
% BxAxC & $\sigma^2_{S,0}$ & 7.33 & 3.92 & 0.00\\
% BxAxC & log\_nbpars & & 7.20 & 10.34\\
% BxAxD & $\sigma^2_{S,0}$ & 1.16 & 0.82 & 0.48\\
% BxAxD & log\_nbpars & & 0.90 & 0.52\\
% BxC & $\sigma^2_{S,0}$ & 11.76 & 4.40 & 10.36\\
% \addlinespace
% BxC & log\_nbpars & & 3.47 & 8.39\\
% BxCxD & $\sigma^2_{S,0}$ & 2.10 & 0.50 & 0.63\\
% BxCxD & log\_nbpars & & 4.93 & 2.92\\
% BxD & $\sigma^2_{S,0}$ & 16.15 & 11.65 & 8.02\\
% BxD & log\_nbpars & & 1.90 & 1.34\\
% \addlinespace
% C & $\sigma^2_{S,0}$ & 29.92 & 5.35 & 10.75\\
% C & log\_nbpars & & 17.75 & 38.53\\
% CxAxD & $\sigma^2_{S,0}$ & 27.98 & 6.36 & 22.98\\
% CxAxD & log\_nbpars & & 7.42 & 19.06\\
% CxD & $\sigma^2_{S,0}$ & 2.90 & 1.88 & 0.98\\
% \addlinespace
% CxD & log\_nbpars & & 1.38 & 1.03\\
% D & $\sigma^2_{S,0}$ & 7.83 & 0.00 & 1.86\\
% D & log\_nbpars & & 3.26 & 0.60\\
% Residual & & 51.62 & 33.24 & 32.41\\
% \bottomrule
% \end{tabular}}
% \vspace{0.25ex}
% \caption*{\small Note: Variance components are reported as raw estimated random effect variances (Eq. \eqref{eq:gstudy_slopes}.}
% \end{table}
\section{Beyond Sum Scores: Modeling Latent Structure and its Covariates}\label{apx:latreg}
% Methods such as Confirmatory Factor Analysis (CFA) are powerful for testing hypotheses about the latent structure of abilities. However, they typically assume that observations—in our case, AI model submissions—are independent and identically distributed (i.id.). This assumption is violated in the AI benchmark ecosystem. The Open LLM Leaderboard, for example, is dominated by fine-tunes, merges, and quantized versions derived from a few foundational architectures (e.g., Llama, Mistral). This creates large pseudo-replication and a hierarchical dependence structure that, if ignored, can severely bias parameter estimates and artificially inflate statistical power, leading to spurious conclusions about the ecosystem's latent dimensionality. Prior work has sought to estimate these pseudo-replications separately yielding poorly defined and interpreted latent constructs \citep{kipnis_textttmetabench_2024,burnell_revealing_2023} which often mismatch known properties of benchmarks, e.g., \citeauthor{polo_sloth_2025} find that a construct of general knowledge is needed 8x more for Winogrande (commonsense pronoun resolution, \cite{sakaguchi_winogrande_2019}) than it is for MMLU Pro (general knowledge). As we will demonstrate, these shortcomings could be addressed through more sophisticated estimations.
% % including controlling for covariates, including .
% % Furthermore, a model's performance is not solely a function of its latent capabilities. It is also determined by observable characteristics such as its size, whether it has been instruction-tuned, and the specifics of its deployment. A comprehensive analysis must therefore not only identify the latent structure of abilities but also explain how that structure is shaped by these external factors. Simply regressing a manifest sum score on model size provides an incomplete and potentially misleading picture, as it confounds the effect of size on a general capability with its potentially distinct effects on specialized abilities.
% To address these challenges, this paper introduces a third, synthesizing method: a
% \paragraph{Hierarchical Mixed-Effects Latent Regression}. This approach provides a powerful fusion of the preceding methods by integrating the structural measurement model of CFA with the hierarchical variance decomposition of mixed-effects modeling. By specifying a regression model \emph{at the latent level}, we can simultaneously:
% \begin{enumerate}[leftmargin=*,nosep]
% \item Estimate the latent bifactor structure of abilities supported by our initial analyses.
% \item Model the hierarchical, non-independent data structure by including random effects for factors like the model's contributor (provenance).
% \item Quantify the precise influence of covariates like model size and tuning strategies on \emph{each specific latent ability} (i.e., the general factor and each benchmark-specific factor), thereby disentangling their effects.
% \end{enumerate}
% This allows us to move from asking ``what is the structure?'' to answering a more profound question: ``what are the systematic drivers of performance \emph{within} that latent structure?'' This provides a de-noised, structurally-informed view of scaling laws and the impact of different development choices, offering a far more nuanced understanding of what truly drives model performance.
\subsection{Hierarchical Mixed-Effects Latent Regression}\label{sec:latent_regression}
\subsubsection{Model Specification}
To simultaneously model the latent dimensional structure, the hierarchical dependencies in the data, and the influence of external covariates, we employ a Hierarchical Mixed-Effects Latent Regression Model. This model extends the bifactor CFA structure by treating the latent traits themselves as outcomes in a mixed-effects regression. This allows us to parse the variance in each latent ability into components attributable to fixed effects (like model size), random effects (like contributor-specific variance), and residual unexplained variance. The model is composed of two interconnected components: a measurement model (Eq. \eqref{eq:irt}) that links observed item responses to latent traits, and a structural model (Eq. \eqref{eq:structural}) that explains the variance in those latent traits.
% \subsection{Method 3: Hierarchical mixed latent regression with bifactor measurement}
\subsubsection{Bifactor IRT measurement with mixed-effects latent regression}\label{sec:method3_latent_mixed}
To support readers who may be more comfortable with more explicit notation, we redefine the relationships of ~\eqref{eq:irt} and ~\eqref{eq:structural}. Let $y_{ij}\in\{0,1\}$ denote the response of model submission $i\in\{1,\dots,N\}$ to item $j\in\{1,\dots,p\}$. Items are partitioned into benchmarks $k(j)\in\{1,\dots,K\}$. We use a bifactor measurement model with one general factor and $K$ benchmark-specific factors. Let the latent ability vector be
\begin{align}
\boldsymbol{\theta}_i \;=\;
\begin{bmatrix}
\theta_{i0} \\ \theta_{i1} \\ \vdots \\ \theta_{iK}
\end{bmatrix}
\in\mathbb{R}^{K+1},
\;
\theta_{i0}
\text{ = general ability},\;\; \theta_{ik} \text{ = benchmark-specific ability}.
\end{align}
For item $j$, define discrimination parameters $(a_{j0},a_{j,k(j)})$ and difficulty $b_j$. Under a 2PL bifactor IRT model in ~\eqref{eq:irt},
\begin{equation*}
\Pr(y_{ij}=1 \mid \boldsymbol{\theta}_i)
\;=\;
\sigma\!\left(
a_{j0}\theta_{i0} + a_{j,k(j)}\theta_{i,k(j)} - b_j
\right),
% \label{eq:irt}
\end{equation*}
where $\sigma(t)=(1+e^{-t})^{-1}$. %This measurement layer is identical in spirit to Method~1’s bifactor CFA, but now expressed in an item-level likelihood appropriate for binary responses.
The key extension is a \emph{mixed-effects regression} for each latent dimension that accounts for ecosystem clustering and covariates. Let $x_i=\log_{10}(\#\text{params})$ and let $\mathbf{w}_i$ collect deployment covariates (e.g., \texttt{type}, \texttt{chat\_template}, \texttt{mo\_e}). Let $c(i)$ denote the contributor/provenance label used in Method~2. We can reformulate the model latent abilities in ~\eqref{eq:structural} as vectors
\begin{equation}
\boldsymbol{\theta}_i
\;=\;
\mathbf{B}\mathbf{z}_i \;+\; \mathbf{U}\mathbf{u}_{c(i)} \;+\; \boldsymbol{\varepsilon}_i,
\qquad
\mathbf{z}_i=\begin{bmatrix}1 & x_i & \mathbf{w}_i^\top\end{bmatrix}^\top,
\label{eq:latent_mixed_reg}
\end{equation}
where $\mathbf{B}\in\mathbb{R}^{(K+1)\times (2+d)}$ are fixed effects (including the scaling slope for each latent dimension), $\mathbf{u}_{c}\sim\mathcal{N}(\mathbf{0},\boldsymbol{\Sigma}_u)$ are contributor-level random effects, and $\boldsymbol{\varepsilon}_i\sim\mathcal{N}(\mathbf{0},\boldsymbol{\Sigma}_\varepsilon)$ captures remaining between-submission latent variation. (Additional random effects for architecture or deployment can be incorporated analogously; in our main specification we include contributor as the dominant non-i.i.d.\ source, consistent with Method~2.)
% \paragraph{Measurement Component.} We adopt an Item Response Theory (IRT) framework, which is a specific form of CFA for dichotomous data. The probability of model $i$ correctly answering item $j$ is given by a 2-Parameter Logistic (2PL) model:
% \begin{equation}
% P(y_{ij} = 1 | \boldsymbol{\theta}_i, \mathbf{a}_j, d_j) = \sigma(\mathbf{a}_j^\top \boldsymbol{\theta}_i + d_j),
% \end{equation}
% where $\sigma(x) = (1+e^{-x})^{-1}$ is the logistic function. Here, $\boldsymbol{\theta}_i$ is the vector of latent ability scores for model $i$, $\mathbf{a}_j$ is a vector of item discrimination parameters specifying how strongly item $j$ relates to each latent factor, and $d_j$ is a scalar related to the item's difficulty. The sparsity pattern of $\mathbf{a}_j$ defines the factor structure; for our bifactor model, each item loads on the general factor and one specific factor.
% \paragraph{Structural Component.} The core innovation in our lies in modeling the latent trait vector $\boldsymbol{\theta}_i$ for each model $i$ as the outcome of a multivariate mixed-effects linear model:
% \begin{equation} \label{eq:latent_regression}
% \boldsymbol{\theta}_i = \boldsymbol{\Gamma} \mathbf{v}_i + \boldsymbol{\Lambda}_{\zeta} \boldsymbol{\zeta}_{c(i)} + \boldsymbol{\epsilon}_i,
% \end{equation}
% where:
% \begin{itemize}[leftmargin=*,nosep]
% \item $\boldsymbol{\theta}_i \in \mathbb{R}^K$ is the vector of $K$ latent trait scores for model $i$ (e.g., for a bifactor model with 6 benchmarks, $K=7$, corresponding to $g, s_1, \dots, s_6$).
% \item $\boldsymbol{\Gamma} \in \mathbb{R}^{K \times F}$ is a matrix of fixed-effect coefficients, where $F$ is the number of fixed covariates. Each row of $\boldsymbol{\Gamma}$ contains the regression coefficients for one latent trait.
% \item $\mathbf{v}_i \in \mathbb{R}^F$ is the vector of fixed covariates for model $i$, such as \texttt{log\_nbpars}, and indicators for \texttt{type}, \texttt{chat\_template}, etc.
% \item $\boldsymbol{\Lambda}_{\zeta} \in \mathbb{R}^{K \times Q}$ is the factor loading matrix for the random effects, where $Q$ is the number of random effects. In our case, with a random intercept per contributor, it is a $K \times K$ diagonal matrix.
% \item $\boldsymbol{\zeta}_{c(i)} \in \mathbb{R}^Q$ is the vector of random effects for the contributor $c(i)$ associated with model $i$. We assume $\boldsymbol{\zeta}_{c(i)} \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Psi})$, where $\boldsymbol{\Psi}$ is the covariance matrix of the random effects. This term captures systematic, unobserved variance attributable to model provenance.
% \item $\boldsymbol{\epsilon}_i \in \mathbb{R}^K$ is the vector of residual latent trait scores for model $i$, assumed to be $\boldsymbol{\epsilon}_i \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Sigma}_{\epsilon})$.
% \end{itemize}
% % This formulation allows us to estimate, for instance, the effect of model size ($\log(\text{params})$) on the general factor ($\theta_g$) and the math-specific factor ($\theta_{\text{math}}$) simultaneously via the respective coefficients in $\mathbf{\Gamma}$, while controlling for the unobserved cluster-level heterogeneity associated with the contributor via $\boldsymbol{\zeta}$.
% \subsubsection{Model Specification}
% We employ a hierarchical latent regression model, a form of explanatory Item Response Theory (IRT), built upon the bifactor structure identified in Method 1. The model has two primary components: a measurement model linking items to latent traits, and a structural model explaining those latent traits.
% \paragraph{Measurement Model.} For an observation from model $i$ on item $j$, we use a 2-Parameter Logistic (2PL) model. The probability of a correct response is:
% \begin{equation}
% P(y_{ij}=1 \mid \boldsymbol{\theta}_i, \mathbf{a}_j, d_j) = \frac{1}{1 + \exp\left(-\left(\mathbf{a}_j^\top \boldsymbol{\theta}_i - d_j\right)\right)},
% \end{equation}
% where $\boldsymbol{\theta}_i \in \mathbb{R}^K$ is the vector of $K$ latent abilities for model $i$ (one general factor, $K-1$ specific factors), $\mathbf{a}_j \in \mathbb{R}^K$ is the vector of item discrimination parameters (loadings), and $d_j$ is the item difficulty parameter. The structure of $\mathbf{a}_j$ is constrained according to the bifactor model: each item loads on the general factor and at most one specific factor.
% \paragraph{Structural Model.} The core of this method lies in modeling the latent trait vector $\boldsymbol{\theta}_i$ as an outcome. We decompose the matrix of latent traits for all $N$ models, $\boldsymbol{\Theta} \in \mathbb{R}^{N \times K}$, into fixed and random components:
% \begin{equation} \label{eq:latent_decomp}
% \boldsymbol{\Theta} = \mathbf{V}\mathbf{\Gamma} + \mathbf{W}\boldsymbol{\zeta} + \mathbf{E},
% \end{equation}
% where:
% \begin{itemize}[leftmargin=*,nosep]
% \item $\mathbf{V}$ is an $N \times F$ design matrix of $F$ fixed-effect covariates for each model (e.g., log-parameters, deployment type, use of a chat template).
% \item $\mathbf{\Gamma}$ is the $F \times K$ matrix of fixed-effect regression coefficients. Each column $\boldsymbol{\gamma}_k$ contains the coefficients predicting the $k$-th latent ability. For example, $\Gamma_{fk}$ is the effect of covariate $f$ on latent ability $k$.
% \item $\mathbf{W}$ is an $N \times R$ design matrix for the random effects, indicating group membership (e.g., which contributor submitted the model).
% \item $\boldsymbol{\zeta}$ is an $R \times K$ matrix of random effects, assumed to follow a multivariate normal distribution $\text{vec}(\boldsymbol{\zeta}) \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Sigma}_\zeta)$, where $\boldsymbol{\Sigma}_\zeta$ is a block-diagonal covariance matrix modeling the variance and covariance of abilities attributable to each random grouping factor.
% \item $\mathbf{E}$ is the $N \times K$ matrix of model-specific residual latent abilities, with rows assumed $\mathbf{e}_i \sim \mathcal{N}(\mathbf{0}, \boldsymbol{\Sigma}_E)$.
% \end{itemize}
% This formulation allows us to estimate, for instance, the effect of model size ($\log(\text{params})$) on the general factor ($\theta_g$) and the math-specific factor ($\theta_{\text{math}}$) simultaneously via the respective coefficients in $\mathbf{\Gamma}$, while controlling for the unobserved cluster-level heterogeneity associated with the contributor via $\boldsymbol{\zeta}$.
% \subsubsection{Estimation and Inference}
% The composite model is a high-dimensional latent variable model that generally cannot be estimated with standard maximum likelihood. We employ a Metropolis-Hastings Robbins-Monro (MH-RM) algorithm \citep{cai_metropolis-hastings_2010,chalmers_mirt_2012}. This stochastic expectation-maximization algorithm is designed for complex IRT models and can efficiently handle the high dimensionality of our item set and the inclusion of mixed effects.
% To ensure robustness, we embed this estimation within our meta-analytic bootstrap-by-item design ($B=500$). In each bootstrap replication, we fit both the standard bifactor CFA and the full latent regression bifactor model. We then perform a likelihood ratio test to formally assess whether the inclusion of the covariates and random effects in the latent regression provides a statistically significant improvement in model fit over the simpler CFA. This allows us to confirm that the additional complexity is empirically justified by the data.
% In each bootstrap replication, we:
% \begin{enumerate}[leftmargin=*,nosep]
% \item Fit both the latent regression bifactor model and the corresponding non-regression bifactor model from Method 1.
% \item Test for the significance of the added regression components using a likelihood ratio test, which confirmed the superiority of the regression model in 100\% of replications.
% \item Extract the fixed-effect coefficients ($\hat{\mathbf{\Gamma}}$) and their standard errors for each latent ability.
% \item Estimate the latent ability scores ($\hat{\boldsymbol{\Theta}}$) for each model under both the regression and non-regression specifications.
% \end{enumerate}
% Final estimates for the fixed effects in $\boldsymbol{\Gamma}$ and the variance components in $\boldsymbol{\Psi}$ are aggregated across the bootstrap replications. Point estimates are taken as the median of the bootstrap distribution, and confidence intervals are constructed from the corresponding quantiles, providing robust inference that accounts for uncertainty from item sampling. Fixed-effect estimates and latent scores are aggregated across bootstraps using inverse-variance weighting to produce robust point estimates and confidence intervals.
% \subsubsection{Estimation and Inference}
% We estimate the parameters of this complex model using a Metropolis-Hastings Robbins-Monro (MH-RM) algorithm, a stochastic expectation-maximization method well-suited for high-dimensional latent variable models. To ensure stability and robustness of our findings, we embed this estimation within a bootstrap-by-item procedure ($B=500$).
% Final fixed-effect estimates and latent scores are aggregated across bootstraps using inverse-variance weighting to produce robust point estimates and confidence intervals.
% \subsection{Results: Decontaminated Scaling Laws and the Structure of Provenance}
% By regressing the latent abilities on model characteristics, we can disentangle the true effect of scaling from the confounding influences of benchmark structure and model provenance. Our analysis confirms that the latent regression model provides a significantly better fit to the data than a standard bifactor model in 100\% of bootstrap replications, validating its use. The results reveal a far more nuanced picture of scaling laws than is visible from manifest leaderboard scores.
% \subsubsection{Noise-Controlled Scaling Laws: Not All Abilities Scale Equally}
% Figure \ref{fig:scaling_law} presents the ``decontaminated'' scaling laws for the general factor and each specific ability, estimated via the fixed effect of log-parameters in the latent regression.
% \paragraph{General Factor and General Knowledge.} The effect of model size on the general factor ($g$) is strong and positive, confirming that larger models possess a greater general capability. The slope of this relationship in the latent regression is substantially steeper than that observed for the manifest leaderboard average. This indicates that raw leaderboard scores, by conflating signal with noise from contributor effects and other sources, systematically \emph{underestimate} the true impact of scale on general ability. Similarly, the specific factor for MMLU-Pro (general knowledge) exhibits a strong positive scaling effect, even after accounting for the general factor. This suggests that factual recall and application is an ability that benefits disproportionately from increased model size.
% \paragraph{Weak Scaling for Complex Reasoning.} In stark contrast, the specific abilities for complex reasoning tasks—GPQA, BBH, and MATH—show markedly weaker, albeit still statistically significant, positive scaling. The practical implication is profound: while larger models do get slightly better at these tasks, the gains are marginal compared to the improvements in general knowledge. Simply increasing model size is an inefficient strategy for advancing these specific reasoning capabilities. The strong performance gains seen on leaderboards for these benchmarks are likely driven more by improvements in the general factor ($g$) than by targeted enhancements to the specific reasoning skills they purport to measure.
% \paragraph{Instruction Following and a Scaling Trade-Off.} Most strikingly, our analysis reveals two critical structural insights:
% \begin{enumerate}[leftmargin=*,nosep]
% \item \textbf{Instruction-following (IFEVAL) does not scale with size.} The coefficient for \texttt{log\_nbpars} on the IFEVAL-specific factor is not statistically different from zero. This capability appears to be a product of specific alignment tuning techniques, not an emergent property of scale.
% \item \textbf{A potential trade-off with so-called ``soft reasoning'' (MUSR).} The MUSR-specific factor shows a mild but statistically significant \emph{negative} relationship with model size. This suggests a structural trade-off: the very training methodologies that are correlated with scaling and that excel on other benchmarks may be subtly detrimental to the flexible, non-literal reasoning required by MUSR. An exploratory analysis of the \texttt{chat\_template} covariate's unstable relationship with MUSR across bootstraps provides a clue: as models become more rigidly aligned to follow instructions, they may lose some capacity for the creative interpretation that some reasoning tasks require.
% \end{enumerate}
\subsubsection{Estimation and evaluation protocol}
We estimate~\eqref{eq:irt}--\eqref{eq:structural}/\eqref{eq:latent_mixed_reg} with the Metropolis-Hastings Robbins-Monro (MH-RM) algorithm \citep{cai_high-dimensional_2010,cai_metropolis-hastings_2010,chalmers_extended_2015}, implemented in \texttt{mirt} \citep{chalmers_mirt_2012}. We fit (i) a baseline bifactor IRT model (measurement only) and (ii) the latent mixed regression bifactor (measurement + covariates + random effect slopes). To ensure conclusions are not driven by a particular subset of items, we repeat estimation across $B=500$ item bootstraps (sampling a fixed number of items per benchmark as in Method~1).
Within each bootstrap, we extract the fixed-effect coefficients ($\hat{\mathbf{\Gamma}}$) the latent ability scores ($\hat{\boldsymbol{\Theta}}$), and their standard errors for each model under both the regression and non-regression specifications, and evaluate:
\begin{enumerate}[leftmargin=*,nosep]
\item \textbf{Model comparability:} likelihood-based comparison between (i) and (ii) to verify that adding covariates/random effects improves fit without distorting the bifactor measurement structure.
\item \textbf{Noise-controlled scaling slopes:} estimates $\{\hat{\beta}_d\}_{d=0}^K$ and their uncertainty, aggregated across bootstraps via inverse-variance weighting.
\item \textbf{Provenance variance:} $\boldsymbol{\Sigma}_u$ as the contribution of contributor/provenance to latent variability after controlling for size and deployment.
\item \textbf{Stability diagnostics:} bootstrap distributions of $\hat{\beta}_d$ and (when applicable) $\hat{\mathcal{R}}_d$.
\end{enumerate}
\subsubsection{Reliability of scaling effects under pseudo-replication}
The object of interest is the scaling law slope on a latent dimension $d\in\{0,\dots,K\}$,
$
\beta_d \;\equiv\; (\mathbf{B})_{d,\;x},
$
and its reliability. As with all aggregations in this study of estimated parameters, inverse squared-standard error weighted means provide the precision-weighted average: $w_i = 1 / \operatorname{SE} (\beta_{di})^2$. % under clustered submissions. % We report two complementary quantities.
% First, a \emph{design-aware effective sample size} that downweights pseudo-replication due to contributor clustering:
% \begin{equation}
% N_{\mathrm{eff}}(d)
% \;\equiv\;
% \frac{\left(\sum_{c} n_c\right)^2}{\sum_{c} n_c^2}\cdot \kappa_d,
% \label{eq:neff}
% \end{equation}
% where $n_c$ is the number of submissions from contributor $c$ and $\kappa_d\in(0,1]$ is the fraction of residual latent variance not absorbed by the contributor random effect on dimension $d$:
% \[
% \kappa_d \;=\; \frac{(\boldsymbol{\Sigma}_\varepsilon)_{dd}}{(\boldsymbol{\Sigma}_\varepsilon)_{dd}+(\boldsymbol{\Sigma}_u)_{dd}}.
% \]
% This provides an interpretable scalar: how many \emph{independent} submissions the ecosystem effectively contains for estimating scaling on dimension $d$.
We quantify slope stability with a \emph{random-slope extension}:
\begin{equation}
\theta_{id}
=
\alpha_d + \beta_d x_i + \mathbf{w}_i^\top\boldsymbol{\gamma}_d + u_{c(i),d} + v_{c(i),d}x_i + \varepsilon_{id},
\label{eq:latent_random_slope}
\end{equation}
with $\mathrm{Var}(v_{\cdot,d})=\tau_d^2$, the weighted variance of the dimension. We then report a \emph{slope reliability index} in Table ~\ref{tab:lreg_reliab}:
\begin{equation}
\mathcal{R}_d
\;\equiv\;
\frac{\beta_d^2}{\beta_d^2+\tau_d^2},
\label{eq:slope_reliability}
\end{equation}
% which is high when the ecosystem exhibits a coherent scaling slope on dimension $d$ and low when “scaling” is primarily contributor-dependent. When random slopes are not included, $\tau_d^2=0$ by construction and $\mathcal{R}_d=1$; in that case stability must be assessed via bootstrapping and posterior predictive checks, using
% \input{tables/slope_reliability}
\subsection{Results and discussion (Method 3): noise-controlled scaling laws differ by latent ability}
\label{sec:results_method3}
% \subsubsection{General factor exhibits the strongest and most reliable scaling}
% Across bootstraps, the fixed effect of $\log$-parameters on the general factor, $\hat{\beta}_0$, is consistently positive and the largest in magnitude among all dimensions. This indicates that much of the leaderboard’s apparent “scaling law” operates through a broad capability that explains shared covariance across benchmarks (Method~1), rather than through benchmark-specific skills. Practically, this means that aggregated leaderboard scaling curves primarily reflect improvements along the latent $g$ axis.
% This finding resolves an apparent paradox: prior PCA-style analyses report a strong first component that scales with size, but cannot determine whether scaling improves “reasoning,” “knowledge,” or “instruction following.” Method~3 shows that scaling acts most strongly on the general factor that cuts across tasks.
% \subsubsection{Benchmark-specific scaling slopes shrink after controlling for deployment and provenance}
% When we regress \emph{latent} benchmark-specific factors on size and deployment (Eq.~\eqref{eq:latent_mixed_reg}), the size slope for several benchmarks is substantially smaller than the slope obtained by regressing raw benchmark scores on size. This is expected from Proposition~\ref{prop:latent_reg_advantage}: manifest benchmark scores mix $g$ with benchmark-specific variance and benchmark-dependent measurement error, inflating or attenuating slopes in benchmark-specific ways.
% Empirically, we observe:
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{General knowledge (MMLU-Pro):} retains a strong positive benchmark-specific slope even after controlling for $g$, suggesting size improves knowledge beyond what is captured by the general factor.
% \item \textbf{Hard QA / expert reasoning (GPQA) and BBH:} benchmark-specific slopes are near zero (or weakly positive) after denoising, indicating that raw-score scaling is largely mediated by the general factor and by correlated deployment/provenance effects.
% \item \textbf{Instruction following (IFEval):} exhibits little to no size slope after controlling for deployment (e.g., chat templating), consistent with the hypothesis that instruction-following gains are driven more by alignment/tuning than parameter count.
% \item \textbf{Soft reasoning (MuSR):} shows an unstable or mildly negative benchmark-specific slope under full controls, suggesting a trade-off: the deployment strategies that improve other dimensions may not improve (and can slightly degrade) this latent ability.
% \end{itemize}
% \subsubsection{Contributor/provenance remains a nontrivial source of latent variation after controls}
% Even with item-level modeling and covariate adjustment, the contributor random effect explains a measurable share of latent variance (on the order reported in the manuscript notes). We emphasize interpretation: this is \emph{signal about unobserved implementation factors} (data mixtures, compute budget, alignment recipes) and \emph{noise only relative to the question “what does architecture/size explain?”}. In other words, provenance variance quantifies the extent to which scaling curves estimated from leaderboard submissions conflate size with who trains and packages models.
\subsubsection{Quantifying Provenance: Training Signal versus Measurement Noise}
A key advantage of the mixed-effects model is its ability to quantify the variance attributable to unobserved, systematic differences between contributors. Even after controlling for item effects, model size, and deployment type, in \eqref{eq:irt}-\eqref{eq:structural}, the inverse-error weighted median random effect for \texttt{contributor} still accounts for 3.0\% of the variance in latent ability scores.
This ``provenance variance'' is not merely measurement noise. It represents a significant signal about the efficacy of unobserved training methods, data mixtures, and resource allocation associated with different model providers. From the perspective of evaluating a base architecture (e.g., assessing \texttt{Llama-3-70B}'s inherent capabilities), this contributor-specific ``secret sauce'' is a confounding factor. However, from the perspective of advancing the field, this variance is a critical indicator of where the most effective (and often proprietary) training innovations lie. Our model successfully isolates this signal, allowing for a clearer interpretation of both architectural capabilities and the impact of training methodology.
% \begin{corollary}[When provenance variance is large, scaling laws are ecosystem-dependent]
% \label{cor:scaling_ecosystem_dependent}
% If $(\boldsymbol{\Sigma}_u)_{dd}$ is large relative to $(\boldsymbol{\Sigma}_\varepsilon)_{dd}$ on dimension $d$, then $N_{\mathrm{eff}}(d)$ in~\eqref{eq:neff} is small, and the uncertainty on $\beta_d$ is dominated by between-contributor variation rather than by within-contributor replication. Consequently, “scaling laws” estimated from such ecosystems generalize poorly to new contributors or training pipelines.
% \end{corollary}
\subsubsection{Significance: scaling is not a single law but a vector over latent abilities}
Method~3 reframes “does performance scale with size?” into a dimension-specific statement:
% \[
$\boldsymbol{\beta} \;=\; (\beta_0,\beta_1,\dots,\beta_K)$
% \]
is a \emph{scaling vector} over latent abilities, not a scalar property of a benchmark average. The observed pattern---strong scaling for $g$, heterogeneous and sometimes negligible scaling for benchmark-specific factors, and persistent provenance variance---explains why leaderboard-based scaling curves can be simultaneously predictive (for aggregate scores) and misleading (for capability-specific claims).
Practically, this implies that benchmark ecosystems should report, at minimum, (i) a general-factor scaling curve, (ii) benchmark-specific residual scaling curves after removing $g$, and (iii) a provenance-adjusted uncertainty estimate (e.g., via $N_{\mathrm{eff}}$ or random-slope reliability $\mathcal{R}_d$). Without these, apparent “laws” may reflect measurement structure and ecosystem composition rather than transferable properties of model scaling.
% \textbf{Introduction Section (Motivation for Method 3)}
% The preceding analyses establish two fundamental properties of the AI benchmark ecosystem. First, by meta-analyzing confirmatory factor models (Method 1), we demonstrated that the ecosystem’s latent structure is not a simple collection of independent modules but is better described by a bifactor model, comprising a strong general capability factor ($g$) and subordinate, benchmark-specific factors. Second, using a generalizability-theoretic variance decomposition (Method 2), we quantified the sources of score variation, revealing that benchmark identity, unobserved contributor practices, and model size are dominant drivers of performance on manifest scores.
% However, these two perspectives remain disconnected. The structural analysis of Method 1, while providing a map of latent abilities, treats each model submission as an independent observation. This assumption is severely violated in an ecosystem characterized by rampant pseudo-replication, where thousands of leaderboard entries are minor variations (fine-tunes, quantizations) of a few base architectures. This inflates statistical power and risks biasing the estimated latent structure toward the idiosyncrasies of dominant model families. Conversely, the variance decomposition of Method 2, while correctly modeling the nested data structure, operates on manifest sum scores. These scores are themselves confounded mixtures of the latent abilities identified in Method 1, meaning we can diagnose the sources of variance in a composite score but cannot attribute that variance to the underlying constructs (e.g., "how much does model size improve the *general factor* versus the *math-specific* factor?").
% To overcome these limitations and synthesize our findings, we introduced (and reintroduce here) our third and final method: \textbf{Latent Mixed-Effects Regression}. This approach unifies the structural insights of CFA with the variance-analytic power of mixed-effects modeling. By treating the latent abilities themselves as outcomes in a regression model, we can simultaneously:
% \begin{enumerate}[leftmargin=*,nosep]
% \item Estimate the latent bifactor structure while explicitly modeling the non-independence of model submissions via random effects (e.g., for contributor/provenance).
% \item Decompose the variance of each latent factor—both general and specific—into components attributable to observable model characteristics like size, architecture, and tuning methodology.
% \item Obtain ``denoised'' estimates of scaling laws that quantify the relationship between model size and a specific latent capability, controlling for other ecosystem factors.
% \end{enumerate}
% This unified model allows us to move beyond describing the ecosystem's structure and variance to explaining how developer choices and model properties shape the latent capabilities we aim to measure, providing the most rigorous and actionable insights into the dynamics of AI progress.
% ---
% \textbf{Method 3: Hierarchical Mixed-Effects Latent Regression}
% \label{sec:latent_regression_method}
% \subsubsection{Motivation: Unifying Structure and Source}
% Our first two methods reveal a critical tension: the latent structure of abilities is multidimensional (Method 1), but the variance in observed scores is dominated by ecosystem factors that are not directly tied to these constructs (Method 2). To bridge this gap, we require a model that can explain variation in the latent abilities themselves. Furthermore, the significant pseudo-replication in the data (e.g., many fine-tunes of one base model) violates the independence assumption of standard CFA, leading to artificially deflated standard errors and overly confident conclusions. A latent regression model with mixed effects directly addresses both challenges by incorporating covariates to explain latent trait variance and random effects to model the clustered nature of the data.
\section{Code Listings}\label{apx:code}
The bifactor model definitions are included here as well as methods 2 and 3 from the paper. All other code containing presented statistical modeling and estimations can be found in the online repository.\footnote{\url{https://github.com/hardy-education/ai_bench_cfa}}
\subsection{Representative CFA Model: Bifactor Structure}
The bifactor model is our recommended structure. Below is the \texttt{lavaan} syntax generator. Other model specifications (unidimensional, correlated factors, hierarchical) follow similar patterns and are available in our code repository.
% \begin{listing}\caption{\texttt{lavaan} Bifactor Model Syntax}
\begin{lstlisting}[language=R, caption=\texttt{lavaan} Bifactor Model Syntax, showstringspaces=false]
generate_bifact_syntax <- function(item_map) {
benchmarks <- unique(item_map$benchmark)
items <- item_map$item_full
# General factor loads on all items
general_spec <- paste0("general =~ ", paste(items, collapse = " + "))
# Specific factors for each benchmark
specific_specs <- map_chr(benchmarks, function(b) {
items_b <- item_map %>% filter(benchmark == b) %>% pull(item_full)
paste0(b, " =~ ", paste(items_b, collapse = " + "))
})
# Orthogonalize: general uncorrelated with specifics
orthog_general <- map_chr(benchmarks, function(b) {
paste0("general ~~ 0*", b)
})
# Orthogonalize specifics with each other
orthog_specific <- combn(benchmarks, 2, function(pair) {
paste0(pair[1], " ~~ 0*", pair[2])
}, simplify = FALSE) %>% unlist()
paste0("# Bifactor Model\n",
general_spec, "\n",
paste(specific_specs, collapse = "\n"), "\n",
paste(orthog_general, collapse = "\n"), "\n",
paste(orthog_specific, collapse = "\n"), "\n")
}
\end{lstlisting}\label{code:lavaan_bifact}
% \end{listing}
\subsection{Bifactor via MH-RM and Latent Regression}
Below is the \texttt{mirt} syntax generator for the bifactor. Other model specifications (unidimensional, correlated factors, hierarchical) follow similar patterns and are available in our code repository. Included in the code listing here is the MH-RM estimation.
% \begin{listing*}\caption{\texttt{mirt} Bifactor Model and Bifactor Latent Mixed Effect Regression Syntax and Estimation}
\begin{lstlisting}[language=R, caption=\texttt{mirt} Bifactor Model and Bifactor Latent Mixed Effect Regression Syntax and Estimation, showstringspaces=false]
#' @param benches Vector of benchmark names
#' @param n_items Number of items per benchmark
#' @return Character string for factor specification
generate_bifactor_string <- function(benches, n_items) {
item_nums <- seq_len(n_items)
bench_nums <- seq_along(benches)
factor_specs <- sapply(bench_nums, function(x) {
min_item <- min(item_nums) + (x - 1) * length(item_nums)
max_item <- max(item_nums) + (x - 1) * length(item_nums)
glue::glue("{benches[x]} = {min_item}-{max_item}")
})
general_str <- glue::glue("G = 1-{length(item_nums)}")
paste(general_str, factor_specs, collapse = "\n")
}
dat = sdf %>% select(-any_of(names(metadf))) # response matrix
covdat = sdf %>% select(any_of(names(metadf))) # covariates
# estimate bifactor model and latent regression bifactor
bifact = dat |>
mirt::mixedmirt(model = generate_bifactor_string(b,ni),
itemtype = "2PL", SE = T, fixed = ~ 0 + items,
accelerate = "squarem", GenRandomPars = TRUE,
technical = list(NCYCLES = 5000))
bifact_regression = dat |>
mirt::mixedmirt(model = generate_bifactor_string(b,ni),
covdata = covdat, itemtype = "2PL", SE = T,
fixed = ~ 0 + items, random = ~ 1|contrib,
lr.fixed = ~ type + chat_template + mo_e + log_nbpars,
accelerate = "squarem", GenRandomPars = TRUE,
technical = list(NCYCLES = 5000))
}
\end{lstlisting}\label{code:mirt_bifact}
% \end{listing*}
\subsection{Variance Decomposition Model}\label{app:facet_estimation}
The G-theory variance decomposition uses a fully-crossed random-effects model with random slopes for model size.
% \begin{listing*}\caption{Generalizability Study: Full Random-Slopes Model}
\begin{lstlisting}[language=R, caption=Generalizability Study: Full Random-Slopes Model, showstringspaces=false]
lmer(formula = val ~ 1
+ log_nbpars
+ (log_nbpars|bench) # (B) Benchmark
+ (log_nbpars|architecture:generation) # (A) Architecture
+ (log_nbpars|author:removed:not_avail) # (C) Contributor
+ (log_nbpars|type:chat_template:mo_e:merged:precision) # (D) Deployment and Tuning
+ (log_nbpars|bench:architecture:generation) # AxB
+ (log_nbpars|bench:author:removed:not_avail) # BxC
+ (log_nbpars|bench:type:chat_template:mo_e:merged:precision) # BxD
+ (log_nbpars|architecture:generation:author:removed:not_avail) # AxC
+ (log_nbpars|architecture:generation:type:chat_template:mo_e:merged:precision) # AxD
+ (log_nbpars|author:removed:not_avail: type:chat_template:mo_e:merged:precision) # CxD
+ (log_nbpars|bench:architecture:generation: type:chat_template:mo_e:merged:precision) # AxBxD
+ (log_nbpars|bench:author:removed:not_avail: type:chat_template:mo_e:merged:precision) # BxCxD
+ (log_nbpars|architecture:generation:author:removed:not_avail: type:chat_template:mo_e:merged:precision) # AxCxD
+ (log_nbpars|bench:architecture:generation:author:removed:not_avail), # AxBxC
data = df)
\end{lstlisting}
\label{code:gstudy_code}
% \end{listing*}
For the reduced model discussed in the main body of the paper, replacing the single \texttt{|}with \texttt{||} specifies uncorrelated random intercepts and slopes. Each facet (A, B, C, D) and their interactions contribute both intercept variance (baseline performance) and slope variance (scaling relationship heterogeneity).
\section{Extended Results and Discussion}
% \subsection{Which Structural Model Best Separates Signal from Noise?}
% \label{sec:results_method1}
% \paragraph{Bifactor-family structures optimally partition signal and noise}
% Across item-resampled replications, bifactor-family models achieve the best average covariance reproduction, indicating they most effectively separate general capability signal ($g$) from benchmark-specific noise. In the mixed-effects meta-analysis of robust/scaled RMSEA, the correlated bifactor structure achieves the lowest expected misfit, followed closely by the orthogonal bifactor. Correlated factors, hierarchical (second-order), and single-$g$ models fit materially worse, while independent benchmark factors perform worst—empirically rejecting the common implicit assumption that benchmarks are mutually exclusive ``modules.''
% \paragraph{Uncontrolled noise persists even under best structure} Despite traditional measures of model fit indicating potential overfit to the covariance structure, degree-one Lagrangian Multiplier tests reveal that on average none of these models is adequately capturing the true relationships between observed item responses from AI models. All latent structures underestimate true dependencies—residual noise remains even after accounting for $g$ and benchmark-specific factors.\footnote{The only relationship that is overestimated by all latent structures is that between the items IFEVAL and MUSR.}
% \paragraph{Interpretation.}
% The dominance of bifactor-family models supports a two-level decomposition of performance:
% (i) a strong cross-item general capability $g$ that explains the majority of shared covariance across the ecosystem, and (ii) benchmark-linked residual group structure capturing within-benchmark dependence not reducible to $g$ alone. This is consistent with prior PCA findings of a dominant first component, but CFA sharpens the claim: the first component is not merely variance-maximizing; it behaves as a \emph{general factor} that improves prediction of the observed tetrachoric dependence under explicit measurement constraints.
% \paragraph{Benchmark labels provide insufficient noise control}
% Randomized benchmark assignment substantially attenuates the apparent advantage of flexible structures, but does not eliminate it. Notably, even under random assignment, the correlated bifactor can achieve strikingly good fit—often surpassing less flexible models under correct assignment. This implies that benchmark metadata alone provides insufficient noise control: some of the correlated bifactor's fit advantage is attributable to generic flexibility in allocating residual dependence rather than to benchmark semantics per se.
% \paragraph{Practical implication (measurement).}
% If a model fits nearly as well when benchmark membership is permuted, then benchmark boundaries are not the primary organizing principle of the residual covariance once $g$ is accounted for. Said differently: individual items appear to carry dependency structure that is only weakly aligned with benchmark names. For benchmark ecosystems, this is a warning sign: “benchmark as construct” is frequently misspecified, and reported per-benchmark sum scores can mix multiple latent sources in ways that are not stable under plausible item perturbations.
% \paragraph{Best fit: (orthogonal) bifactor structure}
% Although correlated bifactor exhibits the best raw fit, the permutation analysis and out-of-sample meta-evaluations (reported later in the paper) suggest it is closer to an interpolation device: powerful but prone to absorbing item idiosyncrasies into cross-specific correlations. The orthogonal bifactor, while slightly less flexible, retains the key decomposition ($g$ plus within-benchmark specifics) and is more stable and interpretable as an ecosystem summary. We therefore adopt the orthogonal bifactor as the most defensible working structure for downstream analyses, while treating the correlated bifactor as an upper bound on achievable covariance fit.
% \paragraph{Localized diagnostics map systematic redundancy as measurable noise}
% Aggregated MI/SEPC diagnostics show that the best-fitting structures also tend to reduce extreme residual dependencies. In particular, bifactor-family models yield fewer and smaller large-magnitude SEPC residual covariances, indicating less unresolved local dependence—systematic noise—after conditioning on $g$ and a benchmark-specific factor.
% \paragraph{Redundancy inflates noise without adding signal}
% Persistently large $|\mathrm{SEPC}|$ for specific item pairs across bootstraps indicates conceptually duplicate or method-coupled items (e.g., templatic phrasing, shared artifacts, near-paraphrases). In benchmark ecosystems, such redundancy inflates apparent reliability without increasing construct coverage—it adds correlated noise, not signal. MI/SEPC thus provides an actionable mechanism for targeted noise reduction through \emph{benchmark slimming}: remove or rewrite items that repeatedly manifest large residual coupling ($|\mathrm{SEPC}| >$ 0.40), thereby increasing parsimony and signal-to-noise ratio.
% \paragraph{Significance: a structural critique of “sum score per benchmark” reporting}
% Method 1 yields a structural finding with direct consequences for evaluation practice:
% \begin{proposition}[Non-equivalence of benchmark totals under multidimensional measurement]
% If the true item dependence is better approximated by a bifactor model than by a benchmark-wise unidimensional model, then benchmark total scores are not sufficient statistics for latent capability profiles; models can be rank-equivalent on totals yet differ on $(g,s_1,\dots,s_K)$, inducing different generalization behavior across items and benchmarks.
% \end{proposition}
% The empirical advantage of bifactor-family structures implies that the ecosystem’s current reporting pipeline (per-benchmark totals and their average) discards latent structure that is both statistically detectable and practically relevant. The general factor $g$ suggests a pervasive ``global capability'' (e.g., broad training/data/scale effects) that drives much of leaderboard covariance, while the specific factors quantify what remains benchmark-unique after removing $g$. This disentangles ``being strong everywhere'' from ``being uniquely strong here,'' enabling sharper scientific questions: which benchmarks truly add information, which are largely redundant, and which items are structurally misassigned to their benchmark’s intended construct.
% Taken together, the meta-analytic bootstrap, permutation control, and MI-based local diagnostics provide convergent evidence that (i) unidimensional and independent-benchmark assumptions are untenable, (ii) a strong general factor is real and stable across item perturbations, and (iii) benchmark labels only partially explain residual structure—motivating the subsequent methods that explicitly decompose ecosystem variance and connect latent structure to sources of variation beyond the benchmarks themselves.
% \begin{figure*}[th]
% \centering
% \includegraphics[width=1\linewidth]{figures/scaling_law_estimates_robust.pdf}
% \caption{Noise-controlled Scaling Laws: Log-log of Performance by Number of Parameters. Shaded regions indicate the 95\% confidence bands for the linear relationship between x and y estimated by robust regression using an M estimator with Tukey's biweight \cite{venables_modern_2002}. Prior to log transformation of the y-axis, each estimated latent score $\Theta_k$ is converted from its assumed distribution to positive values via the inverse cumulative normal distribution function, $S_k= \mathbf{\Phi}^{-1}(\Theta_k)$ before all scores are min-max scaled to the range $s \in S_k : [1\dots100]$.}
% \label{fig:scaling_law}
% \end{figure*}
\subsection{Quantifying Noise Source Magnitudes}
% \subsubsection{Baseline Variance Components}
% Table \ref{tab:varcomp_base} shows the estimated variance components from the random-effects model without covariates. The total variance in normalized scores is 334.2. A striking 44.0\% of this variance is attributable to the Benchmark (B) main effect—predictable noise that can be controlled via benchmark difficulty normalization. The consistency of this difficulty ordering is high; treating models as ``raters'' of benchmark difficulty yields a G-coefficient of 0.74, indicating that the relative difficulty of these six benchmarks is highly stable across the model population.
% Beyond benchmark difficulty, the two largest sources of variance are the Contributor (C) main effect (9.0\%)—systematic noise from human practices—and the unclassified Residual error (15.4\%). Critically, contributor noise alone is larger than that of the Architecture (A) main effect (4.8\%) and the Deployment Type (D) main effect (2.3\%) combined. This reveals that human-introduced noise rivals model-intrinsic signal: metadata about who submitted a model and how it was submitted is a more significant predictor of performance than information about the model's foundational architecture or its deployment configuration. Significant interaction effects, such as Benchmark $\times$ Contributor (B$\times$C, 3.5\%) and Benchmark $\times$ Architecture (B$\times$A, 3.1\%), highlight that model performance is context-dependent; specific contributors and architectures show differential performance on specific benchmarks.
\subsubsection{Without Size Control, Signal Is Buried in Confounded Noise}\label{apx:size_control_implications}
Introducing model size (\texttt{log\_nbpars}) as a covariate dramatically clarifies the sources of variation, as shown in Table \ref{tab:vcov_gstudies}. The residual variance drops by 36\% (from 52.0 to 33.24 in the full model summary), indicating that model size explains a substantial portion of previously unexplained performance differences—without this control, architectural signal is confounded with scale.
We calculate \eqref{eq:omega_size_all} to be $\Omega_{\text{size-all}}=0.402$, suggesting that about 40\% of all observed variation is linked to model size.
% The total non-residual variance is now partitioned into variance of intercepts (score differences independent of size) and variance of slopes (differences in scaling with size). Summing all slope variance components reveals that \textit{model size accounts for 40\% of all explainable (non-residual) variance}. For ranking models on an aggregate score, where benchmark-specific difficulty is averaged out, this effect is even more pronounced: model size accounts for 66\% of the remaining explainable variance.
This confirms that, without explicit size control, aggregate rankings primarily reflect scale rather than architectural innovation; ``size is nearly all you need'' when noise dominates.
Moreover, the influence of other facets is reshaped by their relationship to model size. The Contributor facet (C) has the largest slope variance ($\sigma^2_{C, \text{0}}=17.75$) among all non-benchmark facets, indicating that the performance gap between different contributors widens significantly as model size increases. In contrast, the Architecture facet (A) has a larger intercept variance ($\sigma^2_{A, \text{intcpt}}=4.67$) than slope variance ($\sigma^2_{A, \text{slope}}=14.06$), though both are substantial. This suggests that while some architectures have a higher performance baseline independent of size, the primary differentiator remains how effectively they scale. The main effect of model size is strongly positive and significant, confirming that, on average, larger models perform better.
% \begin{table*}[h]
% \centering
% \caption{Variance Components with Model Size Covariate.}
% \label{tab:varcomp_size}
% \footnotesize
% \begin{tabular}{@{}llrrrr@{}}
% \toprule
% & & \multicolumn{2}{c}{\textbf{Intercept}} & \multicolumn{2}{c}{\textbf{Slope}} \\
% \cmidrule(lr){3-4} \cmidrule(lr){5-6}
% \textbf{Source} & & \textbf{$\sigma^2_{\text{intcpt}}$} & \textbf{\% Total} & \textbf{$\sigma^2_{\text{slope}}$} & \textbf{\% Total} \\ \midrule
% \multicolumn{6}{l}{\textit{Main Effects}} \\
% (A) Architecture & & 4.70 & 1.3\% & 12.79 & 3.5\% \\
% (B) Benchmark & & 103.40 & 28.5\% & 33.26 & 9.2\% \\
% (C) Contributor & & 7.74 & 2.1\% & 18.61 & 5.1\% \\
% (D) Deployment & & 0.00 & 0.0\% & 4.06 & 1.1\% \\
% \midrule
% \multicolumn{6}{l}{\textit{Interactions}} \\
% B $\times$ C & & 4.42 & 1.2\% & 3.58 & 1.0\% \\
% B $\times$ A & & 6.42 & 1.8\% & 4.53 & 1.2\% \\
% A $\times$ C & & 3.71 & 1.0\% & 3.97 & 1.1\% \\
% \multicolumn{2}{@{}l}{Other Interactions} & 17.96 & 4.9\% & 22.61 & 6.2\% \\ \midrule
% \multicolumn{2}{@{}l}{\textbf{Residual} ($\sigma^2_{\epsilon}$)} & 34.30 & 9.5\% & \multicolumn{2}{r}{---} \\
% \multicolumn{2}{@{}l}{\textbf{Total Explained}} & 148.35 & 40.9\% & 103.41 & 28.5\% \\
% \bottomrule
% \end{tabular}
% \\ \footnotesize \textit{Note: Total variance = 363.0. Percentages are relative to this total. Interaction terms are aggregated for brevity; full details are in Appendix \ref{app:facet_estimation}}
% \end{table*}
% \subsection{Variance Components}
% \label{subsec:results_vardecomp}
% We present the results of the variance decomposition, first for the model without covariates to establish a baseline, and second for the full model including model size to understand its pervasive influence.
% \paragraph{Benchmark difficulty dominates total score variance}
% In the fully-crossed decompositions, \eqref{eq:gstudy} and \eqref{eq:gstudy_slopes}, the benchmark main effect $\sigma^2_B$ accounts for nearly half of the total observed variance in per-benchmark scores ($\approx 46\%$). This indicates that, for a fixed evaluated model, a substantial portion of score differences is predictable from which benchmark is being taken. In reliability terms, benchmark difficulty ordering is itself stable: using the observed model sample, the implied reliability of benchmark difficulty ranking is high (reported $G_B\approx 0.74$), consistent with large $\sigma^2_B$ relative to benchmark-by-model interactions.
% \paragraph{Contributor noise dominates after controlling for benchmark difficulty}
% For analyses that aim to generalize \emph{across} benchmarks (i.e., focusing on benchmark-independent rank order), it is helpful to remove the benchmark main effect $B$ and attribute benchmark-interaction variance back to the corresponding facets. Under this view, the largest remaining components are:
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{Contributor/source noise ($C$):} variation attributable to contributor identity and availability metadata explains about $21.3\%$ of total variance (benchmark-independent interpretation). This is substantial: it implies that who submits (and associated submission properties captured by our proxy) is a major noise source—a larger determinant of aggregate standing than architecture or deployment details.
% \item \textbf{Residual error ($\varepsilon$):} also about $21.3\%$, reflecting remaining unmodeled differences at the finest granularity and measurement noise.
% \end{itemize}
% By comparison, variance attributable purely to model architecture (A) and deployment form (D) is smaller than the contributor noise in this dataset. Practically, this means leaderboard rankings can be highly sensitive to ecosystem noise sources that are not intrinsic to architecture choice.
% \subsubsection{Model size explains a large fraction of explained variance across benchmarks}
% Adding model size as a covariate and allowing its effect to vary across facets and interactions~\eqref{eq:gstudy_slopes} shows that a large fraction of systematic rank-order variance is size-linked. In the reported analysis, model size accounts for approximately $36\%$ of the \emph{explained} variance in rank order \emph{across benchmarks}; when benchmark main effects are set aside, roughly $66\%$ of the remaining cross-facet variance is associated with size. Including size reduces the residual variance by roughly one third, indicating that a meaningful amount of what would otherwise appear as unexplained noise is predictable from scale.
% \subsection{Implications for Benchmark Reliability: When Can Rankings Be Trusted?}
% These results point to a central measurement issue: a single leaderboard composite score conflates signal with at least three distinct noise sources:
% \begin{enumerate}
% \item \textbf{Benchmark difficulty (44\%):} predictable noise that can be controlled via normalization,
% \item \textbf{Contributor practices (9\%):} systematic noise from human-introduced factors,
% \item \textbf{Scale effects (36\% of explainable variance):} confounding that buries architectural signal without size control,
% \end{enumerate}
% with additional differential-performance interactions. Consequently, the reliability of \emph{model} rankings depends strongly on the intended generalization and which noise sources are controlled.
% \paragraph{Reliability depends on noise control strategy}
% Our G-theory framework reveals that benchmark ecosystem reliability is conditional:
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{For ranking by benchmark difficulty:} High reliability (G-coefficient = 0.74) when benchmarks are treated as the object of measurement.
% \item \textbf{For ranking models within-contributor:} Architectural differences (4.8\%) and deployment effects (2.3\%) provide signal, but only after controlling for benchmark difficulty and size.
% \item \textbf{For ranking models across-contributors:} Contributor noise (9\%) must be explicitly modeled; otherwise, rankings conflate architectural merit with submission practices.
% \item \textbf{For comparing architectures at different scales:} Size confounding (36\%) must be controlled; aggregate rankings without size adjustment primarily reflect scale, not innovation.
% \end{itemize}
% \paragraph{Uncontrolled aggregation systematically misleads}
% Without explicit noise modeling, benchmark aggregation can overstate architecture- or deployment-specific conclusions by mixing them with contributor noise and scale effects. For example, a ranking difference between two models might reflect: (1) genuine architectural superiority, (2) different submission practices, (3) scale differences, or (4) differential benchmark difficulty. Our variance decomposition quantifies each source, enabling practitioners to assess whether rankings reflect the signal they care about.
% These findings establish a measurement framework for evaluating when benchmark rankings can be trusted and provide actionable diagnostics for improving benchmark ecosystem reliability through targeted noise control.
% \subsection{Extended Discussion Observations}
\subsection{Actionable Recommendations}
% \subsubsection{A Framework for Noise-Aware Benchmark Evaluation}\label{apx:recs}
% Our results reveal three distinct types of noise in AI benchmark ecosystems, each requiring different control strategies. We now synthesize these findings into a framework for noise-aware evaluation.
% \paragraph{1. Systematic Noise from Known Sources}
% Variance decomposition reveals quantifiable noise from:
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{Benchmark difficulty (44\%):} Predictable noise controllable via normalization or stratification. High G-coefficient (0.74) indicates stable difficulty ordering across models.
% \item \textbf{Contribution Characteristics (9\%):} Systematic noise from submission practices and the Hugging Face accounts associated with each leaderboard submission. Nearly double the variance from model architecture (4.8\%).
% \item \textbf{Model size (36\% of explainable variance):} Without explicit control, rankings conflate scale with innovation.
% \end{itemize}
% Our variance decomposition framework (Generalizability Theory) enables practitioners to \emph{measure} these sources rather than assume their absence. Reporting variance component proportions alongside rankings (as in Table \ref{tab:varcomp_base}) provides transparency about what rankings actually reflect.
% \paragraph{2. Structural Noise from Measurement Misalignment}
% Confirmatory Factor Analysis reveals that benchmark labels provide insufficient noise control:
% \begin{itemize}[leftmargin=*,nosep]
% \item Bifactor structures optimally separate general capability signal ($g$) from benchmark-specific noise.
% \item Even under best structures, residual dependencies persist—uncontrolled local noise.
% \item Permutation tests show benchmark metadata only weakly aligns with true dependency structure.
% \end{itemize}
% Modification Indices (MIs) and Standardized Expected Parameter Changes (SEPCs) map this structural noise. In typical use case (non-bootstrapped scenarios), large SEPCs %($|\text{SEPC}| > 0.40$)
% identify item redundancy—correlated noise that inflates apparent reliability without adding signal. This enables \emph{benchmark slimming}: targeted removal of duplicate items to improve signal-to-noise ratio.
% \paragraph{3. Noise control improves measurement quality}
% The residual variance reduction (33\%) achieved by controlling for model size demonstrates the benefit of explicit noise control. While outside the scope of this study, further improvements are possible by reporting noise source decompositions alongside rankings (Table \ref{tab:varcomp_base} provides a template) and reliability coefficients (G-coefficients, SNR$_\beta$, PSI) to quantify trustworthiness, and using MI/SEPC diagnostics ($|\text{SEPC}| > 0.40$) to identify and remove redundant items.
% \begin{itemize}
% \item Reporting noise source decompositions alongside rankings (Table \ref{tab:varcomp_base} provides a template),
% \item Using MI/SEPC diagnostics ($|\text{SEPC}| > 0.40$) to identify and remove redundant items,
% \item Reporting reliability coefficients (G-coefficients, SNR$_\beta$, PSI) to quantify trustworthiness,
% \item Separating contributor, size, and architectural effects in leaderboard displays.
% \end{itemize}
% \paragraph{3. Scaling Law Instability as Noise in Trend Estimates}
% Our novel metrics quantify reliability of scaling relationships:
% \begin{itemize}
% \item \textbf{SNR$_\beta$:} Ratio of average scaling effect to context-specific variation. Low values indicate the average scaling law is misleading.
% \item \textbf{PSI (Proportion of Slope Instability):} Diagnoses which facets undermine scaling reliability. Benchmark PSI (32.35\%) indicates different benchmarks respond heterogeneously to size increases.
% \item \textbf{SRVC (Scaling-Related Variance Component):} Quantifies what fraction of variance is size-linked (36.3\%).
% \end{itemize}
% These metrics move beyond ``do models scale?'' to ``how reliably do they scale, and which noise sources undermine that relationship?''
\subsubsection{Tools for moving the field forward}
Our framework provides concrete tools for noise control. We frame these constructively as methods practitioners can adopt to improve measurement quality:
% \paragraph{For Benchmark Designers}
% \begin{itemize}[leftmargin=*,nosep]
% % \item \textbf{Use MI/SEPC diagnostics before deployment:} Identify redundant items (|SEPC| > 0.40) during pilot testing. Remove or rewrite items with large residual covariances to increase parsimony.
% \item \textbf{Test structural alignment:} Use permutation controls to verify that benchmark boundaries align with latent structure. If fit remains high under randomization, benchmark labels are misspecified with respect to the intended construct.
% \item \textbf{Report reliability coefficients:} Compute and publish G-coefficients to communicate measurement quality. Our analysis yields $G_B = 0.74$ for benchmark difficulty—a transparency standard others can adopt.
% \item \textbf{Consider bifactor scoring:} Report both general ($g$) and specific ($s_k$) factor scores to preserve latent structure. This disentangles globally strong from ``uniquely strong on benchmark $k$.''
% \end{itemize}
\paragraph{For Leaderboard Operators}
\begin{itemize}[leftmargin=*,nosep]
\item \textbf{Decompose and report variance sources:} Use G-theory variance decomposition (Table \ref{tab:varcomp_base} provides a template) to show users what proportion of ranking variance comes from benchmarks, contributors, architecture, and scale.
% \item \textbf{Control for size when comparing architectures:} Report size-adjusted rankings or include model size as a covariate. Without this, aggregate scores primarily reflect scale (66\% of benchmark-independent variance).
% \item \textbf{Distinguish contributor effects:} When feasible, report within-contributor rankings separately from across-contributor rankings. Our analysis shows contributor noise (9\%) exceeds architectural signal (4.8\%).
\item \textbf{Provide reliability context:} Display confidence intervals or G-coefficients alongside ranks to indicate trustworthiness. Reliability is conditional on use case (within- vs. across-contributor, size-controlled vs. uncontrolled).
\end{itemize}
% \paragraph{For Machine Learning Researchers}
% \begin{itemize}[leftmargin=*,nosep]
% % \item \textbf{Control for contributor effects:} When comparing models, account for the 9\% variance from submission practices. Ideally, compare models from the same contributor or statistically adjust for contributor identity.
% \item \textbf{Explicitly model size:} Report architectural comparisons at matched scales or use size as a covariate. Size confounding (36\%) obscures genuine architectural differences.
% \item \textbf{Use variance decomposition to isolate effects:} Our base framework separates architecture (4.8\%), deployment (2.3\%), contributor (9\%), and size (36\%) effects. This enables precise claims about what drives performance.
% \item \textbf{Test generalization across benchmarks:} Large interaction variances (B$\times$A, B$\times$C) indicate context-dependent performance. Claims about ``Architecture X is better'' require specifying the generalization context.
% \end{itemize}
\paragraph{For Scaling Law Analysis}
\begin{itemize}[leftmargin=*,nosep]
\item \textbf{Report SNR$_\beta$ or $\mathcal{R}_\beta$ alongside $\beta$:} A large average effect $\beta$ with low SNR$_\beta$ ($\approx 1$) indicates scaling is context-dependent—the relationship is unreliable across ecosystem facets.
\item \textbf{Use PSI to diagnose instability:} Our analysis reveals Benchmark PSI (30\%) as the largest source of scaling law unreliability. Different benchmarks respond differently to size increases—acknowledge this heterogeneity.
% \item \textbf{Control for benchmark and contributor facets:} Random slopes show contributor slope variance (18.61) exceeds architecture slope variance (12.79). Scaling relationships differ by who submits the model.
% \item \textbf{Quantify SRVC:} Report what fraction of variance is size-related (36.3\% in our analysis). This establishes how informative size is for predicting performance.
\end{itemize}
% \subsection{When Can Benchmark Rankings Be Trusted?}
% As detailed in Section~\ref{subsec:results_vardecomp}, reliability is conditional on the intended use case and which noise sources are controlled. Our G-theory framework makes this explicit:
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{High reliability contexts:} Ranking benchmark difficulty (G = 0.74), comparing models within-contributor at matched scales.
% \item \textbf{Moderate reliability contexts:} Cross-contributor comparisons after controlling for size; architectural comparisons after variance decomposition.
% \item \textbf{Low reliability contexts:} Aggregate rankings without size control (dominated by scale); architectural claims without contributor adjustment (confounded by submission practices); scaling law extrapolations without PSI diagnostics (context-dependent).
% \end{itemize}
The key insight: \emph{reliability is not binary but quantifiable and improvable}. Our metrics (e.g., SNR$_\beta$, PSI) provide numerical targets for noise control efforts.
% \subsection{Future Directions}\label{apx:future_dir}
% \paragraph{Limitations}
% Our analysis focuses on a single leaderboard (Open LLM) at one snapshot in time. While the methods generalize, the specific variance proportions (9\% contributor, 4.8\% architecture, etc.) are dataset-dependent. Different ecosystems may exhibit different noise profiles. Additionally, our contributor facet captures submission metadata rather than causal mechanisms (training procedures, data quality, hyperparameter tuning)—future work should unpack what drives this 9\% variance.
% \paragraph{Future Directions}
% Several extensions would strengthen noise-aware evaluation:
% \begin{itemize}[leftmargin=*,nosep]
% \item \textbf{Automated noise diagnostics:} Develop pipelines that compute MI/SEPC, variance decompositions, and reliability coefficients automatically for new benchmarks. This would enable real-time feedback during benchmark development.
% \item \textbf{Adaptive reliability reporting:} Create tools that compute conditional reliability for user-specified generalization contexts (e.g., ``within my organization's models'' vs. ``across all submitted models'').
% \item \textbf{Noise-aware aggregation:} Design weighting schemes that account for benchmark redundancy (via SEPC patterns) and source-specific reliability (via G-coefficients). Current aggregation treats all benchmarks equally—optimal weighting should reflect information content.
% \item \textbf{Longitudinal noise tracking:} Monitor how noise sources evolve as benchmarks saturate. Do contributor effects increase over time? Does structural misalignment worsen as items are added?
% \item \textbf{Extension to other domains:} Apply these methods to vision model leaderboards, multi-modal evaluation, and human feedback rankings. The framework is domain-agnostic—only the noise sources vary.
% \item \textbf{Causal identification of contributor effects:} Decompose the 9\% contributor variance into specific mechanisms (data quality, hyperparameter choices, training procedures). This requires richer metadata than currently available.
% \end{itemize}
\section{Consolidated Notation}\label{apx:roadmap:notation}
% ============================================================
% \subsubsection{Consolidated notation}\label{apx:roadmap:notation}
% ============================================================
Tables~\ref{tab:notation} and \ref{tab:notation2} provide a comprehensive reference for all mathematical notation used throughout the paper and appendices.
\begin{table}[h]
\centering
\caption{Comprehensive notation reference. Symbols are grouped by the model layer in which they primarily appear.}
\label{tab:notation}
\small
\renewcommand{\arraystretch}{1}
\begin{tabular}{@{}lll@{}}
\toprule
\textbf{Symbol} & \textbf{Domain} & \textbf{Description} \\
\midrule
\multicolumn{3}{@{}l}{\textit{Observed data}} \\
$y_{ij}$, $y_{ikj}$ & $\{0,1\}$ & Binary response of LLM $i$ to item $j$ (in benchmark $k$) \\
$\mathbf{y}_i$ & $\{0,1\}^p$ & Full response vector for LLM $i$ across $p$ items \\
$y^{\star}_{ij}$ & $\mathbb{R}$ & Underlying continuous latent response (probit/logit link) \\
$s_i$ & $\mathbb{R}$ & Manifest (observed) benchmark score for submission $i$ \\
$\mathbf{S}$ & $\mathbb{R}^{p \times p}$ & Observed sample covariance/correlation matrix \\
$x_i$ & $\mathbb{R}$ & Log-size covariate: $\log_{10}(\text{\#parameters}_i)$ \\
$N$ & $\mathbb{N}$ & Number of LLM submissions \\
$p$ & $\mathbb{N}$ & Total number of items across all benchmarks \\
$K$ & $\mathbb{N}$ & Number of benchmarks (= number of specific factors) \\
$B$ & $\mathbb{N}$ & Number of bootstrap replications \\
$r_k$ & $\mathbb{N}$ & Number of items sampled per benchmark $k$ per bootstrap \\
\midrule
\multicolumn{3}{@{}l}{\textit{CFA / SEM measurement model (Method~1)}} \\
$\boldsymbol{\eta}_i$ & $\mathbb{R}^m$ & Latent factor score vector for LLM $i$ \\
$\boldsymbol{\nu}$ & $\mathbb{R}^p$ & Vector of item intercepts \\
$\boldsymbol{\Lambda}$ & $\mathbb{R}^{p \times m}$ & Factor loading matrix (items $\to$ factors) \\
$\lambda_{kj}$ & $\mathbb{R}$ & Loading of item $j$ on factor $k$ \\
$\boldsymbol{\Phi}$ & $\mathbb{R}^{m \times m}$ & Factor covariance matrix; $\boldsymbol{\Phi} = \mathrm{Cov}(\boldsymbol{\eta})$ \\
$\boldsymbol{\Theta}$ & $\mathbb{R}^{p \times p}$ & Residual (unique) covariance matrix; typically diagonal \\
$\Theta_{ab}$ & $\mathbb{R}$ & Residual covariance between items $a$ and $b$ \\
$\boldsymbol{\Sigma}(\boldsymbol{\theta})$ & $\mathbb{R}^{p \times p}$ & Model-implied covariance: $\boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top + \boldsymbol{\Theta}$ \\
$\boldsymbol{\theta}$ & --- & Collected free parameters: $\{\boldsymbol{\Lambda}, \boldsymbol{\Phi}, \boldsymbol{\Theta}\}$ \\
$\boldsymbol{\Psi}$ & $\mathbb{R}^{m \times m}$ & Correlation matrix of latent factors (in correlated-factor models) \\
% $\ell(\boldsymbol{\theta})$ & $\mathbb{R}$ & Log-likelihood (or fit function) \\
$F(\cdot,\cdot)$ & $\mathbb{R}_{\geq 0}$ & Discrepancy function between $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ and $\mathbf{S}$ \\
$s \in \mathcal{S}$ & --- & Candidate CFA structure (one of six) \\
\midrule
\multicolumn{3}{@{}l}{\textit{Bifactor-specific notation}} \\
$g_i$ & $\mathbb{R}$ & General factor score for LLM $i$ \\
$f_{ik}$, $s_{ik}$ & $\mathbb{R}$ & Specific (benchmark) factor score; $k$-th factor for LLM $i$ \\
$\lambda^{(g)}_{kj}$ & $\mathbb{R}$ & Loading of item $j$ on the general factor $g$ \\
$\lambda^{(s)}_{kj}$ & $\mathbb{R}$ & Loading of item $j$ on specific factor $s_k$ \\
$\mathbf{f}_i$ & $\mathbb{R}^K$ & Vector of specific factor scores for LLM $i$ \\
\midrule
\multicolumn{3}{@{}l}{\textit{Hierarchical / second-order model}} \\
$\gamma_k$ & $\mathbb{R}$ & Loading of first-order factor $f_k$ on second-order factor $g$ \\
$\delta_{ik}$ & $\mathbb{R}$ & First-order factor disturbance \\
$\boldsymbol{\Delta}$ & $\mathbb{R}^{K \times K}$ & Disturbance covariance matrix \\
\midrule
\multicolumn{3}{@{}l}{\textit{Local dependence diagnostics}} \\
$\psi$ & $\mathbb{R}$ & Candidate constrained parameter (residual covariance $\Theta_{ab}$) \\
$U_\psi(\hat{\boldsymbol{\theta}})$ & $\mathbb{R}$ & Score (gradient) for $\psi$ at constrained optimum \\
$\mathcal{I}$ & $\mathbb{R}^{q \times q}$ & Expected Fisher information matrix \\
$\mathcal{I}_{\psi\psi\cdot\theta}$ & $\mathbb{R}$ & Schur complement (partial information for $\psi$) \\
$\mathrm{MI}(\psi)$ & $\mathbb{R}_{\geq 0}$ & Modification index (LM test statistic); $\mathrm{MI}(\psi) \xrightarrow{d} \chi^2_1$ \\
$\mathrm{EPC}(\psi)$ & $\mathbb{R}$ & Expected Parameter Change upon freeing $\psi$ \\
$\mathrm{SEPC}$ & $[-1,1]$ & Standardized EPC (on correlation scale) \\
\midrule
\multicolumn{3}{@{}l}{\textit{Meta-analytic bootstrap aggregation}} \\
% $t_{b,s}$ & $\mathbb{R}$ & Fit statistic for structure $s$ in bootstrap $b$ \\
$t_{b,s,r}$ & $\mathbb{R}$ & Fit statistic under condition $r$ (true vs.\ permuted) \\
$\alpha_s$ & $\mathbb{R}$ & Intercept: expected fit of structure $s$ under true assignment \\
$\beta_s$ & $\mathbb{R}$ & Permutation penalty: fit loss from randomized assignment \\
$\Delta T_s^{(b)}$ & $\mathbb{R}$ & $T(s \mid \pi^{(b)}) - T(s \mid \text{id})$; structural signal beyond flexibility \\
$\pi^{(b)}$ & --- & Random item-to-benchmark permutation in bootstrap $b$ \\
$u_b, v_b$ & $\mathbb{R}$ & Random intercept/slope for bootstrap $b$ \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{G-theory variance decomposition (Method~2)}} \\
% $\mu$ & $\mathbb{R}$ & Grand mean score \\
% $a(i), b(i), c(i), d(i)$ & --- & Facet levels: Architecture, Benchmark, Contributor, Deployment \\
% $\mathcal{S}$ & --- & Set of random-effect terms (main effects + interactions) \\
% $u^S_{g_S}$ & $\mathbb{R}$ & Random intercept for level $g_S$ of term $S$ \\
% $v^S_{g_S}$ & $\mathbb{R}$ & Random slope for level $g_S$ of term $S$ \\
% $\sigma^2_{S,0}$ & $\mathbb{R}_{\geq 0}$ & Intercept variance component for term $S$ \\
% $\sigma^2_{S,1}$ & $\mathbb{R}_{\geq 0}$ & Slope variance component for term $S$ \\
% $\sigma^2_\varepsilon$ & $\mathbb{R}_{\geq 0}$ & Residual variance \\
% $p_S$ & $[0,1]$ & Variance share attributable to term $S$ \\
% $\beta$ & $\mathbb{R}$ & Fixed effect of log-size on score \\
% $G_B$ & $[0,1]$ & Generalizability coefficient for benchmark difficulty \\
% $\mathrm{SNR}_\beta$ & $\mathbb{R}_{\geq 0}$ & Slope signal-to-noise ratio: $\beta^2 / \sum_S \sigma^2_{S,1}$ \\
% $\mathrm{PSI}_S$ & $[0,1]$ & Proportion of slope instability from term $S$ \\
% $\Omega_x$ & $[0,1]$ & Size-driven share of marginal variance \\
% $\Omega_x^{(-B)}$ & $[0,1]$ & Benchmark-demeaned size-driven reliability \\
% $R_\beta$ & $[0,1]$ & Reliability coefficient for the fixed slope $\beta$ \\
% $\rho_x$ & $[0,1]$ & Correlation-of-ranks reliability: $\sqrt{\Omega_x}$ \\
% $\mathrm{CV}_\beta$ & $\mathbb{R}_{\geq 0}$ & Coefficient of variation of effective slopes \\
% $H_S$ & $[0,1]$ & Facet-specific slope reliability (intra-class share) \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{Latent mixed-effects regression (Method~3)}} \\
% $\boldsymbol{\theta}_i$ & $\mathbb{R}^{K+1}$ & Latent ability vector: $(\theta_{i0}, \theta_{i1}, \ldots, \theta_{iK})^\top$ \\
% $\theta_{i0}$ & $\mathbb{R}$ & General ability for LLM $i$ \\
% $\theta_{ik}$ & $\mathbb{R}$ & Benchmark-specific ability for LLM $i$, benchmark $k$ \\
% $a_{j0}$ & $\mathbb{R}_{>0}$ & Item $j$ discrimination on the general factor \\
% $a_{j,k(j)}$ & $\mathbb{R}_{>0}$ & Item $j$ discrimination on its benchmark-specific factor \\
% $b_j$ & $\mathbb{R}$ & Item $j$ difficulty \\
% $\boldsymbol{\Theta}$ & $\mathbb{R}^{N \times (K+1)}$ & Matrix of all latent traits \\
% $\mathbf{V}$ & $\mathbb{R}^{N \times F}$ & Fixed-effect design matrix \\
% $\boldsymbol{\Gamma}$ & $\mathbb{R}^{F \times (K+1)}$ & Fixed-effect coefficient matrix \\
% $\beta_k$ & $\mathbb{R}$ & Effect of log-size on latent dimension $k$: $\Gamma_{x,k}$ \\
% $\boldsymbol{\beta}$ & $\mathbb{R}^{K+1}$ & Scaling vector: $(\beta_0, \beta_1, \ldots, \beta_K)$ \\
% $\mathbf{W}$ & $\mathbb{R}^{N \times R}$ & Random-effect design matrix (contributor grouping) \\
% $\boldsymbol{\zeta}$ & $\mathbb{R}^{R \times (K+1)}$ & Contributor-level random effects \\
% $\boldsymbol{\Sigma}_\zeta$ & --- & Covariance of $\mathrm{vec}(\boldsymbol{\zeta})$; block-diagonal \\
% $\boldsymbol{\Sigma}_u$ & --- & Provenance variance (contributor random-effect covariance) \\
% $\boldsymbol{\Sigma}_E$ & $\mathbb{R}^{(K+1)\times(K+1)}$ & Residual latent covariance \\
% $\mathbf{E}$ & $\mathbb{R}^{N \times (K+1)}$ & Residual latent variation \\
% $\mathbf{B}$ & $\mathbb{R}^{(K+1)\times(2+d)}$ & Fixed-effect matrix in IRT parameterization \\
% $\mathbf{z}_i$ & $\mathbb{R}^{2+d}$ & Covariate vector: $[1,\; x_i,\; \mathbf{w}_i^\top]^\top$ \\
% $\mathbf{w}_i$ & $\mathbb{R}^d$ & Deployment covariate vector for LLM $i$ \\
% $\mathbf{u}_{c(i)}$ & $\mathbb{R}^{K+1}$ & Contributor random-effect vector for LLM $i$ \\
% $\mathcal{R}_d$ & $[0,1]$ & Slope reliability index: $\beta_d^2 / (\beta_d^2 + \tau_d^2)$ \\
% $\tau_d^2$ & $\mathbb{R}_{\geq 0}$ & Random-slope variance for latent dimension $d$ \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{Fit indices and model comparison}} \\
% RMSEA & $\mathbb{R}_{\geq 0}$ & Root Mean Square Error of Approximation \\
% CFI & $[0,1]$ & Comparative Fit Index \\
% TLI & $[0,1]$ & Tucker-Lewis Index \\
% SRMR & $\mathbb{R}_{\geq 0}$ & Standardized Root Mean Square Residual \\
% AIC, BIC & $\mathbb{R}$ & Akaike / Bayesian Information Criteria \\
% AUC & $[0,1]$ & Area Under the ROC Curve (out-of-sample prediction) \\
% MAE & $\mathbb{R}_{\geq 0}$ & Mean Absolute Error (out-of-sample prediction) \\
% LRT & $\mathbb{R}_{\geq 0}$ & Likelihood Ratio Test statistic \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{General conventions}} \\
% $\sigma(\cdot)$ & $(0,1)$ & Logistic function: $\sigma(t) = (1+e^{-t})^{-1}$ \\
% $\mathbb{K}[\cdot]$ & $\{0,1\}$ & Indicator function \\
% $\tau_j$ & $\mathbb{R}$ & Item threshold (probit parameterization) \\
% $\mathcal{P}^*(\cdot)$ & --- & Non-empty Proper Power Set \\
% $\xrightarrow{d}$ & --- & Convergence in distribution \\
\bottomrule
\end{tabular}
\end{table}
\begin{table}[h]
\centering
\caption{Comprehensive notation reference. Symbols are grouped by the model layer in which they primarily appear.}
\label{tab:notation2}
\small
\renewcommand{\arraystretch}{1}
\begin{tabular}{@{}lll@{}}
\toprule
\textbf{Symbol} & \textbf{Domain} & \textbf{Description} \\
\midrule
% \multicolumn{3}{@{}l}{\textit{Observed data}} \\
% $y_{ij}$, $y_{ikj}$ & $\{0,1\}$ & Binary response of LLM $i$ to item $j$ (in benchmark $k$) \\
% $\mathbf{y}_i$ & $\{0,1\}^p$ & Full response vector for LLM $i$ across $p$ items \\
% $y^{\star}_{ij}$ & $\mathbb{R}$ & Underlying continuous latent response (probit/logit link) \\
% $s_i$ & $\mathbb{R}$ & Manifest (observed) benchmark score for submission $i$ \\
% $\mathbf{S}$ & $\mathbb{R}^{p \times p}$ & Observed sample covariance/correlation matrix \\
% $x_i$ & $\mathbb{R}$ & Log-size covariate: $\log_{10}(\text{\#parameters}_i)$ \\
% $N$ & $\mathbb{N}$ & Number of LLM submissions \\
% $p$ & $\mathbb{N}$ & Total number of items across all benchmarks \\
% $K$ & $\mathbb{N}$ & Number of benchmarks (= number of specific factors) \\
% $B$ & $\mathbb{N}$ & Number of bootstrap replications \\
% $r_k$ & $\mathbb{N}$ & Number of items sampled per benchmark $k$ per bootstrap \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{CFA / SEM measurement model (Method~1)}} \\
% $\boldsymbol{\eta}_i$ & $\mathbb{R}^m$ & Latent factor score vector for LLM $i$ \\
% $\boldsymbol{\nu}$ & $\mathbb{R}^p$ & Vector of item intercepts \\
% $\boldsymbol{\Lambda}$ & $\mathbb{R}^{p \times m}$ & Factor loading matrix (items $\to$ factors) \\
% $\lambda_{kj}$ & $\mathbb{R}$ & Loading of item $j$ on factor $k$ \\
% $\boldsymbol{\Phi}$ & $\mathbb{R}^{m \times m}$ & Factor covariance matrix; $\boldsymbol{\Phi} = \mathrm{Cov}(\boldsymbol{\eta})$ \\
% $\boldsymbol{\Theta}$ & $\mathbb{R}^{p \times p}$ & Residual (unique) covariance matrix; typically diagonal \\
% $\Theta_{ab}$ & $\mathbb{R}$ & Residual covariance between items $a$ and $b$ \\
% $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ & $\mathbb{R}^{p \times p}$ & Model-implied covariance: $\boldsymbol{\Lambda}\boldsymbol{\Phi}\boldsymbol{\Lambda}^\top + \boldsymbol{\Theta}$ \\
% $\boldsymbol{\theta}$ & --- & Collected free parameters: $\{\boldsymbol{\Lambda}, \boldsymbol{\Phi}, \boldsymbol{\Theta}\}$ \\
% $\boldsymbol{\Psi}$ & $\mathbb{R}^{m \times m}$ & Correlation matrix of latent factors (in correlated-factor models) \\
% $\ell(\boldsymbol{\theta})$ & $\mathbb{R}$ & Log-likelihood (or fit function) \\
% $F(\cdot,\cdot)$ & $\mathbb{R}_{\geq 0}$ & Discrepancy function between $\boldsymbol{\Sigma}(\boldsymbol{\theta})$ and $\mathbf{S}$ \\
% $s \in \mathcal{S}$ & --- & Candidate CFA structure (one of six) \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{Bifactor-specific notation}} \\
% $g_i$ & $\mathbb{R}$ & General factor score for LLM $i$ \\
% $f_{ik}$, $s_{ik}$ & $\mathbb{R}$ & Specific (benchmark) factor score; $k$-th factor for LLM $i$ \\
% $\lambda^{(g)}_{kj}$ & $\mathbb{R}$ & Loading of item $j$ on the general factor $g$ \\
% $\lambda^{(s)}_{kj}$ & $\mathbb{R}$ & Loading of item $j$ on specific factor $s_k$ \\
% $\mathbf{f}_i$ & $\mathbb{R}^K$ & Vector of specific factor scores for LLM $i$ \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{Hierarchical / second-order model}} \\
% $\gamma_k$ & $\mathbb{R}$ & Loading of first-order factor $f_k$ on second-order factor $g$ \\
% $\delta_{ik}$ & $\mathbb{R}$ & First-order factor disturbance \\
% $\boldsymbol{\Delta}$ & $\mathbb{R}^{K \times K}$ & Disturbance covariance matrix \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{Local dependence diagnostics}} \\
% $\psi$ & $\mathbb{R}$ & Candidate constrained parameter (residual covariance $\Theta_{ab}$) \\
% $U_\psi(\hat{\boldsymbol{\theta}})$ & $\mathbb{R}$ & Score (gradient) for $\psi$ at constrained optimum \\
% $\mathcal{I}$ & $\mathbb{R}^{q \times q}$ & Expected Fisher information matrix \\
% $\mathcal{I}_{\psi\psi\cdot\theta}$ & $\mathbb{R}$ & Schur complement (partial information for $\psi$) \\
% $\mathrm{MI}(\psi)$ & $\mathbb{R}_{\geq 0}$ & Modification index (LM test statistic); $\mathrm{MI}(\psi) \xrightarrow{d} \chi^2_1$ \\
% $\mathrm{EPC}(\psi)$ & $\mathbb{R}$ & Expected Parameter Change upon freeing $\psi$ \\
% $\mathrm{SEPC}$ & $[-1,1]$ & Standardized EPC (on correlation scale) \\
% \midrule
% \multicolumn{3}{@{}l}{\textit{Meta-analytic bootstrap aggregation}} \\
% $t_{b,s}$ & $\mathbb{R}$ & Fit statistic for structure $s$ in bootstrap $b$ \\
% $t_{b,s,r}$ & $\mathbb{R}$ & Fit statistic under condition $r$ (true vs.\ permuted) \\
% $\alpha_s$ & $\mathbb{R}$ & Intercept: expected fit of structure $s$ under true assignment \\
% $\beta_s$ & $\mathbb{R}$ & Permutation penalty: fit loss from randomized assignment \\
% $\Delta T_s^{(b)}$ & $\mathbb{R}$ & $T(s \mid \pi^{(b)}) - T(s \mid \text{id})$; structural signal beyond flexibility \\
% $\pi^{(b)}$ & --- & Random item-to-benchmark permutation in bootstrap $b$ \\
% $u_b, v_b$ & $\mathbb{R}$ & Random intercept/slope for bootstrap $b$ \\
% \midrule
\multicolumn{3}{@{}l}{\textit{G-theory variance decomposition (Method~2)}} \\
$\mu$ & $\mathbb{R}$ & Grand mean score \\
$a(i), b(i), c(i), d(i)$ & --- & Facet levels: Architecture, Benchmark, Contributor, Deployment \\
$\mathcal{S}$ & --- & Set of random-effect terms (main effects + interactions) \\
$u^S_{g_S}$ & $\mathbb{R}$ & Random intercept for level $g_S$ of term $S$ \\
$v^S_{g_S}$ & $\mathbb{R}$ & Random slope for level $g_S$ of term $S$ \\
$\sigma^2_{S,0}$ & $\mathbb{R}_{\geq 0}$ & Intercept variance component for term $S$ \\
$\sigma^2_{S,1}$ & $\mathbb{R}_{\geq 0}$ & Slope variance component for term $S$ \\
$\sigma^2_\varepsilon$ & $\mathbb{R}_{\geq 0}$ & Residual variance \\
$p_S$ & $[0,1]$ & Variance share attributable to term $S$ \\
$\beta$ & $\mathbb{R}$ & Fixed effect of log-size on score \\
% $G_B$ & $[0,1]$ & Generalizability coefficient for benchmark difficulty \\
$\mathrm{SNR}_\beta$ & $\mathbb{R}_{\geq 0}$ & Slope signal-to-noise ratio: $\beta^2 / \sum_S \sigma^2_{S,1}$ \\
$\mathrm{PSI}_S$ & $[0,1]$ & Proportion of slope instability from term $S$ \\
% $\Omega_x$ & $[0,1]$ & Size-driven share of marginal variance \\
% $\Omega_x^{(-B)}$ & $[0,1]$ & Benchmark-demeaned size-driven reliability \\
$R_\beta$ & $[0,1]$ & Reliability coefficient for the fixed slope $\beta$ \\
% $\rho_x$ & $[0,1]$ & Correlation-of-ranks reliability: $\sqrt{\Omega_x}$ \\
$\mathrm{CV}_\beta$ & $\mathbb{R}_{\geq 0}$ & Coefficient of variation of effective slopes \\
% $H_S$ & $[0,1]$ & Facet-specific slope reliability (intra-class share) \\
\midrule
\multicolumn{3}{@{}l}{\textit{Latent mixed-effects regression (Method~3)}} \\
$\boldsymbol{\theta}_i$ & $\mathbb{R}^{K+1}$ & Latent ability vector: $(\theta_{i0}, \theta_{i1}, \ldots, \theta_{iK})^\top$ \\
$\theta_{i0}$ & $\mathbb{R}$ & General ability for LLM $i$ \\
$\theta_{ik}$ & $\mathbb{R}$ & Benchmark-specific ability for LLM $i$, benchmark $k$ \\
$a_{j0}$ & $\mathbb{R}_{>0}$ & Item $j$ discrimination on the general factor \\
$a_{j,k(j)}$ & $\mathbb{R}_{>0}$ & Item $j$ discrimination on its benchmark-specific factor \\
$b_j$ & $\mathbb{R}$ & Item $j$ difficulty \\
$\boldsymbol{\Theta}$ & $\mathbb{R}^{N \times (K+1)}$ & Matrix of all latent traits \\
$\mathbf{V}$ & $\mathbb{R}^{N \times F}$ & Fixed-effect design matrix \\
$\boldsymbol{\Gamma}$ & $\mathbb{R}^{F \times (K+1)}$ & Fixed-effect coefficient matrix \\
$\beta_k$ & $\mathbb{R}$ & Effect of log-size on latent dimension $k$: $\Gamma_{x,k}$ \\
$\boldsymbol{\beta}$ & $\mathbb{R}^{K+1}$ & Scaling vector: $(\beta_0, \beta_1, \ldots, \beta_K)$ \\
$\mathbf{W}$ & $\mathbb{R}^{N \times R}$ & Random-effect design matrix (contributor grouping) \\
$\boldsymbol{\zeta}$ & $\mathbb{R}^{R \times (K+1)}$ & Contributor-level random effects \\
$\boldsymbol{\Sigma}_\zeta$ & --- & Covariance of $\mathrm{vec}(\boldsymbol{\zeta})$; block-diagonal \\
$\boldsymbol{\Sigma}_u$ & --- & Provenance variance (contributor random-effect covariance) \\
$\boldsymbol{\Sigma}_E$ & $\mathbb{R}^{(K+1)\times(K+1)}$ & Residual latent covariance \\
$\mathbf{E}$ & $\mathbb{R}^{N \times (K+1)}$ & Residual latent variation \\
$\mathbf{B}$ & $\mathbb{R}^{(K+1)\times(2+d)}$ & Fixed-effect matrix in IRT parameterization \\
$\mathbf{z}_i$ & $\mathbb{R}^{2+d}$ & Covariate vector: $[1,\; x_i,\; \mathbf{w}_i^\top]^\top$ \\
$\mathbf{w}_i$ & $\mathbb{R}^d$ & Deployment covariate vector for LLM $i$ \\
$\mathbf{u}_{c(i)}$ & $\mathbb{R}^{K+1}$ & Contributor random-effect vector for LLM $i$ \\
$\mathcal{R}_d$ & $[0,1]$ & Slope reliability index: $\beta_d^2 / (\beta_d^2 + \tau_d^2)$ \\
$\tau_d^2$ & $\mathbb{R}_{\geq 0}$ & Random-slope variance for latent dimension $d$ \\
\midrule
\multicolumn{3}{@{}l}{\textit{Fit indices and model comparison}} \\
RMSEA & $\mathbb{R}_{\geq 0}$ & Root Mean Square Error of Approximation \\
CFI & $[0,1]$ & Comparative Fit Index \\
TLI & $[0,1]$ & Tucker-Lewis Index \\
SRMR & $\mathbb{R}_{\geq 0}$ & Standardized Root Mean Square Residual \\
AIC, BIC & $\mathbb{R}$ & Akaike / Bayesian Information Criteria \\
AUC & $[0,1]$ & Area Under the ROC Curve (out-of-sample prediction) \\
MAE & $\mathbb{R}_{\geq 0}$ & Mean Absolute Error (out-of-sample prediction) \\
LRT & $\mathbb{R}_{\geq 0}$ & Likelihood Ratio Test statistic \\
\midrule
\multicolumn{3}{@{}l}{\textit{General conventions}} \\
% $\sigma(\cdot)$ & $(0,1)$ & Logistic function: $\sigma(t) = (1+e^{-t})^{-1}$ \\
% $\mathbb{K}[\cdot]$ & $\{0,1\}$ & Indicator function \\
$\tau_j$ & $\mathbb{R}$ & Item threshold (probit parameterization) \\
$\mathcal{P}^*(\cdot)$ & --- & Non-empty Power Set \\
% $\xrightarrow{d}$ & --- & Convergence in distribution \\
\bottomrule
\end{tabular}
\end{table}
\begin{thebibliography}{55}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Bollen(1989)]{bollen_structural_1989}
Bollen, K.~A.
\newblock \emph{Structural equations with latent variables}.
\newblock Wiley series in probability and mathematical statistics. {Applied} probability and statistics. Wiley, New York, 1989.
\newblock ISBN 978-0-471-01171-2 978-1-118-61917-9 978-1-118-61916-2 978-1-118-61903-2.
\newblock \doi{10.1002/9781118619179}.
\bibitem[Brennan(2001)]{brennan_generalizability_2001}
Brennan, R.~L.
\newblock \emph{Generalizability {Theory}}.
\newblock Springer, New York, NY, 2001.
\newblock ISBN 978-1-4419-2938-9 978-1-4757-3456-0.
\newblock \doi{10.1007/978-1-4757-3456-0}.
\newblock URL \url{http://link.springer.com/10.1007/978-1-4757-3456-0}.
\bibitem[Brown(2015)]{brown_confirmatory_2015}
Brown, T.~A.
\newblock \emph{Confirmatory factor analysis for applied research}.
\newblock Methodology in the social sciences. The Guilford Press, New York London, second edition edition, 2015.
\newblock ISBN 978-1-4625-1536-3 978-1-4625-1780-0.
\bibitem[Browne \& Cudeck(1992)Browne and Cudeck]{browne_alternative_1992}
Browne, M.~W. and Cudeck, R.
\newblock Alternative {Ways} of {Assessing} {Model} {Fit}.
\newblock \emph{Sociological Methods \& Research}, 21\penalty0 (2):\penalty0 230--258, November 1992.
\newblock ISSN 0049-1241.
\newblock \doi{10.1177/0049124192021002005}.
\newblock URL \url{https://doi.org/10.1177/0049124192021002005}.
\newblock Publisher: SAGE Publications Inc.
\bibitem[Burnell et~al.(2023)Burnell, Hao, Conway, and Orallo]{burnell_revealing_2023}
Burnell, R., Hao, H., Conway, A. R.~A., and Orallo, J.~H.
\newblock Revealing the structure of language model capabilities, June 2023.
\newblock URL \url{http://arxiv.org/abs/2306.10062}.
\newblock arXiv:2306.10062 [cs].
\bibitem[Buse(1973)]{buse_goodness_1973}
Buse, A.
\newblock Goodness of {Fit} in {Generalized} {Least} {Squares} {Estimation}.
\newblock \emph{The American Statistician}, 27\penalty0 (3):\penalty0 106--108, 1973.
\newblock ISSN 0003-1305.
\newblock \doi{10.2307/2683631}.
\newblock URL \url{https://www.jstor.org/stable/2683631}.
\newblock Publisher: [American Statistical Association, Taylor \& Francis, Ltd.].
\bibitem[Cai(2010{\natexlab{a}})]{cai_high-dimensional_2010}
Cai, L.
\newblock High-dimensional {Exploratory} {Item} {Factor} {Analysis} by {A} {Metropolis}–{Hastings} {Robbins}–{Monro} {Algorithm}.
\newblock \emph{Psychometrika}, 75\penalty0 (1):\penalty0 33--57, March 2010{\natexlab{a}}.
\newblock ISSN 1860-0980.
\newblock \doi{10.1007/s11336-009-9136-x}.
\newblock URL \url{https://doi.org/10.1007/s11336-009-9136-x}.
\bibitem[Cai(2010{\natexlab{b}})]{cai_metropolis-hastings_2010}
Cai, L.
\newblock Metropolis-{Hastings} {Robbins}-{Monro} {Algorithm} for {Confirmatory} {Item} {Factor} {Analysis}.
\newblock \emph{Journal of Educational and Behavioral Statistics}, 35\penalty0 (3):\penalty0 307--335, June 2010{\natexlab{b}}.
\newblock ISSN 1076-9986.
\newblock \doi{10.3102/1076998609353115}.
\newblock URL \url{https://doi.org/10.3102/1076998609353115}.
\newblock Publisher: American Educational Research Association.
\bibitem[Casabianca(2025)]{casabianca_psychometrics_2025}
Casabianca, J.~M.
\newblock Psychometrics is all you need, November 2025.
\newblock URL \url{https://osf.io/preprints/edarxiv/7w6pz_v1/}.
\bibitem[Chalmers(2012)]{chalmers_mirt_2012}
Chalmers, R.~P.
\newblock mirt: {A} {Multidimensional} {Item} {Response} {Theory} {Package} for the {R} {Environment}.
\newblock \emph{Journal of Statistical Software}, 48:\penalty0 1--29, May 2012.
\newblock ISSN 1548-7660.
\newblock \doi{10.18637/jss.v048.i06}.
\newblock URL \url{https://doi.org/10.18637/jss.v048.i06}.
\bibitem[Chalmers(2015)]{chalmers_extended_2015}
Chalmers, R.~P.
\newblock Extended {Mixed}-{Effects} {Item} {Response} {Models} {With} the {MH}-{RM} {Algorithm}.
\newblock \emph{Journal of Educational Measurement}, 52\penalty0 (2):\penalty0 200--222, 2015.
\newblock ISSN 1745-3984.
\newblock \doi{10.1111/jedm.12072}.
\newblock URL \url{https://onlinelibrary.wiley.com/doi/abs/10.1111/jedm.12072}.
\newblock \_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1111/jedm.12072.
\bibitem[Chiang et~al.(2024)Chiang, Zheng, Sheng, Angelopoulos, Li, Li, Zhang, Zhu, Jordan, Gonzalez, and Stoica]{chiang_chatbot_2024}
Chiang, W.-L., Zheng, L., Sheng, Y., Angelopoulos, A.~N., Li, T., Li, D., Zhang, H., Zhu, B., Jordan, M., Gonzalez, J.~E., and Stoica, I.
\newblock Chatbot {Arena}: {An} {Open} {Platform} for {Evaluating} {LLMs} by {Human} {Preference}, March 2024.
\newblock URL \url{http://arxiv.org/abs/2403.04132}.
\newblock arXiv:2403.04132 [cs].
\bibitem[{Clémentine Fourrier [@clefourrier]}(2024)]{clementine_fourrier__is_off_till_dec_2026_hiking_clefourrier_open_2024}
{Clémentine Fourrier [@clefourrier]}.
\newblock The {Open} {LLM} {Leaderboard} is taking a stronger stance on metadata, via 2 things. 1) {If} your model has no model card or license tag, it is now in the "deleted" category \& it won't appear in the main view. {A} model with no explanation or license is not useful to the community., January 2024.
\newblock URL \url{https://x.com/clefourrier/status/1742586339013337119}.
\bibitem[Cronbach \& Meehl(1955)Cronbach and Meehl]{cronbach_construct_1955}
Cronbach, L.~J. and Meehl, P.~E.
\newblock Construct validity in psychological tests.
\newblock \emph{Psychological Bulletin}, 52\penalty0 (4):\penalty0 281--302, 1955.
\newblock ISSN 1939-1455.
\newblock \doi{10.1037/h0040957}.
\bibitem[Cronbach \& Shavelson(2004)Cronbach and Shavelson]{cronbach_my_2004}
Cronbach, L.~J. and Shavelson, R.~J.
\newblock My {Current} {Thoughts} on {Coefficient} {Alpha} and {Successor} {Procedures}.
\newblock \emph{Educational and Psychological Measurement}, 64\penalty0 (3):\penalty0 391--418, June 2004.
\newblock ISSN 0013-1644.
\newblock \doi{10.1177/0013164404266386}.
\newblock URL \url{https://doi.org/10.1177/0013164404266386}.
\newblock Publisher: SAGE Publications Inc.
\bibitem[De~Boeck(2011)]{de_boeck_explanatory_2011}
De~Boeck, P. (ed.).
\newblock \emph{Explanatory item response models: a generalized linear and nonlinear approach}.
\newblock Statistics for social science and public policy. Springer, New York, softcover reprint of the hardcover 1. ed. 2004 edition, 2011.
\newblock ISBN 978-1-4757-3990-9 978-1-4419-2323-3.
\bibitem[Dunn \& McCray(2020)Dunn and McCray]{dunn_place_2020}
Dunn, K.~J. and McCray, G.
\newblock The {Place} of the {Bifactor} {Model} in {Confirmatory} {Factor} {Analysis} {Investigations} {Into} {Construct} {Dimensionality} in {Language} {Testing}.
\newblock \emph{Frontiers in Psychology}, 11, July 2020.
\newblock ISSN 1664-1078.
\newblock \doi{10.3389/fpsyg.2020.01357}.
\newblock URL \url{https://www.frontiersin.org/journals/psychology/articles/10.3389/fpsyg.2020.01357/full}.
\newblock Publisher: Frontiers.
\bibitem[Engel(1984)]{engel_chapter_1984}
Engel, R.~F.
\newblock Chapter 13 {Wald}, likelihood ratio, and {Lagrange} multiplier tests in econometrics.
\newblock In \emph{Handbook of {Econometrics}}, volume~2, pp.\ 775--826. Elsevier, January 1984.
\newblock \doi{10.1016/S1573-4412(84)02005-5}.
\newblock URL \url{https://www.sciencedirect.com/science/chapter/handbook/pii/S1573441284020055}.
\newblock ISSN: 1573-4412.
\bibitem[Fourrier et~al.(2024)Fourrier, Habib, Lozovskaya, Szafer, and Wolf]{fourrier_open_2024}
Fourrier, C., Habib, N., Lozovskaya, A., Szafer, K., and Wolf, T.
\newblock Open {LLM} {Leaderboard} v2, 2024.
\newblock URL \url{https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard}.
\bibitem[Groskurth et~al.(2024)Groskurth, Bluemke, and Lechner]{groskurth_why_2024}
Groskurth, K., Bluemke, M., and Lechner, C.~M.
\newblock Why we need to abandon fixed cutoffs for goodness-of-fit indices: {An} extensive simulation and possible solutions.
\newblock \emph{Behavior Research Methods}, 56\penalty0 (4):\penalty0 3891--3914, June 2024.
\newblock ISSN 1554-3528.
\newblock \doi{10.3758/s13428-023-02193-3}.
\newblock URL \url{https://doi.org/10.3758/s13428-023-02193-3}.
\bibitem[Hardy(2025)]{hardy_all_2025}
Hardy, M.
\newblock “{All} that {Glitters}”: {Techniques} for {Evaluations} with {Unreliable} {Model} and {Human} {Annotations}.
\newblock In Chiruzzo, L., Ritter, A., and Wang, L. (eds.), \emph{Findings of the {Association} for {Computational} {Linguistics}: {NAACL} 2025}, pp.\ 2250--2278, Albuquerque, New Mexico, April 2025. Association for Computational Linguistics.
\newblock ISBN 979-8-89176-195-7.
\newblock \doi{10.18653/v1/2025.findings-naacl.120}.
\newblock URL \url{https://aclanthology.org/2025.findings-naacl.120/}.
\bibitem[Hardy \& Kim(2026)Hardy and Kim]{hardy_knowledge_2026}
Hardy, M. and Kim, Y.
\newblock Knowledge without {Wisdom}: {Measuring} {Misalignment} between {LLMs} and {Intended} {Impact}, April 2026.
\newblock URL \url{http://arxiv.org/abs/2603.00883}.
\newblock arXiv:2603.00883 [cs].
\bibitem[Hendrycks et~al.(2020)Hendrycks, Burns, Basart, Zou, Mazeika, Song, and Steinhardt]{hendrycks_measuring_2020}
Hendrycks, D., Burns, C., Basart, S., Zou, A., Mazeika, M., Song, D., and Steinhardt, J.
\newblock Measuring {Massive} {Multitask} {Language} {Understanding}.
\newblock October 2020.
\newblock URL \url{https://openreview.net/forum?id=d7KBjmI3GmQ}.
\bibitem[Hendrycks et~al.(2021)Hendrycks, Burns, Kadavath, Arora, Basart, Tang, Song, and Steinhardt]{hendrycks_measuring_2021}
Hendrycks, D., Burns, C., Kadavath, S., Arora, A., Basart, S., Tang, E., Song, D., and Steinhardt, J.
\newblock Measuring {Mathematical} {Problem} {Solving} {With} the {MATH} {Dataset}, November 2021.
\newblock URL \url{http://arxiv.org/abs/2103.03874}.
\newblock arXiv:2103.03874 [cs].
\bibitem[Hoffmann et~al.(2022)Hoffmann, Borgeaud, Mensch, Buchatskaya, Cai, Rutherford, Casas, Hendricks, Welbl, Clark, Hennigan, Noland, Millican, Driessche, Damoc, Guy, Osindero, Simonyan, Elsen, Rae, Vinyals, and Sifre]{hoffmann_training_2022}
Hoffmann, J., Borgeaud, S., Mensch, A., Buchatskaya, E., Cai, T., Rutherford, E., Casas, D. d.~L., Hendricks, L.~A., Welbl, J., Clark, A., Hennigan, T., Noland, E., Millican, K., Driessche, G. v.~d., Damoc, B., Guy, A., Osindero, S., Simonyan, K., Elsen, E., Rae, J.~W., Vinyals, O., and Sifre, L.
\newblock Training compute-optimal large language models.
\newblock In \emph{Proceedings of the 36th International Conference on Neural Information Processing Systems}, NeurIPS '22, pp.\ 30016--30030, 2022.
\newblock URL \url{https://arxiv.org/abs/2203.15556}.
\bibitem[Isik et~al.(2025)Isik, Ponomareva, Hazimeh, Paparas, Vassilvitskii, and Koyejo]{isik_scaling_2025}
Isik, B., Ponomareva, N., Hazimeh, H., Paparas, D., Vassilvitskii, S., and Koyejo, S.
\newblock Scaling {Laws} for {Downstream} {Task} {Performance} in {Machine} {Translation}, February 2025.
\newblock URL \url{http://arxiv.org/abs/2402.04177}.
\newblock arXiv:2402.04177 [cs].
\bibitem[Kaplan et~al.(2020)Kaplan, McCandlish, Henighan, Brown, Chess, Child, Gray, Radford, Wu, and Amodei]{kaplan_scaling_2020}
Kaplan, J., McCandlish, S., Henighan, T., Brown, T.~B., Chess, B., Child, R., Gray, S., Radford, A., Wu, J., and Amodei, D.
\newblock Scaling laws for neural language models.
\newblock \emph{arXiv preprint arXiv:2001.08361}, 2020.
\newblock URL \url{https://arxiv.org/abs/2001.08361}.
\bibitem[Kenny et~al.(2015)Kenny, Kaniskan, and McCoach]{kenny_performance_2015}
Kenny, D.~A., Kaniskan, B., and McCoach, D.~B.
\newblock The {Performance} of {RMSEA} in {Models} {With} {Small} {Degrees} of {Freedom}.
\newblock \emph{Sociological Methods \& Research}, 44\penalty0 (3):\penalty0 486--507, August 2015.
\newblock ISSN 0049-1241.
\newblock \doi{10.1177/0049124114543236}.
\newblock URL \url{https://doi.org/10.1177/0049124114543236}.
\bibitem[Kipnis et~al.(2024)Kipnis, Voudouris, Buschoff, and Schulz]{kipnis_textttmetabench_2024}
Kipnis, A., Voudouris, K., Buschoff, L. M.~S., and Schulz, E.
\newblock \${\textbackslash}texttt\{metabench\}\$ -- {A} {Sparse} {Benchmark} to {Measure} {General} {Ability} in {Large} {Language} {Models}, July 2024.
\newblock URL \url{http://arxiv.org/abs/2407.12844}.
\newblock arXiv:2407.12844 [cs] version: 1.
\bibitem[Lalor et~al.(2018)Lalor, Wu, Munkhdalai, and Yu]{lalor_understanding_2018}
Lalor, J.~P., Wu, H., Munkhdalai, T., and Yu, H.
\newblock Understanding deep learning performance through an examination of test set difficulty: A psychometric case study.
\newblock In \emph{Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing}, pp.\ 4711--4716, Brussels, Belgium, 2018. Association for Computational Linguistics.
\newblock \doi{10.18653/v1/d18-1500}.
\newblock URL \url{https://aclanthology.org/D18-1500/}.
\bibitem[Liang et~al.(2023)Liang, Bommasani, Lee, Tsipras, Soylu, Yasunaga, Zhang, Narayanan, Wu, Kumar, Newman, Yuan, Yan, Zhang, Cosgrove, Manning, Ré, Acosta-Navas, Hudson, Zelikman, Durmus, Ladhak, Rong, Ren, Yao, Wang, Santhanam, Orr, Zheng, Yuksekgonul, Suzgun, Kim, Guha, Chatterji, Khattab, Henderson, Huang, Chi, Xie, Santurkar, Ganguli, Hashimoto, Icard, Zhang, Chaudhary, Wang, Li, Mai, Zhang, and Koreeda]{liang_holistic_2023}
Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., Cosgrove, C., Manning, C.~D., Ré, C., Acosta-Navas, D., Hudson, D.~A., Zelikman, E., Durmus, E., Ladhak, F., Rong, F., Ren, H., Yao, H., Wang, J., Santhanam, K., Orr, L., Zheng, L., Yuksekgonul, M., Suzgun, M., Kim, N., Guha, N., Chatterji, N., Khattab, O., Henderson, P., Huang, Q., Chi, R., Xie, S.~M., Santurkar, S., Ganguli, S., Hashimoto, T., Icard, T., Zhang, T., Chaudhary, V., Wang, W., Li, X., Mai, Y., Zhang, Y., and Koreeda, Y.
\newblock Holistic {Evaluation} of {Language} {Models}, October 2023.
\newblock URL \url{http://arxiv.org/abs/2211.09110}.
\newblock arXiv:2211.09110 [cs].
\bibitem[Maia~Polo et~al.(2024)Maia~Polo, Weber, Choshen, Sun, Xu, and Yurochkin]{maia_polo_tinybenchmarks_2024}
Maia~Polo, F., Weber, L., Choshen, L., Sun, Y., Xu, G., and Yurochkin, M.
\newblock {tinyBenchmarks}: evaluating {LLMs} with fewer examples.
\newblock In \emph{Proceedings of the 41st International Conference on Machine Learning}, volume 235 of \emph{Proceedings of Machine Learning Research}, pp.\ 34303--34326. PMLR, 2024.
\newblock URL \url{https://proceedings.mlr.press/v235/maia-polo24a.html}.
\bibitem[McNeish \& Wolf(2023)McNeish and Wolf]{mcneish_dynamic_2023}
McNeish, D. and Wolf, M.~G.
\newblock Dynamic fit index cutoffs for one-factor models.
\newblock \emph{Behavior Research Methods}, 55\penalty0 (3):\penalty0 1157--1174, April 2023.
\newblock ISSN 1554-3528.
\newblock \doi{10.3758/s13428-022-01847-y}.
\newblock URL \url{https://doi.org/10.3758/s13428-022-01847-y}.
\bibitem[Medsker et~al.(1994)Medsker, Williams, and Holahan]{medsker_review_1994}
Medsker, G.~J., Williams, L.~J., and Holahan, P.~J.
\newblock A {Review} of {Current} {Practices} for {Evaluating} {Causal} {Models} in {Organizational} {Behavior} and {Human} {Resources} {Management} {Research}.
\newblock \emph{Journal of Management}, 20\penalty0 (2):\penalty0 439--464, April 1994.
\newblock ISSN 0149-2063.
\newblock \doi{10.1177/014920639402000207}.
\newblock URL \url{https://doi.org/10.1177/014920639402000207}.
\newblock Publisher: SAGE Publications Inc.
\bibitem[Muthén(1993)]{muthen_goodness_1993}
Muthén, B.
\newblock Goodness of fit with categorical and other nonnormal variables.
\newblock In Bollen, K.~A. and Long, J.~S. (eds.), \emph{Testing structural equation models}, pp.\ 205--234. Sage, Newbury Park, CA, 1993.
\newblock ISBN: 0803945078 Publisher: Sage.
\bibitem[Polo et~al.(2025)Polo, Somerstep, Choshen, Sun, and Yurochkin]{polo_sloth_2025}
Polo, F.~M., Somerstep, S., Choshen, L., Sun, Y., and Yurochkin, M.
\newblock Sloth: scaling laws for {LLM} skills to predict multi-benchmark performance across families, December 2025.
\newblock URL \url{http://arxiv.org/abs/2412.06540}.
\newblock arXiv:2412.06540 [cs] version: 5.
\bibitem[Rein et~al.(2023)Rein, Hou, Stickland, Petty, Pang, Dirani, Michael, and Bowman]{rein_gpqa_2023}
Rein, D., Hou, B.~L., Stickland, A.~C., Petty, J., Pang, R.~Y., Dirani, J., Michael, J., and Bowman, S.~R.
\newblock {GPQA}: {A} {Graduate}-{Level} {Google}-{Proof} {Q}\&{A} {Benchmark}, November 2023.
\newblock URL \url{http://arxiv.org/abs/2311.12022}.
\newblock arXiv:2311.12022 [cs].
\bibitem[Reuel et~al.(2024)Reuel, Hardy, Smith, Lamparth, Hardy, and Kochenderfer]{reuel_betterbench_2024}
Reuel, A., Hardy, A., Smith, C., Lamparth, M., Hardy, M., and Kochenderfer, M.~J.
\newblock {BetterBench}: {Assessing} {AI} {Benchmarks}, {Uncovering} {Issues}, and {Establishing} {Best} {Practices}, November 2024.
\newblock URL \url{http://arxiv.org/abs/2411.12990}.
\newblock arXiv:2411.12990 [cs] version: 1.
\bibitem[Rodriguez et~al.(2021)Rodriguez, Barrow, Hoyle, Lalor, Jia, and Boyd-Graber]{rodriguez_evaluation_2021}
Rodriguez, P., Barrow, J., Hoyle, A., Lalor, J.~P., Jia, R., and Boyd-Graber, J.
\newblock Evaluation examples are not equally informative: How should that change {NLP} leaderboards?
\newblock In \emph{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}, pp.\ 4486--4503, Online, 2021. Association for Computational Linguistics.
\newblock \doi{10.18653/v1/2021.acl-long.346}.
\newblock URL \url{https://aclanthology.org/2021.acl-long.346/}.
\bibitem[Rosseel(2012)]{rosseel_lavaan_2012}
Rosseel, Y.
\newblock lavaan: {An} {R} {Package} for {Structural} {Equation} {Modeling}.
\newblock \emph{Journal of Statistical Software}, 48:\penalty0 1--36, May 2012.
\newblock ISSN 1548-7660.
\newblock \doi{10.18637/jss.v048.i02}.
\newblock URL \url{https://doi.org/10.18637/jss.v048.i02}.
\bibitem[Sakaguchi et~al.(2019)Sakaguchi, Bras, Bhagavatula, and Choi]{sakaguchi_winogrande_2019}
Sakaguchi, K., Bras, R.~L., Bhagavatula, C., and Choi, Y.
\newblock {WinoGrande}: {An} {Adversarial} {Winograd} {Schema} {Challenge} at {Scale}, November 2019.
\newblock URL \url{http://arxiv.org/abs/1907.10641}.
\newblock arXiv:1907.10641 [cs].
\bibitem[Salaudeen et~al.(2025)Salaudeen, Reuel, Ahmed, Bedi, Robertson, Sundar, Domingue, Wang, and Koyejo]{salaudeen_measurement_2025}
Salaudeen, O., Reuel, A., Ahmed, A., Bedi, S., Robertson, Z., Sundar, S., Domingue, B., Wang, A., and Koyejo, S.
\newblock Measurement to {Meaning}: {A} {Validity}-{Centered} {Framework} for {AI} {Evaluation}, June 2025.
\newblock URL \url{http://arxiv.org/abs/2505.10573}.
\newblock arXiv:2505.10573 [cs].
\bibitem[Saris et~al.(2009)Saris, Satorra, and van~der Veld]{saris_testing_2009}
Saris, W.~E., Satorra, A., and van~der Veld, W.~M.
\newblock Testing {Structural} {Equation} {Models} or {Detection} of {Misspecifications}?
\newblock \emph{Structural Equation Modeling: A Multidisciplinary Journal}, 16\penalty0 (4):\penalty0 561--582, October 2009.
\newblock ISSN 1070-5511.
\newblock \doi{10.1080/10705510903203433}.
\newblock URL \url{https://doi.org/10.1080/10705510903203433}.
\newblock Publisher: Routledge \_eprint: https://doi.org/10.1080/10705510903203433.
\bibitem[Schaeffer et~al.(2025)Schaeffer, Levi, Miranda, and Koyejo]{schaeffer_pretraining_2025}
Schaeffer, R., Levi, N., Miranda, B., and Koyejo, S.
\newblock Pretraining {Scaling} {Laws} for {Generative} {Evaluations} of {Language} {Models}, September 2025.
\newblock URL \url{http://arxiv.org/abs/2509.24012}.
\newblock arXiv:2509.24012 [cs].
\bibitem[Sprague et~al.(2024)Sprague, Ye, Bostrom, Chaudhuri, and Durrett]{sprague_musr_2024}
Sprague, Z., Ye, X., Bostrom, K., Chaudhuri, S., and Durrett, G.
\newblock {MuSR}: {Testing} the {Limits} of {Chain}-of-thought with {Multistep} {Soft} {Reasoning}, March 2024.
\newblock URL \url{http://arxiv.org/abs/2310.16049}.
\newblock arXiv:2310.16049 [cs].
\bibitem[Srivastava et~al.(2023)Srivastava, Rastogi, Rao, Shoeb, Abid, Fisch, Brown, Santoro, Gupta, Garriga-Alonso, Kluska, Lewkowycz, Agarwal, Power, Ray, Warstadt, Kocurek, Safaya, Tazarv, Xiang, Parrish, Nie, Hussain, Askell, Dsouza, Slone, Rahane, Iyer, Andreassen, Madotto, Santilli, Stuhlmüller, Dai, La, Lampinen, Zou, Jiang, Chen, Vuong, Gupta, Gottardi, Norelli, Venkatesh, Gholamidavoodi, Tabassum, Menezes, Kirubarajan, Mullokandov, Sabharwal, Herrick, Efrat, Erdem, Karakaş, Roberts, Loe, Zoph, Bojanowski, Özyurt, Hedayatnia, Neyshabur, Inden, Stein, Ekmekci, Lin, Howald, Orinion, Diao, Dour, Stinson, Argueta, Ramírez, Singh, Rathkopf, Meng, Baral, Wu, Callison-Burch, Waites, Voigt, Manning, Potts, Ramirez, Rivera, Siro, Raffel, Ashcraft, Garbacea, Sileo, Garrette, Hendrycks, Kilman, Roth, Freeman, Khashabi, Levy, González, Perszyk, Hernandez, Chen, Ippolito, Gilboa, Dohan, Drakard, Jurgens, Datta, Ganguli, Emelin, Kleyko, Yuret, Chen, Tam, Hupkes, Misra, Buzan, Mollo, Yang, Lee, Schrader,
Shutova, Cubuk, Segal, Hagerman, Barnes, Donoway, Pavlick, Rodola, Lam, Chu, Tang, Erdem, Chang, Chi, Dyer, Jerzak, Kim, Manyasi, Zheltonozhskii, Xia, Siar, Martínez-Plumed, Happé, Chollet, Rong, Mishra, Winata, Melo, Kruszewski, Parascandolo, Mariani, Wang, Jaimovitch-López, Betz, Gur-Ari, Galijasevic, Kim, Rashkin, Hajishirzi, Mehta, Bogar, Shevlin, Schütze, Yakura, Zhang, Wong, Ng, Noble, Jumelet, Geissinger, Kernion, Hilton, Lee, Fisac, Simon, Koppel, Zheng, Zou, Kocoń, Thompson, Wingfield, Kaplan, Radom, Sohl-Dickstein, Phang, Wei, Yosinski, Novikova, Bosscher, Marsh, Kim, Taal, Engel, Alabi, Xu, Song, Tang, Waweru, Burden, Miller, Balis, Batchelder, Berant, Frohberg, Rozen, Hernandez-Orallo, Boudeman, Guerr, Jones, Tenenbaum, Rule, Chua, Kanclerz, Livescu, Krauth, Gopalakrishnan, Ignatyeva, Markert, Dhole, Gimpel, Omondi, Mathewson, Chiafullo, Shkaruta, Shridhar, McDonell, Richardson, Reynolds, Gao, Zhang, Dugan, Qin, Contreras-Ochando, Morency, Moschella, Lam, Noble, Schmidt, He, Colón, Metz,
Şenel, Bosma, Sap, Hoeve, Farooqi, Faruqui, Mazeika, Baturan, Marelli, Maru, Quintana, Tolkiehn, Giulianelli, Lewis, Potthast, Leavitt, Hagen, Schubert, Baitemirova, Arnaud, McElrath, Yee, Cohen, Gu, Ivanitskiy, Starritt, Strube, Swedrowski, Bevilacqua, Yasunaga, Kale, Cain, Xu, Suzgun, Walker, Tiwari, Bansal, Aminnaseri, Geva, Gheini, T, Peng, Chi, Lee, Krakover, Cameron, Roberts, Doiron, Martinez, Nangia, Deckers, Muennighoff, Keskar, Iyer, Constant, Fiedel, Wen, Zhang, Agha, Elbaghdadi, Levy, Evans, Casares, Doshi, Fung, Liang, Vicol, Alipoormolabashi, Liao, Liang, Chang, Eckersley, Htut, Hwang, Miłkowski, Patil, Pezeshkpour, Oli, Mei, Lyu, Chen, Banjade, Rudolph, Gabriel, Habacker, Risco, Millière, Garg, Barnes, Saurous, Arakawa, Raymaekers, Frank, Sikand, Novak, Sitelew, LeBras, Liu, Jacobs, Zhang, Salakhutdinov, Chi, Lee, Stovall, Teehan, Yang, Singh, Mohammad, Anand, Dillavou, Shleifer, Wiseman, Gruetter, Bowman, Schoenholz, Han, Kwatra, Rous, Ghazarian, Ghosh, Casey, Bischoff, Gehrmann, Schuster,
Sadeghi, Hamdan, Zhou, Srivastava, Shi, Singh, Asaadi, Gu, Pachchigar, Toshniwal, Upadhyay, Shyamolima, Debnath, Shakeri, Thormeyer, Melzi, Reddy, Makini, Lee, Torene, Hatwar, Dehaene, Divic, Ermon, Biderman, Lin, Prasad, Piantadosi, Shieber, Misherghi, Kiritchenko, Mishra, Linzen, Schuster, Li, Yu, Ali, Hashimoto, Wu, Desbordes, Rothschild, Phan, Wang, Nkinyili, Schick, Kornev, Tunduny, Gerstenberg, Chang, Neeraj, Khot, Shultz, Shaham, Misra, Demberg, Nyamai, Raunak, Ramasesh, Prabhu, Padmakumar, Srikumar, Fedus, Saunders, Zhang, Vossen, Ren, Tong, Zhao, Wu, Shen, Yaghoobzadeh, Lakretz, Song, Bahri, Choi, Yang, Hao, Chen, Belinkov, Hou, Hou, Bai, Seid, Zhao, Wang, Wang, Wang, and Wu]{srivastava_beyond_2023}
Srivastava, A., Rastogi, A., Rao, A., Shoeb, A. A.~M., Abid, A., Fisch, A., Brown, A.~R., Santoro, A., Gupta, A., Garriga-Alonso, A., Kluska, A., Lewkowycz, A., Agarwal, A., Power, A., Ray, A., Warstadt, A., Kocurek, A.~W., Safaya, A., Tazarv, A., Xiang, A., Parrish, A., Nie, A., Hussain, A., Askell, A., Dsouza, A., Slone, A., Rahane, A., Iyer, A.~S., Andreassen, A., Madotto, A., Santilli, A., Stuhlmüller, A., Dai, A., La, A., Lampinen, A., Zou, A., Jiang, A., Chen, A., Vuong, A., Gupta, A., Gottardi, A., Norelli, A., Venkatesh, A., Gholamidavoodi, A., Tabassum, A., Menezes, A., Kirubarajan, A., Mullokandov, A., Sabharwal, A., Herrick, A., Efrat, A., Erdem, A., Karakaş, A., Roberts, B.~R., Loe, B.~S., Zoph, B., Bojanowski, B., Özyurt, B., Hedayatnia, B., Neyshabur, B., Inden, B., Stein, B., Ekmekci, B., Lin, B.~Y., Howald, B., Orinion, B., Diao, C., Dour, C., Stinson, C., Argueta, C., Ramírez, C.~F., Singh, C., Rathkopf, C., Meng, C., Baral, C., Wu, C., Callison-Burch, C., Waites, C., Voigt, C., Manning,
C.~D., Potts, C., Ramirez, C., Rivera, C.~E., Siro, C., Raffel, C., Ashcraft, C., Garbacea, C., Sileo, D., Garrette, D., Hendrycks, D., Kilman, D., Roth, D., Freeman, D., Khashabi, D., Levy, D., González, D.~M., Perszyk, D., Hernandez, D., Chen, D., Ippolito, D., Gilboa, D., Dohan, D., Drakard, D., Jurgens, D., Datta, D., Ganguli, D., Emelin, D., Kleyko, D., Yuret, D., Chen, D., Tam, D., Hupkes, D., Misra, D., Buzan, D., Mollo, D.~C., Yang, D., Lee, D.-H., Schrader, D., Shutova, E., Cubuk, E.~D., Segal, E., Hagerman, E., Barnes, E., Donoway, E., Pavlick, E., Rodola, E., Lam, E., Chu, E., Tang, E., Erdem, E., Chang, E., Chi, E.~A., Dyer, E., Jerzak, E., Kim, E., Manyasi, E.~E., Zheltonozhskii, E., Xia, F., Siar, F., Martínez-Plumed, F., Happé, F., Chollet, F., Rong, F., Mishra, G., Winata, G.~I., Melo, G.~d., Kruszewski, G., Parascandolo, G., Mariani, G., Wang, G., Jaimovitch-López, G., Betz, G., Gur-Ari, G., Galijasevic, H., Kim, H., Rashkin, H., Hajishirzi, H., Mehta, H., Bogar, H., Shevlin, H.,
Schütze, H., Yakura, H., Zhang, H., Wong, H.~M., Ng, I., Noble, I., Jumelet, J., Geissinger, J., Kernion, J., Hilton, J., Lee, J., Fisac, J.~F., Simon, J.~B., Koppel, J., Zheng, J., Zou, J., Kocoń, J., Thompson, J., Wingfield, J., Kaplan, J., Radom, J., Sohl-Dickstein, J., Phang, J., Wei, J., Yosinski, J., Novikova, J., Bosscher, J., Marsh, J., Kim, J., Taal, J., Engel, J., Alabi, J., Xu, J., Song, J., Tang, J., Waweru, J., Burden, J., Miller, J., Balis, J.~U., Batchelder, J., Berant, J., Frohberg, J., Rozen, J., Hernandez-Orallo, J., Boudeman, J., Guerr, J., Jones, J., Tenenbaum, J.~B., Rule, J.~S., Chua, J., Kanclerz, K., Livescu, K., Krauth, K., Gopalakrishnan, K., Ignatyeva, K., Markert, K., Dhole, K.~D., Gimpel, K., Omondi, K., Mathewson, K., Chiafullo, K., Shkaruta, K., Shridhar, K., McDonell, K., Richardson, K., Reynolds, L., Gao, L., Zhang, L., Dugan, L., Qin, L., Contreras-Ochando, L., Morency, L.-P., Moschella, L., Lam, L., Noble, L., Schmidt, L., He, L., Colón, L.~O., Metz, L., Şenel, L.~K.,
Bosma, M., Sap, M., Hoeve, M.~t., Farooqi, M., Faruqui, M., Mazeika, M., Baturan, M., Marelli, M., Maru, M., Quintana, M. J.~R., Tolkiehn, M., Giulianelli, M., Lewis, M., Potthast, M., Leavitt, M.~L., Hagen, M., Schubert, M., Baitemirova, M.~O., Arnaud, M., McElrath, M., Yee, M.~A., Cohen, M., Gu, M., Ivanitskiy, M., Starritt, M., Strube, M., Swedrowski, M., Bevilacqua, M., Yasunaga, M., Kale, M., Cain, M., Xu, M., Suzgun, M., Walker, M., Tiwari, M., Bansal, M., Aminnaseri, M., Geva, M., Gheini, M., T, M.~V., Peng, N., Chi, N.~A., Lee, N., Krakover, N. G.-A., Cameron, N., Roberts, N., Doiron, N., Martinez, N., Nangia, N., Deckers, N., Muennighoff, N., Keskar, N.~S., Iyer, N.~S., Constant, N., Fiedel, N., Wen, N., Zhang, O., Agha, O., Elbaghdadi, O., Levy, O., Evans, O., Casares, P. A.~M., Doshi, P., Fung, P., Liang, P.~P., Vicol, P., Alipoormolabashi, P., Liao, P., Liang, P., Chang, P., Eckersley, P., Htut, P.~M., Hwang, P., Miłkowski, P., Patil, P., Pezeshkpour, P., Oli, P., Mei, Q., Lyu, Q., Chen, Q.,
Banjade, R., Rudolph, R.~E., Gabriel, R., Habacker, R., Risco, R., Millière, R., Garg, R., Barnes, R., Saurous, R.~A., Arakawa, R., Raymaekers, R., Frank, R., Sikand, R., Novak, R., Sitelew, R., LeBras, R., Liu, R., Jacobs, R., Zhang, R., Salakhutdinov, R., Chi, R., Lee, R., Stovall, R., Teehan, R., Yang, R., Singh, S., Mohammad, S.~M., Anand, S., Dillavou, S., Shleifer, S., Wiseman, S., Gruetter, S., Bowman, S.~R., Schoenholz, S.~S., Han, S., Kwatra, S., Rous, S.~A., Ghazarian, S., Ghosh, S., Casey, S., Bischoff, S., Gehrmann, S., Schuster, S., Sadeghi, S., Hamdan, S., Zhou, S., Srivastava, S., Shi, S., Singh, S., Asaadi, S., Gu, S.~S., Pachchigar, S., Toshniwal, S., Upadhyay, S., Shyamolima, Debnath, Shakeri, S., Thormeyer, S., Melzi, S., Reddy, S., Makini, S.~P., Lee, S.-H., Torene, S., Hatwar, S., Dehaene, S., Divic, S., Ermon, S., Biderman, S., Lin, S., Prasad, S., Piantadosi, S.~T., Shieber, S.~M., Misherghi, S., Kiritchenko, S., Mishra, S., Linzen, T., Schuster, T., Li, T., Yu, T., Ali, T.,
Hashimoto, T., Wu, T.-L., Desbordes, T., Rothschild, T., Phan, T., Wang, T., Nkinyili, T., Schick, T., Kornev, T., Tunduny, T., Gerstenberg, T., Chang, T., Neeraj, T., Khot, T., Shultz, T., Shaham, U., Misra, V., Demberg, V., Nyamai, V., Raunak, V., Ramasesh, V., Prabhu, V.~U., Padmakumar, V., Srikumar, V., Fedus, W., Saunders, W., Zhang, W., Vossen, W., Ren, X., Tong, X., Zhao, X., Wu, X., Shen, X., Yaghoobzadeh, Y., Lakretz, Y., Song, Y., Bahri, Y., Choi, Y., Yang, Y., Hao, Y., Chen, Y., Belinkov, Y., Hou, Y., Hou, Y., Bai, Y., Seid, Z., Zhao, Z., Wang, Z., Wang, Z.~J., Wang, Z., and Wu, Z.
\newblock Beyond the {Imitation} {Game}: {Quantifying} and extrapolating the capabilities of language models, June 2023.
\newblock URL \url{http://arxiv.org/abs/2206.04615}.
\newblock arXiv:2206.04615 [cs.CL].
\bibitem[Sun \& Wang(2026)Sun and Wang]{sun_breaking_2026}
Sun, K. and Wang, R.
\newblock Breaking {Myths} in {LLM} scaling and emergent abilities with a comprehensive statistical analysis.
\newblock \emph{Neurocomputing}, 670:\penalty0 132542, March 2026.
\newblock ISSN 0925-2312.
\newblock \doi{10.1016/j.neucom.2025.132542}.
\newblock URL \url{https://www.sciencedirect.com/science/article/pii/S092523122503214X}.
\bibitem[Suzgun et~al.(2022)Suzgun, Scales, Schärli, Gehrmann, Tay, Chung, Chowdhery, Le, Chi, Zhou, and Wei]{suzgun_challenging_2022}
Suzgun, M., Scales, N., Schärli, N., Gehrmann, S., Tay, Y., Chung, H.~W., Chowdhery, A., Le, Q.~V., Chi, E.~H., Zhou, D., and Wei, J.
\newblock Challenging {BIG}-{Bench} {Tasks} and {Whether} {Chain}-of-{Thought} {Can} {Solve} {Them}, October 2022.
\newblock URL \url{http://arxiv.org/abs/2210.09261}.
\newblock arXiv:2210.09261 [cs].
\bibitem[Takane \& De~Leeuw(1987)Takane and De~Leeuw]{takane_relationship_1987}
Takane, Y. and De~Leeuw, J.
\newblock On the {Relationship} between {iTem} {Response} {Theory} and {Factor} {Analysis} of {Discretized} {Variables}.
\newblock \emph{Psychometrika}, 52\penalty0 (3):\penalty0 393--408, September 1987.
\newblock ISSN 0033-3123, 1860-0980.
\newblock \doi{10.1007/BF02294363}.
\newblock URL \url{https://www.cambridge.org/core/product/identifier/S0033312300031495/type/journal_article}.
\bibitem[Thurstone(1935)]{thurstone_vectors_1935}
Thurstone, L.
\newblock \emph{The {Vectors} {Of} {Mind} {Multiple} {Factor} {Analysis} {For} {The} {Isolation} {Of} {Primary} {Traits}}.
\newblock The University Of Chicago Press, 1935.
\newblock URL \url{http://archive.org/details/vectorsofmindmul010122mbp}.
\bibitem[Truong et~al.(2025)Truong, Tu, Hardy, Reuel, Tang, Burapacheep, Perera, Uwakwe, Domingue, Haber, and Koyejo]{truong_fantastic_2025}
Truong, S., Tu, Y., Hardy, M., Reuel, A., Tang, Z., Burapacheep, J., Perera, J., Uwakwe, C., Domingue, B., Haber, N., and Koyejo, S.
\newblock Fantastic {Bugs} and {Where} to {Find} {Them} in {AI} {Benchmarks}, November 2025.
\newblock URL \url{http://arxiv.org/abs/2511.16842}.
\newblock arXiv:2511.16842 [cs].
\bibitem[Vania et~al.(2021)Vania, Htut, Huang, Mungra, Pang, Phang, Liu, Cho, and Bowman]{vania_comparing_2021}
Vania, C., Htut, P.~M., Huang, W., Mungra, D., Pang, R.~Y., Phang, J., Liu, H., Cho, K., and Bowman, S.~R.
\newblock Comparing test sets with item response theory.
\newblock In \emph{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}, pp.\ 1141--1158, Online, 2021. Association for Computational Linguistics.
\newblock \doi{10.18653/v1/2021.acl-long.92}.
\newblock URL \url{https://aclanthology.org/2021.acl-long.92/}.
\bibitem[Venables \& Ripley(2002)Venables and Ripley]{venables_modern_2002}
Venables, W.~N. and Ripley, B.~D.
\newblock \emph{Modern {Applied} {Statistics} with {S}}.
\newblock Springer, New York, fourth edition, 2002.
\newblock URL \url{https://www.stats.ox.ac.uk/pub/MASS4/}.
\bibitem[Wang et~al.(2024)Wang, Ma, Zhang, Ni, Chandra, Guo, Ren, Arulraj, He, Jiang, Li, Ku, Wang, Zhuang, Fan, Yue, and Chen]{wang_mmlu-pro_2024}
Wang, Y., Ma, X., Zhang, G., Ni, Y., Chandra, A., Guo, S., Ren, W., Arulraj, A., He, X., Jiang, Z., Li, T., Ku, M., Wang, K., Zhuang, A., Fan, R., Yue, X., and Chen, W.
\newblock {MMLU}-{Pro}: {A} {More} {Robust} and {Challenging} {Multi}-{Task} {Language} {Understanding} {Benchmark}, November 2024.
\newblock URL \url{http://arxiv.org/abs/2406.01574}.
\newblock arXiv:2406.01574 [cs].
\bibitem[Zhou et~al.(2023)Zhou, Lu, Mishra, Brahma, Basu, Luan, Zhou, and Hou]{zhou_instruction-following_2023}
Zhou, J., Lu, T., Mishra, S., Brahma, S., Basu, S., Luan, Y., Zhou, D., and Hou, L.
\newblock Instruction-{Following} {Evaluation} for {Large} {Language} {Models}, November 2023.
\newblock URL \url{http://arxiv.org/abs/2311.07911}.
\newblock arXiv:2311.07911 [cs].
\end{thebibliography}