SABER-Math: Automated Benchmark for Information Retrieval Evaluation in Mathematics
Extracted source text · arXiv 2606.29894
\documentclass{article}
\input{headers/math_commands.tex}
\input{headers/header-lqa}
\input{headers/header}
\input{headers/header-lqa-tail}
% \title{\bench: Automated Benchmark for Information Retrieval Evaluation in Mathematics}
% ICML running title (short form used in headers)
% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to LaTeX to determine where to break the
% lines. Using \AND forces a line break at that point. So, if LaTeX puts 3 of 4
% authors names on the first line, and the last on the second line, try using
% \AND instead of \And before the third author name.
\icmltitlerunning{\bench: Automated Benchmark for Information Retrieval Evaluation in Mathematics}
\begin{document}
\twocolumn[
\icmltitle{\bench: Automated Benchmark for Information Retrieval Evaluation in Mathematics}
% It is OKAY to include author information, even for blind submissions: the
% style file will automatically remove it for you unless you've provided
% the [accepted] option to the icml2026 package.
% List of affiliations: The first argument should be a (short) identifier you
% will use later to specify author affiliations Academic affiliations
% should list Department, University, City, Region, Country Industry
% affiliations should list Company, City, Region, Country
% You can specify symbols, otherwise they are numbered in order. Ideally, you
% should not use this facility. Affiliations will be numbered in order of
% appearance and this is the preferred way.
\icmlsetsymbol{equal}{*}
\vspace{-0.1in}
\begin{icmlauthorlist}
\icmlauthor{Nikolay Georgiev}{equal,insait}
\icmlauthor{Maria Drencheva}{equal,insait}
\icmlauthor{Kseniia Ibragimova}{insait-intern}
\icmlauthor{Ivo Petrov}{insait}
\icmlauthor{Dimitar I. Dimitrov}{insait}
\icmlauthor{Martin Vechev}{insait,eth}
%\icmlauthor{}{sch}
%\icmlauthor{}{sch}
%\icmlauthor{}{sch}
\end{icmlauthorlist}
\icmlaffiliation{insait}{INSAIT, Sofia University "St. Kliment Ohridski"}
\icmlaffiliation{insait-intern}{This work was done during an internship at INSAIT, Sofia University "St. Kliment Ohridski"}
\icmlaffiliation{eth}{ETH Zurich}
\icmlcorrespondingauthor{Ivo Petrov}{ivo.petrov@insait.ai}
% You may provide any keywords that you find helpful for describing your
% paper; these are used to populate the "keywords" metadata in the PDF but
% will not be shown in the document
\icmlkeywords{Machine Learning, ICML, Information Retrieval, Mathematics, Benchmark, Reasoning, LLMs, Embeddings, Evaluation}
\vskip 0.075in
\begin{center}
\raisebox{-0.16em}{\includegraphics[height=1em]{graphs/assets/github.png}}~
\href{https://github.com/insait-institute/sabermath/}{GitHub}
\quad
\raisebox{-0.16em}{\includegraphics[height=1em]{graphs/assets/huggingface-icon.png}}~
\href{https://huggingface.co/collections/INSAIT-Institute/saber-math}{Hugging Face}
\end{center}
\vskip 0.05in
]
% this must go after the closing bracket ] following \twocolumn[ ...
% This command actually creates the footnote in the first column listing the
% affiliations and the copyright notice. The command takes one argument, which
% is text to display at the start of the footnote. The \icmlEqualContribution
% command is standard text for equal contribution. Remove it (just {}) if you
% do not need this facility.
% Use ONE of the following lines. DO NOT remove the command.
% If you have no special notice, KEEP empty braces:
\printAffiliationsAndNotice{\icmlEqualContribution} % no special notice (required even if empty)
% Or, if applicable, use the standard equal contribution text:
% \printAffiliationsAndNotice{\icmlEqualContribution}
\input{abstract.tex}
\input{introduction.tex}
\input{related.tex}
\input{methodology.tex}
\input{results.tex}
\input{conclusion.tex}
% \input{limitations.tex}
\input{acknowledgements.tex}
\bibliographystyle{icml2026}
\bibliography{references}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix
\onecolumn
\input{appendix.tex}
\end{document}
\usepackage{amsmath,amsfonts,bm}
\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}
\newcommand{\newterm}[1]{{\bf #1}}
\def\figref#1{figure~\ref{#1}}
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
\def\secref#1{section~\ref{#1}}
\def\Secref#1{Section~\ref{#1}}
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
\def\eqref#1{equation~\ref{#1}}
\def\Eqref#1{Equation~\ref{#1}}
\def\plaineqref#1{\ref{#1}}
\def\chapref#1{chapter~\ref{#1}}
\def\Chapref#1{Chapter~\ref{#1}}
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
\def\algref#1{algorithm~\ref{#1}}
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
\def\partref#1{part~\ref{#1}}
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}
\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}
\def\eps{{\epsilon}}
\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}
\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}
\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}
\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}
\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}
\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}
\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}
\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}
\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}
\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}
\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}
\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}
\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}
\newcommand{\pdata}{p_{\rm{data}}}
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}
\newcommand{\laplace}{\mathrm{Laplace}} %
\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\attention}{\mathrm{attention}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}
\newcommand{\parents}{Pa} %
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\svd}{SVD}
\DeclareMathOperator{\sparsity}{sparsity}
\let\ab\allowbreak
\usepackage[utf8]{inputenc} %
\usepackage{lipsum} %
\usepackage[T1]{fontenc}
\usepackage{etoolbox}
\newbool{includeappendix}
\setbool{includeappendix}{true}
\input{headers/lqa/overfull}
\input{headers/lqa/comments}
\input{headers/lqa/colors}
\input{headers/lqa/listing}
\ifdefined\isoverfull
\overfullrule=1cm
\else
\fi
\newcommand{\todo}[1]{
\message{^^JTODO on page \thepage: #1^^J}
\textcolor{red}{\textbf{TODO:} #1}
}
\newenvironment{commentlist}{
\color{gray}
\begin{itemize}
}
{
\end{itemize}
}
\usepackage{xcolor} %
\definecolor{my-full-blue}{HTML}{1F77B4}
\definecolor{my-full-orange}{HTML}{FF7F0E}
\definecolor{my-full-green}{HTML}{2CA02C}
\definecolor{my-full-red}{HTML}{d62728}
\definecolor{my-full-purple}{HTML}{9467bd}
\colorlet{my-blue}{my-full-blue!30}
\colorlet{my-orange}{my-full-orange!30}
\colorlet{my-green}{my-full-green!30}
\colorlet{my-red}{my-full-red!30}
\colorlet{my-purple}{my-full-purple!30}
\usepackage{listings}
\usepackage{textcomp}
\usepackage{xcolor}
\usepackage[scaled=0.8]{beramono}
\definecolor{ckeyword}{HTML}{7F0055}
\definecolor{ccomment}{HTML}{3F7F5F}
\definecolor{cstring}{HTML}{2A0099}
\lstdefinestyle{numbers}{
numbers=left,
framexleftmargin=20pt,
numberstyle=\tiny,
firstnumber=auto,
numbersep=1em,
xleftmargin=2em
}
\lstdefinestyle{layout}{
frame=none,
captionpos=b,
}
\lstdefinestyle{comment-style}{
morecomment=[l]//,
morecomment=[s]{/*}{*/},
commentstyle={\color{ccomment}\itshape},
}
\lstdefinestyle{string-style}{
morestring=[b]",%
morestring=[b]',%
stringstyle={\color{cstring}},
showstringspaces=false,%
}
\lstdefinestyle{keyword-style}{
keywordstyle={\ttfamily\bfseries},
morekeywords={
function,
constructor,
int,
bool,
return,
returns,
uint
},
morekeywords = [2]{},
keywordstyle = [2]{\text},
sensitive=true,
}
\lstdefinestyle{input-encoding}{
inputencoding=utf8,
extendedchars=true,
literate=
{ℝ}{$\reals$}1%
{→}{$\rightarrow$}1%
{α}{$\alpha$}1%
{β}{$\beta$}1%
{λ}{$\lambda$}1%
{θ}{$\theta$}1%
{ϕ}{$\phi$}1%
}
\lstdefinestyle{escaping}{
moredelim={**[is][\color{blue}]{\%}{\%}},
escapechar=|,
mathescape=true
}
\lstdefinestyle{default-style}{
basicstyle=\fontencoding{T1}\ttfamily\footnotesize,
style=numbers,
style=layout,
style=comment-style,
style=string-style,
style=keyword-style,
style=input-encoding,
style=escaping,
tabsize=2,
upquote=true
}
\lstdefinelanguage{BASIC}{
language=C++,
style=default-style
}[keywords,comments,strings]%
\lstset{language=BASIC}
\usepackage{microtype}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs} %
\usepackage{hyperref}
\newcommand{\theHalgorithm}{\arabic{algorithm}}
\usepackage[accepted]{icml2026}
\usepackage[T1]{fontenc}
\usepackage{amsfonts} %
\usepackage{nicefrac} %
\usepackage{url}
\usepackage{xurl}
\usepackage{xcolor}
\usepackage{xspace}
\usepackage{etoolbox}
\usepackage{fontawesome}
\usepackage{enumitem}
\usepackage{amsmath,amssymb}
\usepackage{array}
\usepackage{multirow}
\usepackage{wrapfig}
\usepackage{adjustbox}
\usepackage{siunitx}
\usepackage{caption}
\usepackage{fontawesome}
\usepackage{stackengine}
\usepackage{svg}
\usepackage{mathtools}
\usepackage{xspace}
\usepackage{wrapfig}
\usepackage{makecell}
\usepackage{longtable}
\usepackage[most]{tcolorbox}
\usepackage{diagbox}
\usepackage{ulem}
\usepackage[capitalize,noabbrev]{cleveref}
\usepackage{colortbl}
\input{headers/lqa/tikz}
\usepackage{amsthm}
\usepackage{stmaryrd}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{tikz}
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
\DeclarePairedDelimiter{\iverson}{\llbracket}{\rrbracket}
\newcolumntype{x}[2]{S[table-format=#1.#2,table-auto-round]}
\newcolumntype{M}{>{$}l<{$}}
\renewcommand{\emph}[1]{\textit{#1}}
\newcommand{\bench}{\textsc{SABER-Math}\xspace}
\newcommand{\numina}{\textsc{NuminaMath-1.5}\xspace}
\newcommand{\deeptheorem}{\textsc{DeepTheorem}\xspace}
\newcommand{\maria}[1]{\textcolor{red}{\textbf{Maria:} #1}\xspace}
\newcommand{\niki}[1]{\textcolor{red}{\textbf{Niki:} #1}\xspace}
\newcommand{\ivo}[1]{\textcolor{red}{\textbf{Ivo:} #1}\xspace}
\newcommand{\dimy}[1]{\textcolor{red}{\textbf{Dimy:} #1}\xspace}
\newcommand{\qwenthreefour}{\textsc{Qwen3-4B-Embedding}\xspace}
\newcommand*\bluecircled[1]{\tikz[baseline=(char.base)]{
\node[shape=circle, fill=cyan!70!blue, text=white, inner sep=1.2pt, font=\sffamily\bfseries] (char) {#1};}}
\definecolor{acceptblue}{HTML}{6494EA}
\hypersetup{citecolor=acceptblue}
\definecolor{lightred}{HTML}{ffcbc7}
\definecolor{gemini}{HTML}{4285F4}
\definecolor{claude}{HTML}{f3e9d7}
\definecolor{deepseek}{HTML}{FADA4B}
\definecolor{qwen}{HTML}{FA574B}
\definecolor{oai}{HTML}{10a37f}
\definecolor{LightGreen}{HTML}{CCFFCC}
\lstdefinestyle{promptstyle}{
breaklines=true,
basicstyle=\scriptsize\ttfamily,
numbers=none,
language={},
framextopmargin=0pt,
framexbottommargin=0pt,
breakindent=0pt,
showspaces=false,
keywordstyle=\bfseries,
showstringspaces=false,
columns=fullflexible,
keepspaces=true,
mathescape=false,
texcl=false,
escapechar=,
escapeinside={(*@}{@*)},
morekeywords={Answer},
moredelim=**[is][\bfseries]{!!}{!!},
literate={|}{{\char`\|}}1
}
\newtcblisting{prompt}[2][]{
arc=3pt,
outer arc=3pt,
width=\linewidth,
left=1mm,
top=0mm,
bottom=0mm,
title=#2,
colback=lightred!5!white,
colframe=black,
fonttitle=\bfseries,
listing only,
listing options={style=promptstyle},
breakable,
#1
}
\newtcblisting{smallprompt}[2][]{
arc=3pt, outer arc=3pt,
width=0.9\linewidth,
left=1mm,
top=0mm,
bottom=0mm,
title=#2,
colback=lightred!5!white,
colframe=black,
fonttitle=\bfseries,
listing only,
listing options={style=promptstyle},
breakable,
#1
}
\newtcblisting{response}[2][]{
arc=3pt, outer arc=3pt,
width=0.9\linewidth,
left=1mm,
top=0mm,
bottom=0mm,
title=#2,
colback=lightred!5!white,
colframe=oai,
fonttitle=\bfseries,
listing only,
listing options={style=promptstyle},
breakable,
#1
}
\usepackage{tikz}
\usetikzlibrary{calc,decorations,decorations.pathmorphing}
\usetikzlibrary{positioning,fit,arrows}
\usetikzlibrary{decorations.markings}
\usetikzlibrary{shapes,shapes.geometric}
\usetikzlibrary{shadows,patterns,snakes}
\usetikzlibrary{backgrounds,decorations.pathreplacing,calligraphy,automata}
\usetikzlibrary{intersections}
\usetikzlibrary{angles,quotes}
\usetikzlibrary{plotmarks}
\usetikzlibrary{patterns}
\usetikzlibrary{arrows.meta}
\usetikzlibrary{shapes.misc}
\usetikzlibrary{chains}
\usetikzlibrary{shapes.callouts}
\input{headers/lqa/references}
\usepackage[capitalize]{cleveref}
\crefformat{section}{\S#2#1#3}
\crefrangeformat{section}{\S#3#1#4\crefrangeconjunction\S#5#2#6}
\crefmultiformat{section}{\S#2#1#3}{\crefpairconjunction\S#2#1#3}{\crefmiddleconjunction\S#2#1#3}{\creflastconjunction\S#2#1#3}
\newcommand{\crefrangeconjunction}{--}
\crefname{listing}{Lst.}{listings}
\crefname{line}{Lin.}{Lin.}
\crefname{appendix}{App.}{App.}
\newcommand{\app}[1]{%
\ifbool{includeappendix}{\cref{#1}}{the appendix}%
}
\newcommand{\App}[1]{%
\ifbool{includeappendix}{\cref{#1}}{The appendix}%
}
\begin{abstract}
As agentic AI systems tackle more complex mathematical tasks, they increasingly rely on information retrieval (IR) to search problem databases, theorem libraries, and educational resources. However, choosing the right retriever remains difficult, as it is infeasible to directly isolate its effect on downstream performance. On the other hand, existing retrieval-specific benchmarks often fail to capture fine-grained mathematical relevance, penalizing relevant documents. We address this gap by introducing \bench, the first fully automated benchmark for evaluating mathematical IR without expert annotation. Starting from 283K high-school-level math problems with solutions, \bench builds challenging reranking tasks in three steps: (i) first, LLMs extract concise solution summaries and mathematical topics for each problem; (ii) then, per-query relevant documents are discovered using ontology topic-based and lexical solutions-summary-based similarities, and (iii) finally, a Swiss-style LLM preference tournament produces fine-grained relevance ratings for the documents. We evaluate lexical retrievers, specialized mathematical retrieval systems, and recent embedding models. We find that while modern embedding models substantially outperform classical and math-specific baselines, even the strongest systems struggle in symbol-heavy domains like Algebra and Calculus. Importantly, we show that general-purpose IR benchmarks such as MTEB do not reliably predict mathematical performance, especially for recent embedding models, highlighting the need for math-specific retrieval benchmarks.
\end{abstract}
\begin{figure*}[t]
\centering
\hspace*{-0.22cm}%
\includegraphics[width=1.025\linewidth]{graphs/overview.pdf}
\caption{Overview of \bench construction. First, we source a large mathematical corpus of problems and their solutions \bluecircled{1}. Using LLM annotation, we compute two separate relevance signals for every document pair based on topics and solution summaries \bluecircled{2}. We then select 1000 target query documents, each with 150 relevant documents split evenly across relevance signals \bluecircled{3}. Finally, we rank the candidates per query using an LLM-as-a-judge in a 20-round Swiss tournament based on a Bradley-Terry model \bluecircled{4}. Throughout the figure our query documents are in blue, saturated purple denotes topic relevance, saturated teal denotes solution-summary relevance, and muted colors denote irrelevant documents. }
\label{fig:overview}
\vspace{-1em}
\end{figure*}
\section{Introduction}
Large language models are increasingly used in mathematical reasoning systems, serving both as natural-language assistants and components of formal reasoning pipelines~\citep{reasoningsurvey,reasoningsurvey2}. To improve their effectiveness and reliability, modern agentic systems often supplement proof generation with retrieval of relevant mathematical knowledge. Information retrieval (IR)~\citep{irsurvey} helps these systems find analogous examples, relevant theorems, solution patterns, formal premises, and definitions that can guide multi-step reasoning, making retrieval a core component of mathematical AI systems.
\paragraph{Information retrieval for mathematics}
Retrieval has become important in both formal theorem proving and in informal mathematical problem solving. In formal settings, IR is used to expose relevant definitions, lemmas, declarations~\citep{yang2023leandojo,shen2025real,kozyrev2025rocqstar}, library code~\citep{gao2024semantic,shen2025real}, and previously written proofs to assist a prover or autoformalizer~\citep{liu2025rethinking}. In informal settings, it is used to ground answers in trusted mathematical material and to provide analogous examples, solution schemas, or theorem-level context~\citep{levonian2023retrieval, yang2024analogy}. Although these systems differ in what they retrieve and how they retrieve it, they share a common requirement: retrieved documents should be useful for mathematical reasoning, rather than simply textually similar to the query. Yet, such systems rarely justify their choice of retrieval framework, making rigorous evaluation of mathematical retrieval necessary for building more effective mathematical agents.
\paragraph{Challenges of evaluating mathematical retrieval}
Evaluating mathematical retrieval is challenging because related mathematical problems are relatively infrequent, and their relevance is often not captured by surface similarity. Two problems may use different notation, wording, or terminology while relying on the same method, e.g., an extremal argument, invariant, or auxiliary construction. Conversely, problems with similar objects or formulas may require very different reasoning. Prior work~\citep{bright,xiao2024rar,mathnet} has therefore relied on expert annotation of semantic relevance, but such annotation is costly, hard to scale, and prone to annotator disagreement. Moreover, to keep relevance assessment tractable, many benchmarks use narrow relevance definitions and low-granularity labels~\citep{bright,mirb,mathnet}. This penalizes genuinely useful retrievals that do not match the benchmark's specific notion of relevance. All in all, existing methods thus remain hard to scale to larger corpora and diverse mathematical domains, while preserving a fine-grained notion of relevance.
\paragraph{SABER-Math}
To address this gap, we introduce \textbf{\bench}, the \textbf{S}calable \textbf{A}utomated \textbf{BE}nchmark for \textbf{R}etrieval in \textbf{Math}. To our knowledge, it is the first fully-automated benchmark designed specifically for fine-grained evaluation of mathematical information retrieval without requiring expert annotation.
\paragraph{Scalable relevant problem pair discovery}
To address the issues of rare relevant problem pairs, \bench starts from a large corpus of $283$K high-school and olympiad-style problem-solution pairs and uses two scalable complementary relevance signals to find relevant problem pairs. First, we assign each problem topics in a mathematical ontology and use topic overlap to find problems involving related concepts, theorems, or domains. Second, we extract short LLM-generated summaries of each solution's core idea and use lexical overlap between summaries to find problems with similar solution methods. These signals are not used as final relevance labels. Instead, we use them only for candidate selection: we identify $1000$ query problems across Algebra, Geometry, Number Theory, Combinatorics, and Calculus, such that we can pair each query with $150$ candidate documents spanning different forms of mathematical relatedness.
\paragraph{Automated relevance estimation}
After constructing each query's candidate set, \bench assigns fine-grained relevance labels for each candidate using pairwise LLM judgments. To make this scalable, we avoid exhaustive all-pairs comparison and instead use a Swiss-style tournament, which prioritizes comparisons between candidates of similar estimated relevance. We aggregate the resulting preferences with a Bradley--Terry model to obtain continuous relevance scores, which serve as ground-truth ratings for our benchmark evaluation.
\paragraph{Experimental results}
We evaluate sparse lexical retrievers such as BM25 and TF-IDF, specialized mathematical retrieval frameworks such as Approach Zero~\citep{approachzero}, and embedding-based retrievers ranging from early pretrained encoders to recent state-of-the-art models. Among all evaluated models, recent embedding models substantially outperform everything else, with the Octen open model family~\citep{octen2025rteb} ranking highest for most domains, followed closely by the closed Gemini-Embedding-2 model~\citep{geminiembedding}. Among newer embedding models, however, we show that performance on general-purpose retrieval benchmarks such as MTEB is not a reliable predictor for mathematical performance, demonstrating the need for our work. We also provide a counterfactual analysis of models' reliance on mathematical notation, showing that weaker retrievers over-rely on symbols and formulas without capturing their meaning, which hurts their performance.
\paragraph{Key contributions} Our contributions include:
\begin{itemize}
\item \bench, a scalable automated benchmark for mathematical information retrieval.
\item An extensive evaluation of classical, specialized, and embedding-based retrievers, showing that general-purpose retrieval benchmarks do not fully predict mathematical IR performance.
\item Domain-level and counterfactual analyses of retriever behavior, identifying weaknesses in symbolic domains.
\item Publicly-available code and data to support reproducibility and further research.
\end{itemize}
\section{Related Work} \label{sec:related}
In this section, we discuss related work on evaluating information retrieval, its applications in mathematical reasoning, and the use of LLMs as data processors and judges.
\paragraph{General-purpose retrieval benchmarks}
General-purpose retrieval benchmarks such as BEIR~\citep{beir} and MTEB~\citep{mteb} have become standard for comparing lexical, dense, and reranking models across domains and task types. These resources are useful for testing generalization and overall embedding quality, but they mainly measure general-purpose passage relevance: topical overlap, answer containment, or semantic rephrasing. When retrieval necessitates in-depth reasoning, it remains unclear how well general retrieval capabilities transfer to mathematical domains.
\paragraph{Agentic workflows with retrieval}
Agentic LLM systems interleave planning, tool use~\citep{archon}, memory~\citep{park2023generative,memgpt}, and self-reflection~\citep{selfrefine}, with retrieval serving as a common tool action for grounding intermediate reasoning in external evidence or domain context~\citep{irsurvey,reasoningsurvey,rag,webgpt}. In such workflows, retrieval quality matters, as irrelevant or incomplete context can derail subsequent reasoning, while well-targeted retrieval can support completion of harder tasks and reduce hallucinations~\citep{clasheval,hong2024gullible,yoran2024making}. \bench abstracts away the agentic scaffolding and directly evaluates retriever performance.
\paragraph{Retrieval in mathematical reasoning and theorem proving}
Mathematical retrieval spans a continuum from informal problem similarity to formal premise selection. In proof assistants, accurate retrieval has long been studied as premise selection for advancing the formal proof~\citep{irving2016deepmath,holist}. LLM-based provers continue this line: ReProver~\citep{yang2023leandojo} and REAL-Prover~\citep{shen2025real} retrieve a set of premises conditioned on the formal Lean proof state code. At the same time, newer systems increasingly retrieve over natural-language mathematical statements, even when the final goal is formal reasoning.
\citet{gao2024semantic} and \citet{liu2025rethinking} do so by informalizing mathlib4 statements for proving and autoformalization respectively, while Archon~\citep{archon} combines informal retrieval with formal search to translate informal arguments into machine-checkable developments. Beyond proof assistants, informal retrieval is also used to ground mathematical question answering, retrieve solution schemas or analogous problems, and construct mathematical knowledge graphs~\citep{levonian2023retrieval,dixit2024sbi,yang2024analogy,bian2025automathkg}. However, these works typically evaluate retrieval only on downstream performance, making it difficult to isolate the quality of the retriever itself. Such measurements conflate retriever quality with the generator's reasoning ability, prior knowledge, prompt sensitivity, and robustness to irrelevant context. A model may succeed despite poor retrieval or fail despite useful retrieval, so downstream performance alone does not reliably isolate the retriever's contribution. Our work therefore focuses on informal mathematical IR as a standalone problem.
\paragraph{Mathematical retrieval benchmarks and relevance coverage}
Several recent benchmarks evaluate mathematical or reasoning-intensive retrieval more directly. BRIGHT~\citep{bright} evaluates mathematical relevance through a shared theorem or solving method, and MathNet~\citep{mathnet} studies retrieval of mathematically equivalent or structurally similar Olympiad problems.
However, these works rely on human expert annotation, which is costly and difficult to scale to larger corpora or diverse mathematical domains.
Other works provide automatic pipelines, with MIRB~\citep{mirb} unifying semantic statement, premise, QA, and formula retrieval, while RAR-b~\citep{xiao2024rar} converts reasoning datasets, including MATH~\citep{hendrycksmath} and GSM8K~\citep{gsm8k}, into retrieval over pooled answer candidates.
While these benchmarks improve scalability, their construction can still lead to partial relevance coverage. Pooling-based collections leave many query-document pairs unvalidated, so relevant items surfaced by new systems may be treated as irrelevant or receive no credit, potentially introducing false negatives and affecting model comparisons~\citep{buckley2004retrieval,sakai2008incomplete}. Recent work shows that missing judgments matter, but also that repairing them reliably is difficult~\citep{upadhyay2024patch,abbasiantaeb2024holes}.
This issue is especially acute in mathematics, where useful relationships are sparse and fine-grained: two problems may share a common method without being equivalent.
\bench is designed with this problem in mind, as relevance is estimated on a continuous scale, and evaluates retrievers on a task where each document has been sufficiently validated.
\paragraph{LLMs as data processors and judges}
LLMs are increasingly used to process data at scale, including annotation~\citep{bansal2023annotation,gilardi2023chatgpt}, preference estimation~\citep{thomas2024preferences}, and evaluation~\citep{zheng2023judging,upadhyay2024patch,wildbench}. In particular, strong LLM judges can align well with human judgments in open-ended tasks, but also exhibit known biases and therefore require task-specific validation~\citep{zheng2023judging,justiceorprejudice,selfpreferencebias,jasper,judgingthejudges,verbositybias}. In particular, individual pointwise scores can be noisy or non-transitive~\citep{liu2024aligning,arenahard}, making it difficult to reliably measure subjective criteria such as relevance~\citep{proofrank}. To address these issues, prior work has used pairwise comparisons with Bradley-Terry-style aggregation, converting many local preferences into a global ranking that can reduce noise and mitigate bias~\citep{zheng2023judging,liu2024aligning,jasper}. In \bench, we adopt this framework to generate fine-grained relevance scores, validate it against human annotations, and show that it is preferable to standard ordinal ranking methods. We also use LLMs more broadly for data processing throughout the benchmark.
\section{Methodology}\label{sec:tech}
\input{outline.tex}
\input{sourcing_corpus.tex}
\input{bma.tex}
\input{jaccard.tex}
\input{filtering.tex}
\input{bradley-terry.tex}
In this section, we describe the construction of \bench, our benchmark for evaluating information retrieval in mathematics. \bench is designed to test whether retrieval systems can identify mathematically meaningful relevance between problems, such as shared concepts, reasoning patterns, or solution structure, rather than relying on surface-level textual similarity.
We begin by building \bench from a large corpus of high-school-level problems and solutions (\cref{sec:tech:data}). To identify related problem pairs at scale, we use two complementary automatically extracted relevance signals: topic overlap between problems (\cref{sec:tech:topics}) and lexical similarity between short LLM-extracted solution summaries (\cref{sec:tech:solsummaries}). We then construct reranking tasks by selecting a set of query and candidate problems of varying types and strengths of relatedness~(\cref{sec:tech:seclectbench}). Finally, we assign fine-grained relevance ratings to the candidates using an LLM-as-a-judge pairwise preference framework (\cref{sec:tech:rating}), which serve as reference labels for evaluating performance in \bench. All prompts are included in \cref{sec:app:prompts}. Additionally, we provide information about the computational resources used to construct \bench in \cref{sec:app:compute}.
\subsection{Sourcing a Large Mathematical Corpus}\label{sec:tech:data}
Meaningful mathematical relevance is sparse: few problem pairs share non-trivial concepts, reasoning patterns, or solution structure. To identify such pairs at scale, we build a 283K-sample corpus of mostly high-school-level problems and solutions across Algebra, Geometry, Number Theory, Combinatorics, and Calculus.
The corpus contains $\sim 150K$ AoPS forum problems and community solutions, providing broad but potentially lower-quality heterogeneous coverage. We further curated $\sim 35K$ problems from national and international competitions, such as IMO, USAMO, and RMM, including higher-quality solutions with deeper structure. To reduce reliance on noisy forum data, we also include $\sim 115K$ non-synthetic problems from the olympiad split of \numina.
We deduplicate across sources using DataTrove's MinHash pipeline~\citep{datatrove}, prioritizing official sources, \numina, and AoPS in this order, removing $\sim 17K$ duplicates. We present further data collection and preprocessing details in \cref{sec:app:data}.
\subsection{Analyzing Problem Pairs using Extracted Topics}\label{sec:tech:topics}
Overlapping objects, theorems, subfields and topics provide a strong signal of mathematical relevance beyond textual similarity, despite potentially different phrasing across documents. In this section, we explain how we capture this notion by constructing a MathWorld-derived ontology~\citep{MathWorld} (\cref{sec:tech:ontology}), assigning topics to problems, and computing pairwise relevance from topic sets using the hierarchy (\cref{sec:tech:ontologymatching}).
\subsubsection{Mathematical Ontology}\label{sec:tech:ontology}
We represent each problem by a set of mathematical topics and theorems, which provide good signals of mathematical relatedness. Because related topics may differ lexically while remaining close in the subject hierarchy, we use a fixed ontology tree rather than treating topics as a flat set.
\paragraph{Ontology tree construction}
We build our ontology from Wolfram's MathWorld~\citep{MathWorld}, whose entries provide broad coverage of high-school, olympiad, and university mathematics and are linked hierarchically across mathematical areas, concepts, and theorems. We represent each MathWorld topic as a node and each hierarchical link as a directed edge from a more general topic to a more specific one, yielding a directed acyclic graph (DAG). This graph is not generally a tree: a topic can appear along multiple root-to-topic paths because mathematical concepts often belong to several areas. For example, \textit{Pascal's Triangle} appears under both a combinatorial branch, $\text{Discrete Mathematics} \rightarrow \text{Combinatorics}$, and a number-theoretic branch, $\text{Number Theory} \rightarrow \text{Special Numbers}$, reflecting different uses and contexts. To preserve this information while obtaining a tree structure, we duplicate nodes with multiple root-to-topic paths. This lets us disambiguate topic assignments by context when annotating problems from our corpus. The conversion increases the ontology size from $\sim 10.5K$ to $\sim 17.1K$ nodes, while the maximum root-to-node depth remains 5.
\subsubsection{Scoring Problem Pair Relevance based on Ontologies}\label{sec:tech:ontologymatching}
\paragraph{Automatic topic assignment} To calculate problem pairs relevance signals, we need to first extract relevant ontology topics for each problem in our mathematical corpus. Due to the size of our ontology, presenting it to an LLM in full is infeasible. Instead, for each problem we do a top-down recursive traversal of the ontology. At each visited ontology node we rely on \textsc{GPT-OSS-120B}~\citep{oss} to do local relevance scoring. The model is called with the problem statement, solution, and the ontology labels of the node and its children, and is asked to assign each candidate a relevance score, which we then normalize across nodes. We retain candidates whose normalized scores exceed a threshold $\tau = 0.85$ and recursively apply the same procedure to their descendants. By having the parent node as a reference, we allow the algorithm to terminate early when the children are substantially less relevant than their parent, preventing redundant computation. Finally, we keep only the most specific assigned topics by removing topic ancestors.
\paragraph{Topic similarity over ontologies}
To calculate the relevance signal for a pair of problems with annotated topics, we optimally match the two problems topic sets based on topic similarity. Our final relevance signal is given by the average similarity score within this optimal match. We measure topic similarity using Lin's information-content similarity \citep{linsim}, where two ontology nodes are more similar if their lowest common ancestor is more informative. We estimate information content from topic frequencies over our problem corpus. Concretely, let $c(t)$ be the number of annotations for node $t$ or any of its descendants, and $N$ be the total number of topic annotations. We define
\begin{equation}
p(t)=\frac{c(t)}{N},
\qquad
IC(t)=-\log p(t).
\end{equation}
For two nodes $t_1$ and $t_2$, we compute the similarity:
\begin{equation}
\operatorname{sim}_{\mathrm{Lin}}(t_1,t_2)
=
\frac{
2IC(\operatorname{LCA}(t_1,t_2))
}{
IC(t_1)+IC(t_2)+\varepsilon
},
\end{equation}
where $\operatorname{LCA}(t_1,t_2)$ is the nodes' lowest common ancestor in the ontology tree and $\varepsilon=10^{-12}$ is used for numerical stability. This assigns high similarity to topics sharing a rare, specific ancestor, and low similarity to topics sharing only broad ancestors, capturing both exact and nearby ontology overlap.
\paragraph{Topic set matching}
Given the node-level similarities above, we lift them to problem-level similarity using Best-Match Average (BMA)~\citep{bma}. For problems $x$ and $y$ with topic sets $T_x=\{t_1,\ldots,t_m\}$ and $T_y=\{u_1,\ldots,u_n\}$, we define
\begin{equation}
\begin{aligned}
\operatorname{sim}_{\mathrm{topic}}(x,y)
&=
\frac{1}{|T_x|+|T_y|}
\Bigg(
\sum_{t\in T_x}\max_{u\in T_y}
\operatorname{sim}_{\mathrm{Lin}}(t,u)
\\
&\qquad\qquad
+
\sum_{u\in T_y}\max_{t\in T_x}
\operatorname{sim}_{\mathrm{Lin}}(t,u)
\Bigg).
\end{aligned}
\end{equation}
BMA matches each topic to its closest counterpart in the other problem and averages these matches in both directions. This rewards broad conceptual alignment while penalizing unmatched topics, which is useful when problems combine several mathematical concepts of unequal importance. We use $\operatorname{sim}_{\mathrm{topic}}$ as the topic-based relevance signal for selecting candidate pairs in \cref{sec:tech:seclectbench}.
\subsection{Scoring Problem Pair Relevance using Short Solution Summaries}\label{sec:tech:solsummaries}
While topic relevance captures an important aspect of overall mathematical relevance, problems may share underlying solution strategies despite belonging to completely different topics, or conversely, require substantially different methods even when they share topics. To capture this complementary notion of relevance, we use LLMs to extract concise natural-language summaries from problem solutions, retaining the core methodology while abstracting away formatting and presentation details. We then estimate the methodological relevance using these extracted summaries at scale via the Jaccard similarity~\citep{jaccard}.
\paragraph{Automatic summary extraction}
For each problem-solution pair in our corpus, we use \textsc{GPT-OSS-120B} to extract a summary of the solution's core idea. We prompt the model to produce summaries of at most 30 words in an imperative style, resulting in hint-like descriptions emphasizing the main method without reproducing the full solution, or including computational and algebraic details.
\paragraph{Summary-based relevance estimation}
For this signal, we measure the lexical overlap between the summaries of two problems using Jaccard similarity. For a problem $p$, let $S_p$ denote its extracted short solution summary. Let $T(s)$ be the set of unique words appearing in a sentence $s$. For a pair of problems $x$ and $y$, we define their summary-based relevance score as
\begin{equation}
\operatorname{sim}_{\mathrm{summ}}(x,y)
:=
\frac{
|T(S_x) \cap T(S_y)|
}{
|T(S_x) \cup T(S_y)|
}.
\end{equation}
We use lexical rather than embedding-based similarity to avoid biasing \bench toward models used in the benchmark construction. Further, the choice of lightweight metrics, such as BMA and Jaccard also allows us to scale computation efficiently, as we require deriving the pairwise relevance signals across the entire corpus, the number of which grows quadratically ($\sim7.8\times10^{10}$ total pairs). More details on the implementation can be found in \cref{sec:app:jaccard_details}.
\subsection{Selecting Queries and Document Sets}\label{sec:tech:seclectbench}
We construct \bench's query and document sets from our problem corpus using our topic- and solution-summary signals. A pair is topic-relevant if $\operatorname{sim}_{\mathrm{topic}}>\tau_{topic}$ and summary-relevant if $\operatorname{sim}_{\mathrm{summ}}>\tau_{summ}$, where we choose $\tau_{topic}\approx 0.858$ and $\tau_{summ}\approx 0.211$ to obtain a representative sample spanning multiple domains. More details on how we determined these values can be found in \cref{sec:app:thresholds}. To ensure a balanced and challenging benchmark, we require each query to have at least $K=50$ relevant documents in each of three categories: topic-only, summary-only, and both topic- and summary-relevant. We then sample $K$ documents uniformly from each category, giving $150$ documents per query. The final benchmark contains $1000$ queries, selected to uniformly span the five top-level ontology topics: Algebra, Geometry, Number Theory, Combinatorics, and Calculus and Analysis (see \cref{fig:piechart-benchmark}).
\subsection{Tournament-based Document Relevance Ratings}
\label{sec:tech:rating}
While our topic- and summary-based signals provide broad evidence of mathematical relevance, they still capture only specific forms of relatedness. As a result, using them as final relevance labels in our benchmark could over-penalize other relevant connections between problems. To obtain more fine-grained benchmark ratings, we instead use a pairwise LLM-as-a-judge framework. We aggregate these judgments with a Bradley--Terry model to produce continuous relevance scores, and use a Swiss-style tournament to reduce the computation.
\paragraph{Pairwise preference judging with an LLM-as-a-Judge}
We assign each candidate its final relevance rating using an LLM-as-a-judge framework~\citep{chan2023chateval,zheng2023judging,liu2024aligning,dubois2024length}. Since judging mathematical relevance requires comparing solution ideas, proof structure, and techniques, we use a strong reasoning model, \textsc{GPT-OSS-120B}, as our judge. Rather than eliciting absolute relevance scores, we ask the judge for pairwise preferences, following prior work showing that comparative judgments are more reliable than direct scoring~\citep{arenahard}. This finding is also confirmed by our human evaluation experiment in \cref{sec:app:human_evaluation}. For each query problem and its solution, we ask the judge to compare two candidate problems and their solutions and pick the one that is more mathematically relevant to the query.
\paragraph{Bradley--Terry estimation of latent relevance}
To convert judge preferences into continuous relevance scores, similarly to \citet{jasper}, we fit a Bradley-Terry (BT) model separately for each query. Under this model, the probability that candidate $i$ is judged to be more relevant than candidate $j$ by the LLM is assumed to be:
\begin{equation*}
P(i > j) = \frac{\pi_i}{\pi_i + \pi_j},
\end{equation*}
where $\pi_i$ is candidate $i$'s latent relevance score.
\paragraph{Scaling pairwise comparisons with a Swiss-style tournament}
Exhaustive pairwise judging requires $\frac{n(n-1)}{2}$ LLM calls, which is prohibitively expensive for large reranking sets, and wastes calls on clearly unequal candidates without gaining much information. We therefore use a Swiss-style tournament to select an informative subset of comparisons:
\begin{enumerate}[leftmargin=*, label=\arabic*.]
\item \textbf{Initialization:} Candidates paired uniformly at random.
\item \textbf{Iterative matching:} In each subsequent round, each candidate is paired with an unjudged opponent whose cumulative win count is as close as possible.
\end{enumerate}
The tournament concentrates judge calls on close comparisons, which are the most informative. We use $R=20$ tournament rounds, as we find this to be sufficient in \cref{sec:exp:swiss}. Further details on how we run the tournament are provided in \cref{sec:app:swiss_details}. Finally, we fit the BT model by maximizing the likelihood of the obtained pairwise judgments. These scores are linearly rescaled to the range $[0,5]$ and used as the final relevance ratings for each query.
\section{Results}\label{sec:experimental}
In this section, we present our experimental findings. In \cref{sec:results:main}, we describe the main results of \bench{}, identifying the most effective retrievers for mathematics, and analyzing different task settings across various mathematical domains. Then, in \cref{sec:results:math_vs_words}, we analyze the behaviour of different retrievers with respect to mathematical and textual content. Finally, in \cref{sec:exp:swiss}, we ablate the effectiveness of our Swiss-tournament, while in \cref{sec:results:signal_ablation} we demonstrate that both of our relevance signals contribute significantly to finding relevant problems. %
\input{main_results.tex}
\input{exp_word_vs_text.tex}
\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth, keepaspectratio]{graphs/average_inversions.pdf}
\vspace{-1.75em}
\caption{Average number of inversions between orderings produced by factoring all pairs compared to a Swiss tournament.}
\label{fig:avg-inv}
\vspace{1em}
\includegraphics[width=\columnwidth, keepaspectratio]{graphs/3-color.pdf}
\vspace{-1.75em}
\caption{Cumulative rank distribution of candidates selected by different relevance signals. Higher curves indicate that candidates from that group appear earlier in the final LLM-derived ranking.}
\label{fig:3color-exp}
\vspace{-1em}
\end{figure}
\input{exp_num_rounds.tex}
\input{exp_high_bma_jacc.tex}
\subsection{\bench{} Results and Key takeaways} \label{sec:results:main}
We report the main \bench{} results in \cref{tab:statement-full}. All scores are nDCG@10 with exponential gain, using our relevance ratings as reference scores. We report both aggregate performance and performance across five mathematical domains. Details on how embedding models are run are provided in \cref{sec:app:embedding_details}.
Unless otherwise stated, we present results in the \emph{statement--full} setting: the query contains only a problem statement, while each document contains both a problem statement and its solution. This captures a central use case in mathematical reasoning: given a new problem, a system retrieves solved examples with relevant concepts, theorems, or solution techniques that can guide downstream reasoning~\citep{dixit2024sbi,archon,kozyrev2025rocqstar}.
In \cref{tab:statement-full}, we also evaluate two complementary settings. In \emph{full--full} retrieval, both queries and documents include statements and solutions. This models retrieval over solved mathematical material, where a downstream system may ground answers in trusted sources, retrieve analogous examples, extract solution schemas, or construct mathematical knowledge graphs from problem-solution corpora~\citep{levonian2023retrieval,yang2024analogy,bian2025automathkg}. In \emph{statement--statement} retrieval, both queries and documents contain only problem statements. This captures search over large mathematical corpora that do not necessarily include human-readable proofs or solutions, including theorem-statement retrieval, informalized formal-library search, and retrieval over libraries of mathematical declarations~\citep{yang2023leandojo,shen2025real,gao2024semantic}. Comparing these settings lets us isolate how available context affects mathematical IR.
\input{tables/main_prob_full.tex}
\paragraph{Octen-Embedding-8B is the strongest overall retriever}
\textsc{Octen-Embedding-8B}, an open model, achieves the best aggregate performance and is also among the strongest models in all individual domains. It consistently outperforms other competitive systems, including the slightly smaller \textsc{Octen-Embedding-4B}, the closed \textsc{Gemini-Embedding-2} model, and the \textsc{Qwen3-Embedding} models. We note that even the best score in the main setting remains well below perfect retrieval, leaving substantial room for progress in mathematical IR.
\paragraph{Modern embedding models dominate \bench}
Across tasks and domains, recent embedding models substantially outperform traditional lexical retrievers, such as TF-IDF, as well as older neural encoders. The specialized baseline \textsc{Approach Zero} obtains an nDCG of $0.468$ in the main setting, far below the strongest embedding models and even below most compact recent embedding models. Older contextual encoders such as \textsc{BERT} and \textsc{RoBERTa} perform particularly poorly, suggesting that mathematical retrieval systems should favor modern embedding models when available. Overall, these results indicate that dense retrieval has advanced considerably for mathematical content, while classical lexical matching and earlier math-specific retrieval pipelines are no longer competitive.
\paragraph{Generic retrieval benchmarks cannot solely predict performance on mathematical IR}
Using our main scores, we measure the Pearson and Spearman rank correlations between performance on \bench{} and MTEB multilingual retrieval~\citep{mteb} as $0.77$ and $0.82$ respectfully. However, when restricting the analysis to recent embedding models, these correlations decrease to 0.62 and 0.73, suggesting that general benchmarks are not reliable predictors of mathematical IR performance, especially as models become more capable. For example, \textsc{Harrier-OSS-v1-27b} ranks 1st among our tested models on MTEB, while being behind the Octen, Gemini, and Qwen3 models on \bench{}.
\paragraph{Retrievers struggle more on symbolic tasks}
For the strongest systems, Geometry and Combinatorics tend to be easier, while more symbolic-heavy domains like Algebra, Number Theory, and Calculus are more challenging. For example, \textsc{Octen-Embedding-8B} achieves $0.664$ on Geometry and $0.665$ on Combinatorics, but only $0.594$ on Algebra. This pattern is consistent with the intuition that retrievers are worse at aligning and understanding mathematical formulas vs natural language text.
\paragraph{Performance across domains is not uniform}
Although stronger retrievers generally perform well across the benchmark, domain-level rankings are not uniform. For instance, \textsc{Gemini-Embedding-2} is the best-performing system on Algebra, while \textsc{Octen-Embedding-4B} scores highest in Geometry and Combinatorics. \textsc{Gemini-Embedding-001} performs similarly to the more recent and overall stronger \textsc{Gemini-Embedding-2} on Geometry. While the top models are still close in performance, these heterogeneous differences suggest that a single aggregate score does not fully characterize a retriever's mathematical capabilities.
\paragraph{Stronger retrievers benefit more from additional information}
Comparing the three retrieval settings shows that additional context helps primarily when the retriever can represent it effectively. The top systems improve from statement-statement to statement-full, and improve further in the full-full setting. For example, \textsc{Octen-Embedding-8B} increases from $0.623$ to $0.636$ to $0.672$. In contrast, weaker systems often degrade when solution text is included. \textsc{RoBERTa}, for instance, scores $0.406$ in statement-statement but only $0.311$ in the main setting. This suggests that long text can dilute the retrieval signal unless the model is capable enough to identify and represent the relevant information. Further descriptions of the non-main settings are provided in \cref{sec:app:additional_settings}.
\begin{table*}[!thbp]
\centering
\footnotesize
\caption{Main results of \bench{}. All numbers are reported as nDCG@10.}
\label{tab:statement-full}
\renewcommand{\arraystretch}{1.05}
\setlength{\tabcolsep}{4pt}
\newcommand{\beforerulepad}{\\[4pt]}
\newcommand{\afterrulepad}{\rule{0pt}{\dimexpr\ht\strutbox+4pt\relax}}
\newcommand{\tightmidrule}{\specialrule{\lightrulewidth}{0pt}{0pt}}
\newcommand{\vlinepadl}{6pt}
\newcommand{\vlinepadr}{3pt}
\newcolumntype{R}{>{$}r<{$}}
\resizebox{\linewidth}{!}{
\begin{tabular}{@{}l RRRRRR@{\hspace{\vlinepadl}}|@{\hspace{\vlinepadr}}R@{\hspace{\vlinepadl}}|@{\hspace{\vlinepadr}}R@{}}
\toprule
& \multicolumn{1}{c}{\makecell{Overall}}
& \multicolumn{1}{c}{\makecell{Algebra}}
& \multicolumn{1}{c}{\makecell{Geometry}}
& \multicolumn{1}{c}{\makecell{Number\\Theory}}
& \multicolumn{1}{c}{\makecell{Comb.}}
& \multicolumn{1}{c@{\hspace{\vlinepadl}}|@{\hspace{\vlinepadr}}}{\makecell[c]{Calc./\\Analysis}}
& \multicolumn{1}{c@{\hspace{\vlinepadl}}|@{\hspace{\vlinepadr}}}{\makecell[c]{Statement-\\Statement}}
& \multicolumn{1}{c}{\makecell[c]{Full-Full}} \beforerulepad
\tightmidrule
\afterrulepad Octen-Embedding-8B~\citep{octen2025rteb} & \mathbf{0.636} & 0.594 & 0.664 & \mathbf{0.629} & 0.665 & \mathbf{0.630} & \mathbf{0.623} & \mathbf{0.672} \\
Octen-Embedding-4B~\citep{octen2025rteb} & 0.632 & 0.586 & \mathbf{0.673} & 0.627 & \mathbf{0.667} & 0.609 & 0.619 & 0.663 \\
Gemini-Embedding-2~\citep{geminiembedding} & 0.628 & \mathbf{0.599} & 0.647 & 0.622 & 0.656 & 0.614 & 0.603 & 0.658 \\
Qwen3-Embedding-4B~\citep{qwen3embedding} & 0.615 & 0.576 & 0.652 & 0.610 & 0.642 & 0.597 & 0.597 & 0.667 \\
Qwen3-Embedding-8B~\citep{qwen3embedding} & 0.611 & 0.569 & 0.647 & 0.606 & 0.633 & 0.598 & 0.600 & 0.662 \\
Harrier-OSS-v1-27b~\citep{harrier} & 0.608 & 0.569 & 0.651 & 0.601 & 0.620 & 0.596 & 0.585 & 0.659 \\
Gemini-Embedding-001~\citep{geminiembedding} & 0.605 & 0.573 & 0.650 & 0.604 & 0.626 & 0.577 & 0.591 & 0.667 \\
KaLM-Embedding-Gemma3-12B-2511~\citep{kalm} & 0.585 & 0.548 & 0.617 & 0.583 & 0.606 & 0.569 & 0.579 & 0.599 \\
LLaMa-Embed-Nemotron-8b~\citep{nemotron} & 0.579 & 0.542 & 0.610 & 0.580 & 0.600 & 0.562 & 0.565 & 0.616 \\
Qwen3-Embedding-0.6B~\citep{qwen3embedding} & 0.575 & 0.545 & 0.629 & 0.564 & 0.589 & 0.546 & 0.566 & 0.625 \\
Harrier-OSS-v1-0.6b~\citep{harrier} & 0.572 & 0.538 & 0.613 & 0.566 & 0.581 & 0.557 & 0.549 & 0.632 \\
Jina-Embeddings-v5-Text-Small~\citep{jina} & 0.570 & 0.525 & 0.620 & 0.561 & 0.593 & 0.549 & 0.554 & 0.610 \\
Text-Embedding-3-Large~\citep{openaiembedding} & 0.558 & 0.535 & 0.571 & 0.560 & 0.574 & 0.552 & 0.539 & 0.593 \\
Jina-Embeddings-v5-Text-Nano~\citep{jina} & 0.532 & 0.492 & 0.573 & 0.538 & 0.552 & 0.506 & 0.522 & 0.569 \\
EmbeddingGemma-300m~\citep{embeddinggemma} & 0.519 & 0.496 & 0.550 & 0.518 & 0.540 & 0.485 & 0.511 & 0.588 \\
Text-Embedding-3-Small~\citep{openaiembedding} & 0.512 & 0.487 & 0.557 & 0.504 & 0.526 & 0.491 & 0.497 & 0.554 \\
BGE-m3~\citep{bge-m3} & 0.511 & 0.484 & 0.549 & 0.502 & 0.518 & 0.500 & 0.505 & 0.563 \\
Harrier-OSS-v1-270m~\citep{harrier} & 0.498 & 0.470 & 0.545 & 0.491 & 0.512 & 0.469 & 0.497 & 0.557 \\
Multilingual-E5-Large~\citep{e5} & 0.488 & 0.455 & 0.529 & 0.471 & 0.500 & 0.477 & 0.508 & 0.545 \\
Approach Zero~\citep{approachzero} & 0.468 & 0.485 & 0.443 & 0.454 & 0.480 & 0.490 & 0.469 & 0.481 \\
TF-IDF & 0.427 & 0.409 & 0.433 & 0.397 & 0.451 & 0.429 & 0.435 & 0.458 \\
Jaccard~\citep{jaccard} & 0.403 & 0.387 & 0.426 & 0.394 & 0.443 & 0.358 & 0.424 & 0.444 \\
BM25~\citep{bm25} & 0.398 & 0.394 & 0.420 & 0.381 & 0.422 & 0.368 & 0.409 & 0.410 \\
BERT~\citep{bert} & 0.357 & 0.369 & 0.342 & 0.344 & 0.389 & 0.335 & 0.417 & 0.429 \\
RoBERTa~\citep{roberta} & 0.311 & 0.306 & 0.293 & 0.314 & 0.342 & 0.287 & 0.406 & 0.397 \\
\bottomrule
\end{tabular}
}
\end{table*}
\subsection{Comparison of Mathematical and Textual Content in Embedding Space} \label{sec:results:math_vs_words}
\begin{figure}[!hbt]
\centering
\includegraphics[width=\linewidth]{graphs/maths_vs_words_selected_models.pdf}
\caption{Distribution of retriever preferences for mathematical vs. textual content across domains, as well as the domains avg. token distribution.}
\label{fig:math-vs-words-selected-models}
\vspace{-1em}
\end{figure}
We next examine what content IR methods focus on when aligning relevant pieces of mathematics. For any problem, we construct math-only and word-only variants by segmenting mathematical expressions using \textsc{GPT-OSS-120B}. Our prompt is presented in \cref{sec:app:prompts}. For each retrieval system, we capture the similarity between the query's math-only and word-only representations to that of the problem-and-solution text of the five most relevant documents for each query. We then calculate the preference of a system by measuring the fraction of instances where one variant produces higher similarity than the other. Partial results are shown in \cref{fig:math-vs-words-selected-models}, and we include further details about the implementation in \cref{sec:app:math_vs_words} as well as the full results in \cref{fig:math-vs-words-all-models}.
Overall, we find that weaker retrievers tend to rely more on mathematical content, indicating that they do so without necessarily understanding the meaning behing it. This carries across all domains, where the systems' reliance on mathematical content is correlated to the math-token ratio, the fraction of mathematical tokens in the input. Finally, we see that models from the same family~(\textsc{Qwen3-8B/0.6B}) show nearly identical behaviour across all domains, implying that the preferences may be inherited from the underlying training data. Results across all models are provided in \cref{sec:app:math_vs_words}.
\subsection{Selecting the Number of Rounds in the Swiss Tournament} \label{sec:exp:swiss}
As described in \cref{sec:tech:rating}, we use a Swiss-style tournament to avoid exhaustive comparisons over all $\frac{n(n-1)}{2}=11175$ candidate pairs per query. To quantify the effect of the reduced comparisons, we compare the resulting Swiss-tournament rankings with rankings from exhaustive pairwise comparison. In \cref{fig:avg-inv}, we report the average number of preference inversions relative to the exhaustive ranking as a function of the number of rounds $R$.
We set $R=20$ in the final pipeline, which is near the point where additional rounds give diminishing returns. This reduces the number of comparisons from $11175$ to $1500$, a factor of $\sim 7.5$, while keeping the ranking close to the exhaustive one, with only $700$ inversions out of $11175$ possible pairs. In \cref{sec:app:human_evaluation}, we further confirm through human evaluation that this setting nearly matches the exhaustive ranking in terms of human preference.
\subsection{Relative Performance of Relevance Signals} \label{sec:results:signal_ablation}
In \cref{sec:tech:seclectbench}, we use two proxy signals to select candidate documents: topic-based similarity and solution-summary similarity. Since each query in \bench contains candidates sampled from three groups---topic-only, summary-only, and both---we can ask how well each group fares under the final LLM-derived relevance ranking.
\cref{fig:3color-exp} plots, for each group, the cumulative fraction of candidates appearing within the top $r$ ranked positions, aggregated over all $1000$ queries. Thus, a faster rising curve corresponds to candidates that are placed higher by the final judge. Candidates selected by both signals rise fastest, showing that agreement between the topic and summary signals is a strong indicator of mathematical relevance. The single-signal groups are much closer to each other with summary- based similarity slightly ahead of topic similarity. Overall, the figure suggests that the two signals are complementary, each capturing separate useful relevance notion.
\section{Conclusion}\label{sec:conclusion}
In this work, we introduced \bench{}, an automated benchmark for fine-grained evaluation of mathematical IR. We combine ontology-based topic matching, solution-summary similarity, and LLM pairwise judgments to construct realistic reranking tasks. Our results reveal that modern embedding models outperform classical and math-specific baselines, but still struggle with symbolic domains. Further, we show that general retrieval benchmarks do not reliably predict performance on mathematical retrieval tasks. Together, these results highlight the need for domain-specific evaluation and establish \bench{} as a valuable tool for driving progress in mathematical IR.
\section*{Acknowledgements}
This research was partially funded by the Ministry of Education and Science of Bulgaria (support for INSAIT, part of the Bulgarian National Roadmap for Research Infrastructure). This project was supported with computational resources provided by Google Cloud Platform (GCP).
\clearpage
\input{appendix/app_limitations.tex}
\input{appendix/app_data.tex}
\input{appendix/app_details.tex}
\input{appendix/app_additional.tex}
\input{appendix/app_compute.tex}
\input{appendix/app_prompts.tex}
\input{appendix/app_checklist.tex}
\section{Limitations}\label{sec:app:limitations}
While \bench{} provides a scalable framework for evaluating mathematical information retrieval, several limitations remain. First, the benchmark is built primarily from high-school, olympiad, and early undergraduate-style problems, so performance may not fully transfer to research-level mathematics, formal proof libraries, textbook retrieval, or highly specialized domains. In addition, \bench{} evaluates reranking over preselected candidate sets rather than open-ended retrieval over the full corpus. This design makes it possible to assign fine-grained relevance judgments to every evaluated candidate, while complementary first-stage retrieval evaluation remains an important extension. Finally, the benchmark uses automated LLM-based processing to support construction at scale. Although this can introduce noise into individual relevance signals, our human validation indicates that the resulting judgments are reliable for comparing retrieval systems.
\section{Data Sources}\label{sec:app:data}
In this section we describe in details what datasets we use to compile our problem corpora from \cref{sec:tech:data} and how we preprocess them.
\subsection{Gathering Data from Public Forums}\label{sec:tech:forums}
Large retrieval systems often rely on large-scale curation from public web sources. To establish a broad base for our benchmark, we first collect problems and solutions from the Art of Problem Solving (AoPS) community, which hosts a large collection of mathematical discussions. We parse nearly $500$K threads and filter them to retain only those containing a problem statement and at least one relevant solution. When multiple candidate solutions are available, we use \textsc{GPT-5-mini} to select the most appropriate one. Finally, we deduplicate problem statements using DataTrove's MinHash deduplication pipeline~\citep{datatrove}, yielding $150$K unique entries.
\subsection{Gathering Data from High-Quality Official Sources}\label{sec:tech:officialsrc}
Although AoPS provides a large and diverse set of problems, its quality and difficulty vary substantially. Because AoPS is an open forum, extracted problems and solutions may be incomplete, noisy, or interleaved with out-of-context discussion. We therefore supplement the corpus with problems and solutions from high-quality official sources, including national and international competitions as well as preparation materials.
\begin{figure}
\centering
\begin{minipage}{0.49\linewidth}
\centering
\includegraphics[width=\linewidth]{graphs/piechart_benchmark.pdf}
\caption{Domain and subdomain distribution of the query problems in \bench.}
\label{fig:piechart-benchmark}
\end{minipage}
\hfill
\begin{minipage}{0.49\linewidth}
\centering
\includegraphics[width=0.98\linewidth]{graphs/piechart_databank.pdf}
\caption{Domain and subdomain distribution of the sampling corpus described in \cref{sec:tech:data}.}
\label{fig:piechart-databank}
\end{minipage}
\end{figure}
Starting from official PDFs manually collected from the web, we parse the documents using MathPix OCR, which is specialized for mathematical content. We then apply a multi-step LLM-based parsing pipeline using \textsc{GPT-5-mini} as follows:
\begin{itemize}
\item (Optional) \textbf{Translation:} For documents in languages other than English, we first translate the text into English.
\item \textbf{Segmentation:} We segment the parsed text into individual problems and solutions, removing extraneous information such as formatting artifacts, metadata, and irrelevant text. We do capturing problems and solutions in XML-style tags, without modifying their content. If the number of problem and solution tags do not match, we have a human manually review the document and correct the parsing.
\item \textbf{Cleaning:} We apply a final cleaning step to remove any remaining formatting artifacts, problem metadata, such as the author, source, or number of the problem, and any text that is not part of the problem statement or solution.
\end{itemize}
This process yields a high-quality set of approximately $35$K problems from challenging competitions, including the IMO, USAMO, and RMM. This subset is especially valuable for our benchmark because it contains a higher density of relevant pairs and more complex problems and solutions, making it more likely to challenge current embedding models.
\subsection{Additional Sources}\label{sec:tech:additionaldata}
\begin{wrapfigure}[24]{r}{0.52\textwidth}
\vspace{-2em}
\centering
\includegraphics[width=\linewidth]{graphs/piechart_candidates_NT.pdf}
\caption{Domain and subdomain distribution of the documents retrieved for a representative query from \bench. Shaded subdomains indicate the subdomains associated with the query, while black lines mark documents ranked in the top 10 for the query.}
\label{fig:piechart-candidates-NT}
\end{wrapfigure}
Because the AoPS subset is nearly five times larger than the official-source subset, relying too heavily on AoPS could reduce the overall quality of the corpus. We therefore further supplement the corpus with problems and solutions from another high-quality, but less challenging source, \numina~\cite{numina_math_datasets}. To maintain a similar range of difficulty and mathematical domains, we sample approximately $115$K problems from the \emph{olympiads} split of \numina, discarding synthetic samples, multiple-choice problems, and entries with invalid problem statements or solutions. This completes our initial corpus of approximately $300$K mathematical problem-solution pairs.
\subsection{Cross-Dataset Deduplication}\label{sec:tech:dedup}
Although the sources are collected independently, prominent competition problems may appear in multiple datasets. To avoid including duplicate problems as both queries and documents, we perform cross-source deduplication. We detect duplicates using the same MinHash-based method described in \cref{sec:tech:forums}. When duplicates are found, we retain a single instance according to the following priority order: official sources, \numina, and then AoPS. This process removes approximately $17$K cross-source duplicates, leaving a final corpus of approximately $283$K unique problem-solution pairs.
\subsection{Mathematical Data Distribution} \label{sec:app:data_distribution}
In \cref{fig:piechart-benchmark} and \cref{fig:piechart-databank} we present visualizations of the domain and subdomain distributions of the queries in \bench{} and the sampling corpus respectively. The categories are determined by the mathematical ontology tagging described in \cref{sec:tech:ontology}. Since a single problem may belong to multiple domains or subdomains, each problem contributes a total weight of 1 distributed equally among its assigned categories.
\subsection{Example of Query-Document Topic Distribution} \label{sec:app:candidate_distribution}
To illustrate the distribution of candidate documents retrieved for a given query, \cref{fig:piechart-candidates-NT} shows the domain and subdomain distribution for a representative Number Theory query and its associated documents. The shaded subdomains correspond to those associated with the query, while the black outlines indicate the domains and subdomains of the 10 documents with the highest relevance ratings. As the figure shows, the subdomains highlighted in the query are well represented among the retrieved documents. At the same time, the pipeline also identifies relevant problems from other domains, such as Combinatorics in this example, demonstrating that mathematical relevance is not necessarily confined to the same domain or subdomain.
\subsection{Licensing}\label{sec:app:licensing}
As we collate our corpus from various sources, we release our data under the Creative Commons Attribution-ShareAlike 4.0 International License (CC BY-SA 4.0). This license allows others to share and adapt the material for any purpose, even commercially, as long as they provide appropriate credit, indicate if changes were made, and distribute their contributions under the same license.
All restrictions that apply to the original sources also apply to our compiled corpus. In particular, \numina is released under the Apache License 2.0, which allows for commercial use and modification but requires attribution and does not permit sublicensing. The AoPS content is subject to the site's terms of service, which generally allow for non-commercial use with proper attribution. The official competition problems are typically in the public domain or released under simi, with the exception of using the data for training.
\section{Additional Technical Details}\label{sec:app:technical_details}
Here, we provide additional technical details about our implementation and experimental setup.
\subsection{Running Embedding Models} \label{sec:app:embedding_details}
Embedding models were called through their standard embedding interfaces. In particular, Gemini and OpenAI models were called through their respective embedding APIs, while local models were run with vLLM~\citep{vllm}, when supported, and SentenceTransformers~\citep{sentencetransformers} otherwise. Output dimensionality was never specified manually, so models that support configurable dimensions used their default/full output dimension. If a piece of text exceeded the model's maximum input length, we split it into non-overlapping token chunks that fit the context window. Each chunk was embedded independently, and the final embedding was computed by elementwise averaging these embeddings.
\begin{wrapfigure}{r}{0.60\textwidth}
\vspace{-1.2em}
\centering
\fbox{%
\begin{minipage}{\dimexpr\linewidth-2\fboxsep-2\fboxrule\relax}
\footnotesize
\captionof{algorithm}{Swiss-style greedy pairing.}
\label{alg:swiss_pairing}
\begin{algorithmic}[1]
\Require Query $Q$ Documents $D$, rounds $R$
\State $w_d \gets 0$ for all $d \in D$; \quad $M \gets \emptyset$
\For{$r=1,\ldots,R$}
\If{$r=1$}
\State $P_r \gets$ random matching of $D$
\Else
\State $P_r \gets \emptyset$;
\State \quad $U \gets$ shuffle ties, then sort $D$ by $w_d$
\While{$|U| \geq 2$}
\State $i \gets$ lowest-scoring document in $U$
\State $F_i \gets \{j \in U \setminus \{i\}: \{i,j\} \notin M\}$
\If{$F_i=\emptyset$}
\State remove $i$ from $U$
\Else
\State $j \gets \arg\min_{k \in F_i} |w_i-w_k|$; break ties randomly
\State $P_r \gets P_r \cup \{\{i,j\}\}$
\State \quad $M \gets M \cup \{\{i,j\}\}$
\State remove $i$ and $j$ from $U$
\EndIf
\EndWhile
\EndIf
\State judge similarity of pairs in $P_r$ to $Q$ and increment each winner's score
\EndFor
\end{algorithmic}
\end{minipage}
}
\vspace{-1.2em}
\end{wrapfigure}
\subsection{Computing the Jaccard Similarity} \label{sec:app:jaccard_details}
We use a standard implementation of the Jaccard similarity. We first tokenize the input sentences by filtering only alphanumeric sequences, and convert them to lowercase. We filter out stop words using the Scikit-learn English stop word list~\citep{scikit-learn}. We additionally introduce our own list of mathematics-specific stop words, that do not significantly factor into the meaning of solution summaries. The list of these words can be seen in our supplementary materials. Finally, we compute the Jaccard similarity as the size of the intersection of the two token sets divided by the size of their union.
\subsection{Selecting the Similarity Thresholds} \label{sec:app:thresholds}
In \cref{sec:tech:seclectbench}, we described our use of threshold-based filtering to select candidate documents for our benchmark. The thresholds were determined as follows. We first computed the distribution of similarity scores across all problem pairs in our corpus per domain. Then, for each domain, we derived a minimal threshold, for which at least 150 queries had at least 50 candidate documents above the threshold for each of the three categories: topic-only, summary-only, and both. To determine a final single threshold for each signal, we took the minimal thresholds obtained across the five domains, resulting in $\tau_{topic}\approx 0.858$ and $\tau_{summ}\approx 0.211$.
\subsection{Swiss-style Tournament Details} \label{sec:app:swiss_details}
We used a Swiss-style pairing procedure to assign head-to-head comparisons among the $150$ candidate items. In the first round, documents were paired uniformly at random. After each round, each document was assigned a running score equal to the number of times the LLM judge preferred the candidate up to that point. For all subsequent rounds, pairings were constructed greedily in ascending order of these scores.
We first randomly shuffle the documents among documents with equal scores, then sort them by their current win counts. This prevents potential biases from retaining the relative document order for documents with equal scores across the entire tournament. Starting from the lowest-scoring unpaired document, we select as its opponent an unpaired document with the closest win count among those with whom it had not previously been matched. Because this constraint can make a complete pairing infeasible in later rounds, some documents may remain unpaired in a given round. The full pairing procedure is summarized in \cref{alg:swiss_pairing}.
\section{Additional Experiments and Results} \label{sec:app:additional_experiments}
Here, we display a varying range of additional results and analyses that complement our findings from the main text.
\subsection{Expert Validation of Benchmark and LLM-as-a-Judge} \label{sec:app:human_evaluation}
To validate our benchmark construction, we conduct a human evaluation of both the final benchmark rankings and the reliability of our LLM-as-a-judge procedure. Two authors with experience in high-school mathematical olympiads independently annotated a random sample of 200 examples. Each example consisted of a query problem and two candidate problems, together with their corresponding solutions. Annotators were asked to decide which candidate was more mathematically relevant to the query, with the option to mark a tie when the distinction was unclear or when the two candidates appeared similarly relevant. The instructions are provided in \cref{sec:app:human_evaluation_instructions}.
\paragraph{Inter-annotator agreement}
Of the 200 examples, 20 were judged by both annotators, allowing us to estimate inter-annotator agreement, where the annotators agreed on $75.0\%$ of the cases. This indicates that mathematical relevance judgments are somewhat subjective, especially after candidate prefiltering has removed clearly irrelevant documents. At the same time, the agreement rate suggests that the task contains substantial signal, as most disagreements arise from nuanced comparisons between plausibly related problems.
\paragraph{LLM-as-a-judge agreement with human annotators}
\input{tables/human_eval_regime_agreement_judge.tex}
We next validate our pairwise LLM judge by comparing it against two alternative judging regimes: (i) an ordinal relevance judge, which assigns each query-document pair an absolute relevance score, and (ii) a five-time majority-vote variant of the pairwise judge, where ties are assigned when the vote margin is at most one. All prompts are provided in \cref{sec:app:prompts}. To compare automatic judgments with human annotations, we assign a score of $1$ for exact agreement, $0.5$ when either judgment is a tie and the other expresses a preference, and $0$ otherwise.
The pairwise LLM judge achieves $78.0\%$ agreement with human annotations, outperforming the ordinal judge, which achieves $74.8\%$, and slightly outperforming the majority-vote pairwise judge, which achieves $77.7\%$. These results provide evidence that the pairwise LLM judge is a reliable proxy for expert relevance judgments in this setting. Moreover, the stronger performance of the pairwise judge relative to the ordinal judge supports our design choice of using comparative preferences rather than absolute relevance scores.
We further analyze judge agreement as a function of the final rank difference $\Delta_r$ between the two candidates, where $\Delta_r$ is computed from the full Bradley-Terry ranking. Larger values of $\Delta_r$ correspond to pairs whose estimated relevance differs more substantially, and should therefore be easier to judge. As shown in \cref{tab:judge-agreement-by-method-regime}, all judging regimes achieve higher agreement at larger $\Delta_r$ thresholds. In particular, the pairwise judge reaches $86.3\%$ agreement for pairs with $\Delta_r \geq 50$. This trend is consistent with the intuition that close comparisons are more subjective, while large relevance gaps are easier to identify reliably. The pairwise judge remains among the strongest methods across all $\Delta_r$ thresholds, further supporting its use in our benchmark construction.
\paragraph{Benchmark validation}
Finally, we use the same human annotations to validate our Swiss-style ranking procedure. We compare the preference implied by the final Swiss-tournament rankings against the human judgments and obtain $78.0\%$ agreement. For reference, performing full Bradley-Terry estimation over all candidate pairs yields $78.8\%$ agreement. The small gap between these two procedures suggests that the Swiss tournament recovers most of the relevant ranking information while requiring substantially fewer LLM comparisons.
We also compare against a random tournament, where each pair being compared is selected uniformly at random at each round. Using the same number of rounds as the Swiss tournament, this random scheduling baseline achieves $75.6\%$ agreement, below the Swiss-style procedure. This shows that Swiss-style scheduling is more effective than random comparison sampling at selecting informative pairwise judgments, and validates its use as a scalable approximation to exhaustive pairwise ranking.
\subsection{Comparison of Mathematical vs. Textual Content in the Embedding Space} \label{sec:app:math_vs_words}
We now provide complemetary details and results for the experiment comparing the importance of mathematical versus textual content in the retrieval task, as described in \cref{sec:results:math_vs_words}.
\paragraph{Methodology and motivation}
To investigate whether mathematical notation or surrounding textual content contributes more to retrieving relevant documents, we conduct the following experiment using our benchmark.
For each query problem, we separately extract its mathematical expressions and its textual content. We discard queries for which either component is empty, leaving 969 of the original 1000 query problems. For each retrieval method, including both embedding-based and classical approaches, we then compute two relevance estimates for every remaining query.
First, we compare the query's mathematical content with the problem-and-solution text of each of its five highest-rated relevant candidate problems according to the benchmark. We compute the relevance score assigned by the retrieval method for each of these five candidates and take their average. We repeat the same procedure using only the query's textual content. Thus, each query receives two average relevance scores: one based on its mathematical content and one based on its textual content.
Finally, for each query and retrieval method, we record whether the mathematical or textual component receives the higher average relevance score. Since these scores are averaged real-valued quantities, ties are extremely rare and do not affect the analysis. For each retrieval method, we report the percentage of queries for which the mathematical component is judged more relevant than the textual component. The results for all evaluated embedding models and classical retrieval methods are shown in \cref{fig:math-vs-words-all-models}.
For token-based methods such as Jaccard similarity and TF-IDF, appropriate tokenization is essential. We therefore tokenize mathematical expressions using the Approach Zero mathematics tokenizer.
\begin{figure}
\centering
\includegraphics[width=\linewidth]{graphs/maths_vs_words_all_models.pdf}
\caption{Percentage of query problems for which the mathematical component receives a higher average relevance score than the textual component, across retrieval methods.}
\label{fig:math-vs-words-all-models}
\end{figure}
\subsection{Confidence Intervals for \bench{} Results} \label{sec:app:confidence_intervals}
In \cref{tab:main-results-ci} we include 95\% confidence intervals for the results reported in \cref{sec:results:main}, as computed by $1000$-fold bootstrap resampling of the queries.
\input{tables/main_prob_full_ci.tex}
\subsection{Results on \bench{} on Additional Tasks} \label{sec:app:additional_settings}
We include in \cref{tab:additional_settings} the results for the non-main settings of \bench{}, including statement-statement and full-full retrieval. For the most part, the same trends observed in the main setting hold across these additional settings, and their respective domains. New best models emerge in some domains in the full-full setting, such as \textsc{Gemini-Embedding-001} in Number Theory, and \textsc{Harrier-OSS-v1-27b} in Geometry. However, the overall ranking of models remains relatively consistent across settings, with the strongest models in the main setting generally remaining strong in the additional settings.
\input{tables/additional_settings.tex}
\begin{wraptable}{r}{0.6\columnwidth}
\centering
\footnotesize
\caption{Judge agreement rates across different document rank differences.}
\label{tab:judge-agreement-by-method-regime}
\renewcommand{\arraystretch}{1.05}
\setlength{\tabcolsep}{4pt}
\resizebox{\linewidth}{!}{%
\begin{tabular}{@{}lccc@{}}
\toprule
\multicolumn{1}{c}{Method}
& \multicolumn{1}{c}{$\Delta=0$}
& \multicolumn{1}{c}{$\Delta=25$}
& \multicolumn{1}{c}{$\Delta=50$} \\
\midrule
Inter-annotator & 75.0\% & 75.0\% & 85.7\% \\
\midrule
Pairwise Judge & \textbf{78.0\%} & \textbf{82.8\%} & \textbf{86.3\%} \\
Majority Judge & 77.5\% & 82.0\% & 85.6\% \\
Ordinal Ranking & 74.8\% & 78.9\% & \textbf{86.3\%} \\
\bottomrule
\end{tabular}%
}
\end{wraptable}
\begin{table*}[!thbp]
\centering
\footnotesize
\caption{Results of \bench{} with 95\% confidence intervals. All numbers are reported as nDCG@10.}
\label{tab:main-results-ci}
\renewcommand{\arraystretch}{1.05}
\setlength{\tabcolsep}{4pt}
\newcommand{\beforerulepad}{\\[4pt]}
\newcommand{\afterrulepad}{\rule{0pt}{\dimexpr\ht\strutbox+4pt\relax}}
\newcommand{\tightmidrule}{\specialrule{\lightrulewidth}{0pt}{0pt}}
\newcolumntype{R}{>{$}r<{$}}
\resizebox{0.9\linewidth}{!}{
\begin{tabular}{@{}l RRRRRR@{}}
\toprule
& \multicolumn{1}{c}{\makecell{Overall}}
& \multicolumn{1}{c}{\makecell{Algebra}}
& \multicolumn{1}{c}{\makecell{Geometry}}
& \multicolumn{1}{c}{\makecell{Number\\Theory}}
& \multicolumn{1}{c}{\makecell{Comb.}}
& \multicolumn{1}{c}{\makecell[c]{Calc./\\Analysis}} \beforerulepad
\tightmidrule
\afterrulepad Octen-Embedding-8B~\citep{octen2025rteb} & \mathbf{0.636}^{+\mathbf{0.009}}_{-\mathbf{0.010}} & 0.594^{+0.017}_{-0.017} & 0.664^{+0.017}_{-0.017} & \mathbf{0.629}^{+\mathbf{0.017}}_{-\mathbf{0.018}} & 0.665^{+0.017}_{-0.018} & \mathbf{0.630}^{+\mathbf{0.017}}_{-\mathbf{0.018}} \\
Octen-Embedding-4B~\citep{octen2025rteb} & 0.632^{+0.009}_{-0.010} & 0.586^{+0.017}_{-0.017} & \mathbf{0.673}^{+\mathbf{0.016}}_{-\mathbf{0.016}} & 0.627^{+0.018}_{-0.018} & \mathbf{0.667}^{+\mathbf{0.017}}_{-\mathbf{0.017}} & 0.609^{+0.017}_{-0.017} \\
Gemini-Embedding-2~\citep{geminiembedding} & 0.628^{+0.009}_{-0.009} & \mathbf{0.599}^{+\mathbf{0.016}}_{-\mathbf{0.016}} & 0.647^{+0.016}_{-0.016} & 0.622^{+0.016}_{-0.016} & 0.656^{+0.016}_{-0.017} & 0.614^{+0.016}_{-0.016} \\
Qwen3-Embedding-4B~\citep{qwen3embedding} & 0.615^{+0.009}_{-0.009} & 0.575^{+0.018}_{-0.018} & 0.652^{+0.016}_{-0.017} & 0.610^{+0.017}_{-0.017} & 0.642^{+0.016}_{-0.017} & 0.597^{+0.017}_{-0.017} \\
Qwen3-Embedding-8B~\citep{qwen3embedding} & 0.611^{+0.010}_{-0.010} & 0.569^{+0.018}_{-0.017} & 0.646^{+0.017}_{-0.017} & 0.606^{+0.018}_{-0.017} & 0.633^{+0.017}_{-0.017} & 0.598^{+0.017}_{-0.017} \\
Harrier-OSS-v1-27b~\citep{harrier} & 0.608^{+0.009}_{-0.009} & 0.569^{+0.016}_{-0.016} & 0.651^{+0.016}_{-0.016} & 0.601^{+0.016}_{-0.016} & 0.620^{+0.016}_{-0.017} & 0.596^{+0.016}_{-0.016} \\
Gemini-Embedding-001~\citep{geminiembedding} & 0.605^{+0.009}_{-0.009} & 0.573^{+0.017}_{-0.017} & 0.650^{+0.015}_{-0.015} & 0.604^{+0.017}_{-0.017} & 0.626^{+0.016}_{-0.017} & 0.577^{+0.017}_{-0.017} \\
KaLM-Embedding-Gemma3-12B-2511~\citep{kalm} & 0.585^{+0.009}_{-0.009} & 0.548^{+0.016}_{-0.016} & 0.617^{+0.016}_{-0.016} & 0.583^{+0.017}_{-0.016} & 0.606^{+0.017}_{-0.017} & 0.569^{+0.016}_{-0.016} \\
LLaMa-Embed-Nemotron-8b~\citep{nemotron} & 0.579^{+0.009}_{-0.009} & 0.542^{+0.015}_{-0.015} & 0.610^{+0.016}_{-0.015} & 0.580^{+0.016}_{-0.016} & 0.600^{+0.016}_{-0.017} & 0.562^{+0.017}_{-0.017} \\
Qwen3-Embedding-0.6B~\citep{qwen3embedding} & 0.575^{+0.009}_{-0.009} & 0.545^{+0.016}_{-0.016} & 0.629^{+0.016}_{-0.017} & 0.564^{+0.017}_{-0.017} & 0.589^{+0.017}_{-0.017} & 0.546^{+0.016}_{-0.016} \\
Harrier-OSS-v1-0.6b~\citep{harrier} & 0.572^{+0.009}_{-0.009} & 0.538^{+0.016}_{-0.016} & 0.613^{+0.016}_{-0.016} & 0.566^{+0.017}_{-0.016} & 0.581^{+0.017}_{-0.017} & 0.557^{+0.017}_{-0.017} \\
Jina-Embeddings-v5-Text-Small~\citep{jina} & 0.570^{+0.009}_{-0.009} & 0.525^{+0.016}_{-0.016} & 0.619^{+0.017}_{-0.017} & 0.561^{+0.017}_{-0.017} & 0.593^{+0.017}_{-0.017} & 0.549^{+0.015}_{-0.015} \\
Text-Embedding-3-Large~\citep{openaiembedding} & 0.558^{+0.009}_{-0.009} & 0.535^{+0.016}_{-0.016} & 0.571^{+0.015}_{-0.015} & 0.560^{+0.016}_{-0.017} & 0.574^{+0.016}_{-0.017} & 0.552^{+0.016}_{-0.016} \\
Jina-Embeddings-v5-Text-Nano~\citep{jina} & 0.532^{+0.009}_{-0.009} & 0.491^{+0.016}_{-0.015} & 0.573^{+0.016}_{-0.016} & 0.538^{+0.017}_{-0.017} & 0.552^{+0.017}_{-0.017} & 0.506^{+0.016}_{-0.016} \\
EmbeddingGemma-300m~\citep{embeddinggemma} & 0.519^{+0.009}_{-0.009} & 0.496^{+0.016}_{-0.016} & 0.550^{+0.016}_{-0.016} & 0.519^{+0.017}_{-0.017} & 0.540^{+0.017}_{-0.017} & 0.485^{+0.016}_{-0.016} \\
Text-Embedding-3-Small~\citep{openaiembedding} & 0.513^{+0.009}_{-0.009} & 0.487^{+0.016}_{-0.016} & 0.557^{+0.016}_{-0.016} & 0.505^{+0.016}_{-0.016} & 0.526^{+0.017}_{-0.017} & 0.491^{+0.016}_{-0.016} \\
BGE-m3~\citep{bge-m3} & 0.512^{+0.009}_{-0.009} & 0.484^{+0.017}_{-0.016} & 0.549^{+0.017}_{-0.016} & 0.503^{+0.018}_{-0.018} & 0.518^{+0.017}_{-0.017} & 0.500^{+0.014}_{-0.015} \\
Harrier-OSS-v1-270m~\citep{harrier} & 0.498^{+0.009}_{-0.009} & 0.470^{+0.016}_{-0.016} & 0.545^{+0.016}_{-0.016} & 0.491^{+0.017}_{-0.017} & 0.512^{+0.016}_{-0.016} & 0.468^{+0.015}_{-0.014} \\
Multilingual-E5-Large~\citep{e5} & 0.488^{+0.009}_{-0.009} & 0.455^{+0.015}_{-0.015} & 0.529^{+0.017}_{-0.017} & 0.471^{+0.016}_{-0.016} & 0.500^{+0.016}_{-0.016} & 0.477^{+0.014}_{-0.014} \\
Approach Zero~\citep{approachzero} & 0.446^{+0.009}_{-0.009} & 0.452^{+0.016}_{-0.015} & 0.430^{+0.016}_{-0.015} & 0.436^{+0.017}_{-0.017} & 0.461^{+0.017}_{-0.017} & 0.450^{+0.015}_{-0.015} \\
TF-IDF & 0.430^{+0.008}_{-0.008} & 0.416^{+0.014}_{-0.014} & 0.430^{+0.015}_{-0.015} & 0.399^{+0.014}_{-0.014} & 0.459^{+0.016}_{-0.016} & 0.430^{+0.014}_{-0.014} \\
Jaccard~\citep{jaccard} & 0.404^{+0.008}_{-0.008} & 0.387^{+0.015}_{-0.015} & 0.426^{+0.017}_{-0.017} & 0.394^{+0.015}_{-0.015} & 0.443^{+0.016}_{-0.016} & 0.357^{+0.012}_{-0.013} \\
BM25~\citep{bm25} & 0.398^{+0.008}_{-0.008} & 0.394^{+0.015}_{-0.015} & 0.420^{+0.016}_{-0.016} & 0.382^{+0.014}_{-0.015} & 0.422^{+0.016}_{-0.016} & 0.368^{+0.013}_{-0.013} \\
BERT~\citep{bert} & 0.357^{+0.008}_{-0.007} & 0.369^{+0.014}_{-0.013} & 0.342^{+0.014}_{-0.014} & 0.345^{+0.013}_{-0.013} & 0.389^{+0.016}_{-0.015} & 0.335^{+0.012}_{-0.012} \\
RoBERTa~\citep{roberta} & 0.311^{+0.007}_{-0.007} & 0.306^{+0.011}_{-0.011} & 0.293^{+0.014}_{-0.013} & 0.314^{+0.013}_{-0.012} & 0.342^{+0.013}_{-0.013} & 0.287^{+0.011}_{-0.010} \\
\bottomrule
\end{tabular}
}
\end{table*}
\begin{table*}[!htbp]
\centering
\scriptsize
\caption{Results across the non-main settings of \bench{}. All numbers are reported as nDCG@10.}
\label{tab:additional_settings}
\renewcommand{\arraystretch}{1.05}
\setlength{\tabcolsep}{2.5pt}
\newcolumntype{R}{>{$}r<{$}}
\resizebox{\textwidth}{!}{%
\begin{tabular}{@{}l RRRRRR RRRRRR@{}}
\toprule
& \multicolumn{6}{c}{\textbf{Statement--Statement}}
& \multicolumn{6}{c}{\textbf{Full--Full}} \\
\cmidrule(lr){2-7}
\cmidrule(lr){8-13}
& \multicolumn{1}{c}{\makecell{Overall}}
& \multicolumn{1}{c}{\makecell{Algebra}}
& \multicolumn{1}{c}{\makecell{Geometry}}
& \multicolumn{1}{c}{\makecell{Number\\Theory}}
& \multicolumn{1}{c}{\makecell{Comb.}}
& \multicolumn{1}{c}{\makecell[c]{Calc./\\Analysis}}
& \multicolumn{1}{c}{\makecell{Overall}}
& \multicolumn{1}{c}{\makecell{Algebra}}
& \multicolumn{1}{c}{\makecell{Geometry}}
& \multicolumn{1}{c}{\makecell{Number\\Theory}}
& \multicolumn{1}{c}{\makecell{Comb.}}
& \multicolumn{1}{c}{\makecell[c]{Calc./\\Analysis}} \\
\midrule
Octen-Embedding-8B~\citep{octen2025rteb}
& \mathbf{0.623} & \mathbf{0.586} & 0.648 & \mathbf{0.624} & \mathbf{0.654} & \mathbf{0.606}
& \mathbf{0.672} & \mathbf{0.631} & 0.685 & 0.670 & \mathbf{0.700} & \mathbf{0.673} \\
Octen-Embedding-4B~\citep{octen2025rteb}
& 0.619 & 0.581 & \mathbf{0.659} & 0.615 & 0.651 & 0.591
& 0.663 & 0.616 & 0.695 & 0.658 & 0.695 & 0.653 \\
Gemini-Embedding-2~\citep{geminiembedding}
& 0.603 & 0.569 & 0.623 & 0.602 & 0.630 & 0.587
& 0.658 & 0.630 & 0.665 & 0.659 & 0.675 & 0.652 \\
Qwen3-Embedding-8B~\citep{qwen3embedding}
& 0.600 & 0.562 & 0.643 & 0.600 & 0.622 & 0.569
& 0.662 & 0.621 & 0.693 & 0.660 & 0.679 & 0.657 \\
Qwen3-Embedding-4B~\citep{qwen3embedding}
& 0.597 & 0.554 & 0.638 & 0.591 & 0.626 & 0.573
& 0.667 & 0.624 & 0.693 & 0.670 & 0.693 & 0.656 \\
Gemini-Embedding-001~\citep{geminiembedding}
& 0.591 & 0.553 & 0.629 & 0.598 & 0.619 & 0.560
& 0.667 & 0.630 & 0.690 & \mathbf{0.679} & 0.685 & 0.651 \\
Harrier-OSS-v1-27b~\citep{harrier}
& 0.585 & 0.554 & 0.612 & 0.586 & 0.609 & 0.562
& 0.659 & 0.615 & \mathbf{0.700} & 0.664 & 0.678 & 0.640 \\
KaLM-Embedding-Gemma3-12B-2511~\citep{kalm}
& 0.579 & 0.546 & 0.605 & 0.578 & 0.597 & 0.565
& 0.599 & 0.557 & 0.628 & 0.607 & 0.618 & 0.585 \\
Qwen3-Embedding-0.6B~\citep{qwen3embedding}
& 0.566 & 0.529 & 0.611 & 0.564 & 0.586 & 0.538
& 0.625 & 0.586 & 0.676 & 0.624 & 0.634 & 0.603 \\
LLaMa-Embed-Nemotron-8B~\citep{nemotron}
& 0.565 & 0.535 & 0.584 & 0.578 & 0.588 & 0.540
& 0.616 & 0.569 & 0.641 & 0.629 & 0.638 & 0.602 \\
Jina-Embeddings-v5-Text-Small~\citep{jina}
& 0.554 & 0.513 & 0.603 & 0.549 & 0.577 & 0.528
& 0.610 & 0.556 & 0.664 & 0.608 & 0.631 & 0.591 \\
Harrier-OSS-v1-0.6b~\citep{harrier}
& 0.549 & 0.521 & 0.589 & 0.538 & 0.568 & 0.523
& 0.632 & 0.588 & 0.676 & 0.637 & 0.638 & 0.613 \\
Text-Embedding-3-Large~\citep{openaiembedding}
& 0.539 & 0.522 & 0.554 & 0.541 & 0.560 & 0.526
& 0.593 & 0.564 & 0.609 & 0.598 & 0.611 & 0.588 \\
Jina-Embeddings-v5-Text-Nano~\citep{jina}
& 0.522 & 0.484 & 0.563 & 0.529 & 0.547 & 0.487
& 0.569 & 0.522 & 0.604 & 0.576 & 0.585 & 0.560 \\
EmbeddingGemma-300m~\citep{embeddinggemma}
& 0.511 & 0.488 & 0.547 & 0.502 & 0.545 & 0.472
& 0.588 & 0.548 & 0.629 & 0.594 & 0.604 & 0.559 \\
Multilingual-E5-Large~\citep{e5}
& 0.508 & 0.485 & 0.549 & 0.501 & 0.523 & 0.475
& 0.545 & 0.505 & 0.580 & 0.549 & 0.544 & 0.536 \\
BGE-m3~\citep{bge-m3}
& 0.505 & 0.483 & 0.539 & 0.498 & 0.518 & 0.485
& 0.563 & 0.513 & 0.616 & 0.575 & 0.567 & 0.543 \\
Harrier-OSS-v1-270m~\citep{harrier}
& 0.497 & 0.466 & 0.537 & 0.489 & 0.517 & 0.473
& 0.557 & 0.525 & 0.590 & 0.565 & 0.565 & 0.539 \\
Text-Embedding-3-Small~\citep{openaiembedding}
& 0.497 & 0.470 & 0.540 & 0.483 & 0.519 & 0.473
& 0.554 & 0.520 & 0.597 & 0.552 & 0.565 & 0.536 \\
TF-IDF
& 0.435 & 0.418 & 0.454 & 0.414 & 0.450 & 0.431
& 0.458 & 0.419 & 0.471 & 0.445 & 0.477 & 0.471 \\
Approach Zero~\citep{approachzero}
& 0.428 & 0.425 & 0.433 & 0.432 & 0.451 & 0.392
& 0.473 & 0.478 & 0.467 & 0.457 & 0.492 & 0.475 \\
Jaccard~\citep{jaccard}
& 0.424 & 0.401 & 0.457 & 0.427 & 0.463 & 0.367
& 0.444 & 0.428 & 0.464 & 0.455 & 0.482 & 0.387 \\
BERT~\citep{bert}
& 0.417 & 0.432 & 0.399 & 0.394 & 0.440 & 0.416
& 0.429 & 0.435 & 0.428 & 0.425 & 0.453 & 0.406 \\
BM25~\citep{bm25}
& 0.409 & 0.398 & 0.431 & 0.407 & 0.434 & 0.374
& 0.410 & 0.402 & 0.423 & 0.411 & 0.440 & 0.375 \\
RoBERTa~\citep{roberta}
& 0.406 & 0.415 & 0.394 & 0.399 & 0.423 & 0.392
& 0.397 & 0.409 & 0.390 & 0.382 & 0.416 & 0.381 \\
\bottomrule
\end{tabular}%
}
\end{table*}
\section{Compute Resources}\label{sec:app:compute}
This section details the computational and financial resources required to generate and evaluate our benchmark.
\subsection{Data Collation and Processing}
For the AoPS and NuminaMath datasets, the data was for the most part, readily available in a structured format, requiring only minor cleaning and deduplication. The official sources, however, required a more intensive processing pipeline, costing around \$300 in compute for OpenAI Batch API usage over the course of a total of 72 hours.
\subsection{Benchmark Generation}
After deriving the large corpus of problems as described in \cref{sec:tech:data}, the first stage of benchmark generation consists of extracting topic-based and solution-summary-based signals for each problem in the corpus. Topic extraction for all $\sim$283K problems took approximately two days using 8 H200 GPUs. Extracting summaries for all solutions in the corpus took approximately 12 hours using 4 H200 GPUs. Computing the BMA and Jaccard scores was comparatively efficient, requiring 1 hour on 25 CPU cores. Storing the resulting signals for all problem pairs in the corpus required around 300GB of disk space, where we used a more efficient byte-level storage format.
After filtering for relevant problems, the next stage uses a Swiss-style tournament together with the Bradley--Terry algorithm. Pairwise judging for the 1000 filtered queries took approximately 160 hours on 4 H200 GPU nodes.
\subsection{Applying the Benchmark}
Computing the benchmark results in \cref{tab:statement-full,tab:additional_settings} took approximately 1 hour using 4 H200 GPUs for the open-weight embedding models. For the OpenAI and Gemini models, our usage totalled approximately \$20 in API costs.
\section{Prompts} \label{sec:app:prompts}
In this section, we make available the prompts used to generate our benchmark.
\subsection{Ontology Topic Assignment Prompt}
\begin{prompt}{Ontology topic assignment}
[SYSTEM]
You are an Expert Mathematical Annotator. You will be provided with a mathematical text(problem, solution, theorem, lesson, etc.) and a list of possible tags. You must assign relevance score for each of the tags with respect to the provided text.
# INPUT
You will be provided with:
1. A mathematical text
2. A comma-separated list of tags
# TASK
For each of the provided tags assign a real number from the interval [0.0, 1.0]. The most relevant tag to the text must have a relevance score of 1.0 and the most irrelevant must have a score of 0.0. Align the remaining tags with respect to those.
# OUTPUT SCHEMA
Constrain your output to a pure JSON with no explanations, markdown or comments. Return only pure JSON. Follow the schema - each tag name is a key in the returned JSON object with a value the relevenace score:
```json
{
"<tagName>": <float | tag relevance score 0.0-1.0 to the provided mathematical text>,
...
}
[USER]
# Mathematical Text:
{problem_and_solution}
With respect to the above text, assign relevance scores to the following tags and return the results in JSON:
{list_of_comma_separaed_tags}
\end{prompt}
\subsection{Extracting Short Solution Summaries Prompt}
\begin{prompt}{Extracting short solution summaries}
# INSTRUCTION
You are an expert mathematical annotator tasked with identifying the *core idea*-the central mathematical insight-from a math problem.
# GOALS
1. Identify what makes the solution work conceptually, not how to carry it out. Capture the untrial step or idea that is the greatest hint for the solution.
2. Never include any multi-step reasoning, equations, or numeric computations. Don't include any annotations that are in the solution but not in the original problem statement.
3. Never try to solve the problem on your own, and don't include your reasoning or thoughts.
4. Output a single valid JSON object matching the schema below.
5. Structure the ideas imperatively so they look like you are giving a hint to someone.
6. If the problem seems too easy or straightforward, or you can't identify a core idea, store its value as 'null' and set the 'noCoreIdea' to 'true'.
# SCHEMA
```json
{
"noCoreIdea": <true|false>,
"coreIdea": "<string - one short sentence (up to 30 words) naming the main insight to the problem>",
"supportingIdeas": ["<strings - 0-3 short technique phrases>"],
"keywords": ["<strings - 1-2 word phrases summarizing the ideas, theorems, etc. in the solution>"],
"confidence": <0.0-1.0>
}
Here are the problem statement and solution:
Statement: {problem}
Solution: {solution}
\end{prompt}
\subsection{Pairwise Relevance Judging Prompt}
\begin{prompt}{Pairwise relevance judging}
You are an expert mathematician and analytical-reasoning engine. Your job is to decide which of two sample problems (Sample 1 or Sample 2) is *conceptually closer* to a target problem, judged by the *mathematical ideas and solution techniques actually used* (not surface wording).
You will receive 3 complete items, each with BOTH a problem statement and its correct solution:
[Target]
- Problem: {target_problem}
- Correct Solution: {target_solution}
[Sample 1]
- Problem: {sample1_problem}
- Correct Solution: {sample1_solution}
[Sample 2]
- Problem: {sample2_problem}
- Correct Solution: {sample2_solution}
Your output must follow the steps and formatting rules below exactly.
--------
STEP 1 - Technique Extraction (do this for Target, Sample 1, Sample 2)
For each item, read the Problem + Correct Solution together, then produce a structured list containing:
A. Core topic tags (2-6 tags)
Examples: "modular arithmetic", "similar triangles", "graph invariants", "generating functions", "optimization via calculus", "AM-GM", "inclusion-exclusion".
B. Key tools / theorems / lemmas explicitly used
Name the theorem/identity/strategy when possible (e.g., "Pigeonhole Principle", "Cauchy-Schwarz", "Euclidean algorithm", "double counting", "invariant/monovariant", "casework", "recurrence + induction").
C. Primary solution moves (3-8 bullets)
Describe the actual moves that drive the solution, e.g.:
- "rewrite expression into telescoping sum"
- "introduce substitution of variable to linearize"
- "construct auxiliary line and apply angle chase"
- "count objects in two ways; equate counts"
- "prove monotonicity; then bound endpoints"
D. Structural signature (1-3 sentences)
Summarize the solution's backbone: what is introduced, what is transformed, and what final step seals the result.
Label these sections clearly as:
- Target: Technique Profile
- Sample 1: Technique Profile
- Sample 2: Technique Profile
--------
STEP 2 - Comparative Analysis
Compare each sample to the target using *deep* similarity signals:
1) Technique overlap:
- Are the same kinds of tools central (not merely mentioned)?
- Are the same transformations used (e.g., factoring vs. symmetry/invariants vs. counting)?
2) Problem structure alignment:
- Are the same intermediate objects introduced (auxiliary point, substitution, generating function, invariant quantity, etc.)?
- Do both require the same "shape" of argument (e.g., construction + chase, or setup of recurrence + induction, or extremal argument + contradiction)? This criterion should be weighted lower than the technique overlap but can be a tie-breaker when techniques are similar.
3) Difficulty is NOT the criterion:
- Prefer shared method and structure over "hard/easy".
4) Penalize superficial similarity:
- Do NOT reward matching variable names, story context, or domain language if the underlying method differs. Further, algebraic computations do not imply similarity if the core method is different, or if the algebra is just a technical detail rather than the main driver.
Produce two subsections:
- Sample 1 vs Target -- Similarities / Differences / Overall Match Score (qualitative)
- Sample 2 vs Target -- Similarities / Differences / Overall Match Score (qualitative)
--------
STEP 3 - Decision + Tie-break rules
Choose the sample that is fundamentally closer in methods and structure.
If it's close, use these tie-breakers in order:
1) The sample whose *main* technique is the same as the target's main technique.
2) The sample whose *sequence of moves* (setup -> transformation -> key lemma -> finish) matches more closely.
3) The sample that relies on the same representation (e.g., algebraic manipulation vs. geometric configuration vs. combinatorial counting vs. graph reasoning).
--------
OUTPUT FORMAT (must follow)
- You may include your extracted profiles and comparisons above.
- Output your final decision on the last line, formatted as either:
$\\boxed{{1}}$ -> corresponding to the first sample being more similar to the target,
or
$\\boxed{{2}}$ -> corresponding to the second sample being more similar to the target.
\end{prompt}
\subsection{Ordinal Judge Prompt}
\begin{prompt}{Ordinal ranking prompt for additional ordering experiments \cref{sec:app:human_evaluation}}
You are a helpful mathematical expert and technique-similarity evaluator. You will be given two mathematical items: a Target problem with its correct solution, and a Sample problem with its correct solution.
Your task is to determine how similar the two problems are based on the mathematical techniques, ideas, and solution structure actually used in their solutions --- not based on superficial wording, notation, topic labels, or story context.
You will receive:
[Target]
Problem: {target_problem}
Correct Solution: {target_solution}
[Sample]
Problem: {sample_problem}
Correct Solution: {sample_solution}
Follow the instructions below exactly.
STEP 1 - Technique Extraction
For each item, read the Problem and Correct Solution together and identify all techniques genuinely used in the solution.
For both Target and Sample, produce:
A. Core topic tags (2-8 tags)
Examples: "modular arithmetic", "similar triangles", "double counting", "generating functions", "optimization", "AM-GM", "inclusion-exclusion", "induction", "graph invariants".
B. Key tools / theorems / identities / lemmas used
Name explicit or implicit tools when possible, such as:
- Pigeonhole Principle
- Cauchy-Schwarz
- Euclidean algorithm
- Vieta's formulas
- telescoping
- contradiction
- casework
- recurrence + induction
- symmetry
- invariant / monovariant
- angle chase
- substitution
- extremal argument
- bounding
C. Primary solution moves (3-10 bullets)
Describe the actual moves that drive the solution, for example:
- rewrite the expression into a factorized form
- introduce a substitution to simplify the equation
- construct an auxiliary object
- split into cases
- count the same set in two ways
- derive a recurrence and solve it
- apply a standard inequality after normalization
- use parity or modular reduction
- reduce to a previously established identity
D. Structural signature (1-3 sentences)
Summarize the backbone of the argument: how the solution starts, what key transformation or idea unlocks it, and what final step completes the proof.
Label these sections clearly as:
Target: Technique Profile
Sample: Technique Profile
STEP 2 - Comparative Analysis
Compare the Sample to the Target using the extracted technique profiles.
Your comparison must include:
1. Technique overlap
- Which techniques are shared?
- Which shared techniques are central rather than incidental?
- Are the same theorems, transformations, or proof strategies doing the main work?
2. Technique differences
- Which important techniques appear in one but not the other?
- Do the problems rely on different core ideas even if they are in the same broad subject?
3. Structural alignment
- Do the two solutions have a similar shape, such as:
- setup -> substitution -> simplification -> conclusion
- construction -> theorem application -> chase -> finish
- recurrence -> induction
- extremal argument -> contradiction
- counting representation -> double count -> algebraic cleanup
- This matters, but it should be weighted less than overlap in core techniques.
Important rules:
- Focus on the methods actually used in the solutions.
- Do NOT reward superficial similarity such as matching notation, similar phrasing, same answer form, or broad subject area alone.
- Do NOT treat routine algebraic manipulation as strong similarity unless it is a central technique in both.
- Weight main techniques much more heavily than minor technical steps.
- If one problem uses a tool only incidentally but the other depends on it centrally, count that as weak overlap.
STEP 3 - Similarity Score
Assign a single integer score from 0 to 100 that reflects how similar the Sample is to the Target in terms of mathematical method and solution structure.
Use this scale as guidance:
0-10: essentially no meaningful similarity in method
11-25: very weak similarity; mostly superficial or broad-topic overlap
26-40: limited similarity; some secondary techniques overlap but main approach differs
41-55: moderate similarity; at least one important method overlaps
56-70: fairly strong similarity; core ideas overlap substantially, with notable differences
71-85: strong similarity; main methods and argument structure are closely aligned
86-95: very strong similarity; nearly the same technique and solution pattern
96-100: essentially the same core method, with only minor presentation differences
Give a brief justification for the score in 2-5 sentences.
OUTPUT FORMAT
Use exactly this structure:
Target: Technique Profile
- Core topic tags: ...
- Key tools / theorems / identities / lemmas: ...
- Primary solution moves:
- ...
- ...
- ...
- Structural signature: ...
Sample: Technique Profile
- Core topic tags: ...
- Key tools / theorems / identities / lemmas: ...
- Primary solution moves:
- ...
- ...
- ...
- Structural signature: ...
Comparison
- Technique overlap: ...
- Technique differences: ...
- Structural alignment: ...
- Score justification: ...
Final Score:
\boxed{{N}}
where N is a single integer from 0 to 100.
\end{prompt}
\subsection{Mathematical Equation Formatting Prompt}
\begin{prompt}{Formatting mathematical equations into LaTeX}
Role: You are a LaTeX Formatting Expert specializing in mathematical notation standardization.
Task: You will be provided with mathematical text (problems or solutions). Your task is to standardize the LaTeX formatting and enclose the entire final result within a \boxed{{...}} command.
Strict Constraints:
1. DELIMITER CONVERSION: Replace all instances of \( ... \) and \\( ... \\) with standard dollar signs.
- Use $...$ for inline text.
- Use $$...$$ for display equations.
2. UNIVERSAL MATH TAGGING: Apply math mode ($...$) to every single mathematical element without exception.
3. CONTENT INTEGRITY: Do not solve the problem or edit the prose.
4. FINAL WRAPPING: The entire output must be contained within \boxed{{ <your_formatted_text_here> }}.
5. NO VERBOSITY: Provide ONLY the \boxed{{...}} block.
Example Transformation:
Input: If the radius r is 5, find the area. Use \( \pi \).
Output: \boxed{{If the radius $r$ is $5$, find the area. Use $\pi$.}}
\end{prompt}
\subsection{Human Annotator Instructions} \label{sec:app:human_evaluation_instructions}
\begin{prompt}{Human Annotator Instructions}
Goal: Compare Candidate 1 and Candidate 2 against the Target problem.
Candidate 1: Choose this if Candidate 1 is more similar to the Target.
Candidate 2: Choose this if Candidate 2 is more similar to the Target.
Tie: Choose this if both candidates are equally good / bad / equally similar.
Save: Records your judgment. Completed tasks turn green.
Remove judgment: Deletes your previous answer after confirmation. This removes the task from the list of "Finished Tasks".
\end{prompt}
\section{Use of LLMs in This Work}
In this work, we have not used LLMs for research design, ideation, or decision-making. We have used LLMs to write code for assisting with plot creation and formatting, but all code was reviewed and edited by the authors. We have used LLMs to assist with polishing the writing, but all content was written by the authors and reviewed for accuracy. We have not relied on LLMs for generating any part of the bibliography, and all references were added manually by the authors.
\begin{thebibliography}{75}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Abbasiantaeb et~al.(2024)Abbasiantaeb, Meng, Azzopardi, and
Aliannejadi]{abbasiantaeb2024holes}
Abbasiantaeb, Z., Meng, C., Azzopardi, L., and Aliannejadi, M.
\newblock Can we use large language models to fill relevance judgment holes?
\newblock In Acharya, P., Clarke, C. L.~A., Crestani, F., Fu, X., Jones, G.
J.~F., Kando, N., Kato, M.~P., Lipani, A., and Liu, Y. (eds.), \emph{Joint
Proceedings of the 1st Workshop on Evaluation Methodologies,Testbeds and
Community for Information Access Research {(EMTCIR} 2024) and the 1st
Workshop on User Modelling in Conversational Information Retrieval {(UM-CIR}
2024) co-located with the 2nd International {ACM} {SIGIR} Conference on
Information Retrieval in the Asia Pacific {(SIGIR-AP} 2024), Tokyo, Japan,
December 12, 2024}, {CEUR} Workshop Proceedings. CEUR-WS.org, 2024.
\newblock URL \url{https://ceur-ws.org/Vol-3854/emtcir-2.pdf}.
\bibitem[Agarwal et~al.(2025)Agarwal, Ahmad, Ai, Altman, Applebaum, Arbus,
et~al.]{oss}
Agarwal, S., Ahmad, L., Ai, J., Altman, S., Applebaum, A., Arbus, E., et~al.
\newblock gpt-oss-120b \& gpt-oss-20b model card.
\newblock \emph{arXiv preprint arXiv:2508.10925}, 2025.
\newblock URL \url{https://arxiv.org/abs/2508.10925}.
\bibitem[Akram et~al.(2026)Akram, Sturua, Havriushenko, Herreros, G{\"u}nther,
Werk, and Xiao]{jina}
Akram, M.~K., Sturua, S., Havriushenko, N., Herreros, Q., G{\"u}nther, M.,
Werk, M., and Xiao, H.
\newblock jina-embeddings-v5-text: Task-targeted embedding distillation.
\newblock \emph{arXiv preprint arXiv:2602.15547}, 2026.
\bibitem[Alshammari et~al.(2026)Alshammari, Wen, Zainal, Hamilton, Safaei,
Albarakati, Freeman, and Torralba]{mathnet}
Alshammari, S.~N., Wen, K., Zainal, A., Hamilton, M., Safaei, N., Albarakati,
S., Freeman, W.~T., and Torralba, A.
\newblock Mathnet: a global multimodal benchmark for mathematical reasoning and
retrieval.
\newblock \emph{CoRR}, abs/2604.18584, 2026.
\newblock \doi{10.48550/ARXIV.2604.18584}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2604.18584}.
\bibitem[Babakhin et~al.(2025)Babakhin, Osmulski, Ak, Moreira, Xu, Schifferer,
Liu, and Oldridge]{nemotron}
Babakhin, Y., Osmulski, R., Ak, R., Moreira, G., Xu, M., Schifferer, B., Liu,
B., and Oldridge, E.
\newblock Llama-embed-nemotron-8b: A universal text embedding model for
multilingual and cross-lingual tasks, 2025.
\newblock URL \url{https://arxiv.org/abs/2511.07025}.
\bibitem[Bansal et~al.(2019)Bansal, Loos, Rabe, Szegedy, and Wilcox]{holist}
Bansal, K., Loos, S.~M., Rabe, M.~N., Szegedy, C., and Wilcox, S.
\newblock Holist: An environment for machine learning of higher order logic
theorem proving.
\newblock In Chaudhuri, K. and Salakhutdinov, R. (eds.), \emph{Proceedings of
the 36th International Conference on Machine Learning, {ICML} 2019, 9-15 June
2019, Long Beach, California, {USA}}, Proceedings of Machine Learning
Research, pp.\ 454--463. {PMLR}, 2019.
\newblock URL \url{http://proceedings.mlr.press/v97/bansal19a.html}.
\bibitem[Bansal \& Sharma(2023)Bansal and Sharma]{bansal2023annotation}
Bansal, P. and Sharma, A.
\newblock Large language models as annotators: Enhancing generalization of
{NLP} models at minimal cost.
\newblock \emph{CoRR}, abs/2306.15766, 2023.
\newblock \doi{10.48550/ARXIV.2306.15766}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2306.15766}.
\bibitem[Bian et~al.(2025)Bian, Geng, Yang, and Cheng]{bian2025automathkg}
Bian, R., Geng, Y., Yang, Z., and Cheng, B.
\newblock Automathkg: The automated mathematical knowledge graph based on llm
and vector database.
\newblock \emph{Computational Intelligence}, 41\penalty0 (4):\penalty0 e70096,
2025.
\bibitem[Buckley \& Voorhees(2004)Buckley and Voorhees]{buckley2004retrieval}
Buckley, C. and Voorhees, E.~M.
\newblock Retrieval evaluation with incomplete information.
\newblock In Sanderson, M., J{\"{a}}rvelin, K., Allan, J., and Bruza, P.
(eds.), \emph{{SIGIR} 2004: Proceedings of the 27th Annual International
{ACM} {SIGIR} Conference on Research and Development in Information
Retrieval, Sheffield, UK, July 25-29, 2004}, pp.\ 25--32. {ACM}, 2004.
\newblock \doi{10.1145/1008992.1009000}.
\newblock URL \url{https://doi.org/10.1145/1008992.1009000}.
\bibitem[Chan et~al.(2023)Chan, Chen, Su, Yu, Xue, Zhang, Fu, and
Liu]{chan2023chateval}
Chan, C.-M., Chen, W., Su, Y., Yu, J., Xue, W., Zhang, S., Fu, J., and Liu, Z.
\newblock Chateval: Towards better llm-based evaluators through multi-agent
debate.
\newblock \emph{arXiv preprint arXiv:2308.07201}, 2023.
\bibitem[Chen et~al.(2024)Chen, Xiao, Zhang, Luo, Lian, and Liu]{bge-m3}
Chen, J., Xiao, S., Zhang, P., Luo, K., Lian, D., and Liu, Z.
\newblock Bge m3-embedding: Multi-lingual, multi-functionality,
multi-granularity text embeddings through self-knowledge distillation, 2024.
\bibitem[Cobbe et~al.(2021)Cobbe, Kosaraju, Bavarian, Chen, Jun, Kaiser,
Plappert, Tworek, Hilton, Nakano, Hesse, and Schulman]{gsm8k}
Cobbe, K., Kosaraju, V., Bavarian, M., Chen, M., Jun, H., Kaiser, L., Plappert,
M., Tworek, J., Hilton, J., Nakano, R., Hesse, C., and Schulman, J.
\newblock Training verifiers to solve math word problems.
\newblock \emph{CoRR}, abs/2110.14168, 2021.
\newblock URL \url{https://arxiv.org/abs/2110.14168}.
\bibitem[Dekoninck et~al.(2024)Dekoninck, Baader, and Vechev]{jasper}
Dekoninck, J., Baader, M., and Vechev, M.~T.
\newblock Polyrating: A cost-effective and bias-aware rating system for llm
evaluation.
\newblock \emph{arXiv preprint arXiv:2409.00696}, 2024.
\bibitem[Devlin et~al.(2018)Devlin, Chang, Lee, and Toutanova]{bert}
Devlin, J., Chang, M., Lee, K., and Toutanova, K.
\newblock {BERT:} pre-training of deep bidirectional transformers for language
understanding.
\newblock \emph{CoRR}, abs/1810.04805, 2018.
\newblock URL \url{http://arxiv.org/abs/1810.04805}.
\bibitem[Dixit \& Oates(2024)Dixit and Oates]{dixit2024sbi}
Dixit, P. and Oates, T.
\newblock Sbi-rag: Enhancing math word problem solving for students through
schema-based instruction and retrieval-augmented generation.
\newblock \emph{arXiv preprint arXiv:2410.13293}, 2024.
\bibitem[Dubois et~al.(2024)Dubois, Galambosi, Liang, and
Hashimoto]{dubois2024length}
Dubois, Y., Galambosi, B., Liang, P., and Hashimoto, T.~B.
\newblock Length-controlled alpacaeval: A simple way to debias automatic
evaluators.
\newblock \emph{arXiv preprint arXiv:2404.04475}, 2024.
\bibitem[Ferrag et~al.(2025)Ferrag, Tihanyi, and Debbah]{reasoningsurvey}
Ferrag, M.~A., Tihanyi, N., and Debbah, M.
\newblock From {LLM} reasoning to autonomous {AI} agents: {A} comprehensive
review.
\newblock \emph{CoRR}, abs/2504.19678, 2025.
\newblock \doi{10.48550/ARXIV.2504.19678}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2504.19678}.
\bibitem[Gao et~al.(2024)Gao, Ju, Jiang, Qin, and Dong]{gao2024semantic}
Gao, G., Ju, H., Jiang, J., Qin, Z., and Dong, B.
\newblock A semantic search engine for mathlib4.
\newblock In \emph{Findings of the Association for Computational Linguistics:
EMNLP 2024}, pp.\ 8001--8013, 2024.
\bibitem[Gilardi et~al.(2023)Gilardi, Alizadeh, and Kubli]{gilardi2023chatgpt}
Gilardi, F., Alizadeh, M., and Kubli, M.
\newblock Chatgpt outperforms crowd-workers for text-annotation tasks.
\newblock \emph{CoRR}, abs/2303.15056, 2023.
\newblock \doi{10.48550/ARXIV.2303.15056}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2303.15056}.
\bibitem[Hendrycks et~al.(2021)Hendrycks, Burns, Kadavath, Arora, Basart, Tang,
Song, and Steinhardt]{hendrycksmath}
Hendrycks, D., Burns, C., Kadavath, S., Arora, A., Basart, S., Tang, E., Song,
D., and Steinhardt, J.
\newblock Measuring mathematical problem solving with the {MATH} dataset.
\newblock In Vanschoren, J. and Yeung, S. (eds.), \emph{Proceedings of the
Neural Information Processing Systems Track on Datasets and Benchmarks 1,
NeurIPS Datasets and Benchmarks 2021, December 2021, virtual}, 2021.
\newblock URL
\url{https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/be83ab3ecd0db773eb2dc1b0a17836a1-Abstract-round2.html}.
\bibitem[Hong et~al.(2024)Hong, Kim, Kang, Myaeng, and Whang]{hong2024gullible}
Hong, G., Kim, J., Kang, J., Myaeng, S., and Whang, J.~J.
\newblock Why so gullible? enhancing the robustness of retrieval-augmented
models against counterfactual noise.
\newblock In Duh, K., G{\'{o}}mez{-}Adorno, H., and Bethard, S. (eds.),
\emph{Findings of the Association for Computational Linguistics: {NAACL}
2024, Mexico City, Mexico, June 16-21, 2024}, Findings of {ACL}, pp.\
2474--2495. Association for Computational Linguistics, 2024.
\newblock \doi{10.18653/V1/2024.FINDINGS-NAACL.159}.
\newblock URL \url{https://doi.org/10.18653/v1/2024.findings-naacl.159}.
\bibitem[Irving et~al.(2016)Irving, Szegedy, Alemi, E{\'{e}}n, Chollet, and
Urban]{irving2016deepmath}
Irving, G., Szegedy, C., Alemi, A.~A., E{\'{e}}n, N., Chollet, F., and Urban,
J.
\newblock Deepmath - deep sequence models for premise selection.
\newblock In Lee, D.~D., Sugiyama, M., von Luxburg, U., Guyon, I., and Garnett,
R. (eds.), \emph{Advances in Neural Information Processing Systems 29: Annual
Conference on Neural Information Processing Systems 2016, December 5-10,
2016, Barcelona, Spain}, pp.\ 2235--2243, 2016.
\newblock URL
\url{https://proceedings.neurips.cc/paper/2016/hash/f197002b9a0853eca5e046d9ca4663d5-Abstract.html}.
\bibitem[Jaccard(1912)]{jaccard}
Jaccard, P.
\newblock The distribution of the flora in the alpine zone.
\newblock \emph{New Phytologist}, 11\penalty0 (2):\penalty0 37--50, 1912.
\newblock \doi{10.1111/j.1469-8137.1912.tb05611.x}.
\bibitem[Ju \& Dong(2025)Ju and Dong]{mirb}
Ju, H. and Dong, B.
\newblock {MIRB:} mathematical information retrieval benchmark.
\newblock \emph{CoRR}, abs/2505.15585, 2025.
\newblock \doi{10.48550/ARXIV.2505.15585}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2505.15585}.
\bibitem[Ju et~al.(2026)Ju, Gao, Jiang, Wu, Sun, Chen, Wang, Wang, Wang, He,
Wu, Xiao, Liu, Dai, and Dong]{archon}
Ju, H., Gao, G., Jiang, J., Wu, B., Sun, Z., Chen, L., Wang, Y., Wang, Y.,
Wang, Z., He, W., Wu, P., Xiao, L., Liu, R., Dai, B., and Dong, B.
\newblock Automated conjecture resolution with formal verification.
\newblock \emph{CoRR}, abs/2604.03789, 2026.
\newblock \doi{10.48550/ARXIV.2604.03789}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2604.03789}.
\bibitem[Khramov et~al.(2025)Khramov, Kozyrev, Solovev, and
Podkopaev]{kozyrev2025rocqstar}
Khramov, N., Kozyrev, A., Solovev, G., and Podkopaev, A.
\newblock Rocqstar: Leveraging similarity-driven retrieval and agentic systems
for rocq generation.
\newblock \emph{CoRR}, abs/2505.22846, 2025.
\newblock \doi{10.48550/ARXIV.2505.22846}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2505.22846}.
\bibitem[Kwon et~al.(2023)Kwon, Li, Zhuang, Sheng, Zheng, Yu, Gonzalez, Zhang,
and Stoica]{vllm}
Kwon, W., Li, Z., Zhuang, S., Sheng, Y., Zheng, L., Yu, C.~H., Gonzalez, J.~E.,
Zhang, H., and Stoica, I.
\newblock Efficient memory management for large language model serving with
pagedattention.
\newblock In \emph{Proceedings of the ACM SIGOPS 29th Symposium on Operating
Systems Principles}, 2023.
\bibitem[Lee et~al.(2025)Lee, Chen, Dua, Cer, Shanbhogue, Naim, {\'{A}}brego,
Li, Chen, Vera, Ren, Zhang, Salz, Boratko, Han, Chen, Huang, Rao, Suganthan,
Han, Doumanoglou, Gupta, Moiseev, Yip, Jain, Baumgartner, Shahi, Gomez,
Mariserla, Choi, Shah, Goenka, Chen, Xia, Chen, Duddu, Chen, Walker, Zhou,
Ghiya, Gleicher, Gill, Dong, Seyedhosseini, Sung, Hoffmann, and
Duerig]{geminiembedding}
Lee, J., Chen, F., Dua, S., Cer, D., Shanbhogue, M., Naim, I., {\'{A}}brego,
G.~H., Li, Z., Chen, K., Vera, H.~S., Ren, X., Zhang, S., Salz, D., Boratko,
M., Han, J., Chen, B., Huang, S., Rao, V., Suganthan, P., Han, F.,
Doumanoglou, A., Gupta, N., Moiseev, F., Yip, C., Jain, A., Baumgartner, S.,
Shahi, S., Gomez, F.~P., Mariserla, S., Choi, M., Shah, P., Goenka, S., Chen,
K., Xia, Y., Chen, K., Duddu, S. M.~K., Chen, Y., Walker, T., Zhou, W.,
Ghiya, R., Gleicher, Z., Gill, K., Dong, Z., Seyedhosseini, M., Sung, Y.,
Hoffmann, R., and Duerig, T.
\newblock Gemini embedding: Generalizable embeddings from gemini.
\newblock \emph{CoRR}, abs/2503.07891, 2025.
\newblock \doi{10.48550/ARXIV.2503.07891}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2503.07891}.
\bibitem[Levonian et~al.(2023)Levonian, Li, Zhu, Gade, Henkel, Postle, and
Xing]{levonian2023retrieval}
Levonian, Z., Li, C., Zhu, W., Gade, A., Henkel, O., Postle, M.-E., and Xing,
W.
\newblock Retrieval-augmented generation to improve math question-answering:
Trade-offs between groundedness and human preference.
\newblock \emph{arXiv preprint arXiv:2310.03184}, 2023.
\bibitem[Lewis et~al.(2020)Lewis, Perez, Piktus, Petroni, Karpukhin, Goyal,
K{\"{u}}ttler, Lewis, Yih, Rockt{\"{a}}schel, Riedel, and Kiela]{rag}
Lewis, P., Perez, E., Piktus, A., Petroni, F., Karpukhin, V., Goyal, N.,
K{\"{u}}ttler, H., Lewis, M., Yih, W., Rockt{\"{a}}schel, T., Riedel, S., and
Kiela, D.
\newblock Retrieval-augmented generation for knowledge-intensive {NLP} tasks.
\newblock \emph{CoRR}, abs/2005.11401, 2020.
\newblock URL \url{https://arxiv.org/abs/2005.11401}.
\bibitem[LI et~al.(2024)LI, Beeching, Tunstall, Lipkin, Soletskyi, Huang,
Rasul, Yu, Jiang, Shen, Qin, Dong, Zhou, Fleureau, Lample, and
Polu]{numina_math_datasets}
LI, J., Beeching, E., Tunstall, L., Lipkin, B., Soletskyi, R., Huang, S.~C.,
Rasul, K., Yu, L., Jiang, A., Shen, Z., Qin, Z., Dong, B., Zhou, L.,
Fleureau, Y., Lample, G., and Polu, S.
\newblock Numinamath.
\newblock
\url{[https://huggingface.co/datasets/AI-MO/NuminaMath-1.5](https://github.com/project-numina/aimo-progress-prize/blob/main/report/numina_dataset.pdf)},
2024.
\bibitem[Li et~al.(2024)Li, Chiang, Frick, Dunlap, Wu, Zhu, Gonzalez, and
Stoica]{arenahard}
Li, T., Chiang, W.-L., Frick, E., Dunlap, L., Wu, T., Zhu, B., Gonzalez, J.~E.,
and Stoica, I.
\newblock From crowdsourced data to high-quality benchmarks: Arena-hard and
benchbuilder pipeline.
\newblock \emph{arXiv preprint arXiv:2406.11939}, 2024.
\newblock URL \url{https://arxiv.org/abs/2406.11939}.
\bibitem[Lin et~al.(2025)Lin, Deng, Chandu, Ravichander, Pyatkin, Dziri, Bras,
and Choi]{wildbench}
Lin, B.~Y., Deng, Y., Chandu, K.~R., Ravichander, A., Pyatkin, V., Dziri, N.,
Bras, R.~L., and Choi, Y.
\newblock Wildbench: Benchmarking llms with challenging tasks from real users
in the wild.
\newblock In \emph{The Thirteenth International Conference on Learning
Representations, {ICLR} 2025, Singapore, April 24-28, 2025}. OpenReview.net,
2025.
\newblock URL \url{https://openreview.net/forum?id=MKEHCx25xp}.
\bibitem[Lin(1998)]{linsim}
Lin, D.
\newblock An information-theoretic definition of similarity.
\newblock In \emph{Proceedings of the Fifteenth International Conference on
Machine Learning}, ICML '98, pp.\ 296–304, San Francisco, CA, USA, 1998.
Morgan Kaufmann Publishers Inc.
\newblock ISBN 1558605568.
\bibitem[Liu et~al.(2025)Liu, Zheng, Lu, Cao, and Yan]{liu2025rethinking}
Liu, Q., Zheng, X., Lu, X., Cao, Q., and Yan, J.
\newblock Rethinking and improving autoformalization: Towards a faithful metric
and a dependency retrieval-based approach.
\newblock In \emph{The Thirteenth International Conference on Learning
Representations}, 2025.
\bibitem[Liu et~al.(2019)Liu, Ott, Goyal, Du, Joshi, Chen, Levy, Lewis,
Zettlemoyer, and Stoyanov]{roberta}
Liu, Y., Ott, M., Goyal, N., Du, J., Joshi, M., Chen, D., Levy, O., Lewis, M.,
Zettlemoyer, L., and Stoyanov, V.
\newblock Roberta: {A} robustly optimized {BERT} pretraining approach.
\newblock \emph{CoRR}, abs/1907.11692, 2019.
\newblock URL \url{http://arxiv.org/abs/1907.11692}.
\bibitem[Liu et~al.(2024)Liu, Zhou, Guo, Shareghi, Vulic, Korhonen, and
Collier]{liu2024aligning}
Liu, Y., Zhou, H., Guo, Z., Shareghi, E., Vulic, I., Korhonen, A., and Collier,
N.
\newblock Aligning with human judgement: The role of pairwise preference in
large language model evaluators.
\newblock \emph{arXiv preprint arXiv:2403.16950}, 2024.
\bibitem[Madaan et~al.(2023)Madaan, Tandon, Gupta, Hallinan, Gao, Wiegreffe,
Alon, Dziri, Prabhumoye, Yang, Gupta, Majumder, Hermann, Welleck,
Yazdanbakhsh, and Clark]{selfrefine}
Madaan, A., Tandon, N., Gupta, P., Hallinan, S., Gao, L., Wiegreffe, S., Alon,
U., Dziri, N., Prabhumoye, S., Yang, Y., Gupta, S., Majumder, B.~P., Hermann,
K., Welleck, S., Yazdanbakhsh, A., and Clark, P.
\newblock Self-refine: Iterative refinement with self-feedback.
\newblock In Oh, A., Naumann, T., Globerson, A., Saenko, K., Hardt, M., and
Levine, S. (eds.), \emph{Advances in Neural Information Processing Systems
36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS
2023, New Orleans, LA, USA, December 10 - 16, 2023}, 2023.
\newblock URL
\url{http://papers.nips.cc/paper\_files/paper/2023/hash/91edff07232fb1b55a505a9e9f6c0ff3-Abstract-Conference.html}.
\bibitem[Microsoft(2026)]{harrier}
Microsoft.
\newblock Harrier-oss-v1.
\newblock \url{https://huggingface.co/microsoft/harrier-oss-v1-27b}, 2026.
\bibitem[Muennighoff et~al.(2023)Muennighoff, Tazi, Magne, and Reimers]{mteb}
Muennighoff, N., Tazi, N., Magne, L., and Reimers, N.
\newblock {MTEB:} massive text embedding benchmark.
\newblock In Vlachos, A. and Augenstein, I. (eds.), \emph{Proceedings of the
17th Conference of the European Chapter of the Association for Computational
Linguistics, {EACL} 2023, Dubrovnik, Croatia, May 2-6, 2023}, pp.\
2006--2029. Association for Computational Linguistics, 2023.
\newblock \doi{10.18653/V1/2023.EACL-MAIN.148}.
\newblock URL \url{https://doi.org/10.18653/v1/2023.eacl-main.148}.
\bibitem[Nakano et~al.(2021)Nakano, Hilton, Balaji, Wu, Ouyang, Kim, Hesse,
Jain, Kosaraju, Saunders, Jiang, Cobbe, Eloundou, Krueger, Button, Knight,
Chess, and Schulman]{webgpt}
Nakano, R., Hilton, J., Balaji, S., Wu, J., Ouyang, L., Kim, C., Hesse, C.,
Jain, S., Kosaraju, V., Saunders, W., Jiang, X., Cobbe, K., Eloundou, T.,
Krueger, G., Button, K., Knight, M., Chess, B., and Schulman, J.
\newblock Webgpt: Browser-assisted question-answering with human feedback.
\newblock \emph{CoRR}, abs/2112.09332, 2021.
\newblock URL \url{https://arxiv.org/abs/2112.09332}.
\bibitem[{OpenAI}(2024)]{openaiembedding}
{OpenAI}.
\newblock text-embedding-3-large model, 2024.
\newblock URL
\url{https://developers.openai.com/api/docs/models/text-embedding-3-large}.
\newblock Accessed: May 24, 2026.
\bibitem[Packer et~al.(2023)Packer, Fang, Patil, Lin, Wooders, and
Gonzalez]{memgpt}
Packer, C., Fang, V., Patil, S.~G., Lin, K., Wooders, S., and Gonzalez, J.~E.
\newblock Memgpt: Towards llms as operating systems.
\newblock \emph{CoRR}, abs/2310.08560, 2023.
\newblock \doi{10.48550/ARXIV.2310.08560}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2310.08560}.
\bibitem[Park et~al.(2023)Park, O'Brien, Cai, Morris, Liang, and
Bernstein]{park2023generative}
Park, J.~S., O'Brien, J.~C., Cai, C.~J., Morris, M.~R., Liang, P., and
Bernstein, M.~S.
\newblock Generative agents: Interactive simulacra of human behavior.
\newblock \emph{CoRR}, abs/2304.03442, 2023.
\newblock \doi{10.48550/ARXIV.2304.03442}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2304.03442}.
\bibitem[Pedregosa et~al.(2011)Pedregosa, Varoquaux, Gramfort, Michel, Thirion,
Grisel, Blondel, Prettenhofer, Weiss, Dubourg, VanderPlas, Passos,
Cournapeau, Brucher, Perrot, and Duchesnay]{scikit-learn}
Pedregosa, F., Varoquaux, G., Gramfort, A., Michel, V., Thirion, B., Grisel,
O., Blondel, M., Prettenhofer, P., Weiss, R., Dubourg, V., VanderPlas, J.,
Passos, A., Cournapeau, D., Brucher, M., Perrot, M., and Duchesnay, E.
\newblock Scikit-learn: Machine learning in python.
\newblock \emph{J. Mach. Learn. Res.}, 12:\penalty0 2825--2830, 2011.
\newblock \doi{10.5555/1953048.2078195}.
\newblock URL \url{https://dl.acm.org/doi/10.5555/1953048.2078195}.
\bibitem[Penedo et~al.(2024)Penedo, Kydlíček, Cappelli, Sasko, and
Wolf]{datatrove}
Penedo, G., Kydlíček, H., Cappelli, A., Sasko, M., and Wolf, T.
\newblock Datatrove: large scale data processing, 2024.
\newblock URL \url{https://github.com/huggingface/datatrove}.
\bibitem[Petrov et~al.(2026)Petrov, Dekoninck, Dimitrov, and Vechev]{proofrank}
Petrov, I., Dekoninck, J., Dimitrov, D.~I., and Vechev, M.
\newblock Not all proofs are equal: Evaluating llm proof quality beyond
correctness, 2026.
\newblock URL \url{https://arxiv.org/abs/2605.10379}.
\bibitem[Reimers \& Gurevych(2019)Reimers and Gurevych]{sentencetransformers}
Reimers, N. and Gurevych, I.
\newblock Sentence-bert: Sentence embeddings using siamese bert-networks.
\newblock In \emph{Proceedings of the 2019 Conference on Empirical Methods in
Natural Language Processing}. Association for Computational Linguistics, 11
2019.
\newblock URL \url{https://arxiv.org/abs/1908.10084}.
\bibitem[Robertson \& Zaragoza(2009)Robertson and Zaragoza]{bm25}
Robertson, S.~E. and Zaragoza, H.
\newblock The probabilistic relevance framework: {BM25} and beyond.
\newblock \emph{Found. Trends Inf. Retr.}, 3\penalty0 (4):\penalty0 333--389,
2009.
\newblock \doi{10.1561/1500000019}.
\newblock URL \url{https://doi.org/10.1561/1500000019}.
\bibitem[Saito et~al.(2023)Saito, Wachi, Wataoka, and Akimoto]{verbositybias}
Saito, K., Wachi, A., Wataoka, K., and Akimoto, Y.
\newblock Verbosity bias in preference labeling by large language models.
\newblock \emph{CoRR}, abs/2310.10076, 2023.
\newblock \doi{10.48550/ARXIV.2310.10076}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2310.10076}.
\bibitem[Sakai \& Kando(2008)Sakai and Kando]{sakai2008incomplete}
Sakai, T. and Kando, N.
\newblock On information retrieval metrics designed for evaluation with
incomplete relevance assessments.
\newblock \emph{Inf. Retr.}, 11\penalty0 (5):\penalty0 447--470, 2008.
\newblock \doi{10.1007/S10791-008-9059-7}.
\newblock URL \url{https://doi.org/10.1007/s10791-008-9059-7}.
\bibitem[Schechter~Vera et~al.(2025)Schechter~Vera, Dua, Zhang, Salz, Mullins,
Raghuram~Panyam, Smoot, Naim, Zou, Chen, Cer, Lisak, Choi, Gonzalez,
Sanseviero, Cameron, Ballantyne, Black, Chen, Wang, Li, Martins, Lee,
Sherwood, Ji, Wu, Zheng, Singh, Sharma, Sreepat, Jain, Elarabawy, Co,
Doumanoglou, Samari, Hora, Potetz, Kim, Alfonseca, Moiseev, Han, Palma~Gomez,
Hernández~Ábrego, Zhang, Hui, Han, Gill, Chen, Chen, Shanbhogue, Boratko,
Suganthan, Duddu, Mariserla, Ariafar, Zhang, Zhang, Baumgartner, Goenka, Qiu,
Dabral, Walker, Rao, Khawaja, Zhou, Ren, Xia, Chen, Chen, Dong, Ding, Visin,
Liu, Zhang, Kenealy, Casbon, Kumar, Mesnard, Gleicher, Brick, Lacombe,
Roberts, Sung, Hoffmann, Warkentin, Joulin, Duerig, and
Seyedhosseini]{embeddinggemma}
Schechter~Vera, H., Dua, S., Zhang, B., Salz, D., Mullins, R., Raghuram~Panyam,
S., Smoot, S., Naim, I., Zou, J., Chen, F., Cer, D., Lisak, A., Choi, M.,
Gonzalez, L., Sanseviero, O., Cameron, G., Ballantyne, I., Black, K., Chen,
K., Wang, W., Li, Z., Martins, G., Lee, J., Sherwood, M., Ji, J., Wu, R.,
Zheng, J., Singh, J., Sharma, A., Sreepat, D., Jain, A., Elarabawy, A., Co,
A., Doumanoglou, A., Samari, B., Hora, B., Potetz, B., Kim, D., Alfonseca,
E., Moiseev, F., Han, F., Palma~Gomez, F., Hernández~Ábrego, G., Zhang, H.,
Hui, H., Han, J., Gill, K., Chen, K., Chen, K., Shanbhogue, M., Boratko, M.,
Suganthan, P., Duddu, S. M.~K., Mariserla, S., Ariafar, S., Zhang, S., Zhang,
S., Baumgartner, S., Goenka, S., Qiu, S., Dabral, T., Walker, T., Rao, V.,
Khawaja, W., Zhou, W., Ren, X., Xia, Y., Chen, Y., Chen, Y.-T., Dong, Z.,
Ding, Z., Visin, F., Liu, G., Zhang, J., Kenealy, K., Casbon, M., Kumar, R.,
Mesnard, T., Gleicher, Z., Brick, C., Lacombe, O., Roberts, A., Sung, Y.,
Hoffmann, R., Warkentin, T., Joulin, A., Duerig, T., and Seyedhosseini, M.
\newblock Embeddinggemma: Powerful and lightweight text representations.
\newblock 2025.
\newblock URL \url{https://arxiv.org/abs/2509.20354}.
\bibitem[Shen et~al.(2025)Shen, Huang, Yang, Wang, Gao, Xu, Jiang, He, Yang,
Sun, et~al.]{shen2025real}
Shen, Z., Huang, N., Yang, F., Wang, Y., Gao, G., Xu, T., Jiang, J., He, W.,
Yang, P., Sun, M., et~al.
\newblock Real-prover: Retrieval augmented lean prover for mathematical
reasoning.
\newblock \emph{arXiv preprint arXiv:2505.20613}, 2025.
\bibitem[Shi et~al.(2025)Shi, Ma, Liang, Diao, Ma, and
Vosoughi]{judgingthejudges}
Shi, L., Ma, C., Liang, W., Diao, X., Ma, W., and Vosoughi, S.
\newblock Judging the judges: {A} systematic study of position bias in
llm-as-a-judge.
\newblock In Inui, K., Sakti, S., Wang, H., Wong, D.~F., Bhattacharyya, P.,
Banerjee, B., Ekbal, A., Chakraborty, T., and Singh, D.~P. (eds.),
\emph{Proceedings of the 14th International Joint Conference on Natural
Language Processing and the 4th Conference of the Asia-Pacific Chapter of the
Association for Computational Linguistics, {IJCNLP-AACL} 2025, Mumbai, India,
December 20-24, 2025}, pp.\ 292--314. The Asian Federation of Natural
Language Processing and The Association for Computational Linguistics, 2025.
\newblock URL \url{https://aclanthology.org/2025.ijcnlp-long.18/}.
\bibitem[Su et~al.(2025)Su, Yen, Xia, Shi, Muennighoff, Wang, Liu, Shi, Siegel,
Tang, Sun, Yoon, Arik, Chen, and Yu]{bright}
Su, H., Yen, H., Xia, M., Shi, W., Muennighoff, N., Wang, H., Liu, H., Shi, Q.,
Siegel, Z.~S., Tang, M., Sun, R., Yoon, J., Arik, S.~{\"{O}}., Chen, D., and
Yu, T.
\newblock {BRIGHT:} {A} realistic and challenging benchmark for
reasoning-intensive retrieval.
\newblock In \emph{The Thirteenth International Conference on Learning
Representations, {ICLR} 2025, Singapore, April 24-28, 2025}. OpenReview.net,
2025.
\newblock URL \url{https://openreview.net/forum?id=ykuc5q381b}.
\bibitem[Team(2025)]{octen2025rteb}
Team, O.
\newblock Octen series: Optimizing embedding models to no.1 on rteb
leaderboard, 2025.
\newblock URL
\url{https://octen-team.github.io/octen_blog/posts/octen-rteb-first-place/}.
\bibitem[Thakur et~al.(2021)Thakur, Reimers, R{\"{u}}ckl{\'{e}}, Srivastava,
and Gurevych]{beir}
Thakur, N., Reimers, N., R{\"{u}}ckl{\'{e}}, A., Srivastava, A., and Gurevych,
I.
\newblock {BEIR:} {A} heterogeneous benchmark for zero-shot evaluation of
information retrieval models.
\newblock In Vanschoren, J. and Yeung, S. (eds.), \emph{Proceedings of the
Neural Information Processing Systems Track on Datasets and Benchmarks 1,
NeurIPS Datasets and Benchmarks 2021, December 2021, virtual}, 2021.
\newblock URL
\url{https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/65b9eea6e1cc6bb9f0cd2a47751a186f-Abstract-round2.html}.
\bibitem[Thomas et~al.(2024)Thomas, Spielman, Craswell, and
Mitra]{thomas2024preferences}
Thomas, P., Spielman, S., Craswell, N., and Mitra, B.
\newblock Large language models can accurately predict searcher preferences.
\newblock In Yang, G.~H., Wang, H., Han, S., Hauff, C., Zuccon, G., and Zhang,
Y. (eds.), \emph{Proceedings of the 47th International {ACM} {SIGIR}
Conference on Research and Development in Information Retrieval, {SIGIR}
2024, Washington DC, USA, July 14-18, 2024}, pp.\ 1930--1940. {ACM}, 2024.
\newblock \doi{10.1145/3626772.3657707}.
\newblock URL \url{https://doi.org/10.1145/3626772.3657707}.
\bibitem[Upadhyay et~al.(2024)Upadhyay, Kamalloo, and Lin]{upadhyay2024patch}
Upadhyay, S., Kamalloo, E., and Lin, J.
\newblock Llms can patch up missing relevance judgments in evaluation.
\newblock \emph{CoRR}, abs/2405.04727, 2024.
\newblock \doi{10.48550/ARXIV.2405.04727}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2405.04727}.
\bibitem[Wang et~al.(2007)Wang, Du, Payattakool, Yu, and Chen]{bma}
Wang, J.~Z., Du, Z., Payattakool, R., Yu, P.~S., and Chen, C.-F.
\newblock A new method to measure the semantic similarity of go terms.
\newblock \emph{Bioinformatics}, 23\penalty0 (10):\penalty0 1274--1281, 05
2007.
\newblock ISSN 1367-4803.
\newblock \doi{10.1093/bioinformatics/btm087}.
\newblock URL \url{https://doi.org/10.1093/bioinformatics/btm087}.
\bibitem[Wang et~al.(2024)Wang, Yang, Huang, Yang, Majumder, and Wei]{e5}
Wang, L., Yang, N., Huang, X., Yang, L., Majumder, R., and Wei, F.
\newblock Multilingual e5 text embeddings: A technical report.
\newblock \emph{arXiv preprint arXiv:2402.05672}, 2024.
\bibitem[Wang et~al.(2025)Wang, Liu, Wang, Wang, Yan, Jia, Liu, Chen, Xu, Li,
and Yu]{reasoningsurvey2}
Wang, P., Liu, T., Wang, C., Wang, Y., Yan, S., Jia, C., Liu, X., Chen, X., Xu,
J., Li, Z., and Yu, Y.
\newblock A survey on large language models for mathematical reasoning.
\newblock \emph{CoRR}, abs/2506.08446, 2025.
\newblock \doi{10.48550/ARXIV.2506.08446}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2506.08446}.
\bibitem[Wataoka et~al.(2024)Wataoka, Takahashi, and Ri]{selfpreferencebias}
Wataoka, K., Takahashi, T., and Ri, R.
\newblock Self-preference bias in llm-as-a-judge.
\newblock \emph{CoRR}, abs/2410.21819, 2024.
\newblock \doi{10.48550/ARXIV.2410.21819}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2410.21819}.
\bibitem[Weisstein()]{MathWorld}
Weisstein, E.~W.
\newblock Mathworld---a wolfram web resource.
\newblock \url{https://mathworld.wolfram.com/}.
\newblock Accessed: 2026-05-07.
\bibitem[Wu et~al.(2024)Wu, Wu, and Zou]{clasheval}
Wu, K., Wu, E., and Zou, J.~Y.
\newblock Clasheval: Quantifying the tug-of-war between an llm's internal prior
and external evidence.
\newblock In Globersons, A., Mackey, L., Belgrave, D., Fan, A., Paquet, U.,
Tomczak, J.~M., and Zhang, C. (eds.), \emph{Advances in Neural Information
Processing Systems 38: Annual Conference on Neural Information Processing
Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024},
2024.
\newblock URL
\url{http://papers.nips.cc/paper\_files/paper/2024/hash/3aa291abc426d7a29fb08418c1244177-Abstract-Datasets\_and\_Benchmarks\_Track.html}.
\bibitem[Xiao et~al.(2024)Xiao, Hudson, and Moubayed]{xiao2024rar}
Xiao, C., Hudson, G.~T., and Moubayed, N.~A.
\newblock Rar-b: Reasoning as retrieval benchmark.
\newblock \emph{arXiv preprint arXiv:2404.06347}, 2024.
\bibitem[Yang et~al.(2023)Yang, Swope, Gu, Chalamala, Song, Yu, Godil, Prenger,
and Anandkumar]{yang2023leandojo}
Yang, K., Swope, A., Gu, A., Chalamala, R., Song, P., Yu, S., Godil, S.,
Prenger, R.~J., and Anandkumar, A.
\newblock Leandojo: Theorem proving with retrieval-augmented language models.
\newblock \emph{Advances in Neural Information Processing Systems},
36:\penalty0 21573--21612, 2023.
\bibitem[Yang et~al.(2024)Yang, Lin, Wang, and Zhai]{yang2024analogy}
Yang, X., Lin, J., Wang, Z., and Zhai, C.
\newblock Learning by analogy: Enhancing few-shot prompting for math word
problem solving with computational graph-based retrieval.
\newblock \emph{CoRR}, abs/2411.16454, 2024.
\newblock \doi{10.48550/ARXIV.2411.16454}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2411.16454}.
\bibitem[Ye et~al.(2024)Ye, Wang, Huang, Chen, Zhang, Moniz, Gao, Geyer, Huang,
Chen, Chawla, and Zhang]{justiceorprejudice}
Ye, J., Wang, Y., Huang, Y., Chen, D., Zhang, Q., Moniz, N., Gao, T., Geyer,
W., Huang, C., Chen, P., Chawla, N.~V., and Zhang, X.
\newblock Justice or prejudice? quantifying biases in llm-as-a-judge.
\newblock \emph{CoRR}, abs/2410.02736, 2024.
\newblock \doi{10.48550/ARXIV.2410.02736}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2410.02736}.
\bibitem[Yoran et~al.(2024)Yoran, Wolfson, Ram, and Berant]{yoran2024making}
Yoran, O., Wolfson, T., Ram, O., and Berant, J.
\newblock Making retrieval-augmented language models robust to irrelevant
context.
\newblock In \emph{The Twelfth International Conference on Learning
Representations, {ICLR} 2024, Vienna, Austria, May 7-11, 2024}.
OpenReview.net, 2024.
\newblock URL \url{https://openreview.net/forum?id=ZS4m74kZpH}.
\bibitem[Zhang et~al.(2025)Zhang, Li, Long, Zhang, Lin, Yang, Xie, Yang, Liu,
Lin, Huang, and Zhou]{qwen3embedding}
Zhang, Y., Li, M., Long, D., Zhang, X., Lin, H., Yang, B., Xie, P., Yang, A.,
Liu, D., Lin, J., Huang, F., and Zhou, J.
\newblock Qwen3 embedding: Advancing text embedding and reranking through
foundation models.
\newblock \emph{arXiv preprint arXiv:2506.05176}, 2025.
\bibitem[Zhao et~al.(2025)Zhao, Hu, Shan, Huang, Zhou, Zhang, Sun, Liu, Li,
Wei, Pan, Xiang, Zhang, Wang, Yu, Hu, and Zhang]{kalm}
Zhao, X., Hu, X., Shan, Z., Huang, S., Zhou, Y., Zhang, X., Sun, Z., Liu, Z.,
Li, D., Wei, X., Pan, Y., Xiang, Y., Zhang, M., Wang, H., Yu, J., Hu, B., and
Zhang, M.
\newblock Kalm-embedding-v2: Superior training techniques and data inspire a
versatile embedding model, 2025.
\newblock URL \url{https://arxiv.org/abs/2506.20923}.
\bibitem[Zheng et~al.(2023)Zheng, Chiang, Sheng, Zhuang, Wu, Zhuang, Lin, Li,
Li, Xing, Zhang, Gonzalez, and Stoica]{zheng2023judging}
Zheng, L., Chiang, W.-L., Sheng, Y., Zhuang, S., Wu, Z., Zhuang, Y., Lin, Z.,
Li, Z., Li, D., Xing, E.~P., Zhang, H., Gonzalez, J.~E., and Stoica, I.
\newblock Judging llm-as-a-judge with mt-bench and chatbot arena.
\newblock \emph{arXiv preprint arXiv:2306.05685}, 2023.
\bibitem[Zhong et~al.(2021)Zhong, Zhang, Xin, Zanibbi, and Lin]{approachzero}
Zhong, W., Zhang, X., Xin, J., Zanibbi, R., and Lin, J.
\newblock Approach zero and anserini at the {CLEF-2021} arqmath track: Applying
substructure search and {BM25} on operator tree path tokens.
\newblock In Faggioli, G., Ferro, N., Joly, A., Maistro, M., and Piroi, F.
(eds.), \emph{Proceedings of the Working Notes of {CLEF} 2021 - Conference
and Labs of the Evaluation Forum, Bucharest, Romania, September 21st - to -
24th, 2021}, {CEUR} Workshop Proceedings, pp.\ 133--156. CEUR-WS.org, 2021.
\newblock URL \url{https://ceur-ws.org/Vol-2936/paper-09.pdf}.
\bibitem[Zhu et~al.(2023)Zhu, Yuan, Wang, Liu, Liu, Deng, Dou, and
Wen]{irsurvey}
Zhu, Y., Yuan, H., Wang, S., Liu, J., Liu, W., Deng, C., Dou, Z., and Wen, J.
\newblock Large language models for information retrieval: {A} survey.
\newblock \emph{CoRR}, abs/2308.07107, 2023.
\newblock \doi{10.48550/ARXIV.2308.07107}.
\newblock URL \url{https://doi.org/10.48550/arXiv.2308.07107}.
\end{thebibliography}