Mitigating Premature Exploitation in Particle-based Monte Carlo for Inference-Time Scaling
Extracted source text · arXiv 2510.05825
\documentclass{article}
\usepackage{_ICLR/iclr2026_conference,times}
\usepackage{hyperref}
\usepackage{url}
\input{util/pkg}
\input{util/def}
\usepackage{appendix}
\title{Mitigating Premature Exploitation in Particle-based Monte Carlo for Inference-Time Scaling}
\author{\textbf{Giorgio Giannone}$^{1}$,
\textbf{Guangxuan Xu}$^{1}$,
\textbf{Nikhil Shivakumar Nayak}$^{1}$,
\\
\textbf{Rohan Mahesh Awhad}$^{1}$\textbf{,}
\textbf{Shivchander Sudalairaj}$^{1}$\textbf{,}
\textbf{Kai Xu}$^{1}$\textbf{,}
\textbf{Akash Srivastava}$^{2}$
\\
\\
$^1$Red Hat AI Innovation \quad $^2$Core AI, IBM
}
\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}
\iclrfinalcopy
\begin{document}
\maketitle
\input{sec/0_abstract}
\input{sec/1_intro}
\input{sec/2_background}
\input{sec/3_method}
\input{sec/4_experiment}
\input{sec/5_related_work}
\input{sec/6_conclusion}
\clearpage
\small
\bibliographystyle{_ICLR/iclr2026_conference}
\bibliography{biblio.bib}
\clearpage
\normalsize
\appendix
\input{sec/7_appendix}
\end{document}
\usepackage[utf8]{inputenc} %
\usepackage[T1]{fontenc} %
\usepackage{booktabs} %
\usepackage{amsfonts} %
\usepackage{nicefrac} %
\usepackage{microtype} %
\usepackage{graphicx}
\usepackage{amssymb, mathtools, amsmath}
\usepackage{multirow}
\newcommand{\theHalgorithm}{\arabic{algorithm}}
\usepackage{hyperref}
\usepackage{url}
\usepackage{amsfonts} %
\usepackage{nicefrac} %
\usepackage{microtype} %
\usepackage{xcolor} %
\usepackage[small,bf]{caption}
\usepackage[inline]{enumitem}
\usepackage{rotating}
\usepackage{wrapfig}
\usepackage{multicol}
\usepackage{algorithm}
\usepackage{algpseudocode}
\usepackage{adjustbox}
\usepackage{subcaption}
\usepackage{pifont}
\usepackage{framed}
\usepackage{fancyvrb}
\usepackage{xcolor}
\usepackage{bbm}
\hypersetup{
colorlinks = true,
citecolor=blue,
linkcolor=blue,
urlcolor=blue,
pdfborder = 0 0 0
}
\usepackage{amsthm}
\usepackage{listings}
\usepackage{framed} %
\usepackage[framemethod=TikZ]{mdframed} %
\definecolor{darkgreen}{rgb}{0.0, 0.5, 0.0}
\definecolor{blue}{rgb}{0.0, 0.47, 0.75}
\definecolor{dartmouthgreen}{rgb}{0.05, 0.5, 0.06}
\definecolor{drab}{rgb}{0.59, 0.44, 0.09}
\definecolor{navyblue}{rgb}{0.0, 0.0, 0.5}
\definecolor{darkgreen}{rgb}{0.0, 0.5, 0.0}
\definecolor{blue}{rgb}{0.0, 0.47, 0.75}
\definecolor{dartmouthgreen}{rgb}{0.05, 0.5, 0.06}
\definecolor{drab}{rgb}{0.59, 0.44, 0.09}
\definecolor{navyblue}{rgb}{0.0, 0.0, 0.5}
\newcommand{\mb}[1]{\mathbf{#1}}
\newcommand{\kl}{\mathcal{D}_{\operatorname{KL}}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\KL}{\mathbb{KL}}
\newcommand{\prodl}{\prod^{L-1}_{l=1}}
\newcommand{\suml}{\sum^{L-1}_{l=1}}
\newcommand{\tb}[1]{\textbf{#1}}
\newcommand{\va}{\mathbf{a}}
\newcommand{\vb}{\mathbf{b}}
\newcommand{\vc}{\mathbf{c}}
\newcommand{\vd}{\mathbf{d}}
\newcommand{\ve}{\mathbf{e}}
\newcommand{\vf}{\mathbf{f}}
\newcommand{\vg}{\mathbf{g}}
\newcommand{\vh}{\mathbf{h}}
\newcommand{\vi}{\mathbf{i}}
\newcommand{\vj}{\mathbf{j}}
\newcommand{\vk}{\mathbf{k}}
\newcommand{\vl}{\mathbf{l}}
\newcommand{\vm}{\mathbf{m}}
\newcommand{\vn}{\mathbf{n}}
\newcommand{\vo}{\mathbf{o}}
\newcommand{\vp}{\mathbf{p}}
\newcommand{\vq}{\mathbf{q}}
\newcommand{\vr}{\mathbf{r}}
\newcommand{\vvs}{\mathbf{s}}
\newcommand{\vt}{\mathbf{t}}
\newcommand{\vu}{\mathbf{u}}
\newcommand{\vvv}{\mathbf{v}}
\newcommand{\vw}{\mathbf{w}}
\newcommand{\vx}{\mathbf{x}}
\newcommand{\vy}{\mathbf{y}}
\newcommand{\vz}{\mathbf{z}}
\newcommand{\vA}{\mathbf{A}}
\newcommand{\vB}{\mathbf{B}}
\newcommand{\vC}{\mathbf{C}}
\newcommand{\vD}{\mathbf{D}}
\newcommand{\vE}{\mathbf{E}}
\newcommand{\vF}{\mathbf{F}}
\newcommand{\vG}{\mathbf{G}}
\newcommand{\vH}{\mathbf{H}}
\newcommand{\vI}{\mathbf{I}}
\newcommand{\vJ}{\mathbf{J}}
\newcommand{\vK}{\mathbf{K}}
\newcommand{\vL}{\mathbf{L}}
\newcommand{\vM}{\mathbf{M}}
\newcommand{\vN}{\mathbf{N}}
\newcommand{\vO}{\mathbf{O}}
\newcommand{\vP}{\mathbf{P}}
\newcommand{\vQ}{\mathbf{Q}}
\newcommand{\vR}{\mathbf{R}}
\newcommand{\vS}{\mathbf{S}}
\newcommand{\vT}{\mathbf{T}}
\newcommand{\vU}{\mathbf{U}}
\newcommand{\vV}{\mathbf{V}}
\newcommand{\vW}{\mathbf{W}}
\newcommand{\vX}{\mathbf{X}}
\newcommand{\vY}{\mathbf{Y}}
\newcommand{\vZ}{\mathbf{Z}}
\newcommand{\valpha}{\mathbf{\alpha}}
\newcommand{\vbeta}{\mathbf{\beta}}
\newcommand{\vgamma}{\mathbf{\gamma}}
\newcommand{\vdelta}{\mathbf{\delta}}
\newcommand{\vepsilon}{\mathbf{\epsilon}}
\newcommand{\vzeta}{\mathbf{\zeta}}
\newcommand{\veta}{\mathbf{\eta}}
\newcommand{\vtheta}{\mathbf{\theta}}
\newcommand{\viota}{\mathbf{\iota}}
\newcommand{\vkappa}{\mathbf{\kappa}}
\newcommand{\vlambda}{\mathbf{\lambda}}
\newcommand{\vmu}{\mathbf{\mu}}
\newcommand{\vnu}{\mathbf{\nu}}
\newcommand{\vxi}{\mathbf{\xi}}
\newcommand{\vpi}{\mathbf{\pi}}
\newcommand{\vrho}{\mathbf{\rho}}
\newcommand{\vsigma}{\mathbf{\sigma}}
\newcommand{\vtau}{\mathbf{\tau}}
\newcommand{\vupsilon}{\mathbf{\upsilon}}
\newcommand{\vphi}{\mathbf{\phi}}
\newcommand{\vchi}{\mathbf{\chi}}
\newcommand{\vpsi}{\mathbf{\psi}}
\newcommand{\vomega}{\mathbf{\omega}}
\newcommand{\vGamma}{\mathbf{\Gamma}}
\newcommand{\vDelta}{\mathbf{\Delta}}
\newcommand{\vTheta}{\mathbf{\Theta}}
\newcommand{\vLambda}{\mathbf{\Lambda}}
\newcommand{\vXi}{\mathbf{\Xi}}
\newcommand{\vPi}{\mathbf{\Pi}}
\newcommand{\vSigma}{\mathbf{\Sigma}}
\newcommand{\vUpsilon}{\mathbf{\Upsilon}}
\newcommand{\vPhi}{\mathbf{\Phi}}
\newcommand{\vPsi}{\mathbf{\Psi}}
\newcommand{\vOmega}{\mathbf{\Omega}}
\newcommand{\cA}{\mathcal{A}}
\newcommand{\cB}{\mathcal{B}}
\newcommand{\cC}{\mathcal{C}}
\newcommand{\cD}{\mathcal{D}}
\newcommand{\cE}{\mathcal{E}}
\newcommand{\cF}{\mathcal{F}}
\newcommand{\cG}{\mathcal{G}}
\newcommand{\cH}{\mathcal{H}}
\newcommand{\cI}{\mathcal{I}}
\newcommand{\cJ}{\mathcal{J}}
\newcommand{\cK}{\mathcal{K}}
\newcommand{\cL}{\mathcal{L}}
\newcommand{\cM}{\mathcal{M}}
\newcommand{\cN}{\mathcal{N}}
\newcommand{\cO}{\mathcal{O}}
\newcommand{\cP}{\mathcal{P}}
\newcommand{\cQ}{\mathcal{Q}}
\newcommand{\cR}{\mathcal{R}}
\newcommand{\cS}{\mathcal{S}}
\newcommand{\cT}{\mathcal{T}}
\newcommand{\cU}{\mathcal{U}}
\newcommand{\cV}{\mathcal{V}}
\newcommand{\cW}{\mathcal{W}}
\newcommand{\cX}{\mathcal{X}}
\newcommand{\cY}{\mathcal{Y}}
\newcommand{\cZ}{\mathcal{Z}}
\newcommand{\bA}{\mathbb{A}}
\newcommand{\bB}{\mathbb{B}}
\newcommand{\bC}{\mathbb{C}}
\newcommand{\bD}{\mathbb{D}}
\newcommand{\bE}{\mathbb{E}}
\newcommand{\bF}{\mathbb{F}}
\newcommand{\bG}{\mathbb{G}}
\newcommand{\bH}{\mathbb{H}}
\newcommand{\bI}{\mathbb{I}}
\newcommand{\bJ}{\mathbb{J}}
\newcommand{\bK}{\mathbb{K}}
\newcommand{\bL}{\mathbb{L}}
\newcommand{\bM}{\mathbb{M}}
\newcommand{\bN}{\mathbb{N}}
\newcommand{\bO}{\mathbb{O}}
\newcommand{\bP}{\mathbb{P}}
\newcommand{\bQ}{\mathbb{Q}}
\newcommand{\bR}{\mathbb{R}}
\newcommand{\bS}{\mathbb{S}}
\newcommand{\bT}{\mathbb{T}}
\newcommand{\bU}{\mathbb{U}}
\newcommand{\bV}{\mathbb{V}}
\newcommand{\bW}{\mathbb{W}}
\newcommand{\bX}{\mathbb{X}}
\newcommand{\bY}{\mathbb{Y}}
\newcommand{\bZ}{\mathbb{Z}}
\newcommand{\vzero}{\mathbf{0}}
\newcommand{\vone}{\mathbf{1}}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\abs}{abs}
\DeclareMathOperator*{\sign}{sign}
\DeclareMathOperator*{\sort}{sort}
\DeclareMathOperator*{\argsort}{argsort}
\DeclareMathOperator*{\Cat}{Cat}
\DeclareMathOperator*{\Bern}{Bern}
\DeclareMathOperator*{\Unif}{Unif}
\newcommand{\cmark}{ {\color{dartmouthgreen} \ding{51}} }%
\newcommand{\xmark}{ {\color{red} \ding{55}} }%
\raggedbottom
\usepackage{xcolor,colortbl}
\newcommand{\mc}[2]{\multicolumn{#1}{c}{#2}}
\definecolor{Gray}{gray}{0.5}
\definecolor{LightCyan}{rgb}{0.88,1,1}
\begin{abstract}
Inference-Time Scaling (ITS) improves language models by allocating more computation at generation time. Particle Filtering (PF) has emerged as a strong ITS method for complex mathematical reasoning tasks, but it is vulnerable when guided by process reward models, which often assign overconfident scores early in the reasoning process. This causes PF to suffer from premature exploitation: it myopically commits to locally promising trajectories, prunes potentially correct hypotheses, and converges to suboptimal solutions. This failure mode, known as particle impoverishment, is especially severe under constrained computational budgets. To address this, we analyze the problem and identify two root causes: a lack of diversity in the particle set due to overconfident resampling and consequent inability to assess the potential of a reasoning path. We introduce Entropic Particle Filtering (ePF), an algorithm that integrates two new techniques to solve these issues. The first technique, Entropic Annealing (EA), directly mitigates particle impoverishment by monitoring search diversity via entropy; when diversity drops,
it intervenes by dynamically annealing the resampling distribution to preserve exploration. The second, an enhancement called Look-ahead Modulation (LaM), adds a predictive guide to evaluate a state's potential based on its successors. On several challenging math benchmarks, ePF significantly outperforms strong baselines and achieves up to a 50\% relative improvement in task reward.
Together, these methods improve PF's resilience by balancing the exploration of diverse solution spaces with the exploitation of high-reward regions, ultimately leading to higher-quality solutions.
\end{abstract}
\section{Introduction}
\label{sec:intro}
\begin{wrapfigure}{r}{0.4\textwidth}
\vspace{-4em}
\centering
\includegraphics[width=.9\linewidth]{img/aime_2025_1.7b_median_bar_predictive.pdf}
\label{subfig:aime-bar-intro}
\caption{Task reward comparison on AIME-2025 using Qwen3-1.7B. Our Entropic Particle Filtering (\texttt{ePF}) and its Look-ahead variant (\texttt{ePF} w/ \texttt{LaM}) significantly improve performance over standard Particle Filtering (PF) across all particle budgets. This demonstrates that mitigating premature exploitation leads to significant performance gains.}
\label{fig:reward-predictive}
\vspace{-1em}
\end{wrapfigure}
Inference-Time Scaling (ITS) is a powerful paradigm for improving language model performance by allocating additional computation at generation time. Rather than decoding a single trajectory, ITS reframes generation as a guided search: multiple candidate solutions are explored in parallel, scored, and iteratively refined~\citep{wei2022chain,brown2024large,snell2024scaling,beeching2024scalingtesttimecompute}. This approach has proven especially effective on reasoning tasks, where the search space is vast and correct answers are sparse.
Among ITS methods, \emph{Particle Filtering (PF)} has emerged as a principled and efficient approach~\citep{puri2025probabilistic,feng2024step}. PF maintains a set of candidate trajectories (\emph{particles}), propagates them through the model, weights them using a process reward model (PRM), and resamples candidates with probabilities proportional to their weights. This propagate-weight-resample cycle adaptively focuses computation on promising regions of the search space while preserving some hypothesis diversity, often outperforming well-established methods such as beam search or Best-of-$N$ sampling.
\begin{figure}[ht]%
\centering
\begin{subfigure}[b]{0.32\linewidth}
\centering
\includegraphics[width=\linewidth]{img/calibration/density_plot_prm_gt.pdf}
\caption{Density of the PRM reward $r_i$
assigned to partial trajectories of 1000 tokens in length and the probability $p_i$ of the full response being correct.
$p_i$ are estimated via MC using 20 rollouts for a 128 sample subset of MATH500.}
\label{fig:calibration}
\end{subfigure}
\hfill
\begin{subfigure}[b]{0.32\linewidth}
\centering
\includegraphics[width=\linewidth]{img/variance/variance_aime24_intro.pdf}
\caption{$\mathbb{V}[w_t]$ on AIME 2024.
PF assigns overconfident scores early in the sampling process, generating high variance in the resampling distribution and poor state estimation.}
\label{fig:var-main}
\end{subfigure}
\hfill
\begin{subfigure}[b]{0.32\linewidth}
\centering
\includegraphics[width=\linewidth]{img/pass/ess_aime25_ess_pass_comparison.pdf}
\caption{Expected pass@1 on AIME 2025 as a function of the normalized effective sample size. We aggregate pass@1 and $ESS_{n}$ over trajectories and particle budgets.}
\label{fig:pass-ess}
\end{subfigure}
\caption{PRM overconfidence causes particle impoverishment and harms performance.
\emph{(a)} Uncalibrated PRMs contribute to low diversity by assigning overly optimistic scores to partial solutions, causing Particle Filtering to converge prematurely (Fig.~\ref{fig:reward-entropy}).
\emph{(b)} The variance of the resampling distribution increases with less particle diversity.
\emph{(c)} Task success is strongly correlated with high ESS.}
\label{fig:overconfidence}
\end{figure}
Despite these advantages, PF's effectiveness is often undermined by a critical vulnerability: when guided by imperfect PRMs, it is prone to premature exploitation.
This issue is exacerbated by PF's inherent \emph{myopia} - its inability to assess the long-term potential of a reasoning path beyond the immediate reward.
The early steps of a solution trajectory often carry little information about eventual correctness, yet uncalibrated PRMs frequently assign overconfident scores even at these stages~\citep{park2025know,lightman2023let}. Ideally, rewards at these early stages should be conservative and relatively similar across trajectories, as the eventual solution quality cannot be confidently determined from initial steps alone, yielding a flatter resampling distribution that maintains diversity.
When PF resamples on the basis of these noisy, overconfident signals, it prematurely concentrates probability mass on a small set of trajectories, often trapping the search in a locally optimal, but globally incorrect, solution.
To illustrate the overconfidence phenomena in pretrained PRMs, we plot the histogram of rewards from PRM and estimated ground truth via Monte Carlo in Fig.~\ref{fig:calibration}. It can be seen that PRMs consistently assign higher rewards to partial solutions than is warranted by the final probability of correctness.
A direct result of the overconfident rewards is high variance in the resampling distribution, which we show in Fig~\ref{fig:var-main}
High variance diminishes particle diversity and results in a greedy-like search. This phenomenon, known as \emph{particle impoverishment}, prematurely prunes viable hypotheses before they can reveal their value, causing the search to collapse into suboptimal solutions.
As a final remark, we confirm that ESS, a measure of particle diversity that is inversely related to variance, is indeed correlated with the final performance of the algorithm
(Fig.~\ref{fig:pass-ess}).
performance tracks closely with the effective sample size (ESS) - a direct measure of particle activation and variety - and the entropy of the resampling distribution.
Thus, a gap remains for a search method that is inherently robust against reward miscalibration and overconfidence. Our central hypothesis is that by dynamically preserving search diversity and incorporating forward-looking guidance, we can create a more resilient particle filtering algorithm.
To this end, we introduce \emph{Entropic Particle Filtering (ePF)}, a robust extension of Particle Filtering designed to maintain exploration and prevent premature convergence.
ePF integrates two complementary mechanisms: \emph{(i) Entropic Annealing (EA)}, which dynamically adjusts resampling temperature based on particle diversity to avoid collapse, and \emph{(ii) Look-ahead Modulation (LaM)}, which uses a one-step look-ahead to bias sampling toward trajectories with high long-term potential.
\paragraph{Contribution} Our contributions are:
\begin{itemize}
\item We introduce Entropic Particle Filtering (ePF), which uses Entropic Annealing (EA) to dynamically modulate the resampling step based on particle diversity to prevent premature collapse.
\item We propose Look-ahead Modulation (LaM), a non-myopic guidance mechanism that re-weights particles based on the predicted quality of their successors.
\item We demonstrate a strong correlation between premature exploitation and poor performance, confirming that robust exploration is key to finding high-quality solutions.
\item We show that ePF significantly outperforms strong baselines across several mathematical reasoning benchmarks, especially when operating under limited particle budgets.
\end{itemize}
By improving the exploration-exploitation balance, ePF makes the search more resilient to PRM miscalibration in long-horizon mathematical problems.
\section{Background}
\label{sec:background}
\paragraph{Sequential Importance Sampling}
For sequential models, where the distributions of interest evolve over time, Importance Sampling \citep[IS;][]{kloek1978bayesian,robert1999monte} can be extended into a recursive framework known as Sequential Importance Sampling ~\citep[SIS;][]{doucet2001introduction}.
This is the foundation of particle filters. The goal is to approximate the posterior distribution over a sequence of states $\vz_{1:T}$ given a sequence of observations $\vo_{1:T}$.
In the context of LLM inference, each state $\vz_t$ is an intermediate sampling step, and the observation $\vo_t$ is the scalar score provided by a PRM.
In a standard state-space model with the Markov property, the un-normalized posterior can be factorized recursively.
In particular given a posterior of the form $p(\vz_{1:T} | \vo_{1:T})$, leveraging a proposal $q(\vz_{1:T})$ and using Bayes rule, we can write the importance weights $w_t$ for step $t$ as:
\begin{equation*}
\tilde w_t = \dfrac{\tilde p(\vz_{1:t} | \vo_{1:t})}{q(\vz_{1:t})}
=
\dfrac{p(\vo_t | \vz_t) p(\vo_{1:t-1} | \vz_{1:t-1})~p(\vz_t | \vz_{t-1}) p(\vz_{1:t-1})}{q(\vz_t | \vz_{t-1}) q(\vz_{1:t-1})}
=
\tilde w_{t-1} \dfrac{p(\vo_t | \vz_t) p(\vz_t | \vz_{t-1})}{q(\vz_t | \vz_{t-1})},
\end{equation*}
where $\tilde p(\vz_{1:t} | \vo_{1:t})$ represents the un-normalized posterior.
A powerful computational simplification arises when we choose the proposal to be the model's dynamics, i.e., $q(\vz_t | \vz_{t-1}) = p(\vz_t | \vz_{t-1})$. This choice gives rise to the (forward) Bootstrap Particle Filter~\citep{gordon1993novel}, and reduces the weight update to a simple multiplication by the observation likelihood:
\begin{equation}
\tilde w_t = \tilde w_{t-1}~p(\vo_t \mid \vz_t).
\label{eq:importance-weights}
\end{equation}
This elegant result shows that the un-normalized importance weight at step $t$ is simply the previous weight at $t-1$ modulated by the likelihood of the current observation, efficiently propagating information through the sequence. More details in Appx~\ref{sec:related_work}.
\paragraph{Particle-based Monte Carlo}
Particle Filters (PF) are Sequential Monte Carlo (SMC) methods that use Sequential Importance Resampling ~\citep[SIR;][]{liu2001theoretical} to approximate posterior distributions~\citep{doucet2001sequential}. They work by calculating sequential importance weights (Eq.~\ref{eq:importance-weights}) at each step $t$, which only requires an un-normalized posterior proportional to the likelihood $p(\vo_{t} | \vz_t)$ and the prior dynamics $p(\vz_t | \vz_{t-1})$.
The algorithm iteratively applies three steps to a set of $N$ particles $\vz^{i}$: \emph{(i)} Propagate each particle using the model's dynamics $p(\vz_t | \vz_{t-1})$; \emph{(ii)} Weight each resulting partial trajectory $\vz^{i}_{t}$ using a reward function $g(\vz^{i}_{t})$ that outputs a score $r^{i}_t$; and \emph{(iii)} Resample the particles with replacement from a normalized distribution, typically following a softmax distribution: $w^{i} \propto \exp{(r^{i}_t)}$.
\section{Method}
\label{sec:method}
\begin{figure}[t]
\centering
\centering
\includegraphics[width=\linewidth]{img/intro/pipeline_intro.pdf}
\caption{The Entropic Particle Filtering (ePF) pipeline and its core Mechanisms.
Particle Filtering (left), Entropic Particle Filtering (center), and Entropic Particle Filtering w/ LaM (right).
Each circle represents a particle at step $t$ and the size is proportional to the reward provided by the PRM. EA and LaM help the PF algorithm to mitigate early exploitation and myopic updates, greatly improving the diversity of the particles at step $t+1$. Standard PF often suffers from \emph{particle impoverishment}, where diversity is lost after resampling. Our ePF pipeline incorporates the EA step to maintain particle diversity.
ePF with LaM, further adds the LaM step to guide the search more effectively. Pipeline details in Appx~\ref{appx:pipeline}.}
\label{fig:particle-filtering-pipeline}
\end{figure}
Our method builds on Particle Filtering for posterior estimation in LLMs~\citep{puri2025probabilistic,feng2024step}. Our goal is to address its tendency toward premature exploitation and overconfidence, particularly when guided by a PRM~\citep{park2025know}.
This issue is especially problematic for learning-free ITS methods that rely on frozen models and external feedback, as offline calibration is often infeasible or expensive.
We model sequential mathematical reasoning as a forward generative process over a sequence of latent states $\vz_{1:T}$ conditioned on an input task $\vc$.
At each step $t$, a language model parameterized by $\theta$ defines the transition distribution:
\begin{equation*}
\vz_{t} \sim p_\theta(\vz_{t} | \vz_{1:t-1}, \vc),
\end{equation*}
producing a trajectory of intermediate reasoning steps. A process reward model (PRM) provides feedback by assigning a scalar reward:
\begin{equation*}
r(\vz_{1:t}, \vc) \approx p(\vo_t = 1 | \vz_{1:t}, \vc),
\end{equation*}
which we treat as an un-normalized log-likelihood of correctness for state $\vz_t$. Our goal is to approximate the posterior over trajectories:
\begin{equation}
p(\vz_{1:T} | \vo_{1:T}, \vc) \propto p(\vz_{1} | \vc) \prod_{t=2}^{T} p(\vo_t | \vz_{1:t}, \vc)~p_{\theta}(\vz_t | \vz_{1:t-1}, \vc) = p(\vo_{1:T}, \vz_{1:T} | \vc),
\label{eq:posterior-state}
\end{equation}
where we set $p(\vo_1 | \vz_1, \vc)=1$.
We represent this posterior with $N$ weighted particles $\{(\vz^{i}_t, w^{i}_t)\}_{i=1}^{N}$. Standard particle filtering alternates between:
\emph{(i)} forward propagation using $p_\theta$,
\emph{(ii)} weighting by $r(\vz^i_{1:t}, \vc)$, and
\emph{(iii)} resampling particles proportionally to their normalized weights, $w_t(r_t)$.
The final decoded output is denoted by $\vx$, which is extracted from the final state sequence $\vz_{1:T}$. We now introduce the two main building blocks of our method: \emph{Entropic Annealing} and \emph{Look-ahead Modulation}.
\subsection{Why Particle Filtering Collapses}
Ideally, resampling concentrates computation on promising particles while retaining diversity.
In practice, however, PF often \emph{collapses} early: a few particles acquire nearly all the weight mass, leading to \emph{particle impoverishment} (Fig.~\ref{fig:var-main}).
This effect for particle diversity can be quantified using the normalized entropy $H_n(t)$ and the normalized effective sampling size $ESS_n(t)$:
$$
H_{n}(t) = -\frac{\sum_{i=1}^N w_t^{i} \log w_t^{i}}{\log N},
\quad
ESS_n(t) = \frac{\sum_{i=1}^N (w_t^{i})^{-2}}{N}.
$$
Low $H_n(t)$ or $ESS_n(t)$ indicates that only a small subset of particles are being explored.
In multi-step reasoning, this collapse often occurs prematurely because PRM scores at early steps are overconfident despite being weakly informative (Figure~\ref{fig:calibration}, Figure~\ref{fig:reward-entropy}).
As a result, PF commits to trajectories prematurely, under-exploring paths that may lead to correct, high-reward solutions and thus reducing its overall success probability. This behavior can be quantified by the variance of the resampling weights~\citep{liu1996metropolized,kong1992note}, $\mathbb{V}[w_t]$, which becomes excessively high early in the sampling process (Fig.~\ref{fig:var-main}). High variance indicates that the weight distribution is concentrated on only a few particles, leading to a low $ESS(t)$, a low-entropy state, and ultimately a poor approximation of the posterior in Eq.~\ref{eq:posterior-state}. Details and derivation in Appx~\ref{appx:variance}.
\subsection{Entropic Particle Filtering (ePF)}
\begin{wrapfigure}{r}{0.3\textwidth}
\vspace{-20pt}
\centering
\includegraphics[width=.9\linewidth]{img/entropic_annealing.pdf}
\caption{Adapting the resampling distribution temperature using Entropic Annealing.
}
\label{fig:ea}
\vspace{-25pt}
\end{wrapfigure}
To mitigate premature collapse, we introduce \emph{Entropic Annealing (EA)}, which adaptively modulates the resampling temperature $\beta_t^{-1}$ based on particle diversity:
\begin{equation*}
\beta^{-1}_t = \dfrac{N}{ESS(t)} (1 - t/ T),
\quad
w^{i}_t (r_t, \beta_t) = \frac{\exp(r^{i}_t \cdot \beta_t)}{\sum^{N}_{j=1} \exp(r^{j}_t \cdot \beta_t)},
\end{equation*}
where $ESS(t) = \sum_{i=1}^N (w_t^{i})^{-2}$.
When particle diversity is low, the temperature increases, producing a flatter resampling distribution $w^i_t$ that maintains exploration (Fig.~\ref{fig:ea}).
As $t \to T$, $\beta_t$ gradually anneals to 1, shifting the algorithm from exploration toward exploitation.
By dynamically increasing the temperature when particle diversity is low, EA effectively reduces the variance of the resampling weights, preventing the particle set from collapsing prematurely.
We also explored additional schedules: linear and entropy-based and conclude that ESS-based schedule provides a more direct response to particle impoverishment (Appx~\ref{appx:temperature-schedule-ablation}).
We employ \emph{systematic resampling}~\citep{kitagawa1996monte}, a low-variance sampling technique that uses a single stratified draw to mitigate early estimation errors common in multinomial resampling. This choice better preserves the distribution's structure and reduces random fluctuations, ensuring a more faithful representation for robust exploration.
Details in Appx~\ref{appx:systematic-multinomial}.
\begin{wrapfigure}{r}{0.4\textwidth}
\vspace{-15pt}
\centering
\includegraphics[width=\linewidth]{img/look_ahead_modulation.pdf}
\caption{Non-myopic resampling distribution update using Look-ahead Modulation. We use the dynamics model at step $t$ to predict an intermediate next state $s$.}
\label{fig:lam}
\vspace{-10pt}
\end{wrapfigure}
\subsection{Look-ahead Modulation (LaM)}
While EA preserves exploration, PF remains fundamentally myopic: resampling decisions depend only on current rewards.
To address this, we introduce \emph{Look-ahead Modulation (LaM)}, which adjusts resampling weights using predicted successor quality before resampling.
LaM is a novel, computationally efficient adaptation of the principles of the APF framework~\citep{pitt1999filtering} for language modeling inference.
For each particle $i$ at step $t$, we sample a one-step look-ahead $\vz_s^{i} \sim p_\theta(\vz_s^{i} | \vz_t^{i}, \vc)$, score it with the PRM to obtain $\tilde r_s$, and compute modulated weights:
$$
a^{i}_t = w^{i}_t \cdot \tilde r^{i}_s, \quad
w^{i}_t(a_t) = \frac{a^{i}_t}{\sum_{j=1}^{N} a^{j}_t}.
$$
These forward-looking weights bias resampling toward particles that are likely to produce high-reward successors, making the update less myopic. Crucially, look-ahead states $\vz_s$ are discarded after modulation, so the forward model remains faithful to the propagation dynamics. While LaM introduces a computational overhead of one extra forward pass per particle before resampling, the cost is often offset by significant performance gains, achieving comparable accuracy with smaller particle budgets (see Fig.~\ref{fig:aime24-predictive-appx}).
Algorithm~\ref{alg:epf} and~\ref{alg:aepf} summarizes the combined procedure. Together, EA and LaM transform PF into a guided search that \emph{(i)} maintains exploration until sufficient information has been gathered, and \emph{(ii)} directs computation toward promising trajectories, achieving a better exploration–exploitation balance.
We provide an analysis of the computation overhead for LaM in Appx~\ref{appx:complexity}.
\section{Experiments}
\label{sec:experiments}
\paragraph{Benchmarks and Models}
We conduct our evaluation on six math benchmarks of increasing difficulty: GSM8K~\citep{lightman2023let}, MATH500~\citep{hendrycks2024measuring}, DEEPMATH~\citep{he2025deepmath}, OMNIMATH~\citep{gao2024omni}, and the challenging AIME-2024 and AIME-2025 datasets. The problem of early exploitation becomes more relevant and detrimental for hard benchmarks (AIME), given that the sampling trajectories tend to have an order of magnitude more steps than for easier benchmarks (MATH500).
The four primary models used are generalist models from the Qwen family~\citep{yang2024qwen2,yang2025qwen3}, including Qwen2.5-1.5B-Instruct, Qwen2.5-7B-Instruct, Qwen3-0.6B and Qwen3-1.7B. All generation processes are guided by the same Process Reward Model (PRM), Qwen2.5-Math-PRM-7B~\citep{puri2025probabilistic,park2025know}.
\textbf{Baselines and Setup}
We compare our algorithms against a strong set of baselines, including Self-Consistency with majority voting, weighted and unweighted Best-of-N, Beam Search, and standard Particle Filtering. For all guided methods, the generation is managed by a vLLM sampler with predefined budgets of $N \in \{2, 4, 8, 16, 32\}$, a maximum of 300 generation steps, and a limit of 512 tokens per step. We use ePF for the first 50\% of the steps in the sampling trajectory; we activate ePF (w/o and w/ LaM) with a threshold of $ESS_n(t) \leq 0.5$.
\textbf{Evaluation and Verification}
We assess performance primarily using pass@1 (accuracy) as our main evaluation metric, which allows direct comparison with state-of-the-art methods in iterative test-time search. For output verification and parsing, we use the \texttt{math\_verify} library\footnote{\texttt{https://github.com/huggingface/Math-Verify}}, a deterministic and restrictive verifier chosen to provide a conservative estimate of performance. The final solution from each run is selected using the same PRM that provides intermediate guidance.
\subsection{General Results on Mathematical Benchmarks}
\begin{table}[ht]
\setlength{\tabcolsep}{3pt}
\centering
\caption{
Pass@1 performance comparison of inference-time scaling algorithms on mathematical reasoning benchmarks with increasing complexity. Our proposed method, \texttt{ePF}, demonstrates superior performance over established baselines across multiple datasets of increasing difficulty for Qwen2.5-1.5B-Instruct and Qwen-2.5-7B-Instruct models. Best results are in \textbf{bold}.
We use random subsets of 128 samples for each dataset. ORM: Output Reward Model; PRM: Process Reward Model; MV: Majority Voting.}
\resizebox{\textwidth}{!}{
\begin{tabular}{lcc | cccc | cccc}
\toprule
& & & \multicolumn{4}{c|}{Qwen2.5-1.5B-Instruct} & \multicolumn{4}{c}{Qwen2.5-7B-Instruct} \\
Algorithm & Selection & Scoring & GSM8K & MATH500 & DEEPMATH & OMNIMATH
& GSM8K & MATH500 & DEEPMATH & OMNIMATH \\
\midrule
Base Sampling & - & - & 67.19 & 45.31 & 10.15 & 5.46 & 93.40 & 60.93 & 23.43 & 8.59 \\
Self-Consistency & MV & - & 82.03 & 53.90 & 13.28 & 7.03 & 94.80 & 65.62 & 30.46 & 9.37 \\
Best-of-N & Argmax & ORM & 92.97 & 57.81 & 20.31 & \textbf{10.15} & 96.00 & 67.96 & 32.03 & 9.37\\
Beam-Search & Argmax & PRM & 91.40 & 62.50 & 21.09 & 9.37 & \textbf{96.20} & 66.40 & 32.03 & \textbf{10.93} \\
PF & Argmax & PRM & \textbf{93.75} & 60.15 & 22.65 & 8.59 & \textbf{96.20} & 70.31 & 34.37 & 10.15 \\
\texttt{ePF} (ours) & Argmax & PRM & \textbf{93.75} & \textbf{66.42} & \textbf{25.00} & \textbf{10.15} & 95.80 & \textbf{71.09} & \textbf{35.93} & \textbf{10.93}\\
\bottomrule
\end{tabular}
}
\label{tab:benchmark}
\end{table}
We start by evaluating \texttt{ePF} against leading inference-time scaling algorithms on a suite of four math benchmarks with increasing difficulty (GSM8K, MATH500, DEEPMATH, and OMNIMATH). Our goal was to determine if preventing premature exploitation translates to better reasoning performance.
As shown in Table~\ref{tab:benchmark}, \texttt{ePF} consistently performs well, achieving the highest accuracy across most configurations for both Qwen2.5-1.5B-Instruct and Qwen2.5-7B-Instruct models. This preliminary experiment validates our core hypothesis: by intelligently mitigating early exploitation, \texttt{ePF} consistently boosts a language model's ability to solve mathematical problems.
Fig.~\ref{fig:gsm8k-math500-deepmath} details how performance scales with the particle budget $N \in \{2,4,8,16,32\}$. Our goal was to assess if \texttt{ePF}'s advantage over standard PF grows with task complexity. While \texttt{ePF} consistently matches or outperforms PF, its superiority becomes most apparent on more challenging problems. Specifically, the performance gap widens significantly as dataset complexity increases (MATH500 $\rightarrow$ DEEPMATH) and model size grows (1.5B $\rightarrow$ 7B). \texttt{ePF}'s exploration becomes more useful for more challenging problems, confirming its value in navigating vast and complex solution spaces.
\subsection{Exploration for Hard Problems and Small Budgets}
\begin{table}[ht]
\caption{Entropic Particle Filtering (\texttt{ePF}) outperforms baselines on AIME math benchmarks. The table shows aggregate pass@1 scores (\%) across budgets ($N \in \{2, 4, 8, 16, 32\}$), reweighted to favor large (proportional weighting \texttt{w}), uniform (\texttt{u}), or small (inverse weighting \texttt{iw}) budgets. Averaged over 5 runs; higher is better.}
\centering
\setlength{\tabcolsep}{4pt}
\resizebox{\textwidth}{!}{
\begin{tabular}{l | ccc | ccc | ccc | ccc }
\toprule
& \multicolumn{6}{c}{AIME 2024} & \multicolumn{6}{c}{AIME 2025} \\
& \multicolumn{3}{c}{Qwen2.5-1.5B-In} & \multicolumn{3}{c}{Qwen2.5-7B-In} & \multicolumn{3}{c}{Qwen2.5-1.5B-In} & \multicolumn{3}{c}{Qwen2.5-7B-In} \\
& \texttt{w} & \texttt{u} & \texttt{iw} & \texttt{w} & \texttt{u} & \texttt{iw} & \texttt{w} & \texttt{u} & \texttt{iw} & \texttt{w} & \texttt{u} & \texttt{iw} \\
\midrule
Base Sampling~\citep{yang2025qwen3} & - & 3.33 & - & - & 10.00 & - & - & 3.33 & - & - & 6.66 & - \\
Best-of-N & 9.90 & 7.20 & 4.00 & 23.09 & 20.20 & 17.48 & 5.13 & 3.60 & 2.52 & 17.41 & 15.80 & 14.90 \\
Beam-Search & 10.29 & 8.40 & 5.55 & 17.93 & 14.00 & 11.26 & 9.45 & \textbf{7.40} & \textbf{4.32} & 14.19 & 16.20 & 16.81 \\
PF~\citep{puri2025probabilistic} & 11.16 & 9.00 & \textbf{6.99} & \textbf{26.06} & \textbf{21.60} & \textbf{18.13} & 7.32 & 4.50 & 2.87 & 21.61 & 19.80 & 17.61 \\
\texttt{ePF} (ours) & \textbf{17.06} & \textbf{11.20} & 5.55 & \textbf{26.23} & \textbf{21.00} & 16.06 & \textbf{10.82} & \textbf{7.28} & 3.42 & \textbf{28.83} & \textbf{25.10} & \textbf{21.96} \\
\bottomrule
\end{tabular}
}
\label{tab:benchmark-aime-2.5}
\end{table}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_aime-2024_1.5B_20250810.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-1.5B)}
\label{fig:aime24-1.5b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/eval_benchmark_aime-2024_7B_20250912_new.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-7B)}
\label{fig:aime24-7b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_aime-2025_1.5B_20250802.pdf}
\caption{\scriptsize AIME 2025 (Qwen2.5-1.5B)}
\label{fig:aime25-1.5b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_aime-2025_7B_20250801.pdf}
\caption{\scriptsize AIME 2025 (Qwen2.5-7B)}
\label{fig:aime25-7b}
\end{subfigure}
\caption{pass@1 performance as a function of inference budget ($N$) for Qwen2.5-1.5B-Instruct and Qwen-2.5-7B-Instruct models. The curves illustrate that \texttt{ePF} (orange) not only reaches a higher peak performance but also shows a steeper initial climb, indicating superior efficiency compared to PF (blue) and BoN (green), especially at smaller computation budgets. See Fig.~\ref{fig:aime24-aime25-max-appx} for more results and longer sequence length.}
\label{fig:aime24-aime25}
\end{figure}
We now shift our focus on the AIME benchmarks.
Our goal is to understand how different particle budgets impact exploration for long multi-step mathematical reasoning trajectories.
In Table~\ref{tab:benchmark-aime-2.5} and Fig.~\ref{fig:aime24-aime25}, \texttt{ePF} consistently outperforms established baselines and standard PF across most budgets. Its effectiveness is especially clear on the AIME 2025 benchmark with the Qwen2.5-7B model, where it achieves up to a 28.8\% pass@1 rate across budgets. Crucially, its top performance under the inversely weighted (\texttt{iw}) metric highlights its superior sample efficiency at small budgets. \texttt{ePF}'s enhanced exploration strategy provides a significant advantage to solve difficult mathematical problems.
\subsection{Inference-Time Scaling with Small Reasoning Models}
\begin{wraptable}{r}{0.5\textwidth}
\setlength{\tabcolsep}{4pt}
\centering
\caption{Reasoning and ITS on AIME 2025. \texttt{ePF} effectively leverages an increased sequence length budget (4k $\rightarrow$ 12k) to dramatically boost the Qwen3-1.7B's performance (without thinking), while baselines methods and PF show limited gains. This demonstrates that an efficient search algorithm can elevate a non-thinking model to match specialized reasoning models, highlighting the power of ITS. Higher scores are better.}
\resizebox{\linewidth}{!}{
\begin{tabular}{l c c c c}
\toprule
Model & Budget/Seq & Algorithm & Thinking & AIME 2025 \\
\midrule
Qwen3-1.7B & 32k & Reasoning & w/ & 35.5 \\
R1-Qwen-1.5B & 32k & Reasoning & w/ & 23.1 \\
Nemotron-R-1.5B & 32k & Reasoning & w/ & 33.6 \\
Qwen3-1.7B & 32k & Reasoning + BoN & w/ & 41.1 \\
e3-1.7B & 32k & Reasoning + Tuning & w/ & \textbf{43.8} \\
\midrule
Qwen3-1.7B & 4k & CoT & w/o & 6.66 \\
Qwen3-1.7B & 4k & BoN & w/o & 13.3 \\
Qwen3-1.7B & 4k & PF & w/o & \textbf{20.0} \\
Qwen3-1.7B (ours) & 4k & \texttt{ePF} & w/o & 18.9 \\
\midrule
Qwen3-1.7B & 12k & CoT & w/o & 16.6 \\
Qwen3-1.7B & 12k & BoN & w/o & 28.8 \\
Qwen3-1.7B & 12k & PF & w/o & 26.6 \\
Qwen3-1.7B (ours) & 12k & \texttt{ePF} & w/o & \textbf{38.9} \\
\bottomrule
\end{tabular}
}
\label{tab:reasoning}
\vspace{-3em}
\end{wraptable}
The rise of models trained on reasoning traces~\citep{jaech2024openai,guo2025deepseek}, which learn to perform internalized search and backtracking, raises a key question: is parallel inference-time scaling still necessary? While training for reasoning is powerful, our results in Table~\ref{tab:reasoning} show that ITS is a highly effective complementary approach.
Applying the ePF algorithm, a Qwen3-1.7B model without thinking can elevate its performance to rival or even surpass specialized reasoning models. For instance, with a 12k sequence budget, ePF boosts the standard Qwen3-1.7B model to a score of 38.9, outperforming dedicated models like R1-distilled-Qwen-1.5B and Nemotron-R-1.5B. This highlights ePF's strength: it effectively utilizes a parallel budget to explore longer solution paths, unlike standard PF which converges prematurely. This confirms that parallel search remains a crucial technique for maximizing a model's reasoning potential.
\begin{table}[ht]
\setlength{\tabcolsep}{4pt}
\caption{Look-ahead Modulation boosts performance on the AIME math benchmarks. We compare entropic particle filter (\texttt{ePF}) with and without Look-ahead Modulation (\texttt{ePF w/ LaM}) against baselines using Qwen3 models (no thinking).
The table shows aggregate pass@1 scores (\%) across computational budgets ($N \in \{2, 4, 8, 16, 32\}$), reweighted to favor large (proportional weighting \texttt{w}), uniform (\texttt{u}), or small (inverse weighting \texttt{iw}) budgets. Averaged over 5 runs; higher is better.
}
\centering
\resizebox{\textwidth}{!}{
\begin{tabular}{l | ccc | ccc | ccc | ccc }
\toprule
& \multicolumn{6}{c}{AIME 2024} & \multicolumn{6}{c}{AIME 2025} \\
& \multicolumn{3}{c}{Qwen3-0.6B} & \multicolumn{3}{c}{Qwen3-1.7B} & \multicolumn{3}{c}{Qwen3-0.6B} & \multicolumn{3}{c}{Qwen3-1.7B} \\
& \texttt{w} & \texttt{u} & \texttt{iw} & \texttt{w} & \texttt{u} & \texttt{iw} & \texttt{w} & \texttt{u} & \texttt{iw} & \texttt{w} & \texttt{u} & \texttt{iw} \\
\midrule
Base Sampling~\citep{yang2025qwen3} & - & 3.40 & - & - & 13.40 & - & - & 2.60 & - & - & 9.80 & - \\
Best-of-N & 7.90 & 6.00 & 4.32 & 20.51 & 18.80 & 17.12 & 17.83 & 15.80 & 11.54 & 19.22 & 18.40 & 18.45 \\
PF~\citep{puri2025probabilistic} & 14.46 & 12.20 & 9.38 & 20.45 & 20.40 & 20.45 & 20.25 & 16.40 & 12.61 & 19.10 & 18.60 & \textbf{19.09} \\
\texttt{ePF} (ours) & 14.59 & 11.40 & 8.79 & 28.83 & 25.00 & 21.96 & 22.06 & 18.40 & 13.35 & 23.13 & 20.80 & 18.38 \\
\texttt{ePF w/ LaM} (ours) & \textbf{17.66} & \textbf{13.50} & \textbf{9.67} & \textbf{29.13} & \textbf{25.60} & \textbf{23.13} & \textbf{25.16} & \textbf{19.60} & \textbf{13.55} & \textbf{26.97} & \textbf{22.99} & \textbf{19.03} \\
\bottomrule
\end{tabular}
}
\label{tab:benchmark-aime-3}
\end{table}
\subsection{Guidance with Look-ahead Modulation}
\begin{wrapfigure}{r}{0.6\textwidth}
\vspace{-10pt}
\centering
\begin{subfigure}[b]{0.49\linewidth}
\includegraphics[width=\linewidth]{img/predictive/eval_benchmark_aime-2024_0.6B_20250826_predictive_new.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-0.6B)}
\label{fig:qwen3-aime24}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.49\linewidth}
\includegraphics[width=\linewidth]{img/predictive/eval_benchmark_aime-2025_1.7B_20250827_predictive_new.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:qwen3-aime25}
\end{subfigure}
\caption{AIME 2024 and AIME 2025 results with Qwen3-0.6B and Qwen3-1.7B w/o thinking mode for ePF and ePF w/ LaM.
Look-ahead Modulation boosts \texttt{ePF} and outperforms PF.}
\label{fig:qwen3-aime24-25}
\vspace{-10pt}
\end{wrapfigure}
The previous experiments established that ePF is an effective mechanism for enhancing base models.
We now study LaM and focus our attention on the Qwen3 series without thinking mode.
To overcome the inherent myopia of the particle filtering algorithm, we introduce Look-ahead Modulation (LaM). LaM incorporates a predictive estimate of the next state's value into the current resampling step.
This guidance provides a significant performance boost to our \texttt{ePF} framework, particularly on complex reasoning tasks, as shown in our results in Table~\ref{tab:benchmark-aime-3} and Fig.~\ref{fig:qwen3-aime24-25}.
ePF w/ LaM outperforms all the baselines over datasets, model size, and budget re-weighting, providing strong evidence that a non-myopic resampling, leveraging a relatively cheap single step look-ahead, is an effective mechanism to improve performance.
In Fig.~\ref{fig:reward-predictive} and~\ref{fig:reward-entropy} we provide intrinsic metrics to characterize ePF w/ LaM.
By re-weighting particles based on this forward-looking signal, \texttt{LaM} steers the search toward trajectories with higher long-term potential, rather than those that are only locally optimal, proving crucial for success on complex reasoning tasks. More results in Appx~\ref{appx:lam}, where we show that ePF w/ LaM achieves comparable performance with ePF on AIME 2024 with 1/4 and 1/2 of the particle budget.
\subsection{Characterizing Guided-Search with Intrinsic Metrics}
\begin{figure}[ht]
\centering
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/density/aime_2024_0.6b_hist.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-0.6B)}
\label{fig:hist-24-0.6b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/density/aime_2024_1.7b_hist.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:hist-24-1.7b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/density/aime_2025_0.6b_hist.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-0.6B)}
\label{fig:hist-25-0.6b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/density/aime_2025_1.7b_hist.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:hist-25-1.7b}
\end{subfigure}
\caption{Distribution Response Length using Particle Filtering and Entropic Particle Filtering with Qwen3-0.6B Qwen3-1.7B w/o thinking mode on AIME 2024 and AIME 2025. Both algorithms can use up to 12k tokens for each response. Notice how PF tends to generate shorter answers and converge to local solutions, where ePF explore the search space more, and converges to better solutions.}
\label{fig:length}
\end{figure}
Thus far, our evaluation has focused on benchmarking our proposed method against the state-of-the-art in ITS, primarily using the pass@1 metric. While this metric is a key indicator of model performance, it offers little insight into the solution-finding process or the diversity of the solutions generated.
To better understand the algorithmic behavior of ePF, we now examine several intrinsic metrics: the final output reward, step-by-step trajectory reward, step-wise variance and effective sample size of the resampling distribution, and sequence length. Leveraging the probabilistic interpretation of particle filtering algorithms allows us to study these metrics that characterize the entire generation process and observation likelihood.
Our approach reframes guided decoding as posterior inference (Eq.~\ref{eq:posterior-state}), motivated by the core hypothesis that \emph{the path to a solution is an essential component of the solution itself}. Under this view, analyzing the sampling trajectory through these intrinsic metrics is key to understanding how ePF balances guided search and exploration.
The aggregate measures, shown in Fig.~\ref{fig:reward-predictive},~\ref{fig:length}, and~\ref{fig:reward-entropy}, provide strong evidence for our central hypothesis. For instance,
Fig.~\ref{fig:length} reveals a striking difference in behavior: PF converges on a narrow distribution of shorter answers, whereas \texttt{ePF} produces a bimodal distribution, indicating it explores both simple solutions and significantly longer, more complex reasoning paths that would otherwise be pruned.
This commitment to exploration is further evidenced by the step-reward curves in Fig.~\ref{fig:reward-entropy}.
Notably, ePF's average step-reward initially dips below that of PF as entropic annealing actively forces the search away from greedy, high-reward initial steps (Center-Right of Fig.~\ref{fig:reward-entropy}). This early investment in exploration allows the algorithm to discover superior, high-reward trajectories later on, ultimately leading to higher overall task rewards and better final solutions. The variance of the resampling distribution can also be used to quantify PF exploitation versus ePF exploration (Fig.~\ref{fig:var}).
\begin{figure}[ht]
\centering
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/reward/aime_2024_1.7b_mean_bar.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:reward-mean-2024}
\end{subfigure}%
\hfill
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/reward/aime_2024_1.7b_running_mean.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:reward-running-2024}
\end{subfigure}%
\hfill
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/predictive/step_reward_comparison_closeup_aime_2024_1_7b.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:reward-step-2024}
\end{subfigure}%
\hfill
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=.8\linewidth]{img/diversity/diversity_k_aime_2024_1.7b.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:diversity-2024}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/reward/aime_2025_1.7b_mean_bar.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:reward-mean-2025}
\end{subfigure}%
\hfill
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/reward/aime_2025_1.7b_running_mean.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:reward-running-2025}
\end{subfigure}%
\hfill
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=\linewidth]{img/predictive/step_reward_comparison_closeup_aime_2025_1_7b.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:reward-step-2025}
\end{subfigure}%
\hfill
\begin{subfigure}[b]{0.24\linewidth}
\includegraphics[width=.8\linewidth]{img/diversity/diversity_k_aime_2025_1.7b.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:diversity-2025}
\end{subfigure}
\caption{Intrinsic Metrics Analysis of \texttt{ePF} on AIME 2024 and 2025 benchmarks with Qwen3-1.7B. The plots illustrate how \texttt{ePF} balances exploration and exploitation in the sampling process. \textbf{(Center-Right)}: Step-wise rewards initially dip (first 50/60 steps) as entropic annealing forces exploration, but this allows the algorithm to discover superior, high-reward regions later on. While not directly reward-seeking, this novelty-driven search leads to higher overall task rewards \textbf{(Left, Center-Left)} and a more diverse set of correct solutions across budgets \textbf{(Right)}, effectively mitigating premature convergence and early exploitation. See Fig.~\ref{fig:reward-predictive} for more results for ePF w/ LaM.}
\label{fig:reward-entropy}
\end{figure}
\textbf{Ablations}
We conducted extensive ablation studies to validate our proposed method in Appx~\ref{appx:quantitative}. As shown in Table~\ref{tab:benchmark-aime-sota-appx} and Table~\ref{tab:benchmark-tsmc}, ePF outperforms state-of-the-art baselines, surpassing TSMC~\citep{feng2024step} on the MATH500 and GSM8K benchmarks, and IAS-C~\citep{park2025know} on the combined AIME-24-25 dataset without offline calibration and PRM tuning. Further analyses isolate the contributions of key components, assess generation diversity (Fig.~\ref{fig:diversity-pass-b-appx}), and test various configurations, including alternative LLaMA backbones (Fig.~\ref{fig:llama-ablation}), and different temperature and resampling schedules (Fig.~\ref{fig:schedule-ablation}).
\textbf{Qualitative} Furthermore, the qualitative example in Appx~\ref{appx:qualitative} offers a concrete and tangible illustration of this process, contrasting a failed 3200 token attempt by PF with a successful 5400 token solution found by ePF, making the abstract concept of mitigating early exploitation concrete.
\section{Related Work}
\label{sec:related_work}
\paragraph{Inference-Time Scaling in Language Modeling}
Inference-Time Scaling (ITS) is a class of techniques that improve generative models performance by dedicating more computation at test time~\citep{wang2022self,wei2022chain,brown2024large,snell2024scaling,beeching2024scalingtesttimecompute,kang2025scalable}. These methods reframe generation as a guided search over a vast solution space, using either sequential self-refinement and linearized backtracking~\citep{shinn2024reflexion,yao2022react,jaech2024openai,guo2025deepseek} or, generating and evaluating multiple candidate solutions in parallel~\citep{yao2023tree,brown2024large,snell2024scaling}, performing a form of parallel search.
This search includes many classic ITS algorithms, such as Self-Consistency~\citep{wang2022self,chen2023universal}, Best-of-N sampling~\citep{brown2024large,kang2025scalable,amini2024variational}, and more sophisticated procedures like Beam Search~\citep{snell2024scaling,vijayakumar2016diverse} and Sequential Monte Carlo methods~\citep{lew2023sequential,zhao2024probabilistic,feng2024step,puri2025probabilistic,loula2025syntactic}. These algorithms typically leverage either an external outcome verifier to score complete solutions~\citep{lightman2023let,cobbe2021training} or use process rewards to guide the step-by-step construction of a solution~\citep{puri2025probabilistic}. Ultimately, all ITS methods can be conceptualized as forms of guided search over the sampling space, effectively trading increased computational budget for higher-quality outputs.
However, existing ITS methods have limitations and offer only partial solutions. Beam search and its diverse variants~\citep{vijayakumar2016diverse,snell2024scaling} can improve coverage but remain prone to low-diversity solutions and premature exploitation, while exploration-promoting methods like DVTS~\citep{beeching2024scalingtesttimecompute} tend to under-perform for small budgets. Standard PF and Sequential Monte Carlo (SMC) approaches~\citep{feng2024step,zhao2024probabilistic} require careful per-task tuning of a twist function and remain vulnerable to early overcommitment under miscalibration.
Monte Carlo Tree Search (MCTS), while dominant in domains where simulation is cheap~\citep{silver2016mastering}, remains challenging to apply to long-horizon mathematical tasks due to its computational complexity and the difficulty of training a reliable value function.
\paragraph{Importance Sampling}
Importance Sampling (IS) is a fundamental Monte Carlo technique used across Statistics~\citep{robert1999monte,casella2024statistical}, Physics~\citep{metropolis1949monte,metropolis1953equation,hastings1970monte}, and Engineering~\citep{kalman1960new,kolmogorov1941local} to address the common challenge of sampling from a complex target distribution $\pi(\vz)$~\citep{robert1999monte,Markov1953-MARTTO-31}. This problem is especially prevalent in Bayesian inference, where the goal is to characterize the posterior distribution $p(\vz | \vo)$ of a random variable $\vz$ given an observation $\vo$. Direct sampling from this posterior is often intractable due to high dimensionality or an unknown normalizing constant. However, we can typically evaluate the posterior up to this constant, as Bayes' theorem states $p(\vz | \vo) \propto p(\vo | \vz) p(\vz) = \tilde p(\vz | \vo)$, where the evidence $Z_p = \int p(\vo | \vz) p(\vz) d\vz$ is the intractable component. IS circumvents this by drawing samples from a simpler proposal distribution $q(\vz)$, which is easy to sample from and evaluate, and whose support covers that of $p(\vz | \vo)$. The expectation of a function $f(\vz)$ under the target can then be estimated by re-weighting samples from the proposal:
\begin{equation}
\bE_{p(\vz | \vo)}\left[f(\vz)\right] = \int p(\vz | \vo) f(\vz) d\vz = \int q(\vz) \dfrac{p(\vz | \vo)}{q(\vz)} f(\vz) d\vz = \bE_{q(\vz)}\left[w(\vz) f(\vz)\right].
\end{equation}
The terms $w(\vz) = p(\vz | \vo)/q(\vz)$ are the importance weights that correct for the discrepancy between the proposal and target distributions. Since we can only evaluate $\tilde p(\vz | \vo)$, we compute un-normalized weights $\tilde w(\vz) = \tilde p(\vz | \vo)/q(\vz)$. For a set of $N$ samples $\{\vz_i\}_{i=1}^N$ drawn from $q(\vz)$, these are normalized to yield the estimator $w_i = \tilde w(\vz_i) / \sum_{j=1}^{N} \tilde w(\vz_j)$.
While this estimator introduces a bias for finite $N$, it has lower variance and the estimator is consistent. The choice of proposal is critical; ideally, $q(\vz)$ should be close to the target, e.g., $q(\vz | \vo)$. Notice that in principle we can set the proposal equal to the prior, $q(\vz) = p(\vz)$. This simplifies the un-normalized weights to be the likelihood itself: $\tilde w(\vz) = p(\vo | \vz)p(\vz)/p(\vz) = p(\vo | \vz)$.
IS is a cornerstone of modern computational methods, forming the basis for MCMC sampling algorithms~\citep{metropolis1949monte}, Variational Inference~\citep{jordan1999introduction,kingma2019introduction}, Reinforcement Learning~\citep{peters2007reinforcement,roux2025tapered,schulman2017proximal}, and Monte Carlo Gradient Estimation~\citep{mohamed2020monte,schulman2015gradient,foerster2018dice}.
\paragraph{Auxiliary Particle-based Monte Carlo} Auxiliary Particle Filters (APF) also use a look-ahead mechanism to address the key limitation of standard PF, myopic resampling, i.e. relies only on past and current information for resampling~\citep{pitt1999filtering, pitt1999filtering-sim}. The mechanism biases the resampling distribution toward particles predicted to lead to more promising future states.
\paragraph{Maximum Entropy Algorithms}
Maximum Entropy Algorithms~\citep{jaynes1957information}, seek to maximize a given score while maintaining the highest possible entropy, thus balancing exploitation with exploration. This concept is central to methods like Soft Actor-Critic~\citep{haarnoja2018soft} and is related to quality-diversity algorithms~\citep{pugh2016quality,mouret2015illuminating}. For a given sampling distribution $q(\vz)$, these algorithms typically optimize the following objective:
\begin{equation}
\mathcal{F}(\vz) = \mathbb{S}(\vz) + \alpha~\mathbb{H}(\vz) \propto \mathbb{E}_{q(\vz)}\left[f(\vz)\right] - \mathbb{E}_{q(\vz)}\left[\log q(\vz)\right].
\end{equation}
Here, the scoring function $f(\vz)$ can represent rewards in Reinforcement Learning, where $f(\vz) = r(\vz)$~\citep{williams1991function,mnih2016asynchronous}, or the log-likelihood in Variational Inference, where $f(\vz) = \log p(\vz)$~\citep{jordan1999introduction,mohamed2016learning}.
\texttt{ePF} is inspired by this framework. It aims to find a problem's solution by following a trajectory of high entropy, constrained by a reward signal defined by a likelihood model over observations, $p(\vo | r(\vz))$. In essence, we encourage a policy that acts as randomly as possible while still achieving high rewards. This approach enhances exploration and effectively mitigates the risk of premature convergence to suboptimal solutions.
\paragraph{Premature Exploitation}
Premature convergence is a critical failure mode in evolutionary, reinforce-based, and control algorithms where the process becomes trapped in a suboptimal solution, fundamentally stemming from an imbalance in the exploration-exploitation trade-off~\citep{kaelbling1996RL,pandey2014comparative,lewis2012optimal}. This occurs when an algorithm prioritizes exploiting local good partial solutions over exploring uncharted regions of the search space, causing it to miss the global optimum or higher reward regions. In Particle Filtering, we can frame this issue as a form of poor state estimation in posterior inference, where premature convergence represents a mode collapse - the algorithm's belief about the optimal solution becomes too narrow and overconfident, stifling the discovery of novel, potentially superior solutions. Mixing mechanisms, hybrid algorithms, and multiple chains have been explored in the MCMC literature to counteract such issues~\citep{doucet2001sequential,binder1992monte,hastings1970monte}.
A core challenge in complex multi-step problem solving is the tension between early exploration and exploitation.
In engineering, over-optimization can lead to designs that are theoretically optimal on a single metric but fail on crucial, unmodeled objectives like manufacturability or robustness, limiting the diversity and quality of final designs \citep{chen2016topology, ahmed2016discovering}.
Quality-Diversity (QD~\citep{mouret2015illuminating,chatzilygeroudis2021quality,pugh2016quality}) algorithms shift the goal from finding a single best solution to finding a wide array of diverse, high-performing ones. Similar ideas are captured by entropy methods, particularly in Bayesian optimization~\citep{wang2017max,hernandez2016predictive,hernandez2015predictive}, which use entropy as a measure of uncertainty to actively guide the search. By maximizing information gain - seeking out areas where the outcome is most uncertain - these methods ensure the algorithm maintains full coverage of the search space, effectively preventing over-optimization and ensuring a more robust and comprehensive search for the true best solution. Recently, language modeling and quality-diversity have been scaled to solve algorithmic code-based problems~\citep{novikov2025alphaevolve,lange2024large,lehman2023evolution}.
\section{Limitations and Conclusion}
This paper introduces \emph{Entropic Particle Filtering} (ePF), a guided search algorithm that effectively mitigates premature convergence in language models by balancing exploration and exploitation. Through \emph{Entropic Annealing} (EA) and \emph{Look-ahead Modulation} (LaM), our method preserves hypothesis diversity and incorporates non-myopic guidance, leading to significant performance gains on complex reasoning tasks, especially under tight computational budgets.
\textbf{Limitations} Despite its strong performance, our approach has key limitations. First, its performance advantage is most significant with small computational budgets and diminishes as the number of particles increases. Second, Look-ahead Modulation introduces computational overhead due to its extra forward pass per resampling step. Finally, the method's effectiveness is constrained by the quality of the reward model; ePF can mitigate overconfidence but not a consistently inaccurate signal.
In conclusion, ePF offers a principled and effective strategy for improving inference-time search. By promoting robust exploration and forward-looking guidance, our methods enable the discovery of higher-quality solutions in complex domains.
\clearpage
\input{sec/appendix/estimator}
\clearpage
\input{sec/appendix/resampling}
\clearpage
\input{sec/appendix/complexity}
\clearpage
\input{sec/appendix/experiments_additional}
\clearpage
\input{sec/appendix/diversity}
\clearpage
\input{sec/appendix/pipeline}
\clearpage
\input{sec/appendix/algorithms}
\clearpage
\input{sec/appendix/qualitative}
\clearpage
\input{sec/appendix/details}
\section{Variance of the Resampling Distribution}
\label{appx:variance}
This section explains how Entropic Particle Filtering (ePF) reduces the variance of the resampling distribution. By doing so, ePF increases the effective sample size, leading to a more thorough exploration of the sampling space and a more accurate estimation of the true state.
\subsection{Importance Weights}
In importance sampling~\citep{casella2024statistical}, we draw samples from a tractable proposal distribution, $q(\mathbf{z})$, to estimate properties of an intractable target distribution, $p(\mathbf{z})$. This is achieved by weighting each sample. The importance weight, $w(\mathbf{z})$, for a sample $\mathbf{z}$ is the ratio of the target to proposal densities:
\begin{equation}
w(\mathbf{z}) = \dfrac{p(\mathbf{z})}{q(\mathbf{z})}, \quad \text{where} \quad \mathbb{E}_{q(\mathbf{z})}[w(\mathbf{z})] = 1 \quad \text{and} \quad w(\mathbf{z}) \geq 0.
\end{equation}
These weights allow us to estimate the expectation of any function $f(\mathbf{z})$ under the target distribution using $N$ samples $\{\mathbf{z}_i\}_{i=1}^N$ drawn from the proposal: $\hat{I} = \frac{1}{N}\sum^{N}_{i=1} w(\mathbf{z}_i) f(\mathbf{z}_i)$.
In a sequential context like particle filtering, these weights can be updated recursively. The unnormalized weight $\tilde{w}_t$ for a particle at time $t$ is calculated based on its weight at the previous step, the likelihood of the new observation $\mathbf{o}_t$, and the transition dynamics:
$$\tilde{w}_t = \tilde{w}_{t-1}~\dfrac{p(\mathbf{o}_t | \mathbf{z}_t)~p(\mathbf{z}_t | \mathbf{z}_{t-1})}{q(\mathbf{z}_t | \mathbf{z}_{t-1})}.$$
These quantities are then normalized to produce the final weights $w_t = \tilde{w}_t / \sum^{N}_{j=1} \tilde{w}^j_t$, which form the resampling distribution.
Entropic Particle Filtering operates on the principle that \emph{robust exploration in the early stages of a guided search yields higher-quality solutions later on}. Since the importance weights guide the resampling process, their distribution is critical. The ePF hypothesis implies that, particularly early in the search, the variance of the importance weights should be kept low. A distribution closer to uniform encourages broader exploration of the search space, increases the effective sample size, and makes better use of the available particles.
\subsection{Particle Collapse: Degeneracy and Impoverishment}
Particle collapse occurs when the particle filter loses its ability to approximate the posterior distribution. This failure can manifest in two primary ways: degeneracy and impoverishment.
\emph{Particle degeneracy} occurs when most particles acquire negligible weights, often due to noisy observations or model mismatch. In this state, only a handful of particles contribute to the posterior estimate, wasting computational resources. Degeneracy is typically addressed by resampling, where particles are drawn with replacement from the current set, proportional to their weights.
However, the solution to degeneracy can lead to \emph{particle impoverishment}. If the observation model is overly confident or uncalibrated, resampling may repeatedly select only a few high-weight particles. As these particles are cloned, the diversity of the particle set is drastically reduced. The entire population converges to a small, localized portion of the search space, potentially missing the true state entirely, generating a poor estimate of the posterior.
For instance, consider a system with just $N=4$ particles and normalized weights $w = [0.96, 0.01, 0.02, 0.01]$. During resampling, the first particle is overwhelmingly likely to be selected multiple times. Propagating a state composed almost entirely of clones of this single particle severely restricts the diversity and quality of future solutions.
\begin{figure}[h!]
\centering
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime24_pf_epf_lam_4budget.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B), N=4}
\label{fig:variance-aime-24-4p}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime25_pf_epf_lam_4budget.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B), N=4}
\label{fig:variance-aime-25-4p}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime24_pf_epf_lam_8budget.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B), N=8}
\label{fig:variance-aime-24-8p}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime25_pf_epf_lam_8budget.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B), N=8}
\label{fig:variance-aime-25-8p}
\end{subfigure}
\caption{Running Variance of the resampling distribution weights over the first 50 steps for PF, ePF, and ePF with LaM, using $N \in \{4,8\}$ particles on the AIME 2024 and 2025 dataset and Qwen3-1.7B as the sampler. Each line represents a single run. Standard PF (blue) exhibits a rapid increase in variance as a few particles quickly dominate the weight distribution. In contrast, ePF methods maintain lower variance, promoting greater particle diversity.
}
\label{fig:var}
\end{figure}
\clearpage
\subsection{The Variance of the Weights Distribution}
The health of a particle filter can be quantified by the \emph{Effective Sample Size (ESS)}, which measures the diversity of the particles. Following~\citep{liu1996metropolized,kong1992note}, the ESS at step $t$ for a particle filter with $N$ particles is inversely related to the variance of the normalized importance weights, $\mathbb{V}[w_t]$:
\begin{equation}
ESS(t) = \dfrac{N}{1 + \mathbb{V}[w_t]}
\end{equation}
where the weights $w^i_t$ are typically computed via a softmax over log-rewards $r^i_t$ as $w^i_t = \exp(r^i_t) / \sum^{N}_{j=1} \exp(r^j_t)$. The sample variance of these weights can be estimated as:
$$ \mathbb{\hat V}[w_t] \approx \dfrac{1}{N-1}~\sum^{N}_{i=1}\left(w^i_t - \dfrac{1}{N}\right)^2 $$
This relationship makes it clear that a high variance in the weight distribution leads to a low effective sample size. As variance increases, one or more weights become large while others shrink, indicating that fewer particles are effectively contributing to the state representation.
A more direct and widely used definition of ESS is~\citep{doucet2001sequential,martino2017effective}:
$$ ESS(t) = \dfrac{1}{\sum^{N}_{i=1}(w_t^i)^2} $$
This formulation is intuitive:
\begin{itemize}
\item Maximum ESS: If all weights are uniform ($w_t^i = 1/N$), then $ESS = 1 / \sum(1/N)^2 = 1 / (N/N^2) = N$. The effective size is the total number of particles.
\item Minimum ESS: In the most degenerate case, one weight is 1 and all others are 0. Then $ESS = 1 / (1^2) = 1$. The effective size has collapsed to a single particle.
\end{itemize}
Crucially, low variance in the resampling distribution at step $t$ corresponds to high diversity among the particles selected for propagation to step $t+1$.
\subsection{Entropic Annealing as a Variance Reduction Technique}
High variance in the importance weights is a direct cause of particle impoverishment in Sequential Importance Resampling (SIR) methods. \emph{Entropic Annealing (EA) directly mitigates this by functioning as a dynamic variance reduction technique} integrated into the resampling step.
The core principle is that the variance of a Monte Carlo estimator for the resampling distribution is minimized when the importance weights are uniform. EA operationalizes this by modulating the resampling distribution's "temperature" based on the ESS. When the ESS is low - indicating high weight variance and potential particle collapse - EA increases the entropy of the weight distribution. It does this by applying a temperature parameter, $\beta_t \le 1$, to the log-rewards before the softmax calculation:
$$ w_t^i = \frac{\exp(r_t^i \cdot \beta_t)}{\sum_{j=1}^{N}\exp(r_t^j \cdot \beta_t)} $$
Lowering $\beta_t$ (i.e., "heating up" the distribution) flattens the output of the softmax, pushing the weights closer to a uniform distribution. This action actively reduces the variance of the weights at each step in the early phase of sampling (Fig.~\ref{fig:var}). By preventing the particle set from collapsing onto a few high-reward hypotheses, this proactive management of weight variance ensures a more stable and robust approximation of the posterior distribution throughout the sequential filtering process.
High-variance in the resampling distribution will create a poor estimate for the state and the posterior. In the limit, high-variance over the importance weights will generated a deterministic state.
In summary, the \texttt{ePF} algorithm is a form of variance reduction: limiting extreme overconfidence (see the two blue lines with large variance in Fig.~\ref{fig:var}) over few particles.
Notice that the ESS-based schedule for a given step is closely related to the inverse of the variance, in particular:
$$
\dfrac{\beta^{-1}_t}{t/T} -1 = \frac{N}{ESS(t)} - 1 = \mathbb{\hat V}[w_t],
$$
clearly showing that, for a given $t$, higher the variance, higher the temperature $\beta^{-1}_t$, flattening the resampling distribution and better exploring the state.
Notice that the variance at step $t$ for the importance sampling distribution can also be written as $\mathbb{V}[w_t] = \mathbb{E}[w^2_t] - \mathbb{E}[w_t]^2 \propto \mathbb{E}[w^2_t]$, showing that the quantity $\sum^N_{i=1} (w^i_t)^2$ is intrinsically related to the variance of the importance weight estimator.
\begin{figure}[h!]
\centering
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime24_pf_epf_lam_16budget.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B), N=16}
\label{fig:variance-aime-24-16p}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime25_pf_epf_lam_16budget.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B), N=16}
\label{fig:variance-aime-25-16p}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime24_pf_epf_lam_32budget.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B), N=32}
\label{fig:variance-aime-24-32p}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.4\linewidth}
\includegraphics[width=\linewidth]{img/variance/variance_aime25_pf_epf_lam_32budget.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B), N=32}
\label{fig:variance-aime-25-32p}
\end{subfigure}
\caption{Running Variance of the resampling distribution weights over the first 50 steps for PF, ePF, and ePF with LaM, using $N \in \{16,32\}$ particles on the AIME 2024 and 2025 dataset and Qwen3-1.7B as the sampler. Each line represents a single run. Standard PF (blue) exhibits a rapid increase in variance as a few particles quickly dominate the weight distribution. In contrast, ePF methods maintain lower variance, promoting greater particle diversity.
}
\label{fig:var1}
\end{figure}
\section{Multinomial and Systematic Resampling}
\label{appx:systematic-multinomial}
\begin{wrapfigure}{r}{0.4\textwidth}
\centering
\includegraphics[width=\linewidth]{img/softmax.pdf}
\caption{Different ways to compute the resampling distribution using the PRM output.
$N=12$, $ESS/N \in [0.18,0.43]$, $1/N$ is the uniform.
Our goal is to find a balance between early exploration and late exploitation in guided sampling.}
\label{fig:softmax-systematic}
\end{wrapfigure}
In particle filtering, both \emph{multinomial} and \emph{systematic resampling} aim to generate a new population of $N$ particles from an existing set based on their normalized weights $\{w_k\}_{k=1}^N$, thereby combating particle degeneracy. The two methods are conceptually similar but differ critically in their sampling strategy and resulting statistical properties.
Multinomial resampling~\citep{casella2024statistical} operates by making $N$ independent draws from a categorical distribution defined by the particle weights. For each of the $N$ new particle slots, a random number $u_i \sim U[0, 1)$ is independently drawn, and the particle index $j_i$ is selected such that $c_{j_i-1} \le u_i < c_{j_i}$, where $c_k$ is the cumulative sum of weights. The primary advantage of this method is its simplicity and the statistical independence of each selection.
However, its significant drawback is high variance; the number of offspring for any given particle is binomially distributed, meaning a particle with a reasonably high weight can be lost by chance, introducing unnecessary Monte Carlo error and reducing particle diversity.
Systematic resampling~\citep{kitagawa1996monte}, conversely, is a lower-variance technique that uses a single random draw to select the entire population. It generates one random number $u \sim U[0, 1)$ to create a deterministic, evenly-spaced set of $N$ pointers, $u_i = (i-1+u)/N$ for $i \in \{1, \dots, N\}$. This stratified sequence is then used to select the new particles from the same cumulative weight distribution. The main advantage of this approach is its efficiency and reduced sampling variance. It guarantees that a particle with weight $w_k$ will be selected approximately $N \cdot w_k$ times, ensuring the resampled population is a much more faithful representation of the target distribution, at the cost of introducing a small correlation among the selected indices, as they all depend on the initial draw $u$. Due to its ability to better preserve the distribution's structure and reduce random fluctuations, systematic resampling is often the preferred method in most engineering applications for tracking and robotics.
\section{Computational Cost}
\label{appx:complexity}
The overall computational cost is determined by the forward passes through two models: the main generator (sampler) and the Process Reward Model (PRM). The total complexity scales linearly with the particle budget ($N$), the number of generation steps ($T$), and the maximum number of tokens generated per step ($C$).
\begin{itemize}
\item \emph{PF and ePF:} In these particle filtering methods, each of the $N$ particles requires $C$ forward pass through the sampler for propagation (generation) and one forward pass through the PRM for scoring at each step.
\item \emph{ePF w/ LaM}: The Look-ahead Mechanism (LaM) introduces an additional propagation and scoring pass for each particle during its look-ahead phase. In the worst-case scenario, where look-ahead is used for all the steps in the first half of the trajectory $T$, the computational cost increases by approximately 50\% compared to standard ePF.
\end{itemize}
\begin{table}[ht]
\centering
\caption{Comparison of computational complexity per sampling trajectory.}
\resizebox{\textwidth}{!}{
\begin{tabular}{l c c c}
\toprule
\textbf{Algorithm} & \textbf{Sampler Forwards} & \textbf{PRM Forwards} & \textbf{Worst-Case Total Ops} \\
\midrule
ePF & $N \times T \times C$ & $N \times T$ & $(N \times T \times C) + (N \times T)$ \\
ePF w/ LaM & $N \times (T + T/2) \times C$ & $N \times (T + T/2)$ & $(N \times (T + T/2) \times C) + (N \times (T + T/2))$ \\
\bottomrule
\end{tabular}
}
\label{tab:complexity}
\end{table}
Although the PRM can be a larger model than the sampler, its contribution to the total compute is significantly smaller. This is because the sampler's cost scales with the number of generated tokens per step ($C$), as it operates auto-regressively, while the PRM requires only a single forward pass to score the entire output. Given that $C$ is typically in the range of $10^2$ to $10^3$, the generation cost overwhelmingly dominates the verification cost.
In practice, the practical overhead of LaM is much lower than its worst-case estimate. The look-ahead mechanism is only activated when entropic resampling is required (low effective sample size), which, as shown in Figure~\ref{fig:distro_resampling_steps_appx}, occurs in only about 10-12\% of the steps, not the theoretical maximum of 50\%. This makes the actual added cost quite manageable.
\begin{figure}[ht]
\centering
\includegraphics[width=0.8\linewidth]{img/lam_cost/distro_resampling_steps_aepf.pdf}
\caption{Density of resampling steps with LaM as a fraction of the total steps in the sampling trajectories on the AIME 2025 dataset, aggregated across various budgets. The data shows that resampling with look-ahead typically occurs in only 10-12\% of the generation steps.}
\label{fig:distro_resampling_steps_appx}
\end{figure}
\clearpage
\subsection{Wall-Clock Analysis}
\begin{table}[ht!]
\centering
\caption{Wall-Clock by Budget over the 30 tasks in AIME 2024 using Qwen2.5-1.5B. Benchmark run on a single A100 using \texttt{vllm} for sampling.} \label{tab:algo_stats_revised}
\begin{tabular}{lcccc}
\toprule
Algorithm & Budget & Mean & Median & Max \\
\midrule
\multirow{3}{*}{Best-of-N}
& 8 & 30.252 & 22.457 & 110.173 \\
& 16 & 57.920 & 43.473 & 138.366 \\
& 32 & 117.069 & 95.988 & 341.408 \\
\midrule
\multirow{3}{*}{Beam Search}
& 8 & 37.294 & 35.075 & 117.227 \\
& 16 & 85.077 & 72.572 & 228.574 \\
& 32 & 184.827 & 175.451& 425.077 \\
\midrule
\multirow{3}{*}{Particle Filtering}
& 8 & 37.435 & 35.451 & 91.908 \\
& 16 & 96.992 & 85.207 & 275.590 \\
& 32 & 198.243 & 177.322& 414.395 \\
\midrule
\multirow{3}{*}{Entropic Particle Filtering}
& 8 & 50.163 & 47.149 & 129.590 \\
& 16 & 125.121 & 116.733 & 377.558 \\
& 32 & 271.590 & 244.704 & 559.433 \\
\midrule
\multirow{3}{*}{Entropic Particle Filtering w/ LaM}
& 8 & 53.674 & 49.506 & 155.508 \\
& 16 & 138.884 & 126.072 & 468.172 \\
& 32 & 304.181 & 266.727 & 676.914 \\
\bottomrule
\end{tabular}
\end{table}
\section{Additional Experiments}
\label{appx:quantitative}
\subsection{General Results on Mathematical Benchmarks}
\label{appx:general-math-benchmark}
\begin{figure}[ht]
\centering
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_gsm8k_1.5B_20250820.pdf}
\caption{\scriptsize GSM8K (Qwen2.5-1.5B)}
\label{fig:gsm8k}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_math500_1.5B_20250815.pdf}
\caption{\scriptsize MATH500 (Qwen2.5-1.5B)}
\label{fig:math500}
\end{subfigure}%
\vspace{2em} %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_deepmath_1.5B_20250820.pdf}
\caption{\scriptsize DEEPMATH (Qwen2.5-1.5B)}
\label{fig:deepmath-1.5b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/figures/eval_benchmark_deepmath_7B_20250814.pdf}
\caption{\scriptsize DEEPMATH (Qwen2.5-7B)}
\label{fig:deepmath-7b}
\end{subfigure}
\caption{Math Benchmarks in order of complexity. We run Qwen2.5-1.5b-Instruct (1.5B) and Qwen2.5-7b-Instruct (7B) over a subset of GSM8K, MATH500, and DEEPMATH. ePF is competitive or better than strong baselines on all benchmarks. Table~\ref{tab:benchmark} for more results.}
\label{fig:gsm8k-math500-deepmath}
\end{figure}
\clearpage
\subsection{Exploration for Hard Problems and Small Budgets}
\label{appx:eq-aime-budget-appx}
\begin{table}[ht]
\caption{Baseline performance on AIME 2024 math benchmarks using Qwen2.5-1.5B-Instruct and Qwen2.5-7B-Instruct.
The table shows pass@1 scores for budgets ($N \in \{2, 4, 8, 16, 32\}$).
}
\centering
\begin{tabular}{l | ccccc | ccccc }
\toprule
& \multicolumn{10}{c}{AIME 2024} \\
& \multicolumn{5}{c}{Qwen2.5-1.5B-Instruct} & \multicolumn{5}{c}{Qwen2.5-7B-Instruct} \\
& 2 & 4 & 8 & 16 & 32
& 2 & 4 & 8 & 16 & 32 \\
\midrule
Best-of-N & 0.03 & 0.03 & 0.10 & 0.13 & 0.10 & 0.16 & 0.16 & 0.23 & 0.20 & 0.26 \\
Beam-Search & 0.03 & 0.06 & 0.10 & 0.13 & 0.10 & 0.10 & 0.11 & 0.13 & 0.13 & 0.23 \\
PF & 0.06 & 0.06 & 0.10 & 0.10 & 0.13 & 0.16 & 0.20 & 0.16 & 0.26 & 0.30 \\
\texttt{ePF} & 0.03 & 0.03 & 0.10 & 0.20 & 0.20 & 0.13 & 0.16 & 0.20 & 0.26 & 0.30 \\
\bottomrule
\end{tabular}
\label{tab:benchmark-aime-24-qween2.5-raw-appx}
\end{table}
\begin{table}[ht]
\caption{Baseline performance on AIME 2025 math benchmarks using Qwen2.5-1.5B-Instruct and Qwen2.5-7B-Instruct.
The table shows pass@1 scores for budgets ($N \in \{2, 4, 8, 16, 32\}$).}
\centering
\begin{tabular}{l | ccccc | ccccc}
\toprule
& \multicolumn{10}{c}{AIME 2025} \\
& \multicolumn{5}{c}{Qwen2.5-1.5B-Instruct} & \multicolumn{5}{c}{Qwen2.5-7B-Instruct} \\
& 2 & 4 & 8 & 16 & 32
& 2 & 4 & 8 & 16 & 32 \\
\midrule
Best-of-N & 0.03 & 0.01 & 0.03 & 0.06 & 0.06 & 0.16 & 0.10 & 0.20 & 0.13 & 0.20 \\
Beam-Search & 0.01 & 0.06 & 0.10 & 0.10 & 0.10 & 0.16 & 0.16 & 0.26 & 0.10 & 0.13 \\
PF & 0.03 & 0.01 & 0.03 & 0.04 & 0.11 & 0.16 & 0.20 & 0.16 & 0.23 & 0.20 \\
\texttt{ePF} & 0.01 & 0.02 & 0.10 & 0.10 & 0.13 & 0.20 & 0.23 & 0.23 & 0.26 & 0.33 \\
\bottomrule
\end{tabular}
\label{tab:benchmark-aime-25-qween2.5-raw-appx}
\end{table}
\begin{table}[ht]
\caption{Baseline performance on AIME 2024 math benchmarks using Qwen3-0.6B and Qwen3-1.7B.
The table shows pass@1 scores for budgets ($N \in \{2, 4, 8, 16, 32\}$).}
\centering
\begin{tabular}{l | ccccc | ccccc}
\toprule
& \multicolumn{10}{c}{AIME 2024} \\
& \multicolumn{5}{c}{Qwen3-0.6B} & \multicolumn{5}{c}{Qwen3-1.7B} \\
& 2 & 4 & 8 & 16 & 32
& 2 & 4 & 8 & 16 & 32 \\
\midrule
Best-of-N & 0.03 & 0.05 & 0.06 & 0.06 & 0.10 & 0.16 & 0.16 & 0.23 & 0.16 & 0.23 \\
PF & 0.07 & 0.07 & 0.17 & 0.12 & 0.16 & 0.20 & 0.23 & 0.16 & 0.23 & 0.20 \\
\texttt{ePF} & 0.08 & 0.08 & 0.14 & 0.10 & 0.18 & 0.20 & 0.23 & 0.23 & 0.26 & 0.33 \\
\texttt{ePF w/ LaM} & 0.08 & 0.10 & 0.10 & 0.20 & 0.20 & 0.23 & 0.20 & 0.26 & 0.26 & 0.33\\
\bottomrule
\end{tabular}
\label{tab:benchmark-aime-24-qween3-raw-appx}
\end{table}
\begin{table}[ht!]
\caption{Baseline performance on AIME 2025 math benchmarks using Qwen3-0.6B and Qwen3-1.7B.
The table shows pass@1 scores for budgets ($N \in \{2, 4, 8, 16, 32\}$).}
\centering
\begin{tabular}{l | ccccc | ccccc}
\toprule
& \multicolumn{10}{c}{AIME 2025} \\
& \multicolumn{5}{c}{Qwen3-0.6B} & \multicolumn{5}{c}{Qwen3-1.7B} \\
& 2 & 4 & 8 & 16 & 32
& 2 & 4 & 8 & 16 & 32 \\
\midrule
Best-of-N & 0.07 & 0.13 & 0.20 & 0.23 & 0.16 & 0.20 & 0.16 & 0.16 & 0.20 & 0.20 \\
PF & 0.10 & 0.13 & 0.16 & 0.20 & 0.23 & 0.20 & 0.20 & 0.13 & 0.20 & 0.20 \\
\texttt{ePF} & 0.10 & 0.13 & 0.16 & 0.30 & 0.23 & 0.15 & 0.23 & 0.20 & 0.20 & 0.26 \\
\texttt{ePF w/ LaM} & 0.10 & 0.13 & 0.16 & 0.33 & 0.26 & 0.16 & 0.20 & 0.23 & 0.26 & 0.30\\
\bottomrule
\end{tabular}
\label{tab:benchmark-aime-25-qween3-raw-appx}
\end{table}
\clearpage
\subsection{Look-ahead Modulation Efficiency}
\label{appx:lam}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.6\linewidth}
\includegraphics[width=\linewidth]{img/lam_cost/eval_benchmark_aime-2024_1.5B_20250810_predictive_compute_cost.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-1.5B)}
\label{fig:aime24-cost-1.5b}
\end{subfigure}%
\vspace{2em} %
\begin{subfigure}[b]{0.6\linewidth}
\includegraphics[width=\linewidth]{img/lam_cost/eval_benchmark_aime-2024_7B_20250912_predictive_compute_cost.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-7B)}
\label{fig:aime24-cost-7b}
\end{subfigure}
\caption{AIME-24 results with Qwen-2.5-1.5b-Instruct and Qwen-2.5-7b-Instruct for Entropic Particle Filtering and Look-Ahead Modulation.
We run ePF w/ LaM until it reaches performance within 5\% of the ePF with $N=32$.
We can see that ePF w/ LaM and 8 particles reaches a performance comparable to ePF with 32 particles using a Qwen2.5-1.5B-Instruct model.
And ePF w/ LaM and 16 particles is competitive with ePF with 32 particles using a Qwen2.5-7B-Instruct model.}
\label{fig:aime24-predictive-appx}
\end{figure}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/predictive/eval_benchmark_aime-2024_0.6B_20250826_predictive_new.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-0.6B)}
\label{fig:qwen3-aime24}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/predictive/eval_benchmark_aime-2025_1.7B_20250827_predictive_new.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:qwen3-aime25}
\end{subfigure}
\caption{AIME-24 and AIME-25 results with Qwen3-0.6B and Qwen3-1.7B w/o thinking mode for ePF and ePF w/ LaM.}
\label{fig:qwen3-aime24-25-appx}
\end{figure}
\begin{figure}[ht]
\centering
\vspace{-2em}
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/aime_2025_1.7b_median_bar_predictive.pdf}
\caption{Task Reward by Budget.}
\label{subfig:aime-bar-appx}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/predictive/aime_2025_1.7b_running_median_predictive.pdf}
\caption{Running Task Reward over Budgets.}
\label{subfig:aime-running-appx}
\end{subfigure}
\caption{Task reward comparison on AIME-2025 using Qwen3-1.7B. Our Entropic Particle Filtering (\texttt{ePF}) and its Look-ahead variant (\texttt{ePF} w/ \texttt{LaM}) significantly improve performance over standard Particle Filtering (PF) across all particle budgets. This demonstrates that mitigating premature exploitation leads to significant performance gains.}
\label{fig:reward-predictive-appx}
\end{figure}
\clearpage
\subsection{PF and ePF Max Performance}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_benchmark_aime-2024_1.5B_20250912_long_seq.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-1.5B)}
\label{fig:max-24-1.5b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_benchmark_aime-2024_7B_20250912_long_seq.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-7B)}
\label{fig:max-24-7b}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_benchmark_aime-2025_1.5B_20250912_long_seq.pdf}
\caption{\scriptsize AIME 2025 (Qwen2.5-1.5B)}
\label{fig:max-25-1.5b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_benchmark_aime-2025_7B_20250912_long_seq.pdf}
\caption{\scriptsize AIME 2025 (Qwen2.5-7B)}
\label{fig:max-25-7b}
\end{subfigure}
\caption{Best \texttt{pass@1} performance as a function of inference budget ($N$) across 5 runs. ePF clearly outperforms PF on AIME-2024 and AIME-2025 using models of different sizes (Qwen2.5-1.5B-Instruct and Qwen2.5-7B-Instruct). Max Sequence length of 12288.}
\label{fig:aime24-aime25-max-appx}
\end{figure}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/max/eval_benchmark_aime-2024_0.6B_20250826_max.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-0.6B)}
\label{fig:max-qwen3-24-0.6b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/max/eval_benchmark_aime-2024_1.7B_20250826_max.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:max-qwen3-24-1.7b}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/max/eval_benchmark_aime-2025_0.6B_20250826_max.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-0.6B)}
\label{fig:max-qwen3-25-0.6b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/ablation/max/eval_benchmark_aime-2025_1.7B_20250826_max.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:max-qwen3-25-1.7b}
\end{subfigure}
\caption{Best \texttt{pass@1} performance as a function of inference budget ($N$) across 5 runs. ePF clearly outperforms PF on AIME-2024 and AIME-2025 using models of different sizes (Qwen3-0.6B and Qwen3-0.6B). Max Sequence length of 12288.}
\label{fig:aime24-aime25-max-qwen3-appx}
\end{figure}
\clearpage
\subsection{Specialist and Generalist Models}
\begin{table}[ht!]
\caption{Evaluating Specialized and Generalist Language Models on AIME-2024. We report the best \texttt{pass@1} results among 5 runs. Math-Specialists results from~\citep{puri2025probabilistic}.}
\centering
\begin{tabular}{l c}
\toprule
Method & AIME 2024 \\
\midrule
\emph{Math-Specialists} (Qwen2.5-Math-7B-Instruct)\\
Greedy & 5/30 \\
Self Consistency & 4/30 \\
Best-of-N (W) & 5/30 \\
Beam-Search & 7/30 \\
DVTS~\citep{beeching2024scalingtesttimecompute} & 6/30 \\
PF~\citep{puri2025probabilistic} & 10/30 \\
\midrule
\emph{Generalist} (Qwen2.5-7B-Instruct) \\
Greedy & 3/10 \\
PF & 9/30 \\
\texttt{ePF} (ours) & 11/30 \\
\texttt{ePF w/ LaM} (ours) & 10/30 \\
\midrule
\emph{Generalist} (Qwen3-0.6B, w/o thinking) \\
Greedy & 3/30 \\
PF & 8/30 \\
\texttt{ePF} (ours) & 10/30 \\
\texttt{ePF w/ LaM} (ours) & 10/30 \\
\midrule
\emph{Generalist} (Qwen3-1.7B, w/o thinking) \\
Greedy & 4/30 \\
PF & 10/30 \\
\texttt{ePF} (ours) & 14/30 \\
\texttt{ePF w/ LaM} (ours) & 14/30 \\
\bottomrule
\end{tabular}
\label{tab:benchmark-aime-sota-appx}
\end{table}
\clearpage
\subsection{Guided-Search Ablation}
\begin{table}[ht!]
\caption{Comparison of our method, ePF, against state-of-the-art baselines on the MATH500, GSM8K, and AIME-24-25 mathematical reasoning benchmarks. Our approach consistently achieves higher scores, demonstrating its effectiveness.}
\label{tab:benchmark-tsmc}
\centering
\begin{subtable}{0.32\textwidth}
\centering
\small
\caption{\scriptsize MATH500}
\begin{tabular}{l c}
\toprule
Method & Score \\ \midrule
TSMC~\citep{feng2024step} & 60.8 \\
\texttt{ePF} (ours) & 65.1 \\ \bottomrule
\end{tabular}
\end{subtable}%
\begin{subtable}{0.32\textwidth}
\centering
\small
\caption{\scriptsize GSM8K}
\begin{tabular}{l c}
\toprule
Method & Score \\ \midrule
TSMC~\citep{feng2024step} & 91.7 \\
\texttt{ePF} (ours) & 94.3 \\ \bottomrule
\end{tabular}
\end{subtable}%
\hfill
\begin{subtable}{0.32\textwidth}
\centering
\small
\caption{\scriptsize AIME-24-25}
\begin{tabular}{l c}
\toprule
Method & Score \\ \midrule
IAS-C~\citep{park2025know} & 18.3 \\
\texttt{ePF} (ours) & 26.4 \\ \bottomrule
\end{tabular}
\end{subtable}
\end{table}
\subsection{Backbone Ablation}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.32\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_ablation_aime-2024_1B_20250909.pdf}
\caption{\scriptsize AIME 2024 (Llama3.2-1B)}
\label{fig:ablation-24-1b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.32\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_ablation_aime-2025_1B_20250909.pdf}
\caption{\scriptsize AIME 2025 (Llama3.2-1B)}
\label{fig:ablation-25-1b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.32\linewidth}
\includegraphics[width=\linewidth]{img/ablation/eval_benchmark_aime-2024_8B_20250915.pdf}
\caption{\scriptsize AIME 2024 (Llama3.1-8B)}
\label{fig:ablation-24-8b}
\end{subfigure}
\caption{Llama3.2-1B and Llama3.1-8B ablation for AIME-24 and AIME-25. Mitigating exploitation using ePF improves performance on hard tasks even when using weak models.}
\label{fig:llama-ablation}
\end{figure}
\clearpage
\subsection{Temperature Schedule Ablation}
\label{appx:temperature-schedule-ablation}
While particle filtering ideally explores the search space by resampling from a diverse set of particles, it often suffers from premature collapse. This issue arises when the search process oversamples a few promising particles too early, often due to misspecified rewards or overconfidence. This effectively turns the algorithm into a greedy search, which is particularly detrimental for complex problems as it stifles thorough exploration. To counteract this, we introduce an annealing strategy that modulates the temperature of the resampling distribution to preserve particle diversity and mitigate early exploitation.
To control this process, we introduce a dynamically adjusted temperature, $1/\beta_t$, which follows an annealing schedules at each time step $t$ (Fig.~\ref{fig:schedule}). Two of these schedules adaptively respond to particle diversity, which we quantify using the normalized entropy,
$$H_{n}(t) = - \frac{\sum^{N}_{i=1} w^{i}_t \log w^{i}_t}{\log N},$$
and the effective sample size,
$$ESS(t) = \dfrac{1}{\sum^{N}_{j=1} (w^{j}_{t})^{2}}.$$
ESS and Entropy are closely related~\footnote{One way to define $ESS(t)$ is $\exp{(H(t))}$~\cite{martino2025effective}.}.
The proposed schedules are (Figure~\ref{fig:schedule}):
\begin{itemize}
\item \emph{Linear}: a simple decay schedule, $\beta_t^{-1} = k - t/T$.
\item \emph{ESS-based}: a temperature schedule defined as $\beta_t^{-1} = (N / ESS(t)) \cdot (1 - t/T)$.
\item \emph{Entropy-based}: a convex combination given by $\beta_t = H_{n}(t) + (1 - H_{n}(t)) \cdot t/T$.
\end{itemize}
At each step, the resulting inverse temperature $\beta_t$ modulates the particle rewards $r^{i}_t$ to produce the resampling weights via a softmax function (Fig.~\ref{fig:ea}):
$$
w^{i}_t = \frac{\exp(r^{i}_t \cdot \beta_t)}{\sum^{N}_{j=1} \exp(r^{j}_t \cdot \beta_t)}.
$$
\begin{figure}[ht!]
\centering
\includegraphics[width=.45\linewidth]{img/softmax_schedules.pdf}
\caption{Different Schedules for the resampling distribution temperature annealing increasing the number of sampling steps.
Here we set $k=2$, $H_n=0.5$, $N=16$, and $ess(t)=8$.}
\label{fig:schedule}
\end{figure}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.45\linewidth}
\includegraphics[width=\linewidth]{img/ablation/ci/eval_ablation_schedule_aime-2024_7B_20250911.pdf}
\caption{\scriptsize AIME 2024 (Qwen2.5-7B)}
\label{fig:schedule-24-7b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.45\linewidth}
\includegraphics[width=\linewidth]{img/ablation/ci/eval_ablation_schedule_aime-2024_1.7B_20250911.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:schedule-24-1.7b}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.45\linewidth}
\includegraphics[width=\linewidth]{img/ablation/ci/eval_ablation_schedule_aime-2025_7B_20250911.pdf}
\caption{\scriptsize AIME 2025 (Qwen2.5-7B)}
\label{fig:schedule-25-7b}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.45\linewidth}
\includegraphics[width=\linewidth]{img/ablation/ci/eval_ablation_schedule_aime-2025_1.7B_20250911.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:schedule-25-1.7b}
\end{subfigure}
\caption{Temperature Schedule ablation for AIME-24 and AIME-25 using Qwen2.5-7B-Instruct and Qwen3-1.7B. Among models and datasets, ESS-based temperature annealing is the most consistent.}
\label{fig:schedule-ablation}
\end{figure}
See Algorithm~\ref{alg:epf} for implementation details and Figure~\ref{fig:schedule-ablation} for a temperature schedule ablation over AIME-2024 and AIME-2025.
If not otherwise specified, we use the \emph{ESS-based} schedule for the core experiments.
The ESS-based schedule is designed to react aggressively to particle collapse. When the ESS is low (indicating low diversity), the temperature sharply increases to encourage exploration. The $(1 - t/T)$ term ensures this exploratory pressure is strongest in the early stages and gradually anneals or cools over time, allowing the search to shift from exploration to exploitation as it progresses (Fig.~\ref{fig:schedule}).
\clearpage
\subsection{Effective Sample Size Ablation}
\begin{figure}[ht!]
\centering
\includegraphics[width=0.3\linewidth]{img/budget/ess_aime24_epf_lam_2budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_epf_lam_2budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_aime24_epf_lam_4budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_epf_lam_4budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_aime24_epf_lam_8budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_epf_lam_8budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_aime24_epf_lam_16budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_epf_lam_16budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_aime24_epf_lam_32budget.pdf}
\includegraphics[width=0.3\linewidth]{img/budget/ess_epf_lam_32budget.pdf}
\caption{Running Effective Sample Size (post-resampling) for AIME 2024 and AIME 2025 using Qwen3-1.7B over increasing budgets $N \in \{2,4,8,16,32\}$.}
\label{fig:ess-ablation}
\end{figure}
\subsection{Coverage and Diversity}
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/diversity/diversity_k_aime_2024_1.7b.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:diversity-24}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/diversity/diversity_k_aime_2025_1.7b.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:diversity-25}
\end{subfigure}
\vspace{2em} %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/diversity/diversity_k_aime_2024_1.7b_comparison.pdf}
\caption{\scriptsize AIME 2024 (Qwen3-1.7B)}
\label{fig:diversity-24-comp}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.48\linewidth}
\includegraphics[width=\linewidth]{img/diversity/diversity_k_aime_2025_1.7b_comparison.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:diversity-25-comp}
\end{subfigure}
\caption{Diversity of Correct Solutions for pass@1 and pass@B generating up to 12k tokens/sequence. pass@B is defined as the expected pass@1 aggregated over budgets. The goal of mitigating early exploitation in PF is to maintain a distribution over hypotheses and do not over-optimize early on in the sampling process. Given that ePF is exploring more, we expect the algorithm to find more diverse solutions.}
\label{fig:diversity-pass-b-appx}
\end{figure}
\paragraph{Syntactic Diversity}
For syntactic diversity, we consider the Jaccard coefficient, which measures the similarity between two sets. Given two responses, $r_a$ and $r_b$, let $T_a$ and $T_b$ be their respective sets of unique tokens. The Jaccard coefficient is defined as $J(T_a, T_b) = \frac{|T_a \cap T_b|}{|T_a \cup T_b|}$. A lower Jaccard coefficient indicates less overlap in vocabulary and thus greater syntactic diversity. The Jaccard distance, defined as $1 - J(T_a, T_b)$, provides a complementary view, where a higher value signifies greater dissimilarity. By averaging the pairwise Jaccard distance across all response pairs, we can obtain an overall measure of syntactic diversity for the entire set of responses; see Figure~\ref{fig:diversity-semantic-syntax}.
\begin{figure}[ht!]
\centering
\begin{subfigure}[b]{0.25\linewidth}
\includegraphics[width=\linewidth]{img/diversity/diversity_syntax_aime_2025_1.7b_comparison.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:diversity-syntax}
\end{subfigure}%
\hfill %
\begin{subfigure}[b]{0.45\linewidth}
\includegraphics[width=\linewidth]{img/diversity/diversity_semantic_aime_2025_1.7b_comparison.pdf}
\caption{\scriptsize AIME 2025 (Qwen3-1.7B)}
\label{fig:diversity-semantic}
\end{subfigure}
\caption{Syntactic (left) and Semantic (right) Dissimilarity between selected trajectories using Particle Filtering and Entropic Particle Filtering. Entropic Particle Filtering tends to generate responses that are more syntactically and semantically diverse.}
\label{fig:diversity-semantic-syntax}
\end{figure}
\begin{figure}[ht!]
\centering
\includegraphics[width=1.00\linewidth]{img/diversity/diversity_semantic_l2_aime_2025_1.7b_comparison.pdf}
\caption{Semantic diversity measured using average pairwise $\ell_2$ distance across embeddings from three models. We compare Particle Filtering and Entropic Particle Filtering across all AIME 2025 questions using responses generated by the Qwen3-1.7B model. The analysis is based on the first $N$ tokens of each response. Entropic Particle Filtering tends to generate responses that are more semantically diverse according to the $\ell_2$ distance metric.}
\label{fig:diversity-l2}
\end{figure}
\paragraph{Semantic Diversity}
To quantify the semantic diversity among a set of $N=6$ generated responses, $\{r_1, r_2, \dots, r_N\}$, we introduce a Semantic Entropy score. First, each response $r_i$ is encoded into a normalized embedding vector $\mathbf{e}_i = \text{model}(r_i)$, where $\|\mathbf{e}_i\|_2 = 1$, using one of three sentence-embedding models: \texttt{all-mpnet-base-v2}, \texttt{BAAI/bge-large-en-v1.5}, or \texttt{BAAI/bge-m3}, which map text into a semantic vector space suitable for similarity comparisons. A pairwise cosine similarity matrix, $\mathbf{S}$, is then computed, where $S_{ij} = \mathbf{e}_i \cdot \mathbf{e}_j$. To ensure a response is not compared with itself, the diagonal elements of $\mathbf{S}$ are set to negative infinity, i.e., $S_{ii} = -\infty$. Next, a row-wise softmax function with a temperature parameter $\tau$ is applied to this matrix to obtain a probability distribution for each response over its neighbors: $p(j|i) = \frac{\exp(S_{ij}/\tau)}{\sum_{k \neq i} \exp(S_{ik}/\tau)}$. These individual distributions are then aggregated into a single mean probability distribution, $\bar{\mathbf{p}}$, where $\bar{p}_j = \frac{1}{N} \sum_{i=1}^{N} p(j|i)$. Finally, the Sem-Ent score is calculated as the Shannon entropy of this aggregated distribution, $H(\bar{\mathbf{p}}) = -\sum_{j=1}^{N} \bar{p}_j \log(\bar{p}_j)$, normalized by the maximum possible entropy for $N-1$ choices, which is $\log(N-1)$. The final score is clamped to the range $[0, 1]$. As a complementary measure, we also compute the average pairwise $\ell_2$ distance between response embeddings, again using the same three models. For each set of $N=6$ responses, we calculate the $\ell_2$ norm between every pair of embeddings and average across all $\binom{N}{2}$ pairs and all questions. A higher value indicates greater semantic diversity; see Figures~\ref{fig:diversity-semantic-syntax} and \ref{fig:diversity-l2}.
\section{Entropic Particle Filtering Pipeline}
\label{appx:pipeline}
\begin{figure}[ht]
\centering
\begin{subfigure}[b]{\linewidth}
\centering
\includegraphics[width=\textwidth]{img/intro/ea_lam_iclr.pdf}
\caption{Entropic Annealing (EA) and Look-ahead Modulation (LaM).}
\label{fig:ea-lam-intro-appx}
\end{subfigure}
\vspace{2em}
\begin{subfigure}{\linewidth}
\centering
\includegraphics[width=\linewidth]{img/intro/full_image_pipeline.pdf}
\caption{Entropic Particle Filtering (ePF).}
\label{fig:epf-intro-appx}
\end{subfigure}
\caption{Entropic Particle Filtering Pipeline (ePF; Fig.~\ref{fig:epf-intro-appx}). Our method iteratively refines a set of $N$ particles (states $\vz_t$) for a given prompt $\vc$. In each step $t$, particles are propagated by sampling a language model, $p(\vz_t | \vz_{t-1}, \vc)$, and then weighted by a reward model, $r(\vz_{1:t}, \vc)$. A resampling step, guided by these weights, approximates the posterior $p(\vz_t | \vo_t, \vc)$ to concentrate on promising candidates. After convergence, a final solution $\vx$ is decoded via $p(\vx | \vz_{1:T}, \vc)$. To prevent premature convergence, we introduce two complementary mechanisms: \emph{Entropic Annealing} (EA; Fig.~\ref{fig:ea-lam-intro-appx} left), which preserves diversity by dynamically annealing the resampling distribution $w_t$ based on entropy, and \emph{Look-ahead Modulation} (LaM; Fig.~\ref{fig:ea-lam-intro-appx} right), which steers the search toward more promising future states. This dual strategy guides the algorithm toward regions of both high reward and high diversity, effectively mitigating convergence to suboptimal local solutions.}
\label{fig:particle-filtering-pipeline-appx}
\end{figure}
\section{Algorithms}
\label{appx:algo}
\begin{algorithm}[ht!]
\caption{Particle Filtering (PF)}
\label{alg:pf}
\begin{algorithmic}[1]
\Require
\State $\vc$, $N$, $T$ \Comment{Task Input; Particle Budget; Steps}
\State $p(\vz_{t} | \vz_{t-1}, \vc)$ \Comment{Sampling/Dynamics Model}
\State $p(\vo_t | \vz_t, \vc) = \delta(s - g(\vz_t, \vc))$ \Comment{Reward/Observation Model}
\\
\State $\vz_1 \sim p(\vz_1 | \vc)$ \Comment{Sample initial step}
\State $w_1 \leftarrow 1/N$ \Comment{Initialize weights}
\State $\mathcal{P} = \{(\vz^{i}_1, w^{i}_1)\}^{N}_{i=1}$ \Comment{Initialize Particles}
\For{step $t=2$ to $T$}
\For{particle $i=1$ to $N$} \Comment{State Estimation step $t$}
\State $\vz^{i}_t \sim p(\vz_t | \vz_{t-1}, \vc)$ \Comment{Propagation}
\State $s^{i}_t \leftarrow g(\vz^{i}_{t:T}, \vc)$ \Comment{Scoring}
\State $r^{i}_t \leftarrow \log s^{i}_t $
\EndFor
\State $w^{i}_t \leftarrow \exp(r^{i}_t) / \sum^{N}_{j=1} \exp(r^{j}_t), \quad i=1,...,N$ \Comment{Compute Resampling Distribution}
\State $\{k_i\}^{N}_{i=1} \sim \texttt{Multinomial}\left[w_t\right]$
\Comment{Sample $N$ indexes with Replacement}
\State $\vz_t \leftarrow \vz_t\left[k_{1:N}\right]$ \Comment{Sample Selection}
\State $\mathcal{P} \leftarrow \mathcal{P} \cup \{(\vz_t, w_t)\}^{N}_{i=1}$ \Comment{Update Particles}
\EndFor
\\
\Return $\vx \sim p(\vx | \vz_{1:T}, \vc)$
\end{algorithmic}
\label{algo:pf}
\end{algorithm}
\clearpage
\begin{algorithm}[ht!]
\caption{Particle Filtering with Entropic Annealing (\texttt{ePF})}
\label{alg:epf}
\begin{algorithmic}[1]
\Require
\State $\vc$, $N$, $T$, $\tau$ \Comment{Task Input; Particle Budget; Steps; Threshold}
\State $p(\vz_{t} | \vz_{t-1}, \vc)$ \Comment{Sampling/Dynamics Model}
\State $p(\vo_t | \vz_t, \vc) = \delta(s - g(\vz_t, \vc))$ \Comment{Reward/Observation Model}
\\
\State $\vz_1 \sim p(\vz_1 | \vc)$ \Comment{Sample initial step}
\State $w_1 \leftarrow 1/N$ \Comment{Initialize weights}
\State $\mathcal{P} = \{(\vz^{i}_1, w^{i}_1)\}^{N}_{i=1}$ \Comment{Initialize Particles}
\For{step $t=2$ to $T$}
\For{particle $i=1$ to $N$} \Comment{State Estimation step $t$}
\State $\vz^{i}_t \sim p(\vz_t | \vz_{t-1}, \vc)$ \Comment{Propagation}
\State $s^{i}_t \leftarrow g(\vz^{i}_{t:T}, \vc)$ \Comment{Scoring}
\State $r^{i}_t \leftarrow \log s^{i}_t$
\EndFor
\State $w^{i}_t \leftarrow \exp(r^{i}_t) / \sum^{N}_{j=1} \exp(r^{j}_t), \quad i=1,...,N$ \Comment{Compute Resampling Distribution}
\State $ESS_{n}(t) = \sum^{N}_{j=1} ( w^{i}_t)^{-2} / N$ \Comment{Effective Sample Size Resampling Distribution}
\If{$ESS_n(t) < \tau$}
\State $\beta_t^{-1} = 1/ ESS_{n}(t) (1 - t/T)$ \Comment{Temperature Annealing}
\State $\beta_t^{-1} = \texttt{max}(\beta_t^{-1}, 1)$ \Comment{Bound Temperature}
\State $w^{i}_t \leftarrow \exp(r^{i}_t~\beta_t) / \sum^{N}_{j=1} \exp(r^{j}_t~\beta_t), \quad i=1,...,N$ \Comment{Update Resampling Distribution}
\EndIf
\State $\{k_i\}^{N}_{i=1} \sim \texttt{Systematic}\left[w_t\right]$
\Comment{Sample $N$ indexes with Stratified Replacement}
\State $\vz_t \leftarrow \vz_t\left[k_{1:N}\right]$ \Comment{Sample Selection}
\State $\mathcal{P} \leftarrow \mathcal{P} \cup \{(\vz_t, w_t)\}^{N}_{i=1}$ \Comment{Update Particles}
\EndFor
\\
\Return $\vx \sim p(\vx | \vz_{1:T}, \vc)$
\end{algorithmic}
\label{algo:epf}
\end{algorithm}
\clearpage
\begin{algorithm}[ht!]
\caption{Particle Filtering with Look-ahead Modulation (\texttt{PF w/ LaM})}
\label{alg:aepf}
\begin{algorithmic}[1]
\Require
\State $\vc$, $N$, $T$ \Comment{Task Input; Particle Budget; Steps}
\State $p(\vz_{t} | \vz_{t-1}, \vc)$ \Comment{Sampling/Dynamics Model}
\State $p(\vo_t | \vz_t, \vc) = \delta(s - g(\vz_t, \vc))$ \Comment{Reward/Observation Model}
\\
\State $\vz_1 \sim p(\vz_1 | \vc)$ \Comment{Sample initial step}
\State $w_1 \leftarrow 1/N$ \Comment{Initialize weights}
\State $\mathcal{P} = \{(\vz^{i}_1, w^{i}_1)\}^{N}_{i=1}$ \Comment{Initialize Particles}
\For{step $t=2$ to $T$}
\For{particle $i=1$ to $N$} \Comment{State Estimation step $t$}
\State $\vz^{i}_t \sim p(\vz_t | \vz_{t-1}, \vc)$ \Comment{Propagation}
\State $s^{i}_t \leftarrow g(\vz^{i}_{t:T}, \vc)$ \Comment{Scoring}
\State $r^{i}_t \leftarrow \log s^{i}_t$
\EndFor
\State $w^{i}_t \leftarrow \exp(r^{i}_t~\beta_t) / \sum^{N}_{j=1} \exp(r^{j}_t~\beta_t), \quad i=1,...,N$ \Comment{Resampling Distribution (\texttt{PF}/\texttt{ePF})}
\For{particle $i=1$ to $N$} \Comment{State Estimation step $t$}
\State $\vz^{i}_s \sim p(\vz_s | \vz_{t}, \vc)$ \Comment{Auxiliary Propagation (1-step)}
\State $\tilde r^{i}_s \leftarrow g(\vz^{i}_s, \vz^{i}_{t:T}, \vc)$ \Comment{Auxiliary Scoring}
\State $\tilde w^{i}_s \leftarrow w^{i}_t~\tilde r^{i}_s$ \Comment{Look-ahead Modulation}
\EndFor
\State $w^{i}_t \leftarrow \tilde w^{i}_s / \sum^{N}_{j=1} \tilde w^{j}_s, \quad i=1,...,N$ \Comment{Modulated Resampling Distribution}
\State $\{k_i\}^{N}_{i=1} \sim \texttt{Systematic}\left[w_t\right]$
\Comment{Sample $N$ indexes with Stratified Replacement}
\State $\vz_t \leftarrow \vz_t\left[k_{1:N}\right]$ \Comment{Sample Selection}
\State $\mathcal{P} \leftarrow \mathcal{P} \cup \{(\vz_t, w_t)\}^{N}_{i=1}$ \Comment{Update Particles}
\EndFor
\\
\Return $\vx \sim p(\vx | \vz_{1:T}, \vc)$
\end{algorithmic}
\label{algo:apf}
\end{algorithm}
\section{Qualitative Results}
\label{appx:qualitative}
We report a real example of early exploitation in Particle Filtering and exploration in Entropic Particle Filtering that guides the algorithms towards the correct solution for an AIME-2025 problem.
\begin{mdframed}[
linecolor=gray,
linewidth=2pt,
roundcorner=10pt,
backgroundcolor=lightgray!15,
frametitle={Problem (AIME-2025)},
frametitlerule=true,
frametitlebackgroundcolor=gray!20
]
The parabola with equation $y = x^2 - 4$ is rotated $60^\circ$ counterclockwise around the origin. The unique point in the fourth quadrant where the original parabola and its image intersect has $y$-coordinate $\frac{a - \sqrt{b}}{c}$, where $a$, $b$, and $c$ are positive integers, and $a$ and $c$ are relatively prime. Find $a + b + c$.
\end{mdframed}
\begin{mdframed}[
linecolor=gray,
linewidth=2pt,
roundcorner=10pt,
backgroundcolor=lightgray!15,
frametitle={Solution (Ground Truth)},
frametitlerule=true,
frametitlebackgroundcolor=gray!20
]
62
\end{mdframed}
\begin{mdframed}[
linecolor=blue,
linewidth=2pt,
roundcorner=10pt,
backgroundcolor=lightgray!15,
frametitle={Particle Filtering (3206 tokens)},
frametitlerule=true,
frametitlebackgroundcolor=blue!20
]
Okay, so I need to solve this problem where a parabola $y = x_2 - 4$ is rotated
60 degrees counterclockwise around the origin.
Then, I have to find the unique point in the fourth quadrant
where the original parabola and its image intersect.
The y-coordinate of this point is given in the form $(a - \sqrt{b})/c$,
and I need to find $a + b + c$.
Alright, let's start by understanding the problem step by step.
---
Step 1: Understand the original parabola
The original parabola is given by the equation:
$$
y = x^2 - 4
$$
This is a standard upward-opening parabola with vertex at $(0, -4)$.
It opens upwards and is symmetric about the y-axis.
---
Step 2: Understand the rotation
We are told that the parabola is rotated $60^\circ$ counterclockwise around the origin.
To rotate a point (x, y) counterclockwise by $\theta$, we use the rotation matrix:
$$
\begin{bmatrix}
\cos\theta & -\sin\theta \\
\sin\theta & \cos\theta
\end{bmatrix}
$$
So for $\theta = 60^\circ$, we have:
$$
\cos 60^\circ = \frac{1}{2}, \quad \sin 60^\circ = \frac{\sqrt{3}}{2}
$$
Thus, the rotation matrix becomes:
$$
\begin{bmatrix}
\frac{1}{2} & -\frac{\sqrt{3}}{2} \\
\frac{\sqrt{3}}{2} & \frac{1}{2}
\end{bmatrix}
$$
So, if a point (x, y) is rotated $60^\circ$ counterclockwise, the new coordinates (x', y') are:
$$
x' = \frac{1}{2}x - \frac{\sqrt{3}}{2}y \\
y' = \frac{\sqrt{3}}{2}x + \frac{1}{2}y
$$
---
Step 3: Find the image of the parabola after rotation
We want to find the equation of the image of the parabola $y = x_2 - 4$ after rotation.
Let’s denote the original coordinates as (x, y), and the rotated coordinates as (x', y').
Then:
$$
x' = \frac{1}{2}x - \frac{\sqrt{3}}{2}y \\
y' = \frac{\sqrt{3}}{2}x + \frac{1}{2}y
$$
We can solve these equations for x and y in terms of x' and y':
Let’s write them as a system:
$$
\begin{cases}
x' = \frac{1}{2}x - \frac{\sqrt{3}}{2}y \\
y' = \frac{\sqrt{3}}{2}x + \frac{1}{2}y
\end{cases}
$$
We can solve this system for x and y in terms of x' and y'.
Let me write the equations again:
$$
\frac{1}{2}x - \frac{\sqrt{3}}{2}y = x' \quad \text{(1)} \\
\frac{\sqrt{3}}{2}x + \frac{1}{2}y = y' \quad \text{(2)}
$$
Multiply both equations by 2 to eliminate denominators:
$$
x - \sqrt{3}y = 2x' \quad \text{(1')} \\
\sqrt{3}x + y = 2y' \quad \text{(2')}
$$
Now solve this system. Let’s solve for x and y.
From (1'):
$$
x = 2x' + \sqrt{3}y
$$
Substitute into (2'):
$$
\sqrt{3}(2x' + \sqrt{3}y) + y = 2y'
$$
$$
2\sqrt{3}x' + 3y + y = 2y'
$$
$$
2\sqrt{3}x' + 4y = 2y'
$$
Divide both sides by 2:
$$
\sqrt{3}x' + 2y = y'
$$
Solve for y:
$$
2y = y' - \sqrt{3}x' \Rightarrow y = \frac{y' - \sqrt{3}x'}{2}
$$
Now substitute back into x:
$$
x = 2x' + \sqrt{3} \cdot \frac{y' - \sqrt{3}x'}{2}
$$
$$
x = 2x' + \frac{\sqrt{3}y' - 3x'}{2}
$$
$$
x = \frac{4x' + \sqrt{3}y' - 3x'}{2} = \frac{x' + \sqrt{3}y'}{2}
$$
So the coordinates (x, y) in terms of (x', y') are:
$$
x = \frac{x' + \sqrt{3}y'}{2}, \quad y = \frac{y' - \sqrt{3}x'}{2}
$$
---
Step 4: Find the image of the parabola
We want to find the image of the parabola $y = x_2 - 4$ after rotation.
That is, for each point (x, y) on the original parabola,
we find its image (x', y') and then find the equation of the new curve.
Let’s write the original equation:
$$
y = x^2 - 4
$$
Now, we can express x and y in terms of x' and y' as above, and substitute into the original equation.
So:
$$
y = x^2 - 4 \Rightarrow \frac{y' - \sqrt{3}x'}{2} = \left( \frac{x' + \sqrt{3}y'}{2} \right)^2 - 4
$$
Let’s compute the right-hand side:
$$
\left( \frac{x' + \sqrt{3}y'}{2} \right)^2 = \frac{(x')^2 + 2\sqrt{3}x'y' + 3(y')^2}{4}
$$
So:
$$
\frac{y' - \sqrt{3}x'}{2} = \frac{(x')^2 + 2\sqrt{3}x'y' + 3(y')^2}{4} - 4
$$
Multiply both sides by 4 to eliminate denominators:
$$
2(y' - \sqrt{3}x') = (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16
$$
Simplify left-hand side:
$$
2y' - 2\sqrt{3}x' = (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16
$$
Now bring all terms to one side:
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16 - 2y' + 2\sqrt{3}x' = 0
$$
This is the equation of the rotated parabola.
So the image of the original parabola is:
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16 - 2y' + 2\sqrt{3}x' = 0
$$
---
Step 5: Find the intersection of the original and rotated parabola
We want to find the point where the original parabola and the rotated parabola intersect.
That is, we solve the system:
$$
y = x^2 - 4 \quad \text{(original)}
$$
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16 - 2y' + 2\sqrt{3}x' = 0 \quad \text{(rotated)}
$$
But since we are looking for the **unique point in the fourth quadrant**, we can assume that (x', y') lies in the fourth quadrant. So $x' > 0, y' < 0$.
But maybe it's easier to work in terms of (x, y), and find where
the original parabola and the rotated parabola intersect.
Alternatively, we can consider that the original parabola is $y = x_2 - 4$,
and the rotated parabola is the image of that.
But maybe we can think in terms of coordinates:
if we find the point (x, y) on the original parabola
such that $(x', y')$ is also on the rotated parabola, and that $(x', y')$ is the image of (x, y).
So, for a point (x, y) on the original parabola, its image (x', y') is on the rotated parabola.
So, the point (x, y) lies on the original parabola, and the point (x', y') lies on the rotated parabola.
So, we can write:
1. $ y = x^2 - 4 $
2. $ (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16 - 2y' + 2\sqrt{3}x' = 0 $
But since (x', y') is the image of (x, y), we can express x' and y' in terms of x and y.
From earlier, we had:
$$
x' = \frac{1}{2}x - \frac{\sqrt{3}}{2}y \\
y' = \frac{\sqrt{3}}{2}x + \frac{1}{2}y
$$
So, substituting these into the rotated parabola equation:
\begin{equation*}
\begin{split}
\left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right)^2+ 2\sqrt{3} \left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right) \left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right) + \\
3 \left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right)^2 - 16 - 2\left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right) + 2\sqrt{3} \left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right) = 0
\end{split}
\end{equation*}
This looks complicated, but we can try to simplify step by step.
Let me denote:
Let’s compute each term:
1. First term: $ \left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right)^2 = \frac{1}{4}x^2 - \frac{\sqrt{3}}{2}xy + \frac{3}{4}y^2 $
2. Second term: $ 2\sqrt{3} \cdot \left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right) \left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right) $
Let me compute the product inside:
$$
\left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right) \left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right) = \frac{1}{2}x \cdot \frac{\sqrt{3}}{2}x + \frac{1}{2}x \cdot \frac{1}{2}y - \frac{\sqrt{3}}{2}y \cdot \frac{\sqrt{3}}{2}x - \frac{\sqrt{3}}{2}y \cdot \frac{1}{2}y
$$
$$
= \frac{\sqrt{3}}{4}x^2 + \frac{1}{4}xy - \frac{3}{4}xy - \frac{\sqrt{3}}{4}y^2
$$
$$
= \frac{\sqrt{3}}{4}x^2 - \frac{1}{2}xy - \frac{\sqrt{3}}{4}y^2
$$
Multiply by $ 2\sqrt{3} $:
$$
2\sqrt{3} \cdot \left( \frac{\sqrt{3}}{4}x^2 - \frac{1}{2}xy - \frac{\sqrt{3}}{4}y^2 \right) = \frac{2\sqrt{3} \cdot \sqrt{3}}{4}x^2 - \frac{2\sqrt{3}}{2}xy - \frac{2\sqrt{3} \cdot \sqrt{3}}{4}y^2
$$
$$
= \frac{6}{4}x^2 - \sqrt{3}xy - \frac{6}{4}y^2 = \frac{3}{2}x^2 - \sqrt{3}xy - \frac{3}{2}y^2
$$
3. Third term: $ 3 \left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right)^2 = 3 \left( \frac{3}{4}x^2 + \frac{\sqrt{3}}{2}xy + \frac{1}{4}y^2 \right) = \frac{9}{4}x^2 + \frac{3\sqrt{3}}{2}xy + \frac{3}{4}y^2 $
4. Fourth term: $ -16 $
5. Fifth term: $ -2\left( \frac{\sqrt{3}}{2}x + \frac{1}{2}y \right) = -\sqrt{3}x - y $
6. Sixth term: $ 2\sqrt{3} \left( \frac{1}{2}x - \frac{\sqrt{3}}{2}y \right) = \sqrt{3}x - 3y $
[\texttt{the algorithm generates the stop token and does not find a solution}]
\end{mdframed}
\begin{mdframed}[
linecolor=orange,
linewidth=2pt,
roundcorner=10pt,
backgroundcolor=lightgray!15,
frametitle={Entropic Particle Filtering (5472 tokens)
},
frametitlerule=true,
frametitlebackgroundcolor=orange!20
]
Okay, so I need to solve this problem where a parabola $y = x_2 - 4$ is rotated 60 degrees counterclockwise around the origin. Then, I have to find the unique point in the fourth quadrant where the original parabola and its image intersect. The y-coordinate of this point is given in the form $(a - \sqrt{b})/c$, where a, b, c are positive integers, a and c are coprime, and then find $a + b + c$.
First, let me recall what happens when a graph is rotated around the origin. Rotating a graph 60 degrees counterclockwise involves applying a rotation matrix to the coordinates of each point on the graph.
The rotation matrix for counterclockwise rotation by $\theta$ is:
$$
\begin{pmatrix}
\cos\theta & -\sin\theta \\
\sin\theta & \cos\theta
\end{pmatrix}
$$
For $\theta = 60^\circ, cos(60^\circ) = 0.5, sin(60^\circ) = \sqrt{3}/2$.
So the rotation matrix becomes:
$$
\begin{pmatrix}
\frac{1}{2} & -\frac{\sqrt{3}}{2} \\
\frac{\sqrt{3}}{2} & \frac{1}{2}
\end{pmatrix}
$$
This matrix takes a point (x, y) and maps it to a new point (x', y'):
$$
x' = \frac{1}{2}x - \frac{\sqrt{3}}{2}y \\
y' = \frac{\sqrt{3}}{2}x + \frac{1}{2}y
$$
Now, the original parabola is $y = x_2 - 4$.
We need to find the image of this parabola under rotation. So, we can think of the image of the parabola as the set of points $(x', y')$ such that there exists some (x, y) on the original parabola, and $(x', y')$ is obtained by rotating (x, y) by $60^\circ$ counterclockwise.
But to find the equation of the image of the parabola, we need to express x and y in terms of x' and y', and then substitute into the original equation.
Let me denote:
Let $(x, y)$ be a point on the original parabola, so $y = x^2 - 4$.
After rotation, $(x', y') = ( (1/2)x - (\sqrt{3}/2)y, (\sqrt{3}/2)x + (1/2)y )$
We can solve this system for x and y in terms of x' and y'.
Let me write the equations:
1. $ x' = \frac{1}{2}x - \frac{\sqrt{3}}{2}y $
2. $ y' = \frac{\sqrt{3}}{2}x + \frac{1}{2}y $
Let me write this as a system of linear equations:
$$
\begin{cases}
\frac{1}{2}x - \frac{\sqrt{3}}{2}y = x' \\
\frac{\sqrt{3}}{2}x + \frac{1}{2}y = y'
\end{cases}
$$
To solve this system, we can use substitution or matrix methods. Let me solve for x and y in terms of x' and y'.
Let me multiply the first equation by 2 to eliminate denominators:
1. $ x - \sqrt{3}y = 2x' $
2. $ \sqrt{3}x + y = 2y' $
Now we have:
1. $ x - \sqrt{3}y = 2x' $
2. $ \sqrt{3}x + y = 2y' $
Let me solve this system using substitution or elimination.
Let me solve equation 1 for x:
From equation 1:
$ x = 2x' + \sqrt{3}y $
Substitute into equation 2:
$ \sqrt{3}(2x' + \sqrt{3}y) + y = 2y' $
Compute:
$ 2\sqrt{3}x' + 3y + y = 2y' $
Combine like terms:
$ 2\sqrt{3}x' + 4y = 2y' $
Divide both sides by 2:
$ \sqrt{3}x' + 2y = y' $
Solve for y:
$ 2y = y' - \sqrt{3}x' $
$ y = \frac{y' - \sqrt{3}x'}{2} $
Now substitute back into x:
From equation 1:
$ x = 2x' + \sqrt{3}y $
Substitute y:
$ x = 2x' + \sqrt{3} \cdot \frac{y' - \sqrt{3}x'}{2} $
Simplify:
$ x = 2x' + \frac{\sqrt{3}y' - 3x'}{2} $
Combine terms:
$ x = \frac{4x' + \sqrt{3}y' - 3x'}{2} = \frac{x' + \sqrt{3}y'}{2} $
So we have expressions for x and y in terms of x' and y':
$$
x = \frac{x' + \sqrt{3}y'}{2}, \quad y = \frac{y' - \sqrt{3}x'}{2}
$$
Now, since the original parabola is $y = x_2 - 4$, we can substitute these expressions into that equation.
Let me write that:
Original equation: $ y = x^2 - 4 $
Substitute:
$$
\frac{y' - \sqrt{3}x'}{2} = \left( \frac{x' + \sqrt{3}y'}{2} \right)^2 - 4
$$
Let me simplify this equation.
First, compute the right-hand side:
$$
\left( \frac{x' + \sqrt{3}y'}{2} \right)^2 = \frac{(x')^2 + 2\sqrt{3}x'y' + 3(y')^2}{4}
$$
So:
$$
\frac{y' - \sqrt{3}x'}{2} = \frac{(x')^2 + 2\sqrt{3}x'y' + 3(y')^2}{4} - 4
$$
Multiply both sides by 4 to eliminate denominators:
$$
2(y' - \sqrt{3}x') = (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16
$$
Expand the left side:
$$
2y' - 2\sqrt{3}x' = (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16
$$
Bring all terms to one side:
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 - 16 - 2y' + 2\sqrt{3}x' = 0
$$
Now, group like terms:
- Terms with x'²: $ (x')^2 $
- Terms with y'²: $ 3(y')^2 $
- Terms with x'y': $ 2\sqrt{3}x'y' $
- Terms with x': $ 2\sqrt{3}x' $
- Terms with y': $ -2y' $
- Constant term: $ -16 $
So the equation becomes:
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0
$$
This is the equation of the image of the parabola. Now, we need to find the point of intersection between the original parabola and its image. So we need to solve this equation together with the original equation $y = x_2 - 4$. But we already used the original equation in the derivation. So the image of the parabola is this equation, and the original is $y = x_2 - 4$.
But since we are looking for the intersection of the original parabola and its image, we can set the two expressions for y equal?
Wait, but the image is a rotated version of the original. So the original parabola and its image intersect where the original parabola is mapped to itself.
Alternatively, perhaps it's easier to consider that the intersection points are points that are on both the original and the image parabola.
So, the original parabola is $y = x_2 - 4$.
The image of the parabola is given by the equation we derived:
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0
$$
But since $(x', y')$ are the coordinates of the image, and the original coordinates are $(x, y)$, and we have the expressions for x and y in terms of $(x', y')$, we can also write the original parabola equation as $y = x_2 - 4$. So, we can use the expressions for x and y in terms of x' and y' to substitute into the original equation.
Wait, but we already did that earlier and ended up with the above equation.
But actually, the image is a rotated version of the original, so the intersection points are points that are on both the original and the rotated parabola.
So the intersection points satisfy both:
1. $y = x^2 - 4$
2. The rotated equation: $(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0$
But since we have expressions for x and y in terms of x' and y', we can write:
Original parabola: $ y = x^2 - 4 $
Image parabola: $ (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0 $
But since (x', y') are coordinates of the image, and (x, y) are coordinates of the original, so we can write:
From the rotation, we have:
$$
x = \frac{x' + \sqrt{3}y'}{2}, \quad y = \frac{y' - \sqrt{3}x'}{2}
$$
So substituting these into the original equation $y = x_2 - 4$ gives the equation we had earlier.
So we can now solve the equation:
$$
\frac{y' - \sqrt{3}x'}{2} = \left( \frac{x' + \sqrt{3}y'}{2} \right)^2 - 4
$$
Which we simplified earlier to:
$$
(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0
$$
But this is the equation of the image of the parabola. So the intersection points are the points that satisfy both the original parabola and this rotated equation.
But since we already used the original equation to derive this, the only way to find the intersection points is to solve the system:
1. $ y = x^2 - 4 $
2. $ (x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0 $
But since (x', y') are coordinates of the image, and (x, y) are coordinates of the original, and we have expressions for x and y in terms of x' and y', we can substitute those into the original equation.
But that's what we already did.
Alternatively, perhaps it's easier to think of this as a system of equations where we have the original parabola and the image of the parabola, and we need to find their intersection points.
Alternatively, since the image of the parabola is a rotated version, and the original parabola is $y = x_2 - 4$, the intersection points are points that lie on both.
But to find the intersection points, we can think of solving the system:
1. $y = x_2 - 4$
2. The rotated parabola equation: $(x')^2 + 2\sqrt{3}x'y' + 3(y')^2 + 2\sqrt{3}x' - 2y' - 16 = 0$
But since (x', y') are coordinates of the image, and (x, y) are coordinates of the original, and we have expressions for x and y in terms of x' and y', we can substitute those expressions into the original equation.
Wait, but that's what we did earlier.
Alternatively, maybe I can think of this as a system of equations where we have the original parabola and the image parabola.
But since the image is a rotated version, the intersection points can be found by solving the system:
Original: $ y = x^2 - 4 $
Image: $ y = x^2 - 4 $ rotated by $60^\circ$
But how do we find the image of the parabola? It's the set of points (x', y') such that there exists (x, y) on the original parabola, and (x', y') is the image of (x, y).
So, we can parametrize the original parabola.
Let me try a different approach. Since the original parabola is $y = x_2 - 4$, we can parametrize it as $(x, x_2 - 4)$, for all real x.
Then, the image of this parametrized point under rotation by $60^\circ$ counterclockwise is:
Let’s call the original point $(x, y) = (x, x_2 - 4)$. Then, the image is:
$x' = (1/2)x - (\sqrt{3}/2)y$
$y' = (\sqrt{3}/2)x + (1/2)y$
So, substituting $y = x_2 - 4$:
$x' = (1/2)x - (\sqrt{3}/2)(x_2 - 4)$
$y' = (\sqrt{3}/2)x + (1/2)(x_2 - 4)$
Then, we can write x' and y' in terms of x. Then, the image of the parabola is the set of points (x', y') where x = x, $y = x_2 - 4$, and the above expressions for x' and y'.
So, the image of the parabola is given by:
$x' = (1/2)x - (\sqrt{3}/2)(x_2 - 4)$
$y' = (\sqrt{3}/2)x + (1/2)(x_2 - 4)$
So, the image of the parabola is the set of points (x', y') where x' and y' are expressed in terms of x.
So, to find the intersection between the original and the image parabola, we can set the original parabola equation $y = x_2 - 4$ equal to the image parabola equation.
But since the image is parametrized by x, the image parabola is:
$y' = (\sqrt{3}/2)x + (1/2)(x_2 - 4) = (1/2)x_2 + (\sqrt{3}/2)x - 2$
$So, the image of the parabola is y' = (1/2)x_2 + (\sqrt{3}/2)x - 2$
So, to find the intersection points, we need to set the original $y = x_2 - 4$ equal to the image
$y' = (1/2)x_2 + (\sqrt{3}/2)x - 2$.
So:
$x_2 - 4 = (1/2)x_2 + (\sqrt{3}/2)x - 2$
Bring all terms to one side:
$x_2 - 4 - (1/2)x_2 - (\sqrt{3}/2)x + 2 = 0$
Simplify:
$(1/2)x_2 - (\sqrt{3}/2)x - 2 = 0$
Multiply both sides by 2 to eliminate denominators:
$x_2 - \sqrt{3} x - 4 = 0$
So, we have a quadratic equation in x:
$x_2 - \sqrt{3} x - 4 = 0$
Solve this quadratic equation using the quadratic formula:
$ x = [\sqrt{3} \pm \sqrt{( (\sqrt{3})^2 + 16 )}]/2 $
Compute discriminant:
$(\sqrt{3})^2 + 16 = 3 + 16 = 19$
So,
$x = [\sqrt{3} \pm \sqrt{19}]/2$
So the x-coordinates of the intersection points are
$( [\sqrt{3} + \sqrt{19}]/2 , [\sqrt{3} - \sqrt{19}]/2 )$ and $( [\sqrt{3} - \sqrt{19}]/2 , [\sqrt{3} + \sqrt{19}]/2 )$.
But we are interested in the point in the fourth quadrant, so we need to check which of these x and y coordinates are in the fourth quadrant.
Fourth quadrant is where x > 0 and y < 0.
Let me compute the two x-values:
1. $x_1 = [\sqrt{3} + \sqrt{19}]/2 \approx [1.732 + 4.358]/2 \approx 6.09/2 \approx 3.045 > 0$
2. $x_2 = [\sqrt{3} - \sqrt{19}]/2 \approx [1.732 - 4.358]/2 \approx (-2.626)/2 \approx -1.313 < 0$
So, only $x_1$ is positive, and $x_2$ is negative. So the x-coordinate of the point in the fourth quadrant is $x_1 = [\sqrt{3} + \sqrt{19}]/2$.
Now, let's compute the corresponding y-coordinate for this $x$. Since the original parabola is $y = x_2 - 4$.
So, $y = ([\sqrt{3} + \sqrt{19}]/2)^2 - 4$
Compute that:
First, compute $([\sqrt{3} + \sqrt{19}]/2)^2$:
$= [ (\sqrt{3})^2 + 2\sqrt{3}\sqrt{19} + (\sqrt{19})^2 ] / 4 = [ 3 + 2\sqrt{57} + 19 ] / 4 = [22 + 2\sqrt{57}]/4 = [11 + \sqrt{57}]/2$
Then subtract 4:
$y = [11 + \sqrt{57}]/2 - 4 = [11 + \sqrt{57} - 8]/2 = [3 + \sqrt{57}]/2$
Wait, but this is positive, and we are looking for a point in the fourth quadrant, where y < 0.
But this result is positive, so this can't be the correct intersection point. So there must be a mistake here.
Wait, but we set the original $y = x_2 - 4$ equal to the image $y' = (1/2)x_2 + (\sqrt{3}/2)x - 2$, which is the image of the parabola.
But we got two solutions for x, one positive and one negative, but only the positive one gives a positive y. However, we need the point in the fourth quadrant, where y < 0.
Wait, so maybe we made a mistake in the sign when setting up the equations?
Let me double-check the image y':
Original $y = x_2 - 4$
Image $y' = (\sqrt{3}/2)x + (1/2)(x_2 - 4) = (1/2)x_2 + (\sqrt{3}/2)x - 2$
Yes, that's correct.
So setting original y = image y:
$x_2 - 4 = (1/2)x_2 + (\sqrt{3}/2)x - 2$
Bring everything to the left:
$x_2 - 4 - (1/2)x_2 - (\sqrt{3}/2)x + 2 = 0$
Simplify:
$(1/2)x_2 - (\sqrt{3}/2)x - 2 = 0$
Multiply by 2:
$x_2 - \sqrt{3} x - 4 = 0$
That's correct.
So solutions:
$$
x = [\sqrt{3} \pm \sqrt{( (\sqrt{3})^2 + 16 )}]/2 = [\sqrt{3} \pm \sqrt{(3 + 16)}]/2 = [\sqrt{3} \pm \sqrt{19}]/2
$$
So $x_1 = [\sqrt{3} + \sqrt{19}]/2 \approx$ positive, and $x_2 = [\sqrt{3} - \sqrt{19}]/2 \approx$ negative.
But then, when we compute $y = x_2 - 4$ for $x_1$, we get a positive y, and for $x_2$, we get:
$y = x^2_2 - 4$
But $x_2$ is negative, so $x^2_2$ is positive, so y = positive - 4.
But we need to check if this is negative.
Let me compute y for $x_2 = [\sqrt{3} - \sqrt{19}]/2$:
First, compute $x^2_2$:
$x_2 = [\sqrt{3} - \sqrt{19}]/2$
$x^2_2 = [ (\sqrt{3} - \sqrt{19})^2 ] / 4 = [3 - 2\sqrt{57} + 19]/4 = [22 - 2\sqrt{57}]/4 = [11 - \sqrt{57}]/2$
Then y = $x^2_2 - 4 = [11 - \sqrt{57}]/2 - 4 = [11 - \sqrt{57} - 8]/2 = [3 - \sqrt{57}]/2$
Now, $\sqrt{57} \approx 7.55$, so $3 - \sqrt{57} \approx 3 - 7.55 = -4.55, so y \approx -4.55/2 \approx -2.275$, which is negative.
So, the point with $x = [\sqrt{3} - \sqrt{19}]/2 and y = [3 - \sqrt{57}]/2$ is in the fourth quadrant.
Therefore, the unique point in the fourth quadrant is $(x, y) = ([\sqrt{3} - \sqrt{19}]/2, [3 - \sqrt{57}]/2)$
But the question says the y-coordinate is of the form $(a - \sqrt{b})/c$, where a, b, c are positive integers, a and c are coprime.
So let's write the y-coordinate:
$y = [3 - \sqrt{57}]/2$
So here, $a = 3$, $b = 57$, $c = 2$.
But 3 and 2 are coprime. So $a + b + c = 3 + 57 + 2 = 62$.
But let me check if that's correct. Wait, the question says "the unique point in the fourth quadrant where the original parabola and its image intersect has y-coordinate $(a - \sqrt{b})/c$".
So in our case, the y-coordinate is $[3 - \sqrt{57}]/2$, so a = 3, b = 57, c = 2.
But 3 and 2 are coprime, so that's good.
Therefore, the answer is $3 + 57 + 2 = 62$.
But let me check once again to make sure I didn't make any mistakes.
Original parabola: $y = x_2 - 4$
Image parabola: $y' = (1/2)x_2 + (\sqrt{3}/2)x - 2$
Set equal:
$x_2 - 4 = (1/2)x_2 + (\sqrt{3}/2)x - 2$
Multiply by 2:
$2x_2 - 8 = x_2 + \sqrt{3} x - 4$
Bring all terms to left:
$2x_2 - 8 - x_2 - \sqrt{3} x + 4 = 0 \rightarrow x_2 - \sqrt{3} x - 4 = 0$
Solutions: $x = [\sqrt{3} \pm \sqrt{(3 + 16)}]/2 = [\sqrt{3} \pm \sqrt{19}]/2$
For $x = [\sqrt{3} - \sqrt{19}]/2$, compute $y = x_2 - 4$:
$x = [\sqrt{3} - \sqrt{19}]/2$
$x_2 = [ (\sqrt{3} - \sqrt{19})^2 ] / 4 = [3 - 2\sqrt{57} + 19]/4 = [22 - 2\sqrt{57}]/4 = [11 - \sqrt{57}]/2$
Then $y = [11 - \sqrt{57}]/2 - 4 = [11 - \sqrt{57} - 8]/2 = [3 - \sqrt{57}]/2$
Yes, so that is correct.
So the y-coordinate is $[3 - \sqrt{57}]/2$, so a = 3, b = 57, c = 2, and they are coprime.
Thus, $a + b + c = 3 + 57 + 2 = 62$.
Final Answer
\boxed{62}
\end{mdframed}
\section{Details}
\label{appx:details}
\begin{table}[h]
\centering
\caption{Models.}
\begin{tabular}{lccc}
\toprule
Models & Size & Mode & Hugging Face Path \\
\midrule
Qwen2.5 & 1.5B & Instruct & \texttt{Qwen/Qwen2.5-1.5B-Instruct} \\
Qwen2.5 & 7B & Instruct & \texttt{Qwen/Qwen2.5-7B-Instruct} \\
\midrule
Qwen3 & 0.6B & w/o Thinking & \texttt{Qwen/Qwen3-0.6B} \\
Qwen3 & 1.7B & w/o Thinking & \texttt{Qwen/Qwen3-1.7B} \\
\midrule
Llama3.2 & 1B & Instruct & \texttt{meta-llama/Llama-3.2-1B-Instruct} \\
Llama3.1 & 8B & Instruct & \texttt{meta-llama/Llama-3.1-8B-Instruct} \\
\bottomrule
\end{tabular}
\label{tab:models}
\end{table}
\paragraph{Datasets}
We focus our attention on six math benchmarks.
GSM8K: A dataset of 8,500 high-quality, linguistically diverse grade school math word problems created to test multi-step reasoning.
MATH500: A collection of 500 challenging high-school competition math problems, each with detailed step-by-step solutions.
DEEPMATH: A dataset of advanced mathematical problems and their corresponding solutions.
OMNIMATH: A large-scale, multilingual benchmark featuring math problems covering a vast spectrum of mathematical topics and complexities.
AIME 2024: A dataset containing the 30 problems from the two AIME I and AIME II competitions held in 2024. It is not formally split into training and testing sets. Instead, it serves as a direct evaluation set to test the mathematical reasoning capabilities of advanced AI models.
AIME 2025: Similar to its predecessor, the AIME25 dataset is comprised of the problems from the 2025 AIME competitions and is also a smaller, focused dataset used for benchmarking. Each sample in these datasets includes the problem text, a detailed solution, and the final numerical answer.
\begin{table}[h]
\centering
\caption{Datasets.}
\begin{tabular}{lcccc}
\toprule
Dataset & Split & Sample & Hugging Face Path\\
\midrule
GSM8K & test (subset) & 500 & \texttt{openai/gsm8k-main} \\
MATH500 & test & 500 & \texttt{HuggingFaceH4/MATH-500} \\
DEEPMATH & subset & 500 & \texttt{zwhe99/DeepMath-103K} \\
OMNIMATH & test (subset) & 500 & \texttt{KbsdJames/Omni-MATH} \\
\midrule
AIME 2024 & train & 30 & \texttt{Maxwell-Jia/AIME\_2024} \\
AIME 2025 & train & 30 & \texttt{MathArena/aime\_2025} \\
AIME-24-25 & train & 60 & \texttt{Maxwell-Jia/AIME\_2024}, \texttt{MathArena/aime\_2025}\\
\bottomrule
\end{tabular}
\label{tab:datasets}
\end{table}
\begin{table}[ht!]
\centering
\caption{Hyper-parameters for the main experiments.}
\resizebox{\textwidth}{!}{%
\begin{tabular}{l c c c c}
\toprule
& Qwen2.5-1.5B-Instruct & Qwen2.5-7B-Instruct & Qwen3-0.6B & Qwen3-1.7B \\
\midrule
Budgets $N$ & 2,4,8,16,32 & 2,4,8,16,32 & 2,4,8,16,32 & 2,4,8,16,32 \\
PRM & Qwen2.5-Math-PRM-7B & Qwen2.5-Math-PRM-7B & Qwen2.5-Math-PRM-7B & Qwen2.5-Math-PRM-7B \\
Threshold $\tau$ & 0.5 & 0.5 & 0.5 & 0.5 \\
Repetition Penalty
Temperature & 0.7 & 0.7 & 0.7 & 0.7 \\
TopK & 20 & 20 & 20 & 20 \\
TopP & 0.8 & 0.8 & 0.8 & 0.8 \\
Max Steps & 300 & 300 & 300 & 300 \\
Max Tokens/Step & 512 & 512 & 512 & 512 \\
Max Seq Tokens & 4k/12k & 4k/12k & 4k/12k & 4k/12k \\
\bottomrule
\end{tabular}
}
\label{tab:hparams}
\end{table}
\begin{thebibliography}{76}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Ahmed et~al.(2016)Ahmed, Fuge, and Gorbunov]{ahmed2016discovering}
Faez Ahmed, Mark Fuge, and Lev~D Gorbunov.
\newblock Discovering diverse, high quality design ideas from a large corpus.
\newblock In \emph{International Design Engineering Technical Conferences and Computers and Information in Engineering Conference}, volume 50190, pp.\ V007T06A008. American Society of Mechanical Engineers, 2016.
\bibitem[Amini et~al.(2024)Amini, Vieira, Ash, and Cotterell]{amini2024variational}
Afra Amini, Tim Vieira, Elliott Ash, and Ryan Cotterell.
\newblock Variational best-of-n alignment.
\newblock \emph{arXiv preprint arXiv:2407.06057}, 2024.
\bibitem[Beeching et~al.(2024)Beeching, Tunstall, and Rush]{beeching2024scalingtesttimecompute}
Edward Beeching, Lewis Tunstall, and Sasha Rush.
\newblock Scaling test-time compute with open models, 2024.
\newblock URL \url{https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute}.
\bibitem[Binder et~al.(1992)Binder, Heermann, and Binder]{binder1992monte}
Kurt Binder, Dieter~W Heermann, and K~Binder.
\newblock \emph{Monte Carlo simulation in statistical physics}, volume~8.
\newblock Springer, 1992.
\bibitem[Brown et~al.(2024)Brown, Juravsky, Ehrlich, Clark, Le, R{\'e}, and Mirhoseini]{brown2024large}
Bradley Brown, Jordan Juravsky, Ryan Ehrlich, Ronald Clark, Quoc~V Le, Christopher R{\'e}, and Azalia Mirhoseini.
\newblock Large language monkeys: Scaling inference compute with repeated sampling.
\newblock \emph{arXiv preprint arXiv:2407.21787}, 2024.
\bibitem[Casella \& Berger(2024)Casella and Berger]{casella2024statistical}
George Casella and Roger Berger.
\newblock \emph{Statistical inference}.
\newblock Chapman and Hall/CRC, 2024.
\bibitem[Chatzilygeroudis et~al.(2021)Chatzilygeroudis, Cully, Vassiliades, and Mouret]{chatzilygeroudis2021quality}
Konstantinos Chatzilygeroudis, Antoine Cully, Vassilis Vassiliades, and Jean-Baptiste Mouret.
\newblock Quality-diversity optimization: a novel branch of stochastic optimization.
\newblock In \emph{Black box optimization, machine learning, and no-free lunch theorems}, pp.\ 109--135. Springer, 2021.
\bibitem[Chen et~al.(2023)Chen, Aksitov, Alon, Ren, Xiao, Yin, Prakash, Sutton, Wang, and Zhou]{chen2023universal}
Xinyun Chen, Renat Aksitov, Uri Alon, Jie Ren, Kefan Xiao, Pengcheng Yin, Sushant Prakash, Charles Sutton, Xuezhi Wang, and Denny Zhou.
\newblock Universal self-consistency for large language model generation.
\newblock \emph{arXiv preprint arXiv:2311.17311}, 2023.
\bibitem[Chen et~al.(2016)Chen, Lu, and Wei]{chen2016topology}
Yonghua Chen, Jianan Lu, and Ying Wei.
\newblock Topology optimization for manufacturability based on the visibility map.
\newblock \emph{Computer-Aided Design and Applications}, 13\penalty0 (1):\penalty0 86--94, 2016.
\bibitem[Cobbe et~al.(2021)Cobbe, Kosaraju, Bavarian, Chen, Jun, Kaiser, Plappert, Tworek, Hilton, Nakano, et~al.]{cobbe2021training}
Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, et~al.
\newblock Training verifiers to solve math word problems.
\newblock \emph{arXiv preprint arXiv:2110.14168}, 2021.
\bibitem[Doucet et~al.(2001{\natexlab{a}})Doucet, De~Freitas, and Gordon]{doucet2001introduction}
Arnaud Doucet, Nando De~Freitas, and Neil Gordon.
\newblock An introduction to sequential monte carlo methods.
\newblock In \emph{Sequential Monte Carlo methods in practice}, pp.\ 3--14. Springer, 2001{\natexlab{a}}.
\bibitem[Doucet et~al.(2001{\natexlab{b}})Doucet, De~Freitas, Gordon, et~al.]{doucet2001sequential}
Arnaud Doucet, Nando De~Freitas, Neil~James Gordon, et~al.
\newblock \emph{Sequential Monte Carlo methods in practice}, volume~1.
\newblock Springer, 2001{\natexlab{b}}.
\bibitem[Feng et~al.(2024)Feng, Kong, Ma, Zhang, Yin, Wang, Pang, and Yang]{feng2024step}
Shengyu Feng, Xiang Kong, Shuang Ma, Aonan Zhang, Dong Yin, Chong Wang, Ruoming Pang, and Yiming Yang.
\newblock Step-by-step reasoning for math problems via twisted sequential monte carlo.
\newblock \emph{arXiv preprint arXiv:2410.01920}, 2024.
\bibitem[Foerster et~al.(2018)Foerster, Farquhar, Al-Shedivat, Rockt{\"a}schel, Xing, and Whiteson]{foerster2018dice}
Jakob Foerster, Gregory Farquhar, Maruan Al-Shedivat, Tim Rockt{\"a}schel, Eric Xing, and Shimon Whiteson.
\newblock Dice: The infinitely differentiable monte carlo estimator.
\newblock In \emph{International Conference on Machine Learning}, pp.\ 1529--1538. PMLR, 2018.
\bibitem[Gao et~al.(2024)Gao, Song, Yang, Cai, Miao, Dong, Li, Ma, Chen, Xu, et~al.]{gao2024omni}
Bofei Gao, Feifan Song, Zhe Yang, Zefan Cai, Yibo Miao, Qingxiu Dong, Lei Li, Chenghao Ma, Liang Chen, Runxin Xu, et~al.
\newblock Omni-math: A universal olympiad level mathematic benchmark for large language models.
\newblock \emph{arXiv preprint arXiv:2410.07985}, 2024.
\bibitem[Gordon et~al.(1993)Gordon, Salmond, and Smith]{gordon1993novel}
Neil~J Gordon, David~J Salmond, and Adrian~FM Smith.
\newblock Novel approach to nonlinear/non-gaussian bayesian state estimation.
\newblock In \emph{IEE proceedings F (radar and signal processing)}, volume 140, pp.\ 107--113. IET, 1993.
\bibitem[Guo et~al.(2025)Guo, Yang, Zhang, Song, Zhang, Xu, Zhu, Ma, Wang, Bi, et~al.]{guo2025deepseek}
Daya Guo, Dejian Yang, Haowei Zhang, Junxiao Song, Ruoyu Zhang, Runxin Xu, Qihao Zhu, Shirong Ma, Peiyi Wang, Xiao Bi, et~al.
\newblock Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning.
\newblock \emph{arXiv preprint arXiv:2501.12948}, 2025.
\bibitem[Haarnoja et~al.(2018)Haarnoja, Zhou, Abbeel, and Levine]{haarnoja2018soft}
Tuomas Haarnoja, Aurick Zhou, Pieter Abbeel, and Sergey Levine.
\newblock Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor.
\newblock In \emph{International conference on machine learning}, pp.\ 1861--1870. Pmlr, 2018.
\bibitem[Hastings(1970)]{hastings1970monte}
W~Keith Hastings.
\newblock Monte carlo sampling methods using markov chains and their applications.
\newblock 1970.
\bibitem[He et~al.(2025)He, Liang, Xu, Liu, Chen, Wang, Song, Yu, Liang, Wang, et~al.]{he2025deepmath}
Zhiwei He, Tian Liang, Jiahao Xu, Qiuzhi Liu, Xingyu Chen, Yue Wang, Linfeng Song, Dian Yu, Zhenwen Liang, Wenxuan Wang, et~al.
\newblock Deepmath-103k: A large-scale, challenging, decontaminated, and verifiable mathematical dataset for advancing reasoning.
\newblock \emph{arXiv preprint arXiv:2504.11456}, 2025.
\bibitem[Hendrycks et~al.(2024)Hendrycks, Burns, Kadavath, Arora, Basart, Tang, Song, and Steinhardt]{hendrycks2024measuring}
Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, and Jacob Steinhardt.
\newblock Measuring mathematical problem solving with the math dataset, 2021.
\newblock \emph{URL https://arxiv. org/abs/2103.03874}, 2, 2024.
\bibitem[Hern{\'a}ndez-Lobato et~al.(2016)Hern{\'a}ndez-Lobato, Hernandez-Lobato, Shah, and Adams]{hernandez2016predictive}
Daniel Hern{\'a}ndez-Lobato, Jose Hernandez-Lobato, Amar Shah, and Ryan Adams.
\newblock Predictive entropy search for multi-objective bayesian optimization.
\newblock In \emph{International conference on machine learning}, pp.\ 1492--1501. PMLR, 2016.
\bibitem[Hern{\'a}ndez-Lobato et~al.(2015)Hern{\'a}ndez-Lobato, Gelbart, Hoffman, Adams, and Ghahramani]{hernandez2015predictive}
Jos{\'e}~Miguel Hern{\'a}ndez-Lobato, Michael Gelbart, Matthew Hoffman, Ryan Adams, and Zoubin Ghahramani.
\newblock Predictive entropy search for bayesian optimization with unknown constraints.
\newblock In \emph{International conference on machine learning}, pp.\ 1699--1707. PMLR, 2015.
\bibitem[Jaech et~al.(2024)Jaech, Kalai, Lerer, Richardson, El-Kishky, Low, Helyar, Madry, Beutel, Carney, et~al.]{jaech2024openai}
Aaron Jaech, Adam Kalai, Adam Lerer, Adam Richardson, Ahmed El-Kishky, Aiden Low, Alec Helyar, Aleksander Madry, Alex Beutel, Alex Carney, et~al.
\newblock Openai o1 system card.
\newblock \emph{arXiv preprint arXiv:2412.16720}, 2024.
\bibitem[Jaynes(1957)]{jaynes1957information}
Edwin~T Jaynes.
\newblock Information theory and statistical mechanics.
\newblock \emph{Physical review}, 106\penalty0 (4):\penalty0 620, 1957.
\bibitem[Jordan et~al.(1999)Jordan, Ghahramani, Jaakkola, and Saul]{jordan1999introduction}
Michael~I Jordan, Zoubin Ghahramani, Tommi~S Jaakkola, and Lawrence~K Saul.
\newblock An introduction to variational methods for graphical models.
\newblock \emph{Machine learning}, 37\penalty0 (2):\penalty0 183--233, 1999.
\bibitem[Kaelbling et~al.(1996)Kaelbling, Littman, and Moore]{kaelbling1996RL}
Leslie~Pack Kaelbling, Michael~L. Littman, and Andrew~W. Moore.
\newblock Reinforcement learning: A survey.
\newblock \emph{J. Artif. Int. Res.}, 4\penalty0 (1):\penalty0 237–285, May 1996.
\newblock ISSN 1076-9757.
\bibitem[Kalman(1960)]{kalman1960new}
Rudolph~Emil Kalman.
\newblock A new approach to linear filtering and prediction problems.
\newblock \emph{Journal of basic Engineering}, 82\penalty0 (1):\penalty0 35--45, 1960.
\bibitem[Kang et~al.(2025)Kang, Zhao, and Song]{kang2025scalable}
Zhewei Kang, Xuandong Zhao, and Dawn Song.
\newblock Scalable best-of-n selection for large language models via self-certainty.
\newblock \emph{arXiv preprint arXiv:2502.18581}, 2025.
\bibitem[Kingma \& Welling(2019)Kingma and Welling]{kingma2019introduction}
Diederik~P Kingma and Max Welling.
\newblock An introduction to variational autoencoders.
\newblock \emph{arXiv preprint arXiv:1906.02691}, 2019.
\bibitem[Kitagawa(1996)]{kitagawa1996monte}
Genshiro Kitagawa.
\newblock Monte carlo filter and smoother for non-gaussian nonlinear state space models.
\newblock \emph{Journal of computational and graphical statistics}, 5\penalty0 (1):\penalty0 1--25, 1996.
\bibitem[Kloek \& Van~Dijk(1978)Kloek and Van~Dijk]{kloek1978bayesian}
Teun Kloek and Herman~K Van~Dijk.
\newblock Bayesian estimates of equation system parameters: an application of integration by monte carlo.
\newblock \emph{Econometrica: Journal of the Econometric Society}, pp.\ 1--19, 1978.
\bibitem[Kolmogorov(1941)]{kolmogorov1941local}
Andrey~Nikolaevich Kolmogorov.
\newblock The local structure of turbulence in incompressible viscous fluid for very large reynolds.
\newblock \emph{Numbers. In Dokl. Akad. Nauk SSSR}, 30:\penalty0 301, 1941.
\bibitem[Kong(1992)]{kong1992note}
Augustine Kong.
\newblock A note on importance sampling using standardized weights.
\newblock \emph{University of Chicago, Dept. of Statistics, Tech. Rep}, 348:\penalty0 14, 1992.
\bibitem[Lange et~al.(2024)Lange, Tian, and Tang]{lange2024large}
Robert Lange, Yingtao Tian, and Yujin Tang.
\newblock Large language models as evolution strategies.
\newblock In \emph{Proceedings of the Genetic and Evolutionary Computation Conference Companion}, pp.\ 579--582, 2024.
\bibitem[Lehman et~al.(2023)Lehman, Gordon, Jain, Ndousse, Yeh, and Stanley]{lehman2023evolution}
Joel Lehman, Jonathan Gordon, Shawn Jain, Kamal Ndousse, Cathy Yeh, and Kenneth~O Stanley.
\newblock Evolution through large models.
\newblock In \emph{Handbook of evolutionary machine learning}, pp.\ 331--366. Springer, 2023.
\bibitem[Lew et~al.(2023)Lew, Zhi-Xuan, Grand, and Mansinghka]{lew2023sequential}
Alexander~K Lew, Tan Zhi-Xuan, Gabriel Grand, and Vikash~K Mansinghka.
\newblock Sequential monte carlo steering of large language models using probabilistic programs.
\newblock \emph{arXiv preprint arXiv:2306.03081}, 2023.
\bibitem[Lewis et~al.(2012)Lewis, Vrabie, and Syrmos]{lewis2012optimal}
Frank~L Lewis, Draguna Vrabie, and Vassilis~L Syrmos.
\newblock \emph{Optimal control}.
\newblock John Wiley \& Sons, 2012.
\bibitem[Lightman et~al.(2023)Lightman, Kosaraju, Burda, Edwards, Baker, Lee, Leike, Schulman, Sutskever, and Cobbe]{lightman2023let}
Hunter Lightman, Vineet Kosaraju, Yuri Burda, Harrison Edwards, Bowen Baker, Teddy Lee, Jan Leike, John Schulman, Ilya Sutskever, and Karl Cobbe.
\newblock Let's verify step by step.
\newblock In \emph{The Twelfth International Conference on Learning Representations}, 2023.
\bibitem[Liu(1996)]{liu1996metropolized}
Jun~S Liu.
\newblock Metropolized independent sampling with comparisons to rejection sampling and importance sampling.
\newblock \emph{Statistics and computing}, 6\penalty0 (2):\penalty0 113--119, 1996.
\bibitem[Liu et~al.(2001)Liu, Chen, and Logvinenko]{liu2001theoretical}
Jun~S Liu, Rong Chen, and Tanya Logvinenko.
\newblock A theoretical framework for sequential importance sampling with resampling.
\newblock In \emph{Sequential Monte Carlo methods in practice}, pp.\ 225--246. Springer, 2001.
\bibitem[Loula et~al.(2025)Loula, LeBrun, Du, Lipkin, Pasti, Grand, Liu, Emara, Freedman, Eisner, et~al.]{loula2025syntactic}
Jo{\~a}o Loula, Benjamin LeBrun, Li~Du, Ben Lipkin, Clemente Pasti, Gabriel Grand, Tianyu Liu, Yahya Emara, Marjorie Freedman, Jason Eisner, et~al.
\newblock Syntactic and semantic control of large language models via sequential monte carlo.
\newblock \emph{arXiv preprint arXiv:2504.13139}, 2025.
\bibitem[Markov(1953)]{Markov1953-MARTTO-31}
A.~A. Markov.
\newblock The theory of algorithms.
\newblock \emph{Journal of Symbolic Logic}, 18\penalty0 (4):\penalty0 340--341, 1953.
\newblock \doi{10.2307/2266585}.
\bibitem[Martino \& Elvira(2025)Martino and Elvira]{martino2025effective}
L~Martino and V~Elvira.
\newblock Effective sample size approximations as entropy measures.
\newblock \emph{Computational Statistics}, pp.\ 1--32, 2025.
\bibitem[Martino et~al.(2017)Martino, Elvira, and Louzada]{martino2017effective}
Luca Martino, V{\'\i}ctor Elvira, and Francisco Louzada.
\newblock Effective sample size for importance sampling based on discrepancy measures.
\newblock \emph{Signal Processing}, 131:\penalty0 386--401, 2017.
\bibitem[Metropolis \& Ulam(1949)Metropolis and Ulam]{metropolis1949monte}
Nicholas Metropolis and Stanislaw Ulam.
\newblock The monte carlo method.
\newblock \emph{Journal of the American statistical association}, 44\penalty0 (247):\penalty0 335--341, 1949.
\bibitem[Metropolis et~al.(1953)Metropolis, Rosenbluth, Rosenbluth, Teller, and Teller]{metropolis1953equation}
Nicholas Metropolis, Arianna~W Rosenbluth, Marshall~N Rosenbluth, Augusta~H Teller, and Edward Teller.
\newblock Equation of state calculations by fast computing machines.
\newblock \emph{The journal of chemical physics}, 21\penalty0 (6):\penalty0 1087--1092, 1953.
\bibitem[Mnih et~al.(2016)Mnih, Badia, Mirza, Graves, Lillicrap, Harley, Silver, and Kavukcuoglu]{mnih2016asynchronous}
Volodymyr Mnih, Adria~Puigdomenech Badia, Mehdi Mirza, Alex Graves, Timothy Lillicrap, Tim Harley, David Silver, and Koray Kavukcuoglu.
\newblock Asynchronous methods for deep reinforcement learning.
\newblock In \emph{International conference on machine learning}, pp.\ 1928--1937. PmLR, 2016.
\bibitem[Mohamed \& Lakshminarayanan(2016)Mohamed and Lakshminarayanan]{mohamed2016learning}
Shakir Mohamed and Balaji Lakshminarayanan.
\newblock Learning in implicit generative models.
\newblock \emph{ArXiv preprint}, abs/1610.03483, 2016.
\newblock URL \url{https://arxiv.org/abs/1610.03483}.
\bibitem[Mohamed et~al.(2020)Mohamed, Rosca, Figurnov, and Mnih]{mohamed2020monte}
Shakir Mohamed, Mihaela Rosca, Michael Figurnov, and Andriy Mnih.
\newblock Monte carlo gradient estimation in machine learning.
\newblock \emph{Journal of Machine Learning Research}, 21\penalty0 (132):\penalty0 1--62, 2020.
\bibitem[Mouret \& Clune(2015)Mouret and Clune]{mouret2015illuminating}
Jean-Baptiste Mouret and Jeff Clune.
\newblock Illuminating search spaces by mapping elites.
\newblock \emph{arXiv preprint arXiv:1504.04909}, 2015.
\bibitem[Novikov et~al.(2025)Novikov, V{\~u}, Eisenberger, Dupont, Huang, Wagner, Shirobokov, Kozlovskii, Ruiz, Mehrabian, et~al.]{novikov2025alphaevolve}
Alexander Novikov, Ng{\^a}n V{\~u}, Marvin Eisenberger, Emilien Dupont, Po-Sen Huang, Adam~Zsolt Wagner, Sergey Shirobokov, Borislav Kozlovskii, Francisco~JR Ruiz, Abbas Mehrabian, et~al.
\newblock Alphaevolve: A coding agent for scientific and algorithmic discovery.
\newblock \emph{arXiv preprint arXiv:2506.13131}, 2025.
\bibitem[Pandey et~al.(2014)Pandey, Chaudhary, and Mehrotra]{pandey2014comparative}
Hari~Mohan Pandey, Ankit Chaudhary, and Deepti Mehrotra.
\newblock A comparative review of approaches to prevent premature convergence in ga.
\newblock \emph{Applied Soft Computing}, 24:\penalty0 1047--1077, 2014.
\bibitem[Park et~al.(2025)Park, Greenewald, Alim, Wang, and Azizan]{park2025know}
Young-Jin Park, Kristjan Greenewald, Kaveh Alim, Hao Wang, and Navid Azizan.
\newblock Know what you don't know: Uncertainty calibration of process reward models.
\newblock \emph{arXiv preprint arXiv:2506.09338}, 2025.
\bibitem[Peters \& Schaal(2007)Peters and Schaal]{peters2007reinforcement}
Jan Peters and Stefan Schaal.
\newblock Reinforcement learning by reward-weighted regression for operational space control.
\newblock In \emph{Proceedings of the 24th international conference on Machine learning}, pp.\ 745--750, 2007.
\bibitem[Pitt \& Shephard(1999{\natexlab{a}})Pitt and Shephard]{pitt1999filtering}
Michael~K Pitt and Neil Shephard.
\newblock Filtering via simulation: Auxiliary particle filters.
\newblock \emph{Journal of the American statistical association}, 94\penalty0 (446):\penalty0 590--599, 1999{\natexlab{a}}.
\bibitem[Pitt \& Shephard(1999{\natexlab{b}})Pitt and Shephard]{pitt1999filtering-sim}
Michael~K Pitt and Neil Shephard.
\newblock Filtering via simulation: Auxiliary particle filters.
\newblock \emph{Journal of the American statistical association}, 94\penalty0 (446):\penalty0 590--599, 1999{\natexlab{b}}.
\bibitem[Pugh et~al.(2016)Pugh, Soros, and Stanley]{pugh2016quality}
Justin~K Pugh, Lisa~B Soros, and Kenneth~O Stanley.
\newblock Quality diversity: A new frontier for evolutionary computation.
\newblock \emph{Frontiers in Robotics and AI}, 3:\penalty0 40, 2016.
\bibitem[Puri et~al.(2025)Puri, Sudalairaj, Xu, Xu, and Srivastava]{puri2025probabilistic}
Isha Puri, Shivchander Sudalairaj, Guangxuan Xu, Kai Xu, and Akash Srivastava.
\newblock A probabilistic inference approach to inference-time scaling of llms using particle-based monte carlo methods.
\newblock \emph{arXiv preprint arXiv:2502.01618}, 2025.
\bibitem[Robert et~al.(1999)Robert, Casella, and Casella]{robert1999monte}
Christian~P Robert, George Casella, and George Casella.
\newblock \emph{Monte Carlo statistical methods}, volume~2.
\newblock Springer, 1999.
\bibitem[Roux et~al.(2025)Roux, Bellemare, Lebensold, Bergeron, Greaves, Fr{\'e}chette, Pelletier, Thibodeau-Laufer, Toth, and Work]{roux2025tapered}
Nicolas~Le Roux, Marc~G Bellemare, Jonathan Lebensold, Arnaud Bergeron, Joshua Greaves, Alex Fr{\'e}chette, Carolyne Pelletier, Eric Thibodeau-Laufer, S{\'a}ndor Toth, and Sam Work.
\newblock Tapered off-policy reinforce: Stable and efficient reinforcement learning for llms.
\newblock \emph{arXiv preprint arXiv:2503.14286}, 2025.
\bibitem[Schulman et~al.(2015)Schulman, Heess, Weber, and Abbeel]{schulman2015gradient}
John Schulman, Nicolas Heess, Theophane Weber, and Pieter Abbeel.
\newblock Gradient estimation using stochastic computation graphs.
\newblock In Corinna Cortes, Neil~D. Lawrence, Daniel~D. Lee, Masashi Sugiyama, and Roman Garnett (eds.), \emph{Advances in Neural Information Processing Systems 28: Annual Conference on Neural Information Processing Systems 2015, December 7-12, 2015, Montreal, Quebec, Canada}, pp.\ 3528--3536, 2015.
\newblock URL \url{https://proceedings.neurips.cc/paper/2015/hash/de03beffeed9da5f3639a621bcab5dd4-Abstract.html}.
\bibitem[Schulman et~al.(2017)Schulman, Wolski, Dhariwal, Radford, and Klimov]{schulman2017proximal}
John Schulman, Filip Wolski, Prafulla Dhariwal, Alec Radford, and Oleg Klimov.
\newblock Proximal policy optimization algorithms.
\newblock \emph{arXiv preprint arXiv:1707.06347}, 2017.
\bibitem[Shinn et~al.(2024)Shinn, Cassano, Gopinath, Narasimhan, and Yao]{shinn2024reflexion}
Noah Shinn, Federico Cassano, Ashwin Gopinath, Karthik Narasimhan, and Shunyu Yao.
\newblock Reflexion: Language agents with verbal reinforcement learning.
\newblock \emph{Advances in Neural Information Processing Systems}, 36, 2024.
\bibitem[Silver et~al.(2016)Silver, Huang, Maddison, Guez, Sifre, Van Den~Driessche, Schrittwieser, Antonoglou, Panneershelvam, Lanctot, et~al.]{silver2016mastering}
David Silver, Aja Huang, Chris~J Maddison, Arthur Guez, Laurent Sifre, George Van Den~Driessche, Julian Schrittwieser, Ioannis Antonoglou, Veda Panneershelvam, Marc Lanctot, et~al.
\newblock Mastering the game of go with deep neural networks and tree search.
\newblock \emph{nature}, 529\penalty0 (7587):\penalty0 484--489, 2016.
\bibitem[Snell et~al.(2024)Snell, Lee, Xu, and Kumar]{snell2024scaling}
Charlie Snell, Jaehoon Lee, Kelvin Xu, and Aviral Kumar.
\newblock Scaling llm test-time compute optimally can be more effective than scaling model parameters.
\newblock \emph{arXiv preprint arXiv:2408.03314}, 2024.
\bibitem[Vijayakumar et~al.(2016)Vijayakumar, Cogswell, Selvaraju, Sun, Lee, Crandall, and Batra]{vijayakumar2016diverse}
Ashwin~K Vijayakumar, Michael Cogswell, Ramprasath~R Selvaraju, Qing Sun, Stefan Lee, David Crandall, and Dhruv Batra.
\newblock Diverse beam search: Decoding diverse solutions from neural sequence models.
\newblock \emph{arXiv preprint arXiv:1610.02424}, 2016.
\bibitem[Wang et~al.(2022)Wang, Wei, Schuurmans, Le, Chi, and Zhou]{wang2022self}
Xuezhi Wang, Jason Wei, Dale Schuurmans, Quoc Le, Ed~Chi, and Denny Zhou.
\newblock Self-consistency improves chain of thought reasoning in language models.
\newblock \emph{arXiv preprint arXiv:2203.11171}, 2022.
\bibitem[Wang \& Jegelka(2017)Wang and Jegelka]{wang2017max}
Zi~Wang and Stefanie Jegelka.
\newblock Max-value entropy search for efficient bayesian optimization.
\newblock In \emph{International conference on machine learning}, pp.\ 3627--3635. PMLR, 2017.
\bibitem[Wei et~al.(2022)Wei, Wang, Schuurmans, Bosma, Chi, Le, and Zhou]{wei2022chain}
Jason Wei, Xuezhi Wang, Dale Schuurmans, Maarten Bosma, Ed~Chi, Quoc Le, and Denny Zhou.
\newblock Chain of thought prompting elicits reasoning in large language models.
\newblock \emph{arXiv preprint arXiv:2201.11903}, 2022.
\bibitem[Williams \& Peng(1991)Williams and Peng]{williams1991function}
Ronald~J Williams and Jing Peng.
\newblock Function optimization using connectionist reinforcement learning algorithms.
\newblock \emph{Connection Science}, 3\penalty0 (3):\penalty0 241--268, 1991.
\bibitem[Yang et~al.(2024)Yang, Yang, Hui, Zheng, Yu, Zhou, Li, Li, Liu, Huang, et~al.]{yang2024qwen2}
An~Yang, Baosong Yang, Binyuan Hui, Bo~Zheng, Bowen Yu, Chang Zhou, Chengpeng Li, Chengyuan Li, Dayiheng Liu, Fei Huang, et~al.
\newblock Qwen2 technical report.
\newblock \emph{arXiv preprint arXiv:2407.10671}, 2024.
\bibitem[Yang et~al.(2025)Yang, Li, Yang, Zhang, Hui, Zheng, Yu, Gao, Huang, Lv, et~al.]{yang2025qwen3}
An~Yang, Anfeng Li, Baosong Yang, Beichen Zhang, Binyuan Hui, Bo~Zheng, Bowen Yu, Chang Gao, Chengen Huang, Chenxu Lv, et~al.
\newblock Qwen3 technical report.
\newblock \emph{arXiv preprint arXiv:2505.09388}, 2025.
\bibitem[Yao et~al.(2022)Yao, Zhao, Yu, Du, Shafran, Narasimhan, and Cao]{yao2022react}
Shunyu Yao, Jeffrey Zhao, Dian Yu, Nan Du, Izhak Shafran, Karthik Narasimhan, and Yuan Cao.
\newblock React: Synergizing reasoning and acting in language models.
\newblock \emph{arXiv preprint arXiv:2210.03629}, 2022.
\bibitem[Yao et~al.(2023)Yao, Yu, Zhao, Shafran, Griffiths, Cao, and Narasimhan]{yao2023tree}
Shunyu Yao, Dian Yu, Jeffrey Zhao, Izhak Shafran, Tom Griffiths, Yuan Cao, and Karthik Narasimhan.
\newblock Tree of thoughts: Deliberate problem solving with large language models.
\newblock \emph{Advances in neural information processing systems}, 36:\penalty0 11809--11822, 2023.
\bibitem[Zhao et~al.(2024)Zhao, Brekelmans, Makhzani, and Grosse]{zhao2024probabilistic}
Stephen Zhao, Rob Brekelmans, Alireza Makhzani, and Roger Grosse.
\newblock Probabilistic inference in language models via twisted sequential monte carlo.
\newblock \emph{arXiv preprint arXiv:2404.17546}, 2024.
\end{thebibliography}