GraphSkill: Documentation-Guided Hierarchical Retrieval-Augmented Coding for Complex Graph Reasoning

Extracted source text · arXiv 2603.06620

%%
%% This is file `sample-sigconf.tex',
%% generated with the docstrip utility.
%%
%% The original source files were:
%%
%% samples.dtx  (with options: `all,proceedings,bibtex,sigconf')
%% 
%% IMPORTANT NOTICE:
%% 
%% For the copyright see the source file.
%% 
%% Any modified versions of this file must be renamed
%% with new filenames distinct from sample-sigconf.tex.
%% 
%% For distribution of the original source see the terms
%% for copying and modification in the file samples.dtx.
%% 
%% This generated file may be distributed as long as the
%% original source files, as listed above, are part of the
%% same distribution. (The sources need not necessarily be
%% in the same archive or directory.)
%%
%%
%% Commands for TeXCount
%TC:macro \cite [option:text,text]
%TC:macro \citep [option:text,text]
%TC:macro \citet [option:text,text]
%TC:envir table 0 1
%TC:envir table* 0 1
%TC:envir tabular [ignore] word
%TC:envir displaymath 0 word
%TC:envir math 0 word
%TC:envir comment 0 0
%%
%% The first command in your LaTeX source must be the \documentclass
%% command.
%%
%% For submission and review of your manuscript please change the
%% command to \documentclass[manuscript, screen, review]{acmart}.
%%
%% When submitting camera ready or to TAPS, please change the command
%% to \documentclass[sigconf]{acmart} or whichever template is required
%% for your publication.
%%
%%
\documentclass[sigconf]{acmart}
% \documentclass[manuscript, screen, review]{acmart}
% \documentclass[sigconf,anonymous,review]{acmart}


%%
%% \BibTeX command to typeset BibTeX logo in the docs
% \AtBeginDocument{%
%   \providecommand\BibTeX{{%
%     Bib\TeX}}}

%% Rights management information.  This information is sent to you
%% when you complete the rights form.  These commands have SAMPLE
%% values in them; it is your responsibility as an author to replace
%% the commands and values with those provided to you when you
%% complete the rights form.
\setcopyright{none}
\copyrightyear{2026}
\acmYear{2026}
\acmDOI{XXXXXXX.XXXXXXX}


%% These commands are for a PROCEEDINGS abstract or paper.
% \acmConference[x]{x}{x}{x}
%%
%%  Uncomment \acmBooktitle if the title of the proceedings is different
%%  from ``Proceedings of ...''!
%%
%%\acmBooktitle{Woodstock '18: ACM Symposium on Neural Gaze Detection,
%%  June 03--05, 2018, Woodstock, NY}
% \acmISBN{}


%%
%% Submission ID.
%% Use this when submitting an article to a sponsored event. You'll
%% receive a unique submission ID from the organizers
%% of the event, and this ID should be used as the parameter to this command.
%%\acmSubmissionID{123-A56-BU3}

%%
%% For managing citations, it is recommended to use bibliography
%% files in BibTeX format.
%%
%% You can then either use BibTeX with the ACM-Reference-Format style,
%% or BibLaTeX with the acmnumeric or acmauthoryear sytles, that include
%% support for advanced citation of software artefact from the
%% biblatex-software package, also separately available on CTAN.
%%
%% Look at the sample-*-biblatex.tex files for templates showcasing
%% the biblatex styles.
%%

%%
%% The majority of ACM publications use numbered citations and
%% references.  The command \citestyle{authoryear} switches to the
%% "author year" style.
%%
%% If you are preparing content for an event
%% sponsored by ACM SIGGRAPH, you must use the "author year" style of
%% citations and references.
%% Uncommenting
%% the next command will enable that style.
%%\citestyle{acmauthoryear}


%%
%% end of the preamble, start of the body of the document source.
\usepackage{xcolor}
\usepackage{soul}
% Standard package includes
\usepackage{times}
\usepackage{array}
\usepackage{tabularx}
\usepackage{enumitem}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{multirow}
\usepackage{latexsym}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{makecell}


% \usepackage{algpseudocode}  
\usepackage{siunitx}
\usepackage{booktabs}
\usepackage{longtable}
\usepackage{tabularx, array, colortbl, xcolor}
\definecolor{lightgray}{gray}{0.9}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.


%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}
\usepackage{amsthm}
\theoremstyle{definition}
\newtheorem{definition}{Definition}

% 
\usepackage{tcolorbox}
\tcbuselibrary{skins,breakable}
\newtcolorbox{promptbox}[1]{
  breakable,
  enhanced,
  colback=gray!1,            % subtle background
  colframe=gray!40,          % muted border
  coltitle=black!90,         % title color
  fonttitle=\bfseries\small,
  title={#1},                % e.g., Prompt Template 
  left=3pt, right=3pt, top=4pt, bottom=4pt,
  boxsep=2pt,
  arc=2pt,
  boxrule=0.5pt,
  fontupper=\ttfamily\small, % monospace prompt content
  before skip=4pt,
  after skip=4pt
}

\newtcolorbox{dataexample*}[1]{
  breakable,
  enhanced,
  colback=cyan!8,            % very light cyan background
  colframe=teal!60,          % teal border
  coltitle=teal!40!black,    % dark teal title
  fonttitle=\bfseries\small,
  title={#1},
  left=3pt, right=3pt, top=4pt, bottom=4pt,
  width=0.8\textwidth,          % fits to current line width
  boxsep=2pt,
  arc=2pt,
  boxrule=0.5pt,
  fontupper=\ttfamily\small,
  before skip=4pt,
  after skip=4pt,
  float*=htbp  % makes it span two columns
}


\newcommand{\dataset}{{\color{black}\texttt{ComplexGraph}}}
% GraphCoding
\newcommand{\method}{{\color{black}\textsc{GraphSkill}}}
% GraphTutor
% cyan



\newcommand{\chenglin}[1]{{\color{red}[Chenglin: #1]}}


\newcommand{\fali}[1]{{\color{blue}[Fali: #1]}}
\newcommand{\xianren}[1]{{\color{red}[Xianren: #1]}}

\newcommand{\suhang}[1]{{\color{orange}[SW: #1]}}

\begin{document}

%%
%% The "title" command has an optional parameter,
%% allowing the author to define a "short title" to be used in page headers.
\title{\textsc{GraphSkill}: Documentation-Guided Hierarchical Retrieval-Augmented Coding for Complex Graph Reasoning}
% Equip Agent Graph Skill via
%%
%% The "author" command and its associated commands are used to define
%% the authors and their affiliations.
%% Of note is the shared affiliation of the first two authors, and the
%% "authornote" and "authornotemark" commands
%% used to denote shared contribution to the research.
% \author{Fali Wang, Chenglin Weng}
% \authornote{Both authors contributed equally to this research.}
% \affiliation{%
%   \institution{The Pennsylvania State University}
%   \city{University}
%   \state{Pennsylvania}
%   \country{USA}
% }
% \email{{fqw5095, cvw5844}@psu.edu}
\author{Fali Wang}
\authornote{Both authors contributed equally to this research.}
\affiliation{%
  \institution{The Pennsylvania State University}
  \city{University Park}
  \state{Pennsylvania}
  \country{USA}
}
\email{fqw5095@psu.edu}

\author{Chenglin Weng}
\authornotemark[1]
\affiliation{%
  \institution{The Pennsylvania State University}
  \city{University Park}
  \state{Pennsylvania}
  \country{USA}
}
\email{cvw5844@psu.edu}


% \author{Xianren Zhang, Siyuan Hong}
% \affiliation{%
%   \institution{The Pennsylvania State University}
%   \city{University}
%   \state{Pennsylvania}
%   \country{USA}
% }
% \email{{xzz5508,sjh6636}@psu.edu}


\author{Xianren Zhang}
\affiliation{%
  \institution{The Pennsylvania State University}
  \city{University Park}
  \state{Pennsylvania}
  \country{USA}
}
\email{xzz5508@psu.edu}


\author{Siyuan Hong}
\affiliation{%
  \institution{The Pennsylvania State University}
  \city{University Park}
  \state{Pennsylvania}
  \country{USA}
}
\email{sjh6636@psu.edu}

\author{Hui Liu}
\affiliation{%
  \institution{Michigan State University}
  \city{East Lansing}
  \state{Michigan}
  \country{USA}
}
\email{liuhui7@msu.edu}

\author{Suhang Wang}
\authornote{Corresponding author.}
\affiliation{%
  \institution{The Pennsylvania State University}
  \city{University Park}
  \state{Pennsylvania}
  \country{USA}
}
\email{szw494@psu.edu}

%%
%% By default, the full list of authors will be used in the page
%% headers. Often, this list is too long, and will overlap
%% other information printed in the page headers. This command allows
%% the author to define a more concise list
%% of authors' names for this purpose.
\renewcommand{\shortauthors}{xxx}

%%
%% The abstract is a short summary of the work to be presented in the
%% article.
\begin{abstract}
The growing demand for automated graph algorithm reasoning has attracted increasing attention in the large language model (LLM) community. Recent LLM-based graph reasoning methods typically decouple task descriptions from graph data, generate executable code augmented by retrieval from technical documentation, and refine the code through debugging. However, we identify two key limitations in existing approaches: (i) they treat technical documentation as flat text collections and ignore its hierarchical structure, leading to noisy retrieval that degrades code generation quality; and (ii) their debugging mechanisms focus primarily on runtime errors, yet ignore more critical logical errors.
To address them, we propose {\method}, an \textit{agentic hierarchical retrieval-augmented coding framework} that exploits the document hierarchy through top-down traversal and early pruning, together with a \textit{self-debugging coding agent} that iteratively refines code using automatically generated small-scale test cases. To enable comprehensive evaluation of complex graph reasoning, we introduce a new dataset, {\dataset}, covering small-scale, large-scale, and composite graph reasoning tasks. 
Extensive experiments demonstrate that our method achieves higher task accuracy and lower inference cost compared to baselines\footnote{The code is available at \href{https://github.com/FairyFali/GraphSkill}{\textcolor{blue}{https://github.com/FairyFali/GraphSkill}}.}.
\end{abstract}

%%
%% The code below is generated by the tool at http://dl.acm.org/ccs.cfm.
%% Please copy and paste the code instead of the example below.
%%
% \begin{CCSXML}
% <ccs2012>
%  <concept>
%   <concept_id>00000000.0000000.0000000</concept_id>
%   <concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
%   <concept_significance>500</concept_significance>
%  </concept>
%  <concept>
%   <concept_id>00000000.00000000.00000000</concept_id>
%   <concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
%   <concept_significance>300</concept_significance>
%  </concept>
%  <concept>
%   <concept_id>00000000.00000000.00000000</concept_id>
%   <concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
%   <concept_significance>100</concept_significance>
%  </concept>
%  <concept>
%   <concept_id>00000000.00000000.00000000</concept_id>
%   <concept_desc>Do Not Use This Code, Generate the Correct Terms for Your Paper</concept_desc>
%   <concept_significance>100</concept_significance>
%  </concept>
% </ccs2012>
% \end{CCSXML}

% \ccsdesc[500]{Do Not Use This Code~Generate the Correct Terms for Your Paper}
% \ccsdesc[300]{Do Not Use This Code~Generate the Correct Terms for Your Paper}
% \ccsdesc{Do Not Use This Code~Generate the Correct Terms for Your Paper}
% \ccsdesc[100]{Do Not Use This Code~Generate the Correct Terms for Your Paper}
\begin{CCSXML}
<ccs2012>
   <concept>
       <concept_id>10010147.10010178.10010179.10010182</concept_id>
       <concept_desc>Computing methodologies~Natural language generation</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
   <concept>
       <concept_id>10010147.10010257.10010282.10010290</concept_id>
       <concept_desc>Computing methodologies~Learning from demonstrations</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
   <concept>
       <concept_id>10010147.10010257.10010282.10010291</concept_id>
       <concept_desc>Computing methodologies~Learning from critiques</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
 </ccs2012>
\end{CCSXML}

\ccsdesc[500]{Computing methodologies~Natural language generation}
% \ccsdesc[500]{Computing methodologies~Learning from demonstrations}
% \ccsdesc[500]{Computing methodologies~Learning from critiques}
%%
%% Keywords. The author(s) should pick words that accurately describe
%% the work being presented. Separate the keywords with commas.
\keywords{Graph Reasoning, LLM, Retrieval-Augmented Generation}
%% A "teaser" image appears between the author and affiliation
%% information and the body of the document, and typically spans the
%% page.
% \begin{teaserfigure}
%   \includegraphics[width=\textwidth]{sampleteaser}
%   \caption{Seattle Mariners at Spring Training, 2010.}
%   \Description{Enjoying the baseball game from the third-base
%   seats. Ichiro Suzuki preparing to bat.}
%   \label{fig:teaser}
% \end{teaserfigure}

% \received{xx x xxx}
% \received[revised]{x xx xxxx}
% \received[accepted]{x xx xxxx}

%%
%% This command processes the author and affiliation and title
%% information and builds the first part of the formatted document.
\maketitle

% \vspace{-1em}
% \begin{center}
% {\small \textit{Preprint. Under review.}}
% \end{center}

\section{Introduction}
%%% original
% Graph algorithmic reasoning tasks have been widely studied across domains, including relational modeling in social networks \cite{girvan2002community}, path planning in transportation systems \cite{hart1968formal}, and graph-based ranking such as PageRank \cite{page1999pagerank}. Unlike \textit{graph learning} methods \cite{xia2021graph}, which aim to learn parameterized mappings from graph data distributions for generalization, covering tasks such as node classification \cite{chen2024labelfree, tang2024graphgpt}, knowledge graph reasoning \cite{luo2024reasoning}, and molecular property prediction \cite{le2024molx}, \textit{graph reasoning} focuses on explicit, algorithmic inference over given graph structures, where solutions are typically derived via deterministic graph computations. These tasks require multi-step structured reasoning operations, such as traversal, search, sorting, or compositional computation, and have traditionally relied on manually designed algorithms, which incur high development costs and limit automated adaptation to new settings.
%%%
% as compositional complexity increases.
% , sorting
% task diversity and
% \suhang{the introduction is too long. Consider shortening the related works as the details will be introduced in the related work section}
% The strong general reasoning capabilities of large language models (LLMs) have opened new opportunities for automating graph reasoning tasks, leading to a growing body of research \cite{zhang2024gcoderimprovinglargelanguage, wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms, li2025graphteamfacilitatinglargelanguage, cai2024codegraphenhancinggraphreasoning, fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph, chen2024graphwizinstructionfollowinglanguagemodel, luo2024graphinstruct}. Existing LLM-based graph reasoning methods can be broadly categorized into two paradigms: text-based and coding-based reasoning. \textit{Text-based reasoning} methods, such as Chain-of-Thought (CoT) approaches \cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph, chen2024graphwizinstructionfollowinglanguagemodel}, decompose graph problems into intermediate reasoning steps and derive solutions purely in text. In contrast, \textit{code generation--based reasoning} methods, including GCoder \cite{zhang2024gcoderimprovinglargelanguage}, CodeGraph \cite{cai2024codegraphenhancinggraphreasoning}, PIE \cite{gong2025pseudocodeinjectionmagicenablingllms}, GraphTeam \cite{li2025graphteamfacilitatinglargelanguage}, and GraphTool-Instruction \cite{wang2025GraphToolInstruction}, rely on LLMs to generate executable programs whose execution yields final results.
% The strong reasoning capabilities of large language models (LLMs) have motivated growing interest in graph reasoning. While text-based approaches are limited by context length and performance~\cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph}, code-based methods generate executable programs~\cite{gong2025pseudocodeinjectionmagicenablingllms, wang2025GraphToolInstruction}, decoupling tasks from graph data and leveraging LLMs’ coding ability to achieve substantially better performance.
% Large language models (LLMs) have enabled automated graph reasoning, but text-based methods are limited by context length and performance~\cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph}; thereby, code-based approaches instead generate executable programs that solve the problems~\cite{gong2025pseudocodeinjectionmagicenablingllms, wang2025GraphToolInstruction} in a task-graph decoupling paradigm and achieve substantially better performance.
% Large language models (LLMs), owing to their strong reasoning capabilities, have enabled automated graph reasoning~\cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph, wang2025GraphToolInstruction,gong2025pseudocodeinjectionmagicenablingllms,zhang2024gcoderimprovinglargelanguage,li2025graphteamfacilitatinglargelanguage} and achieved substantial performance. 
% LLM-enabled automated graph reasoning aims to replace manually implemented graph algorithms with language models that can interpret task specifications, synthesize executable reasoning procedures (e.g., code or action sequences), and solve graph problems in a flexible, task-agnostic manner. This paradigm reduces the need for handcrafted algorithms but introduces new challenges in reliability, scalability, and systematic reasoning over complex graph structures.
%%% original 
% Large language models (LLMs), owing to their strong reasoning capabilities, have enabled automated graph reasoning~\cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph, wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms, zhang2024gcoderimprovinglargelanguage, li2025graphteamfacilitatinglargelanguage} and have achieved substantial performance gains. Broadly, LLM-enabled automated graph reasoning seeks to replace manually implemented graph algorithms with language models that can interpret task specifications, synthesize executable reasoning procedures (such as code or reasoning sequences), and solve graph problems in a flexible and graph task-agnostic manner. While this paradigm reduces reliance on handcrafted algorithms, it also introduces new challenges in reliability, scalability, and systematic reasoning over complex graph structures.
% %%% 
Graph algorithmic reasoning tasks play an important role in many real-world applications, including social network analysis~\cite{girvan2002community}, transportation systems~\cite{hart1968formal}, and communication networks~\cite{page1999pagerank}, where solutions rely on the explicit structural and relational properties of graphs. Unlike \emph{graph learning} methods~\cite{xia2021graph,luo2024enhance,wang2024hc,yang2020nargnn}, which learn parameterized mappings from graph data distributions into labels for generalization, \emph{graph reasoning} focuses on deterministic, algorithmic inference over a given graph instance, typically involving multi-step structured operations such as traversal, search, flow computation, or compositional reasoning. Traditionally, these tasks depend on manually designed graph algorithms, which are costly to develop and difficult to adapt as problem complexity increases.



% \suhang{can we give one to two sentences high-level idea of LLM enabled automated graph, this will help readers to understand the issues we mention below}


% 提出数据集
% Despite notable progress, these methods remain limited in \textbf{complex graph reasoning scenarios, particularly for large-scale graphs and composite graph tasks}. For \textit{large-scale graphs}, the context window size and long-context reasoning capability of LLMs constitute major bottlenecks. Many existing approaches, including most coding-based methods, embed the serialized representations of graphs directly into prompts, causing input lengths to grow rapidly with graph size; for example, a fully connected graph with approximately 180 nodes already exceeds the 160k-token context limit of DeepSeek-V3 \cite{liu2024deepseek}. 
% % Moreover, accurate reasoning over long contexts remains challenging even when inputs fit within window limits.
% For \textit{composite graph tasks}, which combine multiple classical graph algorithms, stronger algorithmic knowledge and more sophisticated reasoning planning are required. While some coding-based methods decouple task descriptions from graph data to alleviate context constraints \cite{wang2025GraphToolInstruction}, they still suffer from limited algorithmic knowledge. Prior work \cite{li2025graphteamfacilitatinglargelanguage, zhang2024gcoderimprovinglargelanguage} has attempted to address this by incorporating external technical documentation, such as NetworkX \cite{hagberg2008exploring}. They typically treat such resources as flat text collections and apply semantic retrieval to select relevant content.
% rich
% Despite notable progress, existing LLM-based graph reasoning methods remain limited in complex graph reasoning scenarios, particularly for large-scale graphs and composite graph tasks. For \emph{large-scale graphs}, the context window size and long-context reasoning capability of LLMs constitute major bottlenecks. Many existing approaches, including most code-based methods, embed serialized graph representations directly into the prompt, causing input length to grow rapidly with graph size; for example, a fully connected graph with approximately $180$ nodes already exceeds the $160$k-token context limit of DeepSeek-V3~\cite{liu2024deepseek}. To mitigate this limitation, some code-based methods decouple task descriptions from graph data, thereby alleviating prompt length constraints~\cite{wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms}.
% Beyond scale, \emph{composite graph tasks}, which involve the composition of multiple classical graph algorithms, are both more practical and more challenging. For instance, in a communication network, one may first identify strongly connected components to assess network resilience and then compute shortest communication paths within each component under latency constraints. Such tasks pose additional challenges by requiring stronger algorithmic knowledge as well as more sophisticated reasoning and planning capabilities. While some existing code-based methods rely on human-provided algorithmic guidance and others retrieve knowledge from graph-related technical documentation such as NetworkX~\cite{hagberg2008exploring}, there is a lack of comprehensive benchmarks for evaluating complex graph reasoning. To address this gap, we construct a new complex graph reasoning dataset, {\dataset}, which includes three variants—\texttt{\dataset-S}, \texttt{\dataset-L}, and \texttt{\dataset-C}—covering small-scale, large-scale, and composite graph reasoning tasks, respectively.
% concise
% original
% Despite notable progress, existing LLM-based graph reasoning methods remain limited in \textbf{complex graph reasoning, particularly for large-scale graphs and composite graph tasks.} 
% On the one hand, for large graphs with high structural complexity, context window limitations pose a major bottleneck, as serializing graph structures into prompts can quickly exceed LLM context limits (e.g., a fully connected graph with approximately $180$ nodes already surpasses the $160$k-token context window of DeepSeek-V3~\cite{liu2024deepseek}). Even when large graphs fit within the context window, long-horizon reasoning over lengthy graph representations further challenges the long-context reasoning capabilities of LLMs.
% % decouple task descriptions from graph data to
% On the other hand, while code-based methods partially alleviate this issue by delegating complex structural reasoning to executable programs~\cite{wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms}, they introduce new challenges when applied to composite graph tasks. Such tasks combine multiple classical graph algorithms and capture semantic complexity at the problem level, making them both more practical and substantially harder, as they require stronger algorithmic knowledge and more sophisticated reasoning (e.g., identifying strongly connected components before computing shortest paths under latency constraints in communication networks).
% Existing methods~\cite{gong2025pseudocodeinjectionmagicenablingllms, li2025graphteamfacilitatinglargelanguage} are typically evaluated on relatively simple, small-scale graph tasks that do not sufficiently challenge LLMs’ graph reasoning capabilities, and therefore fail to expose their limitations on large-scale or composite graph reasoning. To address this gap, we introduce a new dataset, {\dataset}, which accounts for both large-scale structural complexity and problem-level semantic complexity. The dataset comprises three subsets, \texttt{\dataset-S}, \texttt{\dataset-L}, and \texttt{\dataset-C}, targeting small-scale, large-scale, and composite graph reasoning tasks, respectively.
% \suhang{I don't get the logic here., i.e., why do we want to mention either rely on or retrieve, then add ``but''. If we want to criticize that existing benchmarks cannot fully test the capability of existing methods, just directly point this out. }. 
% Overall, there lacks a comprehensive dataset for complex graph reasoning. To bridge this gap, we introduce a new dataset, {\dataset}, covering three variants, \texttt{\dataset-S}, \texttt{\dataset-L}, and \texttt{\dataset-C}, that cover small-scale, large-scale, and composite graph reasoning tasks.
% Existing methods either rely on human-provided guidance~\cite{gong2025pseudocodeinjectionmagicenablingllms} or retrieve knowledge from graph-related technical documentation~\cite{li2025graphteamfacilitatinglargelanguage}, but are typically evaluated on relatively simple, small-scale graph tasks that do not sufficiently challenge retrieval or reasoning 
%  such as NetworkX~\cite{hagberg2008exploring}


Recently, large language models (LLMs) have enabled automated graph reasoning~\cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph, wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms, zhang2024gcoderimprovinglargelanguage, li2025graphteamfacilitatinglargelanguage}, achieving substantial performance gains. LLM-based automated graph reasoning aims to replace manually implemented graph algorithms with LLMs that interpret task specifications and generate reasoning procedures (e.g., code or natural language reasoning), thereby solving graph problems automatically and reducing reliance on handcrafted algorithms. LLM-based graph reasoning methods can be broadly categorized into two classes: \emph{text-based} reasoning methods and \emph{code-based} reasoning methods. Text-based approaches serialize graph structures into natural language and leverage LLMs’ strengths in instruction following and multi-step textual reasoning through advanced prompting strategies, achieving competitive performance on small-scale and structurally simple graph tasks~\cite{fatemi2023talklikegraphencoding, wang2024languagemodelssolvegraph, chen2024graphwizinstructionfollowinglanguagemodel, luo2024graphinstruct, chai2025graphllm}. In contrast, code-based graph reasoning methods exploit LLMs’ coding capabilities to generate executable programs and delegate complex structural reasoning to program execution~\cite{cai2024codegraphenhancinggraphreasoning, zhang2024gcoderimprovinglargelanguage, li2025graphteamfacilitatinglargelanguage, wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms}. Many code-based approaches further incorporate retrieval augmentation to compensate for limited or unreliable algorithmic knowledge in LLMs: they retrieve task-relevant algorithmic knowledge from external technical documentation (such as graph library APIs or algorithm textbooks) using flattened vector-based retrieval methods (e.g., TF-IDF~\cite{salton1988term}) and use the retrieved content to guide code generation~\cite{zhang2024gcoderimprovinglargelanguage, li2025graphteamfacilitatinglargelanguage}.

Despite their promise, existing graph reasoning methods remain insufficient for handling complex graph reasoning tasks. On the one hand, for \emph{structurally complex} large graphs, text-based reasoning methods face a major bottleneck due to context window limitations, as serializing graph structures into prompts can quickly exceed LLM context limits (e.g., a fully connected graph with approximately $180$ nodes already surpasses the $160$k-token context window of DeepSeek-V3~\cite{liu2024deepseek}). Even when large graphs fit within the context window, long-horizon reasoning over lengthy graph representations challenges LLMs’ long-context reasoning capabilities.
On the other hand, although code-based methods alleviate the context-window bottleneck by offloading structural reasoning to program execution, they face new difficulties on \textit{semantically complex} composite graph tasks, which place heavier demands on algorithmic knowledge integration and the retrieval module. Such tasks require using multiple classical graph algorithms and introduce problem-level semantic complexity (e.g., identifying strongly connected components before computing shortest paths under latency constraints in communication networks). While these composite tasks are more practical, they are also more challenging, as they require stronger algorithmic understanding and more accurate retrieval of sufficient task-relevant documents to guide code synthesis.


% \begin{table}[htbp]
% \centering
% \caption{Comparison of Graph Reasoning Methods by Key Design Dimensions}
% \label{tab:method_comparison}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccccccc}
% \toprule
% \textbf{Method} 
% & \textbf{Text-based} 
% & \textbf{Code-based} 
% & \textbf{Syntax Debug} 
% & \textbf{Logic Debug} 
% & \textbf{Retrieval} 
% & \textbf{Hierarchical} 
% & \textbf{Not Human-involved} \\
% \midrule
% \textsc{CoT}~\citeyearpar{wei2022chain} 
% & \checkmark &  &  &  &  &  & \checkmark \\
% \textsc{CoT-BaG}~\citeyearpar{wang2024languagemodelssolvegraph}
% & \checkmark &  &  &  &  &  & \checkmark \\
% \textsc{GCoder}~\citeyearpar{zhang2024gcoderimprovinglargelanguage} 
% &  & \checkmark &  &  & \checkmark &  & \checkmark \\
% \textsc{GraphTool-Inst.}~\citeyearpar{wang2025GraphToolInstruction} 
% &  & \checkmark &  &  &  &  & \checkmark \\
% \textsc{GraphTeam}~\citeyearpar{li2025graphteamfacilitatinglargelanguage} 
% &  & \checkmark & \checkmark &  & \checkmark &  & \checkmark \\
% \textsc{CodeGraph}~\citeyearpar{cai2024codegraphenhancinggraphreasoning} 
% &  & \checkmark &  &  &  &  &  \\
% \textsc{PIE}~\citeyearpar{gong2025pseudocodeinjectionmagicenablingllms} 
% &  & \checkmark & \checkmark &  &  &  &  \\
% \textsc{Ours} 
% &  & \checkmark & \checkmark & \checkmark & \checkmark & \checkmark & \checkmark \\
% \bottomrule
% \end{tabular}
% }
% \end{table}
\begin{table}[t]
\centering
\caption{Code-Based Graph Reasoning Comparison (Ret.: retrieval; Hi.: hierarchical; Non-human: no human involvement).
% \fali{add considering large?, text and code-based and ft or not}
}
\vskip -1em
\label{tab:method_comparison}
\resizebox{0.99\linewidth}{!}{
\begin{tabular}{lcccccc}
\hline
\textbf{Method} 
& \textbf{Coding} 
& \textbf{\makecell{Runtime\\Debug}} 
& \textbf{\makecell{Logic\\Debug}} 
& \textbf{Ret.} 
& \textbf{Hi.} 
& \textbf{\makecell{Non-\\Human}} \\
\hline
\textsc{GCoder}~\citeyearpar{zhang2024gcoderimprovinglargelanguage} 
& \checkmark &  &  & \checkmark &  & \checkmark \\
\textsc{GraphTool-Inst.}~\citeyearpar{wang2025GraphToolInstruction} 
& \checkmark &  &  &  &  & \checkmark \\
\textsc{GraphTeam}~\citeyearpar{li2025graphteamfacilitatinglargelanguage} 
& \checkmark & \checkmark &  & \checkmark &  & \checkmark \\
\textsc{CodeGraph}~\citeyearpar{cai2024codegraphenhancinggraphreasoning} 
& \checkmark &  &  &  &  &  \\
\textsc{PIE}~\citeyearpar{gong2025pseudocodeinjectionmagicenablingllms} 
& \checkmark & \checkmark & \checkmark &  &  &  \\
\textsc{Ours} 
& \checkmark & \checkmark & \checkmark & \checkmark & \checkmark & \checkmark \\
\hline
\end{tabular}
}
\vskip -2em
\end{table}



% However, we argue that these approaches suffer from two fundamental limitations.
% \textbf{(i) They largely overlook the hierarchical directory structure of technical documentation.} In practice, technical resources are organized as tree-structured hierarchies, where higher-level directories provide coarse-grained semantic cues and lower-level nodes contain fine-grained algorithmic details. Exploiting this structure is crucial for effective retrieval: pruning irrelevant branches at higher levels can substantially reduce the search space, lower retrieval noise, and improve both efficiency and accuracy. As shown in Fig.~\ref{x}, our preliminary experiments compare semantic embedding-based retrieval methods (TF-IDF and Sentence-BERT) and agent traversal over flat document collections with hierarchical agent traversal over tree-structured documentation. Explicitly modeling the document hierarchy significantly improves the retrieval agent’s accuracy in identifying precise reference documents (from $x$ to $x$), while also reducing retrieval time (from $x$s to $x$s), compared to hierarchy-agnostic agent search.
% \textbf{(ii) Existing methods lack a systematic code debugging mechanism.} Approaches such as PIE \cite{gong2025pseudocodeinjectionmagicenablingllms}, GraphTool-Instruction \cite{wang2025GraphToolInstruction}, and GCoder \cite{zhang2024gcoderimprovinglargelanguage} typically adopt a one-shot code generation strategy, even though the initially generated code frequently contains syntactic or logical errors. Although GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} introduces iterative compiler-based debugging, it primarily addresses syntactic issues and remains limited in resolving logical errors. As shown in Fig.~\ref{x}, our analysis of self-debugging outcomes, with and without retrieval augmentation, indicates that after $x$ rounds of debugging, syntactic errors account for only $x$ of failures, whereas logical errors account for $x$. This observation suggests that logical errors dominate the debugging process and are not adequately handled by existing approaches.
% Motivated by these observations, we investigate the following research question: \emph{Under a task setting where graph task descriptions are decoupled from graph data, how can we effectively exploit the hierarchical structure of technical documentation and integrate systematic code generation and debugging mechanisms to produce robust and scalable programmatic solutions for both large-scale graph reasoning and composite graph reasoning tasks?}
% \suhang{1. We were talking about introducing a new dataset, then suddenly we talk about code-based methods. Somehow, the logic is not smooth. The logic is kindly of jump around. 2. Are code-based methods naturally capable of addressing complex graph reasoning tasks?? I cannot get why code-based can handle complex graph reasoning tasks. Can we add one intuition? Do we have specail design for handling complex graph reasoning tasks?? If yes, we should not say code-based can handle 3. What is our key selling point, the framework or the dataset? If framework/methodology is our key selling point, I think it might be better to not mention dataset in the middle, instead, put it at the end. For example, one story could be: (i). Graph reasoning is important, explain what is graph reasoning and why it is important; (ii). LLMs are used to automate the graph reasoning, categorize existing methods, give high-level idea of each method and issue, then say code-based is the the most promising. Thus, we focus on code-based. (iii). However, existing code-based did not consider xxxx, explain in detail as you did; (iv). To fill the gap, we propose xxx; (v). briefly mention that we also developed a dataset, explain why we construct this dataset (In this way, we do not jump around, but focus on methodology)  } 
% We summarize representative code-based methods capable of addressing complex graph reasoning tasks in Tab.~\ref{tab:method_comparison} and observe that existing non-human-involved and retrieval-based approaches 
% \suhang{there is something missing. what is retrieval-based approaches? we should introduce the high-level idea in the first paragraph that I added the comments, or somewhere appropriate} 
% share two key limitations.
% \textbf{(i) They ignore the hierarchical structure of technical documentation.} 
% Technical documentation is naturally organized as tree-structured hierarchies, with high-level directories providing coarse semantic grouping and lower-level nodes containing detailed algorithmic or interface entries, a structure that is ubiquitous in modern technical documentation, to support systematic navigation.
% Exploiting this structure enables early pruning of irrelevant branches at higher levels, substantially reducing the search space, lowering retrieval noise, and improving both retrieval efficiency and accuracy. As shown in Tab.~\ref{tab:retrieval_results}, hierarchical agent traversal substantially outperforms flat semantic retrieval methods (TF-IDF~\cite{salton1988term}, Sentence-BERT~\cite{reimers2019sentence}), improving retrieval accuracy from $58\%$ to $88\%$ and, compared to flat agent retrieval, significantly reducing search time from $1095$s to $76$s per task.
% \textbf{(ii) Existing methods lack systematic code debugging.} Methods such as GraphTool-Instruction \cite{wang2025GraphToolInstruction} and GCoder \cite{zhang2024gcoderimprovinglargelanguage} rely on one-shot code generation, despite frequent syntactic and logical errors. While GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} introduces compiler-based debugging, it mainly resolves syntactic errors. As shown in Fig.~\ref{fig:err_distribution}, logical errors account for all failures in DeepSeek and LLaMA 70B, indicating that they dominate but remain insufficiently addressed.
% Motivated by these findings, we propose a novel research problem: \emph{Under a task setting where graph task descriptions are decoupled from graph data\suhang{do not emphasize graph task descriptions are decoupled from graph data, it's not a setting, it's just how you handle the problem. Just delete it}, how can hierarchical technical documentation be leveraged to improve retrieval, and how can LLMs integrate the retrieved knowledge to generate and debug robust code solutions for complex graph reasoning tasks?}

As the scalability of code-based methods for complex graph reasoning under both structural and semantic complexity, we focus on this paradigm. Tab.~\ref{tab:method_comparison} summarizes representative code-based methods and highlights two fundamental limitations of existing non-human-involved retrieval-based approaches. 
% \textbf{First, existing methods ignore the hierarchical structure of technical documentation.}
% Technical documentation is naturally organized as a tree, where high-level directories provide coarse semantic grouping and lower-level nodes contain detailed algorithmic or interface descriptions. This hierarchy is pervasive in modern technical documentation and is designed to support systematic navigation. Treating documentation as flat text results in noisy retrieval, inefficient search, and unnecessary reasoning overhead. In contrast, exploiting the hierarchy enables early pruning of irrelevant branches, substantially reducing the search space and improving both retrieval efficiency and accuracy. As later shown in Fig.~\ref{fig:retrieval_results}, hierarchical agent traversal improves retrieval F1 from $\sim\!28\%$ to $79\%$ over flat semantic retrieval (TF-IDF~\cite{salton1988term}, Sentence-BERT~\cite{reimers2019sentence}), while reducing per-task search time from $23.3$s to $9.1$s compared to flat agent retrieval. 
\textbf{First, existing retrieval is brittle under semantic complexity and ignores documentation hierarchy.}
For \emph{semantically complex} composite graph tasks, solving a query often requires retrieving multiple complementary algorithmic documents, placing high demands on semantic retrieval. Most RAG-style retrievers treat documentation as a flat corpus and rank top-$k$ documents by shallow similarity, which frequently misses compositional intent, leading to low recall at small $k$ and heavy noise at large $k$. Flat agentic retrieval can improve relevance via LLM screening, but it is costly and slow at scale. In contrast, technical documentation is inherently hierarchical, with coarse grouping at higher levels and precise algorithmic/API details at lower levels, making it well-suited for efficient agentic navigation. Our agentic hierarchical retrieval exploits this structure via top-down traversal and early pruning, achieving coarse-to-fine disambiguation with fewer LLM calls. It improves retrieval F1 from $\sim\!28\%$ to $79\%$ and reduces per-task search time from $23.3$s to $9.1$s compared to flat agent retrieval (Fig.~\ref{fig:retrieval_results}).
\textbf{Second, existing methods lack systematic code debugging.}
Most approaches, such as GraphTool-Instruction~\cite{wang2025GraphToolInstruction} and GCoder~\cite{zhang2024gcoderimprovinglargelanguage}, rely on one-shot code generation and thus frequently suffer from runtime and logical errors. While GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage} introduces compiler-based debugging, it mainly resolves runtime issues. As shown in Fig.~\ref{fig:pilot}(a), logical errors account for most failures across models, suggesting that logic-level errors dominate yet remain unaddressed. Motivated by these observations, we pose a novel research question: \emph{How can hierarchical technical documentation be exploited to improve retrieval, and how can LLMs integrate the retrieved knowledge to generate and systematically debug robust code solutions for complex graph reasoning tasks?}



% \fali{convince reviewers the hierarchy exists widely, not only in NetworkX}

% and PIE \cite{gong2025pseudocodeinjectionmagicenablingllms} 这个baselines需要手工设计examples and text cases, 因此不在比较之列


% Following the existing task-graph decoupling paradigm~\cite{gong2025pseudocodeinjectionmagicenablingllms} and retrieval from graph-related technical documentation (i.e., NetworkX \cite{hagberg2008exploring}) \cite{li2025graphteamfacilitatinglargelanguage}, to further address the limitations, 
% To this end, we propose {\method}, a \textbf{documentation-guided agentic hierarchical retrieval--augmented coding framework for complex graph reasoning}, consisting of a \texttt{retrieval agent} and a \texttt{coding agent}.
% To exploit the hierarchical structure of technical documentation, the retrieval agent employs a \textit{layer-wise agentic retrieval algorithm} over the document tree. Starting from the root nodes, the retrieval agent traverses the document tree in a top-down manner, retaining only task-relevant nodes and pruning irrelevant branches at higher levels. It then iteratively descends to lower-level nodes until reaching the leaf-level algorithmic entries. The resulting task-relevant entries are then provided to the coding agent as external knowledge.
% Guided by the retrieved results, the coding agent incorporates a \textit{self-debugging mechanism} to improve code robustness. It first autonomously generates small-scale graph test cases with reliable labels, leveraging the LLM’s strong reasoning capability on simple graphs (see \textcolor{black}{Tab.~\ref{tab:text_reasoning_scale} in Appendix}, which shows that LLMs achieve strong performance on small-scale graphs when guided by retrieval, thereby validating the reliability of the self-generated test cases). The agent then generates candidate code and iteratively debugs and refines it in a code compiler environment until all test cases are passed or the maximum debugging limit is reached. The validated code is finally executed on graph instances to produce the final answers. \suhang{shall we mention how do we handle composite/complex graph task, is it part of our novelty???}
% Compared to existing LLM-based retrieval-augmented coding approaches, 
% By shrinking the retrieval search space to the retrieval agent, our method reduces inference cost and minimizes noisy retrieval to the coding agent. 
% By shrinking the retrieval search space to the retrieval agent, our method minimizes noisy retrieval to the coding agent. 
% To systematically evaluate performance under varying complexity, we construct a complex graph reasoning dataset {\dataset}, including three variants: \texttt{\dataset-S}, \texttt{\dataset-L}, \texttt{\dataset-C}, covering small-scale, large-scale, and composite graph reasoning tasks. 


To this end, we propose {\method}, a \textbf{documentation-guided agentic hierarchical retrieval--augmented coding framework for complex graph reasoning}, consisting of a \texttt{retrieval agent} and a \texttt{coding agent}.
To retrieve high-quality documents for complex tasks at low cost, our retrieval agent exploits the hierarchical structure of technical documentation via a \textit{layer-wise agentic retrieval} algorithm over the document tree.
% To retrieve high-quality documents in complex tasks and exploit the hierarchical structure of technical documentation for less cost, the retrieval agent performs a \textit{layer-wise agentic retrieval algorithm} over the document tree. 
Starting from the root nodes, the retrieval agent traverses the document tree in a top-down manner, retaining only task-relevant nodes and pruning irrelevant branches at higher levels. It then iteratively descends to lower-level nodes until reaching the leaf-level algorithmic entries. The resulting task-relevant entries are then provided to the coding agent as external knowledge.
Guided by the retrieved documents, the coding agent integrates a \emph{self-debugging mechanism} to improve code robustness. Since debugging logical errors typically requires labeled test cases, yet task-specific test cases are rarely available in fully automated settings, the agent autonomously generates small-scale graph test cases with reliable labels. This design leverages the observation that LLMs can solve small graphs reliably when supplied with correct algorithmic guidance (Fig.~\ref{fig:pilot} shows $100\%$ accuracy on graphs with fewer than $10$ nodes in our pilot study). The agent then generates candidate code and iteratively debugs and refines it in a code compiler environment until all test cases are passed or the maximum debugging limit is reached.  The validated code is finally executed on graph instances to produce final answers.
To enable systematic evaluation of complex graph reasoning, we additionally introduce a new dataset, {\dataset}, capturing both \emph{large-scale structural complexity} and \emph{problem-level semantic complexity}, which comprises three variants: \texttt{\dataset-S} (small-scale), \texttt{\dataset-L} (large-scale), and \texttt{\dataset-C} (composite).
Experimental results show that text-based reasoning becomes nearly ineffective on our large-scale setting (e.g., DeepSeek-V3 achieves $<15\%$ accuracy), whereas code-based approaches remain effective. Moreover, {\method} consistently outperforms retrieval- and debugging-based coding baselines on composite tasks (e.g., with Qwen-2.5-7B, \textsc{GraphTeam} $56.7\%$ accuracy vs.\ \textsc{Ours} $73.3\%$), validating the benefits of our hierarchical retrieval and logic-aware self-debugging.

Our \textbf{main contributions} are: (i) We identify two key limitations in prior code-based graph reasoning, \emph{flat document retrieval} and missing \emph{logic-level debugging}; (ii) We propose {\method}, a documentation-guided agentic framework that unifies hierarchical retrieval with self-debugging code generation for complex graph reasoning; (iii) We introduce a new dataset, {\dataset}, for controlled evaluation of complex graph reasoning across increasing complexity, comprising small-scale, large-scale, and composite subsets; and (iv) Extensive experiments on \texttt{GTools} and {\dataset} show that our method improves retrieval quality and code robustness, achieving state-of-the-art performance while maintaining low inference cost compared to agentic flat retrieval.
% \begin{itemize}[nosep,leftmargin=*]
% \item We identify two key limitations in prior code-based graph reasoning, \emph{flat document retrieval} and missing \emph{logic-level debugging}. 
% \item We propose {\method}, a documentation-guided agentic framework that unifies hierarchical retrieval with self-debugging code generation for complex graph reasoning.
% \item We introduce a new dataset, {\dataset}, for controlled evaluation of complex graph reasoning across increasing complexity, comprising small-scale, large-scale, and composite subsets.
% \item Extensive experiments on \texttt{GTools} and {\dataset} show that our method improves retrieval quality and code robustness, achieving state-of-the-art performance while maintaining low inference cost compared to agentic flat retrieval.
% \end{itemize}


% \begin{itemize}[nosep,leftmargin=*]
%     % \item We investigate a novel problem of leveraging the hierarchical directory structure of technical documentation to improve code generation quality, and propose \method, a novel documentation-guided, hierarchical agentic retrieval--augmented coding framework with self-debugging for graph reasoning.
%     % \item We study a novel problem of exploiting the hierarchical structure of technical documentation to improve code-based graph reasoning, and propose {\method}, a documentation-guided, agentic hierarchical retrieval–augmented coding framework with self-debugging for graph reasoning.
%     % \item We identify two key limitations of existing code-based graph reasoning methods—ignoring the hierarchical structure of technical documentation during retrieval and lacking logical error checking during debugging—and propose {\method}, which performs retrieval over document hierarchies and incorporates self-debugging to refine code generation.
%     \item We identify two limitations of current code-based graph reasoning, flat document retrieval and missing logic-level debugging, and propose {\method}, performing hierarchical document retrieval and self-debugged code generation.
%     % \item We introduce a new complex graph reasoning benchmark, {\dataset}, consisting of three complementary subsets: \texttt{\dataset-S}, \texttt{\dataset-L}, \texttt{\dataset-C}, which target small-scale graphs, large-scale graphs, and composite graph reasoning tasks, respectively. This benchmark fills an important gap in existing datasets by enabling systematic evaluation of complex graph reasoning scenarios.
%     \item We introduce a new complex graph reasoning dataset, {\dataset}, with three subsets: \texttt{\dataset-S}, \texttt{\dataset-L}, and \texttt{\dataset-C}, thereby enabling systematic evaluation of complex graph reasoning.
%     % \item Extensive experiments on {\dataset} and the GTools benchmark demonstrate that {\method} achieves more effective retrieval and higher-quality code generation for graph reasoning, particularly on large-scale graphs and composite tasks, while also offering lower inference cost compared to existing agent-based retrieval and coding approaches.
%     \item Extensive experiments on GTools and {\dataset} show that our method achieves effective retrieval and high-quality code generation, especially on large-scale and composite graph tasks. 
%     % while \textcolor{red}{reducing inference cost compared to existing agent-based approaches}.
% \end{itemize}


% Large language models (LLMs) have rapidly advanced in natural language understanding, code generation \cite{deepseekai2024deepseekcoderv2breakingbarrierclosedsource, touvron2023llamaopenefficientfoundation, openai2024gpt4technicalreport}, and reasoning tasks  \cite{shao2024deepseekmathpushinglimitsmathematical, touvron2023llamaopenefficientfoundation, openai2024gpt4technicalreport}. Their ability to generalise across domains has opened the possibility of applying them to structured problems that traditionally fall within the scope of algorithmic reasoning. Among these problems, graph reasoning holds particular significance. Graphs are a universal abstraction for representing relationships, spanning various domains. Core graph tasks, including connectivity testing, shortest-path computation, and cycle detection, are fundamental components in algorithms and applications. As such, the ability of LLMs to reason about graph tasks is both of theoretical interest and practical importance.

% Despite these opportunities, existing studies reveal that LLMs face two fundamental barriers when applied directly to graph reasoning in natural language. First, their performance deteriorates as graphs grow larger (i.e., reasoning becomes less accurate) \cite{minaee2025largelanguagemodelssurvey}, and in many cases the graph representation itself exceeds the token context window, preventing the model from fully capturing structural details \cite{chen2024graphwizinstructionfollowinglanguagemodel}. Second, direct reasoning often generates one-off solutions rather than reusable procedures. In practice, this means that the same graph task must be solved separately for each input graph, rather than generating a general solution that applies across multiple instances. Taken together, these limitations reduce both the portability and effectiveness of LLMs for graph reasoning tasks.

% A promising alternative is to shift the problem from direct reasoning to code generation. Instead of requiring the LLM to solve a graph problem step by step, the model can be prompted to produce code for resolving the given graph tasks. This type of approaches offers several advantages. It passes the computation to the computational devices rather than the LLMs' reasoning, thereby bypassing token-length bottlenecks. It also preserves structural consistency across different graph instances, since the same code can be executed on graphs of varying sizes and densities. However, existing work in this direction is constrained by three practical challenges. First, works that require fine-tuning LLMs~ \cite{wang2025GraphToolInstruction,zhang2024gcoderimprovinglargelanguage} for graph reasoning consume substantial computational resources, which hinders the feasibility for graph tasks at scale. Second, existing retrieval-based approaches~ \cite{li2025graphteamfacilitatinglargelanguage} that ignore the hierarchical structure of documentation often suffer from poor alignment between the input graph task description and the underlying graph library, making it difficult to accurately pinpoint the most relevant function documentation. Third, although few-shot prompt-based methods~ \cite{gong2025pseudocodeinjectionmagicenablingllms,cai2024codegraphenhancinggraphreasoning} have showcased extraordinary performance by prompting the LLMs with reliable knowledge to reference, they still require humans to obtain the related documentations.

% To overcome these challenges, this thesis proposes agentic Retrieval-Augmented Generation (agentic RAG), an automated framework for graph reasoning with LLMs. Unlike prior methods, our framework does not require fine-tuning or manual selection. Instead, it combines three cooperating agents. The framework starts from a retrieval agent that navigates the hierarchical structure of NetworkX \cite{hagberg2008exploring} documentation to identify the most relevant function documentation page through a hierarchical search process by narrowing from high-level categories to submodules and individual function pages. This hierarchical search not only improves retrieval accuracy by leveraging structured contextual relationships between documents but also enhances efficiency by avoiding exhaustive keyword matching across the entire corpus. Then, it is followed by a verifier agent for verifying the correctness of retrieved documentation, and a coding agent that integrates this information to generate, test, and iteratively debug executable code for solving the graph task. By unifying structured retrieval with agentic code generation, agentic RAG achieves scalability to stronger generalisability across graph categories, and robustness without human-in-the-loop intervention.

% To evaluate this approach, we extend a existing benchmark dataset GraphEval36K \cite{wu-etal-2025-grapheval36k} that systematically spans diverse graph types and problem time complexity settings. This benchmark enables fine-grained assessment of LLMs performance under different retrieval and reasoning configurations. Empirical studies with multiple state-of-the-art LLMs demonstrate that agentic RAG consistently outperforms direct code generation and generic retrieval methods, underscoring the importance of structured documentation retrieval for enabling effective graph reasoning.

% \paragraph*{Contributions} 
% The main contributions of this thesis are as follows: (i) \textbf{Framework} We introduce \textbf{agentic RAG}, an automated pipeline that integrates hierarchical documentation agentic retrieval with LLM-based code generation and iterative debugging for graph computational tasks; (ii) \textbf{Benchmark Setting} we follow the existing benchmark configuration of GraphEval36K \cite{wu-etal-2025-grapheval36k}, which encompasses eight main graph categories, each further divided into subcategories reflecting connectivity, cyclicity, weightedness, and directionality; and (iii) \textbf{Evaluation} We conduct a systematic empirical study of multiple state-of-the-art LLMs under diverse conditions, showing that agentic RAG improves accuracy and generalisability on average compared to baselines.  

% \begin{itemize}
%     \item \textbf{Framework} We introduce \textbf{agentic RAG}, an automated pipeline that integrates hierarchical documentation agentic retrieval with LLM-based code generation and iterative debugging for graph computational tasks.
        
%     \item \textbf{Benchmark Setting} we follow the existing benchmark configuration of GraphEval36K \cite{wu-etal-2025-grapheval36k}, which encompasses eight main graph categories, each further divided into subcategories reflecting connectivity, cyclicity, weightedness, and directionality.  
        
%     \item \textbf{Evaluation} We conduct a systematic empirical study of multiple state-of-the-art LLMs under diverse conditions, showing that agentic RAG improves accuracy and generalisability on average compared to baselines.
        
% \end{itemize}

% \fali{do we have experimental results to show the efficiency?}
\vskip -3em
\section{Related Work}
% \fali{think carefully about the differences among \textit{graph computation, graph reasoning, graph learning, and graph problem solving}}

% \fali{too long. Learning the writing style of my paper: https://arxiv.org/pdf/2402.11441} \fali{do not simply list the related work, need to analyze their drawbacks that motivate the next line of works.}

% \fali{Firstly, the current framing appears to focus on only one research direction: LLM for Graph reasoning. The related work section should instead cover the multiple research directions relevant to our research question. In our paper, the directions most closely aligned with our research question include RAG(LLM agents represent a related but less related, so we can merge them and introduce representative studies on agentic RAG), and LLMs for graph reasoning. Therefore, we should structure the related work into two research directions: one on LLMs for graph reasoning and another on RAG, with emphasis on works related to agentic RAG and RAG for coding and coding for reasoning. Secondly, current writing is not concise enough, I suggest not to use subsection latex code and instead use ``nointent textbf'' for less space (see example below). Then each research direction only has one to two paragraphs (usually one paragraph is enough). }

% \fali{introduce the motivation of the research direction and then introduce multiple categories. Then list related works briefly. End with the difference and novelty of our paper and we focus on xxx.}

% \fali{consider InstructGLM \cite{ye2024language} that for node classification and explicitly that not our target problem. And \cite{chen2024exploring} used LLMs as enhancers to exploit text attributes to be used in a graph learning model or as predictors for node classification on text-attributed graphs.}

% \fali{do not create terms by yourself, like ``graph task reasoning''. We should prioritize reusing the terms others commonly use.} \fali{we only need to provide the long term and the short term at first time, and then always use the short term for brevity.}\fali{the logic is strange, graph reasoning is not an intuitive task for LLMs. } \fali{give the task description first, like given graph task desc, graph input, the aim is to xxx.} \fali{when giving some arguments, need some cite to show the evidence} \fali{the key is not surface-level text inference, refine the reason, please refer to other LLMs for Graph reasoning papers.}. \fali{give the categories first, and then use explicit separate words, like "\textit{For xxx}, high-level pipeline. Methods such as x which xxx, x which xxx.", "Then"} \fali{one sentence describes the input and output of this task} \fali{categories should explicitly consider how graph input is handled.} \fali{here, explicitly corresponds to the category above.} 

% \noindent\textbf{LLMs based Graph Reasoning} Despite strong performance on NLP tasks \cite{zhao2025surveylargelanguagemodels}, LLMs struggle with graph reasoning, which requires explicit graph-structure modeling to map task descriptions and graph data to task-specific outputs \cite{2024graphmeetllmli}. This is because it requires explicit modeling of graph structure beyond textual task descriptions. To address this gap, growing research efforts aim to enhance LLMs’ graph reasoning capabilities \cite{2023llmlearninggraphchen, jiang-etal-2023-structgpt}. 
% From the perspective of graph data handling, existing approaches can be categorized into two paradigms: prompt-based graph reasoning, which injects graphs into prompts, and code-based graph reasoning, which utilizes executable code to process graph information.
% (1) \textit{Prompt-based} approaches convert task descriptions and graph data into natural language, and treat the problem as direct textual inference. NLGraph \cite{wang2024languagemodelssolvegraph} are among the first works to systematically study prompt-based graph reasoning by designing task-conditioned natural-language prompts that explicitly encode graph structure. In its standard prompting setup, graph data are represented as textual descriptions of edge lists, weights, or capacities placed before the reasoning query, demonstrating preliminary graph reasoning ability. Building on this formulation, NLGraph further proposes two graph-specific prompt designs: Build-a-Graph prompting, which adds an instruction encouraging explicit graph construction before reasoning, and Algorithmic prompting, which leverages task-specific algorithmic descriptions to guide structured, step-by-step reasoning. Later work, Talk-like-a-Graph \cite{fatemi2023talklikegraphencoding} built a benchmark dataset to further explore the varying graph-to-text encodings, by including different node naming schemes (e.g., numeric IDs, common names) and edge verbalisations (e.g., adjacency tuples, directional arrows) along with the task description in the prompt. Despite these extensive explorations, prompt format variation alone remains insufficient for complex graph tasks such as maximum flow. GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} improves reasoning quality through model alignment by applying Direct Preference Optimisation \cite{rafailov2024directpreferenceoptimizationlanguage} to align LLM outputs with the human reasoning process. It considers a simplified graph reasoning setting in which graphs are provided as edge list tuples, without introducing new graph representation strategies. Other methods, such as CodeGraph \cite{cai2024codegraphenhancinggraphreasoning}, PSEUDO \cite{gong2025pseudocodeinjectionmagicenablingllms}, and GCoder \cite{zhang2024gcoderimprovinglargelanguage}, encode graph information directly within prompts and improve program synthesis through injected code exemplars, pseudocode with reflection-based debugging, or supervised fine-tuning with compiler-guided reinforcement learning. \fali{revise them to echo the "prompt-based", not describe their use of prompt engineering, which is so superficial}
% However, they do not explicitly address reasoning over graphs that exceed the context window, restricting reasoning over large graphs. Moreover, they generate one-off answers, which suffer from the problem of reusability. 
% % \fali{this one limitation is enough to transition to the next point. put a little more words for it.} . \fali{use apparent reminder to transition into next point, like \textit{italic} or some signposting words or (1)(2)(3).} 
% % \fali{keep precise and concise expression for the category, use consistent expression in the categories above.}\fali{the expression is not clear, should say ``LLMs generate executable code so that the prompt contains only the task description, rather than lengthy graph inputs.''},\fali{need to highlight how their method implements coding to solve graph reasoning, ignore other irrelevant content.}.
% (2) \textit{Code-based} work leverages LLM-generated code for handling graph data, which mean the graph information is processed with generated code, so that the prompt contains only the task description, rather than lengthy graph inputs. GraphTool-Instruction \cite{wang2025GraphToolInstruction} pioneers large-graph reasoning beyond the context window \fali{unclear, what do you mean beyond the context window} by using a fine-tuned LLM to translate task descriptions into executable graph tool calls and to extract graph information as structured parameters for tool execution.  Although GraphTool-Instruction addresses the problem of large graph reasoning, it still requires fine-tuning at each decomposed sub-task via a human-crafted dataset, which hinders scalability to various graph tasks \fali{use another drawback, rely on existing tool library, which is more aligned with our research question.}\chenglin{For our method, we are using the existing tool library NetworkX as well. I am not sure if we can discuss this drawback} \fali{NX is not like a tool library in GraphTool-Instruction; it is a package with a documentation. If the term "tool library" is general, consider differentiating these two libraries in GraphTool-Instruction and ours in another term. I think the GraphTool-Instruction describes the tool set as a list of tools with Tool Name/Description/Parameters/Return type, but ours focuses on a tool library with a tree-structure documentation. So consider saying the drawback of GraphTool-Instruction is dependent on a list of tools, but in the real world, the tools are documented as a set of functions with a documentation. }. Our method targets scalable graph reasoning by automatic tool documentation retrieval and code generation from solely the graph description, avoiding both explicit graph representation in prompts and manual construction of dataset.
% \fali{I suggest from the perspective of how graph inputs are handled, existing approaches fall into two categories: (1) directly embed graph input into the LLM prompt, and (2) encode graph input through LLM-generated code. In (2), we have direct coding, fine-tuning-based coding, and retrieval-based coding (need to pay particular attention because these are the closest works to ours. need to select the closest work to exclusively state the difference from ours.). }
% \fali{not deep enough, what is the common drawback in these works, motivating us to focus on [our research question]}.

\noindent\textbf{Benchmarking LLMs on Graph Reasoning} Despite the success of LLMs in natural language processing, their ability to perform graph reasoning tasks, such as cycle detection, remains limited. Early benchmarks, including GraphQA \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph}, evaluated LLMs on basic graph reasoning and showed that, while LLMs exhibit preliminary graph reasoning capabilities, their performance falls short of ideal levels. Motivated by these findings, subsequent datasets such as GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel}, GraphInstruct \cite{luo2024graphinstruct}, and LLM4DyG \cite{zhang2024llm4dyg} expanded task coverage to more complex and dynamic graph reasoning settings. However, existing benchmarks remain limited to small-scale graphs and classic graph algorithmic tasks, and do not evaluate large-scale beyond the context window and composite graph reasoning. To address this gap, we introduce a new dataset, covering small- and large-scale and composite graph reasoning tasks.
% spanning graph sizes from $20$ to $10,000$ nodes

\noindent\textbf{LLM-based Graph Reasoning Methods.}
Recent work has sought to improve LLMs’ limited performance on graph reasoning tasks, which can be broadly categorized into \emph{text-based} and \emph{code-based} paradigms. Text-based methods perform natural language reasoning over text-serialized graph structures using prompting strategies such as chain-of-thought and self-consistency, as studied in Talk-like-a-Graph \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph}, with further enhancement through instruction tuning in GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} and GraphInstruct \cite{luo2024graphinstruct}. Code-based approaches generate and execute graph algorithm code, including CodeGraph \cite{cai2024codegraphenhancinggraphreasoning}, and GraphTeam \cite{li2025graphteamfacilitatinglargelanguage}. 
% However, these require serializing graph data into prompts, which becomes infeasible as the graph size exceeds the LLM’s context window. To address this limitation, recent work explores a \emph{task--graph decoupling paradigm} that decouples task descriptions from graph data, such as GraphTool-Instruction \cite{wang2025GraphToolInstruction} and PIE \cite{gong2025pseudocodeinjectionmagicenablingllms}, though these rely on human-designed tools or pseudocode. 
% Therefore, we propose an automatic agentic retrieval-augmented coding framework based on existing technical documents, without human involvement.
However, these methods remain limited by context-window constraints and, more critically, by the lack of logic-level debugging. To address these issues, we propose a self-debugging coding framework that produces robust executable programs by leveraging self-generated test cases and iterative, feedback-driven refinement. Appendix~\ref{sec:appendix_related_reasoning} for more details.
% \noindent\textbf{LLM-based Graph Reasoning Methods.}
% Due to the unsatisfactory performance of LLMs on graph reasoning tasks, recent work has sought to enhance their graph reasoning capabilities. Existing approaches can be broadly categorized into \emph{text-based} and \emph{code-based} graph reasoning. Text-based methods rely on natural language reasoning over text-serialized graph structures using prompting strategies such as chain-of-thought and self-consistency, as evaluated by Talk-like-a-Graph \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph}. To improve performance, instruction tuning has been introduced, including GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} and GraphInstruct \cite{luo2024graphinstruct}, though such methods exhibit limited transferability \cite{zhang2024can}. Code-based approaches generate and execute graph algorithm code, such as CodeGraph \cite{cai2024codegraphenhancinggraphreasoning}, GCoder \cite{zhang2024gcoderimprovinglargelanguage}, and GraphTeam \cite{li2025graphteamfacilitatinglargelanguage}. However, both the above text-based and code-based approaches require serializing graph data into prompts, making them infeasible as the graph size exceeds the LLM’s context window and severely limiting scalability in large-scale graph reasoning. 
% To overcome the context window bottleneck, recent work explores a \emph{task--graph decoupling paradigm}, where prompts contain only task descriptions and graph data is processed externally via generated code, including GraphTool-Instruction \cite{wang2025GraphToolInstruction} and PIE \cite{gong2025pseudocodeinjectionmagicenablingllms}, which alleviate this issue but depend on human-manual tools or human-selected pseudocode. 
% Based on these insights, we propose a retrieval-augmented, code-based reasoning paradigm that decouples task descriptions from graph data, enabling scalable graph reasoning beyond window sizes and human-involving constraints. See more detailed related work in Appendix \ref{sec:appendix_related_reasoning}. 
% \noindent\textbf{LLM-based Graph Reasoning Methods.}
% Recent work has sought to improve LLMs’ limited performance on graph reasoning tasks by adopting two main paradigms: \emph{text-based} and \emph{code-based} graph reasoning. Text-based methods perform natural language reasoning over text-serialized graphs using prompting or instruction tuning, as studied in Talk-like-a-Graph \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{ wang2024languagemodelssolvegraph}, with further improvements via GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} and GraphInstruct \cite{luo2024graphinstruct}, though transferability remains limited \cite{zhang2024can}. Code-based approaches generate and execute graph algorithms, such as CodeGraph, GCoder, and GraphTeam \cite{cai2024codegraphenhancinggraphreasoning, zhang2024gcoderimprovinglargelanguage, li2025graphteamfacilitatinglargelanguage}. However, both paradigms rely on serializing graphs into prompts, limiting scalability. Task–graph decoupling methods, including GraphTool-Instruction and PIE \cite{wang2025GraphToolInstruction, gong2025pseudocodeinjectionmagicenablingllms}, partially address this issue but require manual tools or pseudocode.



% \noindent\textbf{Agentic Retrieval for Tool Usage} 
% \fali{need to check this paper: Crafting Personalized Agents through Retrieval-Augmented Generation on Editable Memory Graphs}
% Recent agentic RAG systems have shown great promise by letting LLMs dynamically retrieve and use tool documentation for modular planning. For example, HuggingGPT \cite{shen2023hugginggpt} uses ChatGPT to plan tasks and select external models from Hugging Face by their descriptions, and ITER-RETGEN \cite{shao-etal-2023-enhancing} introduces an iterative feedback loop where the LLM refines the tool retriever through multi-round feedback. Both methods allow LLMs to interface with large tool ecosystems without task-specific retrieval fine-tuning. However, graph reasoning has lacked such agentic retrieval. Although GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} builds a knowledge base of graph APIs and experiences and uses an LLM search agent to retrieve relevant documentation, it still depends on a ‘question agent’ to rewrite queries and perform basic similarity search. This static retrieval can break down when prompts vary or are ambiguous. Our proposed \textbf{Agentic RAG} framework bridges this gap by unifying hierarchical retrieval with an LLM-guided documentation verification step, improving the correctness and robustness of retrieval for graph reasoning tasks.

% Siyuan
% \fali{check this paper: GraphCoder: Enhancing Repository-Level Code Completion via Code Context Graph-based Retrieval and Language Model}
\noindent\textbf{RAG for Code-based Graph Reasoning}
% LLMs have demonstrated strong performance in language-to-code generation for their large model sizes and mixture of text tokens and code tokens in training data~\cite{ni2024l2ceval}. This enhances LLMs' capability in solving various tasks, including math problems~\cite{gao2023pal, drori2022neural, chen2023program}, tool use~\cite{}, and \textbf{TOBEADDED}. 
% There have been works studying LLM-based graph reasoning via code generation, which solve graph problems by generating executable programs and then running them~\cite{cai2024codegraphenhancinggraphreasoning,wang2025GraphToolInstruction}. However, these approaches rely solely on LLMs and therefore lack access to professional graph algorithm knowledge. To address this limitation, recent works have proposed incorporating retrieval-augmented generation (RAG) to retrieve external knowledge from relevant technical documentation, such as the NetworkX~\cite{hagberg2008exploring}, thereby improving graph reasoning.
% For example, GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage} and GCoder~\cite{zhang2024gcoderimprovinglargelanguage} treat technical documents as a flat text collection and apply semantic similarity-based retrieval. However, none of these works considers the inherent tree-structured organization of technical documentation. In contrast, our approach performs hierarchical retrieval over tree-structured documentation, enabling more efficient and precise localization of relevant materials.
% rich
% Code-based methods have demonstrated substantial advantages in LLM-based graph reasoning. However, approaches that rely solely on LLMs often lack explicit graph algorithm expertise, which can adversely affect code quality. To address this limitation, recent studies have incorporated retrieval-augmented generation (RAG) to access external knowledge from technical documentation, such as NetworkX~\cite{hagberg2008exploring}, thereby improving graph reasoning performance. For example, GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage} and GCoder~\cite{zhang2024gcoderimprovinglargelanguage} treat technical documentation as a flat text corpus and apply semantic similarity–based retrieval. Nevertheless, these methods overlook the inherent hierarchical structure of technical documentation. In contrast, our approach performs hierarchical retrieval over tree-structured documentation, enabling more accurate and efficient localization of task-relevant materials.
% concise
Code-based methods have shown state-of-the-art performance in LLM-based graph reasoning, but approaches that rely solely on LLMs often lack explicit graph algorithm expertise, limiting code quality. Recent retrieval-augmented generation (RAG) approaches retrieve graph knowledge from technical documentation such as NetworkX~\cite{hagberg2008exploring}. For example, methods such as GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage} and GCoder~\cite{zhang2024gcoderimprovinglargelanguage} treat documentation as a flat corpus and apply semantic similarity–based retrieval. Yet, these methods overlook the inherent hierarchical structure of technical documentation. 
In contrast, our approach performs agentic hierarchical retrieval over tree-structured documentation through a layer-wise agentic retrieval framework, enabling more effective and efficient retrieval. More detailed related work is discussed in Appendix~\ref{sec:appendix_related_agentic_rag}.

% GraphTool-Instruction~\cite{wang2025GraphToolInstruction} treats external documents as callable tools and allows the model to decide which tools to invoke, while 

% \fali{consider this paragraph, RAG for LLM coding-based graph reasoning: To address this limitation, prior work has explored enriching LLMs with external technical knowledge. For instance, PIE \cite{gong2025pseudocodeinjectionmagicenablingllms} relies on manually written, task-specific algorithmic pseudocode to guide code generation, but this approach requires substantial human effort and does not scale well. Other methods introduce automated retrieval mechanisms that leverage external technical documentation, such as the NetworkX documentation \cite{hagberg2008exploring}, to provide algorithmic knowledge. GraphTool-Instruction \cite{wang2025GraphToolInstruction} treats external documents as callable tools and allows the model to decide which tools to invoke, while GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} and GCoder \cite{zhang2024gcoderimprovinglargelanguage} treat technical documents as a flat collection and apply semantic retrieval to select relevant content.}


\section{Preliminary}
In this section, we first characterize the technical documentation structure to motivate our agentic retrieval design, then conduct pilot tests on code-based error distributions and the feasibility of self-generated test cases, and finally formalize the problem definition.
% We first characterize the hierarchical technical documentation structure that motivates our agentic retrieval design, then report pilot experiments on code-based error distributions and the feasibility of LLM-generated test cases, and finally formalize the research problem.
% \suhang{Can we add two sentences explaining what preliminary experiments we will do and why we want to do it}


\subsection{Technical Documentation}

% \noindent\textbf{Technical Documentation} 
% Technical documentation is structured material written for technical users, providing reusable functionalities, interfaces, and usage instructions for a specific domain to support efficient software development. Common examples include the official documentation of NumPy \cite{harris2020array}, PyTorch \cite{paszke2019pytorch}, and NetworkX \cite{hagberg2008exploring}. Such documentation is typically organized in a hierarchical directory structure, progressively arranging high-level concepts, module descriptions, and concrete interfaces from abstract to specific. This structure reflects the inherent conceptual hierarchy of technical knowledge and enables users to quickly locate relevant information while reducing retrieval and comprehension costs. During coding, technical documentation serves as an authoritative source of domain-specific knowledge by clearly specifying function semantics, input–output constraints, and representative examples, which improves both coding efficiency and correctness. In complex graph reasoning tasks, NetworkX documentation provides detailed graph algorithm implementations, thereby enhancing the correctness and robustness of LLM-generated code. We formally model technical documentation as a tree structure in Sec.~\ref{sec:hierarchical_retrieval_agent}.
% Technical documentation consists of structured materials written for technical users, providing reusable functions, interfaces, and usage instructions to support efficient software development. Representative examples include NumPy~\cite{harris2020array}, PyTorch~\cite{paszke2019pytorch}, and NetworkX~\cite{hagberg2008exploring}. Such documentation is typically organized in a hierarchical directory structure, progressively arranging high-level concepts, modules, and interfaces from abstract to specific, which facilitates efficient information retrieval and reduces comprehension cost. During coding, technical documentation serves as an authoritative source of domain knowledge by specifying function descriptions, input–output constraints, and usage examples, thereby improving code correctness and efficiency. In graph reasoning tasks, NetworkX documentation provides detailed graph algorithm interfaces that enhance the robustness of LLM-generated code. We model technical documentation as a tree structure in Sec.~\ref{sec:hierarchical_retrieval_agent}.
Technical documentation comprises structured resources written for technical users, describing reusable functions, APIs, and usage guidelines to support efficient software development. Representative examples include NumPy~\cite{harris2020array}, PyTorch~\cite{paszke2019pytorch}, and NetworkX~\cite{hagberg2008exploring}. Such documentation is typically organized in a hierarchical directory structure, arranging concepts, modules, and interfaces from coarse to fine granularity, which supports systematic navigation, improves retrieval efficiency, and reduces comprehension overhead. In coding workflows, technical documentation serves as an authoritative reference by specifying function semantics, input--output constraints, and usage examples, thereby improving implementation correctness and reliability. For graph reasoning tasks, the NetworkX~\cite{hagberg2008exploring} provides rich algorithmic interfaces that facilitate robust LLM-generated code. We formalize technical documentation as a tree structure in Sec.~\ref{sec:hierarchical_retrieval_agent}.



\subsection{Pilot Experiments}

\noindent\textbf{Error Distribution}.
\label{sec:appendix_debug_distribution}
% \suhang{why it's called debug distribution? Isn't it bug distribution or code error distribution?}
% As shown in Figure~\ref{fig:err_distribution}, compared to DeekSeek-V3, which achieves the highest pass rate 50\%, Qwen-7b can only pass 12\% of tests due to the smallest model size. Both DeekSeek-V3 and LLaMA3-70B can generate codes without syntactic errors, while they fail for 50\% and 69\% of tests because of logical errors, respectively. For the least powerful Qwen-7b, it generates codes with syntactic errors in half tests. Moreover, logical errors occur in 38\% of tests without syntactic errors. It is noted that tests with logical errors are only counted when there are no syntactic errors. Therefore, logical errors are more critical in debugging mechanisms, especially for more powerful LLMs, but they are ignored in previous study.
% Fig.~\ref{fig:err_distribution} shows the distribution of \emph{runtime errors}, \emph{logical errors}, and \emph{directly passed} cases for zero-shot code generation on \texttt{\dataset} (introduced in Sec.~\ref{sec:our_dataset}). We define the pass rate as the fraction of cases that pass the test suite directly; logical errors as cases that execute without runtime failures but produce outputs that differ from the ground-truth answers; and the remaining cases as runtime errors. From the results, we observe, for the strongest model, DeepSeek-V3.2, all generated programs are syntactically valid, and every failure is due to logical errors, indicating that logic-level mistakes are the dominant failure mode. For weaker models (LLaMA-3.3-70B and Qwen-2.5-7B), fewer than $7\%$ of cases exhibit runtime errors. Notably, logical errors are counted only when no runtime error occurs, implying that their true prevalence may be higher. Overall, logical errors account for the majority of code generation failures and motivate us to design debugging module to solve it.
Fig.~\ref{fig:pilot}(a) reports the distribution of \emph{directly passed} cases, \emph{logical errors}, and \emph{runtime errors} for zero-shot code generation on \texttt{\dataset} (detailed in Sec.~\ref{sec:our_dataset}). We define the pass rate as the fraction of cases in which the generated program passes the test suite; logical errors as cases that execute successfully but produce outputs that differ from the ground-truth answers; and the remaining cases as runtime errors. We observe that for the strongest model, DeepSeek-V3.2~\cite{liu2024deepseek}, all generated codes execute successfully, and all failures are due to logical errors, indicating that logic-level mistakes dominate. For weaker models (LLaMA-3.3-70B~\cite{dubey2024llama} and Qwen-2.5-7B~\cite{bai2023qwentechnicalreport}), fewer than $7\%$ of cases exhibit runtime errors, while most failures are logical errors ($19.4\%$ and $32.3\%$, respectively). Notably, logical errors are counted only when execution succeeds, suggesting that their true prevalence may be higher. Overall, \textit{logical errors account for the majority of code failures, motivating the need for a logic-aware debugging mechanism}.

% DeepSeek-V3.2~\cite{liu2024deepseek} is consistently employed as the retrieval agent, while the coding agent is selected from DeepSeek-V3.2, LLaMA-3.3-70B~\cite{dubey2024llama}, and Qwen-2.5-7B~\cite{bai2023qwentechnicalreport}.


% \begin{figure}[t]
%     \centering
%     % \vskip -1em
%     \includegraphics[width=\linewidth]{figures/pilot_exp.pdf}
%     \vskip -1em
%     \caption{xxx.}
%     %  on {\dataset-C} across Various LLMs
%     \vskip -1em
%     \label{fig:pilot}
% \end{figure}


\begin{figure}[t]
    \centering
    % \vskip -1em
    \includegraphics[width=\linewidth]{figures/combined_reasoning_error_analysis.pdf}
    \vskip -1.5em
    \caption{(a) Error distribution across various LLMs. (b) Text-based reasoning accuracy (\%) across graph scales on \texttt{\dataset} using different LLMs.}
    %  on {\dataset-C} across Various LLMs
    % \vskip -3em
    \label{fig:pilot}
\end{figure}

% \begin{figure}[t]
%     \centering
%     % \vskip -1em
%     \includegraphics[width=0.76\linewidth]{figures/error_distribution2.pdf}
%     \vskip -1.5em
%     \caption{Error distribution across various LLMs.}
%     %  on {\dataset-C} across Various LLMs
%     \vskip -1em
%     \label{fig:err_distribution}
% \end{figure}


\noindent\textbf{Feasibility of LLM-Generated Test Labels}.
\label{sec:appendix_text_reasoning_various_graph_scales}
To design an automated \emph{self-debugging} module tailored to LLM-based graph reasoning, we propose using LLMs to self-generate labeled test cases for logic-level verification and correction. To assess feasibility, we conduct a pilot study evaluating text-based reasoning on randomly generated graphs. Specifically, given a graph task, a graph instance (edge list), and input arguments, we prompt the LLM to directly output the answer via textual reasoning (prompts in Appendix~\ref{sec:appendix_prompt}). Results on the small-scale and composite subsets of {\dataset}, grouped by graph size, are reported in Fig. \ref{fig:pilot} (b). 
% Tab.~\ref{tab:text_reasoning_scale}.
We observe that (i) on non-composite tasks with fewer than $10$ nodes, all three model scales (7B to 671B) achieve $100\%$ accuracy. On composite tasks, the two larger models also reach $100\%$ accuracy on graphs with fewer than $5$ nodes, and even the 7B model achieves $66.7\%$. These results suggest that LLMs can reliably label small-scale test cases, supporting automated test-case generation for self-debugging.
% In contrast, Qwen-2.5-7B performs worse on composite tasks with small graph sizes, likely due to its limited capacity to parse and integrate more complex task description.
% (ii) As graph size increases, text-based reasoning degrades substantially; when the number of nodes exceeds $200$, accuracy drops to $57\%$, $27.2\%$, and $16.1\%$ across the three models. This decline is driven by longer serialized inputs and higher structural reasoning complexity, reflecting LLMs' limitations in long-horizon graph reasoning. These results indicate that pure text reasoning is insufficient for large-scale graphs and motivate delegating complex reasoning to executable code.
(ii) As graph size grows, text-based reasoning degrades sharply: for graphs with more than $200$ nodes, accuracy drops to $57\%$, $27.2\%$, and $16.1\%$ across the three models on the small-scale subset and $0$ on the composite subset. This reflects textual reasoning's limitations under long serialized inputs and long-horizon structural reasoning for large-scale graphs, motivating delegating complex reasoning to executable code.

% We report the accuracy of text-based graph reasoning across different graph scales in Tab. \ref{tab:text_reasoning_scale}. The results show that when the graph size is small, LLMs achieve an accuracy of approximately 75\%, providing a solid foundation for constructing high-quality self-debugging data. 
% \suhang{1. discuss more observations, e.g., large graph, and why the performance on large graph is worse, the trend, etc; 2. give detailed experiments setting, e.g., prompt used in appendix???}


% \begin{table}[t]
% \centering
% \small
% \caption{Text-based reasoning accuracy (\%) across graph scales on \texttt{\dataset} using different LLMs.}
% % We report results on the small-scale subset (\texttt{\dataset-S}) and the composite subset (\texttt{\dataset-C}).
% \label{tab:text_reasoning_scale}
% \vskip -1em
% \renewcommand{\arraystretch}{1.15}
% \resizebox{0.88\linewidth}{!}{
% \begin{tabular}{lccccc}
% \hline
% \textbf{Graph size} & \textbf{2--5} & \textbf{5--10} & \textbf{10--20} & \textbf{20--200} & \textbf{$>$200} \\
% \hline
% \multicolumn{6}{c}{\textbf{\texttt{\dataset-S} (Small-scale subset)}} \\
% \hline
% {DeepSeek-V3} & 100.0 & 100.0 & 75.0 & 64.5 & 57.0 \\
% {LLaMA-70B}   & 100.0 & 100.0 & 37.5 & 38.2 & 27.2 \\
% {Qwen-7B}     & 100.0 & 100.0 & 50.0 & 31.2 & 16.1 \\
% \hline
% \multicolumn{6}{c}{\textbf{\texttt{\dataset-C} (Composite subset)}} \\
% \hline
% {DeepSeek-V3} & 100.0 & 100.0 & 56.7 & 22.2 & 0.0 \\
% {LLaMA-70B}   & 100.0 & 75.0  & 25.0 & 0.0  & 0.0 \\
% {Qwen-7B}     & 66.7  & 33.3  & 0.0 & 3.3  & 0.0 \\
% \hline
% \end{tabular}
% }
% \vskip -1em
% \end{table}


% for composite 
% 


\subsection{Problem Definition}
% \suhang{The paper doesn't have a good logic. We were talking about preliminary experiments, and got some observations. Then we jump to problem formulation. It's very strange, these two are disconnected. So what insights do we get from these observations, or how these insights guide the design of our framework, or motivate our problem. All these things are missing }

% \suhang{maybe put the datasets construction in the experiment section, and merge the problem definition with Preliminary experiments, calling it Preliminary. If we put it here, it's basically disconnected from the rest of the paper}

% \noindent\textbf{Task Definition} We consider \emph{graph reasoning tasks by coding}, in which the task description is decoupled from the underlying graph data.
% \begin{definition}[Graph Reasoning by Coding]
% Given a natural language description of a graph task $q$, the graph data consist of $n \geq 1$ graph instances, denoted by $\mathcal{G} = \{ G \}$. Each graph $G = (V, E)$ comprises a node set $V = \{ v_i \}$ and an edge set $E = \{ e_{ij} \}$. Each edge is represented as $e_{ij} = (i, j, w)$, where the weight $w$ is defined only for directed graphs.
% In addition to the graph data, the model is provided with a technical document collection $\mathcal{D} = \{ d_i \}$, consisting of a set of passages, and access to an executable code environment $\mathtt{E}$. Let $f_\theta$ denote an LLM parameterized by $\theta$. The task objective is to understand the task description $q$ and retrieve a task-relevant passage subset $\mathcal{D}_q \subseteq \mathcal{D}$, based on which it generates executable code
% $ c = f_\theta(q, {D}_q).$
% The generated code $c$ takes the graph data $G$ as input and is executed in the environment $\mathtt{E}$, yielding the final output of the task
% $\mathcal{A} = \{ a \}, a = \mathtt{E}(c, G), \; G \in \mathcal{G}.$
% \end{definition}

% \noindent\textbf{Task Definition.}
\begin{definition}[Retrieval-augmented Graph Reasoning by Coding]
\label{def:graph_reasoning_coding}
% We study \emph{graph reasoning by coding}, where the task description is decoupled from the graph data. 
Given a task description $q$, the input consists of a set of graph instances $\mathcal{G}=\{G\}, |\mathcal{G}|\geq 1$, where each graph $G=(V,E)$ has nodes $V=\{v_i\}$ and edges $E=\{e_{ij}\}$ with $e_{ij}=(v_i,v_j,w)$, and $w$ is the weight only for weighted graphs.
An LLM denoted by $f_\theta$ is provided with a technical document collection $\mathcal{D}=\{d_i\}$ and an executable environment $\texttt{E}$. The objective is to retrieve a task-relevant subset $\mathcal{D}_q\subseteq\mathcal{D}$ and then augment the LLM to generate executable code
$c = f_\theta(q, \mathcal{D}_q)$. The code $c$ is executed on graph $G$ in environment $\texttt{E}$ to produce the answer (output)
$a = \texttt{E}(c, G), G \in \mathcal{G}.$
\end{definition}








\begin{figure}[tb]
    \centering
    \includegraphics[width=0.99\linewidth]{figures/framework.pdf}
    \vskip -1em
    \caption{Overview of the Agentic Hierarchical Retrieval-Augmented Self-debugging Coding Framework.}
    % \vskip -2em
    \label{fig:overall_framework}
\end{figure}

\section{Proposed Framework - {\method}}
\label{sec:method}

% Existing LLM-based graph reasoning methods exhibit significant limitations when applied to large-scale graph structures and complex graph reasoning tasks. On the one hand, purely text-based reasoning approaches struggle to effectively represent and process large graphs due to inherent context window constraints. On the other hand, although coding-based graph reasoning methods can, in principle, scale to large graph instances through code execution, they often lack the necessary algorithmic knowledge and fail to adequately leverage external technical documentation or algorithmic documentations, resulting in limited performance on complex graph tasks.
% To address these challenges, we propose \textsc{GraphTutor}, a documentation-guided, agentic retrieval-augmented coding framework for graph reasoning. The framework consists of two cooperating components: a retrieval agent and a coding agent, as illustrated in Figure~\ref{fig:overall_framework}. Specifically, \textsc{GraphTutor} first feeds the natural language description of a graph reasoning task to the retrieval agent, which searches a structured technical document tree to retrieve task-relevant algorithm documentations or technical articles. The retrieved documents are then combined with the original task description and provided as input to the coding agent. Through iterative interactions with an executable environment—including code generation, execution, and debugging—the coding agent progressively synthesizes high-quality, executable code that faithfully implements the required graph algorithms.
% During inference, the generated code is executed directly on the target graph instances to produce the final answers. Under this paradigm, the synergy between retrieval-augmented documentation guidance and environment-driven iterative debugging enables the coding agent to produce robust and scalable programmatic solutions, effectively handling complex graph reasoning tasks and naturally supporting graphs of varying scales, including large-scale graphs, without being constrained by the context length limitations of LLMs.
% Existing LLM-based graph reasoning methods face substantial challenges in complex settings, including large-scale graphs and composite graph reasoning tasks. Text-based approaches are constrained by limited context windows and weak long-context reasoning capabilities, making them ineffective for large graphs. Coding-based methods can scale via code execution, but often struggle with composite tasks due to insufficient algorithmic knowledge and inadequate use of external documentation guidance.
% original
% We propose {\method}, a documentation-guided, agentic hierarchical retrieval-augmented coding framework for graph reasoning. The framework consists of a \texttt{retrieval agent} and a \texttt{coding agent} (Fig.~\ref{fig:overall_framework}). Specifically, given a graph reasoning task, the \texttt{retrieval agent} searches a tree-structured technical documentation to identify task-relevant algorithmic functions, which are then combined with the task description and provided to the \texttt{coding agent}.
% To overcome runtime and logical errors in code generation, we propose a \texttt{coding agent} with self-debugging, where the agent first self-generates labeled test cases, then produces an initial code, and subsequently iteratively refines and debugs the code through interaction with a compiler environment until it passes the test cases or reaches a maximum debugging budget. The validated code is finally executed on the graph instances to produce the final answers.
% Through integrating documentation-guided retrieval with compiler-driven debugging in a coding-based reasoning, our method enables robust graph reasoning on both composite and large-scale graphs. 
% \suhang{how do we handle composite tasks? I think the methodology section was written before we tes the composite tasks, right? That means, we did not mention anything for handling composite tasks}
We propose {\method}, a technical documentation-guided agentic hierarchical retrieval--augmented self-debugging coding framework for complex graph reasoning. The framework consists of a \texttt{retrieval agent} and a \texttt{coding agent} (Fig.~~\ref{fig:overall_framework}). Specifically, given a graph reasoning task, the \texttt{retrieval agent} traverses tree-structured technical documentation to identify task-relevant algorithmic entries. Importantly, for semantically complex composite tasks, the \texttt{retrieval agent} can return a set of accurate and less-noisy complementary functions spanning multiple sub-algorithms (e.g., strongly connected components and shortest paths), enabling the \texttt{coding agent} to compose them into a code solution. To mitigate runtime and logical errors in code generation, we equip the \texttt{coding agent} with a self-debugging mechanism. It first autonomously generates labeled test cases, then produces an initial implementation, and iteratively refines the code through interaction with a compiler environment until all test cases pass or a maximum debugging budget is reached. This feedback loop is particularly critical for composite tasks, where the increased complexity often prevents one-shot code generation from being correct. Finally, the validated code is executed on the target graph instances to produce the final answers. By integrating documentation-guided agentic retrieval with compiler-driven self-debugging code generation, {\method} better supports large-scale graph reasoning than text-based reasoning, and improves composite graph reasoning compared to naive one-shot coding with flat retrieval.



\subsection{Hierarchical Retrieval Agent}
\label{sec:hierarchical_retrieval_agent}
% rich
% Previous retrieval-augmented generation methods commonly treat technical documents as a flat corpus, disregarding the hierarchical structure inherent to technical documentation. Such flattening often introduces substantial retrieval noise, which in turn degrades retrieval performance and adversely affects downstream reasoning performance. To overcome this limitation, we introduce a hierarchical retrieval agent that operates directly over a tree-structured technical document collection (see an example in Fig. \ref{fig:retrieval_agent}). The agent adopts a layer-wise agentic retrieval strategy that combines semantic understanding of the task with structured navigation over the document hierarchy. By performing layer-wise traversal and progressively pruning irrelevant branches, the agent is able to more accurately and efficiently identify task-relevant documents, thereby improving the overall system performance.
% fali: note that the efficiency is compared to an agent traveling on the flat, not the TF-IDF. 
% concise
% Prior RAG methods typically treat technical documentation as a flat corpus, ignoring its inherent hierarchical structure and thereby introducing retrieval noise or lacking necessary information. To address this, we propose a hierarchical retrieval agent that operates directly over tree-structured technical documentation, as shown in Fig.~\ref{fig:retrieval_agent}. The agent adopts a layer-wise agentic retrieval algorithm that combines task semantic understanding with structured navigation of the document hierarchy, progressively pruning irrelevant branches to accurately and efficiently identify task-relevant documents and improve overall performance.
Prior RAG methods typically treat technical documentation as a flat text corpus and retrieve the top-$k$ documents via vector similarity such as TF-IDF. Such retrieval often relies on keywords or shallow semantic similarity, making it difficult to capture deeper task relevance; as a result, truly critical entries may not appear among the highest-ranked results. To improve recall, they often increase $k$, which introduces substantial retrieval noise and increases downstream generation cost. A straightforward alternative is to employ an LLM-based agent to filter documents using the LLM’s stronger semantic understanding, but performing document-by-document matching over large corpora incurs high LLM cost and inference latency. To address these issues, we propose a \textbf{hierarchical retrieval agent}, as shown in Fig.~\ref{fig:retrieval_agent}, that performs top-down traversal over tree-structured technical documentation, combining task-level semantic understanding with structured navigation. By pruning irrelevant branches early at higher levels, the agent identifies task-relevant entries with only a few LLM queries, enabling cost-efficient retrieval.

\begin{figure}[tb]
    \centering
    \includegraphics[width=0.99\linewidth]{figures/retrieval_agent.pdf}
    \vskip -1.5em
    \caption{The hierarchical retrieval agent workflow. }
    % \vskip -1em
    \label{fig:retrieval_agent}
\end{figure}


\noindent\textbf{Tree-Structured Technical Documentation}
We model the technical documentation $\mathcal{D}$ as a document tree $\mathcal{T}=(\mathcal{V}, \mathcal{E}, \mathcal{S})$ with maximum depth $L$. The tree is rooted at a virtual root node $v^{(0)}$ and expands level by level into $L$ hierarchical layers. The $l$th layer ($l \in \{0,\dots,L\}$) consists of a set of nodes $\mathcal{V}^{(l)}=\{v_i^{(l)}\}$, where layer $0$ is a virtual root containing a single node. Each non-leaf node $v_i^{(l)}$ is connected to its children in the next layer via edges $(v_i^{(l)}, v_j^{(l+1)}) \in \mathcal{E}$, which represent hyperlinks from the parent node web page to the child node web pages, while leaf nodes correspond to concrete document entries.
Each node $v_i^{(l)}$ can be a chapter, subsection, or individual article, and is associated with textual content ${s}_i^{(l)} \in \mathcal{S}$ that may include a chapter description or a function specification (e.g., input--output parameters, and usage examples). This formulation captures the hierarchical structure of technical documentation and provides a coarse-to-fine abstraction that supports structured navigation.


% \suhang{What's the relationship between the parent node and the child node in terms of the semantic meaning???} \fali{revised. It is a hyperlink.}
% \suhang{Did we construct these parent-child relationships or it naturally exist. Since we used the phrase ``We organize the technical document'', it seems that we constructed. If yes, how do we construct, what tools/criteria/metrics do we use, how do we obtain the chapter/subsection for each node? If we did not construct, we need to convince the reviewers that such the hierarchy widely exists in technical documents to show that our method can be generalized to various tools, not only NetworkX we use (we also need to motivate this in the introduction)} \fali{revised the word. we do not construct, and I supplement a paragraph in Sec. 3 to state the hierarchy's wide existence.}
% \suhang{how do we obtain the semantic representation, use an encoder???} \fali{revised. It is raw content.}
% \suhang{Usually, hierarchy can improve the search efficiency, not search effectiveness. Can you explain why it can improve search effectiveness in more detail, give intuition, and a concrete example???}



% \begin{algorithm}[t]
% \caption{Layer-wise Agentic Retrieval}
% \label{alg:retrieval}
% \small 
% \begin{algorithmic}[1]
% % \REQUIRE Task $q$; document tree $\mathcal{T}=(\mathcal{V},\mathcal{E},\mathcal{S})$ with depth $L$
% \REQUIRE Task $q$; document tree $\mathcal{T}$ with depth $L$, Retrieval Agent \texttt{R}
% \ENSURE Task-relevant document subset $\mathcal{D}_q$

% \STATE Initialize $\mathcal{D}_q \leftarrow \{v^{(0)}\}$

% % \STATE \textbf{Def} \textsc{SelectRelevant}$(q, \mathcal{C})$
% % \STATE \hspace{0.5cm} $\mathcal{D}' \leftarrow \emptyset$
% %     \STATE \hspace{0.5cm} \textbf{for all} $v_i \in \mathcal{C}$ \textbf{do}
% %         \STATE {\hspace{1cm} \textbf{If} \textsc{IsRelevant}$(q, s_i)$} \textbf{do}
% %             \STATE \hspace{1.5cm} $\mathcal{D}' \leftarrow \mathcal{D}' \cup \{v_i\}$
% %     \STATE \hspace{0.5cm} \textbf{return} $\mathcal{D}'$ 
% % \STATE \textbf{End Function} \\ \hspace{2cm}

% \FOR{$l = 0$ \TO $L-1$}
%     % \STATE $\mathcal{D}_q' \leftarrow \emptyset$
%     % \FORALL{$v_i^{(l)} \in \mathcal{D}_q$}
%     %     \STATE $\mathcal{C} \leftarrow \{\, v_j^{(l+1)} \mid (v_i^{(l)}, v_j^{(l+1)}) \in \mathcal{E} \,\}$
%     %     \STATE $\mathcal{D}_q' \leftarrow \mathcal{D}_q' \cup \textsc{SelectRelevant}(q, \mathcal{C})$ \hfill $\triangleright$ \text{\texttt{See Eq.\ref{x}.}}
%     % \ENDFOR
%     % \STATE $\mathcal{D}_q \leftarrow \mathcal{D}_q'$
%     \STATE $\mathcal{C} \leftarrow \{\, v_j^{(l+1)} \mid v_i^{(l)} \in \mathcal{D}_q  ,\,  (v_i^{(l)}, v_j^{(l+1)}) \in \mathcal{E} \}$ %\hfill $\triangleright$ \text{\texttt{Candidate node set.}}
%     \STATE $\mathcal{D}_q \leftarrow \texttt{R.select\_relevant}(q, \mathcal{C})$ \hfill $\triangleright$ \text{\texttt{See Eq.\eqref{eq:select_relevant}.}}
% \ENDFOR

% \STATE $\mathcal{D}_q \leftarrow \texttt{R.global\_filter}(q, \mathcal{D}_q)$
% \RETURN $\mathcal{D}_q$
% \end{algorithmic}
% \end{algorithm}



\noindent\textbf{Layer-wise Agentic Retrieval Algorithm}
Building on the hierarchical document tree $\mathcal{T}$, we introduce a \emph{Layer-wise Agentic Retrieval} algorithm, whose overall procedure is summarized in Algo.~\ref{alg:retrieval} in Appendix. The core idea is to integrate agent-driven semantic relevance assessment with structured, level-wise traversal of the document hierarchy.
Concretely, the retrieval agent $\texttt{R}$ starts from the root node $v^{(0)}$ (initialize the selected node set $\mathcal{D}_q$, Algo.~\ref{alg:retrieval}, Line 1) and examines its first-layer child nodes $\mathcal{V}^{(1)}$. Let $\mathcal{C}=\mathcal{V}^{(1)}$ denote the candidate node set at the current layer. For each node $v_i^{(1)} \in \mathcal{C}$, the agent evaluates its relevance to the task description $q$ based on the node’s semantics ${s}_i^{(1)}$. This relevance filtering operation is formalized as
\begin{equation}
\small 
\begin{aligned}
\mathcal{D}_q
&= \texttt{R}\texttt{.select\_relevant}(q, \mathcal{C}) \\
&= \{\, v_i^{(l)} \mid \texttt{R}.\texttt{is\_relevant}(q, {s}_i^{(l)}) = \texttt{Yes}, v_i^{(l)} \in \mathcal{C} \,\}
\end{aligned}
\label{eq:select_relevant}
\end{equation}
where $\mathcal{D}_q$ is updated to the set of nodes judged relevant at the current layer. The judgement prompt is shown in Appendix \ref{sec:appendix_prompt}.
% The selected node set $\mathcal{D}_q$ is then updated as
% \begin{equation}
% \mathcal{D}_q =
% \begin{cases}
% \{v^{(0)}\}, & \text{initial stage}, \\
% \mathcal{D}_q', & \text{subsequent stages}.
% \end{cases}
% \end{equation}
The algorithm proceeds in a layer-wise manner. At a middle layer $l+1$, given the current selected set $\mathcal{D}_q \subseteq \mathcal{V}^{(l)}$, the agent collects current-layer candidate child node set
\begin{equation}
\small 
    \mathcal{C} = \{ v_j^{(l+1)} \mid v_i^{(l)} \in \mathcal{D}_q,\; (v_i^{(l)}, v_j^{(l+1)}) \in \mathcal{E} \}.
\label{eq:candidate_set}
\end{equation}
The agent then applies the same relevance selection operation in Eq.~\eqref{eq:select_relevant} to $\mathcal{C}$, yielding a new selected set $\mathcal{D}_q \subseteq \mathcal{V}^{(l+1)}$ at the current layer. This procedure is repeated iteratively for layers $l=0$ through $L-1$, until the traversal reaches the leaf level (Algo. Lines 2-5).
After the termination, the nodes remaining in $\mathcal{D}_q$ correspond to leaf nodes and thus form a set of selected technical documents. Finally, for precise selection, the retrieval agent performs a global filtering step over these candidates:
\begin{equation}
\small 
\mathcal{D}_q = \texttt{R}\texttt{.global\_filter}(q, \mathcal{D}_q),
\label{eq:global}
\end{equation}
producing the final retrieved doc subset $\mathcal{D}_q$ (Algo. Line 6). The global filtering prompt is in Appendix \ref{sec:appendix_prompt}.
% rich
% Compared with static retrieval methods that rely solely on flattened semantic similarity, the proposed approach introduces agent-driven relevance assessment, enabling explicit alignment between the task description and both intermediate directory nodes and individual document entries, thereby enhancing the retrieval accuracy. Moreover, by pruning irrelevant branches at higher levels of the hierarchy, the algorithm substantially reduces the downstream search space, thereby improving both retrieval efficiency. 
% concise 
Compared to static retrieval based on flattened semantic similarity, our approach employs agent-driven relevance assessment to align task semantics with both directory nodes and document entries, improving retrieval quality while avoiding excessive cost and latency by reducing the search space and pruning irrelevant branches at higher levels.


\subsection{Coding Agent with Self-Debugging}
Given the task-relevant document subset $\mathcal{D}_q$, we employ a \texttt{Coding Agent} $\texttt{C}$ to synthesize executable code for solving the graph reasoning task. As shown by the green arrows in Fig.~\ref{fig:overall_framework}, the pipeline consists of four stages: test case generation, code generation, debugging, and code refinement. The latter two stages are performed iteratively. The entire process is supported by an external executable environment $\texttt{E}$, which enables program execution and the acquisition of feedback.
Owing to the inherent complexity of certain tasks (e.g., composite tasks), the initially generated code may exhibit logical errors. To mitigate this, we introduce a \emph{self-debugging mechanism} in which the \texttt{Coding Agent} self-constructs a set of small-scale test cases with their labels, and then iteratively refines the code by validating it against these test cases until all tests pass or the debugging budget is exhausted. We describe each stage in detail below.

\noindent\textbf{Test Case Generation}
Given the task description $q$ and the retrieved documents $\mathcal{D}_q$, the \texttt{Coding Agent} exploits its reasoning capability to generate a set of small-scale graph instances (e.g., graphs with three nodes) and their corresponding labels. This design is motivated by the observation that the agent can reliably produce correct solutions on small graphs via textual reasoning (recall in Fig.\ref{fig:pilot}). Formally, this step is defined as
\begin{equation}
\small 
(\mathcal{G}_{\text{test}}, \mathcal{A}_{\text{test}})
= \texttt{C}\texttt{.generate\_test\_cases}(q), % , \mathcal{D}_q
\label{eq:test_case_generation}
\end{equation}
where $\mathcal{G}_{\text{test}}=\{g_i\}$ is the test graphs and $\mathcal{A}_{\text{test}}=\{a_i\}$ is the labels.

\noindent\textbf{Code Generation}
The \texttt{Coding Agent} generates an initial version of executable code $c$ conditioned on the task $q$ and the retrieved documents $\mathcal{D}_q$:
\begin{equation}
c = \texttt{C}\texttt{.generate\_code}(q, \mathcal{D}_q).
\label{eq:code_generation}
\end{equation}

\noindent\textbf{Debugging and Refinement}
To evaluate code correctness, the generated code $c$ is executed in the environment $\texttt{E}$ on the test graphs $\mathcal{G}_{\text{test}}$, and the resulting outputs are compared against the reference answers $\mathcal{A}_{\text{test}}$. The test accuracy is defined as
\begin{equation}
\small 
\text{Acc}_{\text{test}}
= \frac{1}{|\mathcal{G}_{\text{test}}|}
\sum_{(g_i,a_i)\in(\mathcal{G}_{\text{test}},\mathcal{A}_{\text{test}})}
\mathbb{I}\big[\texttt{E}(c, g_i)=a_i\big],
\label{eq:code_refinement}
\end{equation}
where $\mathbb{I}[\cdot]$ denotes the indicator function.
If $\text{Acc}_{\text{test}} = 1.0$, the code $c$ is considered correct on the test cases and is returned as the final solution. Otherwise, the test cases on which the code fails, together with their expected and actual outputs, are aggregated into a feedback signal $F$, which is used by the \texttt{Coding Agent} to refine the code:
\begin{equation}
c \leftarrow \texttt{C}\texttt{.refine\_code}(q, \mathcal{D}_q, c, F).
\end{equation}
The debugging and refinement loop is executed for at most $T_{\max}$ iterations. If no code attains perfect test accuracy within this budget, we select the version with the highest $\text{Acc}_{\text{test}}$ across all iterations as the final output.
Detailed prompts are provided in Appendix~\ref{sec:appendix_prompt}.

\subsection{Code Execution on Graph Instances}
% Upon obtaining a high-quality code $c$, we execute it on the target graph instances to produce the final task outputs. Specifically, given a graph instance $G \in \mathcal{G}$, the code $c$ is executed within the program execution environment $\mathcal{E}$, which provides a deterministic runtime for program interpretation and result generation. 
% The task might involve multiple graph instances; the execution process is applied independently to each graph in the input set $\mathcal{G}=\{G_i\}_{i=1}^n$, yielding a corresponding set of outputs
% \begin{equation}
% \mathcal{A} = \{\, a_i \mid a_i = \mathcal{E}(c, G_i),\; G_i \in \mathcal{G} \,\}.
% \end{equation}
Upon obtaining a high-quality executable code $c$, we execute it in the program execution environment $\texttt{E}$ on the input graph set $\mathcal{G}=\{G_i\}_{i=1}^n$. Each graph is processed independently, yielding the outputs
\begin{equation}
\small 
\mathcal{A} = \{\, a_i \mid a_i = \texttt{E}(c, G_i),\; G_i \in \mathcal{G} \,\}.
\end{equation}
% rich
% By decoupling reasoning from execution, this formulation allows the generated code $c$ to operate on graphs of arbitrary scale, limited only by the computational resources of the execution environment rather than the context length of the language model. As a result, the proposed framework naturally supports both small-scale and large-scale graph reasoning scenarios, while ensuring correctness through explicit program execution.
% concise
The generated code operates on graphs of arbitrary scale, constrained only by computing resources rather than LLM context size, thereby supporting both small- and large-scale graph reasoning with correctness ensured via explicit program execution. For simplicity, we omit task parameters from the notation and assume they are implicitly included in each $G_i$, as part of the task specification.
% , although they are part of the task specification.









% \section{Experiments}
% In this section, we conduct experiments to evaluate whether the proposed \textbf{Agentic RAG} framework enhances the \textit{accuracy}, \textit{robustness}, and \textit{reusability} of large language models (LLMs) in graph reasoning tasks, compared with baseline prompting and retrieval approaches. \fali{using research questions is fine but the space is limited. Let's remove them and directly show the results.} 
% To achieve this goal, we design a series of experiments that address the following research questions:
% \begin{itemize}
%     \item \textbf{RQ1 (Graph Task Solving Capability):} How does the proposed framework perform, in comparison to unfine-tuned code-based and vanilla retrieval-based methods, across diverse graph tasks?
%     \item \textbf{RQ2 (Retrieval Accuracy):} How does the retrieval performance of \textbf{Agentic RAG} compare to existing retrieval methods when identifying relevant documentation and handling description variants for the input graph task description?
%     \item \textbf{RQ3 (Ablation Study):} What is the contribution of each individual component within the proposed framework to the overall task-solving performance?
%     \item \textbf{RQ4 (Context Level Impact):} How does the level of contextual guidance affect the performance of LLMs on graph tasks?
% \end{itemize}

% \begin{figure}[htbp] % [h!] is a placement specifier, trying to place it "here"
%     \centering % Centers the image
%     \includegraphics[width=\linewidth]{figures/retrieval_evaluation.png}
%     \caption{Workflow of the retrieval evaluation process. Given task descriptions as input, retrieval baselines generate queries to search the retrieval database, which contains pre-processed documentation chunks. The top-K retrieved documents are compared with the ground truth documentation to assess retrieval accuracy, measured using the Hit@K metric.}
%     \label{fig:retrieval_evaluation_overview} % A label for referencing the figure later
% \end{figure}

% \begin{figure}[htbp] % [h!] is a placement specifier, trying to place it "here"
%     \centering % Centers the image
%     \includegraphics[width=\linewidth]{figures/code_gen_evaluation_framework.png}
%     \caption{Overall workflow of the evaluation framework. The framework takes task descriptions and graph information as inputs, optionally enhanced with test cases or related documentation. Each baseline model generates intermediate outputs such as code solutions and explanatory ideas. The resulting code extracted by regular expression ([RE]) which are then evaluated against test cases to compute the final pass rate.}
%     \label{fig:code_gen_evaluation_overview} % A label for referencing the figure later
% \end{figure}


\section{Experiments}
In this section, we evaluate the proposed framework for graph reasoning on two datasets with varying graph scales and complexity, across various-size LLMs, including comparative experiments, ablations, retrieval analysis, key hyperparameter analysis, and case studies.

% \suhang{replace some tables by figures. We have too many tables}



% \begin{table}[ht]
% \centering
% \caption{Comparative results on {\dataset-S} and GTools-S using various coding-agent LLMs, using the Accuracy metric. Best in each column is bolded. \fali{need to update the results to match the title.}}
% \label{tab:pass_rate_transposed}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Method} & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7b} \\
% \midrule
% % \textbf{TF-IDF} & 0.51 & 0.47 & 0.48 \\
% % \textbf{Sentence-BERT} & 0.56 & 0.55 & 0.60 \\
% % \textbf{PSEUDO} & 0.91 & 0.74 & 0.45 \\
% % \textbf{GraphTeam} & 0.84 & 0.83 & 0.47 \\
% % \textbf{CodeGraph} & 0.90 & 0.85 & \textbf{0.66} \\
% % \textbf{Agentic RAG (Ours)} & \textbf{1.0} & \textbf{1.0} & \textbf{0.66} \\
% \bottomrule
% \end{tabular}
% }
% \end{table}


% \begin{table*}[t]
% \centering
% \small 
% \caption{Comparative results on {\dataset-S} and GTools-WL using various coding-agent LLMs, evaluated with the Accuracy metric. Best in each column is bolded. Baselines: (i) Text-based Reasoning; (ii) Code-based (No Retrieval, No Debugging); (iii) Code-based (Retrieval, No Debugging); (iv) Code-based (Retrieval + Debugging); and (v) Code-based (Human-involved).}
% \label{tab:main_results_S}
% \vskip -1em
% \resizebox{\linewidth}{!}{
% \renewcommand{\arraystretch}{1.1}  % 1.2–1.5 is typical
% \begin{tabular}{llcccccc}
% \hline
%  &  & \multicolumn{3}{c}{{\dataset-S}} & \multicolumn{3}{c}{GTools-WL} \\
% \cmidrule(lr){3-5} \cmidrule(lr){6-8}
% \textbf{} & \textbf{Method} 
% & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7b}
% & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7b} \\
% \hline
% \multirow{4}{*}{(i)}
%  & \textsc{Zero-shot} & 0.37 & 0.33 & 0.06 & 0.46 & 0.42 & / \\
%  & \textsc{Few-shot} & 0.34 & 0.30 & 0.04 & 0.40 & 0.39 & / \\
%  & \textsc{CoT} & 0.34 & 0.09 & 0.04 & 0.41 & 0.37 & / \\
%  & \textsc{Bag-CoT} & 0.38 & 0.34 & 0.04 & 0.44 & 0.44 & / \\
% \hline
% \multirow{2}{*}{(ii)}
%  & \textsc{Zero-shot Coding} & 0.38 & 0.37 & 0.36 & 0.73 & 0.52 & 0.63 \\
%  & \textsc{Few-shot Coding} & 0.43 & 0.35 & 0.38 & 0.75 & 0.57 & 0.57 \\
% \hline
% \multirow{3}{*}{(iii)}
%  % & \textsc{GCoder} &  &  &  &  &  &  \\
%  & \textsc{GraphTool-Instruction} & 0.97 & 0.73 & \textbf{0.74} & \textbf{0.90} & 0.63 & 0.57 \\
%  & \textsc{TFIDF+Coding} & 0.51 & 0.47 & 0.48 & 0.68 & 0.57 & 0.52 \\
%  & \textsc{SentenceBERT+Coding} & 0.55 & 0.45 & 0.44 & 0.68 & 0.42 & 0.31 \\
% \hline
% \multirow{3}{*}{(iv)}
%  & \textsc{GraphTeam} & 0.84 & 0.82 & 0.47 & 0.63 & 0.73 & 0.42 \\
%  & \textsc{TFIDF+CodeAgent} & 0.70 & 0.90 & 0.38 & 0.78 & 0.78 & 0.57 \\
%  & \textsc{SentenceBERT+CodeAgent} & 0.68 & 0.97 & 0.31 & 0.74 & 0.78 & 0.47 \\
% \hline
% \multirow{2}{*}{(v)}
%  & \textsc{CodeGraph} & 0.90 & 0.85 & 0.66 & 0.63 & 0.84 & 0.67 \\
%  & \textsc{PIE} & 0.90 & 0.73 & 0.06 & 0.52 & 0.81 & 0.57 \\
% \hline
% \rowcolor{lightgray}
%  \multirow{1}{*}{}& \textsc{Ours} & \textbf{1.00} & \textbf{1.00} & {0.66} & {0.80} & \textbf{0.90} & \textbf{0.74} \\
% \hline
% \end{tabular}
% }
% \end{table*}

% \begin{table*}[t]
% \centering
% \small 
% \caption{Comparative results on \texttt{\dataset-S} and \texttt{GTools-WL} using various coding agent LLM backbones, evaluated with the Accuracy. The best result in each column is bolded (except for CodeGraph and PIE, whose coding processes involve human intervention). The Avg.\ column reports the mean over all values. Baselines: (i) Text-based Reasoning; (ii) Code-based (No Retrieval, No Debugging); (iii) Code-based (Retrieval, No Debugging); (iv) Code-based (Retrieval + Debugging); and (v) Code-based (Human-involved).}
% \label{tab:main_results_S}
% \vskip -1em
% \resizebox{0.8\linewidth}{!}{
% \renewcommand{\arraystretch}{1.1}
% \begin{tabular}{llccccccc}
% \hline
%  &  & \multicolumn{3}{c}{{\dataset-S}} & \multicolumn{3}{c}{GTools-WL} & \textbf{Avg.} \\
% \cmidrule(lr){3-5} \cmidrule(lr){6-8}
%  & \textbf{Method} 
% & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
% & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
% &  \\
% \hline
% \multirow{4}{*}{(i)}
%  & \textsc{Zero-shot} & 67.3 & 32.3 & 20.9 & 68.0 & 43.5 & 40.5 &  \\
%  & \textsc{Few-shot} & 64.6 & 35.0 & 16.4 & 78.5 & 52.0 & 45.5 &  \\
%  & \textsc{CoT} & 62.3 & 32.7 & 20.9 & 79.0 & 45.5 & 29.5 &  \\
%  & \textsc{Bag-CoT} & 66.4 & 24.6 & 15.5 & 69.5 & 36.0 & 26.5 &  \\
% \hline
% \multirow{2}{*}{(ii)}
%  % & \textsc{Zero-shot Coding} & 0.38 & 0.37 & 0.36 & 0.73 & 0.52 & 0.63 & 0.50 \\
%   & \textsc{Zero-shot Coding} & 81.8 & 73.2 & 45.4 & 88.0 & 82.5 & 50.0 &  \\
%  % & \textsc{Few-shot Coding} & 0.79 & 0.64 & 0.24 & 0.75 & 0.57 & 0.57 & 0.51 \\
%  & \textsc{Few-shot Coding} & 83.6 & 69.1 & 58.6 & 90.0 & 82.5 & 58.0 &  \\
% \hline
% \multirow{3}{*}{(iii)}
%  % & \textsc{GraphTool-Instruction} & 0.97 & 0.73 & \textbf{0.74} & \textbf{0.90} & 0.63 & 0.57 & 0.76 \\
%  & \textsc{TFIDF+Coding} & 94.1 & 90.0 & 45.0 & 80.0 & 77.5 & 57.5  &  \\
%  & \textsc{SentBERT+Coding} & 94.1 & 80.5 & 50.0 & 85.0 & 75.0 & 45.0 &  \\
% \hline
% \multirow{3}{*}{(iv)}
%  & \textsc{GraphTeam} & 82.7 & 79.1 & 64.6 & 85.0 & 75.0 & 70.0 &  \\
%  % & \textsc{TFIDF+CodeAgent} & 0.63 & 0.84 & 0.38 & 0.81 & 0.90 & 0.69 &  \\
%  % & \textsc{SentenceBERT+CodeAgent} & 0.82 & 0.91 & 0.31 & 0.77 & 0.88 & 0.63 &  \\
%   & \textsc{TFIDF+CodeAgent} & 94.6 & 90.5 & 76.8 & 90.0 & 90.0 & 70.0 &  \\
%  & \textsc{SentBERT+CodeAgent} & 93.7 & 92.3 & 87.7 & 90.0 & 90.0 & 70.0 &  \\
 
% \hline
% \multirow{2}{*}{(v)}
%  & \textsc{CodeGraph} & 94.6 & 94.6 & 58.2 & 90.0 & 95.0 & 45.0 &  \\
%  % & \textsc{PIE} & 0.70 & 0.68 & 0.45 & 0.99 & 0.94 & 0.69 &  \\
%  & \textsc{PIE} & 91.8 & 81.4 & 69.6 & 80.0 & 90.0 & 80.0 &  \\
% \hline
% \rowcolor{lightgray}
%  & \textsc{Ours} & \textbf{96.4} & \textbf{96.4} & \textbf{91.8} & \textbf{100.0} & \textbf{100.0} & \textbf{73.5} & \textbf{} \\
% \hline
% \end{tabular}}
% \end{table*}
\begin{table*}[t]
\centering
\small 
\caption{Comparative results on \texttt{\dataset-S} and \texttt{GTools-WL} using various coding agent LLM backbones, evaluated with Accuracy. The best result in each column is bolded (except for CodeGraph and PIE, whose coding processes involve human intervention). The Avg.\ column reports the mean over all values. Baselines: (i) Text-based Reasoning; (ii) Code-based (No Retrieval, No Debugging); (iii) Code-based (Retrieval, No Debugging); (iv) Code-based (Retrieval + Debugging); and (v) Code-based (Human-involved).}
\label{tab:main_results_S}
\vskip -1.2em
\resizebox{0.82\linewidth}{!}{
\renewcommand{\arraystretch}{1.1}
\begin{tabular}{llccccccc}
\hline
 &  & \multicolumn{3}{c}{\texttt{\dataset-S}} & \multicolumn{3}{c}{\texttt{GTools-WL}} & \textbf{Avg.} \\
\cmidrule(lr){3-5} \cmidrule(lr){6-8}
 & \textbf{Method} 
& \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
& \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
&  \\
\hline
\multirow{4}{*}{(i)}
 & \textsc{Zero-shot} & 67.3 & 32.3 & 20.9 & 68.0 & 43.5 & 40.5 & 45.4 \\
 & \textsc{Few-shot} & 64.6 & 35.0 & 16.4 & 78.5 & 52.0 & 45.5 & 48.7 \\
 & \textsc{CoT} & 62.3 & 32.7 & 20.9 & 79.0 & 45.5 & 29.5 & 45.0 \\
 & \textsc{Bag-CoT} & 66.4 & 24.6 & 15.5 & 69.5 & 36.0 & 26.5 & 39.8 \\
\hline
\multirow{2}{*}{(ii)}
 & \textsc{Zero-shot Coding} & 81.8 & 73.2 & 45.4 & 88.0 & 82.5 & 50.0 & 70.1 \\
 & \textsc{Few-shot Coding} & 83.6 & 69.1 & 58.6 & 90.0 & 82.5 & 58.0 & 73.6 \\
\hline
\multirow{2}{*}{(iii)}
 & \textsc{TFIDF+Coding} & 94.1 & 90.0 & 45.0 & 80.0 & 77.5 & 57.5 & 74.0 \\
 & \textsc{SentBERT+Coding} & 94.1 & 80.5 & 50.0 & 85.0 & 75.0 & 45.0 & 71.6 \\
\hline
\multirow{3}{*}{(iv)}
 & \textsc{GraphTeam} & 82.7 & 79.1 & 64.6 & 85.0 & 75.0 & 70.0 & 76.1 \\
 & \textsc{TFIDF+CodeAgent} & 94.6 & 90.5 & 76.8 & 90.0 & 90.0 & 70.0 & 85.3 \\
 & \textsc{SentBERT+CodeAgent} & 93.7 & 92.3 & 87.7 & 90.0 & 90.0 & 70.0 & 87.3 \\
\hline
\multirow{2}{*}{(v)}
 & \textsc{CodeGraph} & 94.6 & 94.6 & 85.9 & 90.0 & 95.0 & 80.0 & 90.0  \\
 & \textsc{PIE} & 94.2 & 94.6 & 87.2 & 84.4 & 90.0 & 80.0 & 88.4 \\
\hline
\rowcolor{lightgray}
 & \textsc{Ours} & \textbf{96.4} & \textbf{96.4} & \textbf{91.8} & \textbf{100.0} & \textbf{100.0} & \textbf{73.5} & \textbf{93.0} \\
\hline
\end{tabular}}
\end{table*}

\begin{table*}[t]
\centering
\small
\caption{Comparative results on \texttt{\dataset-L} and \texttt{GTools-EL} using various coding agent LLM backbones, evaluated with the Accuracy metric. Best in each column is bolded except for CodeGraph and PIE. (i)–(v) follow the same statements as above Tab. \ref{tab:main_results_S}.}
\label{tab:main_results_L}
\vskip -1.2em
\resizebox{0.82\linewidth}{!}{
\renewcommand{\arraystretch}{1.1}
\begin{tabular}{llccccccc}
\hline
 &  & \multicolumn{3}{c}{\texttt{\dataset-L}} & \multicolumn{3}{c}{\texttt{GTools-EL}} & \textbf{Avg.} \\
\cmidrule(lr){3-5} \cmidrule(lr){6-8}
 & \textbf{Method} 
& \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
& \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
&  \\
\hline
\multirow{4}{*}{(i)}
 & \textsc{Zero-shot} & 12.4 & 3.8 & 0.0 & 78.9 & 42.2 & 41.1 & 29.7 \\
 & \textsc{Few-shot} & 13.3 & 2.4 & 0.0 & 83.3 & 62.8 & 48.3 & 35.0 \\
 & \textsc{CoT} & 11.9 & 5.7 & 0.0 & 84.4 & 53.9 & 46.1 & 33.7 \\
 & \textsc{Bag-CoT} & 8.6 & 3.3 & 0.0 & 71.7 & 33.3 & 24.4 & 23.6 \\
\hline
\multirow{2}{*}{(ii)}
 & \textsc{Zero-shot Coding} & 85.2 & 66.2 & 39.1 & 93.3 & 94.4 & 48.3 & 71.1 \\
 & \textsc{Few-shot Coding} & 84.7 & 44.8 & 48.1 & 83.3 & 91.7 & 64.4 & 69.5 \\
\hline
\multirow{2}{*}{(iii)}
 & \textsc{TFIDF+Coding} & 94.3 & 89.5 & 47.6 & 80.6 & 72.2 & 55.6 & 73.3 \\
 & \textsc{SentBERT+Coding} & 94.3 & 88.1 & 70.5 & 91.7 & 77.8 & 36.1 & 76.4 \\
\hline
\multirow{3}{*}{(iv)}
 & \textsc{GraphTeam} & 87.6 & 81.0 & 70.5 & 94.4 & 75.0 & 77.8 & 81.1 \\
 & \textsc{TFIDF+CodeAgent} & 94.3 & 94.3 & 80.0 & 83.3 & 91.7 & 72.2 & 85.9 \\
 & \textsc{SentBERT+CodeAgent} & \textbf{99.1} & \textbf{99.1} & 97.6 & 94.4 & 94.4 & 83.3 & 94.7 \\
\hline
\multirow{2}{*}{(v)}
 & \textsc{CodeGraph} & 99.1 & 99.1 & 90.0 & 100.0 & 94.4 & 67.8 & 91.7  \\
 & \textsc{PIE} & 97.1 & 96.7 & 90.0 & 94.4 & 100.0 & 77.8 & 92.7 \\
\hline
\rowcolor{lightgray}
 & \textsc{Ours} & \textbf{99.1} & \textbf{99.1} & \textbf{99.1} & \textbf{100.0} & \textbf{100.0} & \textbf{86.1} & \textbf{97.2} \\
\hline
\end{tabular}
}
\vskip -1em
\end{table*}

% \fali{One question: whether we explicitly validate our dataset is necessary? For the large-scale graph subset, it could validate that the textual reasoning is very limited when the graph size scales to a certain number. For the composite graph subset, it can validate the model's ability required for composite graph reasoning, as the 7B model performs significantly weaker than the other two larger models. Where should we state them?  }
% \suhang{Yes, we should explicitly validate that. We can put state these in the dataset construction subsection and put the details in the appendix}

\subsection{Experimental Setup}


% We experiment on the proposed {\dataset} and GTools~\cite{zhang2024gcoderimprovinglargelanguage} datasets using NetworkX \cite{hagberg2008exploring} as technical documentation. Accuracy is used as the metric. We use DeepSeek-V3 \cite{liu2024deepseek} as the retrieval agent, the coding agent is selected from DeepSeek-V3 \cite{liu2024deepseek}, LLaMA-3-70B \cite{dubey2024llama}, and Qwen-7b \cite{huang2025opencoder}, and a fixed temperature of $0.1$.
% Baselines fall into several categories: (i) Text-based reasoning methods include zero-shot, few-shot, chain-of-thought, and build-a-graph~\cite{wang2024languagemodelssolvegraph}. Code-based methods are grouped by retrieval and debugging: (ii) no retrieval or debugging, including LLM Coding and GraphTool-Instruction~\cite{wang2025GraphToolInstruction}; (iii) retrieval without debugging, including GCoder~\cite{zhang2024gcoderimprovinglargelanguage} using BAAI/bge-base-en-v1.5 \cite{xiao2024c}; and (iv) retrieval with debugging, including GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage} using LlamaIndex \cite{llamaindexweb}. We additionally compare TF-IDF + coding agent and Sentence-BERT + coding agent, which retrieve top-$k$ documents from flattened documentation using the same coding agent. (v) Human-involved baselines include CodeGraph~\cite{cai2024codegraphenhancinggraphreasoning} and PIE~\cite{gong2025pseudocodeinjectionmagicenablingllms}. Details are in Appendix~\ref{x}.
% GCoder and GraphTeam originally place graph data in the prompt; for large-graph settings, we modify them to decouple tasks from graph data to fit code generation and execution.

\noindent\textbf{Datasets, Metrics, and LLM Backbones.} We conduct experiments on both \texttt{GTools}~\cite{wang2025GraphToolInstruction} (including two subsets: \texttt{GTools-EL} for large graphs and \texttt{GTools-WL} for small graphs) and the proposed {\dataset} datasets below, using NetworkX~\cite{hagberg2008exploring} as the technical documentation. Accuracy is used as the primary evaluation metric. DeepSeek-V3.2~\cite{liu2024deepseek} is consistently employed as the retrieval agent, while the coding agent is selected from DeepSeek-V3.2, LLaMA-3.3-70B~\cite{dubey2024llama}, and Qwen-2.5-7B~\cite{bai2023qwentechnicalreport}. The temperature is fixed at $0.1$ all the time. Additional details are provided in Appendix~\ref{sec:appendix_experiment_setup}.

\noindent\textbf{Baselines.}
Baselines are grouped into five categories.
(i) \textbf{Pure text-based reasoning} methods include \textsc{Zero-shot}, \textsc{Few-shot}, \textsc{Chain-of-Thought} (\textsc{CoT}), and \textsc{Build-a-Graph} (\textsc{BaG-CoT})~\cite{wang2024languagemodelssolvegraph}.
Code-based methods are further categorized by the use of retrieval and debugging:
(ii) \textbf{No retrieval or debugging}, including \textsc{Zero-shot Coding}, which directly prompts the LLM to generate executable code, and \textsc{Few-shot Coding}, which provides a small number of task-irrelevant example code solutions as in-context demonstrations;
(iii) \textbf{Retrieval without debugging}, including \textsc{TFIDF+ Coding} and \textsc{SentBERT+Coding}, which augment \textsc{Zero-shot Coding} with document retrieval using TF-IDF \cite{salton1988term} and Sentence-BERT~\cite{reimers2019sentence}, respectively, but do not perform iterative code correction.
(iv) \textbf{Retrieval with debugging}, including \textsc{GraphTeam}~\cite{li2025graphteamfacilitatinglargelanguage}, which leverages LlamaIndex~\cite{llamaindexweb} for flattened semantic retrieval and incorporates runtime debugging.
We additionally include \textsc{TFIDF+ CodeAgent} and \textsc{SentBERT+CodeAgent}, which use TF-IDF and Sentence BERT to retrieve top-$k$ documents from flattened documentation and employ the same coding agent with self-debugging as our method.
(v) \textbf{Human-involved} baselines include \textsc{CodeGraph}~\cite{cai2024codegraphenhancinggraphreasoning}, which relies on manually selected documents, and \textsc{PIE}~\cite{gong2025pseudocodeinjectionmagicenablingllms}, which depends on manually provided pseudocodes and labeled test cases for debugging.
% \textsc{GraphTeam} originally places graph data directly in the prompt; for large-graph settings, we modify it to decouple task descriptions from graph data to enable code generation and execution.
Baseline details are provided in Appendix~\ref{appendix:baselines}.

\subsection{The Proposed {\dataset} Dataset}
We introduce a new synthetic benchmark, {\dataset}, for evaluating \emph{complex} graph reasoning under both \emph{structural} and \emph{semantic} complexity. {\dataset} contains three subsets: small-scale \texttt{\dataset-S} with small graphs (3--200 nodes), large-scale \texttt{\dataset-L} with large graphs (5k and 10k nodes) to stress scalability beyond context-window limits, and composite \texttt{\dataset-C} consisting of composite tasks that require composing two or three primitive graph algorithms via sequential, parallel, or conditional structures. All graph instances are generated with NetworkX \cite{hagberg2008exploring}, and labels are produced by executing human-crafted reference implementations. Detailed task lists, construction procedures, and statistics are provided in Appendix \ref{sec:our_dataset}. From the results, existing ``large-graph'' benchmarks (e.g., \texttt{GTools-EL}, $\leq 200$ nodes) remain within typical LLMs' context windows and thus do not challenge text-based reasoning (Tab. \ref{tab:main_results_L}). In contrast, on our truly large-scale graphs, text reasoning degrades sharply, demonstrating that our {\dataset} introduces meaningful structural complexity. For composite tasks, semantic complexity reveals clear limitations of prior retrieval-plus-coding baselines (e.g., \textsc{GraphTeam} in Tab.~\ref{tab:main_results_C}), while human-curated retrieval and test-case debugging yield much larger gains, highlighting deficiencies in existing retrieval/coding components and validating the necessity of our proposed composite dataset (detailed analysis in Appendix \ref{appendix:analysis_our_dataset}).

% \subsection{{\dataset} dataset}
% \label{sec:our_dataset}
% \noindent\textbf{\texttt{\dataset-L(S)}: A Large(Small)-Scale Graph Reasoning Dataset.}
% Prior work has proposed many graph reasoning datasets, but they primarily focus on \emph{small-scale graphs}. For example, GraphQA \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} typically involve graphs with $3$--$10$ nodes, which can be accommodated by existing LLMs. For example, Qwen-7B \cite{bai2023qwentechnicalreport} with an $8$k context window supports complete graphs of $\sim40$ nodes, and DeepSeek-V3 \cite{liu2024deepseek} with a $160$k window supports $\sim180$ nodes. Such settings do not capture the challenges of \emph{large-scale graph reasoning} under long-context abilities and window size limit.
% To address this gap, we construct {\dataset}, a synthetic dataset spanning graph sizes from $3$ to $10$k nodes, including a small-scale subset \texttt{\dataset-S} (graphs with $3$--$200$ nodes) and a large-scale subset \texttt{\dataset-L} (graphs with $5$k and $10$k nodes). Following \cite{wu-etal-2025-grapheval36k}, we select {$14$} classical graph algorithm tasks (see Tables~\ref{tab:small_scale_task_overview}, \ref{tab:task_overview} in Appendix). Task descriptions are written by graph-domain experts manually, and graph instances are generated using NetworkX \cite{hagberg2008exploring}.
% {\dataset} covers diverse graph types (directed or undirected, weighted or unweighted, cyclic or acyclic, connected or disconnected, and sparse or dense). For each task, we generate only semantically appropriate graph types; for instance, connected component counting is evaluated exclusively on unweighted undirected graphs, as the weights do not affect the result. Edge weights for directed graphs are sampled uniformly from $[0.0, 10.0]$, and task-specific inputs (e.g., source--target pairs) are selected uniformly at random. Labels are generated using human-crafted reference code. 

% \noindent\textbf{\texttt{\dataset-C}: A Composite Graph Reasoning Dataset.}
% Existing datasets ignore \emph{complex graph reasoning tasks} at the problem level. Thus, we introduce the \texttt{\dataset-C} subset, which focuses on \emph{composite graph tasks} formed by combining multiple classical graph algorithms and requiring multi-step reasoning. For example, a task may first perform topological sorting to determine precedence constraints and then count paths between a source and a target within the constrained reachable subgraph.
% The construction of \texttt{\dataset-C} follows three steps: 
% (1) {Subtask selection}: randomly sample two or three primitive graph tasks from \texttt{\dataset-S}; 
% (2) {Task composition}: combine the sampled primitive tasks using one of three patterns: sequential (execute A then B), parallel (execute A and B and aggregate outputs), and conditional (execute A, then B or C based on a condition). Only logically and semantically valid combinations are retained. Task descriptions are generated using GPT-5 \cite{openai2025gpt5api}; the prompt and the details are in Appendix \ref{sec:appendix_details_composite}. 
% (3) {Graph Instance and label generation}: graph instances are reused from \texttt{\dataset-S}, and labels are obtained by executing human-crafted reference code. We construct $9$ composite graph tasks with detailed task descriptions and statistics in Tab.~\ref{tab:composite_tasks} in Appendix.

% \fali{check how to generate composite task description?}



% \noindent\textbf{Baselines} Baselines are grouped into several categories. (i) Pure Text-based reasoning methods include zero-shot, few-shot, chain-of-thought (CoT), and build-a-graph (BaG-CoT)~\cite{wang2024languagemodelssolvegraph}. Code-based methods are further categorized by retrieval and debugging: (ii) no retrieval or debugging, including LLM Zero-shot Coding, which directly prompts the LLM to generate executable code, and Few-shot Coding, which xxx; (iii) retrieval without debugging, including 
% % GraphTool-Instruction~\cite{wang2025GraphToolInstruction}, which places retrieved tool descriptions in the prompt; 
% % GCoder~\cite{zhang2024gcoderimprovinglargelanguage}, which performs flattened document retrieval using BAAI/bge-base-en-v1.5~\cite{xiao2024c}
% % and another 
% \textsc{TFIDF+Coding} and \textsc{SentBERT+Coding} using retrieval by TF-IDF and Sentence-BERT, respectively, based on Zero-shot Coding.
% (iv) retrieval with debugging, including GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage}, which leverages LlamaIndex \cite{llamaindexweb} for flattened semantic retrieval and incorporates runtime debugging. We additionally compare TF-IDF + coding agent and Sentence-BERT + coding agent, which retrieve top-$k$ documents from flattened documentation and use the same coding agent as our method. (v) Human-involved baselines include CodeGraph~\cite{cai2024codegraphenhancinggraphreasoning}, which relies on manually selected exemplars without retrieval, and PIE~\cite{gong2025pseudocodeinjectionmagicenablingllms}, which depends on manually provided pseudocode examples and test cases for debugging.
% GraphTeam originally placed graph data in the prompt; for large-graph settings, we modify them to decouple tasks from graph data to fit code generation and execution. Baseline Details are provided in Appendix~\ref{sec:appendix_experiment_setup}.

% TF-IDF indexing~\cite{salton1988term} Sentence-BERT embeddings~\cite{reimers2019sentence} 


% \fali{Under the experimental setup, we need to introduce datasets, metrics, models, baselines.}
% \textbf{Dataset Configuration} 
% \fali{detail how we construct graph: how to generate graphs, how to get the task desc, how to get the label?} We evaluate our method on the dataset built following the GraphEval36K \cite{wu-etal-2025-grapheval36k} benchmark, which contains 8 main graph categories and 4 structural sub-categories \fali{please note that we need to clearly express, do not write like this "contains 8 main graph categories". It is confused that what are the 8 graph categories?}. \xianren{If you have too many details about the setup (e.g., dataset) you can consider put the details in the appendix and refer here. For example, 8 graph categories such as xxx. More details about xxx are in Appendix X. You can also use this strategy for baselines introduction} Building upon this foundation, we extend and reorganise the dataset to better align with the objectives of this study. Each instance in our constructed dataset is represented as a list [task description, input graph, input arguments (optional), ground-truth output].
% The \textit{task description} is expressed in natural language, specifying the computational goal (e.g., \textit{Find the cost of the shortest path from source A to target B.}); the \textit{input graph} is provided in edge list;
% and the \textit{ground-truth output} represents the correct result computed via verified graph algorithms implemented in \texttt{NetworkX}. \fali{give the dataset statistics table.}


% \textbf{Task Selection \fali{we do not need this paragraph. Task selection should be introduced in the dataset configuration.}} We incorporate 15 representative graph reasoning tasks. These tasks collectively cover a spectrum of algorithmic paradigms, including traversal-based reasoning, combinatorial optimisation, and structural pattern
% \& neighbourhood analysis, and graph property detection. The Table~\ref{tab:graph_task_paradigms} shows the classification of algorithmic paradigms.
% Each task instance is further classified according to graph directionality and edge weighting, resulting in four configuration types: (i) Directed and Unweighted; (ii) Directed and Weighted; (iii) Undirected and Unweighted; and (iv) Undirected and Weighted. For weighted graphs, edge weights are sampled from a continuous uniform distribution in the range $[0.0, 10.0)$. This ensures numerical diversity across instances while maintaining consistent scaling across all categories. Table~\ref{tab:wd_graph_tasks} lists the directedness and weightedness of our graph tasks.


% \begin{table}[htbp]
% \centering
% \small 
% \caption{Classification of Graph Tasks by Algorithmic Paradigm.}
% \label{tab:graph_task_paradigms}
% \renewcommand{\arraystretch}{1.5}
% \setlength{\tabcolsep}{6pt}
% \begin{tabular}{
%     | >{\centering\arraybackslash}m{2cm}
%     | >{\centering\arraybackslash}m{5cm} |
% }
% % \hline
% % \textbf{Algorithmic Paradigm} & \textbf{Representative Tasks} \\
% % \hline

% % \textbf{Traversal-Based Reasoning} &
% % \texttt{Connected Components}, \texttt{Strongly Connected Components}, \texttt{Shortest Path Length}, \texttt{Topological Sorts} \\
% % \hline

% % \textbf{Graph Property Detection} &
% % \texttt{Graph Eulerian Path Check}, \texttt{Graph Bipartiteness}, \texttt{Graph Distance Regularity}, \texttt{Graph Diameter}, \texttt{Graph Connectivity} \\
% % \hline

% % \textbf{Combinatorial Optimisation} &
% % \texttt{Travelling Salesman Problem}, \texttt{Maximum Flow}, \texttt{Maximum Clique}, \texttt{Maximum Independent Set} \\
% % \hline

% % \textbf{Structural Pattern \& Neighbourhood Analysis} &
% % \texttt{Clustering Coefficient}, \texttt{Common Neighbours} \\
% % \hline
% \hline
% \textbf{Algorithmic Paradigm} & \textbf{Representative Tasks} \\
% \hline

% \textbf{Traversal-Based Reasoning} &
% \texttt{Connected Components Counting}, \texttt{Strongly Connected Components Counting}, \texttt{Shortest Path}, \texttt{Topological Sorting} \\
% \hline

% \textbf{Graph Property Detection} &
% \texttt{Eulerian Path Existence}, \texttt{Graph Bipartiteness}, \texttt{Distance Regularity Check}, \texttt{Graph Diameter}, \texttt{Graph Connectivity} \\
% \hline

% \textbf{Combinatorial Optimization} &
% \texttt{Traveling Salesman Problem}, \texttt{Maximum Flow}, \texttt{Maximum Clique}, \texttt{Maximum Independent Set} \\
% \hline

% \textbf{Structural Pattern \& Neighborhood Analysis} &
% \texttt{Clustering Coefficient}, \texttt{Common Neighbors} \\
% \hline

% \end{tabular}
% \end{table}

% \begin{table*}[t]
% \centering
% \tiny 
% \begin{tabular}{lccccccccccc}
% \hline
% \textbf{Task} &
% \textbf{Task Type} &
% \textbf{Graph Model} &
% \textbf{Degree Dist.} &
% \textbf{Structural Constraint} &
% \textbf{Density} &
% \textbf{Connectivity} &
% \textbf{Cyclicity} &
% \textbf{Examples (T)} &
% \textbf{Examples (X)} &
% \textbf{\#Node Range} \\
% \hline

% Connected Components Counting
% & Traversal
% & ER
% & Poisson
% & --
% & Sparse
% & Disconnected
% & Acyclic / Cyclic
% & $T_{\text{CC}}$
% & $X_{\text{CC}}^{ER}$
% & $[20,200]$ \\

% Shortest Path
% & Traversal
% & Small-world
% & --
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{SP}}$
% & $X_{\text{SP}}^{SW}$
% & $[20,200]$ \\

% Topological Sorting
% & Traversal
% & ER
% & --
% & --
% & Sparse
% & Connected / Disconnected
% & Acyclic
% & $T_{\text{TS}}$
% & $X_{\text{TS}}^{ER}$
% & $[20,200]$ \\

% Eulerian Path Existence
% & Graph Property
% & Regular
% & --
% & Regular
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{EP}}$
% & $X_{\text{EP}}^{Reg}$
% & $[20,200]$ \\

% Graph Bipartiteness
% & Graph Property
% & ER
% & Poisson
% & --
% & Sparse
% & Connected / Disconnected
% & Acyclic / Cyclic
% & $T_{\text{BI}}$
% & $X_{\text{BI}}^{ER}$
% & $[20,200]$ \\

% Graph Diameter
% & Graph Property
% & Small-world
% & --
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{DI}}$
% & $X_{\text{DI}}^{SW}$
% & $[20,200]$ \\

% Distance Regularity Check
% & Graph Property
% & Regular
% & --
% & Regular
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{DR}}$
% & $X_{\text{DR}}^{Reg}$
% & $[20,200]$ \\

% Traveling Salesman Problem
% & Combinatorial Opt.
% & Complete
% & --
% & --
% & Complete
% & Connected
% & Cyclic
% & $T_{\text{TSP}}$
% & $X_{\text{TSP}}^{Comp}$
% & $[20,200]$ \\

% Maximum Flow
% & Combinatorial Opt.
% & ER
% & Poisson
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{MF}}$
% & $X_{\text{MF}}^{ER}$
% & $[20,200]$ \\

% Maximum Clique
% & Combinatorial Opt.
% & Power-law
% & Power-law
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{MC}}$
% & $X_{\text{MC}}^{PL}$
% & $[20,200]$ \\

% Maximum Independent Set
% & Combinatorial Opt.
% & Power-law
% & Power-law
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{MIS}}$
% & $X_{\text{MIS}}^{PL}$
% & $[20,200]$ \\

% Clustering Coefficient
% & Neighborhood
% & Small-world
% & --
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{CL}}$
% & $X_{\text{CL}}^{SW}$
% & $[20,200]$ \\

% Common Neighbors
% & Neighborhood
% & ER
% & Poisson
% & --
% & Sparse
% & Connected
% & Cyclic
% & $T_{\text{CN}}$
% & $X_{\text{CN}}^{ER}$
% & $[20,200]$ \\

% \hline
% \end{tabular}
% \caption{Dataset taxonomy and statistics across task types, graph generation models, structural constraints, and graph properties.}
% \label{tab:benchmark_taxonomy}
% \end{table*}

% \begin{table*}[t]
% \centering
% \tiny
% \begin{tabular}{lccccccccccc}
% \hline
% \textbf{Task} &
% \textbf{TT} &
% \textbf{Graph Model} &
% \textbf{Degree Dist.} &
% \textbf{SC} &
% \textbf{Density} &
% \textbf{Connectivity} &
% \textbf{Cyclicity} &
% \textbf{Examples (T)} &
% \textbf{Examples (X)} &
% \textbf{\#N} \\
% \hline

% Connected Components Counting
% & T & ER & P & -- & S & D & A/C & $T_{\text{CC}}$ & $X_{\text{CC}}$ & [20,200] \\

% Shortest Path
% & T & SW & -- & -- & S & C & C & $T_{\text{SP}}$ & $X_{\text{SP}}$ & [20,200] \\

% Topological Sorting
% & T & ER & -- & -- & S & C/D & A & $T_{\text{TS}}$ & $X_{\text{TS}}$ & [20,200] \\

% Eulerian Path Existence
% & P & R & -- & R & S & C & C & $T_{\text{EP}}$ & $X_{\text{EP}}$ & [20,200] \\

% Graph Bipartiteness
% & P & ER & P & -- & S & C/D & A/C & $T_{\text{BI}}$ & $X_{\text{BI}}$ & [20,200] \\

% Graph Diameter
% & P & SW & -- & -- & S & C & C & $T_{\text{DI}}$ & $X_{\text{DI}}$ & [20,200] \\

% Distance Regularity Check
% & P & R & -- & R & S & C & C & $T_{\text{DR}}$ & $X_{\text{DR}}$ & [20,200] \\

% Traveling Salesman Problem
% & O & K & -- & -- & K & C & C & $T_{\text{TSP}}$ & $X_{\text{TSP}}$ & [20,200] \\

% Maximum Flow
% & O & ER & P & -- & S & C & C & $T_{\text{MF}}$ & $X_{\text{MF}}$ & [20,200] \\

% Maximum Clique
% & O & PL & PL & -- & S & C & C & $T_{\text{MC}}$ & $X_{\text{MC}}$ & [20,200] \\

% Maximum Independent Set
% & O & PL & PL & -- & S & C & C & $T_{\text{MIS}}$ & $X_{\text{MIS}}$ & [20,200] \\

% Clustering Coefficient
% & N & SW & -- & -- & S & C & C & $T_{\text{CL}}$ & $X_{\text{CL}}$ & [20,200] \\

% Common Neighbors
% & N & ER & P & -- & S & C & C & $T_{\text{CN}}$ & $X_{\text{CN}}$ & [20,200] \\

% \hline
% \end{tabular}
% \caption{
% Dataset taxonomy and statistics.
% \textbf{TT}: task type (T=Traversal, P=Property detection, O=Optimization, N=Neighborhood);
% \textbf{GM}: graph model (ER=Erd\H{o}s--R\'enyi, SW=Small-world, PL=Power-law, R=Regular, K=Complete);
% \textbf{DD}: degree distribution (P=Poisson, PL=Power-law);
% \textbf{SC}: structural constraint;
% \textbf{D}: density (S=Sparse, K=Complete);
% \textbf{Conn}: connectivity (C=Connected, D=Disconnected);
% \textbf{Cyc}: cyclicity (A=Acyclic, C=Cyclic);
% \textbf{T}: task-level examples;
% \textbf{X}: graph instances;
% \textbf{N}: node range.
% }
% \label{tab:benchmark_taxonomy}
% \end{table*}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \begin{table}[t]
% \centering
% \small 
% \setlength{\tabcolsep}{4pt}
% \renewcommand{\arraystretch}{1.25}
% \caption{Mapping between graph configurations and the corresponding computational tasks.}
% \label{tab:wd_graph_tasks}

% \begin{tabular}{p{0.15\linewidth} p{0.15\linewidth} p{0.65\linewidth}}
% \toprule
% \textbf{Directed} & \textbf{Weighted} & \textbf{Tasks} \\
% \midrule

% \textbf{True} & \textbf{True} &
% \texttt{Maximum Flow}, \texttt{Travelling Salesman Problem}, \texttt{Shortest Path Length} \\

% \textbf{True} & \textbf{False} &
% \texttt{Travelling Salesman Problem}, \texttt{Graph Bipartiteness}, 
% \texttt{Topological Sorts}, \texttt{Strongly Connected Components}, 
% \texttt{Graph Eulerian Path Check} \\

% \textbf{False} & \textbf{True} &
% \texttt{Graph Diameter}, \texttt{Clustering Coefficient}, 
% \texttt{Common Neighbours}, \texttt{Connected Components} \\

% \textbf{False} & \textbf{False} &
% \texttt{Clustering Coefficient}, \texttt{Graph Diameter}, 
% \texttt{Graph Distance Regularity}, \texttt{Graph Connectivity}, 
% \texttt{Maximum Independent Set}, \texttt{Maximum Clique}, 
% \texttt{Connected Components}, \texttt{Common Neighbours} \\
% \bottomrule
% \end{tabular}
% \end{table}





%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% \textbf{Randomised Argument Sampling} For graph tasks that require additional parameters (e.g., source/target nodes for shortest path, flow source/sink for maximum flow), these arguments are randomly sampled to cover a wider spectrum of test cases, including edge conditions that may challenge generalisation. This design choice ensures that the evaluation is not biased toward easy or repetitive configurations and provides a more realistic assessment of robustness under variable task parameters.

% \textbf{Model Configuration}
% We test our framework using four LLMs representative of both open and closed-source families: DeepSeek-V3, LLaMA-3.3-70B, and Qwen-7b. All models are initialised at a temperature of 0.1 to ensure determinism. We use DeepSeek-V3 for the retrieval and verifier agent, and DeepSeek-V3, LLaMA-3.3-70B, Qwen-7b for the code generation agent \fali{why different model for two agents?}.

% \subsection{Baselines}
% \fali{only use one short paragraph to show the baselines. Details in Appendix. Refer to my work for introducing baselines. Baselines are part of the Experimental setup.}
% \subsubsection{Graph Task Solving Baselines (RQ1)} \label{sec: graph_task_solving_baselines}
% To comprehensively evaluate the performance of the proposed Agentic RAG framework, we compare it against a diverse set of baseline methods grouped into two categories: (1) retrieval-based baselines, and (3) code-based benchmarks.
% This design allows us to isolate the effect of hierarchical retrieval, agentic coordination, and iterative debugging on the overall reasoning performance of LLMs.


% \textbf{Retrieval-Based Baselines}
% Retrieval-based baselines introduce an intermediate step that retrieves potentially useful information from external sources (e.g., NetworkX documentation or function manuals) before code generation.

% \begin{itemize}
%     \item \textbf{Code generation with TF-IDF Retrieval  \cite{10.5555/106765.106782}} This baseline uses term-frequency–inverse-document-frequency (TF-IDF) matching to select the top-$k$ documentation snippets most relevant to the task description. The retrieved content is concatenated to the prompt before code generation. TF-IDF retrieval provides a simple lexical similarity baseline without semantic understanding.
%     \item \textbf{Code Generation with Sentence-BERT Retrieval  \cite{reimers2019sentencebertsentenceembeddingsusing}} This approach replaces lexical matching with semantic similarity retrieval. Each documentation page and task description is embedded using Sentence-BERT, and cosine similarity is used to identify the most semantically related documentation. This baseline tests whether semantic retrieval alone (without hierarchical structure, agentic verification and debugging) suffices for effective context selection.
% \end{itemize}

% Comparing Agentic RAG to TF-IDF and Sentence-BERT baselines allows us to quantify the contribution of structured retrieval, multi-agent verification and the debugging phase relative to flat lexical or semantic retrieval. 

% \textbf{Code-Based Baselines}
% In addition to prompting and retrieval-based comparisons, we also evaluate Agentic RAG against two recent LLM-based code reasoning systems that target algorithmic and graph-related problem-solving. We selected methods with unfine-tuned models, PSEUDO \cite{gong2025pseudocodeinjectionmagicenablingllms} injects the pseudocode into the prompt and followed by a debugging phase, CodeGraph \cite{cai2024codegraphenhancinggraphreasoning} provides LLMs with the code examples to obtain better performance, GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} leverages a multi-agent framework to resolve graph tasks cooperatively. 

% % \suhang{give high-level idea of each method as you did above}

% \subsubsection{Retrieval Accuracy Baselines (RQ2)} \label{sec: retrieval_baselines}
% We conduct this experiment to evaluate whether Agentic RAG's hierarchical and task-aware retrieval mechanism leads to more accurate identification of relevant documentation compared to traditional methods like TF-IDF and Sentence-BERT, and an existing method for NetworkX documentation retrieval, GraphTeam \cite{li2025graphteamfacilitatinglargelanguage}. Since retrieval quality directly affects code generation accuracy, this experiment is critical for assessing the framework’s ability to interpret diverse task descriptions and dynamically navigate structured documentation.

% \begin{itemize}
%     \item \textbf{TF-IDF (Term Frequency–Inverse Document Frequency)  \cite{10.5555/106765.106782}}: It is a classic lexical retrieval method that ranks documents based on how frequently query terms appear in them, weighted by how unique those terms are across the entire corpus.

%     \item \textbf{Sentence-BERT \cite{reimers2019sentencebertsentenceembeddingsusing}}: Sentence-BERT is a semantic retrieval method that encodes both queries and documents into dense vector embeddings using a transformer-based model, and rank based on Cosine Similarity

%     \item \textbf{GraphTeam \cite{li2025graphteamfacilitatinglargelanguage}}: The retrieval in GraphTeam starts by reformatting the questions through LLMs and leveraging Llama-index \cite{llamaindexweb} for relevance ranking.

% \end{itemize}
    




% \subsection{Context Level Baselines (RQ4)}
% Context-level baselines evaluate the reasoning ability of LLMs under different levels of contextual guidance, without any retrieval mechanism or model fine-tuning. 

% \begin{itemize}
% \item \textbf{No Example (Direct Generation)}
% The model is provided only with a plain-text description of the graph task and the corresponding graph input. This setting assesses the intrinsic problem-solving capability of LLMs without any external hints or prior examples.

% \item \textbf{Use NX No Example (One-Shot Prompting of Tool Hint)}
% In this setting, we append one-shot hint of using NetworkX in the task query. This setup evaluates whether exposure to a tool hint without an actual example helps the model identify relevant functions and generate correct code.

% % \suhang{you mentioned tool usage, can you explain what tools are available}
% \item \textbf{NX Usage Example Provided (Raw Doc Appended)}
% To test whether simple inclusion of external documentation improves reasoning, we append the raw NetworkX documentation relevant to the task directly into the prompt, without retrieval and debugging. This baseline helps determine whether a full.
% \end{itemize}


% \begin{table}[ht]
% \centering
% \small 
% \caption{Comparative results on {\dataset-L} and GTools-L using various coding-agent LLMs, evaluated with the Accuracy metric. Best in each column is bolded. (i)-(v) mean same above.}
% \label{tab:pass_rate_transposed}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{llccc}
% \toprule
% \textbf{} & \textbf{Method} & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7b} \\
% \midrule
% \multirow{4}{*}{(i)}
%  & Zero-shot &  &  &  \\
%  & Few-shot &  &  &  \\
%  & Chain-of-Thought &  &  &  \\
%  & Build-a-Graph &  &  &  \\
% \midrule
% \multirow{2}{*}{(ii)}
%  & LLM Coding &  &  &  \\
%  & GraphTool-Instruction &  &  &  \\
% \midrule
% \multirow{3}{*}{(iii)}
%  & GCoder &  &  &  \\
%  & TF-IDF+Coding &  &  &  \\
%  & Sentence-BERT+Coding &  &  &  \\
% \midrule
% \multirow{1}{*}{(iv)}
%  & GraphTeam &  &  &  \\
% \midrule
% \multirow{2}{*}{(v)}
%  & CodeGraph &  &  &  \\
%  & PIE &  &  &  \\
% \bottomrule
% \end{tabular}
% }
% \end{table}


% original
% \begin{table*}[t]
% \centering
% \small
% \caption{Comparative results on {\dataset-L} and GTools-EL using various coding-agent LLMs, evaluated with the Accuracy metric. Best in each column is bolded. (i)–(v) follow the same category definitions as above.}
% \label{tab:main_results_L}
% \vskip -1em
% \resizebox{0.8\linewidth}{!}{
% \renewcommand{\arraystretch}{1.1}  % 1.2–1.5 is typical
% \begin{tabular}{llcccccc}
% \hline
%  &  & \multicolumn{3}{c}{{\dataset-L}} & \multicolumn{3}{c}{GTools-EL} \\
% \cmidrule(lr){3-5} \cmidrule(lr){6-8}
% \textbf{} & \textbf{Method} 
% & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B}
% & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7B} \\
% \hline
% \multirow{4}{*}{(i)}
%  & \textsc{Zero-shot} & 12.4 & 3.8 & 0 & 78.9 & 42.2 & 41.1 \\
%  & \textsc{Few-shot} & 13.3 & 2.4 & 0 & 83.3 & 62.8 & 48.3 \\
%  & \textsc{CoT} & 11.9 & 5.7 & 0 & 84.4 & 53.9 & 46.1 \\
%  & \textsc{Bag-CoT} & 8.6 & 3.3 & 0 & 71.7 & 33.3 & 24.4 \\
% \hline
% \multirow{2}{*}{(ii)}
%  & \textsc{Zero-shot Coding} & 85.2 & 66.2 & 39.1 & 93.3 & 94.4 & 48.3 \\
%  & \textsc{Few-shot Coding} & 84.7 & 44.8 & 48.1 & 83.3 & 91.7 & 64.4 \\
% \hline
% \multirow{3}{*}{(iii)}
%  % & \textsc{GCoder} &  &  &  &  &  &  \\
%  % & \textsc{GraphTool-Instruction} & 0.96 & 0.78 & 0.57 &  &  &  \\
%  & \textsc{TFIDF+Coding} & 94.3 & 89.5 & 47.6  & 80.6 & 72.2 & 55.6 \\
%  & \textsc{SentenceBERT+Coding} & 94.3 & 88.1 & 70.5 & 91.7 & 77.8 & 36.1  \\
% \hline
% \multirow{3}{*}{(iv)}
%  & \textsc{GraphTeam} & 87.6 & 81.0 & 70.5 & 94.4 & 75.0 & 77.8 \\
%  & \textsc{TFIDF+CodeAgent} & 94.3 & 94.3 & 80.0 & 83.3 & 91.7 & 72.2 \\
%  & \textsc{SentenceBERT+CodeAgent} & 99.1 & 99.1 & 97.6 & 94.4 & 94.4 & 83.3 \\
% \hline
% \multirow{2}{*}{(v)}
%  & \textsc{CodeGraph} & 99.1 & 99.1 & 56.7 & 100.0 & 94.4 & 52.8 \\
%  & \textsc{PIE} & 97.1 & 85.2 & 72.9 & 94.4 & 100.0 & 77.8 \\
% \hline
% \rowcolor{lightgray}
%  % \multirow{1}{*}{}& \textsc{Ours} & 99.1 & 99.1 & 89.5 & 100.0 & 100.0 & 86.1 \\
%  % change the 89.5 to 99.1 becasue considering the more correction budget
%  \multirow{1}{*}{}& \textsc{Ours} & 99.1 & 99.1 & 99.1 & 100.0 & 100.0 & 86.1 \\
% \hline
% \end{tabular}
% }
% \end{table*}





% \begin{table}[t]
% \centering
% \small
% \caption{Comparative results on {\dataset-L} using various coding-agent LLMs, evaluated with the Accuracy metric. Best in each column is bolded. (i)–(v) follow the same category definitions as above.}
% \label{tab:main_results_L}
% \vskip -1em
% \resizebox{\linewidth}{!}{
% \renewcommand{\arraystretch}{1.1}
% \begin{tabular}{llccc}
% \hline
% \textbf{} & \textbf{Method} 
% & \textbf{DeepSeek} & \textbf{LLaMA} & \textbf{OpenCoder} \\
% \hline
% \multirow{4}{*}{(i)}
%  & \textsc{Zero-shot} & / & / & / \\
%  & \textsc{Few-shot} & / & / & / \\
%  & \textsc{CoT} & / & / & / \\
%  & \textsc{Bag-CoT} & / & / & / \\
% \hline
% \multirow{2}{*}{(ii)}
%  % & \textsc{Zero-shot Coding} & 0.56 & 0.76 & 0.68 \\ 
%  & \textsc{Zero-shot Coding} &  &  &  \\ 
%  & \textsc{Few-shot Coding} & 0.82 & 0.65 & 0.23 \\
% \hline
% \multirow{3}{*}{(iii)}
%  % & \textsc{GraphTool-Instruction} & 0.96 & 0.78 & 0.57 \\
%  % & \textsc{TFIDF+Coding} & 0.84 & 0.92 & 0.37 \\
%  % & \textsc{SentenceBERT+Coding} & 0.66 & 0.93 & 0.31 \\
%   & \textsc{TFIDF+Coding} &  &  &  \\
%  & \textsc{SentenceBERT+Coding} &  &  &  \\
% \hline
% \multirow{3}{*}{(iv)}
%  % & \textsc{GraphTeam} & 0.89 & 0.66 & 0.59 \\
%  % & \textsc{TFIDF+CodeAgent} & 0.70 & 0.89 & 0.38 \\
%  % & \textsc{SentenceBERT+CodeAgent} & 0.65 & 0.93 & 0.41 \\
%   & \textsc{GraphTeam} &  &  &  \\
%  & \textsc{TFIDF+CodeAgent} &  &  &  \\
%  & \textsc{SentenceBERT+CodeAgent} &  &  &  \\
% \hline
% \multirow{2}{*}{(v)}
%  & \textsc{CodeGraph} & 0.80 & 0.73 & 0.31 \\
%  & \textsc{PIE} & 0.76 & 0.68 & 0.50 \\
% \hline
% \rowcolor{lightgray}
%  & \textsc{Ours} & \textbf{1.00} & \textbf{1.00} & \textbf{0.73} \\
% \hline
% \end{tabular}
% }
% \end{table}
% \chenglin{remove the None results, and evaluate again. For the tasks that are time-consuming, we evaluate 1-2 samples. Log the prompts and their effects.}
% \chenglin{Zero-Shot change to same graph task in complexgraph-L; Check the data type of the Complex Graph-S and -L. Either provide a reasonable explanation or tune the parameter.}

% \begin{table}[ht]
% \centering
% \caption{Comparative results on {\dataset-C} using various coding-agent LLMs, using the Accuracy metric. Best in each column is bolded. \fali{need to update the results to match the title.}}
% \label{tab:pass_rate_transposed}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Category} & \textbf{Method} & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7b} \\
% \midrule
% \bottomrule
% \end{tabular}
% }
% \end{table}
% \begin{table}[t]
% \centering
% \small 
% \caption{Comparative results on \texttt{\dataset-C} under the same experimental settings as Tab.~\ref{tab:main_results_S}.}
% \label{tab:main_results_C}
% \vskip -1em
% \resizebox{\linewidth}{!}{
% \renewcommand{\arraystretch}{1.1}  % 1.2–1.5 is typical
% \begin{tabular}{llccc}
% \hline
% \textbf{} & \textbf{Method} & \textbf{DeepSeek} & \textbf{LLaMA} & \textbf{Qwen} \\
% \hline
% \multirow{4}{*}{(i)}
%  & \textsc{Zero-shot} & 43.3 & 5.6 & 4.4 \\
%  & \textsc{Few-shot} & 43.3 & 4.4 & 2.2 \\
%  & \textsc{CoT} & 42.2 & 2.2 & 3.3 \\
%  & \textsc{Bag-CoT} & 55.6 & 0 & 11.1 \\
%  % llama 35.2, 30.9, 32.1, 35.6
% \hline
% \multirow{2}{*}{(ii)}
%  & \textsc{Zero-shot Coding} & 88.9 & 56.7 & 6.7 \\
%  & \textsc{Few-shot Coding} & 84.4 & 33.3 & 10.0 \\
% \hline
% \multirow{3}{*}{(iii)}
%  % & \textsc{GraphTool-Inst.} & 0.76 & 0.58 & 0.40 \\
%  % & \textsc{GCoder} &  &  &  \\
%  & \textsc{TFIDF+Coding} & 62.2 &  66.7 & 36.7 \\
%  & \textsc{SentBERT+Coding} & 51.1 & 66.7 & 55.6 \\
% \hline
% \multirow{3}{*}{(iv)}
%  & \textsc{GraphTeam} & 92.2 & 94.4 & 56.7 \\
%  & \textsc{TFIDF+CodeAgent} & 84.4 & 88.9 & 56.7 \\
%  & \textsc{SentBERT+CodeAgent} & 73.3 & 88.9 & 66.7 \\
% \hline
% \multirow{2}{*}{(v)}
%  & \textsc{CodeGraph} & 95.6 & 88.9 & 66.7 \\
%  & \textsc{PIE} & 100.0 & 85.6 & 57.8 \\
% \hline
% \rowcolor{lightgray}
%  \multirow{1}{*}{}& \textsc{Ours} & \textbf{95.6} & \textbf{96.7} &  73.3  \\
% \hline
% \end{tabular}
% }
% \end{table}

\begin{table}[t]
\centering
\small
\caption{Comparative results on \texttt{\dataset-C}.} %under the same statement as Tab.~\ref{tab:main_results_S}
\label{tab:main_results_C}
\vskip -1em
\resizebox{0.95\linewidth}{!}{
\renewcommand{\arraystretch}{1.1}
\begin{tabular}{llccc}
\hline
 & \textbf{Method} 
 & \textbf{DeepSeek} 
 & \textbf{LLaMA-70B} 
 & \textbf{Qwen-7B} \\
\hline
\multirow{4}{*}{(i)}
 & \textsc{Zero-shot}   & 43.3 & 5.6 & 4.4 \\
 & \textsc{Few-shot}    & 43.3 & 4.4 & 2.2 \\
 & \textsc{CoT}         & 42.2 & 2.2 & 3.3 \\
 & \textsc{Bag-CoT}     & 55.6 & 0.0 & 11.1 \\
\hline
\multirow{2}{*}{(ii)}
 & \textsc{Zero-shot Coding} & 64.4 & 56.7 & 6.7 \\
 & \textsc{Few-shot Coding}  & 78.9 & 51.1 & 10.0 \\
\hline
\multirow{2}{*}{(iii)}
 & \textsc{TFIDF+Coding}     & 73.3 & 66.7 & 36.7 \\
 & \textsc{SentBERT+Coding}  & 73.3 & 66.7 & 55.6 \\
\hline
\multirow{3}{*}{(iv)}
 & \textsc{GraphTeam}        & 76.7 & 70.0 & 55.6 \\
 & \textsc{TFIDF+CodeAgent}  & 84.4 & 88.9 & 56.7 \\
 & \textsc{SentBERT+CodeAgent} & 73.3 & 88.9 & 66.7 \\
\hline
\multirow{2}{*}{(v)}
 & \textsc{CodeGraph}        & 95.6 & 88.9 & 66.7 \\
 & \textsc{PIE}              & 100.0 & 92.2 &  70.0 \\
\hline
\rowcolor{lightgray}
 & \textsc{Ours}             & \textbf{95.6} & \textbf{96.7} & \textbf{73.3} \\
\hline
\end{tabular}
}
\vskip -1em
\end{table}


% \chenglin{Reasons for performance difference of GraphTeam in Table4 and Table 3: Because GraphTeam performs extremely badly in Composite tasks documentation retrieval (Table7), which, in this case, makes it generate worse code from irrelevant documentations}

% \vskip -2em
\subsection{Results and Analysis} 
% \fali{waiting for the latest results. plan: compare via model families, retrieval methods, other baselines. } 
% Tables~\ref{tab:main_results_S} and~\ref{tab:main_results_L} compare our method with multiple baselines across graphs of different scales and various LLM backbones. We have the following observations. Overall, our method consistently outperforms all baselines in most settings, including those relying on manually designed in-context learning, and achieves up to 100\% accuracy in some cases. These gains are mainly attributed to high-quality retrieval and effective code-level self-debugging. Compared with category (iv) methods, our approach improves average accuracy over GraphTeam by approximately 18 points and outperforms TF-IDF + Coding Agent and Sentence-BERT + Coding Agent, demonstrating the advantage of tree-structured retrieval. Relative to category (iii) methods, our method achieves an 22–55 point improvement, highlighting the benefit of debugging, while the larger gains over category (ii) methods further confirm the complementary effects of retrieval and debugging. Unlike category (i) text-based reasoning methods, which degrade on large graphs, our method maintains stable performance across both small and large graphs.
% Table~\ref{tab:main_results_C} further reports results on composite graph reasoning tasks, where our method continues to perform strongly. In contrast, most baselines, especially text-based methods and code-based methods without effective retrieval or debugging, have very poor performance in composite settings.
% Tables~\ref{tab:main_results_S} and~\ref{tab:main_results_L} compare our method with a diverse set of baselines across graphs of different scales and multiple large language model (LLM) backbones. Overall, our method consistently outperforms all baselines in most settings, including those relying on manually designed in-context learning strategies, and achieves {100\% accuracy} in several configurations. These improvements primarily stem from {high-quality document retrieval} and {effective code-level self-debugging}. In contrast to category (i) text-based reasoning methods, whose performance degrades substantially as graph size increases, our method maintains stable performance on both small and large graphs.
% For category (ii), {pure code-based approaches} already exhibit stable performance across graph scales, as they do not depend on the LLM context window size. This observation indicates that code-based reasoning paradigms are inherently more scalable than text-based reasoning methods.
% In category (iii), incorporating retrieval into code generation does not yield clear improvements over pure coding, particularly when using TF-IDF or Sentence-BERT for retrieval. We attribute this limitation to the fact that {generic retrieval strategies tend to introduce noisy or weakly relevant documents}, which dilute the benefits of retrieval augmentation.
% Category (iv) methods that combine retrieval with code-level debugging substantially outperform those in category (iii). For example, adding debugging to TF-IDF+Coding improves accuracy by approximately {15\%}, highlighting the effectiveness of {code debugging in correcting logical errors and improving execution reliability}.
% Category (v) represents human-in-the-loop approaches, which outperform most automatic baselines due to the availability of manually curated exemplars. However, such methods rely on human intervention and are therefore difficult to scale in fully automated settings.
% Compared to category (iv), our method further explicitly models the {hierarchical structure of technical documentation} and tightly integrates it with {self-debugging}, leading to consistent and significant performance gains across different configurations. This validates the effectiveness of our proposed design.
% Besides, Table~\ref{tab:main_results_C} reports results on {composite graph reasoning tasks}. Our method continues to perform strongly in these more challenging settings, whereas most baselines, particularly text-based methods and code-based methods lacking effective retrieval or debugging, exhibit severe performance degradation.
Tables~\ref{tab:main_results_S} and~\ref{tab:main_results_L} compare our method with a range of baselines on both \textit{small-scale} and \textit{large-scale} graph data from two datasets, evaluated across multiple coding-agent LLM backbones. Several key observations emerge.
(1) Our method achieves the best performance on both data scales, with average accuracies of $93.0\%$ (small-scale) and $97.2\%$ (large-scale). Among methods without human involvement, it significantly outperforms the strongest baseline, SentBERT+CodeAgent, by $5.7\%$ and $2.5\%$ on small- and large-scale graphs, respectively. Moreover, our method matches or exceeds human-involved approaches (CodeGraph and PIE) in most settings, demonstrating strong effectiveness for automated graph reasoning.
(2) Text-based reasoning baselines are consistently inferior to coding-based methods, with the gap especially pronounced on large-scale graphs. For example, Zero-shot text reasoning achieves only $33.7\%$ accuracy on large-scale datasets, compared to $71.1\%$ for Zero-shot coding. This gap highlights the limitations of pure text-based reasoning for large-scale and structurally complex graph tasks, stemming from the inherent difficulty LLMs face in long-horizon, precise structural reasoning, whereas coding-based approaches mitigate this limitation by reasoning at the algorithmic level and delegating complex execution to programs.
(3) Comparing categories (ii) and (iii), which differ only in the use of retrieval augmentation, we observe consistent performance gains from retrieval; for example, under the Zero-shot coding setting, incorporating TFIDF-based retrieval improves accuracy by $2.9\%$ on small-scale graphs and $2.2\%$ on large-scale graphs, indicating that retrieval provides more relevant contextual information for effective code generation.
(4) Further improvements arise when incorporating explicit debugging. Compared to category (iii), methods in category (iv) integrate self-debugging and achieve substantial gains; for instance, TFIDF+Coding improves from $74.0\%$ to $85.3\%$ on small-scale datasets and from $73.3\%$ to $85.9\%$ on large-scale datasets after applying debugging, confirming the effect of leveraging debugging feedback during coding.
(5) Within category (iv), GraphTeam underperforms TFIDF+CodeAgent and SentBERT+CodeAgent because it relies primarily on runtime checks (such as runtime and semantic errors) and lacks labeled test-case-based logical verification. This limitation results in performance gaps of up to $11.2\%$ on small-scale graphs and $13.6\%$ on large-scale graphs, underscoring the importance of logic-level debugging guided by test cases.


% (3) Comparing categories (ii) and (iii), which differ in the use of retrieval augmentation, we observe consistent gains from retrieval. Under the Zero-shot coding setting, TFIDF-based retrieval improves performance by $2.9\%$ on small-scale graphs and $2.2\%$ on large-scale graphs, indicating that retrieval provides relevant context for better code generation.

% Tab.~\ref{tab:main_results_C} reports results on \textit{composite} graph reasoning tasks, which introduce problem semantic complexity. The same trends hold similar as above, with our method achieving the best overall performance. Notably, small models struggle more on this kind of tasks: Qwen-2.5-7B reaches $73.3\%$, substantially below DeepSeek-V3.2 ($95.6\%$) and LLaMA-3.3-70B ($96.7\%$), indicating that composite graph reasoning demands stronger task understanding and cross-subtask reasoning capabilities, propose challenges for small language models for effective graph reasoning with complex problem.
Tab.~\ref{tab:main_results_C} reports results on \emph{composite} graph reasoning tasks, which introduce additional semantic complexity at the problem level. The overall trends remain consistent with previous results, with our method achieving the best performance. Notably, small models struggle more on this subset: Qwen-2.5-7B attains an accuracy of $73.3\%$, substantially lower than DeepSeek-V3.2 ($95.6\%$) and LLaMA-3.3-70B ($96.7\%$). This gap suggests that composite graph reasoning requires stronger semantic task understanding, where larger models demonstrate clear advantages, thereby challenging the effectiveness of small language models on complex graph reasoning tasks.





% \fali{writing is too verbose. Need to focus on the logic and Conciseness of Expression.} \xianren{Consider using: We have the following observations: \textbf{1)xx:}xxx \textbf{2)} This can make your points more clear}
% To understand the performance of out proposed method in graph task solving compare to code-based and vanilla retrieval-based code generation, we conduct baselines listed in Section\ref{sec: graph_task_solving_baselines} with dataset described in Section\ref{sec:benchmark_dataset}. 
% Table~\ref{tab:pass_rate_transposed} presents the pass rates of different large language models (LLMs) across multiple baseline settings and the proposed \text{\method} framework. The results consistently demonstrate that Agentic RAG achieves the highest overall performance across all tested models, indicating that hierarchical retrieval and iterative debugging substantially enhance the capability of LLMs to solve graph reasoning tasks. In particular, both \textit{DeepSeek} and \textit{LLaMA-70B} attain perfect pass rates (100\%) under the Agentic RAG configuration, significantly outperforming all other baselines. Even the smaller \textit{Qwen-7b} model shows a notable improvement.






% Retrieval-based baselines, such as TF–IDF and Sentence-BERT, provide modest gains over direct prompting but remain limited by their flat retrieval strategies, which lack hierarchical awareness and semantic validation. 

% Code-centric baselines like \textit{PSEUDO} and \textit{CodeGraph} perform competitively, achieving 0.91 and 0.90 respectively for DeepSeek, and 0.74 and 0.85 for LLaMA-70B. These results underscore the effectiveness of algorithmic prompting and code-centric reasoning. Nonetheless, Agentic RAG surpasses with manual selection of reference exmaples. For existing multi-agent baseline \textit{GraphTeam}, although the whole pipeline is fully automated, it still suffers from worse performance due to lower accuracy in related documentation retrieval for later code generation. 

% The consistently superior performance of Agentic RAG across all models confirms its generalisability, efficiency, and robustness in automating graph reasoning without the need for fine-tuning.


\subsection{Ablation Study}
% To evaluate the contribution of each component, we compare our method with variants that remove specific modules: w/o Test Cases, w/o Debugging, w/o Retrieval, and w/o Hierarchy (using TF-IDF filtering followed by agent-based selection). Tab.~\ref{tab:ablation_study} shows, removing retrieval reduces the average accuracy from $89\%$ to $71\%$, confirming the benefit of external knowledge. Disabling the hierarchical structure causes an even larger performance degradation due to increased retrieval noise from flat TF-IDF filtering, highlighting the importance of hierarchy. Removing the debugging module leads to the largest performance drop (from $89\%$ to $70\%$), while removing test cases also significantly degrades performance, underscoring their role in correcting logical errors.
% only -s
% Tab.~\ref{tab:ablation_study} reports ablation results on \texttt{\dataset-S} across different LLM backbones to assess the impact of each component within the {\method}. We compare the full {\method} with five ablated variants: \textsc{w/o Retrieval}, \textsc{w/o Hierarchy} (using TF-IDF filtering instead of hierarchical retrieval), \textsc{w/o Debugging}, \textsc{w/o Test Cases} (debugging without test-case assistance), and \textsc{w/o Deb + Ret}.  
% From the results, we observe that  
% (i) both retrieval and self-debugging are critical. Removing retrieval or debugging reduces average accuracy by $14.5\%$ and $5.5\%$, respectively, while removing both leads to a larger drop of $19.2\%$, indicating strong complementarity;  
% (ii) removing hierarchical traversal in retrieval alone causes a $9.1\%$ performance degradation, highlighting its effect in mitigating noisy retrieval;  
% (iii) excluding self-generated test cases in debugging results in an average $3.8\%$ accuracy drop, confirming their role in correcting code errors.  
% Notably, for larger models (DeepSeek and LLaMA-3-70B), which have few runtime code errors, removing debugging yields a performance drop equal to removing test cases, indicating that their debugging gains primarily stem from labeled test-case–based logical error correction, whereas smaller models (Qwen-2.5-7B) additionally benefit from runtime execution checks.
% -s and -c
Tab.~\ref{tab:ablation_study} reports ablation results on \texttt{\dataset} across different LLM backbones to assess the impact of each component within the {\method}. We compare the full {\method} with five ablated variants: \textsc{w/o Retrieval}, \textsc{w/o Hierarchy} (using TF-IDF filtering instead of hierarchical retrieval), \textsc{w/o Debugging}, \textsc{w/o Test Cases} (debugging without test-case assistance), and \textsc{w/o Deb + Ret}.  
From the results, we observe that  
(i) both retrieval and self-debugging are critical. Removing retrieval or debugging reduces average accuracy by $13.4\%$ and $5.0\%$, respectively, while removing both leads to a larger drop of $19\%$, indicating strong complementarity;  
(ii) removing hierarchical traversal in retrieval alone causes a $8.7\%$ performance degradation, highlighting its effect in mitigating noisy retrieval;  
(iii) excluding self-generated test cases in debugging results in an average $4.0\%$ accuracy drop, confirming their role in correcting code errors.  
Notably, for larger models (DeepSeek and LLaMA-3-70B), which have few runtime code errors, removing debugging yields a performance drop equal to removing test cases, indicating that their debugging gains primarily stem from labeled test-case–based logical error correction, whereas smaller models (Qwen-2.5-7B) additionally benefit from runtime execution checks.


% \begin{table}[t]
% \centering
% \caption{Ablation study on \texttt{\dataset-S}.}
% \vskip -1em
% \label{tab:ablation_study}
% \resizebox{0.85\linewidth}{!}{
% \begin{tabular}{lcccc}
% \hline
% \textbf{Setting} & \textbf{DeepSeek} & \textbf{LLaMA} & \textbf{Qwen} & \textbf{Avg.} \\
% \hline
% \textsc{Full}             & 96.4 & 96.4 & 91.8 & 94.9 \\
% \textsc{w/o Retrieval}    & 92.1 & 84.7 & 64.5 & 80.4 \\
% \textsc{w/o Hierarchy}    & 92.3 & 91.4 & 73.6 & 85.8 \\
% \textsc{w/o Debugging}    & 94.7 & 95.9 & 77.6 & 89.4 \\
% \textsc{w/o Test cases}   & 94.7 & 95.9 & 82.7 & 91.1 \\
% \textsc{w/o Ret + Deb}    & 86.3 & 82.7 & 58.2 & 75.7 \\
% \hline
% \end{tabular}
% }
% % \vskip -2em
% \end{table}

% \begin{table}[t]
% \centering
% \caption{Ablation study on \texttt{\dataset-C}.}
% \vskip -1em
% \label{tab:ablation_study_c}
% \resizebox{0.85\linewidth}{!}{
% \begin{tabular}{lccc}
% \hline
% \textbf{Setting} & \textbf{DeepSeek} & \textbf{LLaMA} & \textbf{Qwen} \\
% \hline
% \textsc{Full}          & 95.6 & 96.7 & 73.3 \\
% \textsc{w/o Retrieval} & 91.6 & 78.9 & 57.8 \\
% \textsc{w/o Hierarchy} & 92.6 & 84.4 & 63.3 \\
% \textsc{w/o Debugging} & 93.6 & 96.0 & 62.2 \\
% \textsc{w/o Test cases}& 93.6 & 96.0 & 63.3 \\
% \textsc{w/o Ret + Deb} & 85.6 & 78.9 & 44.4 \\
% \hline
% \end{tabular}
% }
% \end{table}

\begin{table}[t]
\centering
\caption{Ablation study on \texttt{\dataset}.}
\vskip -1em
\label{tab:ablation_study}
\resizebox{0.77\linewidth}{!}{
\begin{tabular}{lcccc}
\hline
\textbf{Setting} & \textbf{DeepSeek} & \textbf{LLaMA} & \textbf{Qwen} & \textbf{Avg.} \\
\hline
\textsc{Full}             & 96.0 & 96.6 & 82.6 & 91.7 \\
\textsc{w/o Retrieval}    & 91.9 & 81.8 & 61.2 & 78.3 \\
\textsc{w/o Hierarchy}    & 92.5 & 87.9 & 68.5 & 83.0 \\
\textsc{w/o Debugging}    & 94.2 & 96.0 & 69.9 & 86.7 \\
\textsc{w/o Test cases}   & 94.2 & 96.0 & 73.0 & 87.7 \\
\textsc{w/o Ret + Deb}    & 86.0 & 80.8 & 51.3 & 72.7 \\
\hline
\end{tabular}
}
\vskip 0.5em
{\footnotesize 
\textit{Note:} All values are averaged over the small-scale and composite subsets.
}
\end{table}


\subsection{Retrieval Performance Analysis} 
% \fali{need retrieval recall\@1 and research time; need baselines: (1) TFIDF (2) SentenceBERT (3) LlamaIndex (4) agent travels on the flat text collection (5) Layer-wise search.}
% To evaluate retrieval performance, we manually annotate ground-truth documents for tasks in {\dataset} and provide various forms of task descriptions (see Appendix \ref{sec:appendix_prompt}) and compare methods using Recall (see Appendix \ref{sec:appendix_metric}).  As shown in Table \ref{tab:retrieval_results}, we have the following observations. \textbf{Baseline methods have limited performance:} lexical matching with TF-IDF performs poorly (0.20), while Sentence-BERT improves retrieval via semantic embeddings (0.40) but remains limited in aligning algorithmic intent with APIs. GraphTeam further raises performance to 0.74 using LLM-based LlamaIndex. \textbf{Hierarchical traversal is more effective:} In contrast, our {\method} achieves the best Hit@1 score of 0.91, demonstrating that LLM-guided hierarchical traversal and adaptive selection substantially improve retrieval accuracy.
% We evaluate retrieval performance using Recall@1 with a single returned document and per-task retrieval time, based on manually annotated ground-truth documents and five task-description variants. Results in Tab.~\ref{tab:retrieval_results} show that TF-IDF performs poorly (0.20), Sentence-BERT improves retrieval via semantic embeddings (0.40) but remains limited in aligning algorithmic intent with documentation, and GraphTeam further improves performance to 0.74 using LLM-based LlamaIndex. Our method achieves the highest Recall@1 of 0.88, while removing document hierarchy leads to a substantial performance drop, showing the effectiveness of agentic hierarchical traversal. Despite higher latency, a 76-second retrieval time per task is acceptable for high-quality and reusable code generation.
Fig.~\ref{fig:retrieval_results} and \ref{fig:retrieval_results_c} compare our retrieval method with retrieval baselines on \texttt{\dataset}, evaluating Recall, Precision, F1, and average retrieval time per graph problem (see Appendix~\ref{sec:appendix_metric} for metrics).  
We observe that (i) our method consistently outperforms all baselines in retrieval quality, with especially large gains in precision, improving from the strongest vector-based baseline $28.8$ to $70.2$, while maintaining high recall. This demonstrates our method’s effectiveness in reducing noisy retrieval results without sacrificing coverage.  
(ii) These improvements come with increased retrieval cost: vector-based methods complete in under $1$s on average, whereas our method requires approximately $9$s, reflecting a clear performance--efficiency trade-off. We argue that this overhead is acceptable for code-based graph reasoning, where a single generated program can be reused across many graph instances.  
% (iii) Compared with a flat retrieval agent, which uses TF-IDF to retrieve an initial set of documents and then examines them individually, our hierarchical retrieval improves retrieval performance by enabling agent-guided selection of salient directory branches, effectively reducing noisy candidates, followed by more precise document-level decisions. In addition, early branch pruning narrows the agent’s search space, leading to lower retrieval time.
(iii) Compared with a flat retrieval agent that retrieves an initial set of documents via TF-IDF and examines them, our hierarchical retrieval improves performance by enabling agent-guided branch selection to reduce noisy candidates, followed by precise document-level decisions; early branch pruning further narrows the search space and reduces retrieval time.

% \begin{table}[htbp]
% \centering
% \caption{Retrieval Performance Comparison in {\dataset-S}.}
% \label{tab:retrieval_results}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lcc}
% \toprule
% \textbf{Method} & \textbf{Hit@1} & \textbf{Search Time (s)} \\
% \midrule
% \textsc{TF-IDF} & 0.30 & 0.01 \\
% \textsc{Sentence-BERT} & 0.25 & 0.4 \\
% \textsc{GraphTeam} & 0.483 & 31.3 \\
% \textsc{GCoder} &  &  \\
% \textsc{GraphTool-Instruction} & \textbf{0.883} & 2.4 \\
% \textsc{Retrieval Agent (Flat)} & 0.575 & 1094.8 \\
% \textsc{Retrieval Agent (Hierarchical)} & \textbf{0.883} & 75.5 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

% \begin{table}[htbp]
% \centering
% \caption{Retrieval performance and time on \texttt{\dataset-S}.}
% \vskip -1em
% \label{tab:retrieval_results}
% \resizebox{0.8\linewidth}{!}{
% \begin{tabular}{lcc}
% \toprule
% \textbf{Method} & \textbf{F1} & \textbf{Time (s)} \\
% \midrule
% \textsc{TF-IDF} & 24.2 & 0.2 \\
% \textsc{Sentence-BERT} & 28.7 & 0.7 \\
% \textsc{GraphTeam} & 28.8 & 0.5 \\
% \textsc{Retrieval Agent (Flat)} & 56.4 & 23.3 \\
% \textsc{Retrieval Agent (Ours)} & \textbf{79.0} & 9.1  \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

% TF-IDF, Recall 68.2, Precision 22.7
% Sentence-BERT, Recall 86.4, precision 28.8
% GraphTeam, Recall 54.6, Precision 19.7
% Flat, Recall 54.2, Precision 58.3
% Ours, Recall 87.0, precision 70.2 

% original table 
% \begin{table}[htbp]
% \centering
% \caption{Retrieval performance and time on \texttt{\dataset-S}.}
% \vskip -1em
% \label{tab:retrieval_results}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccccc}
% \toprule
% \textbf{Method} & \textbf{Recall} & \textbf{Precision} & \textbf{F1} & \textbf{Time (s)} \\
% \midrule
% \textsc{TF-IDF} & 68.2 & 22.7 & 24.2 & \textbf{0.2} \\
% \textsc{Sentence-BERT} & 86.4 & 28.8 & 28.7 & 0.7 \\
% \textsc{LlamaIndex} & 54.6 & 19.7 & 28.8 & 0.5 \\
% \textsc{Retrieval Agent (Flat)} & 54.2 & 58.3 & 56.4 & 23.3 \\
% \textsc{Retrieval Agent (Ours)} & \textbf{87.0} & \textbf{70.2} & \textbf{79.0} & 9.1 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

\begin{figure}[t]
    \centering
    % \vskip -1em
    \includegraphics[width=0.9\linewidth]{figures/performance_vs_time_annotated.pdf}
    \vskip -1.5em
    \caption{Retrieval performance and time on small-scale subset.}
    % \vskip -1.5em
    \label{fig:retrieval_results}
\end{figure}



% \chenglin{Add precision for this table}

% micro-precision \dataset-S
%  tf-idf: 0.275
%  sentence-bert: 0.244
%  graph-team: 0.358
%  Retrieval Agent (Flat): 0.475
%  Retrieval Agent (Ours): 0.92


% \begin{table}[htbp]
% \centering
% \caption{Retrieval Performance Comparison on {\dataset-C}.}
% \label{tab:retrieval_results}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Method} & \textbf{Micro Recall} & \textbf{Macro Recall} & \textbf{Search Time (s)} \\
% \midrule
% \textsc{TF-IDF} & 0.682 & 0.685 & 0.01 \\
% \textsc{Sentence-BERT} & 0.5 & 0.5 & 0.4 \\
% \textsc{GraphTeam} & 0.182 & 0.185 & 35.2 \\
% \textsc{GCoder} &  &  &  \\
% \textsc{GraphTool-Instruction} & 0.545 & 0.574 & 2.3 \\
% \textsc{Retrieval Agent (Flat)} & 0.682 & 0.704 & 1142.8 \\
% \textsc{Retrieval Agent (Hierarchical)} & 0.909 & 0.907 & 76.9 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

% \begin{table}[htbp]
% \centering
% \caption{Retrieval Performance Comparison on {\dataset-C}. \fali{need precision}}
% \label{tab:retrieval_results}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Method} & \textbf{Micro Recall} & \textbf{Macro Recall} & \textbf{Search Time (s)} \\
% \midrule
% \textsc{TF-IDF} & 0.682 & 0.685 & $<$1 \\
% \textsc{Sentence-BERT} & 0.5 & 0.5 & $<$1 \\
% \textsc{GraphTeam} & 0.182 & 0.185 & 35 \\
% \textsc{GraphTool-Instruction} & 0.545 & 0.574 & 2 \\
% \textsc{Retrieval Agent (Flat)} & 0.682 & 0.704 & 1143 \\
% \textsc{Retrieval Agent (Ours)} & 0.909 & 0.907 & 77 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

% retrieval标准答案不完备，无法评估
% \begin{table}[htbp]
% \centering
% \caption{Retrieval Performance and Time Comparison on {\dataset-C}.}
% \label{tab:retrieval_results_c}
% \vskip -1em
% \resizebox{0.95\linewidth}{!}{
% \begin{tabular}{lccccc}
% \toprule
% \textbf{Method} & \textbf{Recall} & \textbf{Precision} & \textbf{F1} & \textbf{Time (s)} \\
% \midrule
% \textsc{TF-IDF} & 25.9 & 22.2 & 23.7 & \textbf{0.2} \\
% \textsc{Sentence-BERT} & 29.6 & 13.3 & 18.3 & 1.3 \\
% \textsc{LlamaIndex} & 3.7 & 2.5  & 2.2  & 0.4  \\
% \textsc{Retrieval Agent (Flat)} & 54.2 & 54.2 & 53.3 & 44.7 \\
% \textsc{Retrieval Agent (Ours)} & 87.0 & 70.2 & 75.2 & 65.8 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}



% micro precision \dataset-C: 
%     TF-IDF: 0.08
%     Sentence: 0.04
%     Graph-Team: 0.15
%     Flat-Agent: 0.35
%     Ours: 0.26

% micro recall \dataset-C: 
%     TF-IDF: 0.32
%     Sentence: 0.18
%     Graph-Team: 0.09
%     Flat-Agent: 0.27
%     Ours: 0.81

 


\subsection{Hyperparameter Analysis}
% \fali{What is the suitable hyperparameter for analysis? }
% \chenglin{ComplexGraph-C debugging times 1,2,3,4,5}
% \chenglin{line Graph with confidence interval}
Fig.~\ref{fig:hyper_correction} studies the effect of the self-debugging budget $T_{\max}$, which control the number of code updates guided by test-case feedback. We vary $T_{\max}$ in $\{0, 1,2,3,4,5\}$. The results show that:
(i) Increasing $T_{\max}$ from 0 to 1 yields a clear and consistent improvement in code generation quality for both large and small models, confirming the effectiveness of self-debugging.
(ii) The small model (Qwen-7B) improves steadily as $T_{\max}$ increases, from $85\%$ to $99\%$ when $T_{\max}$ grows from $1$ to $5$. In contrast, large models (DeepSeek-V3.2 and LLaMA-70B) benefit mainly from a single self-debugging step, with additional budget yielding negligible gains. This suggests that large models typically converge after one correction, whereas small models require multiple iterations to refine their code. 
We attribute this difference to variations in coding capability.
% We attribute this difference to inherent disparities in coding capability: larger models utilize limited feedback more effectively, while smaller models benefit from extended self-debugging iterations.
(iii) With sufficient self-debugging budget, small models can reach performance comparable to large models, consistent with prior findings on test-time scaling that increased inference-time compute can substantially improve model performance and, in some cases, enable small language models to match or even outperform larger models~\cite{brown2024large, wu2025inference, snell2025scaling, wang2025agenttts,wang2025generalizing}.
% (iii) With sufficient self-debugging budget, small models can achieve performance comparable to that of large models. This observation aligns with recent findings on test-time scaling, which show that allocating additional inference-time compute can substantially enhance model performance and, in some cases, enable small language models to match or even outperform larger models~\cite{brown2024large, wu2025inference, yue2025inference, snell2025scaling, wang2025agenttts}.

\begin{figure}[b]
    \centering
    \vskip -1.5em
    \includegraphics[width=0.8\linewidth]{figures/hyper_correction.pdf}
    \vskip -1em
    \caption{Sensitivity analysis of the hyper-parameter w.r.t. self-debugging budget $T_{\max}$ on \texttt{\dataset-L}. }
    \label{fig:hyper_correction}
    % \vskip -1.5em
\end{figure}


\subsection{Inference Cost}
% To show the inference cost benefits of our method, we compare it with baselines using flat semantic retrieval (TF-IDF+CodeAgent, GraphTeam) and with a flat agent-based variant ({\method}(Flat)) in Tab.~\ref{tab:inference_cost}. Results show that (i) our approach incurs lower inference cost than methods with flat semantic retrieval because it retrieves fewer and more precise documents, thereby reducing coding cost; and (ii) our method further reduces inference cost compared to the flat agent-based variant, mainly on retrieval cost, as hierarchical search with early pruning significantly lowers search space.
Tab.~\ref{tab:inference_cost} compares the per-task inference cost of our method with traditional vector-retrieval-based approaches (TF-IDF and Sentence-BERT) combined with coding, as well as \textsc{GraphTeam}. We decompose the total inference cost into retrieval cost and coding cost. The results show that:
(i) TFIDF- and Sentence-BERT--based methods incur negligible retrieval cost but exhibit the highest code generation cost (approximately \$\(9\times10^{-4}\) per task), as they require a large number of documents to ensure sufficient coverage, substantially increasing the input cost during coding.
(ii) \textsc{GraphTeam} adopts higher-quality vector retrieval via LlamaIndex \cite{Liu_LlamaIndex_2022} with the OpenAI \texttt{text-embedding-ada-002} model \cite{textembeddingada}, which reduces the code generation cost (approximately \$\(5\times10^{-4}\)). However, the overhead from index construction incurs a substantial upfront cost.
(iii) Our method achieves the lowest total inference cost (around \$\(2\times10^{-4}\)). Although it incurs a non-negligible retrieval cost, the precise selection of task-relevant documents significantly lowers the code generation cost, effectively amortizing the retrieval overhead.



\begin{table}[t]
\centering
% \caption{Spilits of Inference Cost per Task on {\dataset-S} Using DeepSeek-V3.}
% \caption{Inference Cost Breakdown per Task on {\dataset-C} Using DeepSeek-V3.}
% \caption{Per-Task Inference Cost Breakdown on {\dataset-C} with DeepSeek-V3.}
\caption{Breakdown of Inference Costs per Task (DeepSeek-V3).} % on {\dataset-C} with DeepSeek-V3
\label{tab:inference_cost}
\vskip -1.1em
\resizebox{0.8\linewidth}{!}{
\begin{tabular}{l@{\hskip 15pt}r@{\hskip 15pt}r@{\hskip 15pt}r}
\toprule
\textbf{Method} 
& \textbf{Retrieval} 
& \textbf{Coding} 
& \textbf{Total} \\
\midrule
\textsc{TFIDF+Coding}    
& $\sim$0    
& 9     
& 9     \\
\textsc{SentBERT+Coding} 
& $\sim$0  
& 9     
& 9     \\
\textsc{GraphTeam}       
& 800$^{\dagger}$ / 0.1 
& 5     
& ${\sim}$5 \\
% \textsc{Ours (Flat)}     
% & 20    
% & 2     
% & ${\sim}$22  \\
\textsc{Ours}            
& 2     
& 2     
& ${\sim}$4   \\
\bottomrule
\end{tabular}
}
\vskip 0.5em
{\footnotesize 
\textit{Note:} All costs in units of $10^{-4}$ USD.
$^{\dagger}$Index building cost (one-time, amortized).
}
\vskip -1em
\end{table}




% \chenglin{TF-IDF Coding Cost Top-10 documentation}




\begin{figure}[h]
    \centering
    \vskip -1em
    \includegraphics[width=\linewidth]{figures/case_study.pdf}
    \vskip -1em
    \caption{Case Study: Retrieval and Debugging.}
    % \vskip -2em
    \label{fig:case}
\end{figure}


\subsection{Case Study}
% \fali{case 1: retrieval; case 2: syntax error; case 3: logic error.}
% \fali{retrieval case: graph\_tutor\_retrieve, and, tf\_idf\_retrieve, highlight our method have more accurate and less noisy retrieval}
% \fali{syntax error case: initial\_fail\_logic\_error, highlight the ability of our method in converting syntax errors into correct. }
% \fali{logic error case: debugged\_success, highlight the ability of our method in converting logic errors into correct.}
% We present results of our method and baselines when performing Endpoint-Aware-Flow task in Figure~\ref{fig:case}.
% For case 1, both clustering and maximum-flow are required to retrieve from NetworkX. However, TF-IDF only retrieves maximum-flow and other noisy documents. In contrast, our method can retrieve both required documents accurately with less noisy retrieval.
% For case 2, compared to zero-shot coding which triggers a syntax error, our method can generate codes without syntax errors.
% For case 3, a logic error occurs when no logic debugging is employed. However, our method can solve the error and output accurate answer using logic debugging.
% Fig.~\ref{fig:case} shows two cases of our method under the \texttt{Endpoint-aware Flow} task using LLaMA3-70B. In Case 1, TF-IDF retrieves only one relevant document and many noisy ones, while our method accurately retrieves all required documents without noise. In Case 2, our self-debugging mechanism corrects both syntactic and logical errors in the generated code, yielding a functionally correct code.
Fig.~\ref{fig:case} presents two cases on the \texttt{Endpoint-aware Flow} task with LLaMA-3-70B. In Case 1, TF-IDF returns only one relevant document alongside substantial noise, whereas our method retrieves all required documents with no irrelevant entries. In Case 2, our self-debugging mechanism fixes both runtime and logical errors in the generated code, producing a functionally correct implementation.


% \begin{table}[htbp]
% \centering
% \caption{Inference Cost (\$) Per Task on {\dataset-S} Using DeepSeek-V3.}
% \label{tab:inference_cost}
% \vskip -1em
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Method} 
% & \textbf{\makecell{Retrieval}} 
% & \textbf{\makecell{Coding}} 
% & \textbf{Total} \\
% \midrule
% \textsc{TFIDF+Coding} & 0 & 9e-4 & 9e-4 \\ %  & 4.2e-4
% \textsc{SentBERT+Coding} & 0 & 4e-4 & 4e-4 \\ %  & 4.2e-4
% \textsc{GraphTeam} & 8e-2/1e-5 & 5e-4 & $\sim$8e-2 \\
% \textsc{Ours (Flat)} & 2e-3 & 1e-4 & $\sim$2e-3 \\
% \textsc{Ours} & 2e-4 & 1e-4 & $\sim$3e-4 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}
% \begin{table}[htbp]
% \centering
% \caption{Inference Cost per Task on {\dataset-S} Using DeepSeek-V3 (Unit: $\times 10^{-4}$ USD).}
% \label{tab:inference_cost}
% \vskip 0.5em
% \resizebox{\linewidth}{!}{
% \begin{tabular}{l c c c}
% \toprule
% \textbf{Method} 
% & \textbf{Retrieval} 
% & \textbf{Coding} 
% & \textbf{Total} \\
% \midrule
% \textsc{TFIDF+Coding}    
% & --    
% & 9     
% & 9     \\

% \textsc{SentBERT+Coding} 
% & --    
% & 4     
% & 4     \\

% \textsc{GraphTeam}       
% & 800$^\dagger$ / 0.1 
% & 5     
% & $\sim$800 \\

% \textsc{Ours (Flat)}     
% & 20    
% & 1     
% & $\sim$20  \\

% \textsc{Ours}            
% & 2     
% & 1     
% & $\sim$3   \\
% \bottomrule
% \end{tabular}
% }
% \vskip 0.3em
% \footnotesize
% $\dagger$ means index building cost. 
% \end{table}


% \subsection{Retrieval Accuracy (RQ2)}


% To answer the comparison of retrieval capability between our method and existing retrieval approaches, we conduct the measurement of retrieval accuracy using the Hit@1 metric across multiple baselines on our task description variants dataset. Table~\ref{tab:retrieval_results} presents the retrieval accuracy (Hit@1) of different baselines compared to the proposed \textbf{Agentic-RAG} framework. Traditional lexical matching methods such as TF-IDF achieve limited performance (0.20), reflecting their inability to capture semantic similarity between task descriptions and documentation content. Sentence-BERT improves retrieval quality (0.40) by leveraging contextual embeddings, yet still struggles to precisely align algorithmic intents with relevant API functions. GraphTeam, which incorporates multi-agent coordination for retrieval and reasoning, further enhances accuracy to 0.74, demonstrating the benefit of structured collaboration among LLMs. In contrast, the proposed \textbf{Agentic-RAG} achieves the highest Hit@1 score of 0.91, showing that its LLM-guided hierarchical traversal and adaptive top-$k$ selection substantially improve retrieval precision by dynamically identifying the most semantically relevant documentation pages.
% \subsection{Ablation Studies (RQ3)}

% To further understand the contribution of each component within the Agentic RAG framework, we conduct an ablation study by progressively disabling key modules and observing the change in pass rate. As shown in Table \ref{tab:ablation_pass_rate}, removing any individual component leads to a noticeable degradation in performance across models. The full version of Agentic RAG achieves the highest pass rate for both DeepSeek and Llama-70B (1.0), demonstrating the effectiveness of the integrated retrieval, debugging, and test-case-guided generation pipeline. Excluding test cases or the retrieval agent causes a moderate decline in accuracy, indicating their importance in providing contextual grounding and relevant documentation. The removal of the debugging module results in the most significant drop, highlighting the necessity of iterative error correction for functional correctness. Disabling the hierarchical structure of the documentation breaks the pipeline, as flattening and feeding the entire documentation into the LLM exceeds the model's token context window. For the smaller Qwen-7b, performance remains lower overall, and certain settings are still under evaluation. These results collectively confirm that each component of the Agentic RAG pipeline contributes to achieving robust and executable solutions for graph-task code generation.



% \subsection{Context Level Impact (RQ4)}

% We conduct this experiment to evaluate how different levels of contextual guidance, ranging from no example, to provided documentation, affect the performance of LLMs on graph reasoning tasks. This helps isolate the impact of prompt design on task-solving ability without fine-tuning. The results in Table~\ref{tab:context_level_pass_rates} show that providing a relevant documentation example significantly boosts performance across all models, confirming that LLMs benefit from concrete related documentation examples.
% \begin{table}[htbp]
% \centering
% \caption{Comparison of Pass Rates Across Prompt Settings and LLMs}
% \label{tab:context_level_pass_rates}
% \renewcommand{\arraystretch}{1.4}
% \resizebox{\linewidth}{!}{\begin{tabular}{|l|c|c|c|}
% \hline
% \textbf{Prompt Setting} & \textbf{DeepSeek} & \textbf{LLaMA-70B} & \textbf{Qwen-7b} \\
% \hline
% No Example & 0.48 & 0.47 & 0.46 \\
% Use NX (No Example) & 0.50 & 0.45 & 0.49 \\
% NX Usage Example Provided & \textbf{0.76} & \textbf{0.79} & \textbf{0.55} \\
% \hline
% \end{tabular}
% }
% \end{table}


\section{Conclusion}
In this study, we identify two core limitations of existing code-based graph reasoning: flat retrieval that is insufficient for handling complex tasks and the lack of debugging mechanisms that address logic-level errors. To address these limitations, we propose {\method}, a documentation-guided framework that couples hierarchical, agentic retrieval with test-case-driven self-debugging code generation. We also introduce {\dataset}, a benchmark spanning small-scale, large-scale, and composite graph reasoning. Across \texttt{GTools} and {\dataset}, {\method} consistently improves retrieval quality and task accuracy without incurring much inference cost or latency, with the largest gains on composite tasks.



% \section{Acknowledgments}

% This section has a special environment:
% \begin{verbatim}
%   \begin{acks}
%   ...
%   \end{acks}
% \end{verbatim}
% so that the information contained therein can be more easily collected
% during the article metadata extraction phase, and to ensure
% consistency in the spelling of the section heading.

% Authors should not prepare this section as a numbered or unnumbered {\verb|\section|}; please use the ``{\verb|acks|}'' environment.

%%
%% The acknowledgments section is defined using the "acks" environment
%% (and NOT an unnumbered section). This ensures the proper
%% identification of the section in the article metadata, and the
%% consistent spelling of the heading.
% \begin{acks}
% \end{acks}

%%
%% The next two lines define the bibliography style to be used, and
%% the bibliography file.
\bibliographystyle{ACM-Reference-Format}
\bibliography{sample-base}


%%
%% If your work has an appendix, this is the place to put it.
\appendix

% \section{Research Methods}

% \subsection{Part One}

\label{sec:appendix}
\section{Appendix}
\label{sec:appendix}


% \begin{table}[htbp]
% \centering
% \caption{Debug Error and Pass Distribution on {\dataset-S} across Various LLMs.}
% \label{tab:debug_distribution}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Model} & \textbf{Syntax Errors} & \textbf{Logic Errors} & \textbf{Total} \\
% \midrule
% DeepSeek-V3 & 0 & 0.50 & 0.50 \\
% LLaMA3-70B & 0 & 0.69 & 0.69 \\
% Qwen-7b & 0.50 & 0.38 & 0.88 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

% \begin{table}[htbp]
% \centering
% \caption{Debug Error and Pass Distribution on {\dataset-C} across Various LLMs.}
% \label{tab:debug_distribution}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{lccc}
% \toprule
% \textbf{Model} & \textbf{Syntax Errors} & \textbf{Logic Errors} & \textbf{Pass} \\
% \midrule
% DeepSeek-V3   & 0.00 & 0.50 & 0.50 \\
% LLaMA3-70B    & 0.00 & 0.69 & 0.31 \\
% Qwen-7b  & 0.50 & 0.38 & 0.12 \\
% \bottomrule
% \end{tabular}
% }
% \end{table}



\subsection{Prompt Design} 
\label{sec:appendix_prompt}



\begin{promptbox}{Prompt for Relevance Judgment in Eq.\eqref{eq:select_relevant}}
\noindent
You will be provided with a description of a graph task and a dictionary of chapter names along with their descriptions.

\textbf{User Query}
\begin{verbatim}
{query}
\end{verbatim}

\textbf{Instructions}
\begin{enumerate}
    \item Determine the most appropriate \texttt{NetworkX} function required to solve the task.
    \item Based on the selected function, identify the relevant documentation chapters.
\end{enumerate}

\textbf{Available Chapters (key $\rightarrow$ description)}
\begin{verbatim}
{chapter_dict}
\end{verbatim}

\noindent
Choose {unique} items from \textbf{Available Chapters} that are essential for the given task.

\textbf{Constraints}
\begin{itemize}
    \item Keys {must match exactly} and are case-sensitive (use the dictionary keys as-is).
    \item If fewer than \texttt{\{top\_k\}} eligible keys exist, return as many as exist ($\geq 0$).
\end{itemize}

\textbf{Selection Guidelines (apply in order)}
\begin{enumerate}
    \item {Direct capability}: Prefer chapters whose descriptions indicate functions, algorithms, or APIs that can directly fulfill the query.
    \item {Specificity over breadth}: Prefer narrowly scoped chapters targeting the requested operation.
    \item {Disambiguation}: When multiple chapters are similar, choose the one most likely to lead to an immediately usable function.
\end{enumerate}

\textbf{Output Format (Strict)}
\begin{itemize}
    \item Return \textbf{only} a Python list literal using double quotes, e.g.,
\end{itemize}

\begin{verbatim}
["key1", "key2", "key3"]
\end{verbatim}

\begin{itemize}
    \item No explanations, no code fences, and no extra text.
    \item If nothing fits, return \texttt{[]}.
\end{itemize}

\paragraph{Internal Validation}
\begin{itemize}
    \item All returned items must be present in the {Available Chapters} keys.
    \item All items must be unique.
\end{itemize}
\end{promptbox}


\begin{promptbox}{Prompt for Global Filtering in Eq.\eqref{eq:global}}
Determine if the function is helpful for addressing the query. Think about the graph type and read carefully the User Query.

Return only one word: Yes or No. If uncertain, answer No. No explanations.

\end{promptbox}
\begin{promptbox}{Prompt for Test Case Generation in Eq.\eqref{eq:test_case_generation}}
{You are a test case generation agent.}

Generate test cases for evaluating Python programs that solve a graph algorithm task.

\textbf{Constraints:}
\begin{itemize}
  \item Do \emph{not} solve the task.
  \item Do \emph{not} include algorithms, reasoning, or hints.
  \item Use \emph{only} the task description and graph specification.
  \item All test cases must be valid and deterministic.
\end{itemize}

\textbf{Requirements:} Each test case must include:
\begin{enumerate}
  \item A concrete graph instance
  \item All required input parameters
  \item The expected output
\end{enumerate}

\textbf{Output format:}
\begin{itemize}
  \item Return a Python-compatible list of dictionaries with keys \texttt{"input"} and \texttt{"expected\_output"}.
  \item Do \emph{not} include explanations or extra text.
\end{itemize}

\textbf{Task description:} \texttt{\{task\_description\}}

\textbf{Graph specification:} \texttt{\{graph\_type\_description\}}

\end{promptbox}
\begin{promptbox}{Prompt for Code Generation in Eq.\eqref{eq:code_generation}}
\noindent
You are tasked with implementing a Python function to solve a graph problem.

\textbf{Relevant Documentation}
The following \texttt{NetworkX} documentation may be helpful for solving this problem:
\begin{verbatim}
{docs_section}
\end{verbatim}

\textbf{Task Description}
\begin{verbatim}
{question_text}
\end{verbatim}

\textbf{Graph Properties}
\begin{itemize}
    \item {Type}: \texttt{\{directed\_text\}}, \texttt{\{weighted\_text\}}
\end{itemize}

\textbf{Input}
\begin{itemize}
    \item \texttt{edge\_list}: A \texttt{\{weighted\_text\}} \texttt{\{directed\_text\}} graph represented as a list of edges.
    \begin{itemize}
        \item If weighted: each edge is \texttt{[source, target, weight]}, where \texttt{weight} is a floating-point value.
        \item If unweighted: each edge is \texttt{[source, target]}.
    \end{itemize}
    \item \texttt{\{args\_desc\}}
\end{itemize}

\textbf{Requirements}
\begin{enumerate}
    \item Implement a function that takes an edge-list representation of the graph.
    \item Convert the edge list into a \texttt{NetworkX} \texttt{\{nx\_graph\_class\}} object.
    \item Use \texttt{NetworkX} functions to solve the task.
    \item Return the result as specified in the task description.
    \item Return \texttt{None} if any error occurs during execution.
\end{enumerate}

\textbf{Important Notes}
\begin{itemize}
    \item Carefully read the provided \texttt{NetworkX} documentation.
    \item Pay close attention to parameter names, input types, and return values.
    \item Handle edge cases such as empty graphs and disconnected components.
    \item Use \texttt{try--except} blocks to catch errors and return \texttt{None} on failure.
\end{itemize}

\textbf{Test Case (for Validation)}
\noindent
\textbf{Input:}
\begin{verbatim}
{Edge List}
{Arguments}
\end{verbatim}

\noindent
\textbf{Expected Output:}
\texttt{\{Answer\}}

\medskip
\noindent
Test your function using the input above and ensure that it produces the expected output. Print the output to verify correctness.

\textbf{Your Task}
Generate a complete Python function that solves this problem. The implementation should include:
\begin{enumerate}
    \item Necessary imports (e.g., \texttt{import networkx as nx})
    \item A clear and well-defined function signature
    \item Conversion from edge list to a \texttt{NetworkX} graph
    \item Appropriate \texttt{NetworkX} operations to solve the task
    \item A return statement with the correct result type
    \item Robust error handling (returning \texttt{None} on failure)
\end{enumerate}

\noindent
Provide the implementation in a Python code block.



\end{promptbox}

\begin{promptbox}{Prompt for Code Refinement in Eq.\eqref{eq:code_refinement}}
\noindent
The following code failed to produce the correct output for a graph problem.

\textbf{Original Task:}

\texttt{\{original\_query\}}

\textbf{Failed Code}

\begin{verbatim}
{error_code}
\end{verbatim}

\textbf{}{Error or Incorrect Output}

\texttt{\{error\_output\}}

\textbf{Test Case}

\textbf{Input:}
\begin{verbatim}
{test_input_str}
\end{verbatim}

\noindent
\textbf{Expected Output:}
\texttt{\{Answer\}}

\textbf{Instructions}
Analyze the error and provide a corrected version of the code that:
\begin{enumerate}
    \item Fixes any syntax or runtime errors
    \item Produces the correct output matching the expected result
    \item Handles edge cases properly
    \item Returns \texttt{None} if any error occurs
    \item Tests the corrected implementation using the provided input and prints the output
\end{enumerate}

\noindent
Provide the corrected implementation in a Python code block.

\end{promptbox}

% \begin{promptbox}{Retrieval Variants using Clustering Coefficient as anExample}

% \textbf{Task Name (Formal)}  
% Consider a weighted, undirected graph $G=\{V, E\}$. The task is to compute the clustering coefficient for a given vertex $v$, which measures the degree to which nodes cluster together.

% \textbf{Task Name (Alternative)}  
% Given a weighted graph $G=\{V, E\}$, determine the clustering coefficient of a node $v$, reflecting how many of its neighbours are connected.

% \textbf{No Task Name (Implicit Intent)}  
% Given a weighted, undirected graph $G=\{V, E\}$, compute a coefficient for a node $v$ that quantifies how strongly its neighbours are connected.

% \textbf{Real-World (Context 1)}  
% Given a social network where nodes are people and edge weights capture interaction frequency, compute the local coefficient for person $v$ to measure how tightly their friends are connected.

% \textbf{Real-World (Context 2)}  
% Imagine an interbank lending network where nodes are banks and weights represent mutual exposures. Compute the local and overall coefficients to assess how often a bank’s counterparties are also connected (forming triangles).
% \end{promptbox}

\begin{promptbox}{\footnotesize Retrieval Prompt Variants (Clustering Coefficient Task)}

\textbf{Task Name (Formal)}  
Given a weighted, undirected graph $G=\{V, E\}$, compute the clustering coefficient of a vertex $v$, which measures the tendency of nodes to form clusters.

\textbf{Task Name (Alternative)}  
For a weighted graph $G=\{V, E\}$, determine the clustering coefficient of node $v$, indicating how many of its neighbors are mutually connected.

\textbf{No Task Name (Implicit Intent)}  
Given a weighted, undirected graph $G=\{V, E\}$, compute a coefficient for node $v$ that quantifies the connectivity among its neighbors.

\textbf{Real-World (Context 1)}  
In a social network where nodes represent people and edge weights denote interaction frequency, compute the local coefficient for person $v$ to measure how tightly their friends are connected.

\textbf{Real-World (Context 2)}  
In an interbank lending network where nodes represent banks and weights indicate exposures, compute local and global coefficients to assess how often a bank’s counterparties are interconnected.

\end{promptbox}



\begin{promptbox}{Prompt for Zero-shot Text-based Reasoning Baseline}
\noindent\texttt{\{task\_description\}}\\
The graph information \texttt{\{graph\_info\}}.\\
Please answer following \texttt{\{answer\_template\}} {WITHOUT any explanation}.\\
The output should be a \texttt{\{output\_type\}}.\\
You must return the correct answer, reasoning {CAREFULLY}.

\end{promptbox}
\begin{promptbox}{Prompt for Few-shot Text-based Reasoning Baseline}
\noindent\texttt{\{task\_description\}}\\
You can check the related info to help you reason about the answer:\\
\texttt{\{prompt\_hint\}}\\
The graph information \texttt{\{graph\_info\}}.\\
Please answer following \texttt{\{answer\_template\}} {WITHOUT any explanation}.\\
The output should be a \texttt{\{output\_type\}}.\\
You must return the correct answer, reasoning {CAREFULLY}.


\end{promptbox}

\begin{promptbox}{Prompt for Zero-shot Code-based Reasoning Baseline}
\noindent
Given the following task description:
\begin{verbatim}
{question_text}
\end{verbatim}

\noindent
Generate a Python function that solves this task for a \texttt{\{weighted\_text\}} \texttt{\{directed\_text\}} graph.

\textbf{Input}
\begin{itemize}
    \item \texttt{edge\_list}: A \texttt{\{weighted\_text\}} \texttt{\{directed\_text\}} graph represented as a list of edges.
    \begin{itemize}
        \item If weighted: each edge is \texttt{[source, target, weight]}, where \texttt{weight} is a floating-point value.
        \item If unweighted: each edge is \texttt{[source, target]}.
        \item \texttt{\{args\_desc\}}
    \end{itemize}
\end{itemize}

\textbf{Output}
\begin{itemize}
    \item Store the final answer in a variable named \texttt{result}.
    \item The result must match the expected return type for the given task.
\end{itemize}

\noindent
Provide the Python implementation below.
\end{promptbox}

\begin{promptbox}{Prompt for Few-shot Code-based Reasoning Baseline}

\noindent
Given the task description:
\begin{verbatim}
{question_text}
\end{verbatim}

\noindent
Generate a Python function that solves this task for a \texttt{\{weighted\_text\}} \texttt{\{directed\_text\}} graph.

\textbf{Input}
\begin{itemize}
    \item \texttt{edge\_list}: A \texttt{\{weighted\_text\}} \texttt{\{directed\_text\}} graph represented as a list of edges.
    \begin{itemize}
        \item If weighted: each edge is \texttt{[source, target, weight]}, where \texttt{weight} is a floating-point value.
        \item If unweighted: each edge is \texttt{[source, target]}.
        \item \texttt{\{args\_desc\}}
    \end{itemize}
\end{itemize}

\textbf{Output}
\begin{itemize}
    \item Store the final answer in a variable named \texttt{result}.
    \item The result must match the expected return type for the given task.
\end{itemize}

\textbf{Example}
\begin{verbatim}
{few_shot_examples}
\end{verbatim}

\noindent
Now, solve the task described above following a similar approach.

\textbf{Your Code:}


\end{promptbox}





% \begin{table*}[t]
% \centering
% \small
% \setlength{\tabcolsep}{5pt}
% \renewcommand{\arraystretch}{1.15}
% \caption{Variants of task descriptions used for retrieval robustness evaluation. Each graph task is expressed in five natural-language forms to assess both lexical and semantic generalisation.}
% \label{tab:retrieval_variants}
% \begin{tabular}{p{0.23\textwidth} p{0.70\textwidth}}
% \toprule
% \textbf{Variant Type} & \textbf{Example Description (Clustering Coefficient Task)} \\
% \midrule

% \textbf{Task Name (Formal)} &
% Consider a weighted, undirected graph $G=\{V,E\}$. The task is to compute the \textit{clustering coefficient} for a given vertex $v$, which measures the degree to which nodes cluster together. \\[3pt]

% \textbf{Task Name (Alternative)} &
% Given a weighted graph $G=\{V,E\}$, determine the clustering coefficient of a node $v$, reflecting how many of its neighbours are connected to one another. \\[3pt]

% \textbf{No Task Name (Implicit Intent)} &
% Given a weighted, undirected graph $G=\{V,E\}$, compute a coefficient for a node $v$ that quantifies how strongly its neighbours are connected to each other. \\[3pt]

% \textbf{Real-World (Context 1)} &
% Given a social network where nodes are people and edge weights capture interaction frequency, compute the local coefficient for person $v$ to measure how tightly their friends are connected. \\[3pt]

% \textbf{Real-World (Context 2)} &
% Imagine an interbank lending network where nodes are banks and weights represent mutual exposures. Compute the local and overall coefficients to assess how often a bank’s counterparties are also connected (forming triangles). \\
% \bottomrule
% \end{tabular}
% \end{table*}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% \begin{table*}[t]
% \centering
% \tiny 

% \caption{Average correct rate per task and model.}
% \label{tab:graph_tasks}
% % paste the tabular here
% \begin{tabular}{lrrrrrrrrrrrrrrrrrrrrrrrrr}
% \toprule
% Task & \multicolumn{2}{l}{clustering} & \multicolumn{2}{l}{common\_neighbors} & \multicolumn{2}{l}{connected\_components} & \multicolumn{2}{l}{diameter} &    hamiltonian\_path & \multicolumn{2}{l}{has\_eulerian\_path} &        is\_bipartite &          is\_connected &   is\_distance\_regular &            is\_regular & \multicolumn{2}{l}{max\_clique} &      maximum\_flow & maximum\_independent\_set & \multicolumn{2}{l}{min\_weighted\_vertex\_cover} & shortest\_path\_length & strongly\_connected\_components &      topological\_sort & traveling\_salesman\_problem \\
% Graph\_Config & undirected\_unweighted & undirected\_weighted & undirected\_unweighted & undirected\_weighted & undirected\_unweighted & undirected\_weighted & undirected\_unweighted & undirected\_weighted & directed\_unweighted & directed\_unweighted & undirected\_unweighted & directed\_unweighted & undirected\_unweighted & undirected\_unweighted & undirected\_unweighted & undirected\_unweighted & undirected\_weighted & directed\_weighted &   undirected\_unweighted &       directed\_unweighted & undirected\_unweighted &  undirected\_weighted &           directed\_unweighted & undirected\_unweighted &        undirected\_weighted \\
% Category\_Model                                     &                       &                     &                       &                     &                       &                     &                       &                     &                     &                     &                       &                     &                       &                       &                       &                       &                     &                   &                         &                           &                       &                      &                               &                       &                            \\
% \midrule
% agentic\_RAG\_code(Llama-3.3-70B-Instruct)           &                 0.400 &               0.562 &                   NaN &               1.000 &                   NaN &               0.000 &                 1.000 &               0.312 &               1.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &               0.000 &             1.000 &                     NaN &                     0.000 &                   NaN &                1.000 &                         1.000 &                 0.000 &                      0.000 \\
% agentic\_RAG\_code(Qwen-7b-Instruct)            &                 0.000 &               0.000 &                   NaN &               0.000 &                   NaN &               0.000 &                 1.000 &               0.312 &               0.909 &               0.273 &                 1.000 &               0.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &               0.000 &             1.000 &                     NaN &                     0.000 &                   NaN &                0.000 &                         0.000 &                 0.000 &                      0.000 \\
% agentic\_RAG\_code(deepseek-chat)                    &                 1.000 &               0.562 &                   NaN &               0.000 &                   NaN &               0.000 &                 1.000 &               0.312 &               1.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &               0.000 &             1.000 &                     NaN &                     1.000 &                   NaN &                0.000 &                         1.000 &                 0.000 &                      0.000 \\
% agentic\_RAG\_debug\_code(Llama-3.3-70B-Instruct)     &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.933 &               1.000 &               1.000 &               0.818 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                   NaN &                        NaN \\
% agentic\_RAG\_debug\_code(Qwen-7b-Instruct)      &                 0.000 &               0.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.933 &               1.000 &               0.091 &               0.000 &                 1.000 &               1.000 &                 1.000 &                 0.867 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                0.000 &                         1.000 &                   NaN &                        NaN \\
% agentic\_RAG\_debug\_code(deepseek-chat)              &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.933 &               1.000 &               0.727 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                   NaN &                        NaN \\
% agentic\_RAG\_debug\_no\_test\_case\_code(Llama-3.3-70B) &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.933 &               0.312 &               1.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 0.333 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                   NaN &                        NaN \\
% agentic\_RAG\_debug\_no\_test\_case\_code(deepseek-chat) &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.000 &               1.000 &               0.727 &               0.818 &                 1.000 &               1.000 &                 0.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                   NaN &                        NaN \\
% code\_only(Llama-3.3-70B-Instruct)                  &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.587 &               0.009 &               0.773 &               0.962 &                 1.000 &               0.956 &                 1.000 &                 0.936 &                 1.000 &                 0.000 &               0.000 &             0.000 &                   0.000 &                     0.218 &                 0.038 &                0.000 &                         0.000 &                 0.000 &                      0.000 \\
% code\_only(Qwen-7b-Instruct)                   &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.587 &               0.688 &               0.756 &               0.795 &                 0.938 &               0.987 &                 0.595 &                 0.000 &                 1.000 &                 0.000 &               0.000 &             0.000 &                   0.000 &                     0.000 &                 0.000 &                0.000 &                         1.000 &                 0.000 &                      0.000 \\
% code\_only(deepseek-chat)                           &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.587 &               0.009 &               0.756 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 0.945 &                 1.000 &                 0.118 &               0.000 &             0.000 &                   0.000 &                     0.005 &                 0.109 &                0.000 &                         0.000 &                 0.000 &                      0.000 \\
% code\_only\_w\_debug(Llama-3.3-70B-Instruct)          &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.600 &               0.688 &               0.727 &               0.909 &                 1.000 &               0.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         0.000 &                   NaN &                        NaN \\
% code\_only\_w\_debug(deepseek-chat)                   &                 1.000 &               0.562 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.533 &               0.688 &               0.727 &               0.818 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         0.000 &                   NaN &                        NaN \\
% codegraph\_code(Llama-3.3-70B-Instruct)             &                 1.000 &               0.562 &                   NaN &               1.000 &                   NaN &                 NaN &                 1.000 &               1.000 &               1.000 &               1.000 &                 1.000 &               0.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                 0.000 &                        NaN \\
% codegraph\_code(Qwen-7b-Instruct)              &                 1.000 &               0.562 &                   NaN &               1.000 &                   NaN &                 NaN &                 1.000 &               0.312 &               0.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 0.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                0.000 &                         1.000 &                 0.000 &                        NaN \\
% codegraph\_code(deepseek-chat)                      &                 1.000 &               0.562 &                   NaN &               1.000 &                   NaN &                 NaN &                 1.000 &               1.000 &               1.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                 0.000 &                        NaN \\
% graph\_team\_code(Llama-3.3-70B-Instruct)            &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.600 &               0.688 &               0.727 &               0.818 &                 0.933 &               1.000 &                 1.000 &                 0.600 &                 1.000 &                   NaN &                 NaN &             0.000 &                     NaN &                       NaN &                   NaN &                0.812 &                         1.000 &                   NaN &                        NaN \\
% graph\_team\_code(Qwen-7b-Instruct)             &                 0.000 &               0.000 &                   NaN &               0.000 &                   NaN &                 NaN &                 0.067 &               0.000 &               0.000 &               0.727 &                 0.933 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             0.000 &                     NaN &                       NaN &                   NaN &                0.562 &                         1.000 &                   NaN &                        NaN \\
% graph\_team\_code(deepseek-chat)                     &                 1.000 &               1.000 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.533 &               0.688 &               0.455 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 0.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                   NaN &                        NaN \\
% nx\_prompt(Llama-3.3-70B-Instruct)                  &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.587 &               0.000 &               0.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 0.998 &                 1.000 &                 0.000 &               0.000 &             0.000 &                   0.000 &                     0.000 &                 0.000 &                0.000 &                         0.000 &                 0.000 &                      0.000 \\
% nx\_prompt(Qwen-7b-Instruct)                   &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.587 &               0.688 &               0.000 &               0.000 &                 1.000 &               1.000 &                 1.000 &                 0.858 &                 1.000 &                 0.218 &               0.000 &             0.000 &                   0.000 &                     0.000 &                 0.000 &                0.000 &                         1.000 &                 0.000 &                      0.000 \\
% nx\_prompt(deepseek-chat)                           &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.587 &               0.000 &               0.100 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 0.998 &                 1.000 &                 0.218 &               0.000 &             0.000 &                   0.000 &                     0.235 &                 0.861 &                0.000 &                         1.000 &                 0.000 &                      0.000 \\
% provided\_example(Llama-3.3-70B-Instruct)           &                 0.389 &               0.564 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.920 &               1.000 &               1.000 &               0.000 &                 0.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                 0.000 &               0.000 &             1.000 &                   1.000 &                     0.000 &                 0.000 &                1.000 &                         1.000 &                 0.000 &                      0.000 \\
% provided\_example(Qwen-7b-Instruct)            &                 0.000 &               0.000 &                 1.000 &               1.000 &                 0.000 &               0.000 &                 0.920 &               1.000 &               0.909 &               0.795 &                 0.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                 0.996 &               0.000 &             0.665 &                   0.000 &                     0.000 &                 0.000 &                0.000 &                         1.000 &                 0.000 &                      0.000 \\
% provided\_example(deepseek-chat)                    &                 1.000 &               0.576 &                 1.000 &               0.000 &                 0.000 &               0.000 &                 0.572 &               0.312 &               1.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                 0.996 &               0.000 &             0.964 &                   1.000 &                     1.000 &                 1.000 &                0.000 &                         1.000 &                 0.000 &                      0.053 \\
% pseudo\_code(Llama-3.3-70B-Instruct)                &                 1.000 &               0.562 &                   NaN &                 NaN &                   NaN &                 NaN &                 0.667 &               0.312 &               0.000 &               1.000 &                 0.000 &               0.818 &                 1.000 &                 0.750 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         0.000 &                 0.000 &                        NaN \\
% pseudo\_code(Qwen-7b-Instruct)                 &                 0.000 &               0.000 &                   NaN &                 NaN &                   NaN &                 NaN &                 0.333 &               0.312 &               0.000 &               0.000 &                 0.000 &               0.273 &                 0.062 &                 1.000 &                 0.938 &                   NaN &                 NaN &             0.818 &                     NaN &                       NaN &                   NaN &                0.000 &                         0.000 &                 0.000 &                        NaN \\
% pseudo\_code(deepseek-chat)                         &                 1.000 &               0.000 &                   NaN &                 NaN &                   NaN &                 NaN &                 1.000 &               0.312 &               0.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                 0.000 &                        NaN \\
% pseudo\_w\_debug\_code(Llama-3.3-70B-Instruct)        &                 1.000 &               0.562 &                   NaN &               1.000 &                   NaN &                 NaN &                 0.533 &               0.688 &               0.727 &               1.000 &                 0.533 &               1.000 &                 1.000 &                 0.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         0.000 &                   NaN &                        NaN \\
% pseudo\_w\_debug\_code(Qwen-7b-Instruct)         &                 0.000 &               0.000 &                   NaN &               0.000 &                   NaN &                 NaN &                 0.333 &               0.312 &               0.091 &               0.000 &                 0.000 &               0.273 &                 0.000 &                 0.000 &                 0.000 &                   NaN &                 NaN &             0.000 &                     NaN &                       NaN &                   NaN &                0.000 &                         0.000 &                   NaN &                        NaN \\
% pseudo\_w\_debug\_code(deepseek-chat)                 &                 1.000 &               1.000 &                   NaN &               0.938 &                   NaN &                 NaN &                 0.533 &               1.000 &               0.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                   NaN &                 NaN &             1.000 &                     NaN &                       NaN &                   NaN &                1.000 &                         1.000 &                   NaN &                        NaN \\
% sentence\_bert\_code(Llama-3.3-70B-Instruct)         &                 0.000 &               0.000 &                 0.000 &               1.000 &                 0.000 &               0.000 &                 0.920 &               0.321 &               1.000 &               1.000 &                 1.000 &               1.000 &                 1.000 &                 0.902 &                 1.000 &                 0.999 &               0.000 &             0.000 &                   1.000 &                     0.000 &                 0.000 &                0.000 &                         0.382 &                 0.000 &                      0.000 \\
% sentence\_bert\_code(Qwen-7b-Instruct)          &                 0.000 &               0.000 &                 0.000 &               0.000 &                 0.000 &               0.000 &                 0.920 &               1.000 &               0.909 &               0.885 &                 0.963 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                 0.997 &               0.000 &             0.000 &                   1.000 &                     0.000 &                 0.000 &                0.000 &                         0.382 &                 0.000 &                      0.000 \\
% sentence\_bert\_code(deepseek-chat)                  &                 0.000 &               0.000 &                 1.000 &               0.000 &                 0.000 &               0.688 &                 0.920 &               0.321 &               1.000 &               0.998 &                 1.000 &               1.000 &                 1.000 &                 1.000 &                 1.000 &                 0.997 &               0.000 &             0.000 &                   0.999 &                     1.000 &                 1.000 &                0.000 &                         0.382 &                 0.000 &                      0.000 \\
% tf\_idf\_code(Llama-3.3-70B-Instruct)                &                 0.000 &               0.000 &                   NaN &                 NaN &                   NaN &                 NaN &                 0.920 &               0.312 &               0.005 &               1.000 &                 1.000 &               0.727 &                 1.000 &                 0.902 &                 1.000 &                   NaN &                 NaN &             0.000 &                     NaN &                       NaN &                   NaN &                0.000 &                         0.000 &                 0.000 &                        NaN \\
% tf\_idf\_code(Qwen-7b-Instruct)                 &                 0.000 &               0.000 &                   NaN &                 NaN &                   NaN &                 NaN &                 0.333 &               0.312 &               0.909 &               0.885 &                 0.963 &               1.000 &                 0.495 &                 1.000 &                 0.764 &                   NaN &                 NaN &             0.000 &                     NaN &                       NaN &                   NaN &                0.000 &                         1.000 &                 0.000 &                        NaN \\
% tf\_idf\_code(deepseek-chat)                         &                 0.000 &               0.000 &                   NaN &                 NaN &                   NaN &                 NaN &                 0.920 &               0.321 &               0.096 &               1.000 &                 1.000 &               0.789 &                 0.839 &                 0.816 &                 1.000 &                   NaN &                 NaN &             0.000 &                     NaN &                       NaN &                   NaN &                0.000 &                         1.000 &                 0.000 &                        NaN \\
% \bottomrule
% \end{tabular}
% \end{table*}

% Preamble (if not already):
% \usepackage{booktabs}
% \usepackage{multirow}
% \usepackage{siunitx}
% \sisetup{detect-weight=true,detect-family=true}

% \begin{table}[t]
% \centering
% \caption{Pass rate on small vs. large graphs (mean $\pm$ std over $N$ runs). 
% We report the proportion of test cases solved correctly under different retrieval/tooling methods.
% Bold indicates the best setting per model and size split.}
% \label{tab:pass_rate_small_large}
% \resizebox{\linewidth}{!}{
% \begin{tabular}{
% l
% *{6}{S[table-format=1.2]}
% }
% \toprule
% \multirow{2}{*}{\textbf{Method}} 
% & \multicolumn{2}{c}{\textbf{DeepSeek}} 
% & \multicolumn{2}{c}{\textbf{LLaMA-70B}} 
% & \multicolumn{2}{c}{\textbf{Qwen-7b}} \\
% \cmidrule{2-3}\cmidrule{4-5}\cmidrule{6-7}
% & {\textbf{Small}} & {\textbf{Large}}
% & {\textbf{Small}} & {\textbf{Large}}
% & {\textbf{Small}} & {\textbf{Large}} \\
% \midrule
% \textbf{TF-IDF}            & 0.51 & 0.96 & 0.47 & 0.92 & 0.48 & 0.41 \\
% \textbf{Sentence-BERT}     & 0.56 & 0.92 & 0.55 & 0.96 & 0.60 & 0.63 \\
% \textbf{PIE}            & 0.91 & 0.80 & 0.74 & 0.76 & 0.45 & 0.26 \\
% \textbf{GraphTeam*}         & 0.84 & 0.88 & 0.83 & 0.65 & 0.47 & 0.58 \\
% \textbf{CodeGraph*}         & 0.90 & 0.93 & 0.85 & 0.80 & \textbf{0.66} & 0.72 \\
% \textbf{Ours}& \textbf{1.00} & \textbf{1.00} & \textbf{1.00} & \textbf{1.00} & \textbf{0.66} & \textbf{0.73} \\
% \bottomrule
% \end{tabular}
% }
% \end{table}

\subsection{Our Algorithm}
Our proposed algorithm is in Algo. \ref{alg:retrieval}.

\begin{algorithm}[t]
\caption{Layer-wise Agentic Retrieval}
\label{alg:retrieval}
\small 
\begin{algorithmic}[1]
% \REQUIRE Task $q$; document tree $\mathcal{T}=(\mathcal{V},\mathcal{E},\mathcal{S})$ with depth $L$
\REQUIRE Task $q$; document tree $\mathcal{T}$ with depth $L$, Retrieval Agent \texttt{R}
\ENSURE Task-relevant document subset $\mathcal{D}_q$

\STATE Initialize $\mathcal{D}_q \leftarrow \{v^{(0)}\}$

% \STATE \textbf{Def} \textsc{SelectRelevant}$(q, \mathcal{C})$
% \STATE \hspace{0.5cm} $\mathcal{D}' \leftarrow \emptyset$
%     \STATE \hspace{0.5cm} \textbf{for all} $v_i \in \mathcal{C}$ \textbf{do}
%         \STATE {\hspace{1cm} \textbf{If} \textsc{IsRelevant}$(q, s_i)$} \textbf{do}
%             \STATE \hspace{1.5cm} $\mathcal{D}' \leftarrow \mathcal{D}' \cup \{v_i\}$
%     \STATE \hspace{0.5cm} \textbf{return} $\mathcal{D}'$ 
% \STATE \textbf{End Function} \\ \hspace{2cm}

\FOR{$l = 0$ \TO $L-1$}
    % \STATE $\mathcal{D}_q' \leftarrow \emptyset$
    % \FORALL{$v_i^{(l)} \in \mathcal{D}_q$}
    %     \STATE $\mathcal{C} \leftarrow \{\, v_j^{(l+1)} \mid (v_i^{(l)}, v_j^{(l+1)}) \in \mathcal{E} \,\}$
    %     \STATE $\mathcal{D}_q' \leftarrow \mathcal{D}_q' \cup \textsc{SelectRelevant}(q, \mathcal{C})$ \hfill $\triangleright$ \text{\texttt{See Eq.\ref{x}.}}
    % \ENDFOR
    % \STATE $\mathcal{D}_q \leftarrow \mathcal{D}_q'$
    \STATE $\mathcal{C} \leftarrow \{\, v_j^{(l+1)} \mid v_i^{(l)} \in \mathcal{D}_q  ,\,  (v_i^{(l)}, v_j^{(l+1)}) \in \mathcal{E} \}$ %\hfill $\triangleright$ \text{\texttt{Candidate node set.}}
    \STATE $\mathcal{D}_q \leftarrow \texttt{R.select\_relevant}(q, \mathcal{C})$ \hfill $\triangleright$ \text{\texttt{See Eq.\eqref{eq:select_relevant}.}}
\ENDFOR

\STATE $\mathcal{D}_q \leftarrow \texttt{R.global\_filter}(q, \mathcal{D}_q)$
\RETURN $\mathcal{D}_q$
\end{algorithmic}
\end{algorithm}




\subsection{Our Proposed {\dataset} Dataset}
\label{sec:our_dataset}
\noindent\textbf{\texttt{\dataset-L(S)}: A Large(Small)-Scale Graph Reasoning Dataset.}
% Prior work has introduced a lot of graph reasoning datasets, but they are primarily in small-scale graphs. For instance, GraphQA \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} typically contain graphs with x to xx nodes, which can be accommodated by existing LLMs. As examples, Qwen-7b \cite{huang2025opencoder} with an $8$k context window can represent a complete graph with about 40 nodes, while DeepSeek-V3 \cite{liu2024deepseek} with a $160$k window can handle roughly $150$ nodes. Such settings, however, do not capture the challenges of large-scale graph reasoning, particularly under long-context constraints.
% Motivated by this limitation, we construct \textit{GraphCoding}, a synthetic dataset spanning a wide range of graph sizes from x to xx nodes. GraphCoding includes a small-scale subset, \textit{GraphCoding-S}, and a large-scale subset, \textit{GraphCoding-L}. Following \cite{wu-etal-2025-grapheval36k}, we select \fali{$14$} classical graph algorithm tasks (Table~\ref{tab:task_overview}, Appendix). Task descriptions are manually written by graph-domain experts, and graph instances are generated using NetworkX \cite{hagberg2008exploring}.
% The dataset covers diverse graph types, including directed/undirected, weighted/unweighted, cyclic/acyclic, connected/disconnected, and sparse/dense/complete graphs. For each task, only semantically appropriate graph types are generated (e.g., unweighted undirected graphs for connected component counting). For directed graphs, edge weights are sampled uniformly from $[0.0, 10.0]$, and for tasks requiring additional inputs (e.g., source--target pairs in shortest path), nodes are selected uniformly at random. Detailed statistics are provided in Tables~\ref{x}--\ref{x} in the appendix.
Prior work has proposed many graph reasoning datasets, but they primarily focus on \emph{small-scale graphs}. For example, GraphQA \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} typically involve graphs with $3$--$10$ nodes, which can be accommodated by existing LLMs. For example, Qwen-7B \cite{bai2023qwentechnicalreport} with an $8$k context window supports complete graphs of $\sim40$ nodes, and DeepSeek-V3 \cite{liu2024deepseek} with a $160$k window supports $\sim180$ nodes. Such settings do not capture the challenges of \emph{large-scale graph reasoning} under long-context abilities and window size limit.
To address this gap, we construct {\dataset}, a synthetic dataset spanning graph sizes from $3$ to $10$k nodes, including a small-scale subset \texttt{\dataset-S} (graphs with $3$--$200$ nodes) and a large-scale subset \texttt{\dataset-L} (graphs with $5$k and $10$k nodes). Following \cite{wu-etal-2025-grapheval36k}, we select {$14$} classical graph algorithm tasks (see Tables~\ref{tab:small_scale_task_overview}, \ref{tab:task_overview} in Appendix). Task descriptions are written by graph-domain experts manually, and graph instances are generated using NetworkX \cite{hagberg2008exploring}.
{\dataset} covers diverse graph types (directed or undirected, weighted or unweighted, cyclic or acyclic, connected or disconnected, and sparse or dense). For each task, we generate only semantically appropriate graph types; for instance, connected component counting is evaluated exclusively on unweighted undirected graphs, as the weights do not affect the result. Edge weights for directed graphs are sampled uniformly from $[0.0, 10.0]$, and task-specific inputs (e.g., source--target pairs) are selected uniformly at random. Labels are generated using human-crafted reference code. A complete example is provided at the end. 

\noindent\textbf{\texttt{\dataset-C}: A Composite Graph Reasoning Dataset.}
% Beside the large-scale data in single task, previous works ignore the complex graph reasoning tasks, so we also construct \texttt{\dataset-C} that xxx\chenglin{composite task }
% Existing datasets ignore \emph{complex graph reasoning tasks} at the problem level. Thus, we introduce the \texttt{\dataset-C} subset, which focuses on \emph{composite graph tasks} formed by combining multiple classical graph algorithms and requiring multi-step reasoning. For example, a task may first perform topological sorting to determine precedence constraints and then count paths between a source and a target within the constrained reachable subgraph.
% The construction of \texttt{\dataset-C} follows three steps: 
% (1) {Subtask selection}: randomly sample two or three primitive graph tasks from \texttt{\dataset-S}; 
% (2) {Task composition}: combine the sampled primitive tasks using one of three patterns: sequential (execute A then B), parallel (execute A and B and aggregate outputs), and conditional (execute A, then B or C based on a condition). Only logically and semantically valid combinations are retained. Task descriptions are generated using GPT-5 \cite{openai2025gpt5api}; the prompt and the details are in Appendix \ref{sec:appendix_details_composite}. 
% (3) {Graph Instance and label generation}: graph instances are reused from \texttt{\dataset-S}, and labels are obtained by executing human-crafted reference code. We construct $9$ composite graph tasks with detailed task descriptions and statistics in Tab.~\ref{tab:composite_tasks} in Appendix.
% 
% \subsection{Detailed Construction of Composite Dataset \texttt{\dataset-C}}
% \label{sec:appendix_details_composite}
Existing datasets do not provide support for \emph{complex graph reasoning tasks}. To address this gap, we introduce \texttt{\dataset-C}, a dataset dedicated to \emph{composite graph tasks}, in which multiple classical graph algorithms are composed into a single task that requires multi-step reasoning. For instance, a composite task may require first performing a topological sort to establish precedence constraints, and subsequently counting the number of paths from a source node to a target node within the reachable subgraph that satisfy these constraints.
The construction of \texttt{\dataset-C} consists of three stages.
(1) \textbf{Subtask selection}: We randomly sample two or three atomic classical graph algorithm tasks from \texttt{\dataset-S} as candidate subtasks.
(2) \textbf{Task composition and description generation}: The selected subtasks are combined according to a valid task logic. Specifically, given algorithms A and B (or A, B, C), we adopt one of three composition patterns: \emph{sequential composition}, where algorithm A is executed first and its output serves as an intermediate result for algorithm B; \emph{parallel composition}, where algorithms A and B are executed concurrently and their outputs are merged via an aggregation operator; and \emph{conditional composition}, where algorithm A is executed first and, depending on whether its output satisfies a predefined condition, either algorithm B or algorithm C is subsequently executed. Only compositions that are semantically and logically coherent are retained; otherwise, the corresponding task instances are discarded. Natural language descriptions of composite tasks are generated using GPT-5, with the prompt template provided below.
(3) \textbf{Graph instance and label generation}: Graph instances are directly reused from \texttt{\dataset-S}, while ground-truth labels are obtained by executing human-crafted reference implementations.
Detailed statistics of the composite tasks are reported in Table~\ref{tab:composite_tasks}. A complete example is provided at the end of the Appendix. Each composite task is described in detail below: 

\noindent\textbf{Sequential Composition Tasks: }

\begin{enumerate}[nosep,leftmargin=*]
\item \textbf{Clustering Coefficient \& Shortest Path.}  
Given an unweighted undirected graph $G$ and two nodes $s$ and $t$:  
(1) compute one shortest path $P$ from $s$ to $t$;  
(2) for each node $v \in P$, compute its local clustering coefficient;  
(3) identify the node on $P$ with the highest clustering coefficient, breaking ties by choosing the node closer to $s$ along the path.  
The output consists of the selected node together with the clustering coefficients of all nodes on the path.

\item \textbf{Strongly Connected Components \& Diameter.}  
Given an directed graph $G$:  
(1) compute all strongly connected components (SCCs) of $G$;  
(2) for each SCC, construct the induced subgraph and treat it as an undirected graph;  
(3) compute the diameter of each induced subgraph.  
The output is the list of diameters for all SCCs.

\item \textbf{Strongly Connected Components \& Maximum Flow.}  
Given a weighted directed graph $G$ and a fixed sink node $t$:  
(1) compute the strongly connected components of $G$;  
(2) for each SCC $C$, select a representative source node $s_C$ (e.g., the node with the smallest ID in $C$);  
(3) compute the maximum flow from $s_C$ to $t$;  
(4) compute the local clustering coefficient of $s_C$ in the undirected projection of $G$;  
(5) define a score $S_C = \mathrm{flow}(s_C,t)\cdot(1+\mathrm{clustering}(s_C))$.  
The output is the SCC with the highest score $S_C$.
\end{enumerate}

\noindent\textbf{Parallel Composition Tasks:}

\begin{enumerate}[nosep,leftmargin=*]
\setcounter{enumi}{3}
\item \textbf{Pair Tightness Score.}  
Given a weighted undirected graph $G$ and two nodes $s$ and $t$:  
(1) compute the weighted shortest-path distance $d(s,t)$;  
(2) compute the number of common neighbors of $s$ and $t$;  
(3) compute the local clustering coefficients $C(s)$ and $C(t)$.  
These quantities are combined into a tightness score
$T(s,t)=\frac{|N_{\mathrm{common}}|+\frac{C(s)+C(t)}{2}}{1+d(s,t)}.$
The output is the score $T(s,t)$.

\item \textbf{Shortest Path \& Clustering Coefficient (Bridge Hub Identification).}  
Given an undirected graph $G$ and an integer $k$:  
(1) compute the local clustering coefficient $C(v)$ for each node $v$;  
(2) compute the average shortest-path distance $\bar{d}(v)$ from $v$ to all other nodes;  
(3) define a bridge-hub score $S(v)=(1-C(v))/\bar{d}(v)$.  
The output is the top-$k$ nodes ranked by $S(v)$ together with their scores.

\item \textbf{Maximum Flow \& Clustering Coefficient.}  
Given a weighted directed graph $G$, a source node $s$, and a sink node $t$:  
(1) compute the maximum flow value $F$ from $s$ to $t$;  
(2) compute the local clustering coefficients $C(s)$ and $C(t)$ in the undirected projection of $G$.  
These results are combined into an endpoint-aware flow score
$S = F \cdot \left(1 + \frac{C(s)+C(t)}{2}\right).$
The output is the score $S$.
\end{enumerate}

\noindent\textbf{Conditional Composition Tasks}

\begin{enumerate}[nosep,leftmargin=*]
\setcounter{enumi}{6}
\item \textbf{Eulerian Path \& Diameter.}  
Given an unweighted undirected graph $G$ and two nodes $s$ and $t$:  
(1) check whether $G$ admits an Eulerian path;  
(2) if an Eulerian path exists, compute and return the diameter of $G$;  
(3) otherwise, compute the local clustering coefficients of $s$ and $t$ and return the larger one.

\item \textbf{Graph Connectivity \& Component Diameter.}  
Given an unweighted undirected graph $G$:  
(1) check whether $G$ is connected;  
(2) if $G$ is connected, compute and return its diameter;  
(3) otherwise, identify the largest connected component (by number of nodes) and compute the diameter of that component only.

\item \textbf{Strongly Connected Components \& Eulerian Path Check.}  
Given an unweighted directed graph $G$:  
(1) check whether $G$ has an Eulerian path;  
(2) if an Eulerian path exists, return the number of nodes in the largest strongly connected component;  
(3) otherwise, return the total number of SCCs.
\end{enumerate}

\begin{promptbox}{Prompt for Composite Task Generation}
You are given a set of graph algorithms.
Compose them into a single composite graph task following one of the allowed composition patterns (sequential, parallel, or conditional).

\textbf{Constraints}:

- Output only the final task description.
- Do not include explanations, algorithms, or solution steps.
- Do not reveal intermediate results or execution order.
- Ensure the task is logically consistent and self-contained.
- Clearly specify the required graph type and all necessary inputs.
- Define a single, well-formed output.
\end{promptbox}




% \begin{table*}[ht]
% \centering
% \small
% \resizebox{\textwidth}{!}{%
% \begin{tabular}{p{2.7cm} l c c p{7.8cm}}
% \hline
% \textbf{Task Name} & \textbf{Type} & \textbf{\#Examples} & \textbf{\#Nodes} & \textbf{Description} \\
% \hline

% Clustering--Shortest Path &
% Sequential &
% $N_{\text{ex}}$ &
% $|V|$ &
% Compute one shortest path from $s$ to $t$ in an unweighted undirected graph. For each node on the path, compute its local clustering coefficient, and return the node with the highest coefficient (ties broken by proximity to $s$), along with all coefficients on the path. \\

% SCC--Diameter &
% Sequential &
% $N_{\text{ex}}$ &
% $|V|$ &
% Compute strongly connected components (SCCs) of a directed graph. For each SCC, treat the induced subgraph as undirected and compute its diameter. Return the diameters of all components. \\

% SCC--Maximum Flow &
% Sequential &
% $N_{\text{ex}}$ &
% $|V|$ &
% For each SCC, select a representative node and compute the maximum flow to a fixed sink $t$, together with its clustering coefficient. Define a combined score and return the SCC with the highest score. \\

% \hline

% Pair Tightness Score &
% Parallel &
% $N_{\text{ex}}$ &
% $|V|$ &
% Compute in parallel the weighted shortest-path distance between $s$ and $t$, the number of common neighbors, and the clustering coefficients of $s$ and $t$. Combine them into a tightness score. \\

% Bridge Hub Identification &
% Parallel &
% $N_{\text{ex}}$ &
% $|V|$ &
% Compute clustering coefficients and average shortest-path distances for all nodes, and rank nodes by a bridge-hub score to identify the top-$k$ bridge hubs. \\

% Endpoint-Aware Flow &
% Parallel &
% $N_{\text{ex}}$ &
% $|V|$ &
% Compute the maximum flow from $s$ to $t$ and the clustering coefficients of the endpoints in parallel, and combine them into an endpoint-aware flow score. \\

% \hline

% Eulerian Path--Diameter &
% Conditional &
% $N_{\text{ex}}$ &
% $|V|$ &
% If the graph has an Eulerian path, return its diameter; otherwise, return the larger clustering coefficient between nodes $s$ and $t$. \\

% Connectivity--Component Diameter &
% Conditional &
% $N_{\text{ex}}$ &
% $|V|$ &
% If the graph is connected, return its diameter; otherwise, identify the largest connected component and return its diameter. \\

% SCC--Eulerian Check &
% Conditional &
% $N_{\text{ex}}$ &
% $|V|$ &
% If the directed graph has an Eulerian path, return the size of the largest SCC; otherwise, return the total number of SCCs. \\

% \hline
% \end{tabular}
% }
% \caption{Composite graph reasoning tasks constructed via sequential, parallel, and conditional composition of classical graph algorithms.}
% \label{tab:composite_tasks}
% \end{table*}

% \begin{table*}[t]
% \centering
% \small
% \caption{Composite graph reasoning tasks constructed via sequential, parallel, and conditional composition of classical graph algorithms.}
% \label{tab:composite_tasks}
% \resizebox{\textwidth}{!}{%
% \begin{tabular}{p{2.7cm} l c c p{7.8cm}}
% \toprule
% \textbf{Task Name} & \textbf{Type} & \textbf{\#Examples} & \textbf{\#Nodes} & \textbf{Description} \\
% \midrule

% Clustering--Shortest Path &
% Sequential &
% &
% &
% Compute one shortest path from $s$ to $t$ in an unweighted undirected graph. For each node on the path, compute its local clustering coefficient, and return the node with the highest coefficient (ties broken by proximity to $s$), along with all coefficients on the path. \\

% SCC--Diameter &
% Sequential &
% &
% &
% Compute strongly connected components of a directed graph. For each SCC, treat the induced subgraph as undirected and compute its diameter. Return the diameters of all components. \\

% SCC--Maximum Flow &
% Sequential &
% &
% &
% For each SCC, select a representative node and compute the maximum flow to a fixed sink $t$, together with its clustering coefficient. Define a combined score and return the SCC with the highest score. \\

% \midrule

% Pair Tightness Score &
% Parallel &
% &
% &
% Compute the weighted shortest-path distance between $s$ and $t$, the number of common neighbors, and the clustering coefficients of $s$ and $t$. Combine them into a tightness score. \\

% Bridge Hub Identification &
% Parallel &
% &
% &
% Compute clustering coefficients and average shortest-path distances for all nodes, and rank nodes by a bridge-hub score to identify the top-$k$ bridge hubs. \\

% Endpoint-Aware Flow &
% Parallel &
% &
% &
% Compute the maximum flow from $s$ to $t$ and the clustering coefficients of the endpoints in parallel, and combine them into an endpoint-aware flow score. \\

% \midrule

% Eulerian Path--Diameter &
% Conditional &
% &
% &
% If the graph has an Eulerian path, return its diameter; otherwise, return the larger clustering coefficient between nodes $s$ and $t$. \\

% Connectivity--Component Diameter &
% Conditional &
% &
% &
% If the graph is connected, return its diameter; otherwise, identify the largest connected component and return its diameter. \\

% SCC--Eulerian Check &
% Conditional &
% &
% &
% If the directed graph has an Eulerian path, return the size of the largest SCC; otherwise, return the total number of SCCs. \\

% \bottomrule
% \end{tabular}%
% }
% \end{table*}

\subsection{Necessity of the Proposed {\dataset}: An Empirical Analysis}
\label{appendix:analysis_our_dataset}
% \paragraph{Why Existing Benchmarks Under-Challenge Text Reasoning, and Why Composite Tasks Matter.}
From Tab.~\ref{tab:main_results_L}, we observe that although \texttt{GTools-EL} is labeled as a large-graph benchmark, its graphs contain no more than $200$ nodes and thus remain well within the context windows of mainstream LLMs. As a result, this setting places limited pressure on text-based reasoning in terms of context capacity and long-horizon structural inference. First, text-based reasoning does not clearly break down on \texttt{GTools-EL}, and in some cases achieves performance comparable to coding-based methods (e.g., Qwen-7B attains similar accuracy under few-shot text prompting and zero-shot coding). Second, the overall results on \texttt{GTools-WL} and \texttt{GTools-EL} are broadly similar, further suggesting that existing ``large-graph'' settings are insufficient to meaningfully separate reasoning paradigms by scalability. In contrast, under our truly large-scale graph setting, text-based reasoning degrades drastically: even the strongest model (DeepSeek) achieves only around $10\%$ accuracy, while a 7B model is nearly unable to solve the tasks (close to $0\%$). This stark contrast indicates that {\dataset} introduces substantial structural complexity that poses a genuine challenge to existing text-based reasoning approaches and better exposes their scalability limits.

Moreover, Tab.~\ref{tab:main_results_C} highlights another critical dimension: \emph{problem-level semantic complexity} in composite tasks substantially amplifies difficulties in both retrieval and code generation, making naïve retrieval-plus-coding pipelines unreliable. For instance, a representative coding-based baseline, \textsc{GraphTeam}, attains only the suboptimal performance on this setting (DeepSeek: $76.7\%$ vs.\ our $95.6\%$), indicating that prior methods do not adequately address two key challenges. First, composite tasks require retrieving complementary knowledge across multiple sub-algorithms, which raises the bar for retrieval accuracy and coverage. Second, semantic complexity introduces more risks on logic-level failures, which demand logic-level debugging beyond runtime checks. Controlled comparisons further support this conclusion: holding the coding agent fixed, merely changing the retrieval strategy (\textsc{TFIDF+CodeAgent} vs.\ \textsc{Ours}) yields large gains, $11.2\%$ on DeepSeek and $16.6\%$ on Qwen. These results show that semantic complexity in composite tasks poses a substantial retrieval challenge to existing coding-based graph reasoning methods. These observations indicates that existing retrieval and coding components are insufficiently robust to semantic complexity, and supports the necessity of our composite dataset.




\subsection{Detailed Experimental Setup: Dataset, Metric, and Baseline}
\label{sec:appendix_experiment_setup}

\subsubsection{Datasets}
We conduct experiments on two datasets: both \texttt{GTools}~\cite{wang2025GraphToolInstruction} and our proposed {\dataset}. Below, we describe \texttt{GTools}; {\dataset} is introduced in detail above.

% \noindent\textbf{GTools}~\cite{wang2025GraphToolInstruction} is a synthetic graph dataset generated by controlling the number of nodes and the probability of edge formation between node pairs. It provides two variants based on graph scale: WL-Graph (within-limited graphs) and EL-Graph (exceed-limited graphs), which we denote as GTools-S and GTools-L, respectively, to align with our dataset naming rule.
% GTools-S consists of graphs with $2$-$40$ nodes and up to $300$ edges, while GTools-L includes larger graphs with $100$-$1{,}000$ nodes and up to $100{,}000$ edges.
% We summarize the dataset statistics based on the official GitHub repository and use the off-the-shelf test set for evaluation. GTools test set covers $11$ graph reasoning tasks, including Cycle Detection, Maximum Triangle Sum, Edge Count, Node Count, Topological Sorting, Degree Count, Edge Existence, Node Existence, Maximum Flow, Path Existence, and Shortest Path. Each task contains $20$ graph instances, with $10$ directed and $10$ undirected graphs. The original purpose of GTools is to evaluate the ability of LLMs to invoke graph-related tools. Thus, each instance comprises a task description, graph instance, and labels specifying tool names, tool parameters, and outputs. We use only the task description, graph instance, and output.
% \footnote{\url{https://github.com/RongzhengWang/GraphTool-Instruction}}


\noindent\textbf{\texttt{GTools}}~\cite{wang2025GraphToolInstruction} is a synthetic graph dataset generated by controlling the number of nodes and the probability of edge formation between node pairs. It provides two variants based on graph scale: WL-Graph (within-limited graphs) and EL-Graph (exceed-limited graphs).
% \texttt{GTools-WL} consists of graphs with $5$-$40$ nodes and up to $400$ edges, while \texttt{GTools-EL} includes larger graphs with $100$-$1{,}000$ nodes and up to $100{,}000$ edges.
We summarize the dataset statistics based on its GitHub repository and use the off-the-shelf test set for evaluation, as in Table \ref{tab:gtools_wl_el_stats}. 
\texttt{GTools} test set covers $11$ graph reasoning tasks.
% , including Cycle Detection, Maximum Triangle Sum, Edge Count, Node Count, Topological Sorting, Degree Count, Edge Existence, Node Existence, Maximum Flow, Path Existence, and Shortest Path. 
Each task contains $20$ graph instances, with $10$ directed and $10$ undirected graphs. The original purpose of \texttt{GTools} is to evaluate the ability of LLMs to invoke graph-related tools. Thus, each instance comprises a task description, graph instance, and labeled tool names, tool parameters, and outputs. We use only the task description, graph instance, and output.
\begin{table}[htbp]
\centering
\small
\caption{Statistics of the GTools dataset. WL denotes small-scale graphs and EL denotes large-scale graphs.}
\label{tab:gtools_wl_el_stats}
\renewcommand{\arraystretch}{1.15}
\resizebox{0.7\linewidth}{!}{
\begin{tabular}{lccc}
\toprule
\textbf{Task Name} & \textbf{Instances} & \textbf{Nodes} & \textbf{Edges} \\
\midrule
\multicolumn{4}{c}{\textbf{GTools-WL (Small-scale)}} \\
\midrule
Maximum Flow              & 20 & 12--39 & 34--150  \\
Shortest Path Length      & 20 & 10--38 & 37--148  \\
Topological Sort          & 10 & 5--37  & 7--265   \\
Node Existence Check      & 20 & 12--40 & 38--300  \\
Edge Existence Check      & 20 & 6--38  & 19--280  \\
Cycle Detection           & 20 & 14--40 & 13--400  \\
Number of Edges           & 20 & 5--40  & 10--274  \\
Number of Nodes           & 20 & 8--40  & 17--269  \\
Path Existence Check      & 20 & 16--39 & 31--277  \\
Node Degree               & 20 & 11--40 & 32--259  \\
Clique Enumeration        & 10 & 11--38 & 24--149  \\
\midrule
\textbf{Total (WL)}        & \textbf{200} & 5--40 & 7--400 \\
\midrule
\multicolumn{4}{c}{\textbf{GTools-EL (Large-scale)}} \\
\midrule
Maximum Flow              & 20 & 11--51  & 17--627   \\
Shortest Path Length      & 20 & 10--88  & 51--728   \\
Node Existence Check      & 20 & 12--66  & 56--921   \\
Edge Existence Check      & 20 & 12--71  & 36--988   \\
Cycle Detection           & 20 & 12--99  & 11--952   \\
Number of Edges           & 20 & 5--71   & 14--818   \\
Number of Nodes           & 20 & 6--62   & 20--803   \\
Path Existence Check      & 20 & 12--97  & 42--1345  \\
Node Degree               & 20 & 13--127 & 53--900   \\
\midrule
\textbf{Total (EL)}        & \textbf{180} & 5--127 & 11--1345 \\
\bottomrule
\end{tabular}
}
\end{table}




% Overall, GTools comprises $$40{,}000$$ instances, with the WL and EL splits each containing $$2{,}000$$ instances. % 这个应是训练集


% For each graph reasoning task, we have prepared five different templates which contain different descriptions to evaluate the task identification capabilities of LLMs.



\subsubsection{Metrics}
\label{sec:appendix_metric}
% We employ answer accuracy as the evaluation metric, which calculates the percentage of questions that the LLM correctly predicts out of the total questions in the test dataset.
\noindent\textbf{Accuracy for graph reasoning.}
We adopt \emph{accuracy} as the evaluation metric, which measures the proportion of test instances for which the model produces correct predictions. Formally, given a test set $\mathcal{D}=\{(q_i, \mathcal{G}_i, \mathcal{A}_i)\}_{i=1}^{N}$ consisting of $N$ tasks, where $q_i$ denotes the task query, $\mathcal{G}_i$ the corresponding set of graph instances, and $\mathcal{A}_i$ the set of ground-truth answers, and given the model predictions $\hat{\mathcal{A}}_i$, the accuracy is defined as
\begin{equation}
\mathrm{Accuracy} = \frac{1}{N} \sum_{i=1}^{N} \frac{1}{|\mathcal{A}_i|} \sum_{j=1}^{|\mathcal{A}_i|} \mathbb{I}\big(\hat{\mathcal{A}}_{ij} = \mathcal{A}_{ij}\big),
\end{equation}
where $\mathbb{I}(\cdot)$ is the indicator function that equals $1$ if the predicted answer $\hat{\mathcal{A}}_{ij}$ matches the ground-truth answer $\mathcal{A}_{ij}$, and $0$ otherwise.


\noindent\textbf{Recall Metric for Retrieval.}
Let $\mathcal{D}$ denote the set of graph tasks, with $|\mathcal{D}| = N$. For each task $t \in \mathcal{D}$, let $\mathcal{R}_t$ be the set of ground-truth required documents, and let $\mathcal{H}_t$ denote the set of documents retrieved by the model. The per-task recall is defined as
\begin{equation}
\mathrm{Recall}_t =
\begin{cases}
\dfrac{|\mathcal{H}_t \cap \mathcal{R}_t|}{|\mathcal{R}_t|}, & \text{if } |\mathcal{R}_t| > 0, \\[6pt]
1, & \text{if } |\mathcal{R}_t| = 0,
\end{cases}
\end{equation}
which measures the fraction of required functions that are successfully retrieved for task $t$.
The overall recall is computed as the average of per-task recall scores across all tasks:
\begin{equation}
\mathrm{Recall} = \frac{1}{N} \sum_{t \in \mathcal{D}} \mathrm{Recall}_t.
\end{equation}

\noindent\textbf{Precision Metric for Retrieval.}
For each task $t \in \mathcal{D}$, the per-task precision is defined as
\begin{equation}
\mathrm{Precision}_t =
\begin{cases}
\dfrac{|\mathcal{H}_t \cap \mathcal{R}_t|}{|\mathcal{H}_t|}, & \text{if } |\mathcal{H}_t| > 0, \\[6pt]
0, & \text{if } |\mathcal{H}_t| = 0,
\end{cases}
\end{equation}
which measures the fraction of retrieved documents that are relevant for task $t$.
The overall precision is computed as the average of per-task precision scores across all tasks:
\begin{equation}
\mathrm{Precision} = \frac{1}{N} \sum_{t \in \mathcal{D}} \mathrm{Precision}_t.
\end{equation}

\noindent\textbf{F1 Metric for Retrieval.}
For each task $t \in \mathcal{D}$, the per-task F1 score is defined as the harmonic mean of per-task precision and recall:
\begin{equation}
\mathrm{F1}_t =
\begin{cases}
\dfrac{2 \cdot \mathrm{Precision}_t \cdot \mathrm{Recall}_t}{\mathrm{Precision}_t + \mathrm{Recall}_t}, & \text{if } \mathrm{Precision}_t + \mathrm{Recall}_t > 0, \\[6pt]
0, & \text{otherwise}.
\end{cases}
\end{equation}
The overall F1 score is computed as the average of per-task F1 scores across all tasks:
\begin{equation}
\mathrm{F1} = \frac{1}{N} \sum_{t \in \mathcal{D}} \mathrm{F1}_t.
\end{equation}


\begin{figure}[t]
    \centering
    % \vskip -1em
    \includegraphics[width=0.8\linewidth]{figures/performance_vs_time_annotated_c.pdf}
    \vskip -1.5em
    \caption{Retrieval performance and time on composite subset.}
    % \vskip -1.5em
    \label{fig:retrieval_results_c}
\end{figure}



\subsubsection{Baselines}
\label{appendix:baselines}
% 
We categorize the baselines into two main groups: {text-based reasoning} and {code-based reasoning}. Due to the diversity of code-based approaches, we further divide them into four subcategories: {code-based reasoning without retrieval or debugging}, {code-based reasoning with retrieval but without debugging}, {code-based reasoning with both retrieval and debugging}, and {human-involved code-based reasoning}. All baselines use NetworkX~\cite{hagberg2008exploring} as the technical documentation if retrieval is included. We detail each category and the corresponding methods below.

\paragraph{(i) Text-based reasoning}
Following Talk-like-a-Graph~\cite{fatemi2023talklikegraphencoding}, we consider several representative text-based reasoning baselines, including zero-shot prompting (\textsc{Zero-shot}), few-shot in-context learning (\textsc{Few-shot})~\cite{brown2020language}, chain-of-thought prompting (\textsc{CoT})~\cite{wei2022chain}, and the build-a-graph prompt (\textsc{CoT-BAG})~\cite{wang2024languagemodelssolvegraph}. Specifically,
\begin{itemize}[nosep,leftmargin=*]
    \item \textsc{Zero-shot}: The model is provided only with the task description and is asked to directly generate the textual reasoning output, without any task-specific demonstrations.
    \item \textsc{Few-shot}: A small number of input–output exemplars are included in the prompt to guide the model in performing the task on new inputs.
    \item \textsc{CoT}: The prompt includes step-by-step reasoning exemplars, enabling the model to generate intermediate reasoning chains when solving tasks.
    \item \textsc{CoT-BAG}: This method augments the task description with an explicit instruction, by appending ``Let us construct a graph with the nodes and edges first'', to encourage the model to reason over the underlying graph structure explicitly.
\end{itemize}
The detailed prompts are provided in Sec.~\ref{sec:appendix_prompt}.
% \fali{give the prompts}

\paragraph{(ii) Code-based reasoning without retrieval or debugging}
This category includes baselines that solve graph reasoning tasks by generating executable code, without employing retrieval augmentation or debugging mechanisms. These methods assess the inherent code-generation capability of LLMs when applied to graph reasoning tasks. Accordingly, we consider both zero-shot and few-shot coding settings, namely \textsc{Zero-shot Coding} and \textsc{Few-shot Coding}. Specifically,
\begin{itemize}[nosep, leftmargin=*]
    \item \textsc{Zero-shot Coding}: The model is provided only with the task description and is required to directly generate executable code, without any additional guidance or augmentation.
    \item \textsc{Few-shot Coding}: A small number of input–output exemplars are included in the prompt to guide the model in generating code-based solutions for graph reasoning.
\end{itemize}
The detailed prompts are provided in Sec.~\ref{sec:appendix_prompt}.
% \fali{give the prompts}


\paragraph{(iii) Code-based reasoning with retrieval but without debugging}
This category comprises baselines that address graph reasoning tasks by generating executable code with retrieval augmentation, but without explicit debugging mechanisms. Methods in this category include \textsc{TFIDF+Coding}, and \textsc{SentBERT+Coding}. Specifically,
% \textsc{GCoder}~\cite{zhang2024gcoderimprovinglargelanguage}, \textsc{GraphTool-Instruction}~\cite{wang2025GraphToolInstruction},
\begin{itemize}[nosep, leftmargin=*]
    % \item \textsc{GCoder}~\cite{zhang2024gcoderimprovinglargelanguage}: This method retrieves relevant documentation using an index built with BAAI/bge-base-en-v1.5~\cite{xiao2024c}, and conditions the LLM on the retrieved content to generate executable code.
    % \item \textsc{GraphTool-Instruction}~\cite{wang2025GraphToolInstruction}: This approach formulates external knowledge as callable tools to assist LLM-based graph reasoning. To adapt it to our setting, where including all NetworkX functions and descriptions would exceed the context window, we employ a TF-IDF retriever to select the \textcolor{red}{top-30} relevant functions and allow the model to choose appropriate tools for solving each task.
    \item \textsc{TFIDF+Coding}: This retrieves relevant documentation using TF-IDF indexing~\cite{salton1988term} and conditions the LLM on the retrieved top-k results for code generation.
    \item \textsc{SentBERT+Coding}: This performs retrieval using Sentence-BERT embeddings~\cite{reimers2019sentence} to obtain semantically relevant top-k documentation, which is then used to guide code generation.
\end{itemize}
% All baselines in this category use NetworkX~\cite{hagberg2008exploring} as the technical documentation.

\paragraph{(iv) Code-based reasoning with both retrieval and debugging}
This category includes baselines that solve graph reasoning tasks by generating executable code with both retrieval augmentation and debugging mechanisms. Representative methods in this category include \textsc{GraphTeam}~\cite{li2025graphteamfacilitatinglargelanguage}, \textsc{TFIDF+CodeAgent}, and \textsc{SentBERT+CodeAgent}. Specifically,
\begin{itemize}[nosep, leftmargin=*]
    \item \textsc{GraphTeam}~\cite{li2025graphteamfacilitatinglargelanguage}: This method employs LlamaIndex~\cite{llamaindexweb} for document retrieval and integrates a coding agent that performs debugging through interaction with a compiler environment for correcting runtime errors.
    \item \textsc{TFIDF+CodeAgent}: This uses TF-IDF indexing~\cite{salton1988term} for document retrieval and adopts the same coding agent as our method for code generation and debugging.
    \item \textsc{SentBERT+CodeAgent}: This performs retrieval using Sentence-BERT embeddings~\cite{reimers2019sentence} and likewise employs the same coding agent as our method for code generation and debugging.
\end{itemize}
% All baselines here also use NetworkX~\cite{hagberg2008exploring} as the technical documentation.


\paragraph{(v) Human-involved code-based reasoning}
This category includes existing methods that rely on explicit human involvement during inference or prompt construction. Owing to their dependence on human-provided guidance, we refer to them as \emph{human-involved code-based reasoning} methods. Representative approaches in this category include \textsc{CodeGraph}~\cite{cai2024codegraphenhancinggraphreasoning} and \textsc{PIE}~\cite{gong2025pseudocodeinjectionmagicenablingllms}. Specifically,
\begin{itemize}[nosep, leftmargin=*]
    \item \textsc{CodeGraph}~\cite{cai2024codegraphenhancinggraphreasoning}: This method relies on human-selected, task-specific input–output code exemplars to guide graph reasoning, without a debugging mechanism. We provide the ground-truth functions in the technical documentation to support this setup.
    % Accordingly, we provide manually selected exemplars when evaluating it.
    \item \textsc{PIE}~\cite{gong2025pseudocodeinjectionmagicenablingllms}: This approach assumes that LLMs are proficient in code generation, and therefore supplies task-specific pseudocode along with human-provided test cases to facilitate debugging.
\end{itemize}





\subsection{Detailed Related Work}
\subsubsection{LLM-based Graph Reasoning}
\label{sec:appendix_related_reasoning}
% \noindent\textbf{Benchmarking LLMs on Graph Reasoning} Despite the remarkable success of LLMs in natural language processing tasks, their capabilities on graph-structured problems need to be explored for artificial general intelligence. A substantial body of prior work has investigated this challenge from the perspective of \textit{LLM-based graph learning}, which focuses on learning parameterized mapping functions over graph-structured data from structural features to the labels to minimize expected risk and achieve generalization to unseen graph samples. Such methods typically target classical graph learning tasks, including node classification \cite{chen2024labelfree, tang2024graphgpt}, link prediction, knowledge graph reasoning \cite{luo2024reasoning}, and molecular learning \cite{le2024molx}. Under this paradigm, LLMs primarily rely on learning statistical patterns from data, rather than performing explicit structural reasoning.
% In contrast, \textit{LLM-based graph reasoning} poses a more challenging problem. Its objective is not to learn a mapping function, but to produce a solution through LLMs' multi-step reasoning, such as finding shortest paths. The correct solutions to such tasks are typically equivalent to, or can be derived from, a sequence of explicit graph-structured derivations, commonly referred to as \emph{graph computation}. Some studies describe these problems as \emph{graph computation tasks} \cite{chen2024graphwizinstructionfollowinglanguagemodel, zhang2024gcoderimprovinglargelanguage, tang2025grapharena}, emphasizing that graph problems admit well-defined algorithmic solutions while we adopt the term \emph{graph reasoning tasks} to highlight the role of LLMs in performing structural reasoning and decision-making. Early benchmarks such as GraphQA \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} were among the first to evaluate LLMs on fundamental graph reasoning tasks, including edge existence, node degree computation, cycle detection, shortest path, and tasks partially computable by graph neural networks. Their empirical results indicate that while LLMs exhibit preliminary graph reasoning capabilities, their performance still falls substantially short of ideal levels. 
% Motivated by these findings, subsequent work has proposed more systematic and comprehensive benchmarks and datasets for graph reasoning, including GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel}, GraphInstruct \cite{luo2024graphinstruct}, GraphWild \cite{zhang2024gcoderimprovinglargelanguage}, GTools \cite{wang2025GraphToolInstruction}, GraphArena \cite{tang2025grapharena}, and LLM4DyG \cite{zhang2024llm4dyg}. Among them, GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} evaluates graph reasoning problems of varying complexity, including NP-complete tasks; GraphInstruct \cite{luo2024graphinstruct} comprises 21 graph analysis tasks spanning node-level, node-pair-level, and graph-level reasoning; LLM4DyG \cite{zhang2024llm4dyg} focuses on dynamic graph reasoning involving temporal and spatial information; and GNN-AutoGL \cite{li2025graphteamfacilitatinglargelanguage} evaluates the ability of LLMs to invoke AutoGL tools for automated GNN deployment. 
% However, existing benchmarks are largely restricted to small-scale graphs and simple canonical graph tasks. They have not systematically examined the ability of LLMs to perform large-scale graph reasoning when graph sizes exceed the model’s context window under constrained token budgets, nor do they adequately consider complex reasoning scenarios that require the composition of multiple graph tasks. To address this gap, we introduce a new graph reasoning dataset that systematically spans multiple graph scales, ranging from 20 to 200 nodes, and includes both simple graph tasks and composite graph reasoning tasks. This dataset is designed to evaluate and analyze the performance of LLMs under large-scale graph structures and compositional graph reasoning settings, thereby providing a more realistic assessment of their applicability and limitations in practical, complex graph problem-solving scenarios.
\noindent\textbf{Benchmarking LLMs on Graph Reasoning.}
Despite the remarkable success of LLMs in natural language processing tasks, their capabilities on graph-structured problems are important, particularly in the context of artificial general intelligence. A substantial body of prior work has investigated this challenge from the perspective of \emph{LLM-based graph learning}, which focuses on learning parameterized mapping functions over graph-structured data, from structural features to labels, to minimize expected risk and achieve generalization to unseen graph samples. Such methods typically target classical graph learning tasks, including node classification \cite{chen2024labelfree, tang2024graphgpt}, knowledge graph reasoning \cite{luo2024reasoning}, and molecular learning \cite{le2024molx}. Under this paradigm, LLMs primarily rely on capturing statistical patterns from data distributions, rather than performing explicit structural reasoning over graphs.
In contrast, \emph{LLM-based graph reasoning} aims to produce correct solutions through multi-step reasoning processes carried out by LLMs themselves over graph algorithm tasks, such as shortest paths or cycle check. The correct solutions to such tasks can be derived from a sequence of explicit graph algorithmic operations. 
Several studies refer to these problems as \emph{graph computation tasks} \cite{chen2024graphwizinstructionfollowinglanguagemodel, zhang2024gcoderimprovinglargelanguage, tang2025grapharena} to emphasize their well-defined algorithmic and executable semantics, whereas we adopt the term \emph{graph reasoning tasks} to highlight the role of LLMs in directly performing structural reasoning and decision-making over graph structures. Early benchmarks such as GraphQA \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} were among the first to evaluate LLMs on graph reasoning tasks, including edge existence, node degree computation, cycle detection, shortest path, and tasks partially computable by graph neural networks. Their empirical results demonstrate that, while LLMs exhibit preliminary graph reasoning capabilities, their performance still falls short of ideal levels.
Motivated by these observations, subsequent work has proposed more systematic and comprehensive benchmarks and datasets for graph reasoning, including GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel}, GraphInstruct \cite{luo2024graphinstruct}, GraphWild \cite{zhang2024gcoderimprovinglargelanguage}, GTools \cite{wang2025GraphToolInstruction}, GraphArena \cite{tang2025grapharena}, and LLM4DyG \cite{zhang2024llm4dyg}. Among them, GraphWiz evaluates graph reasoning problems of varying complexity, including NP-complete tasks; GraphInstruct comprises $21$ graph analysis tasks spanning node-level, node-pair-level, and graph-level reasoning; and LLM4DyG focuses on dynamic graph reasoning involving temporal and spatial information.
However, existing benchmarks are largely restricted to small-scale graphs and single canonical tasks, and do not systematically evaluate LLMs’ ability to perform large-scale graph reasoning when graph sizes exceed the context window, nor complex reasoning that composes multiple graph tasks. To address this gap, we introduce a new graph reasoning dataset spanning multiple graph scales, which includes both simple and composite graph reasoning tasks. This dataset enables a more comprehensive assessment of LLMs in large-scale and complex graph reasoning.

% \noindent\textbf{LLM-based Graph Reasoning Methods} Due to the unsatisfactory performance of LLMs on graph reasoning tasks, a growing body of recent work has sought to enhance their graph reasoning capabilities \cite{tang2024graphgpt, perozzi2024let, chen2024graphwizinstructionfollowinglanguagemodel, cai2024codegraphenhancinggraphreasoning, zhang2024gcoderimprovinglargelanguage, wang2025GraphToolInstruction, li2025graphteamfacilitatinglargelanguage, luo2024graphinstruct, chai2025graphllm, gong2025pseudocodeinjectionmagicenablingllms}. From the perspective of reasoning execution, existing approaches can be broadly categorized into two paradigms: \textit{text-based graph reasoning}, which injects graph structures into prompts in textual form and relies on LLMs to solve graph tasks via natural language reasoning; and \textit{code-based graph reasoning}, which realizes graph algorithmic computation by generating and executing code.
% \textit{(1) Text-based graph reasoning} relies on LLMs’ ability to analyze text-serialized graph structures and typically employs various prompting strategies for natural language reasoning. For example, Talk-like-a-Graph \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} systematically evaluate the effectiveness of standard textual instructions under different prompting settings, including zero-shot, few-shot, Chain-of-Thought \cite{wei2022chain}, and self-consistency \cite{wang2023selfconsistency}, for graph reasoning tasks. NLGraph further introduces Build-a-Graph, which explicitly requires LLMs to output a conceptual representation of the graph to enhance structural awareness, as well as Algorithmic Prompting, which provides algorithmic hints within prompts to facilitate reasoning. However, such purely text-based reasoning approaches are constrained by the limited capacity of LLMs in deep and structured derivation, often leading to unsatisfactory performance. To mitigate this limitation, several works incorporate instruction tuning to enhance graph reasoning ability. For instance, GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} leverages instruction data optimized via Direct Preference Optimization (DPO) to guide LLMs toward more appropriate reasoning paths for graph computation tasks; GraphInstruct \cite{luo2024graphinstruct} proposes GraphLM, which applies LoRA fine-tuning with final-answer prediction as the optimization objective to improve graph understanding, and further introduces GraphLM+, which jointly optimizes key tokens in multi-step reasoning and the final answer to enhance graph reasoning performance; GraphLLM \cite{chai2025graphllm} compresses graphs into structured continuous vector prefixes, which are aligned with the LLM representation space through fine-tuning, enabling the LLM to access a global graph summary at each attention layer. 
% Nevertheless, prior studies indicate that instruction tuning exhibits limited transferability in graph reasoning tasks \cite{zhang2024can}, making it difficult to generalize robustly across different graph tasks.
% \textit{(2) Code-based graph reasoning} adopts a code-based graph reasoning paradigm, which equips LLMs with graph algorithmic capabilities through code generation. For example, CodeGraph \cite{cai2024codegraphenhancinggraphreasoning} generates graph algorithm code within a few-shot exemplars and obtains answers by executing the code with a program interpreter; GCoder \cite{zhang2024gcoderimprovinglargelanguage} further retrieves relevant implementation templates from a code repository based on embedding similarity; GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} employs a multi-agent collaboration framework, in which a search agent retrieves relevant documents and prior experience via semantic retrieval, and a coding agent is responsible for code generation and compilation verification. However, both text-based and code-based approaches above require serializing graph data into prompts, and as the graph size grows beyond the LLM’s context window, these methods become increasingly infeasible, thereby severely limiting their effectiveness in large-scale graph reasoning scenarios.
% To overcome the context window bottleneck, a small number of recent studies have begun to explore a task--graph decoupling paradigm, in which prompts contain only task descriptions, while graph data is processed externally through generated code. GraphTool-Instruction \cite{wang2025GraphToolInstruction} explicitly includes a complete set of graph tool descriptions in the prompt to guide LLMs in generating tool-invoking code; however, this approach relies on manually constructed tool sets and requires enumerating all tool descriptions within the prompt, limiting its scalability to large tool collections. PIE \cite{gong2025pseudocodeinjectionmagicenablingllms} guides code generation by injecting expert-level pseudocode derived from recent research papers and uses test cases to debug and select the optimal implementation, but this method heavily depends on human-provided task-specific pseudocode, restricting its practicality and scalability.
% Based on the above analysis, we propose a code-based reasoning paradigm that decouples task descriptions from graph data and introduces a retrieval-augmented LLM-based agent to enable scalable graph reasoning beyond context window and human-knowledge constraints.
% Based on the above analysis, we focus on a code-based reasoning paradigm that decouples task descriptions from graph data, and further proposes a technical-document-retrieval-augmented LLM-based agent framework for graph reasoning. By doing so, our approach aims to enhance LLMs’ capabilities in large-scale and complex graph reasoning scenarios by retrieving from existing technical document without being constrained by the context window limitation and human-evolving graph knowledge.
% while LLMs have limited capacity to directly read and comprehend graph structures \cite{zhang2024can} in prompts. 
\noindent\textbf{LLM-based Graph Reasoning Methods.}
Due to the unsatisfactory performance of LLMs on graph reasoning tasks, a growing body of recent work has sought to enhance their graph reasoning capabilities \cite{tang2024graphgpt, perozzi2024let, chen2024graphwizinstructionfollowinglanguagemodel, cai2024codegraphenhancinggraphreasoning, zhang2024gcoderimprovinglargelanguage, wang2025GraphToolInstruction, li2025graphteamfacilitatinglargelanguage, luo2024graphinstruct, chai2025graphllm, gong2025pseudocodeinjectionmagicenablingllms}. From the perspective of output reasoning execution, existing approaches can be broadly categorized into two categories: text-based graph reasoning, which solves graph reasoning tasks through natural language reasoning over graph structures serialized in textual form within the prompt; and code-based graph reasoning, which generates code and executes it to realize graph algorithmic computation.
\textit{(1) Text-based graph reasoning.}
This paradigm relies on LLMs’ ability to analyze text-serialized graph structures and typically employs various prompting strategies for natural language reasoning. For example, Talk-like-a-Graph \cite{fatemi2023talklikegraphencoding} and NLGraph \cite{wang2024languagemodelssolvegraph} systematically evaluate the effectiveness of standard textual instructions under different prompting settings, including zero-shot, few-shot, chain-of-thought \cite{wei2022chain}, and self-consistency \cite{wang2023selfconsistency}, for graph reasoning tasks. NLGraph further introduces Build-a-Graph, which explicitly requires LLMs to output a conceptual representation of the graph to enhance structural awareness, as well as Algorithmic Prompting, which provides algorithmic hints within prompts to facilitate reasoning. However, such purely text-based prompting approaches are constrained by the limited capacity of LLMs for deep algorithmic understanding and structured derivation, often resulting in suboptimal performance. To mitigate these limitations, several works incorporate instruction tuning to enhance graph reasoning ability. For instance, GraphWiz \cite{chen2024graphwizinstructionfollowinglanguagemodel} leverages instruction data optimized via direct preference optimization to guide LLMs toward more appropriate reasoning paths for graph computation tasks; GraphInstruct \cite{luo2024graphinstruct} proposes GraphLM, which applies LoRA fine-tuning with final-answer prediction as the optimization objective to improve graph understanding, and further introduces GraphLM+, which jointly optimizes key tokens in multi-step reasoning and the final answer to enhance graph reasoning performance; GraphLLM \cite{chai2025graphllm} compresses graphs into structured continuous vector prefixes aligned with the LLM representation space through fine-tuning, enabling the model to access a global graph summary at each attention layer. Nevertheless, prior studies indicate that instruction tuning exhibits limited transferability in graph reasoning tasks \cite{zhang2024can}, making robust generalization across different graph tasks challenging.
\textit{(2) Code-based graph reasoning.}
This paradigm equips LLMs with graph algorithmic capabilities through code generation and execution, without requiring explicit step-by-step natural language reasoning. For example, CodeGraph \cite{cai2024codegraphenhancinggraphreasoning} generates graph algorithm code using a small set of exemplars and obtains answers by executing the code with a program interpreter; GCoder \cite{zhang2024gcoderimprovinglargelanguage} further retrieves relevant implementation templates from a code repository based on embedding similarity to improve generalization; GraphTeam \cite{li2025graphteamfacilitatinglargelanguage} employs a multi-agent collaboration framework, in which a search agent retrieves relevant documents and prior experience via semantic retrieval, and a coding agent is responsible for code generation and compilation verification. However, both these text-based and code-based approaches still require serializing graph data into prompts. As the graph size grows beyond the LLM’s context window, these methods become increasingly infeasible, thereby severely limiting their effectiveness in large-scale graph reasoning scenarios.
To overcome the context window bottleneck, a small number of recent studies have begun to explore a \emph{task--graph decoupling paradigm}, in which prompts contain only task descriptions while graph data is processed externally through generated code. GraphTool-Instruction \cite{wang2025GraphToolInstruction} explicitly includes a complete set of graph tool descriptions in the prompt to guide LLMs in generating tool-invoking code; however, this approach relies on manually constructed tool sets and requires enumerating all tool descriptions within the prompt, limiting its scalability to large tool collections. PIE \cite{gong2025pseudocodeinjectionmagicenablingllms} guides code generation by injecting expert-level pseudocode derived from recent research papers and uses test cases to debug and select the optimal implementation, but it heavily depends on human-provided task-specific pseudocode, which restricts its practicality.
Based on these, we propose a code-based reasoning paradigm that decouples task descriptions from graph data and introduces a retrieval-augmented LLM-based agent framework to enable scalable graph reasoning beyond context window and human-involving constraints.


\subsubsection{Agents in RAG}
\label{sec:appendix_related_agentic_rag}
% \noindent\textbf{Agents in RAG} 
Autonomous agents are introduced by agentic RAG to enable dynamical retrieval and iterative refinement, which eliminates the constraint of static workflows in regular RAG. Specifically, an agentic RAG system utilizes an agent to decompose tasks, adaptively retrieve external knowledge, synthesize intermediate results, and iteratively performs refinement until a valid result is obtained.
For example, \citeauthor{ravuru2024agentic} propose to use agentic RAG for time series analysis with a hierarchical-agent framework.
CRAG~\cite{yan2024corrective} utilizes an evaluator to assess the quality of retrieved resources and perform corresponding actions based on the results. 
Considering the constraint-rich nature of structured documentation which requires explicit retrieval planning and iterative refinement, some work has proposed to utilize agentic RAG on structured documentation to enhance the reasoning and retrieval accuracy.
For instance, GeAR~\cite{shen2025gear} integrates base retrievals with a graph expansion mechanism and proposes a multi-step retrieval framework.
Similarly, HM-RAG~\cite{liu2025hm} provides parallel retrieval results from vector, graph, and web-based documents and integrates these results to improve answer accuracy. 
% EMG-RAG~\cite{wang2024crafting} introduces an editable memory graph that comprises a tree structure for efficient retrieval.
% Recent work has adapted agentic RAG to graph reasoning, such as GCoder~\cite{zhang2024gcoderimprovinglargelanguage} which proposes a code-based approach to solve graph reasoning problems with hybrid retrieval from external documentation, and GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage} which employs five collaborative agents for question normalization, external knowledge retrieval and solution generation.
Recent work has adapted agentic RAG to graph reasoning, including GraphTeam~\cite{li2025graphteamfacilitatinglargelanguage}. They propose a code-based approach to solve graph reasoning problems with knowledge retrieval from external documentation.
% Difference with our work.
However, they do not consider the hierarchical structure of technical documentation. On the other hand, our proposed framework bridges this gap by utilizing a layer-wise agentic retrieval framework for efficient localization. 
% \fali{need to re-check the logic of this paragraph because our method has changed.}


%%%%%%%%%% graph-s


\begin{table*}[t]
\centering
\small
\setlength{\tabcolsep}{6pt}
\renewcommand{\arraystretch}{1.15}
\caption{Overview of graph reasoning tasks in the small-scale \texttt{\dataset-S}.}
\label{tab:small_scale_task_overview}
\resizebox{0.8\linewidth}{!}{
\begin{tabular}{lllll}
\toprule
\textbf{Task} & \textbf{Task Type} & \textbf{\#Examples} & \textbf{\#Nodes} & \textbf{\#Edges} \\
\midrule
Graph Connectivity Check (GCC) 
& Traversal-Based Reasoning 
& 674 
& 2--200 
& 1--19{,}701 \\

Connected Components Counting (CCC) 
& Traversal-Based Reasoning 
& 674 
& 2--200 
& 1--19{,}701 \\

Strongly Connected Components Counting (SCCC) 
& Traversal-Based Reasoning 
& 550 
& 4--200 
& 2--18{,}336 \\

Shortest Path (SP) 
& Traversal-Based Reasoning 
& 1{,}472 
& 2--200 
& 1--19{,}503 \\
\midrule
Eulerian Path Check (EPC) 
& Graph Property Detection 
& 1{,}224 
& 2--200 
& 1--18{,}336 \\

Graph Bipartiteness (GB) 
& Graph Property Detection 
& 1{,}224 
& 2--200 
& 1--18{,}336 \\

Graph Diameter (GD) 
& Graph Property Detection 
& 457 
& 2--200 
& 1--19{,}701 \\

Regularity Check (RC) 
& Graph Property Detection 
& 1{,}224 
& 2--200 
& 1--19{,}701 \\

Distance Regularity Check (DRC) 
& Graph Property Detection 
& 1{,}226 
& 3--200 
& 2--19{,}900 \\
\midrule
Maximum Flow (MF) 
& Combinatorial Optimization 
& 1{,}450 
& 2--200 
& 1--19{,}503 \\

Maximum Clique (MC) 
& Combinatorial Optimization 
& 674 
& 2--200 
& 1--19{,}701 \\

Maximum Independent Set (MIS) 
& Combinatorial Optimization 
& 674 
& 2--200 
& 1--19{,}701 \\
\midrule
Clustering Coefficient (CC) 
& Neighborhood / Structural Analysis 
& 1{,}117 
& 13--200 
& 14--19{,}701 \\

Common Neighbors (CN) 
& Neighborhood / Structural Analysis 
& 224 
& 16--200 
& 11--19{,}701 \\
\bottomrule
\end{tabular}
}
\end{table*}




% \begin{table*}[t]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Connected Components Counting (CCC)} task statistics in GraphCoding-S with X examples in total.
% % The table enumerates all valid graph configurations after removing \textbf{Graph Model},
% % \textbf{Degree Distribution}, and \textbf{Structural Constraint}.
% \textit{Density}: graph density (Sparse, Dense, Complete);
% \textit{Connectivity}: (Connected, Disconnected);
% \textit{Cyclicity}: (Acyclic, Cyclic);
% \textit{Direction}: edge direction (Undirected, Directed);
% \textit{Weights}: Unweighted only for CCC as edge weights do not influence the connectivity structure.
% Column \textbf{\#Examples} denotes the number of examples per configuration.
% \textit{\#Nodes}: node range.
% }
% \label{tab:ccc_all_combinations}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule

% Sparse   & Connected    & Acyclic & Undirected & Unweighted & 1,200 & [20,200] \\
% Sparse   & Connected    & Acyclic & Directed   & Unweighted & 700 & [20,200] \\
% Sparse   & Connected    & Cyclic  & Undirected & Unweighted & 1,200 & [20,200] \\
% Sparse   & Connected    & Cyclic  & Directed   & Unweighted & 700 & [20,200] \\
% Sparse   & Disconnected & Acyclic & Undirected & Unweighted & 1,200 & [20,200] \\
% Sparse   & Disconnected & Acyclic & Directed   & Unweighted & 700 & [20,200] \\
% Sparse   & Disconnected & Cyclic  & Undirected & Unweighted & 1,100 & [20,200] \\
% Sparse   & Disconnected & Cyclic  & Directed   & Unweighted & 700 & [20,200] \\

% Sparse   & Connected    & Acyclic & Undirected & Weighted & 300 & [20,200] \\
% Sparse   & Connected    & Acyclic & Directed   & Weighted & 300 & [20,200] \\
% Sparse   & Connected    & Cyclic  & Undirected & Weighted & 300 & [20,200] \\
% Sparse   & Connected    & Cyclic  & Directed   & Weighted & 300 & [20,200] \\
% Sparse   & Disconnected & Acyclic & Undirected & Weighted & 300 & [20,200] \\
% Sparse   & Disconnected & Acyclic & Directed   & Weighted & 300 & [20,200] \\
% Sparse   & Disconnected & Cyclic  & Undirected & Weighted & 300 & [20,200] \\
% Sparse   & Disconnected & Cyclic  & Directed   & Weighted & 300 & [20,200] \\


% Dense    & Connected    & Acyclic & Undirected & Unweighted & 1,200 & [20,200] \\
% Dense    & Connected    & Acyclic & Directed   & Unweighted & 700 & [20,200] \\
% Dense    & Connected    & Acyclic & Undirected & Weighted   & 300 & [20,200] \\
% Dense    & Connected    & Acyclic & Directed   & Weighted   & 300 & [20,200] \\
% Dense    & Connected    & Cyclic  & Undirected & Unweighted & 1,200 & [20,200] \\
% Dense    & Connected    & Cyclic  & Directed   & Unweighted & 700 & [20,200] \\
% Dense    & Connected    & Cyclic  & Undirected & Weighted   & 300 & [20,200] \\
% Dense    & Connected    & Cyclic  & Directed   & Weighted   & 300 & [20,200] \\
% Dense    & Disconnected & Acyclic & Undirected & Unweighted & 1,200 & [20,200] \\
% Dense    & Disconnected & Acyclic & Directed   & Unweighted & 700 & [20,200] \\
% Dense    & Disconnected & Acyclic & Undirected & Weighted   & 300 & [20,200] \\
% Dense    & Disconnected & Acyclic & Directed   & Weighted   & 300 & [20,200] \\

% Complete & Connected    & Cyclic  & Undirected & Unweighted & 1,200 & [20,200] \\
% Complete & Connected    & Cyclic  & Directed   & Unweighted & 700 & [20,200] \\
% Complete & Connected    & Cyclic  & Undirected & Weighted   & 300 & [20,200] \\
% Complete & Connected    & Cyclic  & Directed   & Weighted   & 300 & [20,200] \\

% \bottomrule
% \end{tabular}
% \end{table*}


% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Graph Connectivity Check (GCC)} task statistics in \texttt{\dataset-S} with $107$ examples in total.
% \textit{Density}: graph density (Sparse, Dense, Complete);
% \textit{Connectivity}: (Connected, Disconnected);
% \textit{Cyclicity}: (Acyclic, Cyclic);
% \textit{Direction}: edge direction (Undirected, Directed);
% \textit{Weights}: edge weights (Weighted, Unweighted).
% Column \textbf{\#Examples} denotes the number of examples per configuration.
% \textit{\#Nodes}: node count range.
% }
% \label{tab:is_connected_unweighted}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Connected Components Counting (CCC)} task statistics in \texttt{\dataset-S} with $224$ examples in total.
% % \textit{Density}: graph density (Sparse, Dense, Complete);
% % \textit{Connectivity}: (Connected, Disconnected);
% % \textit{Cyclicity}: (Acyclic, Cyclic);
% % \textit{Direction}: edge direction (Undirected, Directed);
% % \textit{Weights}: edge weights (Weighted, Unweighted).
% % Column \textbf{\#Examples} denotes the number of examples per configuration.
% % \textit{\#Nodes}: node count range.
% }
% \label{tab:ccc_all_combinations}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic   & Undirected & Weighted   & 50 & 22--194 \\
% Sparse   & Connected & Acyclic  & Undirected & Weighted   & 17 & 9--35   \\
% Dense    & Connected & Cyclic   & Undirected & Weighted   & 50 & 22--197 \\
% Complete & Connected & Cyclic   & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic  & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic   & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Strongly Connected Components Counting (SCCC)} task statistics in \texttt{\dataset-S} with $150$ examples.
% }
% \label{tab:scc_unweighted}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed & Unweighted & 50 & 30--193 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Shortest Path (SP)} task statistics in \texttt{\dataset-S} with $267$ examples in total.
% }
% \label{tab:spl_all_combinations}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Weighted & 50 & 22--198 \\
% Sparse   & Connected & Acyclic & Directed   & Weighted & 50 & 45--195 \\
% Dense    & Connected & Acyclic & Directed   & Weighted & 50 & 19--193 \\
% Complete & Connected & Cyclic  & Undirected & Weighted & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted & 50 & 22--197 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Topological Sorting (TS)} task statistics in \texttt{\dataset-S} with $150$ examples in total.
% }
% \label{tab:ts_unweighted}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed & Unweighted & 50 & 30--193 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Eulerian Path Check (EPC)} task statistics in \texttt{\dataset-S} with $257$ examples in total.
% }
% \label{tab:ep_unweighted}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed   & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed   & Unweighted & 50 & 30--193 \\
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Graph Bipartiteness (GB)} task statistics in \texttt{\dataset-S} with $257$ examples in total.
% }
% \label{tab:is_bipartite_unweighted}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed   & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed   & Unweighted & 50 & 30--193 \\
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Graph Diameter (GD)} task statistics in \texttt{\dataset-S} with $374$ examples in total.
% }
% \label{tab:diameter_all_combinations}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed   & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed   & Unweighted & 50 & 30--193 \\
% Complete & Connected & Cyclic  & Undirected & Weighted   & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted   & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted   & 50 & 22--197 \\
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Regularity Check (RC)} task statistics in \texttt{\dataset-S} with $257$ examples in total.
% }
% \label{tab:is_regular_unweighted}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed   & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed   & Unweighted & 50 & 30--193 \\
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Distance Regularity Check (DRC)} task statistics in \texttt{\dataset-S} with $208$ examples in total.
% }
% \label{tab:is_distance_regular}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 100 & 19--199 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 8   & 17--38  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 100 & 20--198 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Maximum Flow (MF)} task statistics in \texttt{\dataset-S} with $267$ examples in total.
% }
% \label{tab:maximum_flow}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Weighted & 50 & 22--198 \\
% Sparse   & Connected & Acyclic & Directed   & Weighted & 50 & 45--195 \\
% Dense    & Connected & Acyclic & Directed   & Weighted & 50 & 19--193 \\
% Complete & Connected & Cyclic  & Undirected & Weighted & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted & 50 & 22--197 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Maximum Clique (MC)} task statistics in \texttt{\dataset-S} with $107$ examples in total.
% }
% \label{tab:max_clique}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Maximum Independent Set (MIS)} task statistics in \texttt{\dataset-S} with $107$ examples in total.
% }
% \label{tab:maximum_independent_set}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Clustering Coefficient (CC)} task statistics in \texttt{\dataset-S} with $524$ examples in total.
% }
% \label{tab:clustering_all_combinations}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed   & Weighted   & 50 & 22--198 \\
% Sparse   & Connected & Acyclic & Directed   & Weighted   & 50 & 45--195 \\
% Dense    & Connected & Acyclic & Directed   & Weighted   & 50 & 19--193 \\
% Complete & Connected & Acyclic & Directed   & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed   & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed   & Unweighted & 50 & 30--193 \\
% Complete & Connected & Cyclic  & Undirected & Weighted   & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted   & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted   & 50 & 22--197 \\
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Common Neighbors (CN)} task statistics in \texttt{\dataset-S} with $224$ examples in total.
% }
% \label{tab:common_neighbors}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Weighted   & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted   & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted   & 50 & 22--197 \\
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}





% %%%%%%%%%%%%% large-scale 
\begin{table*}[h]
\centering
\small
\setlength{\tabcolsep}{6pt}
\renewcommand{\arraystretch}{1.15}
\caption{Overview of graph reasoning tasks in the large-scale \texttt{\dataset-L}.}
\label{tab:task_overview}
\resizebox{0.8\linewidth}{!}{
\begin{tabular}{lllll}
\toprule
\textbf{Task} & \textbf{Task Type} & \textbf{\#Examples} & \textbf{\#Nodes} & \textbf{\#Edges} \\
\midrule
Graph Connectivity Check (GCC) 
& Traversal-Based Reasoning 
& 45 
& 200--8{,}000 
& 5{,}400--301{,}158 \\

Connected Components Counting (CCC) 
& Traversal-Based Reasoning 
& 45 
& 200--8{,}000 
& 5{,}400--301{,}158 \\

Strongly Connected Components Counting (SCCC) 
& Traversal-Based Reasoning 
& 45 
& 200--8{,}000 
& 5{,}400--315{,}966 \\

Shortest Path (SP) 
& Traversal-Based Reasoning 
& 54 
& 200--8{,}000 
& 5{,}400--284{,}468 \\
\midrule
Eulerian Path Check (EPC) 
& Graph Property Detection 
& 90 
& 200--8{,}000 
& 5{,}400--315{,}966 \\

Graph Bipartiteness (GB) 
& Graph Property Detection 
& 90 
& 200--8{,}000 
& 5{,}400--315{,}966 \\

Graph Diameter (GD) 
& Graph Property Detection 
& 40 
& 200--8{,}000 
& 10{,}462--301{,}158 \\

Regularity Check (RC) 
& Graph Property Detection 
& 90 
& 200--8{,}000 
& 5{,}400--315{,}966 \\

Distance Regularity Check (DRC) 
& Graph Property Detection 
& 45 
& 200--8{,}000 
& 5{,}400--301{,}158 \\
\midrule
Maximum Flow (MF) 
& Combinatorial Optimization 
& 55 
& 200--8{,}000 
& 5{,}400--284{,}468 \\

Maximum Independent Set (MIS) 
& Combinatorial Optimization 
& 20 
& 200--1{,}800 
& 10{,}462--301{,}158 \\
\midrule
Clustering Coefficient (CC) 
& Neighborhood / Structural Analysis 
& 108 
& 200--8{,}000 
& 10{,}190--315{,}966 \\

Common Neighbors (CN) 
& Neighborhood / Structural Analysis 
& 17 
& 200--1{,}800 
& 12{,}807--301{,}158 \\
\bottomrule
\end{tabular}
}
\end{table*}



% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Graph Connectivity Check (GCC)} task statistics in \texttt{\dataset-L} with $15$ examples in total.
% }
% \label{tab:gcc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Connected Components Counting (CCC)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:ccc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Undirected & Weighted   & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted   & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted   & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Strongly Connected Components Counting (SCCC)} task statistics in \texttt{\dataset-L} with $15$ examples.
% }
% \label{tab:sccc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Shortest Path (SP)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:sp_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Weighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Weighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Weighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Weighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Topological Sorting (TS)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:ts_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Unweighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Eulerian Path Check (EPC)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:epc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Unweighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Graph Bipartiteness (GB)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:gb_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Unweighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Graph Diameter (GD)} task statistics in \texttt{\dataset-L} with $45$ examples.
% }
% \label{tab:gd_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Unweighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Weighted   & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted   & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted   & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Regularity Check (RC)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:rc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Unweighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Distance Regularity Check (DRC)} task statistics in \texttt{\dataset-L} with $15$ examples.
% }
% \label{tab:drc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Maximum Flow (MF)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:mf_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Weighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Weighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Weighted & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Weighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Maximum Clique (MC)} task statistics in \texttt{\dataset-L} with $15$ examples.
% }
% \label{tab:mc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Undirected & Weighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Clustering Coefficient (CC)} task statistics in \texttt{\dataset-L} with $60$ examples.
% }
% \label{tab:cc_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Directed   & Weighted   & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Weighted   & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Weighted   & 5 & 225 \\
% Complete & Connected & Cyclic & Directed   & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Directed   & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Directed   & Unweighted & 5 & 225 \\
% % \midrule
% Complete & Connected & Cyclic & Undirected & Weighted   & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted   & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted   & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Common Neighbors (CN)} task statistics in \texttt{\dataset-L} with $30$ examples.
% }
% \label{tab:cn_all_combinations_L}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic & Undirected & Weighted   & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Weighted   & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Weighted   & 5 & 225 \\
% Complete & Connected & Cyclic & Undirected & Unweighted & 5 & 200 \\
% Sparse   & Connected & Cyclic & Undirected & Unweighted & 5 & 800 \\
% Dense    & Connected & Cyclic & Undirected & Unweighted & 5 & 225 \\
% \bottomrule
% \end{tabular}
% \end{table*}



% %%%% Graph-C %%%%%
\begin{table*}[t]
\centering
\small
\caption{Overview of composite graph reasoning tasks in \texttt{\dataset-C}, including their algorithmic types, and example counts.
% Composite graph reasoning tasks defined by the composition of primitive graph algorithms.
% \fali{change SCC-Maximum\_Flow to CC-Flow-Clustering}
}
\label{tab:composite_tasks}
\resizebox{0.9\linewidth}{!}{%
\begin{tabular}{p{3.2cm} l c c c p{7.8cm}}
\toprule
\textbf{Task Name} & \textbf{Type} & \textbf{\#Examples} & \textbf{\#Nodes} & \textbf{\#Edges} & \textbf{Primitive Tasks} \\
\midrule

Clustering--Shortest Path &
Sequential 
& 572
& 2--200
& 1--19{,}701
& Shortest Path; Clustering Coefficient \\

SCC--Diameter &
Sequential 
& 550
& 4--200
& 2--18{,}336
&Strongly Connected Components; Graph Diameter \\

CC-Flow-Clustering &
Sequential 
& 94
& 2--200
& 1--19{,}701
& Connected Components; Maximum Flow; Clustering Coefficient \\

\midrule

Pair Tightness Score &
Parallel 
& 339
& 3--200
& 1--18{,}915
&Shortest Path; Common Neighbors; Clustering Coefficient \\

Bridge Hub Identification &
Parallel 
& 674
& 2--200
& 1--19{,}701
& Clustering Coefficient; Shortest Path \\

Endpoint-Aware Flow &
Parallel 
& 131
& 8--198
& 5--19{,}503
& Maximum Flow; Clustering Coefficient \\

\midrule

Eulerian Path--Diameter &
Conditional 
& 333
& 2--200
& 1--19{,}701
& Eulerian Path Check; Graph Diameter; Clustering Coefficient \\

Connectivity--Component Diameter &
Conditional 
& 674
& 2--200
& 1--19{,}701
& Graph Connectivity Check; Connected Components; Graph Diameter \\

SCC--Eulerian Check &
Conditional 
& 550
& 4--200
& 2--18{,}336
& Eulerian Path Check; Strongly Connected Components \\

\bottomrule
\end{tabular}%
}
\end{table*}


\begin{dataexample*}{{\dataset}-S and -L Data Example (Shortest Path)}
    \textbf{Task Description:}
    You are given a directed, weighted graph represented by an edge list, where each edge is a triple $(u, v, w)$ indicating a directed connection from node $u$ to node $v$ with weight $w$. Given a source node $\textit{source}$ and a target node $\textit{target}$, your task is to compute the shortest path from $\textit{source}$ to $\textit{target}$. The shortest path is defined as a path that minimizes the sum of edge weights among all possible directed paths connecting the two nodes in the graph. Return only the length of the shortest path as a floating-point value. 

    \textbf{Edge List:}
    $
    \{
    (3,4,5.7),
    (3,1,6.5),
    (3,6,5.5), 
    (3,5,5.2),
    (3,0,8.1),
    (3,2,2.6), 
    \ldots
    \}
    $
    
    \textbf{Arguments:} $\textit{source}=3$, $\textit{target}=5$
    
    \textbf{Answer:} $7.3$ 
\end{dataexample*}


\begin{dataexample*}{\dataset-C Data Example (SCC--Eulerian Check)}
    \textbf{Task Description}:
    You are given a directed, unweighted graph and are asked to compute a \emph{structural complexity score} based on two properties of the graph: the existence of an Eulerian path and the structure of its strongly connected components.
    The graph is provided as an edge list, which is a sequence of ordered pairs $(u, v)$, where each pair represents a directed edge from node $u$ to node $v$. An \emph{Eulerian path} is defined as a directed trail that traverses every edge of the graph exactly once. A \emph{strongly connected component} (SCC) is a maximal set of nodes such that for any two nodes $a$ and $b$ in the set, there exists a directed path from $a$ to $b$ and from $b$ to $a$.
    If the graph admits an Eulerian path, the structural complexity score is defined as the number of nodes in the largest strongly connected component. Otherwise, the score is defined as the total number of strongly connected components in the graph. Return the resulting structural complexity score as an integer.

    \textbf{Edge List:}
    $
    \{\, 
    (0,1),\,
    (0,2),\,
    (0,3),\,
    (0,4),\,
    \ldots
    \,\}
    $
    
    \textbf{Arguments:} None
    
    \textbf{Answer:}$32$ 
\end{dataexample*}

% \begin{table*}[t]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Clustering-Shortest Path} task statistics in \texttt{\dataset-C} with $107$ examples in total.
% }
% \label{tab:highest_clustered_node_shortest_path_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{SCC–Diameter} task statistics in \texttt{\dataset-C} with $150$ examples.
% }
% \label{tab:scc_diameter_all_combinations_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed & Unweighted & 50 & 30--193 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{CC-Flow-Clustering} task statistics in \texttt{\dataset-C} with $117$ examples.
% }
% \label{tab:scc_mf_all_combinations_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Weighted & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted & 50 & 22--197 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[t]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Pair Tightness Score} task statistics in \texttt{\dataset-C} with $117$ examples in total.
% }
% \label{tab:pair_tightness_score_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Weighted & 50 & 22--194 \\
% Sparse   & Connected & Acyclic & Undirected & Weighted & 17 & 9--35   \\
% Dense    & Connected & Cyclic  & Undirected & Weighted & 50 & 22--197 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Bridge Hub Identification} task statistics in \texttt{\dataset-C} with $107$ examples.
% }
% \label{tab:bridge_hub_identification_all_combinations_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[t]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Endpoint-Aware Flow} task statistics in \texttt{\dataset-C} with $150$ examples in total.
% }
% \label{tab:endpoint_aware_flow_score}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed & Weighted & 50 & 22--198 \\
% Sparse   & Connected & Acyclic & Directed & Weighted & 50 & 45--195 \\
% Dense    & Connected & Acyclic & Directed & Weighted & 50 & 19--193 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Eulerian Path–Diameter} task statistics in \texttt{\dataset-C} with $107$ examples.
% }
% \label{tab:eulerian_path_shortest_path_all_combinations_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{Connectivity-Component Diameter} task statistics in \texttt{\dataset-C} with $107$ examples in total.
% }
% \label{tab:largest_component_and_diameter_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Cyclic  & Undirected & Unweighted & 50 & 20--198 \\
% Sparse   & Connected & Acyclic & Undirected & Unweighted & 7  & 14--27  \\
% Dense    & Connected & Cyclic  & Undirected & Unweighted & 50 & 23--195 \\
% \bottomrule
% \end{tabular}
% \end{table*}

% \begin{table*}[h]
% \centering
% \small 
% \setlength{\tabcolsep}{3.5pt}
% \renewcommand{\arraystretch}{1.05}
% \caption{
% \textbf{SCC–Eulerian Check} task statistics in \texttt{\dataset-C} with $150$ examples.
% }
% \label{tab:scc_eulerian_feasibility_all_combinations_C}
% \begin{tabular}{lllllll}
% \toprule
% \textbf{Density} & \textbf{Connectivity} & \textbf{Cyclicity} & \textbf{Direction} & \textbf{Weights} & \textbf{\#Examples} & \textbf{\#Nodes} \\
% \midrule
% Complete & Connected & Acyclic & Directed & Unweighted & 50 & 20--191 \\
% Sparse   & Connected & Acyclic & Directed & Unweighted & 50 & 50--188 \\
% Dense    & Connected & Acyclic & Directed & Unweighted & 50 & 30--193 \\
% \bottomrule
% \end{tabular}
% \end{table*}



\end{document}
\endinput
%%
%% End of file `sample-sigconf.tex'.

%%% -*-BibTeX-*-
%%% Do NOT edit. File created by BibTeX with style
%%% ACM-Reference-Format-Journals [18-Jan-2012].

\begin{thebibliography}{49}

%%% ====================================================================
%%% NOTE TO THE USER: you can override these defaults by providing
%%% customized versions of any of these macros before the \bibliography
%%% command.  Each of them MUST provide its own final punctuation,
%%% except for \shownote{} and \showURL{}.  The latter two
%%% do not use final punctuation, in order to avoid confusing it with
%%% the Web address.
%%%
%%% To suppress output of a particular field, define its macro to expand
%%% to an empty string, or better, \unskip, like this:
%%%
%%% \newcommand{\showURL}[1]{\unskip}   % LaTeX syntax
%%%
%%% \def \showURL #1{\unskip}           % plain TeX syntax
%%%
%%% ====================================================================

\ifx \showCODEN    \undefined \def \showCODEN     #1{\unskip}     \fi
\ifx \showISBNx    \undefined \def \showISBNx     #1{\unskip}     \fi
\ifx \showISBNxiii \undefined \def \showISBNxiii  #1{\unskip}     \fi
\ifx \showISSN     \undefined \def \showISSN      #1{\unskip}     \fi
\ifx \showLCCN     \undefined \def \showLCCN      #1{\unskip}     \fi
\ifx \shownote     \undefined \def \shownote      #1{#1}          \fi
\ifx \showarticletitle \undefined \def \showarticletitle #1{#1}   \fi
\ifx \showURL      \undefined \def \showURL       {\relax}        \fi
% The following commands are used for tagged output and should be
% invisible to TeX
\providecommand\bibfield[2]{#2}
\providecommand\bibinfo[2]{#2}
\providecommand\natexlab[1]{#1}
\providecommand\showeprint[2][]{arXiv:#2}

\bibitem[Bai et~al\mbox{.}(2023)]%
        {bai2023qwentechnicalreport}
\bibfield{author}{\bibinfo{person}{Jinze Bai}, \bibinfo{person}{Shuai Bai}, \bibinfo{person}{Yunfei Chu}, \bibinfo{person}{Zeyu Cui}, \bibinfo{person}{Kai Dang}, \bibinfo{person}{Xiaodong Deng}, \bibinfo{person}{Yang Fan}, \bibinfo{person}{Wenbin Ge}, \bibinfo{person}{Yu Han}, \bibinfo{person}{Fei Huang}, \bibinfo{person}{Binyuan Hui}, \bibinfo{person}{Luo Ji}, \bibinfo{person}{Mei Li}, \bibinfo{person}{Junyang Lin}, \bibinfo{person}{Runji Lin}, \bibinfo{person}{Dayiheng Liu}, \bibinfo{person}{Gao Liu}, \bibinfo{person}{Chengqiang Lu}, \bibinfo{person}{Keming Lu}, \bibinfo{person}{Jianxin Ma}, \bibinfo{person}{Rui Men}, \bibinfo{person}{Xingzhang Ren}, \bibinfo{person}{Xuancheng Ren}, \bibinfo{person}{Chuanqi Tan}, \bibinfo{person}{Sinan Tan}, \bibinfo{person}{Jianhong Tu}, \bibinfo{person}{Peng Wang}, \bibinfo{person}{Shijie Wang}, \bibinfo{person}{Wei Wang}, \bibinfo{person}{Shengguang Wu}, \bibinfo{person}{Benfeng Xu}, \bibinfo{person}{Jin Xu}, \bibinfo{person}{An Yang}, \bibinfo{person}{Hao Yang},
  \bibinfo{person}{Jian Yang}, \bibinfo{person}{Shusheng Yang}, \bibinfo{person}{Yang Yao}, \bibinfo{person}{Bowen Yu}, \bibinfo{person}{Hongyi Yuan}, \bibinfo{person}{Zheng Yuan}, \bibinfo{person}{Jianwei Zhang}, \bibinfo{person}{Xingxuan Zhang}, \bibinfo{person}{Yichang Zhang}, \bibinfo{person}{Zhenru Zhang}, \bibinfo{person}{Chang Zhou}, \bibinfo{person}{Jingren Zhou}, \bibinfo{person}{Xiaohuan Zhou}, {and} \bibinfo{person}{Tianhang Zhu}.} \bibinfo{year}{2023}\natexlab{}.
\newblock \bibinfo{title}{Qwen Technical Report}.
\newblock
\showeprint[arxiv]{2309.16609}~[cs.CL]
\urldef\tempurl%
\url{https://arxiv.org/abs/2309.16609}
\showURL{%
\tempurl}


\bibitem[Brown et~al\mbox{.}(2024)]%
        {brown2024large}
\bibfield{author}{\bibinfo{person}{Bradley Brown}, \bibinfo{person}{Jordan Juravsky}, \bibinfo{person}{Ryan Ehrlich}, \bibinfo{person}{Ronald Clark}, \bibinfo{person}{Quoc~V Le}, \bibinfo{person}{Christopher R{\'e}}, {and} \bibinfo{person}{Azalia Mirhoseini}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Large language monkeys: Scaling inference compute with repeated sampling}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2407.21787}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Brown et~al\mbox{.}(2020)]%
        {brown2020language}
\bibfield{author}{\bibinfo{person}{Tom Brown}, \bibinfo{person}{Benjamin Mann}, \bibinfo{person}{Nick Ryder}, \bibinfo{person}{Melanie Subbiah}, \bibinfo{person}{Jared~D Kaplan}, \bibinfo{person}{Prafulla Dhariwal}, \bibinfo{person}{Arvind Neelakantan}, \bibinfo{person}{Pranav Shyam}, \bibinfo{person}{Girish Sastry}, \bibinfo{person}{Amanda Askell}, {et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Language models are few-shot learners}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}}  \bibinfo{volume}{33} (\bibinfo{year}{2020}), \bibinfo{pages}{1877--1901}.
\newblock


\bibitem[Cai et~al\mbox{.}(2024)]%
        {cai2024codegraphenhancinggraphreasoning}
\bibfield{author}{\bibinfo{person}{Qiaolong Cai}, \bibinfo{person}{Zhaowei Wang}, \bibinfo{person}{Shizhe Diao}, \bibinfo{person}{James Kwok}, {and} \bibinfo{person}{Yangqiu Song}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \bibinfo{title}{CodeGraph: Enhancing Graph Reasoning of LLMs with Code}.
\newblock
\showeprint[arxiv]{2408.13863}~[cs.CL]
\urldef\tempurl%
\url{https://arxiv.org/abs/2408.13863}
\showURL{%
\tempurl}


\bibitem[Chai et~al\mbox{.}(2025)]%
        {chai2025graphllm}
\bibfield{author}{\bibinfo{person}{Ziwei Chai}, \bibinfo{person}{Tianjie Zhang}, \bibinfo{person}{Liang Wu}, \bibinfo{person}{Kaiqiang Han}, \bibinfo{person}{Xiaohai Hu}, \bibinfo{person}{Xuanwen Huang}, {and} \bibinfo{person}{Yang Yang}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Graphllm: Boosting graph reasoning ability of large language model}.
\newblock \bibinfo{journal}{\emph{IEEE Transactions on Big Data}} (\bibinfo{year}{2025}).
\newblock


\bibitem[Chen et~al\mbox{.}(2024a)]%
        {chen2024graphwizinstructionfollowinglanguagemodel}
\bibfield{author}{\bibinfo{person}{Nuo Chen}, \bibinfo{person}{Yuhan Li}, \bibinfo{person}{Jianheng Tang}, {and} \bibinfo{person}{Jia Li}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Graphwiz: An instruction-following language model for graph computational problems}. In \bibinfo{booktitle}{\emph{Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}}. \bibinfo{pages}{353--364}.
\newblock


\bibitem[Chen et~al\mbox{.}(2024b)]%
        {chen2024labelfree}
\bibfield{author}{\bibinfo{person}{Zhikai Chen}, \bibinfo{person}{Haitao Mao}, \bibinfo{person}{Hongzhi Wen}, \bibinfo{person}{Haoyu Han}, \bibinfo{person}{Wei Jin}, \bibinfo{person}{Haiyang Zhang}, \bibinfo{person}{Hui Liu}, {and} \bibinfo{person}{Jiliang Tang}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Label-free Node Classification on Graphs with Large Language Models ({LLM}s)}. In \bibinfo{booktitle}{\emph{The Twelfth International Conference on Learning Representations}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=hESD2NJFg8}
\showURL{%
\tempurl}


\bibitem[Dubey et~al\mbox{.}(2024)]%
        {dubey2024llama}
\bibfield{author}{\bibinfo{person}{Abhimanyu Dubey}, \bibinfo{person}{Abhinav Jauhri}, \bibinfo{person}{Abhinav Pandey}, \bibinfo{person}{Abhishek Kadian}, \bibinfo{person}{Ahmad Al-Dahle}, \bibinfo{person}{Aiesha Letman}, \bibinfo{person}{Akhil Mathur}, \bibinfo{person}{Alan Schelten}, \bibinfo{person}{Amy Yang}, \bibinfo{person}{Angela Fan}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{The llama 3 herd of models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2407.21783}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Fatemi et~al\mbox{.}(2024)]%
        {fatemi2023talklikegraphencoding}
\bibfield{author}{\bibinfo{person}{Bahare Fatemi}, \bibinfo{person}{Jonathan Halcrow}, {and} \bibinfo{person}{Bryan Perozzi}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Talk like a Graph: Encoding Graphs for Large Language Models}. In \bibinfo{booktitle}{\emph{The Twelfth International Conference on Learning Representations}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=IuXR1CCrSi}
\showURL{%
\tempurl}


\bibitem[Girvan and Newman(2002)]%
        {girvan2002community}
\bibfield{author}{\bibinfo{person}{Michelle Girvan} {and} \bibinfo{person}{Mark~EJ Newman}.} \bibinfo{year}{2002}\natexlab{}.
\newblock \showarticletitle{Community structure in social and biological networks}.
\newblock \bibinfo{journal}{\emph{Proceedings of the national academy of sciences}} \bibinfo{volume}{99}, \bibinfo{number}{12} (\bibinfo{year}{2002}), \bibinfo{pages}{7821--7826}.
\newblock


\bibitem[Gong et~al\mbox{.}(2025)]%
        {gong2025pseudocodeinjectionmagicenablingllms}
\bibfield{author}{\bibinfo{person}{Chang Gong}, \bibinfo{person}{Wanrui Bian}, \bibinfo{person}{Zhijie Zhang}, {and} \bibinfo{person}{Weiguo Zheng}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{Pseudocode-Injection Magic: Enabling LLMs to Tackle Graph Computational Tasks}.
\newblock
\showeprint[arxiv]{2501.13731}~[cs.CL]
\urldef\tempurl%
\url{https://arxiv.org/abs/2501.13731}
\showURL{%
\tempurl}


\bibitem[Hagberg et~al\mbox{.}(2008)]%
        {hagberg2008exploring}
\bibfield{author}{\bibinfo{person}{Aric~A Hagberg}, \bibinfo{person}{Daniel~A Schult}, {and} \bibinfo{person}{Pieter~J Swart}.} \bibinfo{year}{2008}\natexlab{}.
\newblock \showarticletitle{Exploring Network Structure, Dynamics, and Function using NetworkX}. In \bibinfo{booktitle}{\emph{Proceedings of the Python in Science Conference}}. SciPy, \bibinfo{pages}{11--15}.
\newblock


\bibitem[Harris et~al\mbox{.}(2020)]%
        {harris2020array}
\bibfield{author}{\bibinfo{person}{Charles~R Harris}, \bibinfo{person}{K~Jarrod Millman}, \bibinfo{person}{St{\'e}fan~J Van Der~Walt}, \bibinfo{person}{Ralf Gommers}, \bibinfo{person}{Pauli Virtanen}, \bibinfo{person}{David Cournapeau}, \bibinfo{person}{Eric Wieser}, \bibinfo{person}{Julian Taylor}, \bibinfo{person}{Sebastian Berg}, \bibinfo{person}{Nathaniel~J Smith}, {et~al\mbox{.}}} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{Array programming with NumPy}.
\newblock \bibinfo{journal}{\emph{nature}} \bibinfo{volume}{585}, \bibinfo{number}{7825} (\bibinfo{year}{2020}), \bibinfo{pages}{357--362}.
\newblock


\bibitem[Hart et~al\mbox{.}(1968)]%
        {hart1968formal}
\bibfield{author}{\bibinfo{person}{Peter~E Hart}, \bibinfo{person}{Nils~J Nilsson}, {and} \bibinfo{person}{Bertram Raphael}.} \bibinfo{year}{1968}\natexlab{}.
\newblock \showarticletitle{A formal basis for the heuristic determination of minimum cost paths}.
\newblock \bibinfo{journal}{\emph{IEEE transactions on Systems Science and Cybernetics}} \bibinfo{volume}{4}, \bibinfo{number}{2} (\bibinfo{year}{1968}), \bibinfo{pages}{100--107}.
\newblock


\bibitem[Le et~al\mbox{.}(2024)]%
        {le2024molx}
\bibfield{author}{\bibinfo{person}{Khiem Le}, \bibinfo{person}{Zhichun Guo}, \bibinfo{person}{Kaiwen Dong}, \bibinfo{person}{Xiaobao Huang}, \bibinfo{person}{Bozhao Nan}, \bibinfo{person}{Roshni Iyer}, \bibinfo{person}{Xiangliang Zhang}, \bibinfo{person}{Olaf Wiest}, \bibinfo{person}{Wei Wang}, {and} \bibinfo{person}{Nitesh~V Chawla}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Molx: Enhancing large language models for molecular learning with a multi-modal extension}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2406.06777}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Li et~al\mbox{.}(2025)]%
        {li2025graphteamfacilitatinglargelanguage}
\bibfield{author}{\bibinfo{person}{Xin~Sky Li}, \bibinfo{person}{Qizhi Chu}, \bibinfo{person}{Yubin Chen}, \bibinfo{person}{Yang Liu}, \bibinfo{person}{Yaoqi Liu}, \bibinfo{person}{Zekai Yu}, \bibinfo{person}{Weize Chen}, \bibinfo{person}{Chen Qian}, \bibinfo{person}{Chuan Shi}, {and} \bibinfo{person}{Cheng Yang}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \bibinfo{title}{GraphTeam: Facilitating Large Language Model-based Graph Analysis via Multi-Agent Collaboration}.
\newblock
\showeprint[arxiv]{2410.18032}~[cs.AI]
\urldef\tempurl%
\url{https://arxiv.org/abs/2410.18032}
\showURL{%
\tempurl}


\bibitem[Liu et~al\mbox{.}(2024)]%
        {liu2024deepseek}
\bibfield{author}{\bibinfo{person}{Aixin Liu}, \bibinfo{person}{Bei Feng}, \bibinfo{person}{Bing Xue}, \bibinfo{person}{Bingxuan Wang}, \bibinfo{person}{Bochao Wu}, \bibinfo{person}{Chengda Lu}, \bibinfo{person}{Chenggang Zhao}, \bibinfo{person}{Chengqi Deng}, \bibinfo{person}{Chenyu Zhang}, \bibinfo{person}{Chong Ruan}, {et~al\mbox{.}}} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Deepseek-v3 technical report}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2412.19437}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Liu.(2022)]%
        {llamaindexweb}
\bibfield{author}{\bibinfo{person}{Jerry Liu.}} \bibinfo{year}{2022}\natexlab{}.
\newblock \bibinfo{title}{Llama Index}.
\newblock
\urldef\tempurl%
\url{https://github.com/jerryjliu/llama_index}
\showURL{%
\tempurl}
\newblock
\shownote{Accessed: 2025-11-09}.


\bibitem[Liu(2022)]%
        {Liu_LlamaIndex_2022}
\bibfield{author}{\bibinfo{person}{Jerry Liu}.} \bibinfo{year}{2022}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{{LlamaIndex}}}.
\newblock
\href{https://doi.org/10.5281/zenodo.1234}{doi:\nolinkurl{10.5281/zenodo.1234}}


\bibitem[Liu et~al\mbox{.}(2025)]%
        {liu2025hm}
\bibfield{author}{\bibinfo{person}{Pei Liu}, \bibinfo{person}{Xin Liu}, \bibinfo{person}{Ruoyu Yao}, \bibinfo{person}{Junming Liu}, \bibinfo{person}{Siyuan Meng}, \bibinfo{person}{Ding Wang}, {and} \bibinfo{person}{Jun Ma}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Hm-rag: Hierarchical multi-agent multimodal retrieval augmented generation}. In \bibinfo{booktitle}{\emph{Proceedings of the 33rd ACM International Conference on Multimedia}}. \bibinfo{pages}{2781--2790}.
\newblock


\bibitem[Luo et~al\mbox{.}(2024b)]%
        {luo2024enhance}
\bibfield{author}{\bibinfo{person}{Haitong Luo}, \bibinfo{person}{Xuying Meng}, \bibinfo{person}{Suhang Wang}, \bibinfo{person}{Tianxiang Zhao}, \bibinfo{person}{Fali Wang}, \bibinfo{person}{Hanyun Cao}, {and} \bibinfo{person}{Yujun Zhang}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{Enhance graph alignment for large language models}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2410.11370}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Luo et~al\mbox{.}(2024a)]%
        {luo2024reasoning}
\bibfield{author}{\bibinfo{person}{Linhao Luo}, \bibinfo{person}{Yuan-Fang Li}, \bibinfo{person}{Gholamreza Haffari}, {and} \bibinfo{person}{Shirui Pan}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Reasoning on Graphs: Faithful and Interpretable Large Language Model Reasoning}. In \bibinfo{booktitle}{\emph{The Twelfth International Conference on Learning Representations}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=ZGNWW7xZ6Q}
\showURL{%
\tempurl}


\bibitem[Luo et~al\mbox{.}(2024c)]%
        {luo2024graphinstruct}
\bibfield{author}{\bibinfo{person}{Zihan Luo}, \bibinfo{person}{Xiran Song}, \bibinfo{person}{Hong Huang}, \bibinfo{person}{Jianxun Lian}, \bibinfo{person}{Chenhao Zhang}, \bibinfo{person}{Jinqi Jiang}, \bibinfo{person}{Xing Xie}, {and} \bibinfo{person}{Hai Jin}.} \bibinfo{year}{2024}\natexlab{c}.
\newblock \showarticletitle{Graphinstruct: Empowering large language models with graph understanding and reasoning capability}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2403.04483}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Neelakantan et~al\mbox{.}(2022)]%
        {textembeddingada}
\bibfield{author}{\bibinfo{person}{Arvind Neelakantan}, \bibinfo{person}{Tao Xu}, \bibinfo{person}{Raul Puri}, \bibinfo{person}{Alec Radford}, \bibinfo{person}{Jesse~Michael Han}, \bibinfo{person}{Jerry Tworek}, \bibinfo{person}{Qiming Yuan}, \bibinfo{person}{Nikolas Tezak}, \bibinfo{person}{Jong~Wook Kim}, \bibinfo{person}{Chris Hallacy}, {et~al\mbox{.}}} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Text and code embeddings by contrastive pre-training}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2201.10005}} (\bibinfo{year}{2022}).
\newblock


\bibitem[Page et~al\mbox{.}(1999)]%
        {page1999pagerank}
\bibfield{author}{\bibinfo{person}{Lawrence Page}, \bibinfo{person}{Sergey Brin}, \bibinfo{person}{Rajeev Motwani}, {and} \bibinfo{person}{Terry Winograd}.} \bibinfo{year}{1999}\natexlab{}.
\newblock \bibinfo{booktitle}{\emph{The PageRank citation ranking: Bringing order to the web.}}
\newblock \bibinfo{type}{{T}echnical {R}eport}. \bibinfo{institution}{Stanford infolab}.
\newblock


\bibitem[Paszke et~al\mbox{.}(2019)]%
        {paszke2019pytorch}
\bibfield{author}{\bibinfo{person}{Adam Paszke}, \bibinfo{person}{Sam Gross}, \bibinfo{person}{Francisco Massa}, \bibinfo{person}{Adam Lerer}, \bibinfo{person}{James Bradbury}, \bibinfo{person}{Gregory Chanan}, \bibinfo{person}{Trevor Killeen}, \bibinfo{person}{Zeming Lin}, \bibinfo{person}{Natalia Gimelshein}, \bibinfo{person}{Luca Antiga}, {et~al\mbox{.}}} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Pytorch: An imperative style, high-performance deep learning library}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}}  \bibinfo{volume}{32} (\bibinfo{year}{2019}).
\newblock


\bibitem[Perozzi et~al\mbox{.}(2024)]%
        {perozzi2024let}
\bibfield{author}{\bibinfo{person}{Bryan Perozzi}, \bibinfo{person}{Bahare Fatemi}, \bibinfo{person}{Dustin Zelle}, \bibinfo{person}{Anton Tsitsulin}, \bibinfo{person}{Mehran Kazemi}, \bibinfo{person}{Rami Al-Rfou}, {and} \bibinfo{person}{Jonathan Halcrow}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Let your graph do the talking: Encoding structured data for llms}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2402.05862}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Ravuru et~al\mbox{.}(2024)]%
        {ravuru2024agentic}
\bibfield{author}{\bibinfo{person}{Chidaksh Ravuru}, \bibinfo{person}{Sagar~Srinivas Sakhinana}, {and} \bibinfo{person}{Venkataramana Runkana}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Agentic retrieval-augmented generation for time series analysis}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2408.14484}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Reimers and Gurevych(2019)]%
        {reimers2019sentence}
\bibfield{author}{\bibinfo{person}{Nils Reimers} {and} \bibinfo{person}{Iryna Gurevych}.} \bibinfo{year}{2019}\natexlab{}.
\newblock \showarticletitle{Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks}. In \bibinfo{booktitle}{\emph{Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)}}. \bibinfo{pages}{3982--3992}.
\newblock


\bibitem[Salton and Buckley(1988)]%
        {salton1988term}
\bibfield{author}{\bibinfo{person}{Gerard Salton} {and} \bibinfo{person}{Christopher Buckley}.} \bibinfo{year}{1988}\natexlab{}.
\newblock \showarticletitle{Term-weighting approaches in automatic text retrieval}.
\newblock \bibinfo{journal}{\emph{Information processing \& management}} \bibinfo{volume}{24}, \bibinfo{number}{5} (\bibinfo{year}{1988}), \bibinfo{pages}{513--523}.
\newblock


\bibitem[Shen et~al\mbox{.}(2025)]%
        {shen2025gear}
\bibfield{author}{\bibinfo{person}{Zhili Shen}, \bibinfo{person}{Chenxin Diao}, \bibinfo{person}{Pavlos Vougiouklis}, \bibinfo{person}{Pascual Merita}, \bibinfo{person}{Shriram Piramanayagam}, \bibinfo{person}{Enting Chen}, \bibinfo{person}{Damien Graux}, \bibinfo{person}{Andre Melo}, \bibinfo{person}{Ruofei Lai}, \bibinfo{person}{Zeren Jiang}, {et~al\mbox{.}}} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Gear: Graph-enhanced agent for retrieval-augmented generation}. In \bibinfo{booktitle}{\emph{Findings of the Association for Computational Linguistics: ACL 2025}}. \bibinfo{pages}{12049--12072}.
\newblock


\bibitem[Snell et~al\mbox{.}(2025)]%
        {snell2025scaling}
\bibfield{author}{\bibinfo{person}{Charlie~Victor Snell}, \bibinfo{person}{Jaehoon Lee}, \bibinfo{person}{Kelvin Xu}, {and} \bibinfo{person}{Aviral Kumar}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{Scaling {LLM} Test-Time Compute Optimally Can be More Effective than Scaling Parameters for Reasoning}. In \bibinfo{booktitle}{\emph{The Thirteenth International Conference on Learning Representations}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=4FWAwZtd2n}
\showURL{%
\tempurl}


\bibitem[Tang et~al\mbox{.}(2024)]%
        {tang2024graphgpt}
\bibfield{author}{\bibinfo{person}{Jiabin Tang}, \bibinfo{person}{Yuhao Yang}, \bibinfo{person}{Wei Wei}, \bibinfo{person}{Lei Shi}, \bibinfo{person}{Lixin Su}, \bibinfo{person}{Suqi Cheng}, \bibinfo{person}{Dawei Yin}, {and} \bibinfo{person}{Chao Huang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Graphgpt: Graph instruction tuning for large language models}. In \bibinfo{booktitle}{\emph{Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval}}. \bibinfo{pages}{491--500}.
\newblock


\bibitem[Tang et~al\mbox{.}(2025)]%
        {tang2025grapharena}
\bibfield{author}{\bibinfo{person}{Jianheng Tang}, \bibinfo{person}{Qifan Zhang}, \bibinfo{person}{Yuhan Li}, \bibinfo{person}{Nuo Chen}, {and} \bibinfo{person}{Jia Li}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{GraphArena: Evaluating and Exploring Large Language Models on Graph Computation}. In \bibinfo{booktitle}{\emph{The Thirteenth International Conference on Learning Representations}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=Y1r9yCMzeA}
\showURL{%
\tempurl}


\bibitem[Wang et~al\mbox{.}(2025a)]%
        {wang2025generalizing}
\bibfield{author}{\bibinfo{person}{Fali Wang}, \bibinfo{person}{Jihai Chen}, \bibinfo{person}{Shuhua Yang}, \bibinfo{person}{Runxue Bao}, \bibinfo{person}{Tianxiang Zhao}, \bibinfo{person}{Zhiwei Zhang}, \bibinfo{person}{Xianfeng Tang}, \bibinfo{person}{Hui Liu}, \bibinfo{person}{Qi He}, {and} \bibinfo{person}{Suhang Wang}.} \bibinfo{year}{2025}\natexlab{a}.
\newblock \showarticletitle{Generalizing Test-time Compute-optimal Scaling as an Optimizable Graph}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2511.00086}} (\bibinfo{year}{2025}).
\newblock


\bibitem[Wang et~al\mbox{.}(2025c)]%
        {wang2025agenttts}
\bibfield{author}{\bibinfo{person}{Fali Wang}, \bibinfo{person}{Hui Liu}, \bibinfo{person}{Zhenwei DAI}, \bibinfo{person}{Jingying Zeng}, \bibinfo{person}{Zhiwei Zhang}, \bibinfo{person}{Zongyu Wu}, \bibinfo{person}{Chen Luo}, \bibinfo{person}{Zhen Li}, \bibinfo{person}{Xianfeng Tang}, \bibinfo{person}{Qi He}, {and} \bibinfo{person}{Suhang Wang}.} \bibinfo{year}{2025}\natexlab{c}.
\newblock \showarticletitle{Agent{TTS}: Large Language Model Agent for Test-time Compute-optimal Scaling Strategy in Complex Tasks}. In \bibinfo{booktitle}{\emph{The Thirty-ninth Annual Conference on Neural Information Processing Systems}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=BuYtcTUMyA}
\showURL{%
\tempurl}


\bibitem[Wang et~al\mbox{.}(2024)]%
        {wang2024hc}
\bibfield{author}{\bibinfo{person}{Fali Wang}, \bibinfo{person}{Tianxiang Zhao}, \bibinfo{person}{Junjie Xu}, {and} \bibinfo{person}{Suhang Wang}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Hc-gst: Heterophily-aware distribution consistency based graph self-training}. In \bibinfo{booktitle}{\emph{Proceedings of the 33rd ACM International Conference on Information and Knowledge Management}}. \bibinfo{pages}{2326--2335}.
\newblock


\bibitem[Wang et~al\mbox{.}(2023a)]%
        {wang2024languagemodelssolvegraph}
\bibfield{author}{\bibinfo{person}{Heng Wang}, \bibinfo{person}{Shangbin Feng}, \bibinfo{person}{Tianxing He}, \bibinfo{person}{Zhaoxuan Tan}, \bibinfo{person}{Xiaochuang Han}, {and} \bibinfo{person}{Yulia Tsvetkov}.} \bibinfo{year}{2023}\natexlab{a}.
\newblock \showarticletitle{Can language models solve graph problems in natural language?}
\newblock \bibinfo{journal}{\emph{Advances in Neural Information Processing Systems}}  \bibinfo{volume}{36} (\bibinfo{year}{2023}), \bibinfo{pages}{30840--30861}.
\newblock


\bibitem[Wang et~al\mbox{.}(2025b)]%
        {wang2025GraphToolInstruction}
\bibfield{author}{\bibinfo{person}{Rongzheng Wang}, \bibinfo{person}{Shuang Liang}, \bibinfo{person}{Qizhi Chen}, \bibinfo{person}{Jiasheng Zhang}, {and} \bibinfo{person}{Ke Qin}.} \bibinfo{year}{2025}\natexlab{b}.
\newblock \showarticletitle{GraphTool-Instruction: Revolutionizing Graph Reasoning in LLMs through Decomposed Subtask Instruction}. In \bibinfo{booktitle}{\emph{Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.1}} (Toronto ON, Canada) \emph{(\bibinfo{series}{KDD '25})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{1492–1503}.
\newblock
\showISBNx{9798400712456}
\href{https://doi.org/10.1145/3690624.3709238}{doi:\nolinkurl{10.1145/3690624.3709238}}


\bibitem[Wang et~al\mbox{.}(2023b)]%
        {wang2023selfconsistency}
\bibfield{author}{\bibinfo{person}{Xuezhi Wang}, \bibinfo{person}{Jason Wei}, \bibinfo{person}{Dale Schuurmans}, \bibinfo{person}{Quoc~V Le}, \bibinfo{person}{Ed~H. Chi}, \bibinfo{person}{Sharan Narang}, \bibinfo{person}{Aakanksha Chowdhery}, {and} \bibinfo{person}{Denny Zhou}.} \bibinfo{year}{2023}\natexlab{b}.
\newblock \showarticletitle{Self-Consistency Improves Chain of Thought Reasoning in Language Models}. In \bibinfo{booktitle}{\emph{The Eleventh International Conference on Learning Representations}}.
\newblock
\urldef\tempurl%
\url{https://openreview.net/forum?id=1PL1NIMMrw}
\showURL{%
\tempurl}


\bibitem[Wei et~al\mbox{.}(2022)]%
        {wei2022chain}
\bibfield{author}{\bibinfo{person}{Jason Wei}, \bibinfo{person}{Xuezhi Wang}, \bibinfo{person}{Dale Schuurmans}, \bibinfo{person}{Maarten Bosma}, \bibinfo{person}{Fei Xia}, \bibinfo{person}{Ed Chi}, \bibinfo{person}{Quoc~V Le}, \bibinfo{person}{Denny Zhou}, {et~al\mbox{.}}} \bibinfo{year}{2022}\natexlab{}.
\newblock \showarticletitle{Chain-of-thought prompting elicits reasoning in large language models}.
\newblock \bibinfo{journal}{\emph{Advances in neural information processing systems}}  \bibinfo{volume}{35} (\bibinfo{year}{2022}), \bibinfo{pages}{24824--24837}.
\newblock


\bibitem[Wu et~al\mbox{.}(2025a)]%
        {wu-etal-2025-grapheval36k}
\bibfield{author}{\bibinfo{person}{Qiming Wu}, \bibinfo{person}{Zichen Chen}, \bibinfo{person}{Will Corcoran}, \bibinfo{person}{Misha Sra}, {and} \bibinfo{person}{Ambuj Singh}.} \bibinfo{year}{2025}\natexlab{a}.
\newblock \showarticletitle{{G}raph{E}val36{K}: Benchmarking Coding and Reasoning Capabilities of Large Language Models on Graph Datasets}. In \bibinfo{booktitle}{\emph{Findings of the Association for Computational Linguistics: NAACL 2025}}, \bibfield{editor}{\bibinfo{person}{Luis Chiruzzo}, \bibinfo{person}{Alan Ritter}, {and} \bibinfo{person}{Lu~Wang}} (Eds.). \bibinfo{publisher}{Association for Computational Linguistics}, \bibinfo{address}{Albuquerque, New Mexico}, \bibinfo{pages}{8095--8117}.
\newblock
\showISBNx{979-8-89176-195-7}
\href{https://doi.org/10.18653/v1/2025.findings-naacl.452}{doi:\nolinkurl{10.18653/v1/2025.findings-naacl.452}}


\bibitem[Wu et~al\mbox{.}(2025b)]%
        {wu2025inference}
\bibfield{author}{\bibinfo{person}{Yangzhen Wu}, \bibinfo{person}{Zhiqing Sun}, \bibinfo{person}{Shanda Li}, \bibinfo{person}{Sean Welleck}, {and} \bibinfo{person}{Yiming Yang}.} \bibinfo{year}{2025}\natexlab{b}.
\newblock \showarticletitle{Inference scaling laws: An empirical analysis of compute-optimal inference for LLM problem-solving}. In \bibinfo{booktitle}{\emph{The Thirteenth International Conference on Learning Representations}}.
\newblock


\bibitem[Xia et~al\mbox{.}(2021)]%
        {xia2021graph}
\bibfield{author}{\bibinfo{person}{Feng Xia}, \bibinfo{person}{Ke Sun}, \bibinfo{person}{Shuo Yu}, \bibinfo{person}{Abdul Aziz}, \bibinfo{person}{Liangtian Wan}, \bibinfo{person}{Shirui Pan}, {and} \bibinfo{person}{Huan Liu}.} \bibinfo{year}{2021}\natexlab{}.
\newblock \showarticletitle{Graph learning: A survey}.
\newblock \bibinfo{journal}{\emph{IEEE Transactions on Artificial Intelligence}} \bibinfo{volume}{2}, \bibinfo{number}{2} (\bibinfo{year}{2021}), \bibinfo{pages}{109--127}.
\newblock


\bibitem[Yan et~al\mbox{.}(2024)]%
        {yan2024corrective}
\bibfield{author}{\bibinfo{person}{Shi-Qi Yan}, \bibinfo{person}{Jia-Chen Gu}, \bibinfo{person}{Yun Zhu}, {and} \bibinfo{person}{Zhen-Hua Ling}.} \bibinfo{year}{2024}\natexlab{}.
\newblock \showarticletitle{Corrective Retrieval Augmented Generation}.
\newblock \bibinfo{journal}{\emph{arXiv preprint arXiv:2401.15884}} (\bibinfo{year}{2024}).
\newblock


\bibitem[Yang et~al\mbox{.}(2020)]%
        {yang2020nargnn}
\bibfield{author}{\bibinfo{person}{Shuang Yang}, \bibinfo{person}{Fali Wang}, \bibinfo{person}{Daren Zha}, \bibinfo{person}{Cong Xue}, {and} \bibinfo{person}{Zhihao Tang}.} \bibinfo{year}{2020}\natexlab{}.
\newblock \showarticletitle{NarGNN: Narrative graph neural networks for new script event prediction problem}. In \bibinfo{booktitle}{\emph{2020 IEEE Intl Conf on Parallel \& Distributed Processing with Applications, Big Data \& Cloud Computing, Sustainable Computing \& Communications, Social Computing \& Networking (ISPA/BDCloud/SocialCom/SustainCom)}}. IEEE, \bibinfo{pages}{481--488}.
\newblock


\bibitem[Zhang et~al\mbox{.}(2025)]%
        {zhang2024gcoderimprovinglargelanguage}
\bibfield{author}{\bibinfo{person}{Qifan Zhang}, \bibinfo{person}{Xiaobin Hong}, \bibinfo{person}{Jianheng Tang}, \bibinfo{person}{Nuo Chen}, \bibinfo{person}{Yuhan Li}, \bibinfo{person}{Wenzhong Li}, \bibinfo{person}{Jing Tang}, {and} \bibinfo{person}{Jia Li}.} \bibinfo{year}{2025}\natexlab{}.
\newblock \showarticletitle{GCoder: Improving Large Language Model for Generalized Graph Reasoning}. In \bibinfo{booktitle}{\emph{Proceedings of the 34th ACM International Conference on Information and Knowledge Management}} (Seoul, Republic of Korea) \emph{(\bibinfo{series}{CIKM '25})}. \bibinfo{publisher}{Association for Computing Machinery}, \bibinfo{address}{New York, NY, USA}, \bibinfo{pages}{4149–4159}.
\newblock
\showISBNx{9798400720406}
\href{https://doi.org/10.1145/3746252.3761066}{doi:\nolinkurl{10.1145/3746252.3761066}}


\bibitem[Zhang et~al\mbox{.}(2024a)]%
        {zhang2024can}
\bibfield{author}{\bibinfo{person}{Yizhuo Zhang}, \bibinfo{person}{Heng Wang}, \bibinfo{person}{Shangbin Feng}, \bibinfo{person}{Zhaoxuan Tan}, \bibinfo{person}{Xiaochuang Han}, \bibinfo{person}{Tianxing He}, {and} \bibinfo{person}{Yulia Tsvetkov}.} \bibinfo{year}{2024}\natexlab{a}.
\newblock \showarticletitle{Can LLM Graph Reasoning Generalize beyond Pattern Memorization?}. In \bibinfo{booktitle}{\emph{Findings of the Association for Computational Linguistics: EMNLP 2024}}. \bibinfo{pages}{2289--2305}.
\newblock


\bibitem[Zhang et~al\mbox{.}(2024b)]%
        {zhang2024llm4dyg}
\bibfield{author}{\bibinfo{person}{Zeyang Zhang}, \bibinfo{person}{Xin Wang}, \bibinfo{person}{Ziwei Zhang}, \bibinfo{person}{Haoyang Li}, \bibinfo{person}{Yijian Qin}, {and} \bibinfo{person}{Wenwu Zhu}.} \bibinfo{year}{2024}\natexlab{b}.
\newblock \showarticletitle{LLM4DyG: can large language models solve spatial-temporal problems on dynamic graphs?}. In \bibinfo{booktitle}{\emph{Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining}}. \bibinfo{pages}{4350--4361}.
\newblock


\end{thebibliography}