You Had One Job: Per-Task Quantization Using LLMs' Hidden Representations
Extracted source text · arXiv 2511.06516
%%%%%%%% ICML 2025 EXAMPLE LATEX SUBMISSION FILE %%%%%%%%%%%%%%%%%
\documentclass{article}
% Recommended, but optional, packages for figures and better typesetting:
\usepackage{microtype}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{booktabs} % for professional tables
\usepackage{multirow} % For \multirow
\usepackage{array} % For column formatting
\usepackage{graphicx} % For \resizebox
\usepackage{float} % For [H] placement specifier
\usepackage{makecell} % (Optional) For better cell formatting
\usepackage{arydshln} % For \cdashline
\usepackage{caption} % For customizing captions
% hyperref makes hyperlinks in the resulting PDF.
% If your build breaks (sometimes temporarily if a hyperlink spans a page)
% please comment out the following usepackage line and replace
% \usepackage{icml2025} with \usepackage[nohyperref]{icml2025} above.
\usepackage{hyperref}
% Attempt to make hyperref and algorithmic work together better:
\newcommand{\theHalgorithm}{\arabic{algorithm}}
% Use the following line for the initial blind version submitted for review:
% \usepackage{icml2025}
% If accepted, instead use the following line for the camera-ready submission:
\usepackage[accepted]{icml2025}
% For theorems and such
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
% if you use cleveref..
\usepackage[capitalize,noabbrev]{cleveref}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% THEOREMS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}
% Todonotes is useful during development; simply uncomment the next line
% and comment out the line below the next line to turn off comments
%\usepackage[disable,textsize=tiny]{todonotes}
\usepackage[textsize=tiny]{todonotes}
% The \icmltitle you define below is probably too long as a header.
% Therefore, a short form for the running title is supplied here:
% \icmltitlerunning{Submission and Formatting Instructions for ICML 2025}
\begin{document}
\twocolumn[
\icmltitle{You Had One Job: Per-Task Quantization Using LLMs’ Hidden Representations}
% It is OKAY to include author information, even for blind
% submissions: the style file will automatically remove it for you
% unless you've provided the [accepted] option to the icml2025
% package.
% List of affiliations: The first argument should be a (short)
% identifier you will use later to specify author affiliations
% Academic affiliations should list Department, University, City, Region, Country
% Industry affiliations should list Company, City, Region, Country
% You can specify symbols, otherwise they are numbered in order.
% Ideally, you should not use this facility. Affiliations will be numbered
% in order of appearance and this is the preferred way.
\icmlsetsymbol{equal}{*}
\begin{icmlauthorlist}
\icmlauthor{Amit LeVi}{tech}
\icmlauthor{Raz Lapid}{deepkeep}
\icmlauthor{Rom Himelstein}{tech}
\icmlauthor{Yaniv Nemcovsky}{tech}
\icmlauthor{Ravid Shwartz Ziv}{nyu}
\icmlauthor{Avi Mendelson}{tech}
\icmlcorrespondingauthor{Amit LeVi}{amitlevi@campus.technion.ac.il}
\icmlcorrespondingauthor{Raz Lapid}{raz.lapid@deepkeep.ai}
\icmlcorrespondingauthor{Rom Himelstein}{romh@campus.technion.ac.il}
\icmlcorrespondingauthor{Yaniv Nemcovsky}{yanemcovsky@gmail.com}
\icmlcorrespondingauthor{Ravid Shwartz Ziv}{ravidziv@gmail.com}
\icmlcorrespondingauthor{Avi Mendelson}{mendlson@technion.ac.il}
\end{icmlauthorlist}
\icmlaffiliation{tech}{Technion -- Israel Institute of Technology, Haifa, Israel}
\icmlaffiliation{deepkeep}{DeepKeep.ai, Tel Aviv, Israel}
\icmlaffiliation{nyu}{Center for Data Science, New York University, New York, USA}
% You may provide any keywords that you
% find helpful for describing your paper; these are used to populate
% the "keywords" metadata in the PDF but will not be shown in the document
\icmlkeywords{Machine Learning, ICML}
\vskip 0.3in
]
% this must go after the closing bracket ] following \twocolumn[ ...
% This command actually creates the footnote in the first column
% listing the affiliations and the copyright notice.
% The command takes one argument, which is text to display at the start of the footnote.
% The \icmlEqualContribution command is standard text for equal contribution.
% Remove it (just {}) if you do not need this facility.
\printAffiliationsAndNotice{} % leave blank if no need to mention equal contribution
% \printAffiliationsAndNotice{\icmlEqualContribution} % otherwise use the standard text.
% RL: abstract might be too long - suggesting to shorten it
\input{sec/0_abstract}
% RL: Need to fix the references to the sections in the end of the intro
\input{sec/1_intro}
% RL: in Task signals within LLM activation space. paragraph, relevance scores aren't defined
% RL: in Layer hidden representations. do we actually use the Shannon matrix entropy? if not - get rid of it.
\input{sec/2_related_work}
% RL: What is prompt entropy? how it is defined? it's in Motivation
% RL: Make sure the objective is correct, not sure about "quantizer parameters"
% RL: combined rank - keep it capital R, plus what is the z???? az(Hl) + bz(Sl)
% RL: combined rank = relevance score? 3.1 there's relevance score with a capital R
% RL: gamma is task dependent - do we have it somewhere?
\input{sec/3_method}
% RL: Need to go over it!
\input{sec/4_experiments}
\input{sec/5_conclusions}
% \section{Electronic Submission}
% \label{submission}
% Submission to ICML 2025 will be entirely electronic, via a web site
% (not email). Information about the submission process and \LaTeX\ templates
% are available on the conference web site at:
% \begin{center}
% \textbf{\texttt{http://icml.cc/}}
% \end{center}
% The guidelines below will be enforced for initial submissions and
% camera-ready copies. Here is a brief summary:
% \begin{itemize}
% \item Submissions must be in PDF\@.
% \item If your paper has appendices, submit the appendix together with the main body and the references \textbf{as a single file}. Reviewers will not look for appendices as a separate PDF file. So if you submit such an extra file, reviewers will very likely miss it.
% \item Page limit: The main body of the paper has to be fitted to 8 pages, excluding references and appendices; the space for the latter two is not limited in pages, but the total file size may not exceed 10MB. For the final version of the paper, authors can add one extra page to the main body.
% \item \textbf{Do not include author information or acknowledgements} in your
% initial submission.
% \item Your paper should be in \textbf{10 point Times font}.
% \item Make sure your PDF file only uses Type-1 fonts.
% \item Place figure captions \emph{under} the figure (and omit titles from inside
% the graphic file itself). Place table captions \emph{over} the table.
% \item References must include page numbers whenever possible and be as complete
% as possible. Place multiple citations in chronological order.
% \item Do not alter the style template; in particular, do not compress the paper
% format by reducing the vertical spaces.
% \item Keep your abstract brief and self-contained, one paragraph and roughly
% 4--6 sentences. Gross violations will require correction at the
% camera-ready phase. The title should have content words capitalized.
% \end{itemize}
% \subsection{Submitting Papers}
% \textbf{Anonymous Submission:} ICML uses double-blind review: no identifying
% author information may appear on the title page or in the paper
% itself. \cref{author info} gives further details.
% \medskip
% Authors must provide their manuscripts in \textbf{PDF} format.
% Furthermore, please make sure that files contain only embedded Type-1 fonts
% (e.g.,~using the program \texttt{pdffonts} in linux or using
% File/DocumentProperties/Fonts in Acrobat). Other fonts (like Type-3)
% might come from graphics files imported into the document.
% Authors using \textbf{Word} must convert their document to PDF\@. Most
% of the latest versions of Word have the facility to do this
% automatically. Submissions will not be accepted in Word format or any
% format other than PDF\@. Really. We're not joking. Don't send Word.
% Those who use \textbf{\LaTeX} should avoid including Type-3 fonts.
% Those using \texttt{latex} and \texttt{dvips} may need the following
% two commands:
% {\footnotesize
% \begin{verbatim}
% dvips -Ppdf -tletter -G0 -o paper.ps paper.dvi
% ps2pdf paper.ps
% \end{verbatim}}
% It is a zero following the ``-G'', which tells dvips to use
% the config.pdf file. Newer \TeX\ distributions don't always need this
% option.
% Using \texttt{pdflatex} rather than \texttt{latex}, often gives better
% results. This program avoids the Type-3 font problem, and supports more
% advanced features in the \texttt{microtype} package.
% \textbf{Graphics files} should be a reasonable size, and included from
% an appropriate format. Use vector formats (.eps/.pdf) for plots,
% lossless bitmap formats (.png) for raster graphics with sharp lines, and
% jpeg for photo-like images.
% The style file uses the \texttt{hyperref} package to make clickable
% links in documents. If this causes problems for you, add
% \texttt{nohyperref} as one of the options to the \texttt{icml2025}
% usepackage statement.
% \subsection{Submitting Final Camera-Ready Copy}
% The final versions of papers accepted for publication should follow the
% same format and naming convention as initial submissions, except that
% author information (names and affiliations) should be given. See
% \cref{final author} for formatting instructions.
% The footnote, ``Preliminary work. Under review by the International
% Conference on Machine Learning (ICML). Do not distribute.'' must be
% modified to ``\textit{Proceedings of the
% $\mathit{42}^{nd}$ International Conference on Machine Learning},
% Vancouver, Canada, PMLR 267, 2025.
% Copyright 2025 by the author(s).''
% For those using the \textbf{\LaTeX} style file, this change (and others) is
% handled automatically by simply changing
% $\mathtt{\backslash usepackage\{icml2025\}}$ to
% $$\mathtt{\backslash usepackage[accepted]\{icml2025\}}$$
% Authors using \textbf{Word} must edit the
% footnote on the first page of the document themselves.
% Camera-ready copies should have the title of the paper as running head
% on each page except the first one. The running title consists of a
% single line centered above a horizontal rule which is $1$~point thick.
% The running head should be centered, bold and in $9$~point type. The
% rule should be $10$~points above the main text. For those using the
% \textbf{\LaTeX} style file, the original title is automatically set as running
% head using the \texttt{fancyhdr} package which is included in the ICML
% 2025 style file package. In case that the original title exceeds the
% size restrictions, a shorter form can be supplied by using
% \verb|\icmltitlerunning{...}|
% just before $\mathtt{\backslash begin\{document\}}$.
% Authors using \textbf{Word} must edit the header of the document themselves.
% \section{Format of the Paper}
% All submissions must follow the specified format.
% \subsection{Dimensions}
% The text of the paper should be formatted in two columns, with an
% overall width of 6.75~inches, height of 9.0~inches, and 0.25~inches
% between the columns. The left margin should be 0.75~inches and the top
% margin 1.0~inch (2.54~cm). The right and bottom margins will depend on
% whether you print on US letter or A4 paper, but all final versions
% must be produced for US letter size.
% Do not write anything on the margins.
% The paper body should be set in 10~point type with a vertical spacing
% of 11~points. Please use Times typeface throughout the text.
% \subsection{Title}
% The paper title should be set in 14~point bold type and centered
% between two horizontal rules that are 1~point thick, with 1.0~inch
% between the top rule and the top edge of the page. Capitalize the
% first letter of content words and put the rest of the title in lower
% case.
% \subsection{Author Information for Submission}
% \label{author info}
% ICML uses double-blind review, so author information must not appear. If
% you are using \LaTeX\/ and the \texttt{icml2025.sty} file, use
% \verb+\icmlauthor{...}+ to specify authors and \verb+\icmlaffiliation{...}+ to specify affiliations. (Read the TeX code used to produce this document for an example usage.) The author information
% will not be printed unless \texttt{accepted} is passed as an argument to the
% style file.
% Submissions that include the author information will not
% be reviewed.
% \subsubsection{Self-Citations}
% If you are citing published papers for which you are an author, refer
% to yourself in the third person. In particular, do not use phrases
% that reveal your identity (e.g., ``in previous work \cite{langley00}, we
% have shown \ldots'').
% Do not anonymize citations in the reference section. The only exception are manuscripts that are
% not yet published (e.g., under submission). If you choose to refer to
% such unpublished manuscripts \cite{anonymous}, anonymized copies have
% to be submitted
% as Supplementary Material via OpenReview\@. However, keep in mind that an ICML
% paper should be self contained and should contain sufficient detail
% for the reviewers to evaluate the work. In particular, reviewers are
% not required to look at the Supplementary Material when writing their
% review (they are not required to look at more than the first $8$ pages of the submitted document).
% \subsubsection{Camera-Ready Author Information}
% \label{final author}
% If a paper is accepted, a final camera-ready copy must be prepared.
% %
% For camera-ready papers, author information should start 0.3~inches below the
% bottom rule surrounding the title. The authors' names should appear in 10~point
% bold type, in a row, separated by white space, and centered. Author names should
% not be broken across lines. Unbolded superscripted numbers, starting 1, should
% be used to refer to affiliations.
% Affiliations should be numbered in the order of appearance. A single footnote
% block of text should be used to list all the affiliations. (Academic
% affiliations should list Department, University, City, State/Region, Country.
% Similarly for industrial affiliations.)
% Each distinct affiliations should be listed once. If an author has multiple
% affiliations, multiple superscripts should be placed after the name, separated
% by thin spaces. If the authors would like to highlight equal contribution by
% multiple first authors, those authors should have an asterisk placed after their
% name in superscript, and the term ``\textsuperscript{*}Equal contribution"
% should be placed in the footnote block ahead of the list of affiliations. A
% list of corresponding authors and their emails (in the format Full Name
% \textless{}email@domain.com\textgreater{}) can follow the list of affiliations.
% Ideally only one or two names should be listed.
% A sample file with author names is included in the ICML2025 style file
% package. Turn on the \texttt{[accepted]} option to the stylefile to
% see the names rendered. All of the guidelines above are implemented
% by the \LaTeX\ style file.
% \subsection{Abstract}
% The paper abstract should begin in the left column, 0.4~inches below the final
% address. The heading `Abstract' should be centered, bold, and in 11~point type.
% The abstract body should use 10~point type, with a vertical spacing of
% 11~points, and should be indented 0.25~inches more than normal on left-hand and
% right-hand margins. Insert 0.4~inches of blank space after the body. Keep your
% abstract brief and self-contained, limiting it to one paragraph and roughly 4--6
% sentences. Gross violations will require correction at the camera-ready phase.
% \subsection{Partitioning the Text}
% You should organize your paper into sections and paragraphs to help
% readers place a structure on the material and understand its
% contributions.
% \subsubsection{Sections and Subsections}
% Section headings should be numbered, flush left, and set in 11~pt bold
% type with the content words capitalized. Leave 0.25~inches of space
% before the heading and 0.15~inches after the heading.
% Similarly, subsection headings should be numbered, flush left, and set
% in 10~pt bold type with the content words capitalized. Leave
% 0.2~inches of space before the heading and 0.13~inches afterward.
% Finally, subsubsection headings should be numbered, flush left, and
% set in 10~pt small caps with the content words capitalized. Leave
% 0.18~inches of space before the heading and 0.1~inches after the
% heading.
% Please use no more than three levels of headings.
% \subsubsection{Paragraphs and Footnotes}
% Within each section or subsection, you should further partition the
% paper into paragraphs. Do not indent the first line of a given
% paragraph, but insert a blank line between succeeding ones.
% You can use footnotes\footnote{Footnotes
% should be complete sentences.} to provide readers with additional
% information about a topic without interrupting the flow of the paper.
% Indicate footnotes with a number in the text where the point is most
% relevant. Place the footnote in 9~point type at the bottom of the
% column in which it appears. Precede the first footnote in a column
% with a horizontal rule of 0.8~inches.\footnote{Multiple footnotes can
% appear in each column, in the same order as they appear in the text,
% but spread them across columns and pages if possible.}
% \begin{figure}[ht]
% \vskip 0.2in
% \begin{center}
% \centerline{\includegraphics[width=\columnwidth]{icml_numpapers}}
% \caption{Historical locations and number of accepted papers for International
% Machine Learning Conferences (ICML 1993 -- ICML 2008) and International
% Workshops on Machine Learning (ML 1988 -- ML 1992). At the time this figure was
% produced, the number of accepted papers for ICML 2008 was unknown and instead
% estimated.}
% \label{icml-historical}
% \end{center}
% \vskip -0.2in
% \end{figure}
% \subsection{Figures}
% You may want to include figures in the paper to illustrate
% your approach and results. Such artwork should be centered,
% legible, and separated from the text. Lines should be dark and at
% least 0.5~points thick for purposes of reproduction, and text should
% not appear on a gray background.
% Label all distinct components of each figure. If the figure takes the
% form of a graph, then give a name for each axis and include a legend
% that briefly describes each curve. Do not include a title inside the
% figure; instead, the caption should serve this function.
% Number figures sequentially, placing the figure number and caption
% \emph{after} the graphics, with at least 0.1~inches of space before
% the caption and 0.1~inches after it, as in
% \cref{icml-historical}. The figure caption should be set in
% 9~point type and centered unless it runs two or more lines, in which
% case it should be flush left. You may float figures to the top or
% bottom of a column, and you may set wide figures across both columns
% (use the environment \texttt{figure*} in \LaTeX). Always place
% two-column figures at the top or bottom of the page.
% \subsection{Algorithms}
% If you are using \LaTeX, please use the ``algorithm'' and ``algorithmic''
% environments to format pseudocode. These require
% the corresponding stylefiles, algorithm.sty and
% algorithmic.sty, which are supplied with this package.
% \cref{alg:example} shows an example.
% \begin{algorithm}[tb]
% \caption{Bubble Sort}
% \label{alg:example}
% \begin{algorithmic}
% \STATE {\bfseries Input:} data $x_i$, size $m$
% \REPEAT
% \STATE Initialize $noChange = true$.
% \FOR{$i=1$ {\bfseries to} $m-1$}
% \IF{$x_i > x_{i+1}$}
% \STATE Swap $x_i$ and $x_{i+1}$
% \STATE $noChange = false$
% \ENDIF
% \ENDFOR
% \UNTIL{$noChange$ is $true$}
% \end{algorithmic}
% \end{algorithm}
% \subsection{Tables}
% You may also want to include tables that summarize material. Like
% figures, these should be centered, legible, and numbered consecutively.
% However, place the title \emph{above} the table with at least
% 0.1~inches of space before the title and the same after it, as in
% \cref{sample-table}. The table title should be set in 9~point
% type and centered unless it runs two or more lines, in which case it
% should be flush left.
% % Note use of \abovespace and \belowspace to get reasonable spacing
% % above and below tabular lines.
% \begin{table}[t]
% \caption{Classification accuracies for naive Bayes and flexible
% Bayes on various data sets.}
% \label{sample-table}
% \vskip 0.15in
% \begin{center}
% \begin{small}
% \begin{sc}
% \begin{tabular}{lcccr}
% \toprule
% Data set & Naive & Flexible & Better? \\
% \midrule
% Breast & 95.9$\pm$ 0.2& 96.7$\pm$ 0.2& $\surd$ \\
% Cleveland & 83.3$\pm$ 0.6& 80.0$\pm$ 0.6& $\times$\\
% Glass2 & 61.9$\pm$ 1.4& 83.8$\pm$ 0.7& $\surd$ \\
% Credit & 74.8$\pm$ 0.5& 78.3$\pm$ 0.6& \\
% Horse & 73.3$\pm$ 0.9& 69.7$\pm$ 1.0& $\times$\\
% Meta & 67.1$\pm$ 0.6& 76.5$\pm$ 0.5& $\surd$ \\
% Pima & 75.1$\pm$ 0.6& 73.9$\pm$ 0.5& \\
% Vehicle & 44.9$\pm$ 0.6& 61.5$\pm$ 0.4& $\surd$ \\
% \bottomrule
% \end{tabular}
% \end{sc}
% \end{small}
% \end{center}
% \vskip -0.1in
% \end{table}
% Tables contain textual material, whereas figures contain graphical material.
% Specify the contents of each row and column in the table's topmost
% row. Again, you may float tables to a column's top or bottom, and set
% wide tables across both columns. Place two-column tables at the
% top or bottom of the page.
% \subsection{Theorems and such}
% The preferred way is to number definitions, propositions, lemmas, etc. consecutively, within sections, as shown below.
% \begin{definition}
% \label{def:inj}
% A function $f:X \to Y$ is injective if for any $x,y\in X$ different, $f(x)\ne f(y)$.
% \end{definition}
% Using \cref{def:inj} we immediate get the following result:
% \begin{proposition}
% If $f$ is injective mapping a set $X$ to another set $Y$,
% the cardinality of $Y$ is at least as large as that of $X$
% \end{proposition}
% \begin{proof}
% Left as an exercise to the reader.
% \end{proof}
% \cref{lem:usefullemma} stated next will prove to be useful.
% \begin{lemma}
% \label{lem:usefullemma}
% For any $f:X \to Y$ and $g:Y\to Z$ injective functions, $f \circ g$ is injective.
% \end{lemma}
% \begin{theorem}
% \label{thm:bigtheorem}
% If $f:X\to Y$ is bijective, the cardinality of $X$ and $Y$ are the same.
% \end{theorem}
% An easy corollary of \cref{thm:bigtheorem} is the following:
% \begin{corollary}
% If $f:X\to Y$ is bijective,
% the cardinality of $X$ is at least as large as that of $Y$.
% \end{corollary}
% \begin{assumption}
% The set $X$ is finite.
% \label{ass:xfinite}
% \end{assumption}
% \begin{remark}
% According to some, it is only the finite case (cf. \cref{ass:xfinite}) that is interesting.
% \end{remark}
% %restatable
% \subsection{Citations and References}
% Please use APA reference format regardless of your formatter
% or word processor. If you rely on the \LaTeX\/ bibliographic
% facility, use \texttt{natbib.sty} and \texttt{icml2025.bst}
% included in the style-file package to obtain this format.
% Citations within the text should include the authors' last names and
% year. If the authors' names are included in the sentence, place only
% the year in parentheses, for example when referencing Arthur Samuel's
% pioneering work \yrcite{Samuel59}. Otherwise place the entire
% reference in parentheses with the authors and year separated by a
% comma \cite{Samuel59}. List multiple references separated by
% semicolons \cite{kearns89,Samuel59,mitchell80}. Use the `et~al.'
% construct only for citations with three or more authors or after
% listing all authors to a publication in an earlier reference \cite{MachineLearningI}.
% Authors should cite their own work in the third person
% in the initial version of their paper submitted for blind review.
% Please refer to \cref{author info} for detailed instructions on how to
% cite your own papers.
% Use an unnumbered first-level section heading for the references, and use a
% hanging indent style, with the first line of the reference flush against the
% left margin and subsequent lines indented by 10 points. The references at the
% end of this document give examples for journal articles \cite{Samuel59},
% conference publications \cite{langley00}, book chapters \cite{Newell81}, books
% \cite{DudaHart2nd}, edited volumes \cite{MachineLearningI}, technical reports
% \cite{mitchell80}, and dissertations \cite{kearns89}.
% Alphabetize references by the surnames of the first authors, with
% single author entries preceding multiple author entries. Order
% references for the same authors by year of publication, with the
% earliest first. Make sure that each reference includes all relevant
% information (e.g., page numbers).
% Please put some effort into making references complete, presentable, and
% consistent, e.g. use the actual current name of authors.
% If using bibtex, please protect capital letters of names and
% abbreviations in titles, for example, use \{B\}ayesian or \{L\}ipschitz
% in your .bib file.
% \section*{Accessibility}
% Authors are kindly asked to make their submissions as accessible as possible for everyone including people with disabilities and sensory or neurological differences.
% Tips of how to achieve this and what to pay attention to will be provided on the conference website \url{http://icml.cc/}.
% \section*{Software and Data}
% If a paper is accepted, we strongly encourage the publication of software and data with the
% camera-ready version of the paper whenever appropriate. This can be
% done by including a URL in the camera-ready copy. However, \textbf{do not}
% include URLs that reveal your institution or identity in your
% submission for review. Instead, provide an anonymous URL or upload
% the material as ``Supplementary Material'' into the OpenReview reviewing
% system. Note that reviewers are not required to look at this material
% when writing their review.
% % Acknowledgements should only appear in the accepted version.
% \section*{Acknowledgements}
% \textbf{Do not} include acknowledgements in the initial version of
% the paper submitted for blind review.
% If a paper is accepted, the final camera-ready version can (and
% usually should) include acknowledgements. Such acknowledgements
% should be placed at the end of the section, in an unnumbered section
% that does not count towards the paper page limit. Typically, this will
% include thanks to reviewers who gave useful comments, to colleagues
% who contributed to the ideas, and to funding agencies and corporate
% sponsors that provided financial support.
% \section*{Impact Statement}
% Authors are \textbf{required} to include a statement of the potential
% broader impact of their work, including its ethical aspects and future
% societal consequences. This statement should be in an unnumbered
% section at the end of the paper (co-located with Acknowledgements --
% the two may appear in either order, but both must be before References),
% and does not count toward the paper page limit. In many cases, where
% the ethical impacts and expected societal implications are those that
% are well established when advancing the field of Machine Learning,
% substantial discussion is not required, and a simple statement such
% as the following will suffice:
% ``This paper presents work whose goal is to advance the field of
% Machine Learning. There are many potential societal consequences
% of our work, none which we feel must be specifically highlighted here.''
% The above statement can be used verbatim in such cases, but we
% encourage authors to think about whether there is content which does
% warrant further discussion, as this statement will be apparent if the
% paper is later flagged for ethics review.
% % In the unusual situation where you want a paper to appear in the
% % references without citing it in the main text, use \nocite
% \nocite{langley00}
\bibliography{main}
\bibliographystyle{icml2025}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% APPENDIX
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% \newpage
% \appendix
% \onecolumn
% \section{You \emph{can} have an appendix here.}
% You can have as much text here as you want. The main body must be at most $8$ pages long.
% For the final version, one more page can be added.
% If you want, you can use an appendix like this one.
% The $\mathtt{\backslash onecolumn}$ command above can be kept in place if you prefer a one-column appendix, or can be removed if you prefer a two-column appendix. Apart from this possible change, the style (font size, spacing, margins, page numbering, etc.) should be kept the same as the main body.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}
% This document was modified from the file originally made available by
% Pat Langley and Andrea Danyluk for ICML-2K. This version was created
% by Iain Murray in 2018, and modified by Alexandre Bouchard in
% 2019 and 2021 and by Csaba Szepesvari, Gang Niu and Sivan Sabato in 2022.
% Modified again in 2023 and 2024 by Sivan Sabato and Jonathan Scarlett.
% Previous contributors include Dan Roy, Lise Getoor and Tobias
% Scheffer, which was slightly modified from the 2010 version by
% Thorsten Joachims & Johannes Fuernkranz, slightly modified from the
% 2009 version by Kiri Wagstaff and Sam Roweis's 2008 version, which is
% slightly modified from Prasad Tadepalli's 2007 version which is a
% lightly changed version of the previous year's version by Andrew
% Moore, which was in turn edited from those of Kristian Kersting and
% Codrina Lauth. Alex Smola contributed to the algorithmic style files.
\begin{abstract}
Large Language Models (LLMs) excel across diverse tasks, yet many applications require only limited capabilities, making large variants inefficient in memory and latency. Existing approaches often combine distillation and quantization, but most post-training quantization (PTQ) methods are task-agnostic, ignoring how task-specific signals are distributed across layers. In this work, we propose to use hidden representations that encode task-salient signals as a guideline for quantization. In order to fully utilize our innovative idea, this paper compares two new task-aware PTQ methods: \textbf{Task-Aware Quantization (TAQ)}, which allocates bitwidths using task-conditioned statistics from hidden activations, and \textbf{TAQO}, which allocates precision based on direct layer sensitivity tests. From a small calibration set, these approaches identify task-relevant layers, preserving their precision while aggressively quantizing the rest. This yields stable task sensitivity profiles and efficient task-specialized models. Across models, TAQ and TAQO outperform the baselines; TAQ leads on Phi-4, while TAQO leads on Llama-3.1, Qwen3, and Qwen2.5. For instances, on Phi-4 it achieves 42.33 EM / 50.81 F1, far surpassing Activation-aware Weight Quantization (AWQ) (2.25 / 7.07), while remaining within $\leq$1.0\% of the original accuracy at lower average precision. % A reference implementation is provided. % A reference implementation is available at
\end{abstract}
\section{Introduction}
\label{section:intro}
Large language models (LLMs) built on the Transformer architecture \citep{vaswani2017attention} continue to scale in size and capability, with empirical scaling laws showing systematic improvements as parameter counts grow \citep{kaplan2020scaling,hoffmann2022chinchilla}. Yet this scaling also amplifies inference- and memory-side costs \citep{dao2022flashattention,kwon2023pagedattention}, making serving efficiency a first-order bottleneck for deployed systems.
Many applications, however, require only a narrow subset of LLM capabilities—for example, code completion \citep{chen2021codex,roziere2023codellama}, mathematical reasoning \citep{cobbe2021gsm8k,hendrycks2021math}, or domain-specific question answering \citep{tsatsaronis2015bioasq}. This motivates fine-tuning and distillation to tailor models to specific tasks \citep{hu2021lora,ouyang2022instruct,sanh2019distilbert,hinton2015distill}, typically at the cost of additional training cycles and ongoing maintenance. Quantization is a complementary approach: it reduces memory and latency without heavy training, enabling rapid, low-cost deployment \citep{gholami2021quantization,zhu2023llmcompresssurvey}. For LLMs, progress spans post-training weight rounding with high numerical fidelity \citep{sharify2024posttrainingmix,frantar2022gptq}, methods for handling activation outliers or redistributing precision \citep{dettmers2022llmint8,xiao2023smoothquant}, and mixed-precision strategies under global bitwidth budgets \citep{dong2019hawq,lin2023awq}.
Concurrently, representation-level studies show that linear directions in hidden states or parameters can encode controllable semantics—such as activation addition \citep{turner2024activationaddition}, task vectors \citep{bowen2024sta,zeng2025taskvectorbases}, and models’ safety mechanisms \citep{arditi2024refusal,levi2025enhancing}—and further indicate that task-salient signals concentrate in specific layers or subspaces, whereas other layers carry more generic features \citep{song2024taskspecific,zhang2024layerimportance,li2024safetylayers,skean2025layer}. This evidence motivates using the same salience signals to drive per-task, per-layer distillation \citep{zhou2024kdsurvey,ni2025caprese}.
In contrast, prevailing post training quantization (PTQ) pipelines are task-agnostic: they optimize layer-wise criteria—such as weight error, activation range, or curvature—rather than end-task performance \citep{nagel2020up,banner2019post,wang2020towards,lin2023awq}. Consequently, bit precision is often assigned using uniform or manually specified schemes that ignore task importance, rather than prioritizing critical layers. This design can under-allocate precision where it matters and over-allocate where it does not, and it typically omits objectives that preserve task-salient hidden representations during quantization \citep{skean2025layer}.
In this work, we ask whether task-salient signals in hidden activations can guide post-training quantization. Unlike prevailing PTQ pipelines—which are task-agnostic and rely on local error surrogates over generic calibration data—we propose \textbf{Task-Aware Quantization (TAQ)} and its oracle variant \textbf{TAQO}, frameworks that convert task-calibrated activations into per-layer precision decisions. Concretely, TAQ computes layer relevance scores from hidden representations on a task-specific calibration set, then solves a budgeted, monotone bit-allocation problem that assigns higher precision to the most task-relevant layers while aggressively quantizing the rest. TAQO, by contrast, bypasses surrogates and allocates precision by directly measuring per-layer task sensitivity.
Our contributions are:
\begin{itemize}
\item \textbf{TAQ:} a PTQ framework that derives layer relevance from task-calibrated hidden activations and allocates bitwidths under an explicit budget.
\item \textbf{TAQO:} a task-aware oracle that selects critical layers via direct task-sensitivity tests, assigning precision according to observed degradation.
\item \textbf{Empirical results:} across code, math, and trivia tasks on open-weight LLMs, TAQ maintains accuracy within $\leq\!1$ point of full precision while reducing model size and latency.
\end{itemize}
The remainder of this paper is organized as follows.
\S\ref{back} reviews post-training quantization, layer-wise activations, and related background.
\S\ref{method} introduces TAQ, its motivation, and the scoring and allocation procedures.
\S\ref{Experiments} reports experimental results.
Finally, \S\ref{sec:discussion} discusses implications, limitations, and future directions.
\section{Background and Related Work}
\label{back}
Recent work indicates that different tasks rely on different layers, with some layers carrying stronger task-specific signals \citep{evci2022head2toe,burns2022latent}. These findings enable weight-space editing via task vectors \citep{ilharco2022taskarithmetic} and behavioral steering through hidden-activation interventions \citep{turner2023activation,zou2023representation}. In parallel, advances in PTQ have achieved low-bit inference from small calibration sets via high-fidelity rounding, range equalization, activation-aware precision allocation, representation-space rotations, and cache compression \citep{frantar2022gptq,xiao2023smoothquant,lin2023awq,shao2023omniquant,ashkboos2024quarot,liu2024spinquant,hooper2024kvquant,he2024zipcache}. Despite their effectiveness, these methods are largely task-agnostic: they optimize layer-local error surrogates on generic calibration data rather than end-task metrics, and they do not exploit task-specific activation signals. We adopt a task-conditioned perspective that uses per-layer evidence from hidden representations to guide precision allocation, linking PTQ techniques with semantic analyses of internal representations.
\paragraph{Quantization: from uniform to layer- and activation-aware.}
LLMs are typically composed of $N$ stacked Transformer blocks~\citep{vaswani2017attention}.
For block $i$, let $a_i(\cdot)$ denote hidden activations. On modern hardware, inference is often constrained by memory bandwidth and latency rather than raw FLOPs~\citep{dao2022flashattention,kwon2023pagedattention}.
Quantization addresses these limitations by mapping floating-point tensors to lower-precision representations, thereby reducing memory footprint and improving throughput.
A standard (asymmetric) uniform PTQ scheme applies
\[
Q(x) \;=\; \operatorname{round}\!\left(\frac{x - z}{s}\right)\!\cdot s \;+\; z,
\]
where $s$ is a scale and $z$ a zero-point offset~\citep{nagel2021white,jacob2018integer,krishnamoorthi2018whitepaper}.
Beyond global uniformity, advances proceed along three axes:
\textit{(i) weight-only PTQ and rounding}, including GPTQ (local-loss–aware rounding), \textsc{LLM.int8} (activation-outlier handling), SmoothQuant (weight–activation trade-off), and OmniQuant/RPTQ (compensation and reordering)~\citep{frantar2022gptq,dettmers2022llmint8,xiao2023smoothquant,shao2023omniquant,yuan2023rptq};
\textit{(ii) sensitivity-driven bit allocation} under a global budget using curvature or error models~\citep{dong2019hawq,yao2022zeroquant,yao2023zeroquantv2,sharify2024posttrainingmix};
and \textit{(iii) activation- and channel-aware calibration} that preserves salient channels/heads and increasingly targets the KV cache for long context~\citep{lin2023awq,xiao2023smoothquant,shao2023omniquant,yuan2023rptq,liu2024kivi}. Within the last category, \textbf{AWQ} (Activation-aware Weight Quantization) is particularly influential: it calibrates decisions using both weight distributions and activation statistics, explicitly preserving activation outliers critical to LLM accuracy, and thereby achieves effective compression and acceleration with minimal downstream degradation~\citep{lin2023awq}.
\paragraph{Per-layer activation analysis and directionality.}
A complementary line of work analyzes the geometry of hidden activations: linear directions have been shown to encode semantic features and afford inference-time control \citep{turner2023steering,repeng2023zou}, while localized edits can reveal and manipulate factual computation \citep{rome2022meng}.
Let $\mathcal{B}=\{B_1,\dots,B_M\}$ denote benchmarks (tasks), and for task $k$ let $D_k=\{(q_j,a_j)\}_{j=1}^{n_k}$ be a labeled set.
A task-conditioned direction at layer $i$ can be estimated as
\[
d_i^{(k)} \;=\; \frac{1}{n_k}\sum_{(q_j,a_j)\in D_k}\!\Big(a_i(q_j)-a_i(\tilde q_j)\Big),
\]
where $\tilde q_j$ are contrast prompts, and cross-task alignment can be measured by the cosine similarity
\[
s_i^{(k,\ell)} \;=\; \frac{\langle d_i^{(k)},\, d_i^{(\ell)}\rangle}{\|d_i^{(k)}\|\,\|d_i^{(\ell)}\|}.
\]
Per-layer activation (PLA) analyses \citep{skean2025layer} relate layer-local norms, sparsity, and directionality to downstream performance, suggesting that compression and precision budgets should follow where task-salient signal resides \citep{turner2023steering,rome2022meng,arditi2024refusal,levi2025enhancing,levi2025jailbreak}.
\paragraph{Layer hidden representations.}
Understanding how information is organized across layers requires a diagnostic that distinguishes compressed from diverse representations. A convenient approach is to examine the spectrum of the token–token Gram matrix, which reflects how hidden states spread across directions at each depth.
Let $X_\ell \in \mathbb{R}^{n\times d}$ denote hidden states at layer $\ell$, and define $K_\ell=\tfrac{1}{n}X_\ell X_\ell^\top$ with eigenvalues $\{\lambda_i\}_{i=1}^n$. Using normalized eigenvalues $\tilde{\lambda}_i=\lambda_i/\sum_{j=1}^n \lambda_j$, the (Shannon) matrix entropy
\[
H(K_\ell) \;=\; -\sum_{i=1}^n \tilde{\lambda}_i \log \tilde{\lambda}_i
\]
quantifies dispersion: larger values indicate more isotropic (diverse) token representations, whereas smaller values indicate concentration (compression).
Recent analyses unify information-theoretic, geometric, and invariance-based diagnostics for hidden states and find that intermediate layers often yield stronger, more task-useful features than the final layer, exhibiting a mid-depth ``compression valley'' that correlates with downstream performance across models and domains \citep{skean2025layer}.
\paragraph{Task signals within LLM activation space.}
Task-conditioned compression concentrates precision on representations that matter for a target task distribution.
Given a task-specific calibration set $C_k \subseteq \mathcal{D}$, we compute per-layer relevance scores $r_i^{(k)}$ (e.g., direction norms or projection energies) from hidden activations and then solve a budgeted allocation problem:
\[
\min_{\mathbf{b}\in\mathcal{B}} \; \mathcal{E}_k(\mathbf{b})
\quad \text{s.t.} \quad \mathrm{Cost}(\mathbf{b}) \le \tau,
\]
where $\mathbf{b}=(b_1,\dots,b_N)$ are per-layer bitwidths, $\mathcal{E}_k$ is a task-conditioned error proxy (e.g., task loss on $C_k$ or a sensitivity surrogate), and $\tau$ enforces a memory/latency budget via $\mathrm{Cost}(\cdot)$ (e.g., model size or estimated throughput).
This formulation prioritizes layers with high $r_i^{(k)}$ by allocating higher precision where task-salient signal is concentrated.
\paragraph{External representation alignment and invariances.}
Beyond PTQ, work on latent-space geometry shows that aligning representations and exploiting invariances can improve efficiency. \citet{moschella2022relative} propose manifold-product and relative-representation methods that enable zero-shot model stitching across tasks; \citet{lahner2024direct} demonstrate alignment and anchoring techniques using linear maps and invariances; \citet{domine2024lazy} examine representation dynamics in linear networks; \citet{zhao2023improving} explore parameter symmetries for data-free quantization; and \citet{williams2021generalized} introduce shape metrics to compare neural representations. Together, these insights underscore the importance of latent-space structure for efficient models and motivate our task-aware quantization.
\paragraph{Connecting PTQ with representation analyses.}
We address the task-agnostic gap in PTQ by linking methods that optimize quantization objectives and allocate per-layer bit precision (e.g., \citet{lin2023awq}) with advances in mechanistic interpretability and semantic analyses of hidden representations \citep{skean2025layer}. Concretely, we instantiate PTQ with task-informed surrogates and per-layer allocation rules derived from representation diagnostics, yielding a high-performance, task-oriented quantization approach.
\section{Method}
\label{method}
We introduce \textbf{TAQ}, a task-aware post-training quantization framework that assigns mixed precision across Transformer layers using task-conditioned activation statistics. This section is organized as follows. \S\ref{subsec:motivation-objective} motivates a task-aware view and formalizes the budgeted objective. \S\ref{subsec:taq-method} details \textbf{TAQ}'s two components: (i) layer relevance scoring and (ii) a lightweight calibration procedure that tunes quantizer parameters on a small, task-specific calibration set. Finally, \S\ref{subsec:taqo} presents \textbf{TAQO}, an oracle-style variant that selects high-precision layers via task-specific sensitivity.
\label{subsec:motivation-objective}
\input{figs/motivation}
\paragraph{Motivation.}
Different tasks stress different parts of a Transformer: some layers are indispensable for capturing semantic diversity, while others can be aggressively quantized with little effect. In Figure~\ref{fig:bbq_full}, we show per-layer relevance signals for Llama-3.1-8B-Instruct across TriviaQA \citep{joshi2017triviaqa} (open-domain question answering with evidence documents), GSM8K \citep{cobbe2021training} (grade-school math word problems requiring multi-step reasoning), and MBPP \citep{austin2021program} (Python code synthesis from natural-language prompts with unit tests), measured via prompt entropy $H_\ell$, stability $S_\ell=-\mathrm{Var}(X_\ell)$, and a combined rank $r_\ell=\alpha\, z(H_\ell)+\beta\, z(S_\ell)$. The resulting profiles are task-dependent: TriviaQA and MBPP concentrate importance near input/output boundaries (early lexical grounding and late semantic control), whereas GSM8K exhibits a broad mid-to-late plateau where entropy grows and stability drops, reflecting the need to preserve intermediate reasoning structure. Mid-layers in TriviaQA and MBPP are comparatively flat and redundant, tolerating more aggressive quantization. Relevance is therefore neither uniform nor universal, motivating task-aware allocation that assigns higher precision where task-specific signals concentrate.
\paragraph{Objective.}
Let a pretrained model with $N$ Transformer layers and parameters $\theta$ be given, and let $\mathcal{D}$ denote the target data distribution. Post-training quantization seeks a compressed parameterization $\hat{\theta}$ that respects a resource budget while preserving task performance:
\begin{equation}
\label{eq:general-ptq}
\min_{\hat{\theta} \in \mathcal{Q}(\theta)} \;\; \mathbb{E}_{x \sim \mathcal{D}}\!\left[\mathcal{L}\big(f_{\theta}(x),\, f_{\hat{\theta}}(x)\big)\right]
\quad \text{s.t.} \quad \mathrm{Cost}(\hat{\theta}) \le \tau,
\end{equation}
where $\mathcal{Q}(\theta)$ is the family of quantized models derived from $\theta$, $\mathcal{L}$ measures discrepancy (e.g., between logits or intermediate activations), and $\mathrm{Cost}(\cdot)$ encodes memory/latency constraints with budget $\tau$.
Let $\mathbf{b}=(b_1,\dots,b_N)$ denote per-layer bitwidths chosen from an admissible set $\mathcal{B}$ (e.g., low/medium/high precision). Given a small calibration set $\mathcal{C}\subset\mathcal{D}$, we formulate task-aware mixed precision as
\begin{equation}
\label{eq:taq-objective}
\min_{\mathbf{b}\in \mathcal{B}^N,\; \phi} \;\; \mathcal{E}\big(\mathbf{b},\phi;\,\mathcal{C}\big)
\quad \text{s.t.} \quad \mathrm{Cost}(\mathbf{b}) \le \tau,
\end{equation}
where $\phi$ collects quantizer parameters (e.g., scales and zero-points), and $\mathcal{E}$ is a proxy error that upper-bounds the discrepancy in~\eqref{eq:general-ptq} when quantizing with $(\mathbf{b},\phi)$. The key design choice is to \emph{allocate larger bitwidth to layers whose activations are both informative and stable for the task}.
\subsection{Task-Aware Quantization (TAQ)}
\label{subsec:taq-method}
\textbf{TAQ} proceeds in two stages. First, each layer $\ell$ receives a relevance score $R_\ell$ computed from task-conditioned activation statistics; these scores guide the bitwidth allocation $\mathbf{b}$ under the resource budget. Second, with $\mathbf{b}$ fixed, the quantizer parameters $\phi$ are calibrated on $\mathcal{C}$ to minimize the mismatch between full-precision and quantized behavior.
\paragraph{Scoring: information and stability.}
For input $x$, let $X_\ell(x)\in\mathbb{R}^{T\times d_\ell}$ denote activations at layer $\ell$. From a small reservoir $\{v_i\}_{i=1}^{r_\ell}\subset\mathbb{R}^{d_\ell}$ sampled across $\mathcal{C}$, construct the centered matrix $Z_\ell$ and the Gram matrix $K_\ell = \tfrac{1}{r_\ell} Z_\ell Z_\ell^\top$. With eigenvalues $\{\lambda_i\}$ and normalized spectrum $\tilde{\lambda}_i=\lambda_i/\sum_j \lambda_j$, define the \emph{information} score
\begin{equation}
\label{eq:entropy}
H_\ell \;=\; -\sum_{i} \tilde{\lambda}_i \log \tilde{\lambda}_i,
\end{equation}
which quantifies representational diversity. In parallel, maintain streaming moments $(S^{(1)}_\ell,S^{(2)}_\ell,n_\ell)$ over token activations and define
\begin{equation}
\label{eq:stability}
\mathrm{Var}_\ell \;=\; \tfrac{S^{(2)}_\ell}{n_\ell} - \Big(\tfrac{S^{(1)}_\ell}{n_\ell}\Big)^2,
\qquad
S_\ell \;=\; -\,\mathrm{Var}_\ell,
\end{equation}
so lower variance yields higher \emph{stability}. After $z$-scoring across layers, the task-conditioned relevance is
\begin{equation}
\label{eq:relevance}
R_\ell \;=\; \alpha\,\widehat{H}_\ell \;+\; \beta\,\widehat{S}_\ell,
\qquad \alpha,\beta \ge 0,\;\; \alpha+\beta=1,
\end{equation}
and layers are ranked by $R_\ell$ to guide the constrained allocation in~\eqref{eq:taq-objective}.%
\footnote{In practice, the allocation reduces to a monotone knapsack under a global bit budget.}
\paragraph{Calibration: quantizer tuning.}
Let $\mathcal{W}_\ell$ be the set of weight matrices in layer $\ell$. For a chosen bitwidth $b_\ell \in \mathcal{B}$, each weight group $W_g \subset \mathcal{W}_\ell$ is mapped to integer codes by a parametric quantizer $q_{b_\ell,\phi_g}$ with parameters $\phi_g$ (e.g., scale and offset). Denote the quantized model by $f_{\hat{\theta}(\mathbf{b},\phi)}$. With $\mathbf{b}$ fixed, \textbf{TAQ} calibrates $\phi$ via a small optimization on $\mathcal{C}$:
\begin{equation}
\label{eq:calibration}
\min_{\phi} \;\; \sum_{x \in \mathcal{C}} \; \mathcal{L}\big( f_{\theta}(x),\, f_{\hat{\theta}(\mathbf{b},\phi)}(x) \big),
\end{equation}
where $\mathcal{L}$ may compare logits and/or intermediate activations. This tunes only $\phi$ while keeping $\theta$ frozen, yielding a task-conditioned quantized model that satisfies the budget $\tau$ and preserves downstream performance.
\subsection{Task-Aware Quantization with Oracle Layer Selection (TAQO)}
\label{subsec:taqo}
While \textbf{TAQ} assigns precision by ranking continuous relevance signals $(H_\ell,S_\ell)$, it is also useful to study a discrete, \emph{oracle-style} baseline that leverages direct task-specific sensitivity. We denote this variant \textbf{TAQO}. For each task, identify a subset of layers $\mathcal{K}_t \subseteq \{1,\dots,N\}$ that are empirically \emph{critical}, as determined by performance degradation when they alone are quantized to low precision. These layers are preserved at high precision, while the remainder are quantized to the target bitwidth.
Formally, let $\Delta_\ell$ denote the task-level performance drop when layer $\ell$ is quantized while all other layers remain full precision. For task $t$, the set of critical layers is
\begin{equation}
\label{eq:oracle-critical}
\mathcal{K}_t \;=\; \big\{ \ell \;\big|\; \Delta_\ell \,\ge\, \gamma \big\},
\end{equation}
where $\gamma$ is a task-dependent sensitivity threshold. The bit allocation rule then becomes
\begin{equation}
b_\ell \;=\;
\begin{cases}
b_{\mathrm{high}}, & \ell \in \mathcal{K}_t, \\
b_{\mathrm{low}}, & \text{otherwise},
\end{cases}
\quad b_{\mathrm{high}}>b_{\mathrm{low}}, \; \mathrm{Cost}(\mathbf{b})\le\tau.
\end{equation}
By construction, \textbf{TAQO} provides an optimistic \emph{upper bound} on task-aware allocation under a given budget. Empirically, $\mathcal{K}_t$ varies substantially across tasks, underscoring that quantization tolerance is not layer-uniform but task-dependent. Although impractical for deployment—since it presumes access to sensitivity curves—\textbf{TAQO} serves as a valuable ceiling that motivates lightweight proxies.
\begin{figure}[H]
\centering
\includegraphics[width=1\linewidth]{imgs/layer_prompt_entropy.png}
\caption{Layers relevance scores per task.}
\label{fig:bbq_full}
\end{figure}
\section{Experiments}
\label{Experiments}
\noindent\textbf{Overview.}
We empirically study task-aware mixed-precision quantization on LLMs across QA-style workloads. Our central hypothesis is that \emph{task-salient evidence is unevenly distributed across layers}, hence precision should follow where this evidence resides. We compare our methods against representative baselines.
\emph{AWQ} (Activation-aware Weight Quantization) calibrates quantization by preserving activation outliers critical to performance, using a small \emph{mixed-task} pool following the original recipe~\citep{awq_neurips,awq_mlsys}.
\emph{AWQ-OT} applies the same procedure but uses a \emph{single task-specific} calibration set identical to that used by our methods.
Our proposed \textbf{\textbf{TAQ}} allocates precision based on hidden-representation statistics computed on a task-specific calibration set, while \textbf{\textbf{TAQO}} uses oracle sensitivity curves in place of surrogate statistics (i.e., \textbf{TAQ} with oracle-derived relevance).
\paragraph{Research questions.}
\begin{enumerate}
\item \textbf{Representation differences.} Do prompt representations \emph{in task context} evolve differently across depth for different tasks, inducing distinct task-specific precision profiles?
\item \textbf{AWQ vs.\ AWQ-OT.} Does task-specific calibration (AWQ-OT) improve over mixed-pool AWQ despite the same quantization rule?
\item \textbf{AWQ-OT vs.\ \textbf{TAQ}/\textbf{TAQO}.} Do task-aware scoring (\textbf{TAQ}) or oracle sensitivities (\textbf{TAQO}) outperform AWQ-OT, indicating that task-aware allocation is the key ingredient?
\item \textbf{Effectiveness.} Under the same budget, do \textbf{TAQ} and \textbf{TAQO} achieve higher accuracy than activation-aware baselines, and how do they differ in practice?
\end{enumerate}
\subsection{Experimental Settings}
\paragraph{Models.}
We evaluate open-weight LLMs representative of current practice:
\emph{Llama-3.1-8B Instruct}~\citep{llama3herd},
\emph{Qwen3-4B Instruct} and \emph{Qwen2.5-7B Instruct}~\citep{qwen25tr},
\emph{Mistral-7B Instruct}~\citep{mistral},
and \emph{Phi-4-mini Instruct}~\citep{phi4tr}.
\paragraph{Baselines.}
\emph{W\textbackslash O} denotes the original model without quantization (full precision).
\emph{AWQ} uses mixed-task calibration as in the original works~\citep{awq_neurips,awq_mlsys}.
\emph{AWQ-OT} uses the identical AWQ recipe but with a single task-specific calibration set matching that of \textbf{TAQ}/\textbf{TAQO}.
\paragraph{Evaluation.}
We report \emph{Exact Match (EM)} and token-level \emph{F1}~\citep{rajpurkar2016squad}.
\paragraph{Datasets and calibration.}
Our main evaluation task is \emph{TriviaQA}. For task-specific calibration we use 512 examples per setting. For mixed calibration, we form an equal-sized pool drawn from \emph{math}, \emph{trivia}, and \emph{code} prompts (total 2048 examples; balanced across the three). % clarify totals if different
\paragraph{Implementation and reproducibility.}
Experiments run on 8$\times$A40 NVIDIA GPUs. Unless stated otherwise, generation uses greedy decoding (\texttt{do\_sample=False}) with \texttt{max\_new\_tokens}=64, \texttt{score\_max\_texts}=256, \texttt{score\_max\_tokens}=512, and batch size 8.
For our methods, defaults are: reservoir size 256; scoring batch size 8; information/variance weights $(\alpha,\beta)=(0.5,0.5)$; first/last two layers left at W\textbackslash O; mid-layer budgets selected automatically (approximately 15\% of non-edge layers at 16-bit and 45\% at 8-bit by rank); per-group size 128; cached dequantization enabled.
\subsection{Experimental Results}
We first compare depth profiles under task-specific calibration pools (Section~\ref{subsec:motivation-objective}).
We observe sharp cross-task differences in the distribution of task-salient signals, arguing against uniform bitwidth allocation and motivating task-aware precision assignment.
Restricting AWQ calibration to the target task (\emph{AWQ-OT}) improves over mixed-pool AWQ on some models (e.g., Qwen2.5 and Mistral-7B), but gains are inconsistent and sometimes negligible. On \emph{Phi-4}, AWQ-OT collapses (0.73 EM / 4.54 F1), indicating that calibration alone cannot ensure stability without task-aware layer selection.
\begin{table*}[t]
\caption{TriviaQA results (Exact Match and F1).
``W\textbackslash O'' denotes performance \emph{without quantization}.
For each model, the best quantized result is \textbf{bolded} and the second-best is \underline{underlined}.}
\centering
\resizebox{\textwidth}{!}{%
\begin{tabular}{lcc|cc|cc|cc|cc}
\toprule
\multirow{2}{*}{Method}
& \multicolumn{2}{c|}{\textbf{Phi-4}}
& \multicolumn{2}{c|}{\textbf{Qwen3}}
& \multicolumn{2}{c|}{\textbf{Llama-3.1}}
& \multicolumn{2}{c|}{\textbf{Qwen2.5}}
& \multicolumn{2}{c}{\textbf{Mistral}} \\
& EM & F1 & EM & F1 & EM & F1 & EM & F1 & EM & F1 \\
\midrule
W\textbackslash O & 43.16 & 51.24 & 11.23 & 26.33 & 57.86 & 66.00 & 11.43 & 21.90 & 18.90 & 35.18 \\
\addlinespace[2pt]\cdashline{1-11}\addlinespace[2pt]
TAQO (Ours) & 0.73 & 4.57 & \textbf{18.51} & \textbf{34.34} & \textbf{57.91} & \underline{65.14} & \textbf{12.65} & \textbf{25.49} & \underline{18.51} & \underline{34.34} \\
TAQ (Ours) & \textbf{42.33} & \textbf{50.81} & \underline{11.82} & \underline{26.12} & \underline{57.03} & \textbf{65.35} & \underline{12.06} & 22.47 & \textbf{18.75} & \textbf{34.74} \\
AWQ & \underline{2.25} & \underline{7.07} & 9.23 & 23.94 & 53.37 & 62.23 & 11.91 & 24.06 & 17.48 & 33.75 \\
AWQ-OT & 0.73 & 4.54 & 9.81 & 24.33 & 55.66 & 62.92 & 10.59 & \underline{24.21} & 17.58 & 26.33 \\
\bottomrule
\end{tabular}
}
\label{tab:all-models}
\end{table*}
Both \textbf{TAQ} and \textbf{TAQO} outperform AWQ-OT across models.
On \emph{Phi-4}, \textbf{TAQ} preserves near-W\textbackslash O accuracy (42.33 EM / 50.81 F1 vs.\ 43.16 / 51.24), a margin of $+41.6$ EM and $+46.3$ F1 over AWQ-OT.
On \emph{Qwen3}, \textbf{TAQ} attains 11.82 / 26.12 (within $\le$1 point of W\textbackslash O), while \textbf{TAQO} reaches 18.51 / 34.34, illustrating the benefit of oracle layer protection.
On \emph{Llama-3.1}, \textbf{TAQ} nearly matches W\textbackslash O (57.03 / 65.35 vs.\ 57.86 / 66.00) and surpasses AWQ and AWQ-OT; \textbf{TAQO} slightly improves F1 (57.91 / 65.14).
On \emph{Qwen2.5}, \textbf{TAQ} yields 12.06 / 22.47, and \textbf{TAQO} improves to 12.65 / 25.49.
Finally, on \emph{Mistral}, \textbf{TAQ} posts 18.75 / 34.74, closely tracking W\textbackslash O (18.90 / 35.18) and surpassing AWQ-OT (17.58 / 26.33).
Across models, \textbf{TAQ} offers a stronger Pareto frontier than baselines: accuracy remains within $\leq$1 EM/F1 point of W\textbackslash O while reducing memory and latency. \textbf{TAQO} validates the layer-selection principle by reaching or exceeding W\textbackslash O on some settings (e.g., Qwen3), confirming that protecting critical layers is the right axis of optimization. Together, the results show that task-aware precision—guided by proxy signals or oracle sensitivities—yields stable and efficient quantization.
\noindent\textbf{Takeaways.}
(1) Task-specific representation profiles differ across domains, requiring task-aware allocation.
(2) AWQ-OT improves on some models but remains unstable on others (e.g., \emph{Phi-4}), indicating that task-specific calibration alone is insufficient without task-aware layer selection.
(3) \textbf{TAQ} consistently recovers near-W\textbackslash O accuracy with large efficiency gains; \textbf{TAQO} can further improve accuracy, supporting the underlying layer-selection principle.
\section{Conclusions}
\label{sec:discussion}
We explored task-aware mixed-precision quantization for LLMs, asking whether precision should follow where task evidence resides. We found that layerwise profiles vary systematically across domains: TriviaQA tends to peak late; MBPP concentrates mid-to-late; and GSM8K exhibits a broader distribution—clarifying why task-agnostic PTQ often misallocates precision. \textbf{TAQ} addresses this by coupling entropy- and stability-based signals with a budgeted allocator, consistently preserving near-W\textbackslash O accuracy within $\leq 1$ point (e.g., Llama-3.1: 57.03/65.35 vs.\ 57.86/66.00; Phi-4: 42.33/50.81 vs.\ 43.16/51.24), while outperforming activation-aware baselines such as AWQ. Compared to task-calibrated AWQ (AWQ-OT), \textbf{TAQ} avoids catastrophic collapse (Phi-4: 0.73/4.54 under AWQ-OT) and delivers stable gains.
In addition, \textbf{TAQO} establishes an upper bound by perfectly retaining critical layers and, in some cases, surpasses W\textbackslash O accuracy (Qwen3: 18.51/34.34 vs.\ 11.23/26.33), confirming that task-specific sensitivity is decisive. While impractical for deployment, \textbf{TAQO} validates the premise for lightweight proxies, showing that quantization can serve not only as compression but also as a lens on how tasks are encoded across depth.
Beyond efficiency, the interpretability angle is notable: task-conditioned profiles surface representational ``pressure points'' aligned with task-salient computation, turning quantization into a tool for both compression and probing hidden structure. Limitations include dependence on small calibration sets and sensitivity to deployment hardware. Promising directions include dynamic, input-conditioned bit schedules and multi-task allocation. Overall, our method demonstrates that lightweight, task-conditioned signals can preserve accuracy where it matters most.
\begin{thebibliography}{72}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
\providecommand{\doi}[1]{doi: #1}\else
\providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi
\bibitem[Abdin \& et~al.(2024)Abdin and et~al.]{phi4tr}
Abdin, M. and et~al.
\newblock Phi-4 technical report.
\newblock \emph{arXiv preprint arXiv:2412.08905}, 2024.
\newblock URL \url{https://arxiv.org/abs/2412.08905}.
\bibitem[AI(2023)]{mistral}
AI, M.
\newblock Mistral 7b.
\newblock \url{https://mistral.ai/news/announcing-mistral-7b/}, 2023.
\bibitem[Arditi et~al.(2024)Arditi, Roberts, Stewart, Turner, and Thiergart]{arditi2024refusal}
Arditi, A., Roberts, O., Stewart, A., Turner, A., and Thiergart, J.
\newblock Refusal in language models is mediated by a single direction.
\newblock In \emph{arXiv preprint arXiv:2406.11717}, 2024.
\bibitem[Ashkboos et~al.(2024)Ashkboos, Frantar, Hoefler, and Alistarh]{ashkboos2024quarot}
Ashkboos, S., Frantar, E., Hoefler, T., and Alistarh, D.
\newblock Quarot: Quantization with rotation for large language models.
\newblock \emph{arXiv preprint arXiv:2404.00456}, 2024.
\newblock URL \url{https://arxiv.org/abs/2404.00456}.
\bibitem[Austin et~al.(2021)Austin, Odena, Nye, Bosma, Michalewski, Dohan, Jiang, Cai, Terry, Le, et~al.]{austin2021program}
Austin, J., Odena, A., Nye, M., Bosma, M., Michalewski, H., Dohan, D., Jiang, E., Cai, C., Terry, M., Le, Q., et~al.
\newblock Program synthesis with large language models.
\newblock \emph{arXiv preprint arXiv:2108.07732}, 2021.
\bibitem[Banner et~al.(2019)Banner, Nahshan, and Soudry]{banner2019post}
Banner, R., Nahshan, Y., and Soudry, D.
\newblock Post training 4-bit quantization of convolutional networks for rapid-deployment.
\newblock \emph{Advances in neural information processing systems}, 32, 2019.
\bibitem[Bowen et~al.(2024)Bowen, Freedman, Zhang, and Belinkov]{bowen2024sta}
Bowen, J., Freedman, S., Zhang, Z., and Belinkov, Y.
\newblock Selective task arithmetic: Per-vector selection for robust model editing and merging.
\newblock In \emph{Proceedings of the 41st International Conference on Machine Learning (ICML Workshop/Proceedings Track)}, 2024.
\newblock URL \url{https://arxiv.org/abs/2404.03592}.
\newblock arXiv:2404.03592.
\bibitem[Burns et~al.(2023)Burns, Ye, Klein, and Steinhardt]{burns2022latent}
Burns, C., Ye, H., Klein, D., and Steinhardt, J.
\newblock Discovering latent knowledge in language models without supervision.
\newblock In \emph{International Conference on Learning Representations}, 2023.
\newblock URL \url{https://openreview.net/forum?id=0c-S0RyWhq}.
\bibitem[Chen et~al.(2021)]{chen2021codex}
Chen, M. et~al.
\newblock Evaluating large language models trained on code.
\newblock \emph{arXiv preprint arXiv:2107.03374}, 2021.
\bibitem[Cobbe et~al.(2021{\natexlab{a}})Cobbe, Kosaraju, Bavarian, Chen, Jun, Kaiser, Plappert, Tworek, Hilton, Nakano, et~al.]{cobbe2021training}
Cobbe, K., Kosaraju, V., Bavarian, M., Chen, M., Jun, H., Kaiser, L., Plappert, M., Tworek, J., Hilton, J., Nakano, R., et~al.
\newblock Training verifiers to solve math word problems.
\newblock \emph{arXiv preprint arXiv:2110.14168}, 2021{\natexlab{a}}.
\bibitem[Cobbe et~al.(2021{\natexlab{b}})]{cobbe2021gsm8k}
Cobbe, K. et~al.
\newblock Training verifiers to solve math word problems.
\newblock \emph{arXiv preprint arXiv:2110.14168}, 2021{\natexlab{b}}.
\bibitem[Dao et~al.(2022)Dao, Fu, Ermon, Rudra, and R{\'e}]{dao2022flashattention}
Dao, T., Fu, D.~Y., Ermon, S., Rudra, A., and R{\'e}, C.
\newblock Flash{A}ttention: Fast and memory-efficient exact attention with {IO}-awareness.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2022.
\newblock URL \url{https://openreview.net/forum?id=JENyE4ZG5b}.
\bibitem[Dettmers et~al.(2022)Dettmers, Lewis, Belkada, and Zettlemoyer]{dettmers2022llmint8}
Dettmers, T., Lewis, M., Belkada, Y., and Zettlemoyer, L.
\newblock Llm.int8(): 8-bit matrix multiplication for transformers at scale.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2022.
\newblock URL \url{https://arxiv.org/abs/2208.07339}.
\bibitem[Domin{\'e} et~al.(2024)Domin{\'e}, Anguita, Proca, Braun, Kunin, Mediano, and Saxe]{domine2024lazy}
Domin{\'e}, C.~C., Anguita, N., Proca, A.~M., Braun, L., Kunin, D., Mediano, P.~A., and Saxe, A.~M.
\newblock From lazy to rich: Exact learning dynamics in deep linear networks.
\newblock \emph{arXiv preprint arXiv:2409.14623}, 2024.
\bibitem[Dong et~al.(2019)Dong, Yao, Gholami, Mahoney, and Keutzer]{dong2019hawq}
Dong, Z., Yao, Z., Gholami, A., Mahoney, M.~W., and Keutzer, K.
\newblock Hawq: Hessian aware quantization of neural networks with mixed-precision.
\newblock In \emph{Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, pp.\ 293--302, 2019.
\bibitem[Dubey \& et~al.(2024)Dubey and et~al.]{llama3herd}
Dubey, A. and et~al.
\newblock The llama 3 herd of models.
\newblock \emph{arXiv preprint arXiv:2407.21783}, 2024.
\newblock URL \url{https://arxiv.org/abs/2407.21783}.
\bibitem[Evci et~al.(2022)Evci, Dumoulin, Larochelle, and Mozer]{evci2022head2toe}
Evci, U., Dumoulin, V., Larochelle, H., and Mozer, M.~C.
\newblock Head2toe: Utilizing intermediate representations for better transfer learning.
\newblock In \emph{Proceedings of the 39th International Conference on Machine Learning}, volume 162 of \emph{Proceedings of Machine Learning Research}. PMLR, 2022.
\newblock URL \url{https://proceedings.mlr.press/v162/evci22a.html}.
\bibitem[Frantar et~al.(2022)Frantar, Ashkboos, Hoefler, and Alistarh]{frantar2022gptq}
Frantar, E., Ashkboos, S., Hoefler, T., and Alistarh, D.
\newblock Gptq: Accurate post-training quantization for generative pre-trained transformers.
\newblock In \emph{Proceedings of the International Conference on Learning Representations (ICLR)}, 2022.
\bibitem[Gholami et~al.(2021)Gholami, Kim, Dong, Yao, Mahoney, and Keutzer]{gholami2021quantization}
Gholami, A., Kim, S., Dong, Z., Yao, Z., Mahoney, M.~W., and Keutzer, K.
\newblock A survey of quantization methods for efficient neural network inference.
\newblock \emph{arXiv preprint arXiv:2103.13630}, 2021.
\newblock URL \url{https://arxiv.org/abs/2103.13630}.
\bibitem[He et~al.(2024)]{he2024zipcache}
He, {\,}. et~al.
\newblock Zipcache: Byte-level kv cache compression for transformer inference.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2024.
\newblock URL \url{https://proceedings.neurips.cc/paper_files/paper/2024/file/7e57131fdeb815764434b65162c88895-Paper-Conference.pdf}.
\bibitem[Hendrycks \& et~al.(2021)Hendrycks and et~al.]{hendrycks2021math}
Hendrycks, D. and et~al.
\newblock {MATH}: Measuring mathematical problem solving with the {MATH} dataset.
\newblock \emph{arXiv preprint arXiv:2103.03874}, 2021.
\newblock URL \url{https://arxiv.org/abs/2103.03874}.
\bibitem[Hinton et~al.(2015)Hinton, Vinyals, and Dean]{hinton2015distill}
Hinton, G., Vinyals, O., and Dean, J.
\newblock Distilling the knowledge in a neural network.
\newblock \emph{arXiv preprint arXiv:1503.02531}, 2015.
\newblock URL \url{https://arxiv.org/abs/1503.02531}.
\bibitem[Hoffmann et~al.(2022)Hoffmann, Borgeaud, Mensch, Buchatskaya, Cai, Rutherford, Casas, Welbl, Clark, Hennigan, et~al.]{hoffmann2022chinchilla}
Hoffmann, J., Borgeaud, S., Mensch, A., Buchatskaya, E., Cai, T., Rutherford, E., Casas, D., Welbl, J., Clark, A., Hennigan, T., et~al.
\newblock Training compute-optimal large language models.
\newblock \emph{arXiv preprint arXiv:2203.15556}, 2022.
\newblock URL \url{https://arxiv.org/abs/2203.15556}.
\bibitem[Hooper et~al.(2024)]{hooper2024kvquant}
Hooper, R. et~al.
\newblock Kvquant: Towards general and efficient kv-cache quantization for large language models.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2024.
\newblock URL \url{https://nips.cc/virtual/2024/poster/97760}.
\bibitem[Hu et~al.(2021)Hu, Shen, Wallis, Allen-Zhu, Li, Wang, Wang, and Chen]{hu2021lora}
Hu, E.~J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., and Chen, W.
\newblock Lora: Low-rank adaptation of large language models.
\newblock \emph{arXiv preprint arXiv:2106.09685}, 2021.
\newblock URL \url{https://arxiv.org/abs/2106.09685}.
\bibitem[Ilharco et~al.(2023)Ilharco, Ribeiro, Wortsman, Gururangan, Schmidt, Hajishirzi, and Farhadi]{ilharco2022taskarithmetic}
Ilharco, G., Ribeiro, M.~T., Wortsman, M., Gururangan, S., Schmidt, L., Hajishirzi, H., and Farhadi, A.
\newblock Editing models with task arithmetic.
\newblock In \emph{International Conference on Learning Representations}, 2023.
\newblock URL \url{https://arxiv.org/abs/2212.04089}.
\bibitem[Jacob et~al.(2018)Jacob, Kligys, Chen, Zhu, Tang, Howard, Adam, and Kalenichenko]{jacob2018integer}
Jacob, B., Kligys, S., Chen, B., Zhu, M., Tang, M., Howard, A., Adam, H., and Kalenichenko, D.
\newblock Quantization and training of neural networks for efficient integer-arithmetic-only inference.
\newblock In \emph{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, pp.\ 2704--2713, 2018.
\newblock URL \url{https://openaccess.thecvf.com/content_cvpr_2018/html/Jacob_Quantization_and_Training_CVPR_2018_paper.html}.
\bibitem[Joshi et~al.(2017)Joshi, Choi, Weld, and Zettlemoyer]{joshi2017triviaqa}
Joshi, M., Choi, E., Weld, D.~S., and Zettlemoyer, L.
\newblock Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension.
\newblock In \emph{Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, pp.\ 1601--1611, 2017.
\bibitem[Kaplan et~al.(2020)Kaplan, McCandlish, Henighan, Brown, Chess, Child, Gray, Radford, Wu, and Amodei]{kaplan2020scaling}
Kaplan, J., McCandlish, S., Henighan, T., Brown, T.~B., Chess, B., Child, R., Gray, S., Radford, A., Wu, J., and Amodei, D.
\newblock Scaling laws for neural language models.
\newblock \emph{arXiv preprint arXiv:2001.08361}, 2020.
\newblock URL \url{https://arxiv.org/abs/2001.08361}.
\bibitem[Krishnamoorthi(2018)]{krishnamoorthi2018whitepaper}
Krishnamoorthi, R.
\newblock Quantizing deep convolutional networks for efficient inference: A whitepaper.
\newblock \emph{arXiv preprint arXiv:1806.08342}, 2018.
\newblock URL \url{https://arxiv.org/abs/1806.08342}.
\bibitem[Kwon et~al.(2023)Kwon, Li, Zhuang, Sheng, Zheng, Yu, Gonzalez, Zhang, and Stoica]{kwon2023pagedattention}
Kwon, W., Li, Z., Zhuang, S., Sheng, Y., Zheng, L., Yu, C.~H., Gonzalez, J.~E., Zhang, H., and Stoica, I.
\newblock Efficient memory management for large language model serving with pagedattention.
\newblock \emph{arXiv preprint arXiv:2309.06180}, 2023.
\newblock URL \url{https://arxiv.org/abs/2309.06180}.
\bibitem[L{\"a}hner \& Moeller(2024)L{\"a}hner and Moeller]{lahner2024direct}
L{\"a}hner, Z. and Moeller, M.
\newblock On the direct alignment of latent spaces.
\newblock In \emph{Proceedings of UniReps: the First Workshop on Unifying Representations in Neural Models}, pp.\ 158--169. PMLR, 2024.
\bibitem[Levi et~al.(2025{\natexlab{a}})Levi, Himelstein, Nemcovsky, Mendelson, and Baskin]{levi2025enhancing}
Levi, A., Himelstein, R., Nemcovsky, Y., Mendelson, A., and Baskin, C.
\newblock Enhancing jailbreak attacks via compliance-refusal-based initialization.
\newblock \emph{arXiv e-prints}, pp.\ arXiv--2502, 2025{\natexlab{a}}.
\bibitem[Levi et~al.(2025{\natexlab{b}})Levi, Himelstein, Nemcovsky, Mendelson, and Baskin]{levi2025jailbreak}
Levi, A., Himelstein, R., Nemcovsky, Y., Mendelson, A., and Baskin, C.
\newblock Jailbreak attack initializations as extractors of compliance directions.
\newblock \emph{arXiv preprint arXiv:2502.09755}, 2025{\natexlab{b}}.
\bibitem[Li et~al.(2024)Li, Yao, Zhang, and Li]{li2024safetylayers}
Li, S., Yao, L., Zhang, L., and Li, Y.
\newblock Safety layers in aligned large language models: The key to llm security.
\newblock \emph{arXiv preprint arXiv:2408.17003}, 2024.
\newblock URL \url{https://arxiv.org/abs/2408.17003}.
\newblock ICLR 2025 (OpenReview).
\bibitem[Lin et~al.(2023{\natexlab{a}})Lin, Tang, Tang, Yang, Dang, and Han]{lin2023awq}
Lin, J., Tang, J., Tang, H., Yang, S., Dang, X., and Han, S.
\newblock Awq: Activation-aware weight quantization for llm compression and acceleration.
\newblock In \emph{Proceedings of the Conference on Neural Information Processing Systems (NeurIPS)}, 2023{\natexlab{a}}.
\bibitem[Lin et~al.(2023{\natexlab{b}})Lin, Tang, Tang, Yang, et~al.]{awq_neurips}
Lin, J., Tang, J., Tang, H., Yang, S., et~al.
\newblock {AWQ}: Activation-aware weight quantization for llm compression and acceleration.
\newblock \emph{arXiv preprint arXiv:2306.00978}, 2023{\natexlab{b}}.
\newblock URL \url{https://arxiv.org/abs/2306.00978}.
\bibitem[Lin et~al.(2024)Lin, Tang, Li, Zhang, et~al.]{awq_mlsys}
Lin, J., Tang, H., Li, Z., Zhang, H., et~al.
\newblock {AWQ}: Activation-aware weight quantization for on-device llm compression and acceleration.
\newblock In \emph{MLSys}, 2024.
\newblock URL \url{https://proceedings.mlsys.org/paper_files/paper/2024/file/42a452cbafa9dd64e9ba4aa95cc1ef21-Paper-Conference.pdf}.
\bibitem[Liu et~al.(2024{\natexlab{a}})Liu, Yuan, Jin, Zhong, Xu, Braverman, Chen, and Hu]{liu2024kivi}
Liu, Z., Yuan, J., Jin, H., Zhong, S., Xu, Z., Braverman, V., Chen, B., and Hu, X.
\newblock {KIVI}: A tuning-free asymmetric 2bit quantization for {KV} cache.
\newblock In \emph{Proceedings of the 41st International Conference on Machine Learning (ICML)}, volume 235 of \emph{Proceedings of Machine Learning Research}, 2024{\natexlab{a}}.
\newblock URL \url{https://proceedings.mlr.press/v235/liu24bz.html}.
\bibitem[Liu et~al.(2024{\natexlab{b}})Liu, Zhao, Fedorov, Soran, Choudhary, Krishnamoorthi, Chandra, Tian, and Blankevoort]{liu2024spinquant}
Liu, Z., Zhao, C., Fedorov, I., Soran, B., Choudhary, D., Krishnamoorthi, R., Chandra, V., Tian, Y., and Blankevoort, T.
\newblock Spinquant: Llm quantization with learned rotations.
\newblock \emph{arXiv preprint arXiv:2405.16406}, 2024{\natexlab{b}}.
\newblock \doi{10.48550/arXiv.2405.16406}.
\newblock URL \url{https://arxiv.org/abs/2405.16406}.
\newblock ICLR 2025.
\bibitem[Meng et~al.(2022)Meng, Bau, Andonian, and Belinkov]{rome2022meng}
Meng, K., Bau, D., Andonian, A., and Belinkov, Y.
\newblock Locating and editing factual associations in {GPT}.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2022.
\newblock URL \url{https://proceedings.neurips.cc/paper_files/paper/2022/file/6f1d43d5a82a37e89b0665b33bf3a182-Paper-Conference.pdf}.
\bibitem[Moschella et~al.(2022)Moschella, Maiorca, Fumero, Norelli, Locatello, and Rodol{\`a}]{moschella2022relative}
Moschella, L., Maiorca, V., Fumero, M., Norelli, A., Locatello, F., and Rodol{\`a}, E.
\newblock Relative representations enable zero-shot latent space communication.
\newblock \emph{arXiv preprint arXiv:2209.15430}, 2022.
\bibitem[Nagel et~al.(2020)Nagel, Amjad, Van~Baalen, Louizos, and Blankevoort]{nagel2020up}
Nagel, M., Amjad, R.~A., Van~Baalen, M., Louizos, C., and Blankevoort, T.
\newblock Up or down? adaptive rounding for post-training quantization.
\newblock In \emph{International conference on machine learning}, pp.\ 7197--7206. PMLR, 2020.
\bibitem[Nagel et~al.(2021)Nagel, Fournarakis, Amjad, and Bondarenko]{nagel2021white}
Nagel, M., Fournarakis, M., Amjad, R.~A., and Bondarenko, Y.
\newblock A white paper on neural network quantization.
\newblock In \emph{arXiv preprint arXiv:2106.08295}, 2021.
\bibitem[Ni et~al.(2025)Ni, Sun, Xiao, Collins, Liu, and Koyejo]{ni2025caprese}
Ni, R., Sun, S., Xiao, Y.-X., Collins, K., Liu, Z., and Koyejo, S.
\newblock Quantifying knowledge distillation for large language models and beyond.
\newblock \emph{arXiv preprint arXiv:2505.13030}, 2025.
\newblock URL \url{https://arxiv.org/abs/2505.13030}.
\bibitem[Ouyang et~al.(2022)Ouyang, Wu, Jiang, Almeida, Wainwright, Mishkin, Zhang, Agarwal, Slama, Ray, Schulman, Hilton, Kelton, Miller, Simens, Askell, Welinder, Christiano, Leike, and Lowe]{ouyang2022instruct}
Ouyang, L., Wu, J., Jiang, X., Almeida, D., Wainwright, C., Mishkin, P., Zhang, C., Agarwal, S., Slama, K., Ray, A., Schulman, J., Hilton, J., Kelton, L., Miller, L., Simens, M., Askell, A., Welinder, P., Christiano, P., Leike, J., and Lowe, R.
\newblock Training language models to follow instructions with human feedback.
\newblock In \emph{Advances in Neural Information Processing Systems (NeurIPS)}, 2022.
\newblock URL \url{https://arxiv.org/abs/2203.02155}.
\bibitem[Rajpurkar et~al.(2016)Rajpurkar, Jia, and Liang]{rajpurkar2016squad}
Rajpurkar, P., Jia, R., and Liang, P.
\newblock {SQuAD}: 100,000+ questions for machine comprehension of text.
\newblock In \emph{EMNLP}, 2016.
\newblock URL \url{https://arxiv.org/abs/1606.05250}.
\bibitem[Roziere et~al.(2023)]{roziere2023codellama}
Roziere, B. et~al.
\newblock Code llama: Open foundation models for code.
\newblock \emph{arXiv preprint arXiv:2308.12950}, 2023.
\bibitem[Sanh et~al.(2019)Sanh, Debut, Chaumond, and Wolf]{sanh2019distilbert}
Sanh, V., Debut, L., Chaumond, J., and Wolf, T.
\newblock Distilbert, a distilled version of bert: Smaller, faster, cheaper (and lighter).
\newblock In \emph{arXiv preprint arXiv:1910.01108}, 2019.
\bibitem[Shao et~al.(2024)Shao, Chen, Zhang, Xu, Zhao, Li, Zhang, Gao, Qiao, and Luo]{shao2023omniquant}
Shao, W., Chen, M., Zhang, Z., Xu, P., Zhao, L., Li, Z., Zhang, K., Gao, P., Qiao, Y., and Luo, P.
\newblock Omni{Q}uant: Omnidirectionally calibrated quantization for large language models.
\newblock In \emph{International Conference on Learning Representations (ICLR)}, 2024.
\newblock URL \url{https://arxiv.org/abs/2308.13137}.
\newblock ICLR 2024 Camera Ready; original preprint arXiv:2308.13137 (2023).
\bibitem[Sharify et~al.(2024)Sharify, Saxena, Xu, Yazar, Soloveychik, and Wang]{sharify2024posttrainingmix}
Sharify, S., Saxena, U., Xu, Z., Yazar, W., Soloveychik, I., and Wang, X.
\newblock Post training quantization of large language models with microscaling formats.
\newblock In \emph{Proceedings of The 4th NeurIPS Efficient Natural Language and Speech Processing Workshop (ENLSP-V)}, volume 262 of \emph{Proceedings of Machine Learning Research}, pp.\ 241--258. PMLR, 2024.
\newblock URL \url{https://proceedings.mlr.press/v262/sharify24a.html}.
\bibitem[Skean et~al.(2025)Skean, Arefin, Zhao, Patel, Naghiyev, LeCun, and Shwartz-Ziv]{skean2025layer}
Skean, O., Arefin, M.~R., Zhao, D., Patel, N., Naghiyev, J., LeCun, Y., and Shwartz-Ziv, R.
\newblock Layer by layer: Uncovering hidden representations in language models, 2025.
\newblock URL \url{https://arxiv.org/abs/2502.02013}.
\bibitem[Song et~al.(2024)Song, He, Jiang, Xian, Gao, Liu, and Yu]{song2024taskspecific}
Song, R., He, S., Jiang, S., Xian, Y., Gao, S., Liu, K., and Yu, Z.
\newblock Does large language model contain task-specific neurons?
\newblock In \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, 2024.
\newblock URL \url{https://aclanthology.org/2024.emnlp-main.403/}.
\bibitem[Team(2024)]{qwen25tr}
Team, Q.
\newblock Qwen2.5 technical report.
\newblock \url{https://qwenlm.github.io/blog/qwen2.5/}, 2024.
\bibitem[Tsatsaronis et~al.(2015)]{tsatsaronis2015bioasq}
Tsatsaronis, G. et~al.
\newblock An overview of the bioasq large-scale biomedical semantic indexing and question answering competition.
\newblock \emph{BMC Bioinformatics}, 16\penalty0 (1):\penalty0 138, 2015.
\bibitem[Turner et~al.(2023{\natexlab{a}})Turner, Thiergart, Rando, Templeton, and Hendrycks]{turner2023steering}
Turner, A., Thiergart, J., Rando, J., Templeton, A., and Hendrycks, D.
\newblock Steering language models with activation engineering.
\newblock In \emph{arXiv preprint arXiv:2308.10248}, 2023{\natexlab{a}}.
\bibitem[Turner et~al.(2023{\natexlab{b}})Turner, Thiergart, Leech, Udell, Vazquez, Mini, and MacDiarmid]{turner2023activation}
Turner, A.~M., Thiergart, L., Leech, G., Udell, D., Vazquez, J.~J., Mini, U., and MacDiarmid, M.
\newblock Steering language models with activation engineering.
\newblock \emph{arXiv preprint arXiv:2308.10248}, 2023{\natexlab{b}}.
\newblock \doi{10.48550/arXiv.2308.10248}.
\newblock URL \url{https://arxiv.org/abs/2308.10248}.
\bibitem[Turner et~al.(2024)Turner, Thiergart, Leech, Udell, Vazquez, Mini, and MacDiarmid]{turner2024activationaddition}
Turner, A.~M., Thiergart, L., Leech, G., Udell, D., Vazquez, J.~J., Mini, U., and MacDiarmid, M.
\newblock Activation addition: Steering language models without optimization.
\newblock \emph{arXiv preprint arXiv:2308.10248}, 2024.
\newblock URL \url{https://arxiv.org/abs/2308.10248}.
\bibitem[Vaswani et~al.(2017)Vaswani, Shazeer, Parmar, Uszoreit, Jones, Gomez, Kaiser, and Polosukhin]{vaswani2017attention}
Vaswani, A., Shazeer, N., Parmar, N., Uszoreit, J., Jones, L., Gomez, A.~N., Kaiser, L.~u., and Polosukhin, I.
\newblock Attention is all you need.
\newblock In \emph{Proceedings of the 31st Conference on Neural Information Processing Systems (NeurIPS)}, pp.\ 5998--6008, 2017.
\bibitem[Wang et~al.(2020)Wang, Chen, He, and Cheng]{wang2020towards}
Wang, P., Chen, Q., He, X., and Cheng, J.
\newblock Towards accurate post-training network quantization via bit-split and stitching.
\newblock In \emph{International Conference on Machine Learning}, pp.\ 9847--9856. PMLR, 2020.
\bibitem[Williams et~al.(2021)Williams, Kunz, Kornblith, and Linderman]{williams2021generalized}
Williams, A.~H., Kunz, E., Kornblith, S., and Linderman, S.
\newblock Generalized shape metrics on neural representations.
\newblock \emph{Advances in neural information processing systems}, 34:\penalty0 4738--4750, 2021.
\bibitem[Xiao et~al.(2023)Xiao, Lin, Seznec, Wu, Demouth, and Han]{xiao2023smoothquant}
Xiao, G., Lin, J., Seznec, M., Wu, H., Demouth, J., and Han, S.
\newblock Smoothquant: Accurate and efficient post-training quantization for large language models.
\newblock In \emph{arXiv preprint arXiv:2211.10438}, 2023.
\bibitem[Yao et~al.(2022)Yao, Aminabadi, Zhang, Li, Zhang, and Wang]{yao2022zeroquant}
Yao, Z., Aminabadi, R.~Y., Zhang, M., Li, X., Zhang, Z., and Wang, Y.
\newblock Zeroquant: Efficient and affordable post-training quantization for large-scale transformers.
\newblock In \emph{Proceedings of the 36th Conference on Neural Information Processing Systems (NeurIPS)}, pp.\ 31468--31482, 2022.
\bibitem[Yao et~al.(2023)Yao, Li, Wu, Youn, and He]{yao2023zeroquantv2}
Yao, Z., Li, C., Wu, X., Youn, S., and He, Y.
\newblock A comprehensive study on post-training quantization for large language models.
\newblock \emph{arXiv preprint arXiv:2303.08302}, 2023.
\newblock URL \url{https://arxiv.org/abs/2303.08302}.
\bibitem[Yuan et~al.(2023)Yuan, Niu, Liu, Liu, Wang, Shang, Sun, Wu, Wu, and Wu]{yuan2023rptq}
Yuan, Z., Niu, L., Liu, J., Liu, W., Wang, X., Shang, Y., Sun, G., Wu, Q., Wu, J., and Wu, B.
\newblock Rptq: Reorder-based post-training quantization for large language models.
\newblock \emph{arXiv preprint arXiv:2304.01089}, 2023.
\newblock URL \url{https://arxiv.org/abs/2304.01089}.
\bibitem[Zeng et~al.(2025)Zeng, Liu, Zhang, Zhu, Chen, Rivera, van Schijndel, Saffari, Poliak, Tsvetkov, and Sedoc]{zeng2025taskvectorbases}
Zeng, H., Liu, S., Zhang, X., Zhu, C., Chen, X., Rivera, C., van Schijndel, M., Saffari, A., Poliak, A., Tsvetkov, Y., and Sedoc, J.
\newblock Efficient model editing with task vector bases.
\newblock \emph{arXiv preprint arXiv:2501.09248}, 2025.
\newblock URL \url{https://arxiv.org/abs/2501.09248}.
\bibitem[Zhang et~al.(2024)Zhang, Dong, and Kawaguchi]{zhang2024layerimportance}
Zhang, Y., Dong, Y., and Kawaguchi, K.
\newblock Investigating layer importance in large language models.
\newblock \emph{arXiv preprint arXiv:2409.14381}, 2024.
\newblock URL \url{https://arxiv.org/abs/2409.14381}.
\bibitem[Zhao et~al.(2023)Zhao, Gower, Walters, and Yu]{zhao2023improving}
Zhao, B., Gower, R.~M., Walters, R., and Yu, R.
\newblock Improving convergence and generalization using parameter symmetries.
\newblock \emph{arXiv preprint arXiv:2305.13404}, 2023.
\bibitem[Zhou et~al.(2024)Zhou, Gao, Song, Zhou, Zha, Li, and et~al.]{zhou2024kdsurvey}
Zhou, L., Gao, R., Song, G., Zhou, L., Zha, H., Li, H., and et~al.
\newblock Knowledge distillation of large language models: A survey.
\newblock \emph{arXiv preprint arXiv:2405.12396}, 2024.
\newblock URL \url{https://arxiv.org/abs/2405.12396}.
\bibitem[Zhu et~al.(2023)Zhu, Li, Liu, Ma, and Wang]{zhu2023llmcompresssurvey}
Zhu, X., Li, J., Liu, Y., Ma, C., and Wang, W.
\newblock A survey on model compression for large language models.
\newblock \emph{arXiv preprint arXiv:2308.07633}, 2023.
\newblock URL \url{https://arxiv.org/abs/2308.07633}.
\bibitem[Zou et~al.(2023{\natexlab{a}})Zou, Phan, Chen, Campbell, Guo, Ren, Pan, Yin, Mazeika, Dombrowski, et~al.]{zou2023representation}
Zou, A., Phan, L., Chen, S., Campbell, J., Guo, P., Ren, B., Pan, A., Yin, X., Mazeika, M., Dombrowski, A.-K., et~al.
\newblock Representation engineering: A top-down approach to ai transparency.
\newblock In \emph{arXiv preprint arXiv:2310.01405}, 2023{\natexlab{a}}.
\bibitem[Zou et~al.(2023{\natexlab{b}})Zou, Wang, Carlini, Nasr, Kolter, Fredrikson, et~al.]{repeng2023zou}
Zou, A., Wang, Z., Carlini, N., Nasr, M., Kolter, J.~Z., Fredrikson, M., et~al.
\newblock Representation engineering: A top-down approach to {AI} transparency.
\newblock \emph{arXiv preprint arXiv:2310.01405}, 2023{\natexlab{b}}.
\newblock URL \url{https://arxiv.org/abs/2310.01405}.
\end{thebibliography}