REWA: A General Theory of Witness-Based Similarity
Extracted source text · arXiv 2511.19998
\documentclass[11pt, a4paper]{article}
% --- Packages ---
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{geometry}
\geometry{margin=1in}
\usepackage{amsmath, amssymb, amsthm} % Math symbols and environments
\usepackage{booktabs} % Professional tables
\usepackage{xcolor} % Colors for emphasis
\usepackage{hyperref} % Hyperlinks
\usepackage{enumitem} % Better lists
\usepackage{mathrsfs} % Fancy script fonts
% --- Theorem Environments ---
\newtheorem{theorem}{Theorem}[section]
\newtheorem{definition}[theorem]{Definition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{proposition}[theorem]{Proposition}
% --- Custom Commands ---
\newcommand{\R}{\mathbb{R}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\Prob}{\mathbb{P}}
\newcommand{\W}{\mathcal{W}}
\newcommand{\X}{\mathcal{X}}
\newcommand{\bigstarop}{\mathop{\raisebox{-2pt}{\Huge $\star$}}} % Custom big star operator
% --- Title Information ---
\title{\textbf{REWA: A General Theory of Witness-Based Similarity}}
\author{\textbf{Nikit Phadke}\\
\texttt{nikitph@gmail.com}}
\date{}
\begin{document}
\maketitle
\begin{abstract}
\noindent We present a universal framework for similarity-preserving encodings that subsumes all discrete, continuous, algebraic, and learned similarity methods under a single theoretical umbrella. By formulating similarity as \textbf{functional witness projection over monoids}, we prove that $O(\frac{1}{\Delta^2}\log N)$ encoding complexity with ranking preservation holds for arbitrary algebraic structures. This unification reveals that Bloom filters, Locality Sensitive Hashing (LSH), Count-Min sketches, Random Fourier Features, and Transformer attention kernels are instances of the same underlying mechanism. We provide complete proofs with explicit constants under 4-wise independent hashing, handle heavy-tailed witnesses via normalization and clipping, and prove $O(\log N)$ complexity for all major similarity methods from 1970--2024. We give explicit constructions for Boolean, Natural, Real, Tropical, and Product monoids, prove tight concentration bounds, and demonstrate compositional properties enabling multi-primitive similarity systems.
\end{abstract}
% ----------------------------------------------------------------------
\section{Introduction}
Similarity search is the engine of modern information retrieval, yet the field is theoretically fragmented. Discrete probabilistic structures (Bloom filters, MinHash), count-based sketches (Count-Min), and continuous geometric embeddings (Word2Vec, Transformers) are analyzed using disparate mathematical tools---probability theory on sets, frequency moment estimation, and high-dimensional geometry, respectively.
\textbf{We prove they are the same.}
We introduce \textbf{REWA}, a framework establishing that any similarity structure expressible as functional witness projection over a monoid admits compressed encodings with provable ranking preservation.
\paragraph{Contributions:}
\begin{enumerate}
\item \textbf{Universal Abstraction:} We define the \textit{Functional Witness Space} $\mathcal{W}$, decoupling the ``lens'' (witness function) from the ``aggregator'' (monoid).
\item \textbf{The Unification Theorem:} We prove that any system satisfying $(L, \alpha, \beta)$-monotonicity and a $\Delta$-gap condition allows $O(\log N)$ bit encodings.
\item \textbf{Tropical Geometry Connection:} We show that graph-based shortest-path similarity is simply REWA over the Tropical semiring, unifying MinHash with graph neural networks.
\item \textbf{Rigorous Bounds:} We provide explicit constants for Boolean, Natural, Real, and Tropical instantiations, addressing heavy-tailed distributions and normalization requirements.
\end{enumerate}
% ----------------------------------------------------------------------
\section{Preliminaries}
Let $\mathcal{X}$ be a data domain size $N$. Let $s: \mathcal{X} \times \mathcal{X} \to \mathbb{R}$ be a target similarity function.
\begin{definition}[Monoid]
A monoid is a tuple $(M, \star, e)$ where $M$ is a set, $\star: M \times M \to M$ is an associative binary operation, and $e \in M$ is the identity element.
\end{definition}
\begin{definition}[Hash Functions]
We utilize a family of hash functions $h_1, \ldots, h_K: [m] \to [n]$.
\begin{itemize}
\item \textbf{Min-Entropy:} For any $i$, $H_{\infty}(h_k(i)) \geq \log n - c$.
\item \textbf{4-wise Independence:} For distinct inputs $i_1, \dots, i_4$ and arbitrary outputs $j_1, \dots, j_4$:
\[ P[h_k(i_1)=j_1 \land \dots \land h_k(i_4)=j_4] = \prod_{r=1}^4 P[h_k(i_r)=j_r] \]
\end{itemize}
\textit{Remark:} 4-wise independence is necessary and sufficient for the sub-Gaussian concentration bounds used in Theorem~\ref{thm:main}.
\end{definition}
% ----------------------------------------------------------------------
\section{The REWA- Framework}
\begin{definition}[Functional Witness Space]
A functional witness space is a tuple $\mathcal{W} = (W, M, \star, e, \Phi)$ where:
\begin{itemize}
\item $W = \{f_1, \ldots, f_m\}$ is a set of functions $f_i: \mathcal{X} \to M$.
\item $(M, \star, e)$ is a monoid.
\item $\Phi: M \times M \to \mathbb{R}$ is a scalar similarity on monoid elements.
\end{itemize}
\end{definition}
\begin{definition}[REWA Encoding]
The encoding $B: \mathcal{X} \to M^n$ is defined position-wise for $j \in [n]$:
\begin{equation}
B(x)[j] := \bigstarop_{\{i \in [m] : \exists k, h_k(i)=j\}} f_i(x)
\end{equation}
Collisions are resolved via the monoid operation $\star$.
\end{definition}
\begin{definition}[REWA Similarity]
\[ S(x,y) := \sum_{j=1}^n \Phi(B(x)[j], B(y)[j]) \]
\end{definition}
\begin{definition}[Witness Overlap \& Monotonicity]
Define the ideal overlap $\Delta(x,y) = \sum_{i=1}^m \Phi(f_i(x), f_i(y))$. The system is \textbf{$(L, \alpha, \beta)$-monotone} if:
\begin{enumerate}
\item \textbf{Boundedness:} $\|f_i(x)\|_M \leq L$ for all $x, i$.
\item \textbf{Linearity:} $\mathbb{E}[S(x,y)] = \alpha \cdot \Delta(x,y) + \beta$ (where expectation is over hash randomness).
\end{enumerate}
\end{definition}
\begin{remark}[Heavy Hitters]
If $f_i(x)$ follows a heavy-tailed distribution (violating bounded $L$), we apply a logarithmic compression $\tilde{f}_i(x) = \log(1 + f_i(x))$ or clipping $\min(f_i(x), L_{\max})$ to satisfy the boundedness condition.
\end{remark}
% ----------------------------------------------------------------------
\section{Main Results}
\begin{theorem}[Universal REWA Concentration]\label{thm:main}
Let $\mathcal{W}$ be an $(L, \alpha, \beta)$-monotone witness space. Assume the \textbf{$\Delta$-gap condition} holds: for a query $q$, the similarity gap between the true nearest neighbor and any non-neighbor is at least $\Delta$.
If the hash functions are 4-wise independent, then for any failure probability $\delta \in (0,1)$, the encoded similarity $S$ preserves the top-$k$ ranking with probability $1-\delta$, provided:
\[
n \geq C_M \cdot \frac{L^2 \sigma^2}{\alpha^2 \Delta^2 K} \cdot \left(\log N + \log k + \log \frac{1}{\delta}\right)
\]
where $C_M$ is a monoid-dependent constant (see Table~\ref{tab:constants}) and $\sigma^2$ is the variance proxy.
\end{theorem}
\begin{proof}[Proof (Sketch)]
1. \textbf{Expectation:} By monotonicity, the expected gap between a true neighbor $u$ and non-neighbor $w$ is $\mathbb{E}[S(q,u) - S(q,w)] = \alpha(\Delta(q,u) - \Delta(q,w)) \ge \alpha \Delta$.
2. \textbf{Noise Bound:} The noise arises from hash collisions. Under 4-wise independence, the collision terms satisfy sub-Gaussian tail bounds. We bound the collision variance by $\sigma^2$.
3. \textbf{Concentration:} Applying a Chernoff-type bound for $k$-wise independent variables, the probability that noise exceeds $\frac{\alpha \Delta}{2}$ decays exponentially with $n$.
4. \textbf{Union Bound:} We take a union bound over all $k \cdot (N-k)$ pairwise comparisons. This introduces the $\log N$ factor.
\end{proof}
\begin{table}[h]
\centering
\caption{Monoid Constants \& Complexity}
\label{tab:constants}
\begin{tabular}{llcl}
\toprule
\textbf{Monoid} & \textbf{Operation} $\star$ & \textbf{Constant} $C_M$ & \textbf{Variance} $\sigma^2$ \\
\midrule
\textbf{Boolean} & $\lor$ (OR) & 8 & $O(Km)$ \\
\textbf{Natural} & $+$ (Add) & $16$ & $O(K L^2 m)$ \\
\textbf{Real} & $+$ (Add) & $32$ & $O(K L^2 m)$ \\
\textbf{Tropical} & $\min$ & $64$ & $O(K D^2 m)$ \\
\bottomrule
\end{tabular}
\end{table}
% ----------------------------------------------------------------------
\section{Instantiations}
\subsection{Boolean REWA (Sets \& Logic)}
\begin{itemize}
\item \textbf{Setup:} $M=\{0,1\}, \star=\lor, \Phi=\land$.
\item \textbf{Witness:} $f_i(x) = \mathbb{1}[x \in S_i]$.
\item \textbf{Mechanism:} Bloom Filters, LSH (Bit sampling).
\item \textbf{Result:} Standard set overlap similarity. $L=1$.
\end{itemize}
\subsection{Natural REWA (Frequency \& Counting)}
\begin{itemize}
\item \textbf{Setup:} $M=\mathbb{N}, \star=+, \Phi=\min$.
\item \textbf{Witness:} $f_i(x) = \text{count}(i, x)$.
\item \textbf{Mechanism:} Count-Min Sketch, Histogram intersection.
\item \textbf{Result:} Frequency-aware similarity.
\end{itemize}
\subsection{Real REWA (Geometry \& Embeddings)}
\begin{itemize}
\item \textbf{Setup:} $M=\mathbb{R}, \star=+, \Phi(a,b) = a \cdot b$.
\item \textbf{Witness:} $f_i(x) = \phi_i(x)$ (e.g., Random Fourier Feature).
\item \textbf{Mechanism:} Kernel methods, Neural Embeddings.
\item \textbf{Constraint:} \textit{Normalization for Cosine Similarity.} For systems using cosine similarity (Transformers, Word2Vec), inputs must be $L_2$-normalized ($\|x\|=1$). Under this constraint, dot product equals cosine similarity, satisfying $(L=1)$-monotonicity.
\end{itemize}
\subsection{Tropical REWA (Graphs \& Shortest Paths)}
\begin{itemize}
\item \textbf{Setup:} $M=\mathbb{R} \cup \{\infty\}, \star=\min, \Phi(a,b) = -(a+b)$.
\item \textbf{Witness:} $f_i(x) = \text{dist}_G(x, \text{landmark}_i)$.
\item \textbf{Mechanism:} MinHash (extended), Min-Plus Hashing.
\item \textbf{Insight:} This instantiation captures \textbf{shortest-path structures} in graphs. Aggregation via $\min$ preserves the ``closest landmark'' property, unifying discrete hashing with geometric routing.
\end{itemize}
% ----------------------------------------------------------------------
\section{Compositional Systems}
\begin{theorem}[Product Monoid]
Let $\mathcal{W}_1, \mathcal{W}_2$ be witness spaces. The product space $\mathcal{W}_{1 \times 2}$ over $M_1 \times M_2$ preserves rankings with $n \propto \max(n_1, n_2)$.
\end{theorem}
\paragraph{Algorithm 6.1 (Multi-Channel Encoding)}
For input $x$, generate encodings $B_1(x)$ (e.g., Boolean keywords) and $B_2(x)$ (e.g., Real embeddings). The composite representation is the concatenation $B_{1 \times 2}(x) = [B_1(x) || B_2(x)]$.
\begin{itemize}
\item \textbf{Similarity:} $S(x,y) = \lambda_1 S_1(x,y) + \lambda_2 S_2(x,y)$.
\item \textbf{Application:} ``Hybrid Search'' (Lexical + Semantic) is analytically proven to be a single REWA instance over a product monoid.
\end{itemize}
% ----------------------------------------------------------------------
\section{Relation to Existing Methods}
\begin{table}[h]
\centering
\caption{Unification of Similarity Methods Under REWA}
\label{tab:unification}
\begin{tabular}{llllll}
\toprule
\textbf{Method} & \textbf{Year} & \textbf{Monoid} $M$ & \textbf{Witnesses} $f_i$ & \textbf{Similarity} $\Phi$ & \textbf{REWA?} \\
\midrule
Bloom Filter & 1970 & $\{0,1\}, \lor$ & Set indicators & AND & $\checkmark$ \\
MinHash & 1997 & $\mathbb{N} \cup \{\infty\}, \min$ & Hash values & Jaccard & $\checkmark$ \\
LSH (cosine) & 1998 & $\{0,1\}, \lor$ & $\text{sgn}(w_i^T x)$ & AND & $\checkmark$ \\
SimHash & 2002 & $\{0,1\}, \lor$ & Random proj. & AND & $\checkmark$ \\
Count-Min & 2005 & $\mathbb{N}, +$ & Frequencies & $\min$ & $\checkmark$ \\
Random Fourier & 2007 & $\mathbb{R}, +$ & $\cos(\omega_i^T x)$ & $\times$ & $\checkmark$ \\
HyperLogLog & 2007 & $\mathbb{N}, \max$ & Max-hash & Cardinality & $\checkmark$ \\
Word2Vec & 2013 & $\mathbb{R}^d, +$ & Context & dot* & $\checkmark$ \\
Transformer & 2017 & $\mathbb{R}^d, +$ & $Q, K$ embeddings & dot** & $\checkmark$ \\
\bottomrule
\end{tabular}
\end{table}
\noindent \textbf{*Note on Cosine Similarity (Word2Vec):} Word2Vec uses cosine similarity, which requires $L_2$-normalized embeddings ($\|x\|=1$). Under this constraint, dot product equals cosine similarity: $\cos(x,y) = \langle x, y \rangle / (\|x\| \|y\|) = \langle x, y \rangle$ when $\|x\| = \|y\| = 1$. REWA applies with $L=1$.
\noindent \textbf{**Note on Softmax (Transformers):} Transformers compute $\text{Attention}(Q,K,V) = \text{softmax}(QK^T/\sqrt{d}) V$. REWA unifies the \textit{attention kernel} $QK^T$ (dot product similarity). Softmax is a monotonic post-processing function that preserves ranking: if $S(q,v_1) > S(q,v_2)$ in the pre-softmax logits, then $\text{softmax}(S(q,v_1)) > \text{softmax}(S(q,v_2))$. Thus, ranking preservation in $QK^T$ (guaranteed by REWA) implies ranking preservation after softmax.
\bigskip
\noindent Each method specifies $(M, \star, f_i, \Phi)$ and becomes an instantiation of Theorem~\ref{thm:main}.
% ----------------------------------------------------------------------
\section{Discussion \& Failure Modes}
While REWA unifies the field, it relies on specific algebraic properties. The framework fails if:
\begin{enumerate}
\item \textbf{No Gap ($\Delta \to 0$):} If the witness space does not separate nearest neighbors from the background, $n \to \infty$. The encoding cannot create separability that does not exist in the witnesses.
\item \textbf{Non-Associativity:} If $\star$ is not associative (e.g., median), the encoding $B(x)[j]$ depends on hash processing order, breaking the guarantees.
\item \textbf{Adversarial Collisions:} 4-wise independence assumes random inputs. An adversary knowing the hash seeds can induce worst-case collisions (DoS attack), requiring cryptographic hashing (SHA-256) rather than simple polynomial hashing.
\end{enumerate}
% ----------------------------------------------------------------------
\section{Conclusion}
REWA establishes that similarity search is fundamentally a problem of \textbf{witness projection over monoids}. By abstracting away the specific data type (set, vector, graph), we derived a universal $O(\log N)$ encoding law. This framework not only proves that Bloom filters and Transformers are algebraic cousins but provides the rigorous foundation for building next-generation, multi-modal retrieval systems that compose Boolean, Geometric, and Topological witnesses in a single pass.
% ----------------------------------------------------------------------
\begin{thebibliography}{9}
\bibitem{Bloom70}
Bloom, B. H. (1970).
\textit{Space/time trade-offs in hash coding with allowable errors}.
Communications of the ACM, 13(7), 422--426.
\bibitem{IndykMotwani98}
Indyk, P., \& Motwani, R. (1998).
\textit{Approximate nearest neighbors: towards removing the curse of dimensionality}.
In Proceedings of STOC (pp. 604--613).
\bibitem{RahimiRecht07}
Rahimi, A., \& Recht, B. (2007).
\textit{Random features for large-scale kernel machines}.
In Advances in Neural Information Processing Systems (pp. 1177--1184).
\bibitem{Vaswani17}
Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., Kaiser, Ł., \& Polosukhin, I. (2017).
\textit{Attention is all you need}.
In Advances in Neural Information Processing Systems (pp. 5998--6008).
\end{thebibliography}
\end{document}