diff --git a/fastlib/u/rriegel/nips07/gnp_nips07.bib b/fastlib/u/rriegel/nips07/gnp_nips07.bib index dd18be7c22..8c8e7663f3 100644 --- a/fastlib/u/rriegel/nips07/gnp_nips07.bib +++ b/fastlib/u/rriegel/nips07/gnp_nips07.bib @@ -121,7 +121,14 @@ Year = "2003" } -% Still looking for nbc phystat and nbc compstat +% Still looking for nbc phystat +@Inproceedings{nbc-compstat, +Author = "Ryan Riegel and Alexander Gray", +Title = "{Large-Scale Kernel Discriminant Analysis with Application + to Quasar Discovery}", +Booktitle = "{Proceedings of Computational Statistics}", +Year = "2006" +} @inproceedings{moore-npt, Title = "Fast Algorithms and Efficient Statistics: N-point Correlation Functions", @@ -131,9 +138,27 @@ Year = "2000" } -% Stuff that is too vague to find: npt 2004, Science 2003, Nature 2005 +% Stuff that is too vague to find: npt 2004, Nature 2005 +@article{science03, +author = "Charles Seife", +Title = "{Breakthrough of the Year: Illuminating the Dark Universe}", +Journal = "Science", +Volume = "302", +Number = "5653", +Month = "December 19", +Year = "2003", +pages = "2017--2172" +} -inproceedings{ hochreiter00beyond, +@Article{nature05, +Author = "Mark Peplow", +Title = "{Quasars Reveal Cosmic Magnification}", +Journal = "Nature", +Month = "April 27", +Year = "2005" +} + +@inproceedings{ hochreiter00beyond, author = "Sepp Hochreiter and Michael Mozer", title = "Beyond Maximum Likelihood and Density Estimation: A Sample-Based Criterion for Unsupervised Learning of Complex Models", booktitle = "{NIPS}", @@ -154,7 +179,13 @@ inproceedings{ hochreiter00beyond, % Jeremy Kubica has a ton of papers that look like what you want, google his name to see them - +@inproceedings{kbc:stch, +author = "Jeremy Kubica and Andrew Moore and Jeff Schneider and Yiming Yang", +title = "{Stochastic Link and Group Detection}", +booktitle = "The Eighteenth National Conference on Artificial Intelligence", +month = "Jul", +pages = "798--804", +year = 2002} % These are the commented references in the paper @@ -166,7 +197,7 @@ booktitle = "NIPS", year = "2005", } -@Inproceedings{klass_fast, +@Inproceedings{klaas_fast, Author = "Mike Klaas and Dustin Lang and Nando de Freitas", Title = "{Fast Maximum a Posteriori Inference in Monte Carlo State Spaces}", Booktitle = "{Artificial Intelligence and Statistics 2005}", diff --git a/fastlib/u/rriegel/nips07/gnp_nips07.tex b/fastlib/u/rriegel/nips07/gnp_nips07.tex index b373c73995..2d0667da0a 100644 --- a/fastlib/u/rriegel/nips07/gnp_nips07.tex +++ b/fastlib/u/rriegel/nips07/gnp_nips07.tex @@ -111,7 +111,7 @@ \newcommand{\ignoreall}[1]{} -\title{Some Awesome Title} +\title{A Mathematical Theory for Scalable Learning} \author{ Ryan N.~Riegel \\ @@ -120,6 +120,12 @@ Georgia Institute of Technology \\ Atlanta, GA 30332 \\ \texttt{rriegel@cc.gatech.edu} \\ \And +Garrett F.~Boyer \\ +College of Computiong \\ +Georgia Institute of Technology \\ +Atlanta, GA 30332 \\ +\texttt{garryb@cc.gatech.edu} \\ +\And Alexander G.~Gray \\ College of Computiong \\ Georgia Institute of Technology \\ @@ -144,76 +150,96 @@ We present mathematical foundations for a highly successful algorithmic strategy that has resulted in the fastest algorithms for many machine learning methods and is broadly applicable to scaling many future methods up to large datasets. We formalize for the first -time a class of computational problems which are very common in -machine learning, called {\em generalized $N$-body problems}, and -subsequently present a template {\em generalized $N$-body algorithm}, -which can be specialized to produce efficient problem-dependent -algorithms. We demonstrate the use of this mathematical framework for -deriving the recent affinity propagation method. +time a class of computational problems which are common in machine +learning, called {\em generalized $N$-body problems} (GNP's), and +provide a calculus for simplification of GNP's in various ways. We +then present a template {\em generalized $N$-body algorithm} applying +to any GNP, which can be specialized to mathematically derive +efficient problem-dependent algorithms using the calculus. We +demonstrate the use of this mathematical framework for deriving a fast +algorithm for the recent affinity propagation method. \end{abstract} \section{Introduction} The ability to apply machine learning methods to large datasets is increasingly important for many applications. Unfortunately, -generally speaking, the more accurate the method, the greater its -computational cost; most nonparametric methods are $O(N^2)$ or +powerful nonparametric methods are commonly $O(N^2)$ or $O(N^3)$, where there are $O(N)$ test (or query) points and training (or reference) points. - -The authors of [[nips2000paper]] presented a new algorithmic strategy +The authors of \cite{nips2000paper} presented a new algorithmic strategy based on the simultaneous traversal of multiple space-partitioning trees, which applies to a number of machine learning methods. The set of problems to which this strategy is applicable was informally named -{\em generalized $N$-body problems}, in analogy to breakthrough -methods for computational physics problems [[Appel's algorithm, -Barnes-Hut, Fast Multipole Method]], all of which have the same -characteristic form. This algorithmic strategy generalizes other -successful algorithms for specific problems in addition to these -physics problems, such as computing the well-separated pair -decomposition in theoretical computer science [[wspd]], computing the -spatial join in databases [[spatial join]], and computing set -intersections [[baeza-yates]]. In each of these problems no superior -algorithmic strategy is known. +{\em generalized $N$-body problems}, in reference to breakthrough +methods for computational physics problems \cite{appel2,barnes_hut,grngard} +which can be seen as special cases. +%all of which have the same +%characteristic form. This algorithmic strategy generalizes other +%successful algorithms for specific problems in addition to these +%physics problems, such as computing the well-separated pair +%decomposition in theoretical computer science [[wspd]], computing the +%spatial join in databases [[spatial join]], and computing set +%intersections [[baeza-yates]]. In each of these problems no superior +%algorithmic strategy is known. % also a kind of dual-tree method in graphics for object collision % detection, but we need the reference - This multi-tree algorithmic strategy has been applied to a succession of well-known statistical learning methods, each representing certain unique challenges for the strategy, including all-$k$-nearest-neighbors (a generalization of $k$-nearest-neighbors) -[[nips2000paper]], kernel density estimation [[nips2000paper, -kde-siamdm, kde-aistats, kde-nips-dong, kde-uai-dong]], -$k$-nearest-neighbor classification [[ting-liu]], kernel discriminant +\cite{nips2000paper}, kernel density estimation \cite{nips2000paper, +kde-siamdm, kde-aistats, kde-nips-dong, kde-uai-dong}, +$k$-nearest-neighbor classification \cite{ting-liu}, kernel discriminant analysis (or nonparametric Bayes classification) [[nbc-phystat, -nbc-compstat]], and $n$-point correlation functions [[nips2000paper, -moore-npt, npt-2004]]. These algorithms have been demonstrated on -large scientific datasets, producing numerous high-profile results, -e.g.~[[science 2003, nature 2005]]. For each of these problems, no -overall faster algorithms are known. In addition, other authors have -directed the strategy toward other machine learning problems, -including dimensionality reduction methods [[hochreiter00beyond]], -nonparametric belief propagation [[alex ihler]], multiple tracking -[[jeremy kubica]], linear algebraic machine learning methods [[nando, -below]], particle filters [[klaas papers, below]]. Aside from speed, -a unique advantage of this algorithmic approach is the ability to -specify and achieve rigorous relative error tolerances when -approximation is necessary, contrary to virtually all other -approximate speedup approaches. +\cite{nbc-compstat}]], and $n$-point correlation functions [[\cite{nips2000paper, +moore-npt}, npt-2004]]. These algorithms have been demonstrated on +large scientific datasets, producing first-of-a-kind high-profile +results, e.g.~\cite{science03, nature05}. For each of these +problems, no overall faster algorithms are known. Other machine +learning problems which have been treated using this strategy include +dimensionality reduction methods \cite{hochreiter00beyond}, nonparametric +belief propagation \cite{alex-ihler}, multiple tracking \cite{kbc:stch}, linear algebraic machine learning methods \cite{freitas_fast}, +and particle filters \cite{klaas_fast, klaas-toward, klaas-fast}. +Many other current and future machine learning methods remain to be +treated in depth with this strategy, not to mention a large array of +related fundamental problems in computational geometry, physics, and +applied mathematics. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. -One paragraph more text more text more text more text mroe text more text. +This paper treats two basic questions: 1) What +is the scope of the problems to which this methodology can be applied? 2) +How can the successful ideas which have been developed over many problem +examples be expertly applied by new researchers wishing to +design fast algorithms for future machine learning methods? +To make the ideas concrete, we begin with examples of some problems +and corresponding algorithms resulting from our approach. We then +mathematically formalize the class of {\em generalized $N$-body +problems} (GNP's) for the first time. We also formalize the +algorithmic approach as a single generic algorithm, {\em the +generalized $N$-body algorithm}, which can be specialized to any problem in the +class, possibly automatically \cite{autobayes}. +We elucidate some of +the abstract mathematical properties of such problems that can be +exploited to achieve computational efficiency, through a number of +lemmas forming a calculus of transformations of GNP's. We then return +to our examples and demonstrate the derivation of each algorithm using +our calculus. As a more sophisticated final example, we apply the +theory to derive a fast algorithm for as recent a method as we could +find, the affinity propagation method of clustering \cite{affinity}, and +demonstrate its efficiency. + +%\footnote{Beyond facilitating application by +%humans to new problems, we strive for a level of precise specification +%which goes even farther, opening up the possibility of automatic +%derivation of algorithms for new problems [[autobayes]], as did the +%specification of the generic EM algorithm by [[dempster et al.]].} + +%Aside from speed, +%a unique advantage of this algorithmic approach is the ability to +%specify and achieve rigorous relative error tolerances when +%approximation is necessary, contrary to virtually all other +%approximate speedup approaches. -% %Nando de Freitas, Yang Wang, Maryam Mahdaviani and Dustin Lang. Fast %Krylov Methods for N-Body Learning . NIPS 2005. % @@ -235,6 +261,32 @@ One paragraph more text more text more text more text mroe text more text. % year = "2000", % url = "citeseer.ist.psu.edu/541475.html" } +%Some fundamental problems to which the multi-tree strategy is +%applicable are shown in Figure~\ref{tab:probs}, including some canonical +%problems in theoretical computer science [[preparata-shamos]]. These +%problems commonly occur in many guises in many fields, including many +%machine learning methods, both classical and modern, such as recent +%manifold learning [[lle, isomap]] methods. + +%In the next section, to make the ideas concrete, we present one of the +%more easily understood problems in our class and an efficient +%multi-tree method for it. We then define generalized $N$-body +%problems, and show that they have a decomposability property which +%ultimately admits a general recursive formulation, the generalized +%$N$-body algorithm. Making the generalized $N$-body algorithm +%efficient requires a problem-dependent pruning step. The following +%two sections discuss problem-dependent properties which admit +%efficient pruning steps in the cases of exact and approximate +%computation, respectively. We conclude with examples of designing +%fast algorithms for machine learning problems, starting with the +%problem of nonparametric density thresholding, a common folk method +%for anomaly detection. Finally, as a demonstration of the practical +%relevance of this methodology to state-of-the-art machine learning +%methods, we considered the most recent method we could find, affinity +%propagation [[affinity]], and designed and implemented in one day a +%fast algorithm for it allowing it to scale to large datasets despite +%its straightforward $O(N^2)$ runtime. + \section{Examples} % Begin figure @@ -682,8 +734,9 @@ This typically involves consideration of $X$ and $Y$ at the abstract level forme \begin{definition} For descriptive statistics $\sigma_x \colon 2^{\mathcal{X}} \to \mathcal{S}_x$ and $\sigma_y \colon 2^{\mathcal{Y}} \to - \mathcal{S}_y$, let {\bf summary results} $\GNP[\sigmahat] \colon - \mathcal{S}_y \times \mathcal{S}_x \to 2^\mathcal{A}$ be such that + \mathcal{S}_y$, let initial {\bf summary results} $\GNP[\sigmahat] + \colon \mathcal{S}_y \times \mathcal{S}_x \to 2^\mathcal{A}$ be such + that \[ \GNP[\sigmahat](\sigma_y(Y),\sigma_x(X)) \supseteq \{\GNP(Y',X') | X' \st \sigma(X') = \sigma(X), Y' \st \sigma(Y') = \sigma(Y)\}. \] @@ -695,46 +748,12 @@ Since the exact set may be costly or impossible to represent, a superset of such Note that functions $\sigma_x$, $\sigma_y$, and $\GNP[\sigmahat]$ are not unique for a given GNP; indeed, choosing the right statistics can significantly impact running time. A common statistic is the bounding radius, from which the summary set of all possible distances between points in $X$ and $Y$ may be computed as upper and lower bounds. %Summaries computed from these may be represented as upper and lower bounds on distances between points in $X$ and $Y$. -Summarization directly leads to a simple pruning technique. -% \subsection{Intrinsic Pruning} - -\begin{lemma}[Intrinsic Pruning] - We may prune when summary results form a singleton set. -\end{lemma} -\killspace -\begin{proof} - Setting $X' = X$ and $Y' = Y$ meets the requirements for inclusion in $\GNP[\sigmahat](\sigma_y(Y),\sigma_x(X))$. - Thus, summary results contain the exact result. - Thus, singleton $\GNP[\sigmahat](\sigma_y(Y),\sigma_x(X)) = \{\GNP(Y,X)\}$ and we are free to shortcut all further computation on $\GNP(Y,X)$ with this value. -\end{proof} -\killspace -\begin{example}[Pruning in the Two-point Correlation] - We may perform intrinsic pruning when considering regions shown to - be completely inside or outside radius $r$ of one another. -\end{example} -\killspace -\begin{proof} - Let $\sigma_x$ find bounding boxes for points in $\mathcal{X}$ and - let $\disthrectmax$ and $\disthrectmin$ find upper and lower bound distances between bounding boxes. - For $x \in X$ and $y \in Y$, bounds on $I(d(x,y) \leq r)$ are then $\left[ - I(\disthrectmax(\sigma_x(Y),\sigma_x(X)) \leq r), - I(\disthrectmin(\sigma_x(Y),\sigma_x(X)) \leq r) \right]$. - We may thus define - \[ - \GNP[\sigmahat](Y,X) = \left[ |Y| \cdot |X| \cdot I(\disthrectmax(\sigma_x(Y),\sigma_x(X)) \leq r), |Y| \cdot |X| \cdot I(\disthrectmin(\sigma_x(Y),\sigma_x(X)) \leq r) \right]. - \] - This set is $\{|Y| \cdot |X|\}$ when - $\disthrectmax(\sigma_x(Y),\sigma_x(X)) \leq r$ and $\{0\}$ when - $\disthrectmin(\sigma_x(Y),\sigma_x(X)) > r$. -\end{proof} -\maybekillspace -% \subsection{Iterative Refinement.} - -{\bf Iterative Refinement.} Further pruning is possible in some -problems by considering results gathered from other parts of -computation. This is assisted by the ability to compose summary -results. +It is often useful to gather summary results from the various components of computation, ultimately composing them into a summary for the entire computation. +% {\bf Iterative Refinement.} Further pruning is possible in some +% problems by considering results gathered from other parts of +% computation. This is assisted by the ability to compose summary +% results. \begin{definition} Let {\bf summary composition operators} $\otimeshat, \odothat \colon 2^{\mathcal{A}} \times 2^{\mathcal{A}} \to 2^{\mathcal{A}}$ be such @@ -780,6 +799,47 @@ Refinement need not be performed in any particular pattern. Depth-first is often a good choice due to its low overhead, though pruning in some problems strongly favors other expansion patterns. +% \subsection{Intrinsic Pruning} + +Summarization directly leads to a simple pruning technique. +\begin{lemma}[Intrinsic Pruning] + We may prune when summary results form a singleton set. +\end{lemma} +\killspace +\begin{proof} + Setting $X' = X$ and $Y' = Y$ meets the requirements for inclusion in $\GNP[\sigmahat](\sigma_y(Y),\sigma_x(X))$. + Thus, summary results contain the exact result. + Thus, singleton $\GNP[\sigmahat](\sigma_y(Y),\sigma_x(X)) = \{\GNP(Y,X)\}$ and we are free to shortcut all further computation on $\GNP(Y,X)$ with this value. +\end{proof} +\killspace +\begin{examplelemma}[Pruning in the Two-point Correlation]\label{lem:2pt} + We may perform intrinsic pruning when considering regions shown to + be completely inside or outside radius $r$ of one another. +\end{examplelemma} +\killspace +\begin{proof} + Let $\sigma_x$ find bounding boxes for points in $\mathcal{X}$ and + let $\disthrectmax$ and $\disthrectmin$ find upper and lower bound distances between bounding boxes. + For $x \in X$ and $y \in Y$, bounds on $I(d(x,y) \leq r)$ are then $\left[ + I(\disthrectmax(\sigma_x(Y),\sigma_x(X)) \leq r), + I(\disthrectmin(\sigma_x(Y),\sigma_x(X)) \leq r) \right]$. + We may thus define + \[ + \GNP[\sigmahat](Y,X) = \left[ |Y| \cdot |X| \cdot I(\disthrectmax(\sigma_x(Y),\sigma_x(X)) \leq r), |Y| \cdot |X| \cdot I(\disthrectmin(\sigma_x(Y),\sigma_x(X)) \leq r) \right]. + \] + This set is $\{|Y| \cdot |X|\}$ when + $\disthrectmax(\sigma_x(Y),\sigma_x(X)) \leq r$ and $\{0\}$ when + $\disthrectmin(\sigma_x(Y),\sigma_x(X)) > r$. +\end{proof} +\killspace +\begin{exampletheorem}[The Two-point Correlation Algorithm] + By Lemma~\ref{lem:2pt}, we may evaluate the two-point correlation + via recursive expansion with pruning as shown in the {\bf tpc} + function of Figure~\ref{fig:allnntpc}. +\end{exampletheorem} +\maybekillspace +% \subsection{Iterative Refinement.} + % \subsection{Extrinsic Pruning} Iterative refinement allows us to prune components when more precise @@ -805,10 +865,10 @@ knowledge of their results cannot affect the global result. $\GNP(Y,X)$ with any $b \in \GNP[\Sigma](Y,X)$. \end{proof} -\begin{example}[Pruning in All-nearest-neighbors] +\begin{examplelemma}[Pruning in All-nearest-neighbors]\label{lem:allnn} We may perform extrinsic pruning when references are shown to be farther away than queries' nearest-neighbors found so far. -\end{example} +\end{examplelemma} \killspace \begin{proof} Again, let $\sigma_x$ find the bounding radius for points in $\mathcal{X}$ and let $\disthrectmax$ and $\disthrectmin$ find upper and lower bound distances between bounding boxes. @@ -845,6 +905,12 @@ knowledge of their results cannot affect the global result. candidate nearest-neighbor distance found so far. \end{proof} \killspace +\begin{exampletheorem}[The All-nearest-neighbors Algorithm] + By Lemma~\ref{lem:allnn}, we may evaluate all-nearest-neighbors via + iterative refinement with pruning as shown in the {\bf all-nn} + function of Figure~\ref{fig:allnntpc}. +\end{exampletheorem} +\killspace % \subsection{Thresholded Pruning} % \killspace % Postprocessing function $h$ introduces a third form of pruning.