
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\documentstyle[12pt,twoside,alltt,psfig]{article}
\documentstyle[12pt,twoside]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecure
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{18}{November 16, 1994}{Ron Rivest}{Andrew Kirmse} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Outline}
\begin{itemize}
\item The VC-dimension of neural networks.
\item Asymptotic error rates of neural networks.
\end{itemize}

\section{The VC-dimension of a neural network}
If we could compute the VC-dimension of a neural network, then we could
determine an upper bound on the number of examples we would need to see before
the neural net is trained to classify according to a target concept.  Below we
will find the VC-dimension of a somewhat restricted set of neural nets.
Instead of using the PAC model of learning, we will derive error and
confidence bounds that depend on a single parameter $\epsilon$.  The results
in this section are from~\cite{BauHaus89}; the theorem numbers match those in
the paper.  Similar proofs appear in~\cite{text} and~\cite{AntBrig92}.

First we will consider neural networks that consist of an acyclic set of edges
and a set of {\em computation nodes}.  Each node computes some arbitrary
function from its real-valued inputs to $\{\pm 1\}$, but there is only one
output node for the entire network.  This may seem to be much more restrictive
than our earlier definition of neural nets, but the computation nodes in this
model are quite general, in that the nodes aren't limited to using the
weighted sum of their inputs.

\newcommand{\dm}[1]{\mbox{$\Delta_{#1}(m)$}}
\newcommand{\pr}{\prod_{i=1}^{N}}

Let $F_{i}$ be the class of functions computable at node $i$, and let $F$ be
the class of functions computed by the entire net as the $F_{i}$ vary.  Then
given a sample $S$ of size $m$, we define
\begin{eqnarray*}
\Delta_{F}(S)  & = & \mbox{the number of dichotomies induced by $F$ on
$S$.} \\
\dm{F}         & = & \max_{S,\ |S| = m} \Delta_{F}(S)
\end{eqnarray*}
($\Delta_{F}(S)$ is what we called $\Pi_{\cal C}(S)$ in our earlier discussion
of VC-dimension, and \dm{F} is what we called $\Pi_{\cal C}(m)$.)  If our
neural network has $N$ nodes, then we have the following first result:

\begin{theorem}
Define $d = \sum_{i=1}^{N}$ VC-dim($F_{i}$).  Then
\[\dm{F} \leq \pr \dm{F_{i}} \leq \left(\frac{Nem}{d}\right)^{d}\]
\end{theorem}
\begin{proof}
For a given sample $S$, the maximum number of labelings as the $F_{i}$ vary is
the number of ways of choosing the net's output based on its inputs.
Certainly we can't have more labelings than the number of ways of choosing
each node's outputs independently; we may have fewer if some choices induce
the same labelings.  Each node has \dm{F_{i}} ways of labeling $S$.  So the
whole network has at most \dm{F_{1}}\dm{F_{2}}\ldots ways of labeling $S$.
This gives the first inequality in the theorem.

To establish the second inequality, define $d_{i} =$ VC-dim$(F_{i})$.  Then $d
= \sum_{i=1}^{N} d_{i}$ by definition.  Now recall a result that we derived in
our first lecture on VC-dimension.  If $F$ has (finite) VC-dimension $k$, then
$\dm{F} \leq \left(\frac{em}{k}\right)^{k}$.  Applying this to each $F_{i}$
individually, we have that
\[\pr \dm{F_{i}} \leq \pr \left(\frac{em}{d_{i}}\right)^{d_{i}} \]
This product is maximized when all the $d_{i}$ are equal; {\em i.e.} when
$d_{i} = d/N$ for all $i$.  So
\begin{eqnarray*}
\pr \dm{F_{i}} & \leq & \pr \left(\frac{em}{d/N}\right)^{d/N} \\
               & =    & \left(\frac{em}{d/N}\right)^{d} \\
               & =    & \left(\frac{Nem}{d}\right)^{d}
\end{eqnarray*}

\end{proof}

For the remainder of this section, we will restrict ourselves to computation
nodes that compute linear threshold functions.  That is, a node outputs $+1$ if
the weighted sum of its inputs is greater than some threshold, and $-1$
otherwise.  The following corollary gives an upper bound on the VC-dimension
of such nets:

\setcounter{corollary}{2}
\begin{corollary}
Let $E$ be the number of edges in a neural net, and $W = E+N$ the total number
of weights (one per edge plus the thresholds).  If $N \geq 2$, then
\begin{itemize}
\item $\dm{F} \leq \left(\frac{Nem}{W}\right)^{W}$ for $m \geq W$.
\item VC-dim$(F) < 2W\log (eN)$
\end{itemize}
\end{corollary}
\begin{proof}
\begin{itemize}
\item Recall that the VC-dimension of the class of half-spaces in ${\cal
R}^{n}$ is $n+1$.  Since each node computes a linear threshold in
${\cal R}^{k}$, where $k$ is the number of inputs to the node, the
class of functions computed by a node has VC-dimension at most $k+1$.
Thus $d = \sum d_{i} =
\sum$ VC-dim$(F_{i}) \leq E+N = W$.  Applying Theorem~1 gives the inequality.
\item If we take $m \geq 2W\log (eN)$, then we can show that 
\[ \left(\frac{Nem}{W}\right)^{W} < 2^{m} \] 
as long as $N \geq 2$.  Using the first part of the corollary, this
shows that $\dm{F}~<~2^{m}$.  Thus no set of $m$ points is shattered,
so $F$ has VC-dimension less than $m$.
\end{itemize}
\end{proof}

Here are two other results from~\cite{BauHaus89}:

\begin{corollary}
If $m \geq \frac{32W}{\epsilon}\log\frac{32W}{\epsilon}$ and we can
find a choice of weights that correctly labels at least a fraction
$1-\epsilon/2$ of the $m$ training examples, then with probability at
least $1-8^{-1.5W}$, the net will correctly classify at least a
fraction $1-\epsilon$ of examples drawn from the same distribution as
the training examples.
\end{corollary}

\setcounter{theorem}{3}

\begin{theorem}
The class of one hidden layer linear threshold nets with k hidden
nodes and n inputs has VC-dimension at least
$2\lfloor\frac{k}{2}\rfloor n$.
\end{theorem}
Note that this bound is approximately equal to the total number of
weights if the net is highly connected ($n$ edges from the inputs to
each of the $k$ hidden nodes, plus one edge from each hidden node to
the output node, for a total of $k(n+1)$ in a fully-connected net).

We never mentioned back propagation in the discussion above.  The results hold
for an arbitrary training algorithm; in particular, back propagation performs
better in practice than these results would suggest.

\section{Asymptotic error rates of neural networks}
In this section we are interested in determining the error rate of a
neural network on a sample as the size of the sample and the number of
hidden nodes in the net increase.  We would like to compare our actual
(or {\em empirical}) error rate with the optimal error rate (also
known as the {\em Bayes risk}).  The results below are
from~\cite{FarLug93}.

In the discussion below we will consider neural nets with $k$ hidden
nodes in a single hidden layer and $d$ {\em fixed} inputs.  There is a
single output node which outputs either 0 or 1.  Each node computes a
step function $\sigma$ of the weighted sum of its inputs:
\[ \sigma(x) = \left\{ \begin{array}{ll}
			1   & \mbox{if $x \geq b_{i}$} \\
			-1  & \mbox{if $x < b_{i}$}
		       \end{array}
	       \right. \]
We label each edge from an input to a hidden node by $a_{i}$, and each
edge from a hidden node to the output node by $c_{i}$.  Considering
$a$, $b$, and $c$ as column vectors, we can write the output of the net as a
function $f$ of its inputs and weights:
\[ f(x, a, b, c) = \sum_{i=1}^{k} c_{i}\sigma(a_{i}^{T}x + b_{i}) +
c_{0} \] 
Here $x$ is a vector of the inputs,  and $c_{0}$ is a constant input to
the output node.

We can use these neural nets in situations where the inputs don't
completely determine a classification.  For example, suppose that the
inputs are characteristics of a lung X-ray, and the output should
predict whether or not a tumor is present.  Under such circumstances,
there is only a probability (not a certainty) of an output being
correct given a set of inputs.  It's easy to see that the best
possible classification rule is the one that agrees with the true
outcome most of the time.  Let $(X, Y)$ be an input/output pair.
Then we can write the optimal classification rule as
\[ g^{*}(x) =  \left\{ \begin{array}{ll}
			0 & \mbox{if Pr$\{Y = 0 | X = x\} \geq 1/2$} \\
			1 & \mbox{if Pr$\{Y = 1 | X = x\} > 1/2$}
		       \end{array}
	       \right. \]

Any given neural net corresponds to some classification rule, given by the
mapping of its inputs to its outputs.  Let $g_{n}$ be the classification rule
of a neural net after it has seen $n$ examples.  This rule will have a certain
error rate on the set of training examples, denoted by $L(g_{n})$, and another
(possibly different) error rate on the set of all examples in the sample
space, written $r(g_{n})$.  These can be written formally as
%\begin{eqnarray*}
\[ L(g_{n}) = \mbox{Pr$\{g_{n}(X) \neq Y | $ some
particular training sequence$\}$} \]
\[ r(g_{n}) = E(L(g_{n})) = \mbox{Pr$\{g_{n}(X) \neq Y\}$} \]
%r(g_{n}) & E(L(g_{n})) & = & \mbox{Pr$\{g_{n}(X) \neq Y\}$}
%\end{eqnarray*}
The optimal classification rule $g^{*}$ has the minimum possible error
rate on the whole sample space.  We denote this rate by $L^{*}$, which
is also known as the {\em Bayes risk} of the sample space.
\newcommand{\limn}{\lim_{n \rightarrow \infty}}
\begin{definition}
A sequence $g_{n}$ of classification rules is universally consistent
if
\[ \limn r(g_{n}) = L^{*} \]
regardless of the distribution on the sample space $X$.

A sequence $g_{n}$ of classification rules is strongly universally
consistent if
\[ \limn L(g_{n}) = L^{*} \]
with probability one, regardless of the distribution on the sample
space $X$.
\end{definition}
Strong universal consistency does indeed imply universal consistency:
if a sequence of values chosen from some distribution always converges
to a certain value, then the sequence of expected values chosen from
the same distribution converges to the same value.

Let $g_{k, n}^{*}$ denote the optimal neural net with k hidden nodes
after the first n examples.
\setcounter{theorem}{0}
\begin{theorem}
If the number of hidden nodes $k$ satisfies $k \rightarrow \infty$ and
$\frac{k\log n}{n} \rightarrow 0$ as $n \rightarrow \infty$, then with
probability one,
\[ \limn L(g^{*}_{k, n}) = L^{*} \]
\end{theorem}
This theorem is somewhat useful in that it provides a guarantee of strong
universal convergence, even when no neural net is a perfect classifier.
However, Theorem~1 says nothing about the rate of convergence; the following
theorem is more useful in practice:
\begin{theorem}
If $k = \sqrt{\frac{n}{d\log n}}$, then 
\[ r(g^{*}_{k, n}) - L^{*} = O\left(\left(\frac{d\log
n}{n}\right)^{\frac{1}{4}}\right)\]
\end{theorem}
This theorem tells you roughly how fast the optimal neural net with $k$ hidden
nodes after $n$ examples converges to the optimal error rate.  Notice that the
rate of convergence has a constant exponent; in particular, it doesn't depend
on the number of inputs $d$.

\begin{thebibliography}{99}

\bibitem{AntBrig92} Anthony and Briggs. {\em Computational Learning
Theory}.  Cambridge University Press, 1992.

\bibitem{BauHaus89} Eric B. Baum and David Haussler.  What size net gives
valid generalization?  In {\em Advances in Neural Information Processing
Systems I}, pages 81-90.  Morgan Kaufmann, 1989.

\bibitem{FarLug93} Andr\'{a}s Farag\'{o} and G\'{a}bor Lugosi.  Strong
universal consistency of neural network classifiers.  {\em IEEE Transactions
on Information Theory}, IT-39(4):1146-1151, July 1993.

\bibitem{text} Michael J. Kearns and Umesh V. Vazirani. {\em An
Introduction to Computational Learning Theory}, pages 64-67.  MIT
Press, 1992.

\end{thebibliography}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes end here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}














