
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[12pt,twoside,alltt,psfig]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecure
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{13}{October 26, 1994}{Ron Rivest}{Rob Miller} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\section*{Outline}

\begin{itemize}
\item Finish the proof that weak learnability implies strong 
learnability.
\end{itemize}

\section{Weak Learnability Implies Strong Learnability}

Today we finish the proof that weak learnability implies strong
learnability, a theorem due originally to Shapire~\cite{Shap90}.  This
is a rather surprising result; essentially it says that if we can find
an efficient learning algorithm that guarantees only a fixed accuracy
$\epsilon_0$ ($\epsilon_0 \leq \frac{1}{2} -
\frac{1}{p(n)}$) with only a fixed confidence $1-\delta_0$, then we can
turn it into a general PAC algorithm for arbitrary $\epsilon$ and
$\delta$.

Last time, we saw a procedure for boosting the confidence of an
algorithm, from a fixed $\delta_0$ to arbitrary $\delta$.  We also saw
how an algorithm's accuracy can be boosted by running the algorithm
three times, on three different distributions: the original
distribution ${\cal D}_1$; a distribution ${\cal D}_2$ which is drawn
half from examples where $h_1$ is correct, and half from where $h_1$
is incorrect; and a distribution ${\cal D}_3$, which consists only of
examples where $h_1$ and $h_2$ disagree.  Taking the majority of
$h_1$, $h_2$, and $h_3$ yields a hypothesis with error rate less than
$g(\epsilon)=3\epsilon^2 - 2\epsilon^3$.  This one-shot
accuracy-boosting procedure has been applied at Bell Labs in teaching
a neural net to recognize characters, with some success in reducing
its error rate.  So hypothesis boosting is not only interesting in
theory, but useful in practice.~\cite{DSS92}

Today we will show how to recursively apply the accuracy-boosting
procedure to achieve arbitrary accuracy, in time polynomial in
$\frac{1}{\epsilon}$ and $n$.


\section{Recursive Hypothesis-Boosting Algorithm}

Suppose $L$ is a learning algorithm which produces hypotheses with
guaranteed error rate less than $\frac{1}{2} - \frac{1}{p(n)}$ (for
some polynomial $p(n)$) with probability at least $1-\delta$.  We will
define a recursive algorithm $SL(\alpha, EX(c, {\cal D}'))$, which
uses the example oracle $EX(c, {\cal D}')$ to find a hypothesis with
error rate less than $\alpha$, for any $\alpha > 0$.

\subsection*{Algorithm $SL(\alpha, EX(c, {\cal D}'))$}

\begin{enumerate}
\item If $\alpha > \frac{1}{2} - \frac{1}{p(n)}$, then return 
$L(EX(c,{\cal D}'))$.
\item $\beta \leftarrow g^{-1}(\alpha)$.
\item $h_1 \leftarrow SL(\beta, EX(c, {\cal D}'))$.
\item Create $EX(c,{\cal D}'_2)$ by filtering $EX(c,{\cal D}')$ so that half
of its examples are drawn subject to the constraint $h_1 = c$, and half are 
subject to $h_1 \neq c$.
\item $h_2 \leftarrow SL(\beta, EX(c, {\cal D}'_2))$.
\item Create $EX(c,{\cal D}'_3)$ by filtering $EX(c,{\cal D}')$ subject to
$h_1 \neq h_2$.
\item $h_3 \leftarrow SL(\beta, EX(c, {\cal D}'_3))$.
\item Return $h = majority(h_1, h_2, h_3)$.
\end{enumerate}


To achieve an arbitrary input accuracy of $\epsilon$ on the original
distribution ${\cal D}$, we run $SL(\epsilon, EX(c, {\cal D}))$.  The
execution of $SL$ performs a recursive computation like that shown in
Figure~\ref{fig:tree}.  The error rate required of the root node is
$\epsilon$, but thanks to the hypothesis-boosting at each node, its
children have a weaker error-rate requirement, $g^{-1}(\epsilon) >
\epsilon$ (see Figure~\ref{fig:g}).  Intuitively, $SL$ finds a
sufficiently accurate hypothesis for the root by searching depth-first
down the tree until the required error rate is greater than
$\frac{1}{2} - \frac{1}{p(n)}$, at which point $L$ itself can produce
acceptable hypotheses.  The hypotheses from the leaves are then
combined by majority vote at each node to produce the final
hypothesis.

\begin{figure}[hbtp]
\begin{center}
\mbox{\psfig{figure=lecture13-fig-2.ps}}
\caption{The tree of recursive calls made by $SL$.  The recursion bottoms out 
when the required error rate $\alpha$ is greater than $\frac{1}{2} -
\frac{1}{p(n)}$, for which the weak learning algorithm $L$ is able
to generate a satisfactory hypothesis.}
\label{fig:tree}
\end{center}
\end{figure}

\begin{figure}[hbtp]
\begin{center}
\mbox{\psfig{figure=lecture13-fig-1.ps}}
\caption{The error rate requirement becomes less stringent for nodes lower
in the tree.  Although the root is required to output a hypothesis
with error rate less than $\epsilon$, its children need to achieve an
error rate of $g^{-1}(\epsilon)$, which the graph above shows is greater than
$\epsilon$.}
\label{fig:g}
\end{center}
\end{figure}


We care about the depth $B$ of this tree, since it will play a major
role in the bounds of the running time and sample size of $SL$.

\nopagebreak

{\lemma{$B = O(\log p(n) + \log \log \frac{1}{\epsilon})$.}}

\begin{proof}
Let $\beta_i$ be the required error rate at depth $i$ in the tree,
where the root has depth 0.  So $\beta_0 = \epsilon$, and $\beta$ of a
node is related to the $\beta$ of its parent by $\beta_{i+1} =
g^{-1}(\beta_i)$.

We will break the analysis into two parts, each giving one term of the
result $O(\log p(n) + \log \log \frac{1}{\epsilon})$.  First, consider
how many levels of the tree are sufficient to get $\beta$ up from
$\epsilon$ to $\frac{1}{4}$.  We will do the analysis in the reverse
direction, considering how $\beta$ decreases as we go up the tree from
$\frac{1}{4}$.  In one step up the tree, $\beta$ decreases to
$g(\beta) = 3\beta^2 - 2\beta^3 \leq 3\beta^2$.  In $k$ steps, it is
less than $\frac{1}{3}(3\beta)^{2^k}$.  If initially $\beta \leq
\frac{1}{4}$, then after $k$ steps it falls below
$\frac{1}{3}(\frac{3}{4})^{2^k}$.  Setting this less than $\epsilon$
and solving for $k$ gives $k \geq c_0 \log \log \frac{1}{\epsilon}$,
for some constant $c_0 > 0$.  So $O(\log \log \frac{1}{\epsilon})$
levels are sufficient to get $\beta$ up from $\epsilon$ to
$\frac{1}{4}$.

For the second part, we consider how quickly $\beta$ approaches
$\frac{1}{2}$, as we go down the tree from $\beta \geq
\frac{1}{4}$ to $\beta \geq \frac{1}{2} - \frac{1}{p(n)}$.  We
 want to compare the deviation $\frac{1}{2} - \beta$ with the
deviation for its parent, $\frac{1}{2} - g(\beta)$.  \[\frac{1}{2} -
g(\beta) = \frac{1}{2} - \left(3\beta^2 - 2\beta^3\right) = \left(\frac{1}{2} -
\beta\right)\left(1 + 2\beta - 2\beta^2\right)\]

When $\frac{1}{4} \leq \beta \leq \frac{1}{2}$, then $(1 + 2\beta -
2\beta^2) \geq \frac{11}{8}$.  (This is just its minimum on the
specified interval, found by basic calculus.)  So the distance from
$\beta$ to $\frac{1}{2}$ decreases by a factor of at least
$\frac{11}{8}$ for every level down the tree, so \[k \geq
\log_{\frac{11}{8}}\left(\frac{1/4}{1/p(n)}\right) = O(\log p(n))\]

\end{proof}

The size of the tree is $3^B = O(p(n) \log \frac{1}{\epsilon})$.


\section{Filtering Efficiency}

An issue that has been glossed over in the discussion to this point is
the efficiency of the filtering process that creates ${\cal D}'_2$ and
${\cal D}'_3$.  After all, the running time of $SL$ is proportional
not only to the size of the computation tree, but also to the number
of examples that must be drawn from the original distribution ${\cal
D}$ in order to satisfy draws on the filtered distributions.  In fact,
in the worst case, the algorithm $SL$ as given will fail to terminate!
Suppose $h_1$ at some node is a perfect hypothesis, with zero error
rate.  The filter for ${\cal D}'_2$ will run forever, looking for
examples where $h_1 \neq c$.  A similar problem occurs if $h_1 = h_2$,
since the filter for ${\cal D}'_3$ will never find an example where
the two hypotheses disagree.  We need to patch the $SL$ algorithm so
that it avoids these pitfalls.

The basic idea is to test $h_1$ and $h_2$ before using them to filter
examples.  If either of the hypotheses has a small estimated error
rate, we return it immediately; otherwise we can show that the
filtering process will terminate in polynomial time with high
probability.  We will consider $h_1$ and $h_2$ in turn.

For $h_1$, the dangerous situation occurs when $error_{{\cal
D}'}(h_1)$ is close to zero (where $error_{{\cal D}'}$ is measured
with respect to ${\cal D}'$). So we calculate an estimate $\hat{e}_1$
of $error_{{\cal D}'}(h_1)$ by drawing enough examples from
$EX(c,{\cal D})$ to make the estimate accurate within $\pm
\frac{\alpha}{3}$ with high probability.\footnote{By the additive
Chernoff bound, the number of examples required will be polynomial in
$\frac{3}{\alpha}$ and $log
\frac{1}{\delta'}$, where $1-\delta'$ is the desired confidence for
the estimate.  A sufficient value for $\delta'$ will be derived in the
next section.} If $\hat{e}_1
\leq \frac{2\alpha}{3}$, then with high probability, the true error rate 
of $h_1$ is less than $\alpha$, so we can just return it.  Otherwise, we 
can be confident that the true error rate $error_{{\cal D}'}(h_1) \geq 
\frac{\alpha}{3}$, so generating an example for ${\cal D}'_2$ will 
take at most $\frac{3}{\alpha}$ examples from ${\cal D}'$ (on average).

To test $h_2$, calculate an estimate $\hat{e}_2$ of $error_{{\cal
D}'}(h_2)$ to within $\pm \tau$, where $\tau = \frac{\alpha}{8} (1 -
2g^{-1}(\alpha))$.  If $\hat{e}_2 \leq \alpha - \tau$, then $h_2$ is a
sufficiently good hypothesis, so we can return it.  Otherwise
$error_{{\cal D}'}(h_2) \geq \alpha - 2\tau$, so we can apply the
following lemma.

{\lemma{If $error_{{\cal D}'}(h_1) \geq \frac{\alpha}{3}$ and $error_{{\cal D}'}(h_2) \geq 
\alpha - 2\tau$, then $Pr_{x \in {\cal D}'}(h_1(x) \neq h_2(x)) \geq 
\frac{\alpha}{24}$. (For a proof of this result, see Lemma 4.3 in 
\cite{KeaVaz94}.)}
\label{lem:alpha24}}

Lemma~\ref{lem:alpha24} implies that generating a ${\cal D}'_3$
example requires at most $\frac{24}{\alpha}$ draws from ${\cal D}'$ on
the average.  We are interested in finding an upper bound on the
filtering efficiency, which is the expected number of draws on the
root oracle ${\cal D}$ for each draw by a node lower in the tree.  The
filtering overhead introduced by a single node is at most
$\frac{24}{\alpha}$, so the filtering efficiency at level $i$ is at
most 
\[\frac{24}{\epsilon}
\frac{24}{g^{-1}(\epsilon)} 
\frac{24}{g^{-2}(\epsilon)} \cdots
\frac{24}{g^{-i}(\epsilon)}\].

{\lemma{$g^{-i}(\epsilon) \geq \frac{1}{3}\epsilon^{2^{-i}}$.}
\label{lem:g-i}}

\begin{proof}
Proof by induction.  For the base case $i=1$, we have $g(x) \leq 3x^2$, 
so $x \geq \sqrt{\frac{g(x)}{3}}$.  Substituting $x=g^{-1}(\epsilon)$ gives 
$g^{-1}(\epsilon) \geq \sqrt{\frac{\epsilon}{3}} > 
\frac{1}{3}\sqrt{\epsilon}$ ($\ast \ast$).

For the induction step, suppose the claim 
is true for $i-1$.  Then
\begin{eqnarray*}
g^{-i}(\epsilon) & = & g^{-1}\left(g^{-(i-1)}(\epsilon)\right) \\ 
& \geq & g^{-1}\left(\frac{1}{3}\epsilon^{2^{-(i-1)}}\right) \\ 
& \geq & \frac{1}{\sqrt{3}}\left(\frac{1}{3}\epsilon^{2^{-(i-1)}}\right)^\frac{1}{2} \mbox{\hspace{.3 in}   (using $\ast \ast$)}\\ 
& \geq & \frac{1}{3}\epsilon^{2^{-i}}
\end{eqnarray*}

\end{proof}


Applying Lemma~\ref{lem:g-i} to the upper bound on filtering efficiency gives

\begin{eqnarray*}
\frac{24}{\epsilon} \frac{24}{g^{-1}(\epsilon)} \cdots
\frac{24}{g^{-i}(\epsilon)}
& \leq & \frac{72}{\epsilon} \frac{72}{\epsilon^{1/2}}
\frac{72}{\epsilon^{1/4}} \cdots \frac{72}{\epsilon^{2^{-(i-1)}}} \\
& = & \frac{72^i}{\epsilon^{1+2^{-1}+2^{-2}+\cdots+2^{-(i-1)}}} \\
& = & \frac{72^i \epsilon^{2^{-(i-1)}}}{\epsilon^2} \\
& \leq & \frac{72^i(3g^{-(i-1)}(\epsilon))}{\epsilon^2} \\
& \leq & \frac{72^i \cdot 3g\left(g^{-i}(\epsilon)\right)}{\epsilon^2} \\
& \leq & \frac{72^i \cdot 3\left(3\left(g^{-i}\left(\epsilon\right)\right)^2\right)}{\epsilon^2} \\
& \leq & \frac{9 \cdot 72^i\left(g^{-i}(\epsilon)\right)^2}{\epsilon^2}
\end{eqnarray*}


Since $i$ is bounded by $B = O(\log p(n) + \log \log
\frac{1}{\epsilon})$, and $g^{-i}(\epsilon) < 1$, the filtering 
efficiency is polynomial in $\frac{1}{\epsilon}$ and $p(n)$.

The revised algorithm is shown below.

\subsection*{Revised Algorithm $SL(\alpha, EX(c, {\cal D}'))$}

\begin{enumerate}
\item If $\alpha > \frac{1}{2} - \frac{1}{p(n)}$, then return 
$L(EX(c,{\cal D}'))$.
\item $\beta \leftarrow g^{-1}(\alpha)$.
\item $h_1 \leftarrow SL(\beta, EX(c, {\cal D}'))$.
\item $\hat{e}_1 \leftarrow$ an estimation of $error_{{\cal D}'}(h_1)$ 
to within $\pm \frac{\alpha}{3}$.  \\
If $\hat{e}_1 \leq \frac{2\alpha}{3}$, then return $h_1$.
\item Create $EX(c,{\cal D}'_2)$ by filtering $EX(c,{\cal D}')$ so that half
of its examples are drawn subject to the constraint $h_1 = c$, and half are 
subject to $h_1 \neq c$.
\item $h_2 \leftarrow SL(\beta, EX(c, {\cal D}'_2))$.
\item $\hat{e}_2 \leftarrow$ an estimation of $error_{{\cal D}'}(h_2)$ 
to within $\pm \tau$.  \\
If $\hat{e}_2 \leq \alpha - \tau$, then return $h_2$.
\item Create $EX(c,{\cal D}'_3)$ by filtering $EX(c,{\cal D}')$ subject to
$h_1 \neq h_2$.
\item $h_3 \leftarrow SL(\beta, EX(c, {\cal D}'_3))$.
\item Return $h = majority(h_1, h_2, h_3)$.
\end{enumerate}

\section{Confidence Analysis}

Repeatedly throughout the construction of $SL$ we have said ``with
high probability,'' without specifying the actual probability
required.  We turn now to this question.  In the end, $SL$ must output
an accurate hypothesis with a chance of failure no greater than
$\delta$.  Each node in the recursive computation of $SL$ can make an
mistake in any of five ways: in estimating the hypotheses $h_1$,
$h_2$, or $h_3$, or in estimating the error rates $\hat{e}_1$ and
$\hat{e}_2$.  The simplest solution divides $\delta$ among all of the
possible points of failure, requiring each to have chance of failure
less than $\delta' = \frac{\delta}{5 \cdot 3^B}$.  Then applying the
union bound will limit the probability that the entire algorithm fails
to at most $\delta$.

Note that we require $L$ to accept an arbitrary confidence parameter
$\delta$, so if we start with a weak learning algorithm $L$ with fixed
accuracy $\epsilon_0 < \frac{1}{2}$ and fixed confidence $\delta_0$,
we need to boost the confidence of its hypotheses.  Unfortunately, the
confidence-boosting procedure we saw last lecture delivers
less-accurate hypotheses $L$ (error rate at most $2\epsilon_0$), which
works only if $\epsilon_0 < \frac{1}{4}$.  It is possible to tweak the
confidence-boosting procedure so that it outputs hypotheses with error
rates at most $\epsilon_0 + \gamma$, where $\gamma$ can be arbitrary
small.  (For a proof, see section 4.2 in~\cite{KeaVaz94}.)  This makes
the confidence-boosted $L$ a candidate for the accuracy-boosting
procedure described here.


\section{Overall Performance of $SL$}

To finish up, we present some results on the running time, sample
size, and hypothesis-evaluation time of $SL$, as functions of the
input parameters and the corresponding quantities for $L$.
Let $t(\delta, n)$ be the running time of $L$, $m(\delta, n)$ be the 
sample size required by $L$, and $u(\delta, n)$ be the time to evaluate 
a hypothesis returned by $L$.

{\lemma{The time to evaluate a hypothesis returned by $SL$ is 
\[U = O\left(3^B u(\delta',n)\right)\]}
\label{claim:U}}

\begin{proof}
$U$ is comprised of the time to evaluate the hypotheses at each of the
$O(3^B)$ leaves of the tree, plus a majority operation at each of the
$O(3^B)$ internal nodes, so $U = O(3^B u(\delta',n) + 3^B) = O(3^B
u(\delta',n))$.
\end{proof}

{\lemma{The expected sample size drawn by $SL$ from its root oracle
$EX(c,{\cal D})$ is \[M = O\left(\frac{216^B}{\epsilon^2}
\left(m(\delta',n) + p^2(n) \log \frac{1}{\delta'}\right)\right)\]}
\label{claim:M}}

\begin{proof}
Each leaf of the tree runs $L$, which draws $m(\delta',n)$ examples.
Multiplying by the filtering ratio $\frac{9 \cdot 72^B
(g^{-i}(\epsilon))^2}{\epsilon^2} \leq \frac{9 \cdot 72^B}{\epsilon^2}$
and the number of leaves $3^B$ gives the first term of the upper
bound on $M$.

Each internal node must estimate the error rates of $h_1$ and $h_2$,
with a confidence of $1-\delta$.  Actually, the estimation of
$\hat{e}_2$ must be more accurate than $\hat{e}_1$ (within $\pm \tau =
\frac{\alpha}{8} (1 - 2g^{-1}(\alpha))$, which is tighter than the
$\pm \frac{\alpha}{3}$ allowed for $\hat{e}_2$), so the sample size
required by $\hat{e}_2$ dominates.  Using the additive Chernoff bound,
this sample size is $O(\frac{1}{\tau^2} \log
\frac{1}{\delta'})$.  Since \[1-2\beta \geq \left(1-2\left(\frac{1}{2} -
\frac{1}{p(n)}\right)\right) = \frac{2}{p(n)}\] 
we have $\tau \geq \frac{\alpha}{4p(n)}$.  For a node at level $i$,
$\alpha = g^{-i}(\epsilon)$, so the sample size drawn by the node is
$O(\frac{p^2(n)}{g^{-i}} \log \frac{1}{\delta'})$.  Multiplying by the
filtering ratio and the number of internal nodes gives the second
term of $M$.
\end{proof}

{\lemma{The running time of $SL$ is
\[T = O(B  U  M +  3^B t(\delta',n))\]}
\label{claim:T}}

\begin{proof}
The second term in $T$ is simply the running time of $L$ in each leaf,
multiplied by the number of leaves.  The first term is an upper bound
on the cost of filtering examples.  As each of the $M$ examples drawn
from the root distribution travels down the tree, it must be tested by
$h_1$ and/or $h_2$ at each node on the path to determine if it should
be passed or rejected.  Since the path has length at most $B$, and
each hypothesis takes at most $U$ time to evaluate, the resulting cost
is $O(B  U)$ per example, for a total cost of $B U M$.
\end{proof}

It is easy to see that if $t$, $u$, and $m$ are polynomial in $n$,
$\frac{1}{\epsilon}$, and $\log \frac{1}{\delta}$, then so are $T$,
$U$, and $M$.  So we have found an efficient strong PAC algorithm for
${\cal C}$.


\begin{thebibliography}{99}

\bibitem{DSS92} H. Drucker, R. Shapire, and P. Simard.  Improving performance
in neural networks using a boosting algorithm.  In S.J. Hanson, J.D. Cowan, 
and C.L. Giles, editors, {\em Advances in Neural Information Processing 
Systems}, pages 42-49.  Morgan Kaufmann, San Mateo, CA, 1992.

\bibitem{KeaVaz94} M. Kearns and U. Vazirani.  {\em An Introduction to 
Computational Learning Theory.} MIT Press, 1994.

\bibitem{Shap90} R.E. Shapire.  The strength of weak learnability.  
{\em Machine Learning}, 5(2):197-227, 1990.

\end{thebibliography}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes end here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}
