
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[12pt,twoside,alltt,psfig]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecure
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{8 }{October 5, 1994}{Ron Rivest}{Lisa Zhang} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

In today's lecture we show the PAC learnability of infinite concept classes 
of finite VC-dimension.

\section {Background and Definitions}
Last time we introduced notations $\Pi_C(S)$ and $\Pi_C(m)$ defined by
\begin{eqnarray*}
\Pi_C(S) & = & \{c\cap  S: c\in C\},\\
\Pi_C(m) & = & \max \{ |\Pi_C (S)| : |S|=m\}
\end{eqnarray*}
for any concept class $C$ over instance class $X$ and any $S\subseteq X$.
The value of $\Pi_C(m)$ is upper bounded by function $\Phi$, that is 
$\Pi_C(m) \le \Phi_d(m) = \sum_{i=0}^{d} {m\choose i} \le 
\left({em \over d}\right) ^d$.

To show an infinite concept class $C$ of finite VC-dimension $d$ is
PAC learnable, the learning algorithm $L$
\begin {enumerate}
\item
draws a large sample $S$ of size $|S| =m$, where $m$ is a function of $d, 
\varepsilon$ and $\delta$.
\item 
returns any concept $h\in C$ that is consistent with the sample.
\end{enumerate}
\noindent
We would like a similar Occam algorithm for infinite concept classes.

Hypothesis $h$ is {\em bad} if $er(h)\ge \varepsilon$; otherwise $h$ is
$\em good$.
Let $c\Delta h = \{ x: c(x)\neq h(x)\}$ be the region where concept $c$
and hypothesis $h$ differ.  Notice that $Pr_{x\in D}[x\in c\Delta h] = er(h)$.
For a fixed target concept $c\in C$, let $\Delta(c)=\{ h\Delta c: h \in C\}$
be the set of {\em error regions} with respect to $c$ and $C$.  
Furthermore, let $\Delta_\varepsilon(c)=\{h\Delta c: h \in C 
\wedge Pr_{x\in D}[x \in h\Delta c] \ge \varepsilon\}$ be the error regions 
with weight at least $\varepsilon$ under the fixed target distribution $D$.
We can now make the following definition:

\begin{definition}
For any $\varepsilon >0$ a set $S$ is an $\varepsilon$-{\em net} for $\Delta
(c)$ if every region in $\Delta_{\varepsilon}(c)$ is hit by 
a point in $S$, that is $\forall r\in \Delta_{\varepsilon}(c)$, 
$S\cap r \neq \emptyset$.
\end{definition}

As an example, let us look at the concept class $C$ consisting of all closed 
intervals on
$[0,1]$ under the uniform distribution.  For any hypothesis $h$,  $h\Delta 
c = I_1 \cup I_2$, where $I_1$ and $I_2$ are two closed intervals and
$I_1=\{x : x\in h \wedge x\notin c\}$, $I_2=\{ x: x\notin h \wedge x\in c\}$.  
Under the uniform density $Pr_{x\in D}[x \in I]$ for any interval $I$ is
just the length of $I$.  So if $h$ is bad,  
either $|I_1|$ or $|I_2|$ is at least $\varepsilon/2$.  
Hence the set $S=\{x=k\varepsilon /2: k=0,1,\dots, \lceil 2/\varepsilon \rceil\}$
forms an $\varepsilon$-net.

\section{ The Double Sampling Method}
Notice that any hypothesis $h$ consistent with an $\varepsilon$-net is good.
Hence if we can upper bound the probability that a random sample $S$ fails to form
an $\varepsilon$-net, then we have upper bounded the probability
of $er(h)\ge \varepsilon$.  In order for $Pr[S$ to be an 
$\varepsilon$-net$]\ge 1-\delta$ we adopt the method of double sampling.

Let $S_1$ be a random sample of size $m$, and let $A$ be the event that 
$S_1$ fails to be an $\varepsilon$-net.  Clearly, we want $Pr[A]\le\delta$.
Let $S_2$ be a second random sample of size $m$.  
If event $A$ happens, then there exists region $r\in \Delta_\varepsilon(c)$ 
such that $S_1\cap r = \emptyset$ by the definition of the $\varepsilon$-net.  
For a fixed region $r$ missed by $S_1$,
each element in $S_2$ has probability $\varepsilon' \ge \varepsilon$ to hit $r$. 
By the multiplicative form of the Chernoff bound (See Appendix) 
\begin{eqnarray}
& & Pr[|S_2\cap r| > {\varepsilon m \over 2}] \nonumber \\
& \ge & Pr[|S_2\cap r| > {\varepsilon' m \over 2}] \nonumber \\
& = & 1-Pr[|S_2\cap r| \le {\varepsilon' m \over 2}] \nonumber \\
& \ge & 1- e^{-{\varepsilon' m\over8}} \label{eq:b1}\\
& \ge & {1\over 2} \nonumber
\end{eqnarray}
for $m\ge {8\over\varepsilon}\ln{2}\ge {8\over\varepsilon'}\ln{2}$.
Now let $B$ be the event that $A$ happens and $S_2$ has at least 
$\varepsilon m \over 2$ hits in some region $r\in \Delta_\varepsilon
(c)$ that is missed by $S_1$.  We have just proved that $Pr[B|A] \ge 
{1\over2}$. Since $Pr[B]=Pr[A\wedge B]=Pr[B|A]Pr[A]$, $Pr[A]\le
2Pr[B]$.  Therefore, we want $Pr[B]\le {\delta\over 2}$.

Equivalently, $B$ is the event that there is some $r\in\Pi_{\Delta_
\varepsilon(C)}(S_1\cup S_2)$ such that $|r|\ge \varepsilon m /2$ and
$r\cap S_1=\emptyset$.  Thus, instead of directly analyzing the
probability of event $A$ by considering
all regions of the infinite class $\Delta_\varepsilon(c)$ that $S_1$ 
might miss, we can now analyze the probability of event $B$ by only 
considering the regions of $\Pi_{\Delta_\varepsilon(C)}(S_1\cup S_2)$.

To bound $Pr[r\in\Pi_{\Delta_\varepsilon(C)}(S_1\cup S_2):
|r|\ge \varepsilon m /2  \wedge r\cap S_1=\emptyset]$,  we draw a random
sample of size $2m$ and randomly divide it into $S_1$ and $S_2$ of equal
size.  The resulting distribution of $S_1$ and $S_2$ is the same
as drawing 2 samples of size $m$ randomly and independently.  The
probability for a fixed region $r\in S_1\cup S_2$ of size 
$l=|r|\ge \varepsilon m/2$ to be entirely in $S_2$ is
${{m \choose l} \over {2m \choose l}}\le 2^{-l} \le 2^{-\varepsilon m/2}$.
Therefore, 
\begin{eqnarray*}
Pr[B] & = & Pr[r\in\Pi_{\Delta_\varepsilon(C)}(S_1\cup S_2):
|r|\ge \varepsilon m /2  \wedge r\cap S_1=\emptyset]\\
& \le & |\Pi_{\Delta_\varepsilon(c)}(S_1\cup S_2)|2^{-\varepsilon m/2} \\
& \le & |\Pi_{\Delta(c)}(S_1\cup S_2)|2^{-\varepsilon m/2}.
\end{eqnarray*}

We need the following lemma to finish the analysis:
\begin{lemma}\label{le:vcd}
VC-dimension$(\Delta(c))$=VC-dimension$(C)$.
\end{lemma}
To see the correctness of Lemma \ref{le:vcd}, for any set $S$ we can 
map each element
$c'\in \Pi_C(S)$ to $c'\Delta (c\cup S) \in \Pi_{\Delta(c)}(S)$.  Since 
this is a bijective mapping of $\Pi_C(S)$ to $ \Pi_{\Delta(c)}(S)$ for
any $S$, $|\Pi_C(S)|=|\Pi_{\Delta(c)}(S)|$, and 
VC-dimension$(\Delta(c))$=VC-dimension$(C)$ follows.

Hence we have
\begin{eqnarray*}
Pr[B] & \le & |\Pi_{\Delta(c)}(S_1\cup S_2)|2^{-\varepsilon m/2}\\
& \le & \Phi_d(2m)2^{-\varepsilon m/2}\\
& \le & \left( {2em \over d} \right)^d 2^{-\varepsilon m/2}. 
\end{eqnarray*}

\begin{lemma} \label{le:m}
The inequality $\left( {2em \over d} \right)^d 
2^{-\varepsilon m/2} \le {\delta \over 2}$ is satisfied for 
$m=\max({4\over \varepsilon}\lg{2\over \delta}, 
       {8d\over \varepsilon}\lg{13\over \epsilon})$.
\end{lemma}
\begin{proof}
To show that $m$ satisfies the given inequality, we take logs and
verify that it satisfies 
$$ 
d\lg{2em \over d} - {\varepsilon m \over 2} \le \lg{\delta \over 2}.
$$
That is,  $m$ satisfies
$$
m \ge {2\over \varepsilon}\lg{2\over \delta}+
{2d\over \varepsilon}\lg{2em \over d}.
$$
Since ${m \over 2} \ge {2\over \varepsilon}\lg{2\over \delta}$ by our
choice of $m$, we simply need to verify that ${m \over 2} \ge
{2d\over \varepsilon}\lg{2em \over d}$.  By plugging in
$m={8d\over \varepsilon}\lg{13\over \varepsilon}$,
we obtain the following equivalent set of inequalities:
\begin{eqnarray*}
{m\over 2} & \ge & {2d\over \varepsilon}\lg{2em \over d}\\
{4d\over \varepsilon}\lg{13\over \varepsilon} & \ge & 
{2d\over \varepsilon}\lg{\left( {2e \over d} {8d\over\varepsilon}
\lg{13\over \varepsilon}\right)}\\
2\lg{13\over \varepsilon} & \ge & 
\lg{\left( {16e \over \varepsilon} \lg{13\over \varepsilon}\right)}\\
\left( 13\over \varepsilon\right)^2 & \ge &
{16e \over \varepsilon} \lg{13\over \varepsilon}\\
{13^2 \over 16e\varepsilon} & \ge & \lg{13 \over \varepsilon}
\end{eqnarray*}
It can be easily verified that the last inequality holds for any
$\varepsilon \le 1$.  Notice that ${m\over 2}$ grows faster than
${2d\over \varepsilon}\lg{2em \over d}$. Since  
$m = {8d\over \varepsilon}\lg{13\over \epsilon}$ satisfies
${m\over 2} \ge  {2d\over \varepsilon}\lg{2em \over d}$, any 
$m\ge {8d\over \varepsilon}\lg{13\over \epsilon}$ satisfies the
inequality.
Thus, $m=\max({4\over \varepsilon}\lg{2\over \delta}, 
       {8d\over \varepsilon}\lg{13\over \epsilon})$
satisfies the original inequality.
\end{proof}

Combining Lemma \ref{le:m} with the bound 
$m\ge {8\over\varepsilon} \ln{2}$ obtained
from inequality  \ref{eq:b1}, we have proved the following result:
\begin{theorem}
Let $C$ be any concept class of VC-dimension $d$.  Let $L$ be any
algorithm that takes as input a set $S$ of $m$ labeled examples
of a concept in $C$, and produces as output a concept $h\in C$ that
is consistent with $S$.  Then $L$ is PAC learnable for $C$ provided
it is given a random sample of $m$ examples from EX(c,D), where
$m$ obeys
$$
m\ge \max({8\over \varepsilon}\lg{2\over \delta}, 
       {8d\over \varepsilon}\lg{13\over \epsilon}).
$$
\end{theorem}


\section*{Appendix}
The Chernoff bound is a fundamental result from probability theory that
appears repeatedly in theoretical computer science.  

\begin{theorem}
Let $X_1, \dots, X_m$ be a sequence of $m$ independent Bernoulli trials
(coin flips), each with probability of heads $E[X_i]=p$.  Let 
$S=X_1 + \dots +X_m$ be a random variable indicating the total number
of heads, so $E[S]=pm$.  Then for $0\le\gamma\le 1$, the following
bounds hold:

(Additive Form)
$$
Pr[S>(p+\gamma)m]\le e^{-2m\gamma^2}
$$
and
$$
Pr[S<(p-\gamma)m]\le e^{-2m\gamma^2}.
$$

(Multiplicative Form)
$$
Pr[S>(1+\gamma)pm]\le e^{-mp\gamma^2/3}
$$
and
$$
Pr[S<(1-\gamma)pm]\le e^{-mp\gamma^2/2}.
$$
  
\end{theorem}

\begin{thebibliography}{99}

\bibitem{KeaVaz94} M. Kearns and U. Vazirani.  An Introduction to 
Computational Learning Theory. 57-62.

\bibitem{BEHW89} A. Blumer, A. Ehrenfeucht, D. Haussler, and M.K. Warmuth.
Learnability and the Vapnik-Chervonenkis dimension.  {\em Journal of
the ACM}, 36(4): 929-965, 1989.
\end{thebibliography}







%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes end here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}














