%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[12pt,twoside,alltt,psfig]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecture
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{6 }{September 28, 1994}{Ron Rivest}{Marcos Kiwi} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Outline}

\begin{itemize}
\item
Comments on learning $3$-term DNF by $3$-CNF.

\item
Learning $k$-DL.

\item
Hierarchies.

\item
Occam's Razor (general case).

\item
Learning conjunction with few literals.
\end{itemize}

\section{Comments on learning $3$-term DNF by $3$-CNF.}

Last lecture we showed that $3$-term DNF could be rewritten as
  $3$-CNF.  
Since the concept class $3$-CNF is efficiently PAC learnable,
  we concluded that $3$-term DNF are also efficiently PAC learnable
  using $3$-CNF.

In general, the following holds:
\begin{theorem}
For any constant $k \geq 2$, the concept class of $k\mbox{\em -term DNF}$ 
  is efficiently PAC learnable using $k\mbox{\em -CNF}$.
\end{theorem}

In the next section we will introduce a concept class that contains
  both the concept classes $k\mbox{-CNF}$ and $k\mbox{-DNF}$, which
  can still be efficiently learned. 

\section{Learning $k$-Decision Lists.}

A decision list is basically an ordered sequence of if-then-else
  statements. 
The sequence of if-then-else conditions are tested in order, and the
  answer associated to the first satisfied condition is output 
  (see Figure~\ref{fig:decision_list}).
\begin{figure}
\begin{center}
\mbox{\psfig{figure=lecture-06-fig-1.ps,height=2.9in,width=1.5in}}
\caption{A $2$-decision list.}
\label{fig:decision_list}
\end{center}
\end{figure}

More precisely,
\begin{definition}
A {\em $k$-decision list} over the variables $x_{1}, \ldots, x_{n}$ is an
  ordered sequence $L=(c_{1}, b_{1}), \ldots, (c_{l},b_{l})$ and 
  a bit $b$, in which each $c_{i}$ is a conjunction of $k$ literals over
  $x_{1},\ldots, x_{n}$.
The bit $b$ is called the {\em default} value, and $b_{i}$ is 
  referred to as the bit {\em associated} with condition $c_{i}$.
For any input $x \in \{0, 1\}^{n}$, $L(x)$ is defined to be the bit
  $b_{j}$, where $j$ is the smallest index satisfying $c_{j}(x)=1$;
  if no such index exists, then $L(x)=b$.

We denote by $k\mbox{\em -DL}$
  the class of concepts that can be represented by a 
  $k$-decision list.
\end{definition}

We now consider the issue of what is the expressive power of $k$-decision
  lists.
\begin{lemma}\label{le:k_decision_list_contains}
\[
k\mbox{\em-DNF} \cup k\mbox{\em-CNF} \subseteq k\mbox{\em-DL}.
\]
\end{lemma}
\begin{proof}
First notice that if a concept $c$ can be represented as a
  $k$-decision list, then, so can $\neg c$ (just complement the values
  $b_{i}$ and $b$ of the decision list representing the concept $c$).
Thus, it suffices to show that $k\mbox{-DNF} \subseteq k\mbox{-DL}$.
But a $k\mbox{-DNF}$ can be represented as a $k$-decision list by
  simply choosing an arbitrary ordering in which to evaluate the terms
  of the $k\mbox{-DNF}$, setting all the $b_{i}$'s to $1$ and the
  default to $0$.
\end{proof}

The containment of Lemma~\ref{le:k_decision_list_contains}
  is strict. That is, there exist
  functions that can be represented by a $k$-decision list, but not by
  either $k$-DNF or $k$-CNF.

The main result of this section is,
\begin{theorem}
For any constant $k \geq 1$, the representation class of $k$-decision
  lists is efficiently PAC learnable.
\end{theorem}
\begin{proof}
Consider an input sample $S$, where $|S|=m$ and $S$ is consistent with
  some $k$-decision list.
We build a decision list consistent with $S$ as follows;
  we first find a 
  conjunction $c(\cdot)$ of $k$ literals (by going over all the 
  $\Theta(n^{k})$ 
  conjunctions of $k$ literals if necessary) such that the set 
  $S_{c} = \{ x \in S : c(x) = 1\}$ is both non-empty, and contains
  only positive examples, or negative examples.
We call such conjunction $c(\cdot)$ a {\em good} 
  conjunction.
If $S_{c} = S$ then we set the default bit to $1$ 
  if $S_{c}$  contains only positive examples, or to $0$ if
  $S_{c}$ contains only negative examples.
Otherwise 
  we add the conjunction $c(\cdot)$ (with the associated bit $1$ 
  if $S_{c}$  contains only positive examples, and the associated bit
  $0$ if $S_{c}$ contains only negative examples)
  as the last condition in the current hypothesis decision list, 
  update $S$ to $S - S_{c}$, and iterate until $S = \phi$.

Clearly all the examples in the initial sample $S$ are correctly classified
  by the hypothesis $k$-decision list built by the process described above.
Also, the process will always succeed in finding a consistent 
  hypothesis, because it succeeds at each iteration in 
  correctly setting the value of the default bit or in finding a good
  conjunction.
Note that the first conjunction  of the target $k$-decision
  list is a good conjunction.

Since any $k$-decision list on $n$ variables can be encoded in 
  $\Theta( k n^{k} \log n )$ bits, we can apply the simple case
  of Occam's Razor to obtain a sample size bound of 
  $m = \Omega( (1/\epsilon)( \log (1/\delta) + k n^{k} \log n))$ 
  for PAC learning.
Since the described procedure runs in time polynomial in $m$, we get 
  efficient PAC learning.
\end{proof}

\section{Hierarchies}

In this section we review our understanding of the relationships between 
  the representation classes that we have discussed. 
First, we need the following definitions:
\begin{definition}
A {\em boolean circuit} over $x_{1}, \ldots, x_{n} \in \{0, 1\}$ of size
  $N$, is a
  directed acyclic graph on $N$ vertices, 
  in which each vertex has indegree either
  $0,1$ or $2$, and unbounded outdegree. 
Each vertex of indegree $0$ is labeled with one of the 
  {\em input variables} $x_{1}, \ldots, x_{n}$.
Each vertex of indegree $1$ by the symbol $\neg$, and each vertex of
  indegree $2$ is labeled by one of the symbols $\vee$ and $\wedge$.
There is a single designated {\em output vertex} of outdegree $0$.
When the input variables are assigned values the graph computes a 
  boolean function in the obvious way.
\end{definition}

\begin{definition}
The class {\em Poly-size Circuits}, is the concept class $\cal{C}$
  in which each concept in ${\cal C}_{n}$ 
  is computed by a boolean circuit of size at most 
  $p(n)$, where $p(\cdot)$ is some fixed polynomial.
\end{definition}

Our knowledge about the relationship between the concept classes that
  we have so far defined can be summarized as follows:
\[
\begin{array}{ccccccccccc}
\mbox{Conjunc.} & \hspace{-.08in}\subseteq\hspace{-.08in} 
	& k\mbox{-term DNF} & \hspace{-.08in}\subseteq\hspace{-.08in} 
	& k\mbox{-CNF}& \hspace{-.08in}\subseteq\hspace{-.08in} 
	& k\mbox{-DL}& \hspace{-.08in}\subseteq\hspace{-.08in} 
	& \mbox{Poly-size Circuits}& \hspace{-.08in}\subseteq\hspace{-.08in} 
	& \mbox{DNF} \\
(easy) && (hard) && (easy) && (easy) && (hard) 
\end{array}
\]


\section{Occam's Razor (General Case).}

The purpose of this section is to formalize the notion of Occam
  learning.
In Occam learning, an algorithm takes a sample  
  $S=\{ <x_{1},c(x_{1})>, \ldots, <x_{m},c(x_{m})>\}$, and outputs
  a ``succinct'' hypothesis $h$ that is consistent with the sample,
  {\em i.e.} $h(x_{i})=c(x_{i})$ for each $i$, and $size(h)$ is a 
  slowly growing function of $n$, $size(c)$, and $m$.
Formally,
\begin{definition}
For constants $\alpha \geq 0$ and $0 \leq \beta < 1$. 
$L$ is an $(\alpha, \beta)$-Occam algorithm for the concept class 
  $\cal{C}$ using $\cal{H}$ if on input sample $S$ of cardinality
  $m$ consistent with $c \in {\cal C}_{n}$, $L$ outputs a hypothesis 
  $h \in \cal{H}$ such that:
\begin{itemize}
\item
$h$ is consistent with $S$.

\item
$size(h) \leq (n \cdot size(c))^{\alpha}m^{\beta}$.
\end{itemize}
We say that $L$ is an efficient $(\alpha,\beta)$-Occam algorithm if
  its running time is bounded by a polynomial in $n$, $m$ and 
  $size(c)$.
\end{definition}

Observe that the definition of an Occam algorithm captures the notion
  of succinctness by requiring that for $m \gg  n$, the size of the
  output is effectively $O( m^{\beta})$ for some $\beta < 1$, and thus
  $o(m)$.
In other words, the hypothesis $h$ is a succinct representation of the 
  bits $c(x_{1}), \ldots, c(x_{m})$.

Our main goal is to prove the following theorem, which shows that
  efficient Occam algorithms are also efficient PAC learning
  algorithms.

\begin{theorem}[Occam's Razor]\label{th:occam_razor}
Let $L$ be an efficient $(\alpha,\beta)$-Occam algorithm for ${\cal C}$
  using ${\cal H}$.
Let ${\cal D}$ be the target distribution over the instance space $X$,
  let $c \in {\cal C}_{n}$ be the target concept, and 
  $\epsilon > 0$, $\delta \leq 1$. 
Then there is a constant $a > 0$ such that if $L$ is given as input a
  random  sample $S$ of $m$ examples drawn from $EX( c, {\cal D} )$, 
  where $m$ satisfies
\[
m \geq a\left( \frac{1}{\epsilon}\log\frac{1}{\delta} +
       \left(\frac{1}{\epsilon} (n \cdot size(c))^{\alpha}
       \right)^{\frac{1}{1-\beta}}
       \right)
\]
then, with probability at least $1-\delta$ the output $h$ of $L$
  satisfies $error(h) \leq \epsilon$.
Moreover, $L$ runs in time polynomial in $n$, $size(c)$, $1/\epsilon$
  and $1/\delta$.
\end{theorem}

The proof of Occam's Razor is a consequence of the result concerning
  the simple case of Occam's Razor covered in the preceding lecture
  which we state below in a slightly more general version.

\begin{theorem}[Occam's Razor, Cardinality version]
\label{th:cardinality_version}
Let ${\cal C}$ be a concept class and ${\cal H}$ a representation class.
Let $L$ be an algorithm such that for any $n$ and any 
  $c\in {\cal C}_{n}$, if $L$ is given as input a sample $S$ of $m$ 
  labeled examples of $c$, then $L$ runs in time polynomial in $n$,
  $m$ and $size(c)$, and outputs an $h\in {\cal H}_{n,m}$ that is
  consistent with $S$.
Then, there is a constant $b > 0$ such that for any $n$, any
  distribution ${\cal D}$ over $X_{n}$, and any target concept 
  $c \in {\cal C}_{n}$, if $L$ is given as input a random sample from
  $EX(c,{\cal D})$ of $m$ examples, where $|{\cal H}_{n,m}|$ satisfies
\[
m \geq \frac{1}{b\epsilon}
	\left(\log |{\cal H}_{n,m}| + \log\frac{1}{\delta}\right)
\]
then, $L$ is guaranteed to find a hypothesis $h \in {\cal H}_{n}$ that
  with probability at least $1-\delta$ satisfies 
  $error(h) \leq \epsilon$.
\end{theorem}

We now prove Theorem~\ref{th:occam_razor}.

\begin{proof}
Let ${\cal H}_{n,m}$ be the class of possible output hypothesis that 
  the algorithm $L$ can output on sample input $S$, where $|S|=m$.
By Theorem~\ref{th:cardinality_version} the output 
  $h$ of $L$ is such
  that $error(h) \leq \epsilon$ with confidence $1-\delta$ given that
\[
m \geq \frac{1}{b\epsilon}
	\left(\log |{\cal H}_{n,m}| + \log\frac{1}{\delta}\right).
\]
The previous inequality is implied by the following inequality,
  if we require that for a sufficiently large
  constant $a$:
\[
m \geq a\left( \frac{1}{\epsilon}\log\frac{1}{\delta} +
       \left(\frac{1}{\epsilon} (n \cdot size(c))^{\alpha}
       \right)^{\frac{1}{1-\beta}}
       \right)
\]
and recalling that since $L$ is an $(\alpha, \beta)$-Occam algorithm, 
  $\log |{\cal H}_{n,m}| \leq (n \cdot size(c))^{\alpha}m^{\beta}$.
\end{proof}

\section{Learning Conjunctions with few Literals.}

Although we already have an efficient PAC algorithm for learning boolean
  conjunctions, we can imagine a situation in which the target conjunction
  of $n$ literals, $c$, that we desire to learn, is such that 
  $size(c) \ll  n$.
This situation could represent, for example, the case in which an
  object is described by referring to only a few of its attributes out
  of a long list of potential attributes.
Our intention is to give an algorithm for learning conjunctions with
  few literals (relative to the total number of literals).
The new algorithm will make use of the negative examples, as opposed
  to our previous algorithm for learning conjunctions.

The algorithm will use a well known approximation algorithm for the Set
  Cover Problem.
Below we describe both the Set Cover Problem, and its approximation 
  algorithm.

{\bf Set Cover Problem}
\begin{itemize}
\item
{\bf Input:}
A collection ${\cal S}$ of subsets of $U=\{1,\ldots,m\}$.

\item
{\bf Output:}
A collection ${\cal S}^{\prime} \subseteq {\cal S}$, which covers $U$, 
  {\em i.e.} $\bigcup_{s\in {\cal S}^{\prime}} s = U$, 
  and such that $|{\cal S}^{\prime}|$
  is minimized.
\end{itemize}


We denote by $opt({\cal S})$ the number of sets in a minimum
  cardinality cover.
It is well known that the Set Cover Problem is an NP-hard problem.
However, there is an efficient greedy heuristic that is guaranteed to
  find a cover of cardinality at most $opt({\cal S}) \cdot \ln m$.

The latter heuristics initializes ${\cal S}^{\prime}$ to the empty
  collection. 
It then updates ${\cal S}^{\prime}$ by adding the set 
  $s\in {\cal S} - {\cal S}^{\prime}$ which covers the largest
  number of elements of $U$ that remain uncovered.
The process is then repeated until all the elements of $U$ are covered
  by ${\cal S}^{\prime}$.

Let $u_{i}$ denote the number of elements that remain uncovered after 
  $i$ iterations of the above described process. 
Clearly, $u_{0}=m$.
Furthermore, 
\[
u_{i} \leq u_{i-1}\left( 1 - \frac{1}{opt({\cal S})} \right)
        \leq m \left( 1 - \frac{1}{opt({\cal S})} \right)^{i}
        \leq m \cdot \exp{(-i/opt({\cal S}))}.
\]
The first inequality is a consequence of the fact that whatever the
  subset $U^{\prime} \subseteq U$, since $U$ has a set cover of size
  $opt(S)$, then there must be a set in $S$ that covers at least a
  $1/opt(S)$ fraction of $U^{\prime}$.
Hence, if $i >  opt({\cal S}) \cdot \ln m$, then $u_{i} < 1$.
That is the greedy heuristics outputs a cover of $U$ of
  cardinality at most $opt({\cal S}) \cdot \ln m$ as we wanted to 
  prove.\footnote{
                  A tighter analysis of the described greedy
heuristics shows that it actually is guaranteed to find a cover of
cardinality at most $opt(S) \cdot H_{\max{ \{ |s| : s \in S } \} }$ where
$H_n = 1 + \frac{1}{2} + \frac{1}{3} + \ldots + \frac{1}{n}$, 
(see~\cite{CLR}).
                }

We now return to the problem of learning conjunctions with few literals.
We shall give an Occam algorithm for this problem.
This algorithm, given a sample $S$ of size $m$, first applies the 
  algorithm for learning conjunctions that we have already seen, and 
  outputs a hypothesis $h$.
Recall that $h$ has the property that it is consistent with the positive
  examples.
The new algorithm for learning conjunctions will now make use of the 
  negative examples in $S$ to exclude literals from $h$ and 
  produce a hypothesis $h^{\prime}$.
Thus, $h^{\prime}$ will be consistent with the positive examples.
In particular, $h^{\prime}$ is built as follows; 
\begin{itemize}
\item 
For each literal $z$ of $h$, let
  $S_{z}  = \{ <x,0> \in S : z(x) = 0 \}$, ({\em i.e.} $S_{z}$ is
  the set of negative examples for which the value of $z$ is $0$).

\item 
Then, find a collection of $z$'s 
  (using the greedy heuristics described above) such that 
  the $z$'s are literals of $h$ and the $S_{z}$'s cover the set of 
  negative examples of $S$.

\item 
Let $h^{\prime}$ be the conjunction of all such literals. 
\end{itemize}

Observe that $h^{\prime}$ is consistent with the examples of $S$.
Also, note that among the $S_{z}$'s such that $z$ is a literal 
  of $h$, a cover of $size(c)$ sets exists.
Thus, $h^{\prime}$ has at most $size(c)\log m$ literals.
Hence, our hypothesis class ${\cal H}_{n,m}$ is the set of all
  conjunctions of at most $size(c)\log m$ literals, and 
  $|{\cal H}_{n,m}| \leq 2^{size(c) \log m \log n}$.
It follows that for some sufficiently large constant $c_{1}$ we have that 
\[
m \geq \frac{c_{1}}{\epsilon} \left(
                \log \frac{1}{\delta} + 
                size(c)\log n \log( size(c)\log n)
                          \right)
\]
implies that for $b>0$
\[
m  \geq   \frac{1}{b\epsilon}\left( 
                \log \frac{1}{\delta} + 
                size(c) \log m \log n
               \right)
   \geq \frac{1}{b\epsilon}\left( 
                \log \frac{1}{\delta} + 
                \log |{\cal H}_{n,m}| 
               \right).
\]
Thus, the sample size bound of Theorem~\ref{th:cardinality_version} 
  is insured and PAC learning implied.


\begin{thebibliography}{99}
\bibitem{CLR} T. Cormen, C. Leiserson, and R. Rivest. {\em
Introduction to Algorithms.} The MIT Press, Cambridge, Massachusetts,
1990.
\end{thebibliography}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes end[224z here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}


