%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[12pt,twoside,alltt,psfig]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecure
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{19 }{November 21, 1994}{Javed Aslam}{Gillian Lee} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Outline}
\begin{itemize}
\item Standard PAC Model vs. Noisy PAC Model
\item Noise Models
\item Minimizing disagreements to learn with classification noise 
\item Minimizing disagreements for conjunctions is NP-hard
\end{itemize}

The PAC model assumes access to a noise-free oracle for examples of the 
target concept.  In reality, we need learning algorithms with at least some 
tolerance for mislabeled examples.  In this lecture, we show that all
finite concept classes can be learned with classification noise if we
minimize disagreements.  However, then we show that for the
simple concept class of monotone conjunctions, minimizing
disagreements is NP-hard.  In the following lecture, we will show how
to get {\em efficient} learning algorithms via the statistical query model.

\section{Standard PAC Model vs. Noisy PAC Model}

The standard PAC model assumes that the learning algorithms have access to a
noise-free examples oracle for the target concept (see Figure~\ref{fig:picture1}).  In the
noisy PAC model, however, we may have noise which corrupts  examples
and/or the classification of examples (see
Figure~\ref{fig:picture2}).    We consider two types of noise models:
the {\em malicious noise} model and the {\em classification noise} model.

%%%%% PostScript PACmodel %%%%%

\begin{figure}
\begin{center}
%\mbox{\epsfxsize=4in \epsffile{lecture19-fig1.eps}}
\mbox{\psfig{figure=lecture19-fig1.eps}}
\caption{Standard PAC Model}
\label{fig:picture1}
\end{center}
\end{figure}



\subsection{Malicious Noise}

$EX^\beta_{MN}(c_*,{\cal D}) =  \left\{ \begin{array}{ll}
	\mbox{with probability $1-\eta$, returns }
        \langle x,c_*(x)\rangle \mbox{ from $EX(c_*,{\cal D})$}\\
	\mbox{with probability $\eta$, returns }
        \langle x,l \rangle \mbox{ for maliciously chosen}\\ 
         \mbox{\hspace{.2in} } \mbox{ } x \in X \mbox{ and } l \in \{0,1\}
	\end{array}\right.$ \\


The malicious error noise model was first introduced by Valiant (1985).  His results for
CNF($n$,$k$) hold only for a small rate of noise.  Kearns and Li
(1987) show that the malicious noise model can tolerate only a small
rate of noise for any domain.

%%%%% PostScript noisyPACmodel %%%%%

\begin{figure}
\begin{center}
%%\mbox{\epsfxsize=4in \epsffile{lecture19-fig2.eps}}
\mbox{\psfig{figure=lecture19-fig2.eps}}
\caption{Noisy PAC Model}
\label{fig:picture2}
\end{center}
\end{figure}


\subsection{Random Classification Noise}

$EX^\eta_{CN}(c_*,{\cal D}) =  \left\{ \begin{array}{ll}
	\mbox{with probability $1-\eta$, returns $\langle x,c_*(x) \rangle$ from $EX(c_*,{\cal D})$}\\
	\mbox{with probability $\eta$, returns $\langle x,\neg {c_*(x)} \rangle$ from $EX(c_*,{\cal D})$}
	\end{array}\right.$ \\

\begin{itemize}
\item label flipped with probability $\eta$
\item example unchanged
\end{itemize}

This model was first introduced by Angluin and Laird (1988).  We shall discuss their 
results below.

\section{Learning With Classification Noise}

We assume:
\begin{itemize}
\item The learner knows which type of noise exists.
\item The learner has an upper bound $\eta_b$ on noise rate $(0\leq\eta\leq\eta_b<0.5)$
\end{itemize}

As the noise rate approaches 0.5, the labels provided by the noisy
oracle are providing less and less information about the target
concept.  The learning alorithm thus needs more oracle calls and more
computation time as the noise rate approaches 0.5.  When the noise
rate is equal to 0.5, PAC learning becomes impossible, because
every label seen by the algorithm is the outcome of an unbiased coin
flip, and gives no information about the target concept.



{\definition{
${\cal C}$ is PAC-learnable by ${\cal H}$ in the presence of noise 
if there exists a learning algorithm ${\cal L}$  with the property that 
$(\forall c\in{\cal C}) (\forall {\cal D} \mbox{ on } X)
 (\forall \varepsilon$, $0<\varepsilon<1$) ($\forall\delta$, $0<\delta<1$) 
 ($\forall\eta$, $0\leq\eta<0.5)$,
if ${\cal L}$ is given inputs $n$,$\varepsilon$, $\delta$, $\eta_b$ ($\eta\leq\eta_b<0.5)$ and access to 
$EX^\eta(c_*,{\cal D})$, then it will with probability $\geq 1-\delta$ produce an ouptut hypothesis
$h\in{\cal H}$ s.t. $error(h)\leq\varepsilon$.}}

{\definition{
${\cal C}$ is efficiently PAC-learnable if the running time of ${\cal L}$ is polynomial
in $n$, $\frac{1}{\varepsilon}$, $\frac{1}{\delta}$, $\frac{1}{1-2\eta_b}$.}}

Note:
\begin{itemize}
\item The error rate is measured with respect to the target concept and distribution.
	$$error(h) = \sum_{x:c_*(x)\neq h(x)} \Pr_D(x)$$
\item The learner is allowed more time as $\eta_b\rightarrow0.5$.
\end{itemize}


\subsection{Angluin and Laird Method}

The typical noise-free PAC algorithm draws a large number of samples and outputs a
consistent hypothesis. With classification noise, however, there may not be a
consistent hypothesis. Angluin and Laird propose the following method.

\begin{itemize}
\item Draw a ``large enough'' sample.
\item Output hypothesis $c\in{\cal C}$ which minimizes disagreements
with the sample.
\end{itemize}

Suppose concept $c_i$ has true error rate $d_i$.
What is the probability $p_i$ that $c_i$ disagrees with a labelled example drawn from  
$EX^\eta_{CN}(c_*,{\cal D})$?  We have two cases. 

\begin{enumerate}
\item $EX^\eta_{CN}(c_*,{\cal D})$ reports correctly, but~$c_i$ is incorrect:
	$d_i(1-\eta)$
\item $EX^\eta_{CN}(c_*,{\cal D})$ reports incorrectly, but~$c_i$ is correct:
	$(1-d_i)\eta$
\end{enumerate}

$$p_i = d_i(1-\eta) + (1-d_i)\eta$$
$$p_i = \eta + d_i(1-2\eta)$$

An $\varepsilon$-good hypothesis has expected disagreement rate 
$\leq \eta + \varepsilon(1-2\eta)$.\\
An $\varepsilon$-bad hypothesis has expected disagreement rate 
$> \eta + \varepsilon(1-2\eta)$.

%%%%% Disagreement Rates %%%%%

\begin{figure}
\begin{center}
%\mbox{\epsfxsize=4in \epsffile{lecture19-fig3.eps}}
\mbox{\psfig{figure=lecture19-fig3.eps}}
\caption{Disagreement Rates}
\label{fig:picture3}
\end{center}
\end{figure}

We need $m$ large enough such that an $\varepsilon$-bad hypothesis
will not minimize disagreements with the hypothesis.  Consider the
point $\eta$ + $\frac{\varepsilon(1-2\eta_b)}{2}$.  If an
$\varepsilon$-bad hypothesis minimizes disagreements, then the target
function must have at least as large of a disagreement rate.  Thus, at
least one of the following events must hold:

\begin{enumerate}
\item Some $\varepsilon$-bad hypothesis $c_i$ has empirical disagreement rate $\leq 
	\eta + \frac{\varepsilon(1-2\eta_b)}{2}$.  
\item Target concept $c_*$ has empirical disagreement rate $\geq
	\eta + \frac{\varepsilon(1-2\eta_b)}{2}$.  
\end{enumerate}
 
We can make these probabilities small (i.e., $\leq\frac{\delta}{2}$)
using Hoeffding Bounds:
$$GE(p, m, p + \alpha) \leq e^{-2m\alpha^2}$$
$$LE(p, m, p - \alpha) \leq e^{-2m\alpha^2}$$



\mbox{We have: }
\begin{eqnarray*}
\Pr\mbox{(event 2)} 
         &=& GE(\eta, m, \eta +\frac{\varepsilon(1-2\eta_b)}{2})\\
         & \leq & e^{-2m\left(\frac{\varepsilon(1-2\eta_b)}{2}\right)^2} \\
\end{eqnarray*}

We want $\Pr \mbox{(event 2)} \leq\frac{\delta}{2}$, so choose 
	$$m\geq\frac{2}{\varepsilon^2(1-2\eta_b)^2}\ln\left(\frac{2}{\delta}\right).$$

\mbox{We have: }
\begin{eqnarray*}
\Pr\mbox{(event 1)} & \leq & LE(\eta + \varepsilon(1-2\eta), m, 
		\eta + \frac{\varepsilon(1-2\eta_b)}{2})|{\cal C}|\\  
 & 	\leq & LE(\eta + \varepsilon(1-2\eta_b), m, 
		\eta + \frac{\varepsilon(1-2\eta_b)}{2})|{\cal C}|\\  
 &	\leq & |{\cal C}| e^{-2m(\frac{\varepsilon(1-2\eta_b)}{2})^2}\\  
\end{eqnarray*}

We want $\Pr${(event 1)}$\leq\frac{\delta}{2}$, so choose
	$$m\geq\frac{2}{\varepsilon^2(1-2\eta_b)^2}\ln\left(\frac{2|{\cal C}|}{\delta}\right).$$

If we choose $m$ larger than both values, then the probability that
either of the events occurs is at most $\delta$.  That is, the
probability that an $\epsilon$-bad hypothesis minimizes disagreements
is at most $\delta$.  Thus, if we draw a sample of size $m$ as
specified above, and then find a hypothesis which minimizes
disagreements with the sample, then we have an algorithm which PAC
learns in the presence of classification noise. This gives the
following theorem.

\begin{theorem}
For all finite concept classes ${\cal C}$, we can PAC learn in the
presence of classification noise.  \qed
\end{theorem}

\subsection{Minimizing disagreements can be NP-hard}

\begin{theorem}
Finding monotone conjunctions which minimize disagreements with a given sample is NP-hard.
\end{theorem}

\begin{proof_sketch}
Reduce the vertex cover problem to that of finding a monotone
conjunction which minimizes disagreements with a given sample.  Given
a graph $G=(V,E)$, and a constant $c$, the vertex cover problem is to
determine whether there is a vertex cover of size at most $c$ (i.e.,
whether there is a subset of the nodes of size at most $c$ such that
every edge in the graph is adjacent to some vertex in this subset).
Our reduction is given by the following table.

\begin{tabular}{ll}
Graph & Monotone Conjunction \\ \hline
$V=\{v_1, v_2,...v_n\}$ & $\{x_1, x_2,...x_n\}$ define the instance space.\\
$\forall v_i \in V$  & $\mbox{ example } (\langle 11 \cdots 101
\cdots 11 \rangle,+)$, with $0$ in $i$-th position. \\
$\forall \mbox{ edges } (v_i, v_j) \in E$ & ${c+1} \mbox { examples } 
(\langle 11 \cdots 101 \cdots 101 \cdots 11 \rangle, -)$,\\
& $\mbox{ \hspace{.1in}}$  with $0$'s in
$i$-th and $j$-th positions. \\
\end{tabular}

If there are $n$ vertices in the graph, then our instance space is
$\{0,1\}^n$.  For each vertex in the graph, we introduce a positive
example, and for each edge in the graph we introduce $c+1$ negative
examples.  It is straightforward to show the following.
\begin{claim}
$G$  has a vertex cover of size at most $c$ iff there is a monotone
conjunction with at most $c$ disagreements. 
\end{claim}

\end{proof_sketch}
 
\begin{thebibliography}{99}

\bibitem{AngLai88} D. Angluin and P. Laird.  Learning from noisy
examples.  {\em Machine Learning}, 2(4):343-370, 1988.

\bibitem{KeaLi87} M. Kearns and M. Li.  {\em Learning in the presence of malicious errors.} (Technical Report TR-03-87).  Cambridge, MA: Harvard University, Center for Research in Computing Technology.

\bibitem{KeaVaz94} M. Kearns and U. Vazirani.  {\em An Introduction to 
Computational Learning Theory.} MIT Press, 1994.

\bibitem{Val85} L. G. Valiant.  Learning disjunctions of conjunctions.  {\em Proceedings of the Ninth International Joint Conference of Artificial Intelligence}  (pp. 560-566). Los Angeles, CA: Morgan Kaufmann.




\end{thebibliography}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes end here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{document}














