%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[12pt,twoside,alltt]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecure
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{21 }{November 28, 1994}{Ron Rivest}{Andreas Argiriou} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Outline}

\begin{itemize}
\item Learning decision trees with Fourier theory
\end{itemize}

In this lecture we see how we can apply Fourier techniques to learn decision trees, as described by Kushilevitz and Mansour in \cite{KM}. Such techniques have also been used in other problems, like DNF formulas. 



\section{Introduction}

The type of learning game we need to consider now is special in the following ways:

\begin{enumerate}
\item The distribution is {\em uniform} and not arbitrary.
\item We want to learn {\em decision trees}.
\item There is no source of random examples.
\item The algorithm uses {\em membership queries}.

\end{enumerate}

In the model of membership queries, the learner makes up the questions
so that it may ask for the label of an example. With {\em equivalence
queries}, on the other hand, the learner presents a hypothesis $h$ and
asks if $h$ is the same as the target concept~$f$.  If $h \neq f$, the
learner receives a counterexample as an answer. In this problem we
essentially need statistical queries, but we can use membership
queries to get information of the type a statistical query would give.

We will consider boolean functions with a slight modification: a boolean function now maps $\{0, 1\}^n$ to $\{\pm1\}$ instead of $\{0, 1\}$ in the following way $$x \in \{0, 1\} \mapsto (-1)^x.$$
In other words, negative examples map to $+1$, and positive examples
map to $-1$.

The concept class being learned is the class ${\cal C}$ of boolean functions computed by {\em decision trees with linear operations}. In this model each node computes a sum (\mbox{modulo} 2) of a subset of the $n$ variables and if the result is 1 branches to the right subtree, else branches to the left subtree. The output is the value of the leaf where we end up.  

Learning this class in a passive model of learning is an open problem. In our new model, when given $\epsilon, \delta$ we need to produce a hypothesis $h$ such that $\mbox{Pr}[h(x) \neq f(x)] \leq \epsilon$ with probability at least $1-\delta$ (where $x$ is drawn according to the uniform distribution and $f$ is the target function). 

\section{The orthonormal basis}

The basis functions for the Fourier transform we will use are given by the following definition.

{\definition{For each $z \in \{0, 1\}^n$, define the function $\chi_z:\{0, 1\}^n \rightarrow \{\pm1\}$ by $$\chi_z(x)=(-1)^{z \cdot x \bmod 2 }$$ }
\label{chi}
}

\begin{table}
\begin{center}
\begin{tabular}
{|l|r|r|r|r|}
\hline
{} & \multicolumn{4}{c|}{$x$} \\ \hline
$z$ & 00 & 01 & 10 & 11 \\ \hline
00 & 1 & 1 & 1 & 1 \\ \hline
01 & 1 & -1 & 1 & -1 \\ \hline
10 & 1 & 1 & -1 & -1 \\ \hline
11 & 1 & -1 & -1 & 1  \\ \hline
\end{tabular}
\caption{The $\chi$ functions for $n=2$.}
\label{tbl}
\end{center} 
\end{table}

The value of the function depends on the exclusive or of those variables ``selected'' by $z$ in the dot product. If  $\sum_{i=1}^{n} z_i x_i \equiv 0 \bmod 2 $ the output is $+1$, else $-1$. Table \ref{tbl} shows the values of the $\chi$ functions if $n=2$. 

Our objective now is to define an inner product of functions such that 
\begin{equation}
\langle \chi_{z_1}, \chi_{z_2} \rangle = 
\left\{ \begin{array}{ll}             
   0 & \mbox{if $z_1 \neq z_2$} \\
   1 & \mbox{if $z_1 = z_2$}
   \end{array}
   \right. 
\label{orth}
\end{equation}

and such that a function $f$ on $\{0, 1\}^n$ can be uniquely expressed as

\begin{equation}
	f = \sum_{z \in \{0, 1\}^n} \hat{f}(z) \chi_z.
\label{fourier}
\end{equation}
where $\hat{f}(z)$ are real constants.
%%It is easy to show (\ref{fourier}) and table \ref{tbl} demonstrates for the case $n=2$ that the $\chi$ functions form a basis.

The Fourier transform of $f$ is the expansion of $f$ as a linear
combination of the $\chi_z$'s.
Equations (\ref{orth}) and (\ref{fourier}) together state that the $\chi$ functions form an {\bf orthonormal basis}. Then the Fourier coefficients are given by
\[
\hat{f}(z) = \langle f, \chi_z \rangle.
\]

The method of using Fourier coefficients is helpful because now the objective will be constructing an algorithm that approximates a function $f$ as a linear combination of the basis functions. The algorithm should find enough of the biggest Fourier coefficients to meet restrictions on the error. 

To define the inner product, we use the concept of the expectation of $f$ (over uniform distribution) $$\mbox{E}[f] = \frac{1}{2^n}\sum_{x \in \{0, 1\}^n}f(x).$$

{\definition{For any functions $f, g:\{0, 1\}^n \rightarrow \Re$, define the {\em inner product} of $f$, $g$ as
$$ \langle f, g \rangle = \mbox{E}[fg] = \frac{1}{2^n}\sum_{x \in \{0, 1\}^n}f(x)g(x)$$
and the ${\cal L}_2$ {\em norm} of $f$ as
$$\|f\|_2 = \sqrt{\langle f, f \rangle}.$$
}
\label{norm}
}

Note that for {\em boolean functions} $\|f\|_2 = 1$. 

It is also clear from definition \ref{chi} that for all $z_1, z_2 \in \{0, 1\}^n$, 
\[	
\chi_{z_1} \chi_{z_2} = \chi_{z_1 \oplus z_2}
\]
where $\oplus$ denotes exclusive or.

Using the above, one can see that definition \ref{norm} verifies equations (\ref{orth}), so that indeed the $\chi$ functions form an orthonormal basis. 

Another useful relation is {\em Parseval's identity}:  
\begin{equation}
\|f\|_2^2 = \sum_{z \in \{0, 1\}^n} \hat{f}^2(z)  
\label{Parceval}
\end{equation}

or for boolean functions 
\begin{equation}
\sum_{z \in \{0, 1\}^n} \hat{f}^2(z) = 1.  
\label{BParceval}
\end{equation}

Finally, we can define the ${\cal L}_1$ {\em norm} of $f$, as follows:

{\definition{The ${\cal L}_1$ {\em norm} of $f$ is 
$$ {\cal L}_1(f) = \sum_{z \in \{0, 1\}^n}|\hat{f}(z)|. $$
}}

The ${\cal L}_1$ norm is a measure of how spread out the coefficients are and ranges from 1 to $2^{\frac{n}{2}}$.

\section{Approximation by sparse functions}

The following theorem will be proven later:
{\theorem{If the function $f$ is represented by a decision tree with $m$ nodes, then ${\cal L}_1(f) \leq m$.}}

We will thus try to learn decision trees with a small number of nodes, since it is easier to approximate a function with a small ${\cal L}_1$ norm (intuitively, weight is not spread a lot and there are just a few big coefficients).  

The approximation of the target function $f$ will be another function $g$, real valued and not boolean. However 
\[ \mbox{sign}(g(x)) =  
\left\{ \begin{array}{ll}             
   +1 & \mbox{if $g(x) \geq 0$} \\
   -1 & \mbox{otherwise}
   \end{array}
   \right. 
\]

is a boolean function. So we want $\mbox{sign}(g(x))$ to be a good approximation of $f$. We see now that this happens when {\em $g$ $\epsilon$-approximates $f$ in the ${\cal L}_2$ norm}.

{\definition{We say that a function $g$ $\epsilon$-approximates $f$ in ${\cal L}_2$ norm, when $$\mbox{E}[(f-g)^2] \leq \epsilon$$. 
}}
 
Notice that when $\mbox{sign}(g(x)) \neq f(x)$, then $(f(x)-g(x))^2 \geq 1$ and hence $\mbox{Pr}[f \neq \mbox{sign}(g)] \leq \mbox{E}[(f-g)^2]$. Thus we have the following lemma:

{\lemma{If $g$ $\epsilon$-approximates $f$ in ${\cal L}_2$ norm, then $\mbox{Pr}[f \neq \mbox{sign}(g)] \leq \epsilon$.}}    

So, now the question is how many of the Fourier coefficients are needed to $\epsilon$-approximate the target function. For this reason it is helpful to define the concept of a {\em $t$-sparse} function.

{\definition{The function $g$ is {\em $t$-sparse} if it has at most $t$ non-zero Fourier coefficients.
}} 

As we mentioned earlier, we prefer approximations with large coefficients and the next lemma shows that this can be achieved.

{\lemma{If $f$ can be $\epsilon$-approximated (in ${\cal L}_2$ norm) by a $t$-sparse function $g$ then $f$ can be $\epsilon+\mbox{O}(\epsilon^2)$-approximated by a $t$-sparse function $h$ whose non-zero coefficients are at least $\epsilon/t$. 
}
\label{t-sparce}
}

%% Generate file with:
%%   fig2dev -L latex lecture03-numline.fig >! lecture03-numline.tex
\begin{figure}
\begin{center}
\include{lecture21-fig}
\caption{Search for large coefficients on the $n$-dimensional cube}
\label{fig:tree}
\end{center}
\end{figure}

{\proof{ Write $g(x) = \sum_{i=1}^{t} \hat{g}(z_i) \chi_{z_i}(x)$. Parseval's identity (\ref{Parceval}) gives $\mbox{E}[(f-g)^2] = \sum_{z} (\hat{f}(z) - \hat{g}(z))^2$ and this has a minimum when $\hat{g}(z_i) = \hat{f}(z_i), i=1, \ldots, t$. Let $g'(x) = \sum_{i=1}^{t} \hat{f}(z_i)\chi_{z_i}(x)$. We can show that $$h(x) = \sum_{\hat{f}(z_i) \geq \frac{\epsilon}{t}} \hat{f}(z_i)\chi_{z_i}(x)$$ is as desired. Indeed, $\mbox{E}[(f-h)^2] - \mbox{E}[(f-g)^2] \leq \mbox{E}[(f-h)^2] - \mbox{E}[(f-g')^2] =  \sum_{\hat{f}(z_i) < \frac{\epsilon}{t}} \hat{f}^2(z_i) < t (\frac{\epsilon}{t})^2 = \mbox{O}(\epsilon^2)$. Since $\mbox{E}[(f-g)^2] \leq \epsilon$ we are done.
\hfill $\Box$
}}

A consequence of lemma \ref{t-sparce} is that our algorithm can look only for the larger Fourier coefficients. However, the space is huge and this is not an easy task. What we can do is use a recursive procedure to implement tree search over the values of $z$ and find the large coefficients (see Figure \ref{fig:tree}). The problem is solved if at each node we can say whether there is a large coefficient in the subtree. With a {\em perfect} test like that we would need $\mbox{O}(tn)$ work to find the $t$ largest coefficients in a tree of depth $n$. We need only to figure out 
\begin{itemize}
\item how to do the test
\item how many coefficients to find.
\end{itemize}

The idea is to define a function that includes coefficients under a certain node.

{\definition{For each $\alpha \in \{0, 1\}^k$, define the function $f_{\alpha}: \{0, 1\}^{n-k} \rightarrow \Re$ by $$ f_{\alpha}(x) = \sum_{\beta \in \{0, 1\}^{n-k}} \hat{f}(\alpha\beta)\chi_{\beta}(x).$$
}}

By estimating $\mbox{E}[f_{\alpha}^2]$, which equals $\sum_{\beta \in \{0, 1\}^{n-k}} \hat{f}^2(\alpha\beta)$ (Parseval), it will be possible to make a prediction about the size of the coefficients $\hat{f}(\alpha\beta)$. The following lemma will be necessary for the approximation of $\mbox{E}[f_{\alpha}^2]$:

{\lemma{If $1 \leq k < n$ and $\alpha \in \{0, 1\}^k$, then $$f_{\alpha}(x) = \mbox{E}_{y \in \{0, 1\}^k}[f(yx)\chi_{\alpha}(y)].$$ 
}}

{\proof{ It is easy to see from the definitions that if $z = z_1z_2$ and $|z_1| = k$, then $\chi_z(yx) = \chi_{z_1}(y)\chi_{z_2}(x)$. So, we have
$$ \mbox{E}_y[f(yx)\chi_{\alpha}(y)] = \mbox{E}_y[\sum_z \hat{f}(z)\chi_z(yx)\chi_{\alpha}(y)] = \mbox{E}_y[\sum_{z_1}\sum_{z_2}\hat{f}(z_1z_2)\chi_{z_1}(y)\chi_{z_2}(x)\chi_{\alpha}(y)] $$ 
$$= \sum_{z_1}\sum_{z_2}\hat{f}(z_1z_2)\chi_{z_2}(x)\mbox{E}_y[\chi_{z_1}(y)\chi_{\alpha}(y)].$$ 
Now from equations (\ref{orth}) we see that 
\[ \mbox{E}_y[\chi_{z_1}(y)\chi_{\alpha}(y)]= 
\left\{ \begin{array}{ll}             
   0 & \mbox{if $z_1 \neq \alpha$} \\
   1 & \mbox{if $z_1 = \alpha$}
   \end{array}
   \right. 
\]

and therefore 
$$\mbox{E}_y[f(yx)\chi_{\alpha}(y)] = \sum_{z_2} \hat{f}(\alpha z_2)\chi_{z_2}(x) = f_{\alpha}(x)$$
\hfill $\Box$
}}

The remaining issues are now the following:

\begin{itemize}
\item How to use an approximation technique to pick out large Fourier coefficients
\item Showing the approximation is good enough
\item Showing the algorithm applies to decision trees.  

\end{itemize}


\begin{thebibliography}{99}

\bibitem{KM}
Eyal Kushilevitz and Yishay Mansour. 
\newblock Learning Decision Trees using the Fourier Spectrum.
\newblock In {\em Proceedings of the 23rd Annual ACM Symposium on Theory of Computing}, pages 455-464, 1991. 

\end{thebibliography}

\end{document}














