%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NOTE: delete ,psfig here if using on athena
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[12pt,twoside,alltt,fleqn]{article}
\textwidth=6in
\oddsidemargin=0.25in
\evensidemargin=0.25in
\topmargin=-0.1in
\footskip=0.8in
\parindent=0.0cm
\parskip=0.3cm
\textheight=8.00in
\setcounter{tocdepth} {3}
\setcounter{secnumdepth} {2}
\sloppy

\newenvironment{proof}{\noindent{\bf Proof:}}{\qed\bigskip}
\newenvironment{proof_sketch}{\noindent{\bf Sketch of Proof:}\hspace*{1em}}{\qed\bigskip}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{claim}{Claim}
\newtheorem{fact}{Fact}
\newtheorem{definition}{Definition}
\newtheorem{assumption}{Assumption}
\newtheorem{observation}{Observation}
\newtheorem{example}{Example}
\newcommand{\qed}{\rule{7pt}{7pt}}



\newcounter{lecnum}
\newcommand{\omt}[1]{}
\renewcommand{\thepage}{\thelecnum-\arabic{page}}
\renewcommand{\thesection}{\thelecnum.\arabic{section}}
\renewcommand{\theequation}{\thelecnum.\arabic{equation}}
\renewcommand{\thefigure}{\thelecnum.\arabic{figure}}
\renewcommand{\thetable}{\thelecnum.\arabic{table}}



\newcommand{\lecture}[6]{
   \pagestyle{myheadings} \thispagestyle{plain} \newpage
\setcounter{lecnum}{#1} \setcounter{page}{1} \noindent \begin{center}
\framebox{ \vbox{ \hbox to 6.28in { {\bf 6.858/18.428 Machine Learning
\hfill Lecture #1: #2} } \vspace{4mm} 
\vspace{2mm} \hbox to 6.28in { {\it
Lecturer: #3 \hfill Scribe: #4} } } } \end{center}
\markboth{Lecture #1: #2}{Lecture #1: #2} \vspace*{4mm} }



\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Lecture command takes 4 arguments:
%		ordinal number of the lecture
%		date of the lecure
%		lecturer
%		scribe---that is you.
%
% Fill out the next line with this information !!!
\lecture{25}{Decmeber 12, 1994}{Ron Rivest}{Jesse Hull} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes start here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% For theorems, lemmas, definitions, remarks, etc. use commands
% {\theorem{...}}, {\lemma{...}}, {\definition{...}}, etc.
% For proofs, use \begin{proof} ... \end{proof}
%
% For figures use the following block:
%
% \begin{figure}
% \begin{center}
% \mbox{\psfig{figure=lecture-nn-fig-mm.ps}}
% \caption{A very nice picture.}
% \label{fig:picture}
% \end{center}
% \end{figure}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Outline}

\begin{itemize}
\item Learning Hidden Markov Models
\item Application to speech recognition
\end{itemize}

\section{Description of Hidden Markov Models}

In this lecture we describe Hidden Markov Models and give a method for
learning them. Their application to speech recognition is also
discussed.

\begin{definition}
A Hidden Markov Model is a 5-tuple $\{Q, V, \pi(i), A, B\}$ where
\end{definition}
\begin{description}
\item $Q$ = the set of states = $\{q_1, q_2,..., q_n\}$  
\item $V$ = the output alphabet = $\{v_1, v_2,..., v_m\}$ 
\item $\pi(i)$ = probability of being in state $q_i$ at time $t=0$
\item $A$ = transition probabilities = $\{a_{ij}\}$,  \\
       where $a_{ij}$ = $Pr[$entering state $q_j$ at time $t+1$ $|$ in state $q_i$ at time $t]$ 
\item $B$ = output probabilites = $\{b_j(k)\}$, \\
       where $b_j(k)$ = $Pr[$producing $v_k$ at time $t$ $|$ in state $q_j$ at time $t]$
\end{description}
%  \end{eqnarray}

\section{Example of a Hidden Markov Model}

\begin{figure}[htb]
\begin{center}
\input{lecture25-fig-1.tex}
\caption{A Hidden Markov Model}
\label{fig:markov}
\end{center}
\end{figure}

If we let $\pi(q_1) = 1$ and $\pi(q_2) = 0$ then the following is a example
of a possible transition sequence and output sequence for the Hidden Markov Model in figure \ref{fig:markov}:

{\begin{tabular}{cccccccccccccc}
 &$0.8$&&$0.8$&&$0.2$&&$0.6$&&$0.4$&&$0.8$&    \\
 $q_1$&$\longrightarrow$&$q_1$&$\longrightarrow$&$q_1$&$\longrightarrow$
 &$q_2$&$\longrightarrow$&$q_2$&$\longrightarrow$&$q_1$&$\longrightarrow$
 &$q_1$ \\
 $\Downarrow$&&$\Downarrow$&&$\Downarrow$&&$\Downarrow$&&$\Downarrow$&&
 $\Downarrow$&&$\Downarrow$ \\
 H&&H&&T&&T&&T&&T&&H \\
\end{tabular}} 

We can easily calculate probabilities for the following events.
\begin{enumerate}
\item The probability of the above state transition sequence:
\[ Pr[q_1 q_1 q_1 q_2 q_2 q_1 q_1] = \pi(q_1) a_{11} a_{11} a_{12} a_{22} a_{21} a_{11} \approx 0.025 \]
\item The probability of the above output sequence given the above transition sequence:
\[ Pr[(H H T T T T H) | (q_1 q_1 q_1 q_2 q_2 q_1 q_1)] = \frac{2}{3} \frac{2}{3} \frac{1}{3} \frac{5}{6} \frac{5}{6} \frac{1}{3} \frac{2}{3} \approx 0.23 \] 
\item The probability of the above output sequence and the above transition sequence:
\[ Pr[(H H T T T T H) \wedge (q_1 q_1 q_1 q_2 q_2 q_1 q_1)] \approx (0.025) (0.023) \approx 5.7 x 10^{-4} \]
\end{enumerate}

\section{Using Hidden Markov Models to generate speech}

Markov models can be used to generate speech by making the output
symbols phonemes of speech. A Hidden Markov Model might then capture
the pronounciation of a word. By changing the structure and transition
probabilies in the Markov model one can modify the legal
pronounciation of a word.

\section{Using Markov Models to recognize words}
 
Suppose one has a set of Hidden Markov Models for a set of words. Then
given a speech signal one could ask which model was the most likely to
have generated that signal. We need to be able to calculate the
probability of a given output sequence for a Hidden Markov Model. So,
given a output sequence $O_1 O_2 ... O_T$ we have the following
equation:

\[ Pr[O_1 O_2 ... O_T] = \sum_{i_1, i_2, ..., i_T} Pr[i_1, i_2, ..., i_T] Pr[O_1 O_2 ... O_T | i_1, i_2, ..., i_T] \]

This is simply the probability of a given state transition sequence
times the probability of the output sequence given that transition
sequence, summed over all possible state transtions. We can use dynamic
programming to calculate this probability efficiently.

\begin{definition}
Let $\alpha_t(i)$ = $Pr[(O_1 O_2 ... O_t) \wedge $($i_t = q_i)]$
\end{definition}

Then we can use the following formulas to calculate $\alpha_t(i)$ where $N$ is the number of states in the Hidden Markov Model.

\begin{eqnarray*}
 \alpha_1(i) &= &\pi(i) b_i(O_1) \\
 \alpha_{t+1}(i) &= &\left(\sum_{j=1}^{N}\alpha_t(j) a_{ji} \right) b_i(O_{t+1}) \\
\end{eqnarray*}

From these equations it should be clear that $\alpha_T(i)$ can be
computed in $O(N^2 T)$ time. Furthermore since $\alpha_T(i)$ is the
probability that we get the correct output and end in state $q_i$, the
probability that we get the correct output is simply the sum of the
$\alpha_T(i)$'s. Namely,

\[ Pr[O_1 O_2 ... O_T] = \sum_{i=1}^N \alpha_T(i) \]

Hence we can calculate $Pr[O_1 O_2 ... O_T]$ in $O(N^2 T)$ time.

\section{Using Markov Models to recognize sentences}

Now suppose that we not only have models for recognizing words but we
have a model that tells us how words should be put together. Namely we
have encoded a gramatical model of speech into our Hidden Markov
Model. In this case we may ask two questions:
\begin{enumerate}
\item For each $t$, what state is most likely? (Note that the sequence of states computed by this criterion might be impossible.) 
\item What single sequence of states has the largest posterior probability?
\end{enumerate}

To answer the first question we define $\beta_t(i)$, which is
essentially the time reversal of $\alpha_t(i)$, and we define
$\gamma_t(i)$, which is the probability of being in state $q_i$ at
time $t$ given the observation sequence.

\begin{definition}
Let $\beta_t(i)$ = $Pr[(O_{t+1}O_{t+2} ... O_T) | ${\rm (at time $t$ in state} $q_i)]$
\end{definition}

\begin{definition}
Let $\gamma_t(i)$ = $Pr[i_t = q_i | O_1, O_2, \ldots, O_{T}]$
\end{definition}

Then we have the following formulas for $\beta_t(i)$ and $\gamma_t(i)$:
\begin{eqnarray*}
 \beta_T(i) &= &1 \\
 \beta_t(i) &= & \sum_{j=1}^{N}\beta_{t+1}(j) a_{ij}  b_j(O_{t+1}) \\
 \gamma_t(i) &= &\frac{\alpha_t(i) \beta_t(i)}{\sum_{j=1}^{N}\alpha_t(j) \beta_t(j)} 
\end{eqnarray*}

Now we have the answer to question 1: for time $t$, the state
$q_i$ which maximizes $\gamma_t(i)$ is the state with highest
probability.  To answer question 2, we make one further definition.

\begin{definition}
Let $\delta_t(i)$ = $Pr[${\rm that the most likely single sequence of states ends with state} $q_i${\rm , given observed output}$]$
\end{definition}

And the equations for $\delta_t(i)$ are
\begin{eqnarray*}
 \delta_1(i) &= &\pi(i) b_i(O_1) \\
 \delta_{t+1}(i) &= &\left( \max_{j \in [1, N]}\delta_t(j) a_{ji} \right) b_i(O_{t+1}) \\
\end{eqnarray*}
Backtracking the calculation $\delta_T(i)$ will allow you to calculate the answer to question 2. This method is called the Viterbi Algorithm.

\section{Learning Hidden Markov Models}

We have shown that given a Hidden Markov Model we can 
\begin{enumerate}
\item produce output
\item see which sequence of states produces the best fit
\end{enumerate}

Now we need to learn how to build a model. We have to find a method
for calculating $a_{ij}$, $b_i$, and $\pi(i)$. The algorithm that we
present is similiar to the back-propagation algorithm of neural nets in
that it is susceptible to local minima and needs lots of data to
converge. It does, however, have the advantage that in every step of
the algorithm it improves (or at least keeps constant) its accuracy on
the test data.

The Baum/Welch training algorithm works as follows: It is given $O_1 O_{2}
... O_T$ and initial estimates for parameters $a_{ij}$, $b_i$, and $\pi(i)$. We
define new parameters $\bar{a}_{ij}$, $\bar{b}_i$, and $\bar{\pi}(i)$.
First we define $\Upsilon_t(i, j)$, the estimator of the probabilty of a
transition from $q_i$ to $q_j$ at time $t$ given output $O_1 O_{2}
... O_T$.

\[ \Upsilon_t(i,j) = \frac{\alpha_t(i) a_{ij} b_j(O_{t+1}) \beta_{t+1}(j)}
			  {Pr[O_1 O_{2} ... O_T]} \]

then $\bar{\pi}(i)$, $\bar{a}_{ij}$, and $\bar{b}_i$ are calculated as:
\begin{eqnarray*}
 \bar{\pi}(i) & = & \gamma_1(i) \\
 \bar{a}_{ij} & = & \frac{\sum_{t=1}^{T-1} \Upsilon_t(i,j)}
		      {\sum_{t=1}^{T-1} \gamma_t(i)} \\
              & = & {\frac {{\rm expected\; number\; of\; transitions\;
from\; state\;} q_i {\rm  to\;} q_j}
	               { {\rm expected\; number\; of\; transitions\; out\;
of\; state\;} q_i }}
		\\
 \bar{b}_j(k)    & = & \frac{\sum_{t=1; O_t = v_k}^{T} \gamma_t(j)}
		       {\sum_{t=1}^{T} \gamma_t(j)} \\
	         & = & \frac{ {\rm expected\; number\; of\; times\;  in\; state\;} 
			       q_j {\rm \; with\; observed\; output\;} v_k }
		        { {\rm expected\; number\; of\; times\; of\;
being\; in\;  state\;} q_j }
		\\
\end{eqnarray*}

These values $\bar{\pi}(i)$, $\bar{a}_{ij}$, and $\bar{b}_i$ are used
for our estimates of $a_{ij}$, $b_i$, and $\pi(i)$ for the next
iteration. Using this algorithm we can learn Hidden Markov Models.
Baum and Welch showed that every iteration of this algorithm will not
make the parameters worse. However, the algorithm must still be
given reasonable initial parameters. Furthermore, the algorithm says
nothing about how many states our Hidden Markov Model should have or
how they should be connected.

\begin{thebibliography}{99}
\bibitem{Lee} Kai-Fu Lee. Automatic peec recognition. 
In {\it Computer Science Research Review}, pages 7-16. School of
Computer Science, Carnegie Mellon University, 1989.

\bibitem{levin} S. E. Levinson, L. R. Rabiner, and M. M. Sondhi. An 
introduction to the application of the theory of probabilistic
functions of Markov process to automatic speech recognition. In {\it Bell System Technical Journal}, 62(4):1035-1074, April 1983.

\bibitem{rabin} L. R. Rabiner and B. H. Juang. An introduction to hidden markov models. {\it IEEE ASSP Magzine}, 3(1):4-16, January 1986.

\end{thebibliography}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Your notes end here!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}

