\documentclass[11pt,twoside]{article}

\usepackage{macros-course-lectures}

\usepackage{graphics}
\usepackage[pdflatex]{graphicx}

\newenvironment{verify}{\par{\noindent\em Verification.}\hspace{1em}\setlength{\parindent}{3ex}}{\qed}

\newtheorem{guess}{Guess}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{method}{Method}

\begin{document}

\lecture{notes12}{Lecture Notes}{14 Mar 00}

\begin{outline}

This lecture is about solving recurrences and recurrent problems.
Needless to say, recurrent problems come up again and again.  In
particular, recurrences often arise in the analysis of recursive
algorithms.

\section{The Towers of Hanoi}

In the Towers of Hanoi problem, there are three posts and seven disks
of different sizes.  Each disk has a hole through the center so that
it fits on a post.  At the start, all seven disks are on one post as
shown in Figure~\ref{fig:hanoi}.  The disks are arranged by size so
that the smallest is on top and the largest is on the bottom.  The
goal is to end up with all seven disks in the same order, but on a
different post.  This is not trivial because of two restrictions.
First, the only permitted action is removing the top disk from a post
and dropping it onto another post.  Second, a larger disk can never
lie above a smaller disk on any post.  (These rules imply, for
example, that it is not fair to pick up the whole stack of disks at
once and then to drop them all on another post!)

\end{outline}

\begin{figure}[htbp]
\centerline{
%\psfig{figure=figures/hanoi.eps,height=1.5in}
\includegraphics[height=1.5in]{figures00/hanoi}
}
\caption{\em This is the starting position in the Towers of Hanoi
problem.  All seven disks are stacked on the leftmost post.  The goal
is to move all the disks to a different post.  At each step the top
disk on a post can be removed and dropped onto another post.  At no
time, however, can a larger disk lie above a smaller disk.}
\label{fig:hanoi}
\end{figure}

\begin{outline}
It is not immediately clear that a solution to this problem exists;
maybe the rules are so stringent that the disks cannot all be moved
to another post!

% We will represent the state of the Towers of Hanoi using diagrams like
% the one below.  The picture on the left is the initial state of the
% Towers as shown in Figure~\ref{fig:hanoi}.  In the picture on the
% right, all of the disks are on the middle post.  This is one of the
% two possible final states.  (In the other permissible final state, all
% disks are on the rightmost post.)
%
% \[
% \begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
% & 1 & & \\
% & 2 & & \\
% & 3 & & \\
% & 4 & & \\
% & 5 & & \\
% & 6 & & \\
% & 7 & & \\ \hline \hline
% \end{array}
% \Rightarrow
% \begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
% & & 1 & \\
% & & 2 & \\
% & & 3 & \\
% & & 4 & \\
% & & 5 & \\
% & & 6 & \\
% & & 7 & \\ \hline \hline
% \end{array}
% \]

One approach to this problem is to consider a simpler variant with
only three disks.  We can quickly exhaust the possibilities of this
simpler puzzle and find a 7-move solution like the one shown below.
(The disks on each post are indicated by the numbers immediately to
the right.  Larger numbers correspond to larger disks.)
\end{outline}

\begin{eqnarray*}
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
& 1 & & \\
& 2 & & \\
& 3 & & \\ \hline \hline
\end{array}
& \Rightarrow & 
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   &   & \\
& 2 &   & \\
& 3 & 1 & \\ \hline \hline
\end{array}
\Rightarrow
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   &   &   \\
&   &   &   \\
& 3 & 1 & 2 \\ \hline \hline
\end{array} \\
& \Rightarrow &
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   &   &   \\
&   &   & 1 \\
& 3 &   & 2 \\ \hline \hline
\end{array}
\Rightarrow
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   &   &   \\
&   &   & 1 \\
&   & 3 & 2 \\ \hline \hline
\end{array} \\
& \Rightarrow &
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   &   &   \\
&   &   &   \\
& 1 & 3 & 2 \\ \hline \hline
\end{array}
\Rightarrow
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   &   &   \\
&   & 2 &   \\
& 1 & 3 &   \\ \hline \hline
\end{array} \\
& \Rightarrow &
\begin{array}{p{0.4cm}|p{0.75cm}|p{0.75cm}|p{0.4cm}}
&   & 1 &   \\
&   & 2 &   \\
&   & 3 &   \\ \hline \hline
\end{array}
\end{eqnarray*}

\begin{outline}
This problem was invented in 1883 by the French mathematician Edouard
Lucas.  In his original account, there were 64 disks of solid gold.
At the beginning of time, all 64 were placed on a single post, and
monks were assigned the task of moving them to another post according
to the rules described above.  According to legend, when the monks
complete their task, the Tower will crumble and the world will end!

The questions we must answer are, ``Given sufficient time, can the
monks succeed?'' and if so, ``How long until the world ends?'' and,
most importantly, ``Will this happen before the 6.042 final?''

\subsection{Finding a Recurrence}

The Towers of Hanoi problem can be solved in a recurrent fashion as
follows.  Let $T_n$ be the minimum number of steps needed to move an
$n$-disk tower.  For example, it is not hard to see that $T_1 = 1$ and
$T_2 = 3$.  For 3 disks, the solution given above shows that $T_3 \leq
7$.  We can generalize the approach used for 3 disks to the following
recursive algorithm for $n$ disks.

{\bf Step 1.  } Move the top $n-1$ disks from the first post to the
third post.  This can be done recursively in $T_{n-1}$ steps.

{\bf Step 2.  }  Move the largest disk from the first post to the to
the second post.  This requires just 1 step.

{\bf Step 3.  } Move the $n-1$ disks from the third post onto the
second post.  Again, $T_{n-1}$ steps are required.

This algorithm shows that $T_n$, the number of steps required for $n$
disks, is at most $T_n \leq 2 T_{n-1} + 1$.  We can now compute upper
bounds on the number of steps required for various numbers of disks.

\begin{eqnarray*}
T_3 & \leq & 2 \cdot T_2 + 1 \\
    & = & 7 \\
T_4 & \leq & 2 \cdot T_3 + 1 \\
    & \leq & 15 \\
\end{eqnarray*}

The algorithm answers our first question; given sufficient time, the
monks can finish their task and end the world.

\subsection{A Lower Bound}

We can not yet compute the exact number of steps that the monks need
to move the 64 disks; we can only show an upper bound.  Perhaps---
having pondered the problem since the beginning of time--- the monks
have devised a better algorithm.

In fact, there is no better algorithm, and here is why.  At some step,
we must move the $n$-th disk from the first post to a different post.
For this to happen, the $n-1$ smaller disks must all be stacked out of
the way on the only remaining post.  Arranging the $n-1$ smaller disks
this way requires at least $T_{n-1}$ moves.  After the largest disk is
moved, at least another $T_{n-1}$ moves are required to pile the $n-1$
smaller disks on top.

This argument shows that the number of steps required is at least $2
T_{n-1} + 1$.  Since we gave an algorithm using exactly this number of
steps, we now have a recurrence for $T_n$, the number of moves
required to complete the Tower of Hanoi problem with $n$ disks:

\[
T_n = 2 T_{n-1} + 1
\]

Knowing the $T_1 = 1$, we can use this recurrence to conclude that
$T_2 = 3, T_3 = 7, T_4 = 15, \ldots$.

\subsection{Guess and Verify}

Computing $T_{64}$ from the recurrence would require a lot of work.
It would be nice to have a closed form expression for $T_n$, so that
we could quickly compute the number of steps required to solve the
Towers of Hanoi problem for any given number of disks.  (For example,
we might want to know how much faster moving the disks would be if the
monks donate one of the golden disks to their favorite political party
as ``soft money''.)

There are several different methods for solving recurrences.  The
simplest method is to guess the solution and then to verify that the
guess is correct, usually with an induction proof.  This method is
called ``Guess and Verify'' or ``substitution''.  To help us make an
intelligent guess, we can tabulate $T_n$ for small values of $n$:

\[
\begin{array}{c|c}
n & T_n \\ \hline
1 & 1 \\
2 & 3 \\
3 & 7 \\
4 & 15 \\
5 & 31 \\
\end{array}
\]

From the table, a natural guess is that $T_n = 2^n - 1$.  Whenever you
guess a solution to a recurrence, you should always verify it with a
proof by induction or by some other technique; after all, your guess
might be wrong.  (But why bother to verify in this case?  After all,
if we're wrong, its not the end of the... no, let's check.)

\begin{guess*}
$T_n = 2^n - 1$
\end{guess*}

\begin{verify}
The proof is by induction on $n$.  Let $P(n)$ be the predicate that
$T_n = 2^n - 1$.  In the base case, $P(1)$ is true because $T_1 = 1 =
2^1 - 1$.  In the inductive step, assume $T_n = 2^n - 1$ to prove that
$T_{n+1} = 2^{n+1} - 1$.

\begin{eqnarray*}
T_{n+1} & = & 2 T_n + 1 \\
        & = & 2 (2^n - 1) + 1 \\
        & = & 2^{n+1} - 1
\end{eqnarray*}

The first equality holds by assumption, and the second follows by
application of the induction hypothesis.  The final step is just a
simplification.
\end{verify}

Our guess is now verified.  This shows, for example, that the 7-disk
puzzle will require $2^7 - 1 = 127$ moves to complete.  We can also
now resolve our remaining questions about the 64-disk puzzle.  Since
$T_{64} = 2^{64} - 1$, the monks must complete more than 18 billion
billion steps before the world ends.  Better study for the final.

\subsection{The Plug and Chug Method}

In general, Guess and Verify is the best way to solve a recurrence.
The only problem with the method is guessing the right solution.  In
the Towers of Hanoi example, guessing the solution was easy, but
sometimes the solution has a very strange form that is quite hard to
guess.  Practice helps, of course, but so can some other methods.

Plug and Chug is one such alternative method for solving recurrences.
Plug and Chug is also sometimes called ``expansion'', ``iteration'',
or ``brute force''.  The method consists of four calculation-intensive
steps.  These steps are listed for reference, but will only make real
sense after we work through the Towers of Hanoi example below.

\begin{enumerate}

\item Expand the recurrence equation by alternately ``plugging''
(applying the recurrence equation) and ``chugging'' (simplifying the
resulting expression).

\item Identify a pattern for the recurrence equation after $i$
rounds of plug and chug.  Verify that this pattern is correct by
carrying out one additional round of plug and chug.

\item Substitute a value of $i$ into the pattern so that $T_n$
is expressed as a function of just the first few terms (e.g. $T_1,
T_2, \ldots$).  Substitute computed values for these terms to obtain
an ordinary, non-recurrent expression for $T_n$.

\item Find a closed form for $T_n$ from the ordinary
expression.
\end{enumerate}

The Plug and Chug calculations for the Towers of Hanoi recurrence are
worked out below.

\subsubsection*{Step 1: Plug and Chug}

\begin{eqnarray*}
T_n & = & 1 + 2 T_{n-1} \\
(plug) \quad & = & 1 + 2 (1 + 2 T_{n-2}) \\
(chug) \quad & = & 1 + 2 + 4 T_{n-2}) \\
(plug) \quad & = & 1 + 2 + 4 (1 + 2 T_{n-3}) \\
(chug) \quad & = & 1 + 2 + 4 + 8 T_{n-3} \\
(plug) \quad & = & 1 + 2 + 4 + 8 (1 + 2 T_{n-4}) \\
(chug) \quad & = & 1 + 2 + 4 + 8 + 16 T_{n-4}
\end{eqnarray*}

The first equation is the recurrence that we want to solve.  In the
second step, we transform the original recurrence by {\em plugging};
that is, we apply the recurrence equation to make $T_n$ depend on
earlier terms.  In this case, we eliminate $T_{n-1}$ and make $T_n$
depend on $T_{n-2}$ instead.  Next, we {\em chug}; that is, we
simplify the new expression for $T_n$.  Then we plug again so that
$T_n$ depends on a still earlier term, $T_{n-3}$.  Then we chug and
plug and chug some more.

\subsubsection*{Step 2: Identify and Verify a Pattern}

\begin{eqnarray*}
T_n          & = & 1 + 2 + 4 + \ldots + 2^{i-1} + 2^i T(n-i) \\
(plug) \quad & = & 1 + 2 + 4 + \ldots + 2^{i-1} + 2^i (1 + 2 T_{n-(i+1)}) \\
(chug) \quad & = & 1 + 2 + 4 + \ldots + 2^{i-1} + 2^i + 2^{i+1} T_{n-(i+1)})
\end{eqnarray*}

After enough work, a pattern becomes clear: $T_n$ is always a sum of
consecutive powers of two together with an earlier $T$ term.

The next step is to describe this pattern by writing down the
expression obtained after $i$ rounds of plug and chug.  Then we do one
last round to confirm that the pattern is correct.  This check is very
similar to the inductive step in a proof by induction.

Be careful in the ``chug'' stage; too much simplification can obscure
an emerging pattern.  For example, summing $1 + 2 + 4 + \ldots$ at
every stage would have concealed the geometric series.  The rule to
remember--- indeed, a rule applicable to the whole of college life---
is {\em chug in moderation}.

\subsubsection*{Step 3: Express $n$-th Term Using Early Terms}

\begin{eqnarray*}
T_n & = & 1 + 2 + 4 + \ldots + 2^{i-1} + 2^i T(n-i) \\
T_n & = & 1 + 2 + 4 + \ldots + 2^{n-2} + 2^{n-1} T(1) \\
    & = & 1 + 2 + 4 + \ldots + 2^{n-2} + 2^{n-1}
\end{eqnarray*}

Now we can use the pattern to express $T_n$ as a function of small,
computable terms.  In particular, if we set $i = n - 1$ in the above
pattern, then $T_n$ becomes a function only of $T_1$, which we know is
equal to 1.  This leaves an ordinary, non-recurrent expression for
$T_n$.

\subsubsection*{Step 4: Find a Closed Form for the $n$-th Term}

All that remains is to reduce the ordinary expression for $T_n$ to a
closed form.  We are fortunate in this case, because $T_n$ is the sum
of a geometric series.  We learned how to tackle these last week!

\begin{eqnarray*}
T_n & = & 1 + 2 + 4 + 8 + \ldots + 2^{n-2} + 2^{n-1} \\
    & = & \sum_{i=0}^{n-1} 2^i \\
    & = & 2^n - 1
\end{eqnarray*}

We are done!  When using the Plug and Chug method, be sure to go back
and verify that the answer is correct.  It is very easy to make a
mistake when observing the general pattern.


\section{Graduate Student Job Prospects}

In a certain new academic field (say computer science), there are only
so many faculty positions available in all the universities of the
world.  Initially, there were not enough qualified candidates, and
many positions were unfilled.  But, over time, new graduates are
filling the positions, making job prospects for later computer science
students ever more bleak.  Worse, the increasing number of professors
are able to train an increasing number of graduate students, causing
positions to fill ever more rapidly.  Eventually, the universities
will be saturated; new computer science graduates will have no chance
at an academic career.  Our problem is to determine when the
universities will stop hiring new computer science faculty and, in
particular, to answer the question, ``Are the 6.042 TAs doomed?''
Here are the details of the problem.

\begin{itemize}
\item There are a total of $N$ faculty positions available worldwide.  This
number never changes due to budgetary constraints.

\item Congress has passed a law forbidding forced retirement, and no one
will retire voluntarily.  (This is true and a problem!)  In our
analysis, therefore, once a faculty position is filled, it never opens
up again.

\item Each year, every professor trains exactly 1 student who will
go on to become a professor the following year.  The only exception is
that first year professors do not train students; they are too busy
publishing, teaching, getting grants, and serving on committees.

\item In year 0, there are no computer science professors in the world.  In
year 1, the first professor is hired.
\end{itemize}

\subsection{Finding a Recurrence}

Ideally, we could find a formula for the number of professors in the
world in a given year.  Then we could determine the year in which all
$N$ faculty positions are filled.  Let $f(n)$ be the number of
professors during year $n$.  To develop some intuition about the
problem, we can compute values of this function for small $n$ by hand.
\end{outline}

\begin{eqnarray*}
f(0)=0 &\quad& \mbox{No CS professors; the dark ages.} \\
f(1)=1 &\quad& \mbox{1 new professor; too busy to train a student.} \\
f(2)=1 &\quad& \mbox{1 old professor; now training a student.} \\
f(3)=2 &\quad& \mbox{1 new prof, 1 old prof; new prof too busy, old prof training.} \\
f(4)=3 &\quad& \mbox{1 new prof, 2 old profs; new prof too busy, both old profs training} \\
f(5)=5 &\quad& \mbox{2 new profs, 3 old profs} \\ 
f(6)=8 &\quad& \mbox{3 new profs, 5 old profs} \\ 
\end{eqnarray*}

\begin{outline}

In general, the number of professors in year $n$ is equal to the
number of professors last year plus the number of new hires.  The
number of professors last year is $f(n-1)$.  The number of new hires
is equal to the number of professors two years ago, $f(n-2)$, since
each of these professors trained a student last year.  These
observations give the following recurrence equation for the number of
professors:

\begin{eqnarray*}
f(n) & = & f(n-1) + f(n-2) \quad \quad (n \geq 2)
\end{eqnarray*}

This is the familiar Fibonacci recurrence.  Looking back, the values
of $f(n)$ that we computed by hand are indeed the first few Fibonacci
numbers.  Fibonacci numbers arise in all sorts of applications.  So
how big is $f(n)$ anyway?  Of course, we could compute as many
Fibonacci numbers as we like using the recurrence, but it would be
much nicer to find a closed form.

{\bf Note:} When Fibonacci introduced this recurrence, the academic
environment was a little bit different.  So he used rabbits instead.
A pair of rabbits takes one year to mature, then starts producing one
pair of rabbits as offspring forever.

\subsection{Solving the Recurrence}

Solving the Fibonacci recurrence is easy because the recurrence is
linear.  (Well, ``easy'' in the sense that you can learn the technique
in one lecture; discovering it actually took six centuries.)

\begin{defin*}
A recurrence is {\em linear} if it is of the form

\begin{eqnarray*}
f(n)    & = & a_1 f(n-1) + a_2 f(n-2) + \ldots + a_d f(n-d) \\
        & = & \sum_{i=1}^d a_i f(n-i)
\end{eqnarray*}

where $a_1, a_2, \ldots a_d$ are constants.  The {\em order} of the
recurrence is $d$.
\end{defin*}

For example, the Fibonacci recurrence is order 2 and has $a_1 = a_2 =
1$.

For now, let's try to solve just the Fibonacci recurrence; we'll see
how to solve general linear recurrences later in the lecture.  Our
rule of thumb is that Guess and Verify is the first method to apply to
a recurrence.  It turns out that for a linear recurrence, an
exponential solution is a good guess.  However, since we know nothing
beyond this, our initial Guess and Verify attempt will really only be
a ``dry run''; that is, we will not make an exact guess and will not
verify it with a complete proof.  Rather, the goal of this first
attempt is only to clarify the form of the solution.

\begin{guess*}
$f(n) = c \alpha^n$
\end{guess*}

Here $c$ and $\alpha$ are parameters introduced to improve our odds of
having a correct guess; in the verification step, we can pick values
that make the proof work.  To further improve our odds, for now let's
neglect the boundary conditions, $f(0) = 0$ and $f(1) = 1$.

\begin{verify} {\em (dry run) }
Plugging our guess into the recurrence $f(n) = f(n-1) + f(n-2)$ gives:

\begin{eqnarray*}
c \alpha^n & = & c \alpha^{n-1} + c \alpha^{n-2} \\
\alpha^2 & = & \alpha  + 1 \\
\alpha^2 - \alpha  - 1 & = & 0\\
\alpha & = & \frac{1 \pm \sqrt{5}}{2}
\end{eqnarray*}

In the first step, both sides of the equation are divided by $c
\alpha^{n-2}$.  The second step only involves rearranging terms.  In
the final step, we can compute $\alpha$ by solving the quadratic
equation.

This calculation suggests that $c$ can be anything, but that $\alpha$
must be $(1 \pm \sqrt{5})/2$.  Evidently, there are two solutions to
the recurrence:

\[
f(n) = c \left( \frac{1 + \sqrt{5}}{2} \right)^n
\quad \quad \mbox{or} \quad \quad
f(n) = c \left( \frac{1 - \sqrt{5}}{2} \right)^n
\]
\end{verify}

There are actually more solutions than suggested by our ``dry run'' of
Guess and Verify; in fact, any linear combination of the above two
solutions is also a solution.  The following theorem states that this
is true in general for linear recurrences.

\begin{theorem}
If $f(n) = \alpha_1^n$ and $f(n) = \alpha_2^n$ are two solutions to a
linear recurrence (without boundary conditions), then $f(n) = c_1
a_1^n + c_2 a_2^n$ is also a solution for all constants $c_1$ and
$c_2$.
\end{theorem}

\begin{proof}
Every linear recurrence has the following form:

\[
f(n) = \sum_{i=1}^d a_i f(n-i)
\]

We must prove that $f(n) = c_1 \alpha_1^n + c_2 \alpha_2^n$ is a
solution; that is, we must prove:

\[
c_1 \alpha_1^n + c_2 \alpha_2^n =
\sum_{i=1}^d a_i (c_1 \alpha_1^{n-i} + c_2 \alpha_2^{n-i})
\]

Substituting the two solutions $f(n) = \alpha_1^n$ and $f(n) =
\alpha_2^n$ into the recurrence gives the following two equations:

\begin{eqnarray*}
\alpha_1^n = \sum_{i=1}^d a_i \alpha_1^{n-i} \\
\alpha_2^n = \sum_{i=1}^d a_i \alpha_2^{n-i} \\
\end{eqnarray*}

Multiplying the first equation by $c_1$, multiplying the second
equation by $c_2$, and adding the two gives the desired result.
\end{proof}

The theorem implies that for all $c_1$ and $c_2$, the following is a
solution to the Fibonacci recurrence without boundary conditions:

\[
f(n) = c_1 \left( \frac{1 + \sqrt{5}}{2} \right)^n +
       c_2 \left( \frac{1 - \sqrt{5}}{2} \right)^n
\]

All that remains is to find a solution that satisfies the boundary
conditions $f(0) = 0$ and $f(1) = 1$.  The first boundary condition
requires that:

\begin{eqnarray*}
& & f(0) = c_1 \left( \frac{1 + \sqrt{5}}{2} \right)^0 +
       c_2 \left( \frac{1 - \sqrt{5}}{2} \right)^0 = 0 \\
& \Rightarrow & c_1 + c_2 = 0
\end{eqnarray*}

Similarly, the second boundary condition requires that:

\begin{eqnarray*}
& & f(1) = c_1 \left( \frac{1 + \sqrt{5}}{2} \right)^1 +
       c_2 \left( \frac{1 - \sqrt{5}}{2} \right)^1 = 1 \\
& \Rightarrow & c_1 \left( \frac{1 + \sqrt{5}}{2} \right) +
       c_2 \left( \frac{1 - \sqrt{5}}{2} \right) = 1
\end{eqnarray*}

We now have two linear equations in two unknowns!  The system of
equations is not degenerate, so there is a unique solution: $c_1 =
1/\sqrt{5}$ and $c_2 = -1/\sqrt{5}$.  We now have a complete solution to
the Fibonacci recurrence {\em with} boundary conditions:

\[
f(n) = \frac{1}{\sqrt{5}} \left( \frac{1 + \sqrt{5}}{2} \right)^n -
       \frac{1}{\sqrt{5}} \left( \frac{1 - \sqrt{5}}{2} \right)^n
\]

This looks completely wrong!  All Fibonacci numbers are integers, but
this expression is full of square roots of five!  Amazingly, however,
the square roots always cancel out.  This expression really does give
the Fibonacci numbers if we plug in $n = 0, 1, 2, \ldots$.  If one
were required to guess the solution to the Fibonacci recurrence, then
it is easy to imagine taking at least six centuries to come up with
this!

\subsection{Job Prospects}

Let's return to the original question: how long until all $N$ faculty
positions are taken?

To answer this question, we must find the smallest $n$ such that $f(n)
\geq N$.  This will imply that in year $n$, there are as many professors
as university positions.  Therefore, graduates after year $n$ will
have to find other employment, e.g. shuffling golden disks in an
obscure monastic community for the next $10^{19}$ years.

Because $f(n)$ has such a complicated form, it is hard to compute the
right value of $n$ exactly.  However, we can make a good
approximation.  Notice that $f(n)$ is the sum of two terms.  The key
observation is that the second term tends to zero as $n$ grows large,
since $|\frac{1 - \sqrt{5}}{2}| = 0.618\ldots < 1$.  This means that we can
rewrite $f(n)$ with asymptotic notation as follows:

\[
f(n) = \frac{1}{\sqrt{5}} \left( \frac{1 + \sqrt{5}}{2} \right)^n + o(1)
\]

Using this approximation for $f(n)$, we can determine the approximate
year in which all faculty positions will be taken as follows:

\begin{eqnarray*}
& & f(n) \geq N \\
& \Leftrightarrow & \frac{1}{\sqrt{5}} \left( \frac{1 + \sqrt{5}}{2} \right)^n + o(1) \geq N \\
& \Leftrightarrow & n \ge \frac{\log(\sqrt{5} \ N + o(1))}{\log(\frac{1 + \sqrt{5}}{2})}
\end{eqnarray*}

This implies that all the faculty jobs will be filled in $\Theta(\log
N)$ years.  This makes sense, since the number of professors is
increasing exponentially.  For example, if there are $N = 10000$ jobs,
then all will be taken in $n = 20.8$ years.  Now, as to the fate of
the TAs, there are fewer than ten thousand faculty jobs in computer
science and hiring started about 20 years ago...

The solution to the Fibonacci recurrence has an interesting corollary:
for large $n$, $\frac{1}{\sqrt{5}} \left(\frac{1 +
\sqrt{5}}{2} \right)^n$ is very close to an integer--- namely, the
$n$-th Fibonacci number.  This expression is the first term in the
closed form for the $n$-th Fibonacci number.  Since the other term of
the closed form goes to zero exponentially fast and Fibonacci numbers
are always integers, the first term must be very close to an integer!
This is one of many strange facts about the ``Golden Ratio'', $\frac{1
+ \sqrt{5}}{2} = 1.61803\ldots$.

\section{General Linear Recurrences}

The method we used to solve the Fibonacci recurrence can be used to
solve any linear recurrence!  Recall that every linear recurrence has
the following form:

\[
f(n) = \sum_{i=1}^d a_i f(n-i)
\]

Substituting the guess $f(n) = \alpha^n$ into the recurrence gives:

\begin{eqnarray*}
\alpha^n & = & a_1 \alpha^{n-1} + a_2 \alpha^{n-2} + \ldots + a_d \alpha^{n-d} \\
\alpha^d & = & a_1 \alpha^{d-1} + a_2 \alpha^{d-2} + \ldots + a_{d-1} \alpha + a_d \\
\alpha^d - a_1 \alpha^{d-1} - a_2 \alpha^{d-2} - \ldots - a_{d-1} \alpha - a_d & = & 0\\
\end{eqnarray*}

The equation is divided by $\alpha^{n-d}$ in the first step, and terms
are rearranged in the second step.  The last equation is called the
{\em characteristic equation} of the recurrence.  The characteristic
equation of a recurrence can be read of very quickly; the coefficients
of the equation are the same as the coefficients of the recurrence.
In general, the characteristic equation will have $d$ roots, where $d$
is the order of the recurrence.  The solution to the recurrence
depends on whether all roots of the characteristic equation are
distinct.

\subsection{Simple Case:  Distinct Roots}

If the characteristic equation has $d$ distinct roots $r_1, r_2,
\ldots, r_d$, then the solution to the recurrence has the form:

\[
f(n) = c_1 r_1^n + c_2 r_2^n + \ldots + c_d r_d^n
\]

Note that the roots of the characteristic equation need not even be
real!  In the end, imaginary parts will cancel just as the
$\sqrt{5}$'s canceled in the solution to the Fibonacci recurrence.

The constants $c_1, c_2, \ldots, c_d$ depend on the boundary
conditions.  Each boundary condition generates a linear equation in
these constants.  For example, the boundary conditions $f(0) = b_0$,
$f(1) = b_1$, $\ldots$, $f(d-1) = b_{d-1}$ generate the linear
equations:

\begin{eqnarray*}
b_0 & = & c_1 + c_2 + \ldots + c_d \\
b_1 & = & c_1 r_1 + c_2 r_2 + \ldots + c_d r_d \\
\ldots & \ldots & \ldots \\
b_{d-1} & = & c_1 r_1^{d-1} + c_2 r_2^{d-1} + \ldots + c_d r_d^{d-1}
\end{eqnarray*}

If there are $d$ boundary conditions, then we obtain $d$ linear
equations in $d$ unknowns.  A neat linear algebra fact is that a
system of equations of this particular type is never degenerate.  This
means that for every set of constants $b_0, b_1, \ldots b_{d-1}$ in
the boundary conditions, we can find a unique set of constants $c_0,
c_1, \ldots c_{d-1}$ that give a solution to the recurrence and
satisfy the boundary conditions.

\subsection{Tricky Case:  Repeated Roots}

If the characteristic equation of a recurrence has a repeated root,
then the solution is slightly more complicated.  The main tool is the
following theorem, which we will not prove.

\begin{theorem}
If $r$ is a root of the characteristic equation of a recurrence with
multiplicity $m$, then all of $r^n, n r^n, n^2 r^n, \ldots n^{m-1}
r^n$ are solutions to the recurrence.
\end{theorem}

For example, if the characteristic equation of a recurrence has root
$r_1$ with multiplicity 2, root $r_2$ with multiplicity 1, and root
$r_3$ with multiplicity 3, then the solution to the recurrence has the
form:

\[
f(n) = c_1 r_1^n + c_2 n r_1^n +
       c_3 r_2^n +
       c_4 r_3^n + c_5 n r_3^n + c_6 n^2 r_3^n
\]

As before, the constants depend on the boundary conditions.  Each
boundary condition generates a linear equation, and the constants are
computed by solving the resulting system of linear equations.  In this
respect, solving linear recurrences is very similar to solving linear
differential equations.

\subsection{An Example of the Tricky Case}

To understand recurrences of this tricky type, let's work an example.

Suppose that there is a type of plant that lives forever, but only
reproduces during the first year of life.  How fast will the plant
population grow?  (Notice that this is just the reverse of the
graduate student job problem where faculty ``reproduce'' in every year
except the first.)

Let $f(n)$ be the number of plants in year $n$.  As boundary
conditions, define $f(0) = 0$ and $f(1) = 1$.  The plant population in
year $n$ is equal to the population from the year before plus the
number of new plants.  The population from the year before is
$f(n-1)$.  The number of new plants this year is equal to the number
of new plants last year or $f(n-1) - f(n-2)$.  We can now form a
recurrence equation for the plant population:

\begin{eqnarray*}
f(n) & = & f(n-1) + (f(n-1) - f(n-2)) \\
     & = & 2 f(n-1) - f(n-2) \\
f(n) - 2 f(n-1) + f(n-2) & = & 0
\end{eqnarray*}

The characteristic equation is $\alpha^2 - 2 \alpha + 1 = 0$, which
has the single root $\alpha = 1$ with multiplicity 2.  Therefore, the
solution to the recurrence has the form:

\begin{eqnarray*}
f(n) & = & c_1 (1)^n + c_2 n (1)^n \\
     & = & c_1 + c_2 n
\end{eqnarray*}

The boundary conditions, $f(0) = 0$ and $f(1) = 1$, generate the
following linear equations:

\begin{eqnarray*}
0 & = & c_1 + c_2 (0) \\
1 & = & c_1 + c_2 (1)
\end{eqnarray*}

The solution to the linear system is $c_1 = 0$ and $c_2 = 1$.
Therefore, the solution to the recurrence is:

\begin{eqnarray*}
f(n) & = & c_1 + c_2 n \\
     & = & 0 + (1) n \\
     & = & n
\end{eqnarray*}

The answer turns out to be very simple!  In year $n$, there are
exactly $n$ plants.  Of course, we probably could have solved this
problem more easily with Guess and Verify, but for many other
examples, guessing is not so easy.

\section{Inhomogeneous Recurrences}

We can now solve all recurrences of the form:

\[
f(n) - a_1 f(n-1) - \ldots - a_d f(n-d) = 0
\]

Strictly speaking, this is the family of linear {\em homogeneous}
recurrences.  These recurrences are called homogeneous because the
right side of the equation is 0.  If the right side were 1 or $n^2$ or
some arbitrary function $g(n)$, then the recurrence would be called
{\em inhomogeneous}.

\begin{defin*}
An inhomogeneous linear recurrence has the form:

\[
f(n) - a_1 f(n-1) - a_2 f(n-2) - \ldots - a_d f(n-d) = g(n)
\]

where $a_1, a_2, \ldots a_d$ are constants.
\end{defin*}

Inhomogeneous recurrences are trickier than homogeneous ones.
Remember that we used the following fact: if two distinct functions
satisfy a linear recurrence, then their sum does too.  This is no
longer true for inhomogeneous recurrences: if we have two functions
$f_1$ and $f_2$ that satisfy the inhomogeneous recurrence, that is to
say,
\begin{align*}
f_1(n)-\sum_{i=1}^d a_if_1(n-d) &= g(n)\\
f_2(n)-\sum_{i=1}^d a_if_2(n-d) &= g(n)\\
\intertext{then their sum $s(n)=f_1(n)+f_2(n)$ satisfies}\\
s(n)-\sum_{i=1}^d a_is(n-d) &= 2g(n)\\
\end{align*}
(we can see this by adding the equations for $f_1$ and $f_2$.
However, if we \emph{substract} the equations for $f_1$ and $f_2$, and
define $t(n)=f_1(n)-f_2(n)$, then we find that
\[
t(n)-\sum_{i=1}^d a_i t(n-d) = 0,
\]
that is, the difference $t(n)$ satisfies the \emph{homogeneous}
portion of the inhomogeneous equation.  This leads to an approach for
inhomogeneous linear recurrences: solve the inhomogeneous part once,
and then add in the homogeneous part to deal with the boundary
conditions.


There are three steps to solving an inhomogeneous linear recurrence.

\begin{enumerate}

\item Replace $g(n)$ by 0 and solve the resulting homogeneous recurrence.
(Ignore boundary conditions for now; that is, do not solve for
constants $c_1, c_2, \ldots, c_d$.)  The solution to the homogeneous
recurrence is called the {\em homogeneous solution}.

\item Restore $g(n)$ and find a single solution to the recurrence, again
ignoring boundary conditions.  This is called the {\em particular
solution}.  (This is the hard part!)

\item Add the homogeneous and particular solutions together to obtain the
{\em general solution}.  Now use the boundary conditions to determine
the constant factors by the usual method of generating and solving a
system of linear equations.

\end{enumerate}

Notice that the method for solving inhomogeneous linear recurrences is
closely analogous to the method for solving inhomogeneous linear 
differential equations!

\subsection{An Example of an Inhomogeneous Recurrence}

Let $f(n) = 4 f(n-1) + 3^n$ and, as a boundary condition, let $f(1) =
1$.  This is an inhomogeneous linear recurrence.

\subsubsection*{Step 1:  Solve the Homogeneous Recurrence}

The homogeneous recurrence is $f(n) - 4 f(n-1) = 0$.  The
characteristic equation is $\alpha - 4 = 0$.  The only root is $\alpha
= 4$.  Therefore, the homogeneous solution is $c_1 4^n$.

\subsubsection*{Step 2:  Find a Particular Solution}

We must find a single solution to the recurrence $f(n) = 4 f(n-1) +
3^n$.  This is usually the hard part of solving an inhomogeneous linear
recurrence.  Let's guess that there is a solution of the form $c 3^n$.
Substituting this into the recurrence gives:

\begin{eqnarray*}
f(n) & = & 4 f(n-1) + 3^n \\
c 3^n & = & 4 c 3^{n-1} + 3^n \\
3 c & = & 4 c  + 3 \\
c & = & -3
\end{eqnarray*}

The equation was divided by $3^{n-1}$ in the first step.  The second
and third steps were only simplification.  We now have a particular
solution, $f(n) = -3 \cdot 3^n = -3^{n+1}$.

\subsubsection*{Step 3:  Add Solutions and Find Constants}

We now add the homogeneous solution and the particular solution to
obtain the general solution:

\[
f(n) = c_1 4^n - 3^{n+1}
\]

The value of the constant is obtained from the boundary condition,
$f(1) = 1$:

\begin{eqnarray*}
1 & = & c_1 4^1 - 3^{1+1} \\
  & = & 4 c_1 - 9 \\
c_1 & = & 5/2
\end{eqnarray*}

The solution to the recurrence is $f(n) = \frac{5}{2} 4^n - 3^{n+1}$!

Since we could easily have made a mistake, let's check to make sure
that our solution at least works for $n = 2$.  From the recurrence,
$f(2) = 4 f(1) + 3^2 = 13$.  From our closed form, $f(2) = \frac{5}{2}
4^2 - 3^3 = 40 - 27 = 13$.  It looks right!

\subsection{Hanoi}

Now we can sledgehammer the Hanoi problem.  Recall the recurrence:
\[
T(n)=2T(n-1)+1.
\]
Let's apply our approach:

\subsubsection*{Step 1:  Solve the Homogeneous Recurrence}

The homogeneous recurrence is $T(n) - 2 T(n-1) = 0$.  The
characteristic equation is $\alpha - 2 = 0$.  The only root is $\alpha
= 2$.  Therefore, the homogeneous solution is $c 2^n$.


\subsubsection*{Step 2:  Find a Particular Solution}

Now we need a single solution to the recurrence $T(n) = 2T(n-1) + 1$
Let's guess a \emph{constant} solution $b$.  Is there a solution to
$b=2b+1$?  Yes!  Namely, $b=-1$.  So this is particular solution.

\subsubsection*{Step 3:  Add Solutions and Find Constants}

Adding the homogeneous solution and the particular solution gives the
general solution:
\[
T(n) = c 2^n -1
\]

Plugging in the boundary conditions gives $0=T(0)=c-1$ which tells us
that $c=1$.  So we get the expected answer $T(n)=2^n-1$.


\subsection{A Final Example}

Let $f(n) = 4 f(n-1) - 4 f(n-2) + n$ and, as a boundary condition, let
$f(0) = 4$ and $f(1) = 7$.

\subsubsection*{Step 1:  Solve the Homogeneous Recurrence}

The homogeneous recurrence is $f(n) - 4 f(n-1) + 4 f(n-2) = 0$.  The
characteristic equation is $\alpha^2 - 4 \alpha + 4$.  The only root
is $\alpha = 2$ with multiplicity 2.  Therefore, the homogeneous
solution is $c_1 2^n + c_2 n 2^n$.

\subsubsection*{Step 2:  Find a Particular Solution}

Now we need a single solution to the recurrence $f(n) = 4 f(n-1) - 4
f(n-2) + n$.  Let's guess a solution of the form $a n + b$ and
substitute into the recurrence:

\begin{eqnarray*}
f(n) & = & 4 f(n-1) - 4 f(n-2) + n \\
a n + b & = & 4 (a (n-1) + b) - 4 (a (n-2) + b) + n \\
        & = & 4 a n - 4 a + 4 b - 4 a n + 8 a - 4 b + n \\
        & = & 4 a + n \\
& \Rightarrow & a = 1, b = 4a = 4
\end{eqnarray*}

This gives the particular solution $f(n) = n + 4$.

\subsubsection*{Step 3:  Add Solutions and Find Constants}

Adding the homogeneous solution and the particular solution gives the
general solution:

\[
f(n) = c_1 2^n + c_2 n 2^n + n + 4
\]

The values of the constants are obtained from the boundary conditions,
$f(0) = 4$ and $f(1) = 7$.

\begin{eqnarray*}
4 & = & c_1 2^0 + 0 c_2 2^0 + 0 + 4 \\
  & = & c_1 + 4 \\
c_1 & = & 0 \\
7 & = & c_1 2^1 + 1 c_2 2^1 + 1 + 4 \\
  & = & 2 c_1 + 2 c_2  + 5 \\
c_2 & = & 1
\end{eqnarray*}

The solution to the recurrence is $f(n) = n 2^n + n + 4$.

\subsection{Finding a Particular Solution}

The hardest part in solving inhomogeneous recurrences is finding a
particular solution.  There are some good rules of thumb, however.
Suppose the recurrence has the form:

\[
f(n) - a_1 f(n-1) - a_2 f(n-2) - \ldots - a_d f(n-d) = g(n)
\]

If $g(n)$ is a polynomial of degree $r$, then guess that $f(n)$ is a
polynomial of degree $r$.  For example, if $g(n)= n^2$, then guess
$f(n) = a n^2 + b n + c$ and determine $a$, $b$, and $c$ by
substituting and verifying.  If a degree $r$ polynomial does not work,
then try degree $r+1$ and so on.

Similarly, if $g(n)$ is an exponential, then guess that $f(n)$ is an
exponential.  For example, if $g(n) = 2^n + 3^n$, then guess $f(n) =
a2^n + b3^n$.  In general, guess a particular solution with the same
form as $g(n)$.


\section{Divide and Conquer Recurrences}

We now turn to another family of recurrences known as \emph{Divide and
  Conquer} recurrences.  These arise in recursive algorithms that
partition their inputs into smaller pieces in order to solve the
problem.


\subsection{The Merge Sort Algorithm}

As a motivating example, we will consider Merge Sort.  There are many
algorithms for sorting a list of $n$ items; in fact, you will see a
dozen of them in 6.046. Merge sort is one of the most popular.

Here is how Merge Sort works.  The input is a list of $n
\geq 1$ items $x_1, x_2, \ldots, x_n$.  If $n = 1$, then the algorithm
returns the single item $x_1$.  If $n > 1$, then the original list is
broken into two lists, $x_1, \ldots, x_{n/2}$ and $x_{n/2+1}, \ldots,
x_n$.  Both of these lists are sorted recursively, and then they are
merged to form a complete, sorted list of the original $n$ items.  All
work is in the merging operation; the details are explained in the
following example.

Suppose we want to sort the list 10, 7, 23, 5, 2, 4, 3, 9.  Since the
number of items is greater than 1, we divide into two lists; one is
10, 7, 23, 5, and the other is 2, 4, 3, 9.  Each list is sorted
recursively.  The results are 5, 7, 10, 23 and 2, 3, 4, 9.

Now we must merge these two small sorted lists into one big sorted
list.  (In this case, the ``small'' lists have 4 items apiece, and the
``big'' list will eventually have 8 items.)  We start with the big
list empty and add one item at a time.  At each step, we compare the
first items in the small lists.  We remove the smaller item and append
it to the end of the big list.  This process repeats until one of the
small lists becomes empty.  At that point, the remaining small list is
appended to the big list and we are done.

For the example, the contents of the three lists after each step are
shown in the table below.

\begin{eqnarray*}
\begin{array}{l|l|l}
small \ list \ 1 & small \ list \ 2 & big \ list \\ \hline
5, 7, 10, 23 & 2, 3, 4, 9 & \\
5, 7, 10, 23 & 3, 4, 9 &  2 \\
5, 7, 10, 23 & 4, 9 & 2, 3 \\
5, 7, 10, 23 & 9 & 2, 3, 4 \\
7, 10, 23 & 9 & 2, 3, 4, 5 \\
10, 23 & 9 & 2, 3, 4, 5, 7 \\
10, 23 & & 2, 3, 4, 5, 7, 9 \\
 & & 2, 3, 4, 5, 7, 9, 10, 23
\end{array}
\end{eqnarray*}

\subsection{Finding a Recurrence}

In the analysis of a sorting algorithm, a traditional question is,
``What is the maximum number comparisons used in sorting $n$ items?''
The number of comparisons is taken as an estimate of the run time.  In
the case of Merge Sort, we can find a recurrence for this quantity.
Solving the recurrence will allow us to study the asymptotic behavior
of the algorithm.

To make the analysis easier, assume for now that the number of items
we are sorting is a power of 2.  This ensures that we can divide the
list of items exactly in half at every stage of the recursion.

Let $T(n)$ be the maximum number of comparisons used by Merge Sort in
sorting a list of $n$ items.  If $n > 1$, then this is equal to the
number of comparisons used in sorting both halves of the list plus the
number of comparisons used in merging the two halves.  

The number of comparisons used in sorting each half of the list is
$T(n/2)$.  Sorting both halves requires $2 T(n/2)$ comparisons.  The
number of comparisons used in merging the two halves is at most $n-1$.
The reason is that one item is appended to the big list after each
comparison, and at least one additional item is appended to the big
list in the final step when one small list becomes empty.  Since the
big list eventually contains $n$ items, there can be at most $n-1$
comparisons.  (Of course, there might be fewer comparisons, but we are
analyzing the worst case.)

Adding all comparisons gives the following recurrence:

\[
T(n) = 2T(n/2) + n - 1
\]

No comparisons are required to sort a list with a single item, so
$T(1) = 0$.  From this fact and the recurrence, we can compute $T(2) =
2T(1) + 2 - 1 = 1$, $T(4) = 2T(2) + 4 - 1 = 5$, $T(8) = 2T(3) + 8 - 1
= 17$, $T(16) = 2T(4) + 16 - 1 = 49$, etc.

\subsection{Solving the Recurrence}

We would like to find a closed form for the number of comparisons used
by Merge Sort in sorting a list of $n$ items.  This requires solving
the recurrence $T(n) = 2T(n/2) + n - 1$.  The first method to try is
Guess and Verify.  The problem is that there is no obvious pattern to
the values from small cases, 0, 1, 5, 17, 49.  Our only other method
is Plug and Chug.  Here we go:

\subsubsection*{Step 1: Plug and Chug}

\begin{eqnarray*}
T(n) & = & n-1 + 2T(n/2) \\
(plug) \quad & = & n-1 + 2(n/2 - 1 + 2T(n/4)) \\
(chug) \quad & = & n-1 + n-2 + 4T(n/4) \\
(plug) \quad & = & n-1 + n-2 + 4(n/4 - 1 + 2T(n/8)) \\
(chug) \quad & = & n-1 + n-2 + n-4 + 8T(n/8) \\
(plug) \quad & = & n-1 + n-2 + n-4 + 8(n/8 - 1 + 2T(n/16)) \\
(chug) \quad & = & n-1 + n-2 + n-4 + n-8 + 16T(n/16)
\end{eqnarray*}

A pattern has emerged.  Note that too much simplification would have
obscured the pattern.

\subsubsection*{Step 2: Identify and Verify a Pattern}

\begin{eqnarray*}
T(n) & = & n-1 + n-2 + \ldots + n-2^{i-1} + 2^iT(n/2^i) \\
(plug) \quad & = & n-1 + n-2 + \ldots + n-2^{i-1} + 2^i(n/2^i - 1 + 2T(n/2^{i+1})) \\
(chug) \quad & = & n-1 + n-2 + \ldots + n-2^{i-1} + n-2^i + 2^{i+1}T(n/2^{i+1}) \\
\end{eqnarray*}

The additional round of plug and chug confirms that the pattern is
correct.

\subsubsection*{Step 3: Express $n$-th Term Using Early Terms}

We must substitute a value for $i$ into the pattern so that $T(n)$
depends on only small terms.  A natural choice is $i = \log_2 n$,
since then $T(n/2^i) = T(1)$.  This substitution makes $T(n)$
dependent only on $T(1)$, which we know is 0.

\begin{eqnarray*}
T(n) & = & n-1 + n-2 + \ldots + n-2^{i-1} + 2^iT(n/2^i) \\
     & = & n-1 + n-2 + \ldots + n-2^{\log_2(n)-1} + 2^{\log_2 n}T(n/2^{\log_2 n}) \\
     & = & n-1 + n-2 + \ldots + n-2^{\log_2(n)-1} + 2^{\log_2 n}T(1) \\
     & = & n-1 + n-2 + \ldots + n-2^{\log_2(n)-1}
\end{eqnarray*}

In the final equation, we used the fact that $T(1) = 0$; this
completely eliminates the last term in the summation.

\subsubsection*{Step 4: Find a Closed-Form for the $n$-th Term}

Now we have an ordinary, non-recurrent expression for $T(n)$.  We can
reduce this to a closed form by summing a series.

\begin{eqnarray*}
T(n) & = & n-1 + n-2 + \ldots + n-2^{i-1} + n-2^{\log_2(n)-1} \\
     & = & \sum_{i=0}^{\log_2(n) - 1} n - 2^i \\
     & = & \sum_{i=0}^{\log_2(n) - 1} n - \sum_{i=0}^{\log_2(n) - 1} 2^i \\
     & = & n \log_2 n - (2^{\log_2 n} - 1) \\ 
     & = & n \log_2 n - n + 1 \\
     & \sim & n \log_2 n
\end{eqnarray*}

What a weird answer--- we never would have guessed this!  To make sure
that there is no error, we can check that the formula gives the same
values for $T(n)$ that we computed earlier:

\[
\begin{array}{c|c}
n & n \log_2 n - n + 1 \\ \hline
1 & 1 \log_2 1 - 1 + 1 = 0 \\
2 & 2 \log_2 2 - 2 + 1 = 1 \\
4 & 4 \log_2 4 - 4 + 1 = 5 \\
8 & 8 \log_2 8 - 8 + 1 = 17 \\
16 & 16 \log_2 16 - 16 + 1 = 49
\end{array}
\]

Sure enough!  Note that the $n \log_2 n$ comparisons used by Merge
Sort are far fewer than the total of $n(n-1)/2$ possible comparisons.

\section{More Recurrences}

Let's compare the Tower of Hanoi and Merge Sort recurrences.

\begin{eqnarray*}
T(n) = 2 T(n-1) + 1 & \quad \quad & T(n) \sim 2^n \\
T(n) = 2 T(n/2) + n - 1 & \quad \quad & T(n) \sim n \log_2 n
\end{eqnarray*}

In the Towers of Hanoi, we divided a problem of size $n$ into two
subproblems of size $n-1$ and did only 1 additional step (moving the
largest disk).  In Merge Sort, we divided a problem of size $n$ into
two subproblems of size just $n/2$, but we did as many as $n-1$
additional steps (merging two sublists).  Which algorithm runs faster?
Merge Sort by a mile!  The key point is that generating smaller
subproblems is far more important than reducing the additional steps
per recursive call.

Let's vary the above recurrences slightly and see how the solutions
change.

\subsection{A Speedy Algorithm}

Suppose we have a speedy algorithm with the best properties of both
earlier algorithms; that is, at each stage the problem is divided in
half {\em and} we do only one additional step.  The run time is
described by the following recurrence:

\begin{eqnarray*}
S(1) & = & 0 \\
S(n) & = & 2 S(n/2) + 1
\end{eqnarray*}

(Another interpretation of this recurrence is that $S(n)$ is the number
recursive calls made when Merge Sort is run on a list of $n$ items.)

Guess and Verify is the first method to try.  (Plug and Chug also
works.)  To motivate a guess, let's tabulate a few values of $S(n)$.
As before, assume that $n$ is a power of two.

\[
\begin{array}{c|c}
n & S(n) \\ \hline
1 & 0 \\
2 & 2S(1) + 1 = 1 \\
4 & 2S(2) + 1 = 3 \\
8 & 2S(4) + 1 = 7
\end{array}
\]

\begin{guess*}
$S(n) = n - 1$
\end{guess*}

\begin{verify}
The proof is by strong induction on $n$.  Let $P(n)$ be the predicate
that if $n$ is a power of two, then $S(n) = n - 1$.  In the base case,
$P(1)$ is true because $S(1) = 0$ by assumption.  In the inductive
step, for $n \geq 2$ assume $P(1)$, $P(2)$, $\ldots$, and $P(n-1)$ to
prove $P(n)$.  If $n$ is not a power of two, then $P(n)$ is vacuously
true.  Otherwise, we have:

\begin{eqnarray*}
S(n) & = & 2 S(n/2) + 1 \\
       & = & 2 (n/2 - 1) + 1 \\
       & = & n - 1
\end{eqnarray*}

The first equation is the recurrence.  The second equality
follows from the induction hypothesis, and the last step is
simplification only.
\end{verify}

The run time of this speedy algorithm is $\sim n$.  This is better
than the $\sim n \log n$ run time of Merge Sort, but only slightly so.
This is consistent with the idea that decreasing the number of
additional steps per recursive call is much less important than
reducing the size of subproblems.

\subsection{A Verification Problem}

Sometimes verifying the solution to a recurrence using induction can
be tricky.  For example, suppose that we have the same recurrence as
before, but we only try to prove that $S(n) \leq n$.  This is true,
but the proof goes awry!

\begin{guess}
$S(n) \leq n$
\end{guess}

{\em Verification (attempt). }
The proof is by strong induction.  Let $P(n)$ be the predicate that if
$n$ is a power of two, then $S(n) \leq n$.  In the base case, $P(1)$
is true because $S(1) = 0 \leq 1$.  In the inductive step, for $n \geq
2$, assume $P(1)$, $P(2)$, $\ldots$, $P(n-1)$ to prove $P(n)$.  If $n$
is not a power of two, then $P(n)$ is vacuously true.  Otherwise, we
have:

\begin{eqnarray*}
S(n)    & = & 2 S(n/2) + 1 \\
        & \leq & 2 (n/2) + 1 \\
        & = & n + 1 \\
        & \not\leq & n
\end{eqnarray*}

The first equation is the recurrence.  The second equality follows by
induction.  The third step is a simplification, and in the fourth step
we crash and burn spectacularly.

We know that the result is true, but the proof did not work!  The
natural temptation is to ease up and try to prove something weaker,
say $S(n) \leq n+1$ or even $S(n) \leq 2n$.  Bad plan!

As was often the case with other induction proofs, the trick is to
choose a {\em stronger} induction hypothesis!  In particular, if we
use the stronger hypothesis that $S(n) \leq n - 1$, then the proof
goes through.  The crux of the argument in the inductive step is shown
below.

\begin{eqnarray*}
S(n)    & = & 2 S(n/2) + 1 \\
        & \leq & 2 (n/2 - 1) + 1 \\
        & = & n - 1 \\
        & \leq & n - 1
\end{eqnarray*}

As before, the first equation is the recurrence, the second follows by
induction, and the third is a simplification.  The final inequality
shows that the induction hypothesis holds for $P(n)$ as desired.

\subsection{A False Proof}

What happens if we try an even stronger induction hypothesis?
Shouldn't the proof work out even more easily?  For example, suppose
our hypothesis is that $S(n) \leq n - 2$.  This hypothesis is false,
since we proved that $S(n) = n - 1$.  But let's see where the proof
breaks.  Here again is the crux of the argument:

\begin{eqnarray*}
S(n)    & = & 2 S(n/2) + 1 \\
        & \leq & 2 (n/2 - 2) + 1 \\
        & = & n - 3 \\
        & \leq & n - 2
\end{eqnarray*}

Something is wrong; we proved a false statement!  The problem is that
we were lazy and did not write out the full proof; in particular, we
ignored the base case.  Since $S(1) = 0 \not\leq -1$, the induction
hypothesis is actually false in the base case.  This is why we cannot
construct a proof with a ``too strong'' induction hypothesis.

\subsection{Altering the Number of Subproblems}
\label{sec:multiples}

Some variants of the Merge Sort recurrence have truly peculiar
solutions!  The main difference in these variants is that we replace
the constant 2 (arising because we create 2 subproblems) by a
parameter $a$.

\begin{eqnarray*}
T(1) & = & 1 \\
T(n) & = & a T(n/2) + n
\end{eqnarray*}

Intuitively, $a$ is the number of subproblems of size $n/2$ generated
at each stage of the algorithm; however, $a$ is actually not required
to be an integer.  This recurrence can be solved by Plug and Chug, but
we will give only the solutions here:

\begin{eqnarray*}
0 \leq a < 2 & \quad & T(n) \sim \frac{2}{2-a}n \\
a = 2 & \quad & T(n) \sim n \log_2 n \\
a > 2 & \quad & T(n) \sim \frac{a}{a-2} n^{\log_2 a}
\end{eqnarray*}

The examples below show that the Merge Sort recurrence is extremely
sensitive to the multiplicative term, especially when it is near 2.

\begin{eqnarray*}
a = 1.99 & \quad & T(n) = \Theta(n) \\
a = 2 & \quad & T(n) = \Theta(n \log n) \\
a = 2.01 & \quad & T(n) = \Theta(n^{1.007\ldots})
\end{eqnarray*}

The constant $1.007\ldots$ is equal to $\log_2 2.01$.


\subsection{A False Theorem}

People often write recurrences with asymptotic notation as in the
previous section, but be careful!  As we have seen before, asymptotic
notation mixed with induction leads to faulty proofs.

\begin{falsethm}
If $T(1) = 1$ and $T(n) = 2 T(n/2) + n$, then $T(n) = O(n)$.
\end{falsethm}

This theorem is false; we know that the correct solution to the
recurrence is $T(n) = \Theta(n \log n)$.

\begin{proof}
The proof is by strong induction.  Let $P(n)$ be the predicate that
$T(n) = O(n)$.  In the base case, $P(1)$ is true because $T(1) = 1 =
O(1)$.  In the inductive step, for $n \geq 2$ assume $P(1), P(2),
\ldots P(n-1)$ to prove $P(n)$.  We have:

\begin{eqnarray*}
T(n) & = & 2 T(n/2) + n \\
     & = & 2 O(n/2) + n \\
     & = & O(n)
\end{eqnarray*}

The first equation is the recurrence, the second follows by induction,
and the third is a simplification.
\end{proof}

The general problem is that Big Oh should not be used in an induction
hypothesis.  More specifically, the proof shows that for all $n \geq
1$, $T(n)$ is $O(n)$; that is, $T(1)$ is $O(1)$, $T(2)$ is $O(2)$,
$T(47)$ is $O(47)$, etc.  This is perfectly true, but wholly unrelated
to the theorem statement; it says, in effect, that all constants are
in fact constants.

Recalling the definition of Big Oh notation, the theorem implies that
there exists a constant $c > 0$ and an $n_0$ such that for all $n \geq
n_0$, $|T_n| \leq c n$.  If we pick a $c$ and $x_0$ and then attempt a
proof by induction, then we fail as shown below.

{\em Proof (failed attempt). } The proof is by strong induction.  Let
$P(n)$ be the predicate that $|T_n| \leq c n$.  In the base case,
$P(n_0)$ is true because $T(n_0) \leq cn_0$.  {\em (Suppose that we
chose $c$ large enough to make this true.)} In the inductive step, for
$n \geq n_0 + 1$ assume $P(n_0), \ldots, P(n-1)$ to prove $P(n)$.  We
have:

\begin{eqnarray*}
T(n) & = & 2 T(n/2) + n \\
     & \leq & 2 c (n/2) + n \\
     & = & cn + n \\
     & = & (c + 1) n \\
     & \not\leq & cn
\end{eqnarray*}

The first equation is the recurrence.  The next equality follows by
applying the induction hypothesis.  Thereafter we simplify until the
proof collapses.

\section{A Master Theorem}

We've looked at divide and conquer recurrences in order to practice
solving recurrences.  But as with linear recurrences, you can often
just hit the recurrence with a sledgehammer:

\begin{theorem}
Let $T$ be an increasing function that satisfies the recurrence
\[
T(n) = aT(n/b)+c
\]
where $a \ge 1$, $b>1$ is an integer, and $c>0$.  Then
\begin{equation*}
T(n) = 
\begin{cases}
O(n^{\log_b a}) & \text{if $a>1$},\\
O(\log n) & \text{if $a=1$}
\end{cases}
\end{equation*}
\end{theorem}


\begin{theorem}
Let $T$ be an increasing function that satisfies the recurrence
\[
T(n) = aT(n/b)+cn^d
\]
whenever $n=b^k$, where $a \ge 1$, $b>1$ is an integer, and $c>0$ and
$d>0$ are real numbers.  Then
\begin{equation*}
T(n) = 
\begin{cases}
O(n^{\log_b a}) & \text{if $a>b^d$,}\\
O(n^d \log n) & \text{if $a=b^d$,}\\
O(n^d) &\text{if $a < b^d$}
\end{cases}
\end{equation*}
\end{theorem}


Both of these theorems have proofs in the text.


\section{Diversion on $\pi$}

It's $\pi$ day.  Someone asked me to prove something about $\pi$.
Here we go:

\begin{theorem} $\pi^2$ is irrational
\end{theorem}

To prove this theorem I will use a lemma about a special function.
\begin{lemma}
Define 
\[
f(x) = \frac{x^n(1-x)^n}{n!}.
\]
Then
\begin{enumerate}
\item \label{i:poly} $f(x)$ is a polynomial
  $\frac{1}{n!}\sum_{i=n}^{2n}c_ix^i$.
\item \label{i:small} For $-<x<1$ we have $0<f(x)<1/n!$.
\item \label{i:int} For any $k$, we the $k^{th}$ derivatives
  $f^{(k)}(0)$ and $f^{(k)}(1)$ are integers
\end{enumerate}
\end{lemma}
\begin{proof}
  For (\ref{i:poly}), ignore the $n!$ term.  The coefficients of
  $x^n(1-x)^n$ are integers by induction on $n$.  Also $x^n$ divides
  every term, so the smallest term is $x^n$, and the max degree is
  $2n$.

  For (\ref{i:small}), note that $x(1-x)<1$ so clearly any integer
  power is less than 1 (induction again).

  For (\ref{i:int}), note that by (\ref{i:poly}) the $k^{th}$
  derivative is 0 unless $n \le k \le 2n$.  In this range the
  derivative $f^{(k)}(0) = \frac{k!}{n!}c_k$ is an integer since $k>n$
  and $c_k$ is an integer.  Also since $f(x)=f(1-x)$ we have
  $f^{(k)}(x) = (-1)^kf^{(k)}(1-x)$ so $f^{(k)}(1)=(-1)^{k}f^{(k)}(0)$
  is an integer too.
\end{proof}

With this lemma we can prove the theorem.  Let's assume $\pi^2=a/b$
for integer $a$ and $b$.  Consider the polynomial
\[
F(x) = b^n\sum_{k=0^n}(-1)^k\pi^{2(n-k)}f^{(k)}(x).
\]
Note that $b^n\pi^{2(n-k)}$ is an integer for every $k \le n$, so this
is actually an integer sum of $f$ terms.
And from (\ref{i:int}) in the lemma each $f^{(k)}(x)$ is an integer at 0
and 1.  Thus $F(0)$ and $F(1)$ are integers.

Elementary differentiation says
\[
\frac{d}{dx}[F'(x)\sin(\pi x)-\pi F(x)\cos(\pi x)] =
b^n\pi^{2n+2}f(x)\sin(\pi x)=\pi^2a^n\sin(\pi x).
\]
It follows that
\begin{align*}
N &= \pi \int_0^1 a^n f(x)\sin(\pi x)dx\\
&= \left[\frac1\pi F'(x)\sin(\pi x)-F(x)\cos(\pi x)\right]_0^1\\
&= F(0)+F(1)\\
\end{align*}
is an integer.
Also, $N$ is positive since it is the integral of a function that is
positive (except at 0 and 1 where it is 0).  We are going to show that
also $N<1$, this contradicts $N$ being an integer.

To see that, recall that $f(x) < 1/n!$.  So the integral is at most
\begin{align*}
N &= \pi \int_0^1 a^n f(x)\sin(\pi x)dx\\
&<\pi a^n/n!\\
&<\pi a^n(e/n)^n/\sqrt{2\pi}e^{1/12n}
&<(ae/n)^n.
\end{align*}
Note, however, that by taking $n>ae$. this is at most 1.

\end{outline}
\end{document}
