\documentstyle[12pt,twoside,amstex,eepic,psfig]{article}
\input{macros-course}
\newcommand{\remove}{}

\parskip1.5ex
\parindent0pt
\newcommand{\var}[1]{{\rm Var}[#1]}
\begin{document}
\handout{notes23}{Notes 24}{December 3, 1996}


\section{Expectations of Products}

Now consider the expectation of the {\em product\/} of two 
random variables.  \\
\\
We might hope for the identity: \\
$E[R_1 R_2] = E[R_1]\cdot E[R_2]$ \\
\\
But this does not hold in general. \\
However, it does hold in the important special case when 
$R_1$ and $R_2$ are independent:

\begin{theorem}
Let $R_1$ and $R_2$ be independent random variables.  \\
Then $E[R_1 R_2] = E[R_1]\cdot E[R_2]$.
\label{thm:product}
\end{theorem}

\begin{proof}
We have
\begin{align*}
E[R_1 R_2] & =  \sum_{r} r \cdot Pr(R_1 R_2 = r) \\
& =  \sum_{r} r \sum_{r_1,r_2|r_1 r_2=r} Pr(R_1 = r_1 ~{\rm and}~ R_2
= r_2) \\
& =  \sum_{r} \sum_{r_1,r_2|r_1 r_2=r} r Pr(R_1 = r_1 ~{\rm and}~ R_2
= r_2) \\
& =  \sum_{r} \sum_{r_1,r_2|r_1 r_2=r} r_1 r_2 Pr(R_1 = r_1 ~{\rm
and}~ R_2 = r_2) \\
& =  \sum_{r_1,r_2} r_1 r_2 Pr(R_1 = r_1 ~{\rm and}~ R_2 = r_2) \\
& =  \sum_{r_1,r_2} r_1 r_2 Pr(R_1 = r_1) \cdot Pr(R_2 = r_2).
\end{align*}
The first equality arises by decomposing the probability of an event
into the sum of the probabilities of disjoint events whose union is
the original event. \\
The next 3 steps are just calculations. \\
The last equality follows from the independence of $R_1$ and $R_2$. \\
\\
Now, we can factor this expression as
$$\left( \sum_{r_1} r_1 \cdot Pr(R_1 = r_1) \right)
\left( \sum_{r_2} r_2 \cdot Pr(R_2 = r_2) \right).$$
which is equal to $E[R_1] E[R_2]$. \\
Therefore, $$E[R_1 R_2] = E[R_1]\cdot E[R_2].$$
\end{proof}

\begin{example}
Rolling 2 fair, 6-sided dice independently. \\
Let $R_1$, $R_2$ be values of the first and second dice, respectively.
\\
Since these are independent, we have that
$$E[R_1 R_2] = E[R_1]\cdot E[R_2] = \frac{7}{2} \cdot \frac{7}{2} =
\frac{49}{4}.$$ \\
\\
Now, what if the two dice are rigged so that they always give the
same value as one another?  \\
In this case, $R_1$ and $R_2$ always assume the same values, and we have
$$E[R_1 R_2] = \sum_{i=1}^6 i^2 \cdot \frac{1}{6} = \frac{91}{6} \neq 
E[R_1]^2 = (\frac{7}{2})^2.$$
So this is a case of two dependent random variables for which the
expectation of their product is not the product of their expectations.
\end{example}

Finally, by induction on $n$, it is easy to prove the following
corollary of Theorem~\ref{thm:product}

\begin{theorem}
If $R_1, \ldots, R_n$ are mutually independent, then 
$$E[R_1 R_2 \cdots R_n] = (E[R_1]) (E[R_2]) \cdots (E[R_n]).$$
\label{cor:product}
\end{theorem}

\section{Conditional Expectation}

Can also define a conditional version of expectation, conditioned on a
particular event:

\begin{defin}
The conditional expectation of $R$ given an event $A$ is defined as
$$E[R \mid A] = \sum_{s \in A} R(s) \Pr(s \mid A)$$.
\end{defin}

\begin{theorem}
$$E[R \mid A] = \sum_r r \cdot Pr(R = r \mid A).$$
\end{theorem}

\begin{proof}
Analogous to the unconditional version of this statement, in Theorem
\ref{thm2}.
\end{proof}

\begin{example} Let $R$ be the number of spots we see in a typical die roll.
Then 
$$E[R \mid R \le 4] = 2.5$$
\end{example}

An important use of conditional expectation is in doing a ``case
analysis'' to compute an expectation. \\
Depends on a theorem that breaks down the expectation:

\begin{theorem}
Let $\{A_i\}$ be a set of disjoint events that exhaustively partition
the sample space.  Then for any random variable $R$,
\[
E[R] = \sum_i E[R \mid A_i]\Pr(A_i)
\]
\end{theorem}

The proof of this theorem depends on the following basic conditional
probability lemma:

\begin{lemma}
If $\{ A_i \}$ partition the sample space, then 
$$\Pr(B) = \sum_i \Pr(B|A_i) \Pr(A_i)$$.
\end{lemma}

\begin{proof}
RHS is just $\sum_i \Pr(B \cap A)$, which adds up to $\Pr(B)$ by
definition of a probability measure.
\end{proof}
We are now ready to prove the theorem (5.3):
\begin{proof} 
\begin{align*}
E[R] &= \sum_r r \cdot \Pr(R=r) \\
&= \sum_r r \cdot \left( \sum_i \Pr(R=r \mid A_i) \Pr(A_i) \right) \\
&= \sum_i \left(\sum_r r \cdot \Pr(R=r \mid A_i)\right) \Pr(A_i) \\
&= \sum_i E[R \mid A_i] \Pr(A_i)
\end{align*}
The first equation follows from the lemma just above. 
\end{proof}

\begin{example}
(Assembly plants)

Suppose there are two computer assembly plants $F$ and $S$. \\
\\
Plant $F$ is fast but sloppy: \\
They make $70\%$ of the computers, but
a computer from their plant has a $2\%$ probability of breaking down
on any given day of use.  \\
\\
Plant $S$ is slow and careful: \\
They make $30\%$ of the computers, but the breakdown probability is
only $1\%$. \\
\\
These computers get randomly mixed, so I buy one at random from the
store. \\
What is the expected number of days my computer works? \\
\\
Let $D$ be the random variable giving the number of days my computer
works. \\
Let $I_S$ be the event that I bought the computer from $S$, and let
$I_F$ be that I bought it from $F$. \\
\\
We decompose this problem using conditional expectations. \\
That is, we calculate separately the condition expectation of the
number of working days for $S$ and for $F$, \\
then combine using the conditional expectation theorem. \\
\\
The expected number of working days for a computer from $S$ is
obtained from an MTF calculation, that is, a negative binomial
distribution with failure probability $p=.01$. \\
Recall this is just 
$$E[D \mid I_S] = \frac{1-p}{p} = 99$$
Similarly, the expected number of working days for a computer from $F$
is 
$$E[D \mid I_F] =\frac{1-.02}{.02} = 49$$.
\\
Now to calculate the expected number of working days for the computer
when we don't know which factory it came from, we use the conditional
expectation theorem:

$$E[D] 
  = E[D|I_S] \Pr(I_S) + E[D|I_F] \Pr(I_F) = 99 (.30) + 49(.70) = 64$$
\end{example}

\section{Wald's Theorem}

The final topic is a theorem that allows another kind of combination
of experiments -- one in which a probability distribution determines
how many experiments are performed. \\
Begin with an example:

\begin{example}
Suppose I flip 3 standard coins and then roll a die for each coin that
comes up heads.  \\
How many total spots do I expect to see?  \\
Well, if $H$ is a random variable denoting the number of heads I see,
and $D_1,D_2,\ldots$ are random variables for die rolls, then the random
variable describing the number of spots can be written as
\[
N = \sum_{i=1}^H D_i.
\]
This notation is a bit strange, because the upper summation index is a
random variable.  \\
It makes sense, though -- for each element of the sample space, I
calculate the value of the random variable $N$ by summing up as
indicated, based on the values of the other random
variables at the same point. \\
\\
How can I compute the expectation?  One way is by
conditional expectations, conditioning on the number of coins that
come up heads:
\begin{align*}
E[N] &= \sum_{k=0}^{3} E[N \mid H = k] \Pr(H=k)\\
&=  \sum_{k=0}^{3} E[\sum_{i=1}^k D_i] \Pr(H=k)\\
&= \frac{1}{8}0 + \frac38(E[D_1])+ \frac38(E[D_1]+E[D_2])\\ 
&\quad +\frac18(E[D_1]+E[D_2]+E[D_3])\\
&= \frac383.5+\frac387+\frac1810.5\\
&= \frac{21}{4}
\end{align*}
The first equality uses the conditional expectation theorem. \\
The second specializes the sum to the case where the upper bound is
$k$. \\
\end{example}

Is there an easier way to figure this out?  \\
Notice that $E[H]=\frac72$, and $E[D]=\frac32$, so the expectation
turns out to be $E[H]\cdot E[D]$. \\
But why? \\
\\
Doesn't follow from linearity of expectation (rule for expectation
sums), because we don't know how many copies of $E[D]$ to add up. \\
\\
Doesn't follow from the product rule. \\
That would apply if I multiplied the number of heads by the outcome of
rolling one die. \\
\\
But it is true.  \\
Need a new result called {\em Wald's Theorem}:

\begin{theorem}
Let $Q$ and $R$ be independent random variables, where $Q$ takes on
natural number values. \\
Let $T$ be the sum of $Q$ samples of $R$, that is:
\[
T=\sum_{i=1}^Q R_i.
\]
Then $E[T] = E[Q]\cdot E[R]$.
\label{thm:wald}
\end{theorem}

\begin{proof}
Picture the random experiment as follows. \\
First we generate an infinite number of samples of $R$, call them
$R_1,R_2,\ldots$.; then we use $Q$ to decide how many of these copies
to add up in computing $T$. \\
This experiment works on a different sample space from the
original, but gives the same distribution.  \\
Then:
\begin{align*}
E[T] &= \sum_k E[T \mid Q=k] \cdot \Pr(Q=k) \\
  &= \sum_k E[\sum_{i=1}^k R_i] \cdot \Pr(Q=k) \qquad\text{(*)} \\
  &= \sum_k \left( \sum_{i=1}^k E[R_i]\right) \cdot \Pr(Q=k) \\
  &= \sum_k \left( \sum_{i=1}^k E[R]\right) \cdot \Pr(Q=k) \\
  &= \sum_k ( k E[R]) \cdot \Pr(Q=k) \\
  &= E[R] \sum_k  k  \cdot \Pr(Q=k) \\
  &= E[R] E[Q]
  \end{align*}
\end{proof}
One of the lines in this proof -- needs a little closer attention. \\
Namely, look at the line labeled (*), where we say that
\[
E[\sum_{i=1}^Q R_i \mid Q=k] = E[\sum_{i=1}^k R_i].
\]
Let's look at this more closely. \\
In fact, this is the place where we use the independence of the random
variables $Q$ and $R$. \\
\\
A bit more carefully: \\
The LHS can be rewritten as
\[
E[\sum_{i=1}^k R_i \mid Q=k] 
\]
We want to show this is equal to the RHS. \\
Now, independence of $R$ and $Q$ can be used to show that, for any $r$: \\
\[
\Pr(\sum_{i=1}^k R_i = r | Q = k) = \Pr(\sum_{i=1}^k R_i = r)
\]
This can be used to show that the conditional expectation is the same
as the unconditional expectation:
\[
E[\sum_{i=1}^k R_i \mid Q=k] = E[\sum_{i=1}^k R_i]
\]
as needed. \\
\\
If $Q$ and $R$ are dependent, the theorem might not hold.

We close this section with an application of Wald's Theorem to 
{\em branching processes}. \\
Branching processes are used to model population growth, nuclear chain
reactions, mutations, and contagious diseases. \\

\begin{example}
(Contagious disease)

Suppose someone is infected with a new, incurable disease. \\
Every day, he encounters $n$ healthy people and infects each of
them with probability $p$.  \\
What is the expected number of infected people after $k$ days? \\
\\
Let $S_k$ be a random variable giving  the number infected after $k$
days. \\
We can write a recurrence for $S_{k+1}$ in terms of $S_k$. \\
\\
Let the sick people after day $k$ be numbered $1,\ldots,S_k$. \\
Let $H_i$ be a random variable giving the number of healthy 
people infected by sick person $i$ in one day. \\
Then $H_i$ has a binomial distribution with parameters $n$ and $p$. \\
Then: \\
\[
S_{k+1} = S_k+ \sum_{i=1}^{S_k} H_i
\]
It follows from linearity of expectation that
\[
E[S_{+1}] = E[S_k]+E[ \sum_{i=1}^{S_k} H_i].
\]
Then by Wald's theorem,
\[
E[S_{k+1}] = E[S_k]+E[S_k]\cdot E[H]
\]
where $H$ is a variable with the binomial distribution with
parameters $n$ and $p$. \\
Thus, $E[H] = np$.  \\
\\
So
\[
E[S_{k+1}] = E[S_k]+E[S_k]\cdot np = (1 + np) E[S_k]
\]
Also, $E[S_0] = 1$. \\
So we have a recurrence involving these expectations. \\
Writing $e_i$ for $E[S_i]$, we have:
\[
e_{k+1} = (1+np)e_k
\]
and 
\[
e_0 = 1
\]
It follows that $e_k = (1+np)^k$.
\end{example}

\remove{
Eventually, we will have more sick people than people.  What erroneous
assumption is in our model?  That each sick person meets a different
collection of 10 healthy people.  Once there are enough sick people, it
will become less likely for a sick person to meet so many healthy
people.  Also, many sick people will encounter the same healthy person.
But our ``approximation'' will be valid as long as the sick
population is not too big.
%
Now suppose any sick person recovers from the disease each day with
probability $r$.  Let $B_i$ be the indicator variable for whether the
$i^{th}$ sick person gets better.  Then $E[B_i] = r$.  Then
\[
S_{k+1} = S_k+ \sum_{i=1}^{S_k} (H_i-B_i)
\]
It follows that
\[
E[S_{k+1}] = E[S_k]+E[S_k]\cdot E[H]-E[S_k]E[B_i]
\]
and so
\begin{align*}
e_{k+1} &= e_k(1+np-r)\\
&=(1+np-r)^k
\end{align*}
%
Therefore, the progress of the disease is determined entirely by the
parameter $(1+np-r)$.  If this exceeds $1$, it looks like we have an
epidemic on our hands (but expectation isn't telling the whole story:
there's a chance the first person gets better without making anyone
sick).  If less than $1$, it looks pretty innocuous.  In fact, next
time we will see that if less than $1$, the disease almost certainly
dies out after a certain number of days.
} % end remove



\section{Variance}


Last week we spent a lot of time talking about the expected values of
random variables.  One thing we saw is that it is possible for a
random variable to have expected value 1 even though it is never
actually anywhere near 1 (e.g., if I flip two coins that are tied
together, what is the expected number of heads?).  This week, we are
going to study {\em deviations from expectation.} We will study the
likelihood that a random variable has a value distant from its
expectation and give methods for proving that probability is small.
Intuitively, the expected value of a random variable is the value that
we expect it to obtain.\\ 

For example, if we flip $100$ fair coins, the expected number of heads
is $50$. In fact, the probability that we get exactly $50$ heads is
low, but there is a pretty good chance that we get close to $50$
heads. In particular, the probability that we deviate substantially
from the expected value in this case is extremely small: The
probability of getting more than $75$ or less than $25$ heads is less
than one in a billion. This is because the distribution of the number
of heads, which is a binomial distribution, is very tightly centered
around its mean. This property holds for any binomial distribution,
even biased ones. Now, for arbitrary random variables, it is not
always the case that the distribution is tightly centered around the
mean. For example, if we glue $100$ coins together, the probability of
getting close to the mean of $50$ vanishes, as the random variable
always takes on either the value $0$ or $100$. Even so, it turns out
that we can usually say something about the probability of deviating
from the mean. For example, in the latter example, the random variable
never takes on values that are more than $2$ times the mean.\\

\section{Markov's Theorem}

Markov's Theorem gives the most general bound on the probability of
deviating from the mean. Markov's Theorem says that if a random
variable takes on only non-negative values, then it is unlikely to
greatly exceed its mean;

\begin{theorem} If $R:S\rightarrow R^+$ is a random variable, then for
  all $x \geq 0$,
\[Pr(R \geq  x)\leq \frac{E[R]}{x}. \]
\end{theorem}

\begin{proof} (of theorem)  
\begin{align*}
E[R]&=\sum_z z\Pr(R=z)\\
&\geq \sum_{z\geq x} z\Pr(R\geq z)\\
&\geq \sum_{z\geq x} x\Pr(R=z)\\
& =x\sum_{z\geq x}\Pr(R=z)\\
&=x\Pr(R\geq x) 
\end{align*}
\end{proof}

In other words, if $E[R]$ is small, then R is not likely to be large.
Markov's Theorem is often expressed in the follwoing alternative form,
which can be stated as a corollary:

\begin{corollary}
If  $R:S\rightarrow R^+$, then for all positive real numbers c,
\[\Pr(R\geq  cE[R])\leq \frac{1}{c}. \]
\end{corollary}

\begin{proof} (of corollary) Just write x=cE[R] in the theorem. Then
  $$\Pr (R \geq cE[R]) \leq \frac{E[R]}{cE[R]} = 1/c$$
\end{proof}

\begin{example}

  Let $R$ be the weight of a randomly selected person. Suppose we know
  that $E[R] = 100$, but we don't know what the distribution of $R$
  looks like. Then the probability that $R$ is greater than or equal
  to $200$ is, by Markov's Theorem, $\Pr (R \geq 200) = \Pr (R \geq 2
  \cdot E[R]) \leq 1/2$. Similarly, $\Pr (R \geq 300) = \Pr (R \geq 3
  \cdot E[R]) \leq 1/3$. Why can't everyone have weight $\geq 200$ in
  this example? The answer is that, if this were the case, then the
  expected weight of a randomly selected person would be greater than
  $200$. Not everyone can ``beat'' the mean!
\end{example}

\begin{example}
  If I give each person a random hat back at a hat check, what is the
  probability that at least $k$ people get their own hat back? Let $R$
  be the number of people who get their own hat back.  Markov's
  Theorem doesn't give us an exact answer, but, since $E[R] = 1$, it
  tells us that $\Pr(R \geq k) \leq 1/k$. 
\end{example}

Note that Markov's theorem doesn't hold if $R$ can be negative.  For
example, consider a variable $R$ that is $\pm 1000$ with probabilities
$1/2$ each.  It's expectation is 0!  So Markov's theorem is clearly
violated.\\

So where in the proof of Markov's Theorem did we use the
non-negativity of $R$? If we go back and check, we see that
non-negativity is required for $E[R] =\sum_x x\Pr(R=x) \geq
\sum_{x\geq K} x\Pr(R\geq x)$ to hold.\\

There is a symmetric result that gives an upper bound on the
probability that that a random variable takes on a small value.


\begin{theorem}
If $R \leq L$ for some $L \in \reals$, then, for all $x \leq L$:
\[
\Pr[R \leq x] \leq \frac{L - E[R]}{L-x}
\]
\end{theorem}

\begin{proof}
Consider the event $L-R$. Since $R \leq L$, we have that $L - R \geq
0$, and hence the conditions of Markov's Theorem are satisfied. Hence
we have that
$\Pr(R \leq x) = \Pr(L-R \geq L-x) \leq \frac{E[L-R]}{L-x} = \frac{L - E[R]}{L-x}$
\end{proof}

\begin{example}

Let $R$ be the score of a randomly selected student on a test, and
suppose the average score on a test is $E[R] = 75$, and the max score
was $100$. Then we can compute $\Pr(R \leq 50)$.
\[
\Pr[R \leq 50] \leq \frac{100-75}{100-50}=1/2
\]
\end{example}
One problem with Markov's Theorem and its corollaries is that it does
not give an exact probability, just an upper bound. Sometimes the
upper bound it gives is exactly the right answer, but sometimes it can
also be pretty far off from the actual value.

\begin{example}

  Suppose we flip $100$ fair coins, and let $R$ be the number of
  heads.  Then $E[R] = 50$, and so from Markov's Theorem, $\Pr(R \geq
  75) \leq 2/3$. However, $\Pr(R \geq 75)$ is in fact less than one in
  a billion, so Markov's Theorem is way off the mark! 
\end{example}

In general, just knowing the range and expected value of a random
variable is not enough to give a really good idea of what the
distribution looks like. In an effort to get a better handle on the
shape and nature of a distribution, mathematicians have defined
several other properties of a distribution which help describe it.
After the expected value, the next most important measure of a random
variable is its variance.\\

\section{Variance and Chebyshev's Theorem}

\begin{defin} 
The variance of R is 
\[\var{R}=E[(R-E[R])^2] .
\] 
\end{defin}

The variance gives us the average of the squares of the deviations
from the mean. The idea behind variance is that it is large when $R$
is likely to be far from its expectation, that is, when its
distribution is ``spread out''. If $R$ is usually close to its
expectation, the variance will be small.\\

\begin{example}

Consider the follwing two gambling games, $A$ and $B$:\\

\begin{itemize}
\item Game A: With probability $2/3$, you win $2\$$.  Otherwise, you
  lose $1\$$
\item Game B: With probability $2/3$, you win $1002\$$. Otherwise, you
  lose $2001\$$
\end{itemize}

Which would you play? First, note that in either game, you win with
probability $2/3$. Let $A$, $B$ be the returns from playing games A
and B, respectively. Then for game $A$, $E[A] = 2 \cdot 2/3 - 1
\cdot 1/3 = 1$, and for game $B$, $E[B] = 1002 \cdot 2/3 - 2001 \cdot
1/3 = 1$. So the expected return is the same for both games, namely
$1$! Still, the games are very different. This difference is captures
by the variances. Let's see what the variance is for each game:\\ 

\begin{itemize}
\item Game A:
\begin{align*}
A-E[A] &=\begin{cases}
   1& \text{with probability 2/3},\\
   -2 &\text{with prob. $1/3$}.
   \end{cases}\\
(A-E[A])^2 &=\begin{cases}
   1 &\text{with probability $2/3$}\\
   4 &\text{with prob. $1/3$}
   \end{cases}\\
\var{A} &= 1\cdot2/3+4\cdot1/3\\
&= 2
\end{align*}

\item Game B:
\begin{align*}
B-E[B] &=\begin{cases}
   1001 &\text{with probability $2/3$},\\
   -2002 &\text{with prob. $1/3$}
   \end{cases}\\
(B-E[B])^2 &=\begin{cases}
   1002001 &\text{with probability $2/3$}\\
   4008004 &\text{with prob. $1/3$}
   \end{cases}\\
\var{B} &= 1002001\cdot2/3)+4008004\cdot1/3 \\
&= 2004002
\end{align*}
\end{itemize}

Hence we se that game $B$ is much more risky than game $A$. Risk
averse people should stay away from game $B$!!!  In short, playing a
game with high variance is a serious risk: we can win a lot, but we
can loose a lot, too. Playing another game with the same expectation
of winnings but less variance we can hope for the same average gains
risking less.  In investments, many people prefer bonds to stocks
because, although the expected earnings in bonds are lower, the
variance is much lower too.  They are willing to pay for
risk-reduction.\\
\end{example} 

Why do we bother to square the deviation before taking expectations?
Why not just compute the expected deviation from the mean? Let's
consider a random variable $R$ with mean $E[R]$.  What we are
interested in, then, is the deviation of the random variable from its
mean, namely $R-E[R]$. Unfortunately, $E[R-E[R]] = E[R]-E[R] = 0$, so
the positive and negative deviations cancel out and give us no
information. This problem occurs because the expected value is the
center of mass for the distribution, so the positive and negative
deviations {\em must} balance out, by definition of expectation. So,
one advantage of squaring is that all deviations become positive, and
hence there is no cancelling. Of course, we could also get around this
cancellation problem by computing $E[|R-E[R]|]$.  However, absolute
values are painful in analysis (not differentiable, etc.). We could
also take higher powers of R-E[R]. If we take the $4$th power, for
instance, we get what is called the {\em kurtosis} of $R$. It turns
out however, for reasons what we will not go into, that squaring is
pretty much the best way to go.\\

People often like to ``renormalize'' the variance:

\begin{defin} 
The {\em standard deviation} of R is
  \[\sigma_R=\sqrt{\var{R}}.\] 
\end{defin}

Thus the variance is the square of the standard deviation.  It follows
that the standard deviation has the same units (days, people, etc.) as
the original random variable and its expectation.  On an {\em
  intuitive} level, the standard deviation measures $E[|R-E[R]|]$
(careful---they aren't really equal!). The standard deviation is a
very popular measure in statistics, particularly when fitting curves
to experimental data.\\

\begin{example}

Returning to game $B$, we can compute its standard deviation
$\sigma_B$:\\
$$ \sigma_B = \sqrt{\var{R}} = \sqrt{2004002} \approx 1416$$

\end{example}

So far, we have given a precise definition of variances of random
variables, as well as their standard deviations. The intuition is that
these measures tell you how likely a random variable is to be far from
its mean. Unfortunately, this is not always true. Just like the
expectation of a random variable does not completely specify the
distribution, the expectation of the deviations does not completely
specify the distribution of deviations. However, the variance and
standard deviation of a distribution are still useful tools for
analysis.

\subsection{The Chebyshev Bound}
 
A variable with small variance is unlikely to be far from its mean.
With Markov's Theorem, we can deduce, for example, that given a mean
IQ of 100, only $1/2$ the people can have IQ bigger than $200$.  But
suppose we know that the standard deviation in IQ is $10$.  Then
intuitively, it should quite rare to have an IQ bigger than $110$.  We
can show that in fact only $1/100$ people have IQ bigger than $200$.
  
Chebyshev's theorem formalizes this observation:

\begin{theorem}
For any random variable R, 
\[\Pr(|R-E[R]|\geq x)\leq \frac{\var{R}}{x^2} .\] 
\end{theorem}
 
\begin{proof}
\begin{align*}
\Pr(|R - E[R]| \geq x) &=\Pr((R-E[R])^2 \geq x^2\\
&\leq \frac{E[(R-E[R]^2]}{x^2}\\
&=\frac{\var{R}}{x^2}
\end{align*}


\end{proof}

\begin{corollary}
$\Pr(|R-E[R]|>c \cdot \sigma_R) \leq 1/c^2$.
\end{corollary}

\begin{proof}
Set $x = c \cdot \sigma_R$
\end{proof}

Chebyshev's theorem is very similar to Markov's Theorem, but uses the
added information supplied by the standard deviation to get a better
upper bound. In fact, the proof uses Markov's Theorem.

\begin{example}

Let $R$ be the IQ of a random person. Then, assume $R =geq 0$,
$\sigma_R=10$ and  $E[R]=100$.  Therefore, using Chebyshev:\\ 
\begin{align*}
\Pr(R \ge 200( &=\Pr((R-100)\ge100)\\
&=\Pr(|R-E[R]| \ge 10\sigma_R)\\
&\le 1/100
\end{align*}

Had we used Markov's Theorem here, we would have gotten a lower bound
of 1/2 on this same probability. 

\end{example}

In the previous example, we used $\Pr(|R-E[R]|) $ to give an upper bound
on $\Pr(R-E[R])$. Maybe the probability $\Pr(R-E[R])$ is in fact half
the probability  $\Pr(|R-E[R]|) $? In this case, the answer is no. If
the distribution were symmetric about the mean, however, it would be
true. We do not, however, have any reason to believe that the
distribution is symmetric. In general, if we only want the probability
mass of one of the tails, we can do a little better than what we just
did, however.\\

\begin{theorem}
For any random variable $R$, the following holds:\\
$$ \Pr(R - E[R] \geq x) \leq \frac{\var{R}}{\var{R}+x^2}$$
$$ \Pr(R - E[R] \leq -x) \leq \frac{\var{R}}{\var{R}+x^2}$$
\end{theorem}

We will not prove this theorem. The following is a useful corollary:\\

\begin{corollary}
For any random variable $R$, the following holds:\\
$$ \Pr(R - E[R] \geq c\sigma_R) \leq
\frac{\sigma_R^2}{\sigma_R^2+c^2\sigma_R^2} = \frac{1}{1+c^2}$$
$$ \Pr(R - E[R] \leq -c\sigma_R) \leq
\frac{\sigma_R^2}{\sigma_R^2+c^2\sigma_R^2} = \frac{1}{1+c^2}$$
\end{corollary}
\begin{example}
Returning to our IQ example, we can now recompute the probability of
having an IQ greater than $200$;\\

$$\Pr[R \ge 200] \leq 1/101$$

\end{example}

\begin{example} 

  At most 1/4 of the students score more than 2 standard deviations
  away from the mean, whatever the set of scores is.  Indeed, if R is
  the score of a random student, then
\[|Pr(R-E(R)|\geq 2\sigma_R)\leq \frac{1}{4},\]
which means that at most 1/4 of all the scores can be that far from
the average. Moreover, at most 1/5 of the students score more than 2 standard deviations
  above the mean, whatever the set of scores is.  Indeed, if R is
  the score of a random student, then
\[Pr(R-E(R)\geq 2\sigma_R)\leq \frac{1}{1+2^2} = 1/5,\]
\end{example}

Note that the results we get from Markov's Theorem and Chebyshev's
Theorem are only upper bounds on the probability of deviating from the
mean. If there is more information available about the distribution,
it might be possible to prove better bounds. For example, if we know
we are dealing with an unbiased binomial distribution, then it is
possible to prove a much better upper bound:\\

\begin{theorem}
For an unbiased binomially distributed random variable $R$,\\
$$\Pr[R-E[R] \ge c\sigma_R] < e^{-c^2}$$
\end{theorem}

\begin{example}
  Suppose that the test scores are binomially distributed. Then the
  probability of a randomly selected student having a score greater
  than or equal to two times the standard deviation is bounded above by
  $1/e^4$.
\end{example}


\subsection{A Different Variance Formula}

There is another popular way of defining the variance of a random
variable, which we will state as a theorem.\\

\begin{theorem}
For all R, we have
\[Var(R)=E[R^2]-E[R]^2.\] 
\end{theorem}

\begin{proof}
\begin{align*}
Var(R)&=E[(R-E[R])^2]\\
&=E[R^2-2RE[R]=E[R]^2] \\
&=E[R^2]-2E[R]E[R]+E[R]^2\\
&=E[R^2]-E[R]^2.
\end{align*}
\end{proof}


This formula is often more useful to compute variances than the previous one.
\begin{example}
  Let $R$ be the outcome of rolling a $6$-sided die. Then the variance
  of $R$ is given by

\begin{align*}
\var{R} &= \frac16(1+4+9+\cdots+36)-(3.5)^2\\
&= 15\frac16-12\frac14 = 2\frac{11}{12}
\end{align*}
\end{example}

This theorem is also useful from a computational point of view.
Suppose, say, that you are given a stream of data (like test scores),
and you are trying to compute the expectation and variance on the fly
with only one pass through the data. Computing a running average is
easy, we just compute a running sum and divide by the number of data
points seen so far. Similarly, to compute the variance, we compute two
running sums, one for the scores and one for the square of scores.\\

The theorem has a nice corollary:\\


\begin{corollary} 
For all R, $E(R^2)\geq E(R)^2$. 
\end{corollary}
\begin{proof} 
This follows from the previous theorem, since Var(R) is always
nonnegative, being the expectation of a square. \end{proof}


\subsection{Manipulating Variances}

\begin{theorem} 
If Var(R)=0, then R is a constant scalar.
\end{theorem}

\begin{proof} Suppose that \[\var{R}=E(R-E[R])^2]=0 .\]
Then $(R-E[R])^2$ must be always 0, thus R must always be equal to E[R].
\end{proof}

\begin{theorem}
For any constants $a$ and $b$, $\var{aX+b} = a^2\var{X}$.
\end{theorem}

\begin{proof}
\begin{align*}
\var{aX+b} &= E[(aX+b)^2]-(E[aX+b])^2\\
&= E[a^2X^2 + 2abX + b^2]-(aE[X]+b)^2\\
&= a^2E[X^2]+2abE[X]+b^2 - a^2E[X]^2 - 2abE[X]-b^2\\
&= a^2(E[X^2]-E[X]^2)\\
&= a^2\var{X}
\end{align*}
\end{proof}

A Bernoulli Random variable with probability $p$ has variance
\[
p\cdot1^2+(1-p)\cdot 0^2-p^2 = p(1-p)
\]

What about the binomial distribution?  This is messy to do directly,
but easy if we figure out how to sum variances:

\begin{theorem}
If $X$ and $Y$ are {\em independent}, then 
\[
\var{X+Y}=\var{X}+\var{Y}
\]
\end{theorem}

\begin{proof}
The assumption that $X$ and $Y$ are independent means that
$E[XY]=E[X]E[Y]$.  Therefore,
\begin{align*}
\var{X+Y} &= E[(X+Y)^2]-E[X+Y]^2\\
&= E[X^2+2XY+Y^2]-(\mu_X+\mu_Y)^2\\
&= E[X^2]+2E[XY]+E[Y^2]-\mu_X^2-2\mu_X\mu_Y-2\mu_Y^2\\
&= E[X^2]-\mu_X^2 +E[Y^2]-\mu_Y^2+2E[XY]-2\mu_X\mu_Y\\
&=  E[X^2]-\mu_X^2 +E[Y^2]-\mu_Y^2\\
\intertext{(since $E[XY] = \mu_X\mu_Y$ by independence)}
&= \var{X}+\var{Y}
\end{align*}
\end{proof}

\begin{corollary}
If $X_1,\ldots,X_n$ are {\em pairwise} independent random variables,
then
\[
\var{\sum{X_i}} = \sum\var{X_i}.
\]
\end{corollary}

Note that mutual independence is {\em not} required, because the only
fact we used that relied on independence was that
$E[X_iX_j]=E[X_i]E[X_j]$.  This follows from pairwise independence,
since it means $X_i$ and $X_j$ are independent.

Therefore, the variance of the binomial distribution on $n$ Bernoulli
trials with probability $p$ is $np(1-p)$.  It follows that the
standard deviation is $\sigma=\sqrt{np(1-p)}$.  


\end{document} 
