\documentclass[11pt]{article} 
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage{amsxtra}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{graphicx}
\usepackage{url}
\usepackage{fancyvrb}
\usepackage{color,hyperref}
\usepackage{enumerate}
\hypersetup{colorlinks,breaklinks,
             linkcolor=blue,urlcolor=blue,
           anchorcolor=blue,citecolor=blue}

\usepackage[margin=3cm]{geometry}

\usepackage{preamble}

\begin{document}

\section*{Math 5490 -- Homework 6: Due May 10 by 11:59pm}
\thispagestyle{empty}

\subsection*{Instructions:}

\begin{itemize}
\item  Complete the problems below, and submit your solutions and Python code by uploading them to the Google form: \url{https://forms.gle/B3EfYpSU8jQPYhiS9}
\item Submit all your Python code in a single .py file using the function templates given in each problem. I will import your functions from this file and test your code.
\item If you use LaTeX to write up your solutions, upload them as a pdf file. Students who use LaTeX to write up their solutions will receive bonus points on the homework assignment (equivalent to 1/3 of a letter grade bump).
\item If you choose to handwrite your solutions and scan them, please either use a real scanner, or use a smartphone app that allows scanning with you smartphone camera. It is not acceptable to submit photos of your solutions, as these can be hard to read.
\end{itemize}


\subsection*{Problems:}



\begin{enumerate}
\item Complete the following exercises. 
\begin{enumerate}[(i)]
\item Let $g(\x) = A \x + \b$, where $\x\in \R^n$ and $A\in \R^{n\times n}$. Show that the Jacobian matrix of $g$ is $Jg(\x)=A$. 
\item Let $h(\x) = \sigma(\x)$, where $\sigma:\R\to \R$ acts componentwise on $\x$, so that $\sigma(\x)_i = \sigma(x_i)$. Show that $Jh(\x)$ is the diagonal matrix $D =\diagm \sigma'(\x)$.
\item Use the first two parts and the chain rule to show that the Jacobian of $g(\x) = A\sigma(B\x + \b)$ is $Jg(\x) = ADB$, where $D = \diagm \sigma'(B\x + \b)$.
\end{enumerate}
\item  The traditional neural network architecture 
\[\Fv_{k} = \sigma_k(W_k \Fv_{k-1} + \b_k), \ \  k=1,\dots,L,\]
often yields worse performance for deeper networks with more layers compared to shallower networks. The main issue it that training is difficult, due to vanishing gradients or gradient blowup (where the gradients either become very small and training does not progress, or become very large and training is unstable). To understand why, consider the case where $\sigma_k(t)=t$ is the identity and the biases $\b_k=0$ all vanish. Then 
  \[\Fv_L(\x) = W_L W_{L-1}\cdots W_2W_1\x.\]
  The $L$-fold product is very sensitive to the spectral norms of the matrices; when the eigenvalues are larger than one in magnitude it blows up exponentially with $L$, while when they are less than one it will decay exponentially.

  The \emph{Residual Neural Network (ResNet)} architecture solves this problem by changing the architecture to
  \begin{equation}\label{eq:resnet}
\Fv_{k} = \Fv_{k-1} + V_k\sigma_k(\z_k), \ \ \text{where}  \ \ \z_k=W_k\Fv_{k-1} + \b_k, \ \ k=1,\dots,L.
  \end{equation}
  The idea is to have each layer learn the \emph{residual}  $\Fv_k - \Fv_{k-1}$, which allows the network to easily skip layers, by setting $\Fv_k = \Fv_{k-1}$. Thus, a deeper network with ResNet architecture should not perform worse than a shallower network. Note that in ResNet the dimensions of each layer must be the same. 

Follow an argument similar to that in the proof of Theorem 10.1 to establish the ResNet back propagation equations
\begin{align*}
\Lf{k-1} &= \Lf{k} + W_k^T D_k V_k^T\Lf{k}\\
\LV{k} &= (\Lf{k}) \:\sigma(\z_k)^T\\
\Lz{k} &= D_kV_k^T\Lf{k}
\end{align*}
and as in Theorem 10.1  $\Lw{k} = (\Lz{k}) \:\Fv_{k-1}^T$ and $\Lb{k} = \Lz{k}$.  Hint: For the first identity, let $h_k:\R^{n}\to \R$ denote the loss $\Loss$ as a function $\Fv_k$, so that $h_k(\Fv_k) = \Loss$ and $\nabla h_k(\Fv_k) = \Lf{k}$. Then show that
\[h_{k-1}(\Fv_{k-1}) = h_k(\Fv_{k-1} + V_k\sigma_k(W_k\Fv_{k-1} + \b_k)), \]
and use the chain rule. 
\end{enumerate}

\end{document}

