47 lines
3.1 KiB
TeX
47 lines
3.1 KiB
TeX
\chapter{Conclusions and Further Research}
|
|
|
|
We will present three avenues of further research and related work on parameter estimates here.
|
|
|
|
\section{Further operations and further kinds of neural networks}
|
|
|
|
Note, for instance, that several classical operations are done on neural networks that have yet to be accounted for in this framework and talked about in the literature. We will discuss two of them \textit{dropout} and \textit{dilation} and provide lemmas that may be useful to future research.
|
|
|
|
\subsection{Mergers and Dropout}
|
|
|
|
\begin{definition}[Hadamard Product]
|
|
Let $m,n \in \N$. Let $A,B \in \R^{m \times n}$. For all $i \in \{ 1,2,\hdots,m\}$ and $j \in \{ 1,2,\hdots,n\}$ define the Hadamard product $\odot: \R^{m\times n} \times \R^{m \times n} \rightarrow \R^{m \times n}$ as:
|
|
\begin{align}
|
|
A \odot B \coloneqq \lb A \odot B \rb _{i,j} = \lb A \rb_{i,j} \times \lb B \rb_{i,j} \quad \forall i,j
|
|
\end{align}
|
|
\end{definition}
|
|
|
|
\begin{definition}[Scalar product of weights]
|
|
Let $\nu \in \neu$, $L\in \N$, $i,j,k \in \N$, and $c\in \R$. Assume also that $\lay \lp \nu\rp = \lp l_0,l_1,l_2,\hdots, l_L\rp$. Assume then that the neural network is given by $\nu = \lp \lp W_1,b_1\rp, \lp W_2,b_2\rp,\hdots, \lp W_L,b_L\rp\rp$. We will denote by $c\circledast^{i,j}\nu$ as the neural network which, for $i \in \N \cap \lb 1,L-1\rb$, $j \in \N \cap \lb 1,l_i\rb$, is given by $c \circledast^{i,j} \nu = \lp \lp W_1,b_1 \rp, \lp W_2,b_2\rp, \hdots,\lp \tilde{W}_i,b_i \rp,\lp \tilde{W}_{i+1},b_{i+1}\rp,\hdots \lp W_L,b_L\rp\rp$ where it is the case that:
|
|
\begin{align}
|
|
\tilde{W}_i = \lp \mymathbb{k}^{j,j,c-1}_{l_i,l_{i}} + \mathbb{I}_{l_i}\rp W_i
|
|
\end{align}
|
|
\end{definition}
|
|
\begin{definition}[The Dropout Operator]
|
|
Let $\nu \in \neu$, $L\in \N$, $i_1,i_2,\hdots, i_k,j,k \in \N$, and $c_1,c_2,\hdots,c_k\in \R$. Assume also that $\lay \lp \nu\rp = \lp l_0,l_1,l_2,\hdots, l_L\rp$. Assume then that the neural network is given by $\nu = \lp \lp W_1,b_1\rp, \lp W_2,b_2\rp,\hdots, \lp W_L,b_L\rp\rp$. We will denote by $\dropout_n^{\unif}\lp \nu \rp$ the neural network that is given by:
|
|
\begin{align}
|
|
0\circledast^{i_1,j_1} \lp 0 \circledast^{i_2,j_2}\lp \hdots 0\circledast^{i_n,j_n}\nu \hdots \rp\rp
|
|
\end{align}
|
|
Where for each $k \in \{1,2,\hdots,n \}$ it is the case that $i \sim \unif \{ 1,L-1\}$ and $j\sim \unif\{1,l_j\} $
|
|
\end{definition}
|
|
|
|
We will also define the dropout operator introduced in \cite{srivastava_dropout_2014}.
|
|
|
|
|
|
\begin{definition}[Realization with dropout]
|
|
Let $\nu \in \neu$, $L,n \in \N$, $p \in \lp 0,1\rp$, $\lay \lp \nu\rp = \lp l_0,l_1,\hdots, \l_L\rp$, and that $\neu = \lp \lp W_1,b_1\rp, \lp W_2,b_2\rp, \hdots , \lp W_L,b_L\rp \rp$. Let it be the case that for each $n\in \N$, $\rho_n = \{ x_1,x_2,\hdots,x_n\} \in \R^n$ where for each $i \in \{1,2,\hdots,n\}$ it is the case that $x_i \sim \bern(p)$. We will then denote $\real_{\rect}^{D} \lp \nu \rp \in C\lp \R^{\inn\lp \nu\rp},\R^{\out\lp \nu \rp}\rp$, the continuous function given by:
|
|
\begin{align}
|
|
\real_{\rect}^D\lp \nu \rp = \rho_{l_L}\odot \rect \lp W_l\lp \rho_{l_{L-1}} \odot \rect \lp W_{L-1}\lp \hdots\rp + b_{L-1}\rp\rp + b_L\rp
|
|
\end{align}
|
|
\end{definition}
|
|
|
|
|
|
|
|
|
|
|
|
|