diff --git a/.DS_Store b/.DS_Store index 2a669bb..b5a38f7 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/Dissertation/Brownian_motion_monte_carlo.tex b/Dissertation/Brownian_motion_monte_carlo.tex index b442c79..d5f2f40 100644 --- a/Dissertation/Brownian_motion_monte_carlo.tex +++ b/Dissertation/Brownian_motion_monte_carlo.tex @@ -483,3 +483,23 @@ Note that $\mathfrak{C}_{d,\mathfrak{N}_{d,\epsilon},\mathfrak{N}_{d,\epsilon}}$ % \end{align} \end{proof} + + + + + + + + + + + + + + + + + + + + diff --git a/Dissertation/ann_product.tex b/Dissertation/ann_product.tex index 2b3e5f1..1344cea 100644 --- a/Dissertation/ann_product.tex +++ b/Dissertation/ann_product.tex @@ -8,6 +8,7 @@ We will build up the tools necessary to approximate $e^x$ via neural networks in \item whether their parameter estimates are bounded at most polynomially on the type of accuracy we want, $\ve$. \item The accuracy of our neural networks. \end{enumerate} +The sections pertaining to squaring and taking the product of neural networks derive mostly from \cite{yarotsky_error_2017} via \cite{bigbook}. \subsection{The squares of real numbers in $\lb 0,1 \rb$} One of the most important operators we will approximate is the product operator $\times$ for two real numbers. The following sections takes a streamlined version of the proof given in \cite[Section~3.1]{grohs2019spacetime}. In particular we will assert the existence of the neural network $\Phi$ and $\phi_d$ and work our way towards its properties. \begin{definition}[The $\mathfrak{i}_d$ Network]\label{def:mathfrak_i} diff --git a/Dissertation/ann_rep_brownian_motion_monte_carlo.tex b/Dissertation/ann_rep_brownian_motion_monte_carlo.tex index a9dbc96..c9df940 100644 --- a/Dissertation/ann_rep_brownian_motion_monte_carlo.tex +++ b/Dissertation/ann_rep_brownian_motion_monte_carlo.tex @@ -4,7 +4,7 @@ We will now take the modified and simplified version of Multi-level Picard intro \begin{lemma}[R\textemdash,2023] Let $d,M \in \N$, $T \in (0,\infty)$ , $\act \in C(\R,\R)$, $ \Gamma \in \neu$, satisfy that $\real_{\act} \lp \mathsf{G}_d \rp \in C \lp \R^d, \R \rp$, for every $\theta \in \Theta$, let $\mathcal{U}^\theta: [0,T] \rightarrow [0,T]$ and $\mathcal{W}^\theta:[0,T] \rightarrow \R^d$ be functions , for every $\theta \in \Theta$, let $U^\theta: [0,T] \rightarrow \R^d \rightarrow \R$ satisfy satisfy for all $t \in [0,T]$, $x \in \R^d$ that: \begin{align} - U^\theta(t,x) = \frac{1}{M} \lb \sum^M_{k=1} \lp \real_{\act} \lp \Gamma \rp \rp \lp x+ \mathcal{W}^{\lp \theta,0,-k \rp } \rp \rb + U^\theta(t,x) = \frac{1}{M} \lb \sum^M_{k=1} \lp \real_{\act} \lp \mathsf{G}_d \rp \rp \lp x+ \mathcal{W}^{\lp \theta,0,-k \rp } \rp \rb \end{align} Let $\mathsf{U}^\theta_t \in \neu$ , $\theta \in \Theta$ satisfy for all $\theta \in \Theta$, $t \in [0,T]$ that: \begin{align} @@ -79,6 +79,7 @@ Items (ii)--(iii) together shows that for all $\theta \in \Theta$, $t \in [0,T]$ \end{align} This proves Item (v) and hence the whole lemma. \end{proof} +While we realize that the modified Multi-Level {Picard may approximate solutions to non-linear PDEs we may chose a more circuitous route. It is quite possible, now that we have networks $\pwr_n^{q,\ve}$, to approximate polynomials using these networks. Once we have polynomials we may approximate more sophisticated PDEs. \section{The $\mathsf{E}^{N,h,q,\ve}_n$ Neural Networks} \begin{lemma}[R\textemdash, 2023]\label{mathsfE} Let $n, N\in \N$ and $h \in \lp 0,\infty\rp$. Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$, satisfy that $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. Let $a\in \lp -\infty,\infty \rp$, $b \in \lb a, \infty \rp$. Let $f:[a,b] \rightarrow \R$ be continuous and have second derivatives almost everywhere in $\lb a,b \rb$. Let $a=x_0 \les x_1\les \cdots \les x_{N-1} \les x_N=b$ such that for all $i \in \{0,1,...,N\}$ it is the case that $h = \frac{b-a}{N}$, and $x_i = x_0+i\cdot h$ . Let $x = \lb x_0 \: x_1\: \cdots \: x_N \rb$ and as such let $f\lp\lb x \rb_{*,*} \rp = \lb f(x_0) \: f(x_1)\: \cdots \: f(x_N) \rb$. Let $\mathsf{E}^{N,h,q,\ve}_{n} \in \neu$ be the neural network given by: @@ -332,7 +333,7 @@ This proves Item (v) and hence the whole lemma. % Text Node \draw (122,262.4) node [anchor=north west][inner sep=0.75pt] [font=\LARGE] {$\vdots $}; % Text Node -\draw (41,250.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Cpy}_{n}{}_{,}{}_{1}$}; +\draw (41,250.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Sum}_{n}{}_{,}{}_{1}$}; \end{tikzpicture} @@ -791,7 +792,7 @@ Let $t \in \lp 0,\infty\rp$ and $T \in \lp t,\infty\rp$. Let $\lp \Omega, \mathc It is then the case that for all $\fX \in \R^{\fn \lp N+1\rp} \times \R^{\fn d}$: \begin{enumerate}[label = (\roman*)] - \item $\real_{\rect} \lp \mathsf{UES}^{N,h,q,\ve}_{n,\mathsf{G}_d, \Omega, \fn} \rp \in C \lp \R^{\mathfrak{n}\lp N+1 \rp}\times \R^{\mathfrak{n} d}, \R \rp$ + \item $\real_{\rect} \lp \mathsf{UES}^{N,h,q,\ve}_{n,\mathsf{G}_d, \Omega, \fn} \rp \lp \fX \rp\in C \lp \R^{\mathfrak{n}\lp N+1 \rp}\times \R^{\mathfrak{n} d}, \R \rp$ \item $\dep \lp \mathsf{UES}^{N,h,q,\ve}_{n,\mathsf{G}_d, \Omega, \fn}\rp \les \begin{cases} \frac{q}{q-2} \lb \log_2 \lp \ve^{-1}\rp +q \rb +\dep \lp \mathsf{G}_d\rp-1 &:n = 0\\ \frac{q}{q-2} \lb \log_2 \lp \ve^{-1}\rp +q \rb +\max\left\{\dep \lp \mathsf{E}^{N,h,q,\ve}_{n}\rp,\dep \lp \mathsf{G}_d\rp\right\}-1 &:n \in \N\\ @@ -802,7 +803,7 @@ Let $t \in \lp 0,\infty\rp$ and $T \in \lp t,\infty\rp$. Let $\lp \Omega, \mathc \end{align} \item It is also the case that: \begin{align} - &\left| \frac{1}{\mathfrak{n}}\lb \sum^{\mathfrak{n}}_{i=1}\lb \exp \lp \int_t^T f\lp \mathcal{X}^{d,t,x}_{r,\omega_i}\rp ds \cdot u_d^T\lp \mathcal{X}^{d,t,x}_{r,\omega_i}\rp\rp\rb \rb - \real_{\rect}\lp \mathsf{UES}^{N,h,q,\ve}_{n,\mathsf{G}_d,\Omega, \fn}\rp\right| \nonumber\\ + &\left| \frac{1}{\mathfrak{n}}\lb \sum^{\mathfrak{n}}_{i=1}\lb \exp \lp \int_t^T f\lp \mathcal{X}^{d,t,x}_{r,\omega_i}\rp ds \cdot u_d^T\lp \mathcal{X}^{d,t,x}_{r,\omega_i}\rp\rp\rb \rb - \real_{\rect}\lp \mathsf{UES}^{N,h,q,\ve}_{n,\mathsf{G}_d,\Omega, \fn}\rp \lp \fX \rp\right| \nonumber\\ &\les 3\ve +2\ve \left| \fu^T_d\lp x\rp\right|^q+2\ve \left| \exp \lp \int^b_afdx\rp\right|^q + \ve \left| \exp \lp \int^b_afdx\rp - \mathfrak{e}\right|^q -\mathfrak{e}\fu^T_d\lp x \rp\nonumber \end{align} Where, as per Lemma \ref{mathsfE}, $\mathfrak{e}$ is defined as: @@ -828,8 +829,7 @@ Let $t \in \lp 0,\infty\rp$ and $T \in \lp t,\infty\rp$. Let $\lp \Omega, \mathc \begin{align} &\param \lp \sm_{\mathfrak{n},1}\bullet\lb \boxminus_{i=1}^{\mathfrak{n}} \mathsf{UEX}^{N,h,q,\ve}_{n,\mathsf{G}_d,\omega_i}\rb\rp \les \param \lp \boxminus_{i=1}^{\mathfrak{n}} \mathsf{UEX}^{N,h,q,\ve}_{n,\mathsf{G}_d,\omega_i} \rp \nonumber\\ &\les \mathfrak{n}^2\cdot \param \lp \mathsf{UEX}^{N,h,q,\ve}_{n,\mathsf{G}_d,\omega_i}\rp \nonumber\\ - &\les \fn^2 \cdot \lb \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +324+ 48n\right. \nonumber\\ &\left. +24 \wid_{\hid\lp \mathsf{G}_d\rp}\lp \mathsf{G}_d\rp + 4\max \left\{\param \lp \mathsf{E}^{N,h,q,\ve}_{n}\rp, \param \lp \mathsf{G}_d\rp \right\} \rb - \end{align} +\ \end{align} and therefore that: \begin{align} &\param \lp \frac{1}{\mathfrak{n}} \triangleright\lp \sm_{\mathfrak{n},1}\bullet\lb \boxminus_{i=1}^{\mathfrak{n}} \mathsf{UEX}^{N,h,q,\ve}_{n,\mathsf{G}_d, \omega_i}\rb\rp \rp \nonumber\\ @@ -867,7 +867,7 @@ Let $t \in \lp 0,\infty\rp$ and $T \in \lp t,\infty\rp$. Let $\lp \Omega, \mathc Let $N,n,\fn \in \N$, $h,\ve \in \lp 0,\infty\rp$, $q\in\lp 2,\infty\rp$, given $\mathsf{UES}^{N,h,q,\ve}_{n,\mathsf{G}_d, \Omega, \fn} \subsetneq \neu $, it is then the case that: \begin{align} &\lp \E\lb \left| \E \lb \exp\lp \int^T_t \alpha_d \circ \cX^{d,t,x}_{r,\Omega } ds\rp \cdot \fu_d^T\lp \cX^{d,t,x}_{r,\Omega}\rp\rb \right.\right.\right.\nonumber\\ &\left. \left.\left.-\frac{1}{\mathfrak{n}}\lp \sum^{\mathfrak{n}}_{i=1}\lb \exp \lp \int_t^T \alpha_d \circ \mathcal{X}^{d,t,x}_{r,\omega_i} ds \rp \cdot \fu_d^T\lp \mathcal{X}^{d,t,x}_{r,\omega_i}\rp\rb \rp \right|^2\rb\rp^{\frac{1}{2}} \nonumber \\ - &\les \frac{\fk_p }{n^{\frac{1}{2}}} \cdot \fL \lp T+1\rp \exp \lp LT\rp \lb \sup_{s\in \lb 0,T\rb} \lp \E \lb \lp 1+\left\| x + \cW_s\right\|^p\rp^2\rb\rp^{\frac{1}{2}}\rb + &\les \frac{\fk_2 }{\fn^{\frac{1}{2}}} \cdot \fL \lp T+1\rp \exp \lp LT\rp \lb \sup_{s\in \lb 0,T\rb} \lp \E \lb \lp 1+\left\| x + \cW_s\right\|^p\rp^2\rb\rp^{\frac{1}{2}}\rb \end{align} \end{corollary} @@ -881,13 +881,13 @@ Let $t \in \lp 0,\infty\rp$ and $T \in \lp t,\infty\rp$. Let $\lp \Omega, \mathc Note that \cite[Corollary~3.8]{hutzenthaler_strong_2021} tells us that: \begin{align}\label{kk_application} &\lp \E\lb \left| \E \lb \exp\lp \int^T_t \alpha_d \circ \cX^{d,t,x}_{r,\Omega } ds\rp \cdot \fu_d^T\lp \cX^{d,t,x}_{r,\Omega}\rp\rb \right.\right.\right.\nonumber\\ &\left. \left.\left.-\frac{1}{\mathfrak{n}}\lp \sum^{\mathfrak{n}}_{i=1}\lb \exp \lp \int_t^T \alpha_d \circ \mathcal{X}^{d,t,x}_{r,\omega_i} ds \rp \cdot \fu_d^T\lp \mathcal{X}^{d,t,x}_{r,\omega_i}\rp\rb \rp \right|^2\rb\rp^{\frac{1}{2}} \nonumber \\ - &\les \frac{\fk_p }{n^{\frac{1}{2}}} \lp \E \lb \left| \exp\lp \int^T_t \alpha_d \circ \cX^{d,t,x}_{r,\Omega } ds\rp \cdot \fu_d^T\lp\cX^{d,t,x}_{r,\Omega}\rp \right|^2\rb \rp^{\frac{1}{2}} + &\les \frac{\fk_2}{\fn^{\frac{1}{2}}} \lp \E \lb \left| \exp\lp \int^T_t \alpha_d \circ \cX^{d,t,x}_{r,\Omega } ds\rp \cdot \fu_d^T\lp\cX^{d,t,x}_{r,\Omega}\rp \right|^2\rb \rp^{\frac{1}{2}} \end{align} For the purposes of this proof let it be the case that $\ff: [0,T] \rightarrow \R$ is the function represented for all $t \in \lb 0,T \rb$ as: \begin{align} \ff\lp t\rp = \int^T_{T-t} \alpha_d\circ \cX^{d,t,x}_{r,\Omega} ds \end{align} - In which case we have that $\ff\lp 0\rp = 0$, and thus, stipulating $g\lp x\rp = \fu^T\lp \cX^{d,t,x}_{r,\Omega}\rp$ we may define $u\lp t,x\rp$ as the function given by: + In which case we haved that $\ff\lp 0\rp = 0$, and thus, stipulating $g\lp x\rp = \fu^T\lp \cX^{d,t,x}_{r,\Omega}\rp$ we may define $u\lp t,x\rp$ as the function given by: \begin{align} u\lp t,x\rp &= \exp \lp \ff\lp t\rp\rp \cdot g\lp x\rp \nonumber\\ &= \lb \exp\lp \ff\lp 0\rp\rp + \int_0^s \ff'\lp s\rp\cdot \exp \lp \ff\lp s\rp\rp ds\rb \cdot g\lp x\rp\nonumber \\ diff --git a/Dissertation/main.bib b/Dissertation/main.bib index 8f34dba..d4cc59b 100644 --- a/Dissertation/main.bib +++ b/Dissertation/main.bib @@ -813,6 +813,23 @@ year = {2021} year={2016} } +@article{yarotsky_error_2017, + title = {Error bounds for approximations with deep {ReLU} networks}, + volume = {94}, + issn = {0893-6080}, + url = {https://www.sciencedirect.com/science/article/pii/S0893608017301545}, + doi = {10.1016/j.neunet.2017.07.002}, + abstract = {We study expressive power of shallow and deep neural networks with piece-wise linear activation functions. We establish new rigorous upper and lower bounds for the network complexity in the setting of approximations in Sobolev spaces. In particular, we prove that deep ReLU networks more efficiently approximate smooth functions than shallow networks. In the case of approximations of 1D Lipschitz functions we describe adaptive depth-6 network architectures more efficient than the standard shallow architecture.}, + urldate = {2024-03-22}, + journal = {Neural Networks}, + author = {Yarotsky, Dmitry}, + month = oct, + year = {2017}, + keywords = {Approximation complexity, Deep ReLU networks}, + pages = {103--114}, + file = {ScienceDirect Snapshot:/Users/shakilrafi/Zotero/storage/4HS3Z6ZE/S0893608017301545.html:text/html;Submitted Version:/Users/shakilrafi/Zotero/storage/C6KQ6BFJ/Yarotsky - 2017 - Error bounds for approximations with deep ReLU net.pdf:application/pdf}, +} + diff --git a/Dissertation/main.pdf b/Dissertation/main.pdf index 11d56a2..e978354 100644 Binary files a/Dissertation/main.pdf and b/Dissertation/main.pdf differ diff --git a/Dissertation/neural_network_introduction.tex b/Dissertation/neural_network_introduction.tex index abfb857..f81c448 100644 --- a/Dissertation/neural_network_introduction.tex +++ b/Dissertation/neural_network_introduction.tex @@ -631,6 +631,65 @@ Affine neural networks present an important class of neural networks. By virtue \begin{remark} For an \texttt{R} implementation, see Listing \ref{nn_sum}. \end{remark} +\begin{remark} + We may diagrammatically refer to this network as: + + \begin{figure}[h] + \begin{center} + +\tikzset{every picture/.style={line width=0.75pt}} %set default line width to 0.75pt + +\begin{tikzpicture}[x=0.75pt,y=0.75pt,yscale=-1,xscale=1] +%uncomment if require: \path (0,433); %set diagram left start at 0, and has height of 433 + +%Shape: Rectangle [id:dp9509582141653736] +\draw (470,170) -- (540,170) -- (540,210) -- (470,210) -- cycle ; +%Shape: Rectangle [id:dp042468147108538634] +\draw (330,100) -- (400,100) -- (400,140) -- (330,140) -- cycle ; +%Shape: Rectangle [id:dp46427980442406214] +\draw (330,240) -- (400,240) -- (400,280) -- (330,280) -- cycle ; +%Straight Lines [id:da8763809527154822] +\draw (470,170) -- (401.63,121.16) ; +\draw [shift={(400,120)}, rotate = 35.54] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da9909123473315302] +\draw (470,210) -- (401.63,258.84) ; +\draw [shift={(400,260)}, rotate = 324.46] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da8497218496635237] +\draw (570,190) -- (542,190) ; +\draw [shift={(540,190)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Shape: Rectangle [id:dp11197066111784415] +\draw (210,170) -- (280,170) -- (280,210) -- (210,210) -- cycle ; +%Straight Lines [id:da5201326815013356] +\draw (330,120) -- (281.41,168.59) ; +\draw [shift={(280,170)}, rotate = 315] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da4370325799656589] +\draw (330,260) -- (281.41,211.41) ; +\draw [shift={(280,210)}, rotate = 45] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da012890543438617508] +\draw (210,190) -- (182,190) ; +\draw [shift={(180,190)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; + +% Text Node +\draw (481,182.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Cpy}_{n}{}_{,}{}_{k}$}; +% Text Node +\draw (351,110.4) node [anchor=north west][inner sep=0.75pt] {$\nu _{1}$}; +% Text Node +\draw (351,252.4) node [anchor=north west][inner sep=0.75pt] {$\nu _{2}$}; +% Text Node +\draw (574,180.4) node [anchor=north west][inner sep=0.75pt] {$x$}; +% Text Node +\draw (441,132.4) node [anchor=north west][inner sep=0.75pt] {$x$}; +% Text Node +\draw (437,232.4) node [anchor=north west][inner sep=0.75pt] {$x$}; +% Text Node +\draw (221,180.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Sum}_{n}{}_{,}{}_{k}$}; + + +\end{tikzpicture} +\end{center} +\caption{Neural Network diagram of a neural network sum.} +\end{figure} +\end{remark} \subsection{Neural Network Sum Properties} \begin{lemma}\label{paramsum} @@ -1084,6 +1143,72 @@ Affine neural networks present an important class of neural networks. By virtue This is a consequence of a finite number of applications of Lemma \ref{lem:diamondplus}. This proves the Lemma. \end{proof} +\begin{remark} + We may represent this kind of sum as the neural network diagram shown below: + +\begin{figure}[h] +\begin{center} + +\tikzset{every picture/.style={line width=0.75pt}} %set default line width to 0.75pt + +\begin{tikzpicture}[x=0.75pt,y=0.75pt,yscale=-1,xscale=1] +%uncomment if require: \path (0,433); %set diagram left start at 0, and has height of 433 + +%Shape: Rectangle [id:dp9509582141653736] +\draw (470,170) -- (540,170) -- (540,210) -- (470,210) -- cycle ; +%Shape: Rectangle [id:dp042468147108538634] +\draw (200,100) -- (400,100) -- (400,140) -- (200,140) -- cycle ; +%Shape: Rectangle [id:dp46427980442406214] +\draw (330,240) -- (400,240) -- (400,280) -- (330,280) -- cycle ; +%Straight Lines [id:da8763809527154822] +\draw (470,170) -- (401.63,121.16) ; +\draw [shift={(400,120)}, rotate = 35.54] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da9909123473315302] +\draw (470,210) -- (401.63,258.84) ; +\draw [shift={(400,260)}, rotate = 324.46] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da8497218496635237] +\draw (570,190) -- (542,190) ; +\draw [shift={(540,190)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Shape: Rectangle [id:dp11197066111784415] +\draw (80,170) -- (150,170) -- (150,210) -- (80,210) -- cycle ; +%Straight Lines [id:da5201326815013356] +\draw (200,130) -- (151.56,168.75) ; +\draw [shift={(150,170)}, rotate = 321.34] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da4370325799656589] +\draw (330,260) -- (312,260) ; +\draw [shift={(310,260)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Straight Lines [id:da012890543438617508] +\draw (80,190) -- (52,190) ; +\draw [shift={(50,190)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; +%Shape: Rectangle [id:dp2321426611089945] +\draw (200,240) -- (310,240) -- (310,280) -- (200,280) -- cycle ; +%Straight Lines [id:da03278204116412775] +\draw (200,260) -- (151.41,211.41) ; +\draw [shift={(150,210)}, rotate = 45] [color={rgb, 255:red, 0; green, 0; blue, 0 } ][line width=0.75] (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29) ; + +% Text Node +\draw (481,182.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Cpy}_{n}{}_{,}{}_{k}$}; +% Text Node +\draw (301,110.4) node [anchor=north west][inner sep=0.75pt] {$\nu _{1}$}; +% Text Node +\draw (351,252.4) node [anchor=north west][inner sep=0.75pt] {$\nu _{2}$}; +% Text Node +\draw (574,180.4) node [anchor=north west][inner sep=0.75pt] {$x$}; +% Text Node +\draw (441,132.4) node [anchor=north west][inner sep=0.75pt] {$x$}; +% Text Node +\draw (437,232.4) node [anchor=north west][inner sep=0.75pt] {$x$}; +% Text Node +\draw (91,180.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Sum}_{n}{}_{,}{}_{k}$}; +% Text Node +\draw (238,252.4) node [anchor=north west][inner sep=0.75pt] {$\mathsf{Tun}$}; + + +\end{tikzpicture} +\caption{Neural network diagram of a neural network sum of unequal depth networks.} +\end{center} +\end{figure} +\end{remark} \section{Linear Combinations of ANNs and Their Properties} \begin{definition}[Scalar left-multiplication with an ANN]\label{slm} Let $\lambda \in \R$. We will denote by $(\cdot ) \triangleright (\cdot ): \R \times \neu \rightarrow \neu$ the function that satisfy for all $\lambda \in \R$ and $\nu \in \neu$ that $\lambda \triangleright \nu = \aff_{\lambda \mathbb{I}_{\out(\nu)},0} \bullet \nu$. diff --git a/MLP and DNN Material/.DS_Store b/MLP and DNN Material/.DS_Store index d3433ca..0e7ac27 100644 Binary files a/MLP and DNN Material/.DS_Store and b/MLP and DNN Material/.DS_Store differ diff --git a/MLP and DNN Material/HJKP21/HJKP21.pdf b/MLP and DNN Material/HJKP21/HJKP21.pdf index a0848bf..ace5b43 100644 Binary files a/MLP and DNN Material/HJKP21/HJKP21.pdf and b/MLP and DNN Material/HJKP21/HJKP21.pdf differ diff --git a/Templates/.DS_Store b/Templates/.DS_Store index 713dfaa..7393d66 100644 Binary files a/Templates/.DS_Store and b/Templates/.DS_Store differ