\chapter{ANN Product Approximations and Their Consequences}\label{chp:ann_prod}

\section{Approximation for Products of Two Real Numbers}
We will build up the tools necessary to approximate $e^x$ via neural networks in the framework described in the previous sections. While much of the foundation comes from, e.g., \cite{grohs2019spacetime} way, we will, along the way, encounter neural networks not seen in the literature, such as the $\tay$, $\pwr$, $\tun$, and finally a neural network approximant for $e^x$. For each of these neural networks, we will be concerned with at least the following:
\begin{enumerate}[label = (\roman*)]
	\item whether their instantiations using the ReLU function (often just continuous functions) are continuous.
	\item whether their depths are bounded, at most polynomially, on the type of accuracy we want, $\ve$.
	\item whether their parameter estimates are bounded at most polynomially on the type of accuracy we want, $\ve$.
	\item The accuracy of our neural networks.
\end{enumerate}
The sections pertaining to squaring and taking the product of neural networks derive mostly from \cite{yarotsky_error_2017} via \cite{bigbook}. 
\subsection{The squares of real numbers in $\lb 0,1 \rb$}
One of the most important operators we will approximate is the product operator $\times$ for two real numbers. The following sections takes a streamlined version of the proof given in \cite[Section~3.1]{grohs2019spacetime}. In particular we will assert the existence of the neural network $\Phi$ and $\phi_d$ and work our way towards its properties. 
\begin{definition}[The $\mathfrak{i}_d$ Network]\label{def:mathfrak_i}
	For all $d \in \N$ we will define the following set of neural networks as ``activation neural networks'' denoted $\mathfrak{i}_d$ as:
	\begin{align}
		\mathfrak{i}_d = \lp \lp \mathbb{I}_d, \mymathbb{0}_d\rp, \lp \mathbb{I}_d, \mymathbb{0}_d\rp \rp 
	\end{align}
\end{definition}
\begin{lemma}\label{lem:mathfrak_i}
	Let $d \in \N$. It is then the case that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \mathfrak{i}_4\rp \in C \lp \R^d, \R^d\rp$.
		\item $\lay \lp \mathfrak{i}_d\rp = \lp d,d,d\rp$
		\item $\param \lp \mathfrak{i}_4\rp = 2d^2+2d$
	\end{enumerate}
\end{lemma}
\begin{proof}
	Item (i) is straightforward from the fact that for all $d \in \N$ it is the case that  $\real_{\rect} \lp \mathfrak{i}_d\rp = \mathbb{I}_d\lp \real_{\rect} \lp \lb \mathbb{I}_d\rb_*\rp + \mymathbb{0}_d\rp + \mymathbb{0}_d$. Item (ii) is straightforward from the fact that $\mathbb{I}_d \in \R^{d \times d}$. We realize Item (iii) by observation.
\end{proof}


\begin{lemma}\label{lem:6.1.1}\label{lem:phi_k}
	Let $\lp c_k \rp _{k \in \N} \subseteq \R$, $\lp A_k \rp _{k \in \N} \in \R^{4 \times 4},$ $\mathbb{B}\in \R^{4 \times 1}$, $\lp C_k \rp _{k\in \N}$ satisfy for all $k \in \N$ that:
	\begin{align}\label{(6.0.1)}
		A_k = \begin{bmatrix}
			2 & -4 &2 & 0 \\
			2 & -4 & 2 & 0\\
			2 & -4 & 2 & 0\\
			-c_k & 2c_k & -c_k & 1
		\end{bmatrix} \quad B=\begin{bmatrix}
			0 \\ -\frac{1}{2} \\ -1 \\ 0
		\end{bmatrix} \quad C_k = \begin{bmatrix}
			-c_k & 2c_k &-c_k & 1
		\end{bmatrix}
	\end{align}
	and that:
	\begin{align}
		c_k = 2^{1-2k}
	\end{align}
	Let $\Phi_k \in \neu$, $k\in \N$ satisfy for all $k \in [2,\infty) \cap \N$ that $\Phi_1 = \lp \aff_{C_1,0} \bullet \mathfrak{i}_4 \rp \bullet \aff_{\mymathbb{e}_4,B}$, that for all $d \in \N$, $\mathfrak{i}_d = \lp \lp \mathbb{I}_d, \mymathbb{0}_d \rp, \lp \mathbb{I}_d, \mymathbb{0}_d \rp \rp$  and that:
	\begin{align}
		\Phi_k =\lp \aff_{C_k,0}\bullet \mathfrak{i}_4 \rp \bullet \lp \aff_{A_{k-1},B} \bullet \mathfrak{i}_4\rp \bullet \cdots \bullet \lp  \aff_{A_1,B} \bullet \mathfrak{i}_4 \rp \bullet \aff_{\mymathbb{e}_4,B} ,
	\end{align}
	It is then the case that:
	\begin{enumerate}[label = (\roman*)]
	\item for all $k \in \N$, $x \in \R$ we have $\real_{\rect}\lp \Phi_k\rp\lp x \rp \in C \lp \R, \R \rp $
	\item for all $k \in \N$ we have $\lay \lp \Phi_k \rp = \lp 1,4,4,...,4,1 \rp \in \N^{k+2}$
	\item for all $k \in \N$, $x \in \R \setminus \lb 0,1 \rb $ that $\lp \real_{\rect} \lp \Phi_k \rp \rp \lp x \rp = \rect \lp x \rp$
	\item for all $k \in \N$, $x \in \lb 0,1 \rb$, we have $\left| x^2 - \lp \real_{\rect} \lp \Phi_k \rp \rp \lp x \rp \right| \les 2^{-2k-2}$, and 
	\item for al $k \in \N$ , we have that $\param \lp \Phi_k \rp = 20k-7$ 
	\end{enumerate} 
\end{lemma}
\begin{proof}
	Firstly note that Lemma \ref{aff_prop}, Lemma \ref{comp_prop}, and Lemma \ref{lem:mathfrak_i}
	 ensure that for all $k \in \N$, $x \in \R$ it is the case that $\real_{\rect}\lp \Phi_k\rp \lp x\rp \in C\lp \R, \R\rp$. This proves Item (i).
	 
	 Note next that Lemma \ref{aff_prop}, Lemma \ref{lem:mathfrak_i}, and Lemma \ref{comp_prop} tells us that:
	 \begin{align}
	 	\lay \lp \Phi_1 \rp = \lay \lp \aff_{\mymathbb{e}_4},B\rp = \lp 1,4,1\rp
	 \end{align}
	 and for all $k \in \N$ it is the case that:
	 \begin{align}
	 	\lay \lp \aff_{A_k,B} \bullet \mathfrak{i}_4\rp = \lp 4,4,4,4\rp
	 \end{align}
	
	 Whence it is straightforward to see that for $\Phi_k$ where $k \in \N \cap \lb 2,\infty \rp$, Lemma \ref{comp_prop} tells us then that:
	 \begin{align}
	 	\lay \lp \Phi_k\rp &= \lay \lp \lp \aff_{C_k,0}\bullet \mathfrak{i}_4 \rp \bullet \lp \aff_{A_{k-1},B} \bullet \mathfrak{i}_4\rp \bullet \cdots \bullet \lp  \aff_{A_1,B} \bullet \mathfrak{i}_4 \rp \bullet \aff_{\mymathbb{e}_4,B} \rp \nonumber\\
	 	&= (1,\overbrace{4) \: \overbrace{( 4}^{merged},4,4,\overbrace{4) \:( 4}^{merged},4,4,\overbrace{4)\: }^{merged}\hdots \overbrace{\: ( 4}^{merged},4,4,\overbrace{4) \:}^{merged} (4}^{k-1 \text{ many}},1)
	 \end{align}
	 This thus finally yields that:
	 \begin{align}
	 	\lay \lp \Phi_k\rp = \lp 1,4,4,\hdots, 4,1\rp \in \N^{k+2}
	 \end{align}





	Let $g_k: \R \rightarrow \lb 0,1 \rb$, $k \in \N$ be the functions defined as such, satisfying for all $k \in \N$, $x \in \R$ that:
	\begin{align}\label{(6.0.3)}
		g_1 \lp x \rp &= \begin{cases}
			2x & : x \in \lb 0,\frac{1}{2} \rp \\
			2-2x &: x\in \lb \frac{1}{2},1\rb \\
			0 &: x \in \R \setminus \lb 0,1 \rb 
		\end{cases} \\
		g_{k+1} &= g_1(g_{k}) \nonumber
	\end{align}
	and let $f_k: \lb 0,1 \rb \rightarrow \lb 0,1 \rb$, $k \in \N_0$ be the functions satisfying for all $k \in \N_0$, $n \in \{0,1,\hdots,2^k-1\}$, $x \in \lb \frac{n}{2^k}, \frac{n+1}{2^k} \rp$ that $f_k(1)=1$ and:
	\begin{align}\label{(6.0.4.2)}
		f_k(x) = \lb \frac{2n+1}{2^k} \rb x-\frac{n^2+n}{2^{2k}}
	\end{align} 
	and let $r_k = \lp r_{k,1},r_{k,2},r_{k,3},r_{k,4} \rp: \R \rightarrow \R^4$, $k \in \N$ be the functions which which satisfy for all $x \in \R$, $k \in \N$ that:
	\begin{align}\label{(6.0.5)}
		r_1\lp x \rp &= \begin{bmatrix}
			r_{1,1}(x) \\ r_{2,1}(x) \\ r_{3,1}(x) \\ r_{4,1}(x)
		\end{bmatrix}= \rect  \lp \begin{bmatrix}
			x \\ x-\frac{1}{2} \\ x-1 \\ x 
		\end{bmatrix} \rp  \\
		r_{k+1} &= \rect \lp A_{k+1}r_k(x) +B \rp \nonumber
	\end{align}
	Note that since it is the case that for all $x \in \R$ that $\rect(x) = \max\{x,0\}$, (\ref{(6.0.3)}) and (\ref{(6.0.5)}) shows that it holds for all $x \in \R$ that:
	\begin{align}\label{6.0.6}
		2r_{1,1}(x) -4r_{2,1}(x) + 2r_{3,1}(x) &= 2 \rect(x) -4\rect \lp x-\frac{1}{2}\rp+2\rect\lp x-1\rp \nonumber \\
		&= 2\max\{x,0\} -4\max\left\{x-\frac{1}{2} ,0\right\}+2\max\{x-1,0\} \nonumber \\
		&=g_1(x) 
	\end{align}
	Note also that combined with (\ref{(6.0.4.2)}), the fact that for all $x\in [0,1]$ it holds that $f_0(x) = x = \max\{x,0\}$ tells us that for all $x \in \R$:
	\begin{align}\label{6.0.7}
		r_{4,1}(x) = \max \{x,0\} = \begin{cases}
			f_0(x) & :x\in [0,1] \\
			\max\{x,0\}& :x \in \R \setminus \lb 0,1\rb 
		\end{cases}
	\end{align} 
	We next claim that for all $k \in \N$, it is the case that:
	\begin{align}\label{6.0.8}
		\lp \forall x \in \R : 2r_{1,k}(x)-4r_{2,k}(x) + 2r_{3,k}(x) =g(x) \rp 
	\end{align}
	and that:
	\begin{align}\label{6.0.9}
		\lp \forall x \in \R: r_{4,k} (x) = \begin{cases}
			f_{k-1}(x) & :x \in \lb 0,1 \rb \\
			\max\{x,0\} & : x \in \R \setminus \lb 0,1\rb 
		\end{cases} \rp 
	\end{align}
	We prove (\ref{6.0.8}) and (\ref{6.0.9}) by induction. The base base of $k=1$ is proved by (\ref{6.0.6}) and (\ref{6.0.7}) respectively. For the induction step $\N \ni k \rightarrow k+1$ assume there does exist a $k \in \N$ such that for all $x \in \R$ it is the case that:
	\begin{align}
		2r_{1,k}(x) - 4r_{2,k}(x) + 2r_{3,k}(x) = g_k(x)
	\end{align}
	and:
	\begin{align}\label{6.0.11}
		r_{4,k}(x) = \begin{cases}
			f_{k-1}(x) & : x \in [0,1] \\
			\max\{x,0\} &: x \in \R \setminus \lb 0,1 \rb 
		\end{cases}
	\end{align}
	Note that then (\ref{(6.0.3)}), (\ref{(6.0.5)}), and (\ref{6.0.6}) then tells us that for all $x \in \R$ it is the case that:
	\begin{align}\label{6.0.12}
		g_{k+1}\lp x \rp &= g_1(g_k(x)) = g_1(2r_{1,k}(x)+4r_{2,k}(x) + 2r_{3,k}(x)) \nonumber \\
		&= 2\rect \lp 2r_{1,k}(x)) + 4r_{2,k} +2r_{3,k}(x) \rp \nonumber \\
		&-4\rect \lp 2r_{1,k}\lp x \rp -4r_{2,k}+2r_{3,k}(x) - \frac{1}{2} \rp \nonumber \\
		&+ 2\rect \lp 2r_{1,k} (x) - 4r_{2,k}(x) + 2r_{3,k}(x)-1 \rp \nonumber \\
		&=2r_{1,k+1}(x) -4r_{2,k+1}(x) + 2r_{3,k+1}(x)
	\end{align} 
	In addition note that (\ref{(6.0.4.2)}), (\ref{(6.0.5)}), and (\ref{6.0.7}) tells us that for all $x \in \R$:
	%TODO: Ask about the extra powers of 2 and b_k
	\begin{align}\label{6.0.13}
		r_{4,k+1}(x) &= \rect \lp \lp -2 \rp ^{3-2 \lp k+1 \rp }r_{1,k} \lp x \rp + 2^{4-2 \lp k+1 \rp}r_{2,k} \lp x \rp  + \lp -2 \rp^{3-2\lp k+1\rp }r_{3,k} \lp x \rp  + r_{4,k} \lp x\rp \rp \nonumber \\
		&= \rect \lp \lp -2 \rp ^{1-2k}r_{1,k} \lp x \rp + 2^{2-2k}r_{k,2}\lp x \rp + \lp -2 \rp ^{1-2k}r_{3,k} \lp x \rp + r_{4,k}\lp x \rp \rp \nonumber \\
		&=\rect \lp 2^{-2k} \lb -2r_{1,k}\lp x \rp + 2^2r_{2,k} \lp x \rp -2r_{3,k} \lp x \rp \rb +r_{4,k}\lp x \rp \rp \nonumber \\
		&= \rect \lp - \lb 2^{-2k} \rb \lb 2r_{1,k}\lp x \rp -4r_{2,k} \lp x \rp +2r_{3,k}\lp x \rp \rb +r_{4,k}\lp x \rp \rp \nonumber \\
		&= \rect\lp -\lb 2^{-2k} \rb g_k \lp x \rp +r_{4,k}\lp x \rp \rp 
	\end{align}
	This and the fact that for all $x\in \R$ it is the case that $\rect \lp x \rp = \max\{x,0\}$, that for all $x\in \lb 0 ,1 \rb$ it is the case that $f_k \lp x \rp \ges 0$, (\ref{6.0.11}), shows that for all $x \in \lb 0,1 \rb$ it holds that:
	\begin{align}\label{6.0.14}
		r_{4,k+1}\lp x \rp &= \rect \lp -2 \lb 2^{-2k} g_k \rb + f_{k-1}\lp x \rp  \rp = \rect \lp -2 \lp 2^{-2k}g_k \lp x \rp \rp +x-\lb \sum^{k-1}_{j=1} \lp 2^{-2j}g_j \lp x \rp \rp \rb \rp \nonumber \\
		&= \rect \lp x - \lb \sum^k_{j=1}2^{-2j}g_j \lp x \rp \rb \rp = \rect \lp f_k \lp x \rp \rp =f_k \lp x \rp 
	\end{align}
	Note next that (\ref{6.0.11}) and (\ref{6.0.13}) then tells us that for all $x\in \R \setminus \lb 0,1\rb$:
	\begin{align}
		r_{4,k+1}\lp x \rp = \max \left\{ -\lp 2^{-2k}g_x \lp x \rp \rp + r_{4,k}\lp x \rp \right\} = \max\{\max\{x,0\},0\} = \max\{x,0\}
	\end{align}
	Combining (\ref{6.0.12}) and (\ref{6.0.14}) proves (\ref{6.0.8}) and (\ref{6.0.9}). Note that then (\ref{(6.0.1)}) and (\ref{6.0.8}) assure that for all $k\in \N$, $x\in \R$ it holds that $\real_{\rect} \lp \Phi_k \rp \in C \lp \R,\R \rp$  and that:
	\begin{align}\label{(6.0.17)}
		&\lp \real_{\rect} \lp \Phi_k \rp \rp \lp x \rp \nonumber \\
		&= \lp \real_{\rect} \lp \lp \aff_{C_k,0} \bullet \mathfrak{i}_4 \rp \bullet \lp \aff_{A_{k-1},B} \bullet \mathfrak{i}_4 \rp \bullet \cdots \bullet\lp \aff_{A_1,B} \bullet \mathfrak{i}_4 \rp \bullet \aff_{\mymathbb{e}_4,B} \rp \rp \lp x \rp \nonumber \\
		&= \lp -2\rp^{1-2k}r_{1,k}\lp x \rp + 2^{2-2k} r_{2,k} \lp x \rp + \lp -2 \rp ^{1-2k} r_{3,k} \lp x \rp + r_{4,k} \lp x \rp \nonumber \\
		&=\lp -2 \rp ^{2-2k} \lp \lb \frac{r_{1,k}\lp x \rp +r_{3,k} \lp x \rp }{-2} \rb + r_{2,k}\lp x \rp \rp +r_{4,k}\lp x \rp \nonumber \\
		&=2^{2-2k} \lp \lb \frac{r_{1,k}\lp x \rp+r_{3,k} \lp x \rp }{-2} \rb + r_{2,k} \lp x \rp \rp +r_{4,k} \lp x \rp \nonumber \\
		&=2^{-2k}\lp 4r_{2,k} \lp x \rp -2r_{1,k}\lp x \rp -2r_{3,k} \lp x \rp \rp +r_{4,k} \lp x \rp \nonumber \\
		&=-\lb 2^{-2k} \rb \lb 2r_{1,k} \lp x \rp -4r_{2,k} \lp x \rp +2r_{3,k} \lp x \rp \rb +r_{4,k} \lp x \rp = -\lb 2^{-2k} \rb g_k \lp x \rp + r_{4,k} \lp x \rp 
	\end{align}
	This and (\ref{6.0.9}) tell us that:
	\begin{align}
		\lp \real_{\rect} \lp \Phi_k \rp \rp \lp x \rp = - \lp 2^{-2k}g_k \lp x \rp \rp +f_{k-1}\lp x \rp &= -\lp 2^{-2k}g_k \lp x \rp \rp +x-\lb \sum^{k-1}_{j=1} 2^{-2j}g_j \lp x \rp \rb \nonumber \\
		&=x-\lb \sum^k_{j=1}2^{-2j}g_j \lp x \rp \rb =f_k\lp x\rp \nonumber
	\end{align}
	Which then implies for all $k\in \N$, $x \in \lb 0,1\rb$ that it holds that:
	\begin{align}
		\left| x^2-\lp \real_{\rect} \lp \Phi_k \rp \rp \lp x \rp \right| \les 2^{-2k-2}
	\end{align}
	This, in turn, establishes Item (i). 
	
	Finally observe that (\ref{(6.0.17)}) then tells us that for all $k\in \N$, $x \in \R \setminus \lb 0,1\rb$ it holds that:
	\begin{align}
		\lp \real_{\rect} \lp \Phi_k \rp \rp \lp x \rp = -2^{-2k}g_k \lp x \rp +r_{4,k} \lp x \rp =r_{4,k} \lp x \rp = \max\{x,0\} = \rect(x)
	\end{align}
	This establishes Item(iv). Note next that Item(iii) ensures for all $k\in \N$ that $\dep\lp \xi_k \rp = k+1$, and:
	\begin{align}
		\param \lp \Phi_k \rp = 4(1+1) + \lb \sum^k_{j=2} 4 \lp 4+1\rp \rb + \lp 4+1 \rp =8+20\lp k-1\rp+5 = 20k-7
	\end{align}
	This, in turn, proves Item(vi). The proof of the lemma is thus complete. 
\end{proof}
\begin{remark}
	For an \texttt{R} implementation see Listing \ref{Phi_k}
\end{remark}

\begin{figure}[h]
	\includegraphics[width = \linewidth]{/Users/shakilrafi/R-simulations/Phi_k_properties/diff.png}
	\caption{Plot of $\log_{10}$ of the $L^1$ difference between $\Phi_k$ and $x^2$ over $\lb 0,1\rb$ for different values of $k$}
\end{figure}

\begin{corollary}\label{6.1.1.1}\label{cor:phi_network}
	Let $\ve \in \lp 0,\infty\rp$, $M= \min \{ \frac{1}{2}\log_2 \lp \ve^{-1} \rp -1,\infty\}\cap \N$, $\lp c_k\rp_{k \in \N} \subseteq \R$, $\lp A_k\rp_{k\in\N} \subseteq \R^{4 \times 4}$, $B \in \R^{4\times 1}$, $\lp C_k\rp_{k\in \N}$ satisfy for all $k \in \N$ that:
	\begin{align}
		A_k = \begin{bmatrix}
			2&-4&2&0 \\
			2&-4&2&0\\
			2&-4&2&0\\
			-c_k&2c_k & -c_k&1
		\end{bmatrix}, \quad B = \begin{bmatrix}
			0\\ -\frac{1}{2}\quad \\ -1 \\ 0
		\end{bmatrix}\quad C_k = \begin{bmatrix}
			-c_k &2c)_k&-c_k&1
		\end{bmatrix}
	\end{align}
	where:
	\begin{align}
		c_k = 2^{1-2k}
	\end{align}
	and let $\Phi \in \neu$ be defined as:
	\begin{align}
		\Phi = \begin{cases}\label{def:Phi}
			\lb \aff_{C_1,0}\bullet \mathfrak{i}_4\rb \bullet \aff_{\mymathbb{e}_4,B} & :M=1 \\
			\lb \aff_{C_M,0} \bullet \mathfrak{i}_4\rb\bullet \lb \aff_{A_{M-1},0} \bullet \mathfrak{i}_4 \rb \bullet \cdots \bullet \lb \aff_{A_1,B}\bullet \mathfrak{i}_4\rb \bullet \aff_{\mymathbb{e}_4,B} &: M \in \lb 2,\infty \rp \cap \N
		\end{cases}
	\end{align}
	it is then the case that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \Phi\rp \in C \lp \R,\R\rp$
		\item $\lay \lp \Phi\rp = \lp 1,4,4,...,4,1\rp \in \N^{M+2} $
		\item it holds for all $x \in \R \setminus\lb 0,1 \rb$ that $\lp \real_{\rect} \lp \Phi\rp\rp \lp x \rp = \rect(x)$
		\item it holds for all $x \in \lb 0,1 \rb$ that $\left| x^2 - \lp \real_{\rect} \lp \Phi \rp \rp\lp x \rp \right| \les 2^{-2M-2} \les \ve$
		\item $\dep \lp \Phi \rp \les M+1 \les \max\{ \frac{1}{2}\log_2 \lp \ve^{-1}\rp+1,2\}$, and
		\item $\param \lp \Phi\rp = 20M-7 \les \max\left\{ 10\log_2 \lp \ve^{-1}\rp-7,13\right\}$
	\end{enumerate}
	\end{corollary}
	\begin{proof}
		Items (i)--(iii) are direct consequences of Lemma \ref{lem:6.1.1}, Items (i)--(iii). Note next the fact that $M = \min \left\{\N \cap \lb \frac{1}{2} \log_2 \lp \ve^{-1}\rp-1\rb,\infty\right\}$ ensures that:
		\begin{align}
			&M = \min \left\{ \N \cap \lb \frac{1}{2}\log_2\lp \ve^{-1}\rp-1\rb, \infty\right\}\\ &\ges \min \left\{ \lb\max \left\{ 1,\frac{1}{2}\log_2 \lp\ve^{-1} \rp-1\right\},\infty \rb\right\}\\ &\ges \frac{1}{2}\log_2 \lp \ve^{-1}\rp-1
		\end{align}
		This and Item (v) of Lemma \ref{lem:6.1.1} demonstrate that for all $x\in \lb 0,1\rb$ it then holds that:
		\begin{align}
			\left| x^2 - \lp \real_{\rect}\lp \Phi\rp\rp \lp x\rp \right| \les 2^{-2M-2} = 2^{-2(M+1)} \les 2^{-\log_2\lp\ve^{-1} \rp} = \ve
		\end{align}
		Thus establishing Item (iv). The fact that $M = \min \left\{ \N \cap \lb \frac{1}{2}\log_2 \lp \ve^{-1}\rp -1,\infty\rb\right\}$ and Item (ii) of Lemma \ref{lem:6.1.1} tell us that:
		\begin{align}
			\dep \lp \Phi \rp = M+1 \les \max \left\{ \frac{1}{2} \log_2 \lp \ve^{-1}\rp+1,2\right\}
		\end{align}
		Which establishes Item(v). This and Item (v) of Lemma \ref{lem:6.1.1} then tell us that:
		\begin{align}
			\param \lp \Phi_M\rp \les 20M-7 \les 20 \max\left\{ \frac{1}{2}\log_2\lp\ve^{-1}\rp,2\right\}-7 = \max\left\{ 10\log_2 \lp\ve^{-1} \rp-7,13\right\}
		\end{align}
		This completes the proof of the corollary. 
	\end{proof}
\begin{remark}
	For an implementation in \texttt{R}, see Listing \ref{Phi}
\end{remark}
\begin{figure}[h]
	\centering
	\includegraphics[width = \linewidth]{/Users/shakilrafi/R-simulations/Phi_properties/Phi_diff_contour.png}
	\caption{Contour plot of the $L^1$ difference between $\Phi$ and $x^2$ over $\lb 0,1 \rb$ for different values of $\ve$.}
\end{figure}
\begin{remark}
	Note that (\ref{def:Phi}) implies that $\dep \lp \Phi \rp \ges 4$.
\end{remark}
Now that we have neural networks that perform the squaring operation inside $\lb -1,1\rb$, we may extend to all of $\R$. Note that this neural network representation differs somewhat from the ones in \cite{grohs2019spacetime}. 

\subsection{The $\sqr^{q,\ve}$ Neural Networks and Squares of Real Numbers}
\begin{lemma}\label{6.0.3}\label{lem:sqr_network}
	Let $\delta,\epsilon \in (0,\infty)$, $\alpha \in (0,\infty)$, $q\in (2,\infty)$, $ \Phi \in \neu$ satisfy that $\delta = 2^{\frac{-2}{q-2}}\ve ^{\frac{q}{q-2}}$, $\alpha = \lp \frac{\ve}{2}\rp^{\frac{1}{q-2}}$, $\real_{\rect}\lp\Phi\rp \in C\lp \R,\R\rp$, $\dep(\Phi) \les \max \left\{\frac{1}{2} \log_2(\delta^{-1})+1,2\right\}$, $\param(\Phi) \les \max\left\{10\log_2\lp \delta^{-1}\rp \right.\\\left. -7,13\right\}$, $\sup_{x \in \R \setminus [0,1]} | \lp \real_{\rect} \lp \Phi \rp -\rect(x) \right| =0$, and $\sup_{x\in \lb 0,1\rb} |x^2-\lp \real_{\rect} \lp \Phi \rp \rp \lp x\rp | \les \delta$, let $\Psi \in \neu$ be the neural network given by:
	\begin{align}
		 \Psi = \lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{\alpha,0} \rp \bigoplus\lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{-\alpha,0}\rp 
	\end{align}
	\begin{enumerate}[label = (\roman*)]
		\item it holds that $\real_{\rect} \lp \Psi \rp \in C \lp \R,\R \rp$.
		\item it holds that $\lp \real_{\rect} \lp \Psi \rp \rp \lp 0\rp=0$
		\item it holds for all $x\in \R$ that $0\les \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp  \les \ve + |x|^2$
		\item it holds for all $x \in \R$ that $|x^2-\lp \real_{\rect} \lp \Psi  \rp \rp \lp x \rp |\les \ve \max\{1,|x|^q\}$
		\item it holds that $\dep (\Psi)\les \max\left\{1+\frac{1}{q-2}+\frac{q}{2(q-2)}\log_2 \lp \ve^{-1} \rp,2\right\}$, and
		\item it holds that $\param\lp \Psi \rp \les \max\left\{ \lb \frac{40q}{q-2} \rb \log_2 \lp \ve^{-1} \rp +\frac{80}{q-2}-28,52 \right\}$ 
	\end{enumerate}
\end{lemma}
\begin{proof}
	Note that for all $x\in \R$ it is the case that:
	\begin{align}\label{6.0.21}
		\lp \real_{\rect}\lp \Psi  \rp \rp\lp x \rp &= \lp \real_{\rect} \lp \lp  \aff_{\alpha^{-2}}\bullet \Phi \bullet \aff_{\alpha,0}\rp  \oplus\lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{-\alpha,0} \rp \rp \rp \lp x \rp \nonumber\\
		&=  \lp \real_{\rect}\lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{\alpha,0} \rp \rp \lp x\rp + \lp \real_{\rect}\lp \aff_{\alpha^{-2},0} \bullet  \Phi \bullet \aff_{-\alpha,0}\rp \rp \lp x\rp \nonumber \\
		&= \frac{1}{\alpha^2}\lp \real_{\rect}\lp \Phi \rp \rp \lp \alpha x\rp + \frac{1}{\alpha^2}\lp \real_{\rect} \lp \Phi \rp \rp \lp -\alpha x\rp \nonumber\\
		&= \frac{1}{\lp \frac{\ve}{2}\rp^{\frac{2}{q-2}}}\lb \lp \real_{\rect}\lp \Phi \rp \rp \lp \lp \frac{\ve}{2}\rp ^{\frac{1}{q-2}}x \rp + \lp \real_{\rect}\lp \Phi \rp \rp \lp -\lp \frac{\ve}{2}\rp^{\frac{1}{q-2}}x\rp \rb 
	\end{align}
	This and the assumption that $\Phi \in C\lp \R, \R \rp$ along with the assumption that $\sup_{x\in \R \setminus \lb 0,1\rb } \left| \lp \real_{\rect} \lp \Phi \rp \rp \right.\\ \left.\lp x \rp -\rect\lp x\rp \right| =0$ tells us that for all $x\in \R$ it holds that:
	\begin{align}
		\lp \real_{\rect}\lp \Psi \rp \rp \lp 0 \rp &= \lp \frac{\ve}{2}\rp^{\frac{-2}{q-2}}\lb \lp \real_{\rect}\lp \Phi \rp \rp  \lp 0 \rp +\lp \real_{\rect} \lp \Phi\rp \rp \lp 0 \rp \rb \nonumber \\
		&=\lp \frac{\ve}{2}\rp ^{\frac{-2}{q-2}} \lb \rect (0)+\rect(0) \rb \nonumber \\
		&=0
	\end{align} 
	This, in turn, establishes Item (i)--(ii). Observe next that from the assumption that $\real_{\rect} \lp \Phi \rp \in C\lp \R,\R \rp$ and the assumption that $\sup_{x\in \R \setminus \lb 0,1\rb} | \lp \real_{\rect}\lp \Phi \rp \rp \lp x \rp -\rect(x) |=0$ ensure that for all $x\in \R \setminus \lb -1,1 \rb$ it holds that:
	\begin{align}\label{6.0.23}
		\lb \real_{\rect}\lp \Phi \rp \rb \lp x\rp + \lb \real_{\rect}\lp \Phi \rp  \lp -x \rp\rb   = \rect\lp x\rp +\rect(-x) &= \max\{x,0\}+\max\{-x,0\} \nonumber\\
		&=|x|
	\end{align}
	The assumption that for all $\sup_{x\in \R \setminus \lb 0,1\rb }|\lp \real_{\rect} \lp \Phi \rp \rp \lp x\rp -\rect\lp x\rp |=0$ and the assumption that $\sup_{x\in\lb 0,1\rb} |x^2-\lp \real_{\rect} \lp \Phi \rp \rp \lp x\rp |\les \delta$ show that:
	\begin{align}\label{6.0.24}
		&\sup_{x \in \lb -1,1\rb} \left|x^2 - \lp \lb \real_{\rect}\lp \Phi \rp \rb \lp x\rp +\lb \real_{\rect}\lp \Phi \rp \lp x \rp \rb \rp \right| \nonumber \\
		&= \max\left\{ \sup_{x\in \lb -1,0 \rb} \left| x^2-\lp \rect(x)+ \lb \real_{\rect}\lp \Phi \rp \rb \lp -x \rp \rp \right|,\sup _{x\in \lb 0,1 \rb} \left| x^2-\lp \lb \real_{\rect} \lp \Phi \rp \rb \lp x \rp + \rect \lp -x \rp \rp \right| \right\} \nonumber\\
		&= \max\left\{\sup_{x\in \lb -1,0 \rb}\left|\lp -x \rp^2 - \lp \real_{\rect}\lp \Phi \rp \rp \lp -x \rp \right|, \sup_{x\in \lb 0,1\rb} \left| x^2-\lp \real_{\rect} \lp \Phi \rp \rp \lp x \rp \right| \right\} \nonumber \\
		&=\sup_{x\in \lb 0,1 \rb}\left| x^2 - \lp \real_{\rect}\lp \Phi \rp \rp \lp x\rp \right| \les \delta 
	\end{align}
	Next observe that (\ref{6.0.21}) and (\ref{6.0.23}) show that for all $x \in \R \setminus \lb -\lp \frac{\ve}{2} \rp^{\frac{-1}{q-2}}, \lp \frac{\ve}{2}\rp ^{\frac{-1}{q-2}} \rb$ it holds that:
	\begin{align}\label{6.0.25}
		0 \les \lb \real_{\rect} \lp \Psi \rp \rb \lp x \rp &= \lp \frac{\ve}{2} \rp ^{\frac{-2}{q-2}}\lp \lb \real_{\rect} \lp \Phi \rp \rb \lp \lp \frac{\ve}{2}\rp ^{\frac{1}{q-2}}x \rp + \lb \real_{\rect} \lp \Phi \rp \rb \lp -\lp \frac{\ve}{2}\rp^{\frac{1}{q-2}} x\rp \rp \nonumber \\
		&= \lp \frac{\ve}{2} \rp ^{\frac{-2}{q-2}} \left| \lp \frac{\ve}{2} \rp^{\frac{1}{q-2}}x \right| = \lp \frac{\ve}{2} \rp^{\frac{-1}{q-2}|x|} \les |x|^2
	\end{align}
	The triangle inequality then tells us that for all $x\in \R \setminus \lb - \lp \frac{\ve}{2} \rp^{\frac{-1}{q-2}}, \lp \frac{\ve}{2} \rp^{\frac{-1}{q-2}} \rb$ it holds that:
	\begin{align} \label{6.0.25}
		\left| x^2- \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp \right| &= \left| x^2 - \lp \frac{\ve}{2} \rp ^{\frac{-1}{q-2}}\left|x\right| \right| \les \lp \left|x \right|^2 + \lp \frac{\ve}{2} \rp ^{\frac{-1}{q-2}} \left| x \right| \rp   \nonumber\\
		&= \lp \left| x \right|^q \left|x\right|^{-(q-2)} + \lp \frac{\ve}{2} \rp^{\frac{-1}{q-2}} \left| x \right|^q\left| x \right|^{-(q-1)} \rp \nonumber \\
		&\les \lp \left| x \right|^q \lp \frac{\ve}{2} \rp^{\frac{q-2}{q-2}} + \lp \frac{\ve}{2} \rp ^{\frac{-1}{q-2}} \left| x \right|^q \lp \frac{\ve}{2} \rp ^{\frac{q-1}{q-2}} \rp \nonumber \\
		&= \lp \frac{\ve}{2}+ \frac{\ve}{2} \rp \left| x \right|^q = \ve \left| x \right|^q \les \ve \max \left\{ 1, \left| x \right|^q \right\} 
	\end{align}

Note that (\ref{6.0.24}), (\ref{6.0.21}) and the fact that $\delta = 2^{\frac{-2}{q-2}}\ve^{\frac{q}{q-2}}$ then tell for all $x \in \\ \lb -\lp \frac{\ve}{2} \rp ^{\frac{-1}{q-2}}, \lp \frac{\ve}{2} \rp ^{\frac{-1}{q-2}} \rb$ it holds that:
\begin{equation}
\begin{aligned}\label{6.0.26}
%	&\left| x^2-\lp \real_{\rect} \lp \Phi \rp \rp \lp x \rp \right| \\
%	&= \lp \frac{\ve}{2} \rp^{\frac{-2}{q-2}} \left| \lp \lp \frac{\ve}{2} \rp ^{\frac{1}{q-2}}x \rp^2 - \lp \lb \real_{\rect} \lp \Phi \rp \rb \lp \lp \frac{\ve}{2} \rp ^{\frac{1}{q-2}}x \rp + \lb \real_{\rect} \lp \Phi \rp \rb \lp -y \rp \rp \right| \\
%	&\les \lp \frac{\ve}{2} \rp^{\frac{-2}{q-2}} \lb \sup_{y \in \lb -1,1\rb} \left| y^2 - \left \lb \real_{\rect} \lp \Phi \rp \rb \lp y \rp + \lb \real_{\rect} \lp \Phi \rp \rb \lp -y \rp \right| \rb \\
	&\left| x^2-\left( \real_{\rect} (\Phi) \right) (x) \right| \\
	&= \left( \frac{\varepsilon}{2} \right)^{\frac{-2}{q-2}} \left| \left( \left( \frac{\varepsilon}{2} \right) ^{\frac{1}{q-2}}x \right)^2 - \left( \left[ \real_{\rect} (\Phi) \right] \left( \left( \frac{\varepsilon}{2} \right) ^{\frac{1}{q-2}}x \right) + \left[ \real_{\rect} (\Phi) \right] (-y) \right) \right| \\
	&\les \left( \frac{\varepsilon}{2} \right)^{\frac{-2}{q-2}} \left[ \sup_{y \in \left[-1,1\right]} \left| y^2 - \left[ \real_{\rect} (\Phi) \right] (y) + \left[ \real_{\rect} (\Phi) \right] (-y) \right| \right] \\
	&\les \lp \frac{\ve}{2} \rp^{\frac{-2}{q-2}} \delta = \lp \frac{\ve}{2} \rp^{\frac{-2}{q-2}} 2^{\frac{-2}{q-2}} \ve^{\frac{q}{q-2}} = \ve \les \ve \max \{ 1, \left| x \right|^q \}
\end{aligned}
\end{equation}
Now note that this and (\ref{6.0.25}) tells us that for all $x\in \R$ it is the case that:
\begin{align}
	\left| x^2-\lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp \right| \les \ve \max\{1,|x|^q \}
\end{align}
This establishes Item (v). Note that, (\ref{6.0.26}) tells that for all $x \in \lb - \lp \frac{\ve}{2} \rp ^{\frac{-1}{q-2}}, \lp \frac{\ve}{2} \rp ^{\frac{1}{q-2}} \rb $ it is the case that:
\begin{align}
	\left| \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp \right| \les \left| x^2 - \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp \right| + \left| x \right|^2 \les \ve + \left| x \right| ^2
\end{align}
This and (\ref{6.0.25}) tells us that for all $x\in \R$:
\begin{align}
	\left| \lp \real_{\rect} \rp \lp x \rp \right| \les \ve + |x|^2  
\end{align}
This establishes Item (iv). 

Note next that by Corollary \ref{affcor}, Remark \ref{5.3.2}, the hypothesis, and the fact that $\delta = 2^{\frac{-2}{q-2}}\ve ^{\frac{q}{q-2}}$ tells us that:
\begin{align}
	\dep \lp \Psi \rp = \dep \lp \Phi \rp &\les \max \left\{\frac{1}{2} \log_2(\delta^{-1})+1,2\right\} \nonumber \\
	&= \max \left\{  \frac{1}{q-2} + \lb \frac{q}{q-2}\rb\log_2 \lp \ve \rp +1,2\right\}
\end{align}
This establishes Item (v). 

Notice next that the fact that $\delta = 2^{\frac{-2}{q-2}}\ve^{\frac{q}{q-2}}$ tells us that:
\begin{align}
	\log_2 \lp \delta^{-1} \rp = \log_2 \lp 2^{\frac{2}{q-2}} \ve^{\frac{-q}{q-2}}\rp = \frac{2}{q-2} + \lb \lb \frac{q}{q-2}\rb \log_2 \lp \ve^{-1}\rp  \rb
\end{align}
Note that by , Corollary \ref{affcor} we have that:
\begin{align}
	\param \lp \Phi \bullet \aff_{-\alpha,0} \rp &\les \lb \max\left\{ 1, \frac{\inn \lp \aff_{-\alpha,0}\rp+1}{\inn\lp \Phi\rp+1}\right\}\rb \param \lp \Phi\rp = \param \lp \Phi\rp 
\end{align}
and further that:
\begin{align}
	\param \lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{-\alpha,0} \rp &= \lb \max\left\{ 1, \frac{\out \lp \aff_{-\alpha^2,0}\rp}{\out\lp \Phi \bullet \aff_{-\alpha,0}\rp}\right\}\rb \param \lp \Phi \bullet \aff_{-\alpha,0}\rp \nonumber\\
	&\les \param \lp \Phi\rp
\end{align}
By symmetry note also that $ \param \lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{\alpha,0}\rp = \param \lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{-\alpha,0}\rp $ and also that $ \lay \lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{\alpha,0}\rp = \lay \lp \aff_{\alpha^{-2},0} \bullet \Phi \bullet \aff_{-\alpha,0}\rp $. Thus Lemma \ref{paramsum}, Corollary \ref{cor:sameparal}, and the hypothesis tells us that:
\begin{align}\label{(6.1.42)}
	\param \lp \Psi \rp &= \param \lp \Phi \boxminus \Phi \rp \nonumber \\
	&\les 4\param \lp \Phi\rp \nonumber \\
	&= 4\max\left\{10\log_2\lp \delta^{-1}\rp-7,13\right\}
\end{align}
This, and the fact that $\delta = 2^{\frac{-2}{q-2}}\ve ^{\frac{q}{q-2}}$ renders (\ref{(6.1.42)}) as:
\begin{align}
	4\max\left\{10\log_2\lp \delta^{-1}\rp-7,13\right\} &= 4\max\left\{10\log_2\lp \delta^{-1}\rp-7,13\right\} \nonumber\\
	&= 4\max \left\{ 10 \lp \frac{2}{q-2} +\frac{q}{q-2}\log_2 \lp \ve^{-1}\rp\rp-7,13\right\} \nonumber \\
	&=\max \left\{ \lb \frac{40q}{q-2}\rb \log_2 \lp \ve^{-1}\rp + \frac{80}{q-2}-28,52\right\}
\end{align}
\end{proof}
\begin{remark}
	We will often find it helpful to refer to this network for fixed $\ve \in \lp 0, \infty \rp$ and $q \in \lp 2,\infty\rp$ as the $\sqr^{q,\ve}$ network.
\end{remark}
\begin{remark}
	For an \texttt{R} implementation see Listing \ref{Sqr}
\end{remark}



\begin{figure}[h]
	\centering
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Sqr_properties/experimental_deps.png}
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Sqr_properties/dep_theoretical_upper_limits.png}
		\caption{Left: $\log_{10}$ of depths for a simulation with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points. Right: The theoretical upper limits over the same range of values}
\end{figure}

\begin{figure}[h]
	\centering
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Sqr_properties/experimental_params.png}
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Sqr_properties/param_theoretical_upper_limits.png}
		\caption{Left: $\log_{10}$ of params for a simulation with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points. Right: The theoretical upper limits over the same range of values}
\end{figure}

% Please add the following required packages to your document preamble:
% \usepackage{booktabs}

\begin{table}[h]
\begin{tabular}{@{}l|llllll@{}}
\toprule
           & Min.   & 1\textsuperscript{st} Qu. & Median & Mean    & 3\textsuperscript{rd} Qu.  & Max. \\ \midrule
Experimental $|x^2 - \real_{\rect}(\mathsf{Sqr}^{q,\ve})(x)$                 & 0.00000 & 0.08943 & 0.33787 & 3.14893 & 4.67465 & 20.00 \\ \midrule
Theoretical  $|x^2 - \real_{\rect}(\mathsf{Sqr})^{q,\ve}(x)$ & 0.010    & 1.715    & 10.402   & 48.063   & 45.538   & 1250.00  \\ \midrule
Forward Difference & 0.01 & 1.6012  & 9.8655 & 44.9141 & 40.7102 & 1230
\end{tabular}
\caption{Theoretical upper bounds for $L^1$ error, experimental $L^1$ error and their forward difference, with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points.}
\end{table}


\subsection{The $\prd^{q,\ve}$ Neural Networks and Products of Two Real Numbers}
We are finally ready to give neural network representations of arbitrary products of real numbers. However, this representation differs somewhat from those found in the literature, especially \cite{grohs2019spacetime}, where parallelization (stacking) is used instead of neural network sums. This will help us calculate $\wid_1$ and the width of the second to last layer.
\begin{lemma}\label{prd_network}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$, $A_1,A_2,A_3 \in \R^{1\times 2}$, $\Psi \in \neu$ satisfy for all $x\in \R$ that $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$, $A_1 = \lb 1 \quad 1 \rb$, $A_2 = \lb 1 \quad 0 \rb$, $A_3 = \lb 0 \quad 1 \rb$, $\real_{\rect} \in C\lp \R, \R \rp$, $\lp \real_{\rect} \lp \Psi \rp \rp \lp 0\rp = 0$, $0\les \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp \les \delta+|x|^2$, $|x^2-\lp \real_{\rect}\lp \Psi \rp \rp \lp x \rp |\les \delta \max \{1,|x|^q\}$, $\dep\lp \Psi \rp \les \max\{ 1+\frac{1}{q-2}+\frac{q}{2(q-2)}\log_2 \lp \delta^{-1} \rp ,2\}$, and $\param \lp \Psi \rp \les \max\left\{\lb \frac{40q}{q-2} \rb \log_2\lp \delta^{-1} \rp +\frac{80}{q-2}-28,52\right\}$, then:
	\begin{enumerate}[label=(\roman*)]
		\item there exists a unique $\Gamma \in \neu$ satisfying:
		\begin{align}
			\Gamma = \lp \frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_1,0} \rp \rp \bigoplus \lp \lp -\frac{1}{2}\rp \triangleright\lp \Psi \bullet \aff_{A_2,0} \rp \rp \bigoplus\lp \lp -\frac{1}{2}\rp \triangleright \lp \Psi \bullet \aff_{A_3,0} \rp \rp 
		\end{align}
		\item it  that $\real_{\rect} \lp \Gamma \rp \in C \lp \R^2,\R \rp$
		\item it holds for all $x\in \R$ that $\lp \real_{\rect}\lp \Gamma \rp \rp \lp x,0\rp = \lp \real_{\rect}\lp \Gamma \rp \rp \lp 0,y\rp  =0$
		\item it holds for any $x,y \in \R$ that $\left|xy - \lp \real_{\rect} \lp \Gamma \rp \rp \lp \begin{bmatrix}
			x \\
			y
		\end{bmatrix} \rp \right| \les \ve \max \{1,|x|^q,|y|^q \}$
		\item it holds that $\param(\Gamma) \les \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb -252$
		\item it holds that $\dep\lp \Gamma \rp \les \frac{q}{q-2} \lb \log_2 \lp \ve^{-1}\rp +q \rb $ 
		\item it holds that $\wid_1 \lp \Gamma \rp=24$
		\item it holds that $\wid_{\hid \lp\Gamma\rp} = 24$
	\end{enumerate}
\end{lemma}
\begin{proof}
	Note that:
	\begin{align}
		&\lp \real_{\rect} \lp \Gamma \rp \rp \lp \begin{bmatrix}
			x\\y
		\end{bmatrix} \rp =  \real_{\rect} \lp \lp \frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_1,0} \rp \rp \bigoplus \lp \lp -\frac{1}{2}\rp \triangleright\lp \Psi \bullet \aff_{A_2,0} \rp \rp \bigoplus \right. \\
		&\left. \lp \lp -\frac{1}{2}\rp \triangleright \lp \Psi \bullet \aff_{A_3,0} \rp \rp \rp \nonumber \lp \begin{bmatrix}
			x \\ y
		\end{bmatrix} \nonumber\rp\\
		&= \real_{\rect} \lp \frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_1,0} \rp \rp \lp \begin{bmatrix}
			x\\y 
		\end{bmatrix} \rp  + \real_{\rect}\lp \lp -\frac{1}{2}\rp \triangleright\lp \Psi \bullet \aff_{A_2,0} \rp \rp \lp \begin{bmatrix}
			x \\ y
		\end{bmatrix} \rp \nonumber \\
		&+\real_{\rect}\lp \lp -\frac{1}{2}\rp \triangleright \lp \Psi \bullet \aff_{A_3,0} \rp \rp \lp \begin{bmatrix}
			x\\y
		\end{bmatrix} \rp \nonumber \\
		&= \frac{1}{2} \lp \real_{\rect} \lp \Psi \rp \rp \lp \begin{bmatrix}
			1 && 1
		\end{bmatrix} \begin{bmatrix}
			x \\ y
		\end{bmatrix}\rp - \frac{1}{2} \lp \real_{\rect} \lp \Psi  \rp \rp \lp \begin{bmatrix}
			1 && 0
		\end{bmatrix} \begin{bmatrix}
			x \\ y
		\end{bmatrix} \rp \nonumber\\
		&-\frac{1}{2} \lp \real_{\rect}\lp \Psi \rp \rp \lp \begin{bmatrix}
			0 && 1
		\end{bmatrix} \begin{bmatrix}
			x \\y
		\end{bmatrix} \rp 	\nonumber \\
		&=\frac{1}{2} \lp \real_{\rect}\lp \Psi \rp \rp \lp x+y \rp -\frac{1}{2} \lp \real_{\rect}\lp \Psi \rp \rp \lp x \rp - \frac{1}{2} \lp \real_{\rect}\lp \Psi \rp \rp \lp y \rp \label{6.0.33}
		%TODO: Revisit this estimate
	\end{align}
	Note that this, and the assumption that $\lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp  \in C \lp \R, \R \rp$ and that $\lp \real_{\rect}\lp \Psi \rp \rp  \lp 0 \rp = 0$ ensures:
	\begin{align}
		\lp \real_{\rect} \lp \Gamma \rp \rp \lp \begin{bmatrix}
			x \\0
		\end{bmatrix} \rp &= \frac{1}{2} \lp \real_{\rect} \lp \Psi \rp \rp \lp x+0 \rp -\frac{1}{2} \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp - \frac{1}{2} \lp \real_{\rect} \lp \Psi \rp \rp \lp 0 \rp \nonumber \\
		&= 0 \nonumber\\
		&= \frac{1}{2} \lp \real_{\rect} \lp \Psi \rp \rp \lp 0+y \rp -\frac{1}{2} \lp \real_{\rect} \lp \Psi \rp \rp \lp 0 \rp  - \frac{1}{2}\lp \real_{\rect} \lp \Psi \rp \rp \lp y \rp \nonumber \\
		&=\lp \real_{\rect} \lp \Gamma \rp \rp \lp \begin{bmatrix}
			0 \\y 
		\end{bmatrix} \rp 
	\end{align}
	Next, observe that since by assumption it is the case for all $x,y\in \R$ that $|x^2 - \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp | \les \delta \max\{1,|x|^q\}$, $xy = \frac{1}{2}|x+y|^2-\frac{1}{2}|x|^2-\frac{1}{2}|y|^2$,  triangle Inequality and from (\ref{6.0.33}) we have that:
	\begin{align}
		&\left| \lp \real_{\rect} \lp \Gamma\rp\lp x,y \rp  \rp -xy\right| \nonumber\\
		&=\left|\frac{1}{2}\lb \lp \real_{\rect} \lp \Psi \rp \rp \lp x + y \rp - \left|x+y\right|^2 \rb - \frac{1}{2} \lb \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp -\left| x \right|^2\rb  - \frac{1}{2} \lb \lp \real_{\rect} \lp \Psi\rp \rp \lp x \rp -\left|y\right|^2\rb \right| \nonumber \\
		&\les \left|\frac{1}{2}\lb \lp \real_{\rect} \lp \Psi \rp \rp \lp x + y \rp - \left|x+y\right|^2 \rb + \frac{1}{2} \lb \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp -\left| x \right|^2\rb  + \frac{1}{2} \lb \lp \real_{\rect} \lp \Psi\rp \rp \lp x \rp -\left|y\right|^2\rb \right| \nonumber \\
		&\les \frac{\delta}{2} \lb \max \left\{ 1, |x+y|^q\right\} + \max\left\{ 1,|x|^q\right\} + \max \left\{1,|y|^q \right\}\rb\nonumber
	\end{align}
	Note also that since for all $\alpha,\beta \in \R$ and $p \in \lb 1, \infty \rp$ we have that $|\alpha + \beta|^p \les 2^{p-1}\lp |\alpha|^p + |\beta|^p \rp$ we have that:
	\begin{align}
		&\left| \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp - xy \right| \nonumber \\
		&\les \frac{\delta}{2} \lb \max \left\{1, 2^{q-1}|x|^q+ 2^{q-1}\left| y\right|^q\right\} + \max\left\{1,\left|x\right|^q \right\} + \max \left\{1,\left| y \right|^q \right\}\rb \nonumber \\
		&\les \frac{\delta}{2} \lb \max \left\{1, 2^{q-1}|x|^q \right\}+ 2^{q-1}\left| y\right|^q + \max\left\{1,\left|x\right|^q \right\} + \max \left\{1,\left| y \right|^q \right\}\rb \nonumber \\
		&\les \frac{\delta}{2} \lb 2^q + 2\rb \max \left\{1, \left|x\right|^q, \left| y \right|^q \right\} = \ve \max \left\{ 1,\left| x \right|^q, \left| x \right|^q\right\} \nonumber
	\end{align} 
	This proves Item (iv). 	
	
	By symmetry it holds that $\param \lp \frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_1,0} \rp \rp \\ = \param  \lp -\frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_2,0} \rp \rp = \param \lp -\frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_3,0} \rp \rp$ and further that $\lay \lp \frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_1,0} \rp \rp = \lay  \lp -\frac{1}{2}\triangleright \lp \Psi \bullet \aff_{A_2,0} \rp \rp = \lay \lp -\frac{1}{2}\triangleright\lp \Psi \bullet \aff_{A_3,0} \rp \rp$. 
	Note also that Corollary \ref{affcor} tells us that for all $i \in \{1,2,3\}$ and $a \in \{ \frac{1}{2},-\frac{1}{2}\}$ it is the case that:
	\begin{align}
		\param \lp a \triangleright \lp \Psi \bullet \aff_{A_i,0}\rp \rp = \param \lp \Psi \rp
	\end{align}
	This, together with Corollary \ref{corsum} indicates that:
	\begin{align}\label{(6.1.49)}
		\param \lp \Gamma \rp &\les 9\param\lp \Psi \rp \nonumber \\
		&\les 9\max\left\{\lb \frac{40q}{q-2} \rb \log_2\lp \delta^{-1} \rp +\frac{80}{q-2}-28,52\right\}
	\end{align}
	Combined with the fact that $\delta =\ve \lp 2^{q-1} +1\rp^{-1}$, this is then rendered as:
	\begin{align}\label{(6.1.50)}
		&9\max\left\{\lb \frac{40q}{q-2} \rb \log_2\lp \delta^{-1} \rp +\frac{80}{q-2}-28,52\right\} \nonumber \\
		&= 9\max \left\{ \lb \frac{40q}{q-2}\rb \lp \log_2 \lp \ve^{-1}\rp  +\log_2 \lp 2^{q-1}+1\rp\rp + \frac{80}{q-2}-28,52 \right\}
	\end{align}
	Note that:
	\begin{align}
		\log_2 \lp 2^{q-1}+1\rp &= \log_2\lp 2^{q-1}+1\rp - \log_2 \lp 2^q\rp + q \nonumber\\
		&=\log_2 \lp \frac{2^{q-1}+1}{2^q}\rp + q = \log_2 \lp 2^{-1}+2^{-q}\rp +q\nonumber \\
		&\les \log_2 \lp 2^{-1} + 2^{-2}\rp + q = \log_2 \lp \frac{3}{4}\rp + q = \log_2 \lp 3\rp-2+q
	\end{align}
	Combine this with the fact that for all $q\in \lp 2,\infty\rp$ it is the case that $\frac{q(q-1)}{q-2} \ges 2$ then gives us that:
	\begin{align}
		\lb \frac{40q}{q-2}\rb \log_2 \lp 2^{q-1}+1\rp -28\ges \lb \frac{40q}{q-2}\rb \log_2 \lp 2^{q-1}\rp -28= \frac{40q(q-1)}{q-2}-28 \ges 52
	\end{align}
	This then finally renders (\ref{(6.1.50)}) as:
	\begin{align}
		&9\max \left\{ \lb \frac{40q}{q-2}\rb \lp \log_2 \lp \ve^{-1}\rp  +\log_2 \lp 2^{q-1}+1\rp\rp + \frac{80}{q-2}-28,52 \right\} \nonumber \\
		&\les 9 \lb \lb \frac{40q}{q-2}\rb \lp \log_2\lp \ve^{-1}\rp + \log_2\lp 3\rp-2+q\rp +\frac{80}{q-2}-28\rb \nonumber\\
		&= 9 \lb \lb \frac{40q}{q-2}\rb \lp \log_2\lp \ve^{-1}\rp + \log_2\lp 3\rp-2+\frac{2}{q}\rp-28\rb \nonumber\\
		&\les 9 \lb \lb \frac{40q}{q-2}\rb \lp \log_2\lp \ve^{-1}\rp + \log_2\lp 3\rp-1\rp -28\rb \nonumber\\
		&= \frac{360q}{q-2}\lb \log_2 \lp \ve^{-1} \rp +q+\log_2\lp 3\rp-1\rb -252
	\end{align}
	Note that Lemma \ref{depth_prop}, Lemma \ref{5.3.3}, the hypothesis, and the fact that $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$ tell us that:
	\begin{align}
		\dep \lp \Gamma \rp = \dep\lp \Psi \rp &\les \max\left\{ 1+\frac{1}{q-2}+\frac{q}{2(q-2)}\log_2 \lp \delta^{-1} \rp ,2\right\} \nonumber\\
		&= \max \left\{1+\frac{1}{q-2} +\frac{q}{2(q-2)}\lb \log_2\lp \ve^{-1}\rp + \log_2 \lp 2^{q-1}+1\rp\rb,2 \right\} \nonumber\\
		&= \max \left\{ 1+\frac{1}{q-2}+\frac{q}{2(q-2)} \lp \log_2\lp \ve^{-1}\rp +q-1\rp,2\right\}
	\end{align}
	Since it is the case that $\frac{q(q-1)}{2(q-2)} > 2$ for $q \in \lp 2, \infty \rp$ we have that:
	\begin{align}
		& \max \left\{ 1+\frac{1}{q-2}+\frac{q}{2(q-2)} \lp \log_2\lp \ve^{-1}\rp +q-1\rp,2\right\} \nonumber \\
		&=  1+\frac{1}{q-2}+\frac{q}{2(q-2)} \lp \log_2\lp \ve^{-1}\rp +q-1\rp \nonumber \\
		&\les \frac{q-1}{q-2} +\frac{q}{2\lp q-2\rp} \lp \log_2 \lp \ve^{-1}\rp+q\rp \nonumber \\
		&
	\end{align}
	
Observe next that for $q\in \lp 0,\infty\rp$, $\ve \in \lp 0,\infty \rp$, $\Gamma$ consists of, among other things, three stacked $\lp \Psi \bullet \aff_{A_i,0}\rp$ networks where $i \in \{1,2,3\}$. Corollary \ref{affcor} tells us therefore, that $\wid_1\lp \Gamma\rp = 3\cdot \wid_1 \lp \Psi \rp$. On the other hand, note that each $\Psi$ networks consist of, among other things, two stacked $\Phi$ networks, which by Corollary \ref{affcor} and Lemma \ref{lem:sqr_network}, yields that $\wid_1 \lp \Gamma\rp = 6 \cdot \wid_1 \lp \Phi\rp$. Finally from Corollary \ref{cor:phi_network}, and Corollary \ref{affcor}, we see that the only thing contributing to the $\wid_1\lp \Phi\rp$ is $\wid_1 \lp \mathfrak{i}_4\rp$, which was established from Lemma \ref{lem:mathfrak_i} as $4$. Whence we get that $\wid_1\lp \Gamma\rp = 6 \cdot 4 = 24$, and that $\wid_{\hid\lp \Gamma\rp}\lp \Gamma\rp = 24$. This proves Item (vii)\textemdash(viii). This then completes the proof of the Lemma. 
\end{proof}

\begin{corollary}\label{cor_prd}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$, $A_1,A_2,A_3 \in \R^{1\times 2}$, $\Psi \in \N$ satisfy for all $x\in \R$ that $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$, $A_1 = \lb 1 \quad 1 \rb$, $A_2 = \lb 1 \quad 0 \rb$, $A_3 = \lb 0 \quad 1 \rb$, $\real_{\rect} \in C\lp \R, \R \rp$, $\lp \real_{\rect} \lp \Psi \rp \rp \lp 0\rp = 0$, $0\les \lp \real_{\rect} \lp \Psi \rp \rp \lp x \rp \les \delta+|x|^2$, $|x^2-\lp \real_{\rect}\lp \Psi \rp \rp \lp x \rp |\les \delta \max \{1,|x|^q\}$, $\dep\lp \Psi \rp \les \max\{ 1+\frac{1}{q-2}+\frac{q}{2(q-2)}\log_2 \lp \delta^{-1} \rp ,2\}$, and $\param \lp \Psi \rp \les \max\left\{\lb \frac{40q}{q-2} \rb \log_2\lp \delta^{-1} \rp +\frac{80}{q-2}-28,52\right\}$, and finally let $\Gamma$ be defined as in Lemma \ref{prd_network}, i.e.:
	\begin{align}
		\Gamma = \lp \frac{1}{2}\circledast \lp \Psi \bullet \aff_{A_1,0} \rp \rp \bigoplus \lp \lp -\frac{1}{2}\rp \circledast\lp \Psi \bullet \aff_{A_2,0} \rp \rp \bigoplus\lp \lp -\frac{1}{2}\rp \circledast \lp \Psi \bullet \aff_{A_3,0} \rp \rp 
	\end{align}
	
	
	
	It is then the case for all $x,y \in \R$ that:
	\begin{align}
		\real_{\rect} \lp \Gamma \rp \lp x,y \rp \les \frac{3}{2} \lp \frac{\ve}{3} +x^2+y^2\rp \les \ve + 2x^2+2y^2
	\end{align}
\end{corollary}
\begin{proof}
	Note that the triangle inequality, the fact that $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$, the fact that for all $x,y\in \R$ it is the case that $|x+y|^2 \les 2\lp |x|^2+|y|^2\rp $ and (\ref{6.0.33}) tell us that:
	\begin{align}
		\left| \real_{\rect} \lp \Gamma \rp\lp x,y\rp  \right| &\les \frac{1}{2}\left| \real_{\rect} \lp \Psi \rp\lp x+y \rp \right| + \frac{1}{2}\left| \real_{\rect} \lp \Psi \rp\lp x \rp \right| + \frac{1}{2}\left| \real_{\rect} \lp \Psi \rp\lp y \rp \right| \nonumber \\
		&\les \frac{1}{2} \lp \delta + |x+y|^2 \rp + \frac{1}{2}\lp \delta + |x|^2\rp + \frac{1}{2}\lp \delta + |y|^2\rp\nonumber \\
		&\les \frac{3\delta}{2} +\frac{3}{2}\lp |x|^2+|y|^2\rp = \lp \frac{3\ve}{2}\rp \lp 2^{q-1}+1\rp^{-1} + \frac{3}{2}\lp |x|^2+|y|^2\rp \nonumber\\
		&= \frac{3}{2}\lp \frac{\ve}{2^{q-1}+1} + |x|^2 + |y|^2 \rp \les \frac{3}{2} \lp \frac{\ve}{3}+|x|^2+|y|^2\rp \nonumber \\
		&\les \ve + 2x^2+2y^2
	\end{align}
\end{proof}
\begin{remark}
	We shall refer to this neural network for a given $q \in \lp 2,\infty \rp$ and given $\ve \in \lp 0,\infty \rp$ from now on as $\prd^{q,\ve}$. 
\end{remark}
\begin{remark}
	For an \texttt{R} implementation see Listing \ref{Prd}
\end{remark}
\begin{remark}
	Diagrammatically, this can be represented as:
\end{remark}
\begin{figure}[h]	
\begin{center}
\tikzset{every picture/.style={line width=0.75pt}} %set default line width to 0.75pt        

\begin{tikzpicture}[x=0.75pt,y=0.75pt,yscale=-1,xscale=1]
%uncomment if require: \path (0,475); %set diagram left start at 0, and has height of 475

%Shape: Rectangle [id:dp5102621452939872] 
\draw   (242,110.33) -- (430.67,110.33) -- (430.67,162.33) -- (242,162.33) -- cycle ;
%Shape: Rectangle [id:dp5404063577476766] 
\draw   (238.67,204.33) -- (427.33,204.33) -- (427.33,256.33) -- (238.67,256.33) -- cycle ;
%Shape: Rectangle [id:dp36108799479514775] 
\draw   (240,308.33) -- (428.67,308.33) -- (428.67,360.33) -- (240,360.33) -- cycle ;
%Shape: Rectangle [id:dp8902718451088835] 
\draw   (515.33,202.67) -- (600.67,202.67) -- (600.67,252.33) -- (515.33,252.33) -- cycle ;
%Shape: Rectangle [id:dp787158651575801] 
\draw   (74,204.67) -- (159.33,204.67) -- (159.33,254.33) -- (74,254.33) -- cycle ;
%Straight Lines [id:da7097969194866411] 
\draw    (515.33,202.67) -- (433.55,136.26) ;
\draw [shift={(432,135)}, rotate = 39.08] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da06987054821586158] 
\draw    (514.67,226) -- (432,226.98) ;
\draw [shift={(430,227)}, rotate = 359.32] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da6649718583556108] 
\draw    (515.33,252.33) -- (430.79,331.63) ;
\draw [shift={(429.33,333)}, rotate = 316.83] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da522975332769982] 
\draw    (240.67,136) -- (160.86,203.38) ;
\draw [shift={(159.33,204.67)}, rotate = 319.83] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da23420272890635796] 
\draw    (238.67,230.67) -- (160.67,231.64) ;
\draw [shift={(158.67,231.67)}, rotate = 359.28] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da3786949398178764] 
\draw    (239.33,333.33) -- (160.76,255.74) ;
\draw [shift={(159.33,254.33)}, rotate = 44.64] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da6573206574101601] 
\draw    (640.67,228.33) -- (602.33,228.33) ;
\draw [shift={(600.33,228.33)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da2877353538717321] 
\draw    (74,227.67) -- (35.67,227.67) ;
\draw [shift={(33.67,227.67)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;

% Text Node
\draw (286,124) node [anchor=north west][inner sep=0.75pt]    {$\frac{1}{2} \rhd \lp \Phi \bullet \aff_{A_1,0}\rp$};
% Text Node
\draw (286,220) node [anchor=north west][inner sep=0.75pt]    {$\frac{1}{2} \rhd \lp \Phi \bullet \aff_{A_2,0}\rp$};
% Text Node
\draw (286,326) node [anchor=north west][inner sep=0.75pt]    {$\frac{1}{2} \rhd \lp \Phi \bullet \aff_{A_2,0}\rp$};
% Text Node
\draw (543,220) node [anchor=north west][inner sep=0.75pt]    {$\cpy$};
% Text Node
\draw (100,225) node [anchor=north west][inner sep=0.75pt]    {$\sm$};

\end{tikzpicture}
	
\end{center}
\caption{Neural network diagram of the $\prd^{q,\ve}$ network.}
\end{figure}

\begin{figure}[h]
	\centering
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Prd_properties/experimental_deps.png}
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Prd_properties/dep_theoretical_upper_limits.png}
		\caption{Left: $\log_{10}$ of deps for a simulation of $\prd^{q,\ve}$ with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points. Right: The theoretical upper limits over the same range of values.}
\end{figure}

\begin{figure}[h]
	\centering
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Prd_properties/experimental_params.png}
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Prd_properties/param_theoretical_upper_limits.png}
		\caption{Left: $\log_{10}$ of params for a simulation of $\prd^{q,\ve}$ with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points. Right: The theoretical upper limits over the same range of values.}
\end{figure}

\begin{figure}[h]
	\centering
		\includegraphics[width = \linewidth]{/Users/shakilrafi/R-simulations/Sqr_properties/iso.png}
		\caption{Isosurface plot showing $|x^2 - \sqr^{q,\ve}|$ for $q \in [2.1,4]$, $\ve \in [0.01,2]$, and $x \in [-5,5]$ with 50 mesh-points in each.}
\end{figure}

\begin{table}[h]
\begin{tabular}{l|llllll}
\hline
                                  & Min   & 1st. Qu & Median & Mean    & 3rd Qu  & Max. \\ \hline
Experimental \\ $|x^2 - \inst_{\rect}\lp \sqr^{q,\ve}\rp(x)|$                 & 0.0000 & 0.0894 & 0.3378 & 3.1489 & 4.6746 & 20.0000 \\ \hline
Theoretical upper limits for\\ $|x^2 - \mathfrak{R}_{\mathfrak{r}}(\mathsf{Sqr})(x)$ & 0.010    & 1.715    & 10.402   & 48.063   & 45.538   & 1250.000  \\ \hline
\textbf{Forward Difference}       & 0.001 & 1.6012  & 9.8655 & 44.9141 & 40.7102 & 1230 \\ \hline
Experimental depths               & 2     & 2       & 2      & 2.307   & 2       & 80   \\ \hline
Theoretical upper bound on\\ depths & 2     & 2       & 2      & 2.73    & 2       & 91   \\ \hline
\textbf{Forward Difference}       & 0     & 0       & 0      & 0.423   & 0       & 11   \\ \hline
Experimental params               & 25    & 25      & 25     & 47.07   & 25      & 5641 \\ \hline
Theoretical upper limit on \\ params & 52    & 52      & 52     & 82.22   & 52      & 6353 \\ \hline
\textbf{Forward Differnce}        & 27    & 27      & 27     & 35.16   & 27      & 712  \\ \hline
\end{tabular}
\caption{Table showing the experimental and theoretical $1$-norm difference, depths, and parameter counts respectively for $\sqr^{q,\ve}$ with $q\in [2.1,4]$, $\ve \in [0.01,2]$, and $x \in [-5,5]$ all with $50$ mesh-points, and their forward differences.}
\end{table}



\section{Higher Approximations}\label{sec_tun}
We take inspiration from the $\sm$ neural network to create the $\prd$ neural network. However, we first need to define a special neural network called \textit{tunneling neural network} to stack two neural networks not of the same length effectively.
\subsection{The $\tun^d_n$ Neural Networks and Their Properties}
\begin{definition}[R\textemdash,2023, The Tunneling Neural Networks]\label{def:tun}
	We define the tunneling neural network, denoted as $\tun_n$ for $n\in \N$ by:
	\begin{align}
		\tun_n = \begin{cases}
			\aff_{1,0} &:n= 1 \\
			\id_1 &: n=2 \\
			\bullet^{n-2} \id_1 & n \in \N \cap [3,\infty)
		\end{cases}
	\end{align}	
	Where $\id_1$ is as in Definition \ref{7.2.1}.
\end{definition}
\begin{remark}
	For an \texttt{R} implementation see Listing \ref{Tun}
\end{remark}
\begin{lemma}\label{6.2.2}\label{tun_1}
	Let $n\in \N$, $x \in \R$ and $\tun_n \in \neu$. For all $n\in \N$ and $x\in \R$, it is then the case that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \tun_n \rp \in C \lp \R, \R \rp$
		\item $\dep \lp \tun_n \rp =n$
		\item $\lp \real_{\rect} \lp \tun_n \rp \rp \lp x \rp  = x$
 		\item $\param \lp \tun_n \rp = \begin{cases}
 			2 &:n=1 \\
 			7+6(n-2) &:n \in \N \cap [2,\infty)
 		\end{cases}$ 
 		\item $\lay \lp \tun_n \rp = \lp l_0, l_1,...,l_{L-1}, l_L \rp = \lp 1,2,...,2,1 \rp $ 
	\end{enumerate}
\end{lemma}
\begin{proof}
	Note that $\aff_{0,1} \in C \lp \R, \R\rp$ and by Lemma \ref{idprop} we have that $\id_1 \in C\lp \R, \R\rp$. Finally, the composition of continuous functions is continuous, hence $\tun_n \in C\lp \R, \R\rp$ for $n \in \N \cap \lb 2,\infty\rp$. This proves Item (i).
	
	Note that by Lemma \ref{5.3.2} it is the case that $\dep\lp \aff_{1,0} \rp = 1$ and by Lemma \ref{7.2.1} it is the case that $\dep \lp \id_1 \rp = 2$. 
	Assume now that for all $n \les N$ that $\dep\lp \tun_n \rp = n$, then for the inductive step, by Lemma \ref{comp_prop} we have that:
	\begin{align}
		\dep \lp \tun_{n+1} \rp  &= \dep \lp \bullet^{n-1} \id_1 \rp  \nonumber \\
		&= \dep \lp \lp \bullet^{n-2} \id_1 \rp \bullet \id_1 \rp \nonumber \\
		&=n+2-1 = n+1
	\end{align}
	This completes the induction and proves Item (i)\textemdash(iii). 
	Note next that by (\ref{5.1.11}) we have that:
	\begin{align}
		\lp \real_{\rect} \lp \aff_{1,0} \rp \rp \lp x \rp = x
	\end{align}
	Lemma \ref{idprop}, Item (iii) also tells us that:
	\begin{align}
		\lp \real_{\rect} \lp \id_1 \rp \rp \lp x \rp = \rect(x) - \rect(-x) = x
	\end{align}
	Assume now that for all $n\les N$ that $\tun_n \lp x \rp = x$. For the inductive step, by Lemma \ref{idprop}, Item (iii), and we then have that:
	\begin{align}
		\lp \real_{\rect} \lp \tun_{n+1} \rp \rp \lp x \rp &= \lp \real_{\rect} \lp \bullet^{n-1} \id_1 \rp \rp \lp x \rp \lp x \rp \nonumber\\
		&= \lp \real_{\rect} \lp \lp   \bullet^{n-2} \id_1 \rp \bullet \id_1 \rp \rp \nonumber\\
		&= \lp \lp \real_{\rect} \lp \bullet^{n-2} \id_1 \rp \rp \circ \lp \real_{\rect} \lp \id_1 \rp \rp \rp \lp x \rp \nonumber \\
		&= \lp \lp \real_{\rect} \lp \tun_n \rp \rp \circ \lp \real_{\rect} \lp \id_1 \rp \rp \rp \lp x \rp \nonumber \\
		&= x
	\end{align}
	This proves Item (ii). Next note that $\param\lp \tun_1\rp = \param\lp \aff_{1,0}\rp = 2$. Note also that:
	\begin{align}
		\param\lp \tun_2\rp = \param \lp \id_1 \rp &= \param \lb \lp \lp \begin{bmatrix}
			1 \\ -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1
		\end{bmatrix}, \begin{bmatrix}
			0
		\end{bmatrix}\rp \rp \rb \nonumber \\
		&= 7 \nonumber
	\end{align}
	And that by definition of composition:
	\begin{align}
		&\param \lp \tun_3 \rp \\ &= \param \lb \lp \lp \begin{bmatrix}
			1 \\ -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1
		\end{bmatrix}, \begin{bmatrix}
			0
		\end{bmatrix}\rp \rp \bullet \lp \lp \begin{bmatrix}
			1 \\ -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1
		\end{bmatrix}, \begin{bmatrix}
			0
		\end{bmatrix}\rp \rp \rb \nonumber \\
		&= \param \lb \lp \lp \begin{bmatrix}
			1 \\ -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix} \rp, \lp \begin{bmatrix}
			1 & -1 \\ -1 & 1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1&-1
		\end{bmatrix},\begin{bmatrix}
			0
		\end{bmatrix}\rp \rp \rb \nonumber \\
		&=13 \nonumber
	\end{align}
	Now for the inductive step assume that for all $n\les N\in \N$, it is the case that $\param\lp \tun_n \rp = 7+6(n-2)$. For the inductive step, we then have:
	\begin{align}
		&\param \lp \tun_{n+1} \rp = \param \lp \tun_n \bullet \id_1 \rp =\nonumber\\
		&\param \lb \lp \lp \begin{bmatrix}
			1 \\ -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1 \\ -1 & 1
		\end{bmatrix}, \begin{bmatrix}
			0 \\0
		\end{bmatrix}\rp, \cdots, \lp \begin{bmatrix}
			1 & -1
		\end{bmatrix}, \begin{bmatrix}
			0
		\end{bmatrix}\rp \rp \bullet \id_1  \rb =\nonumber \\
		&\param \lb \lp \lp \begin{bmatrix}
			1 \\ -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1 \\ -1 & 1
		\end{bmatrix}, \begin{bmatrix}
			0 \\0
		\end{bmatrix}\rp, \cdots, \lp \begin{bmatrix}
			1 & -1 \\ -1 & 1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0
		\end{bmatrix} \rp, \lp \begin{bmatrix}
			1 & -1
		\end{bmatrix}, \begin{bmatrix}
			0
		\end{bmatrix}\rp \rp  \rb \nonumber \\
		&=7+6(n-2)+6 = 7+6\lp \lp n+1 \rp -2 \rp  
	\end{align}
	This proves Item (iv).
	
	Note finally that Item (v) is a consequence of Lemma \ref{idprop}, Item (i), and Lemma \ref{comp_prop}
\end{proof} 

\begin{definition}[R\textemdash, 2023, The Multi-dimensional Tunneling Network]\label{def:tun_mult}
	We define the multi-dimensional tunneling neural network, denoted as $\tun^d_n$ for $n\in \N$ and $d \in \N$ by:
	\begin{align}
		\tun_n^d = \begin{cases}
			\aff_{\mathbb{I}_d,\mymathbb{0}_d} &:n= 1 \\
			\id_d &: n=2 \\
			\bullet^{n-2} \id_d & :n \in \N \cap [3,\infty)
		\end{cases}
	\end{align}	
	Where $\id_d$ is as in Definition \ref{7.2.1}.
\end{definition}
\begin{remark}
	We may drop the requirement for a $d$ and write $\tun_n$ where $d=1$, and it is evident from the context.
\end{remark}
\begin{lemma}\label{tun_mult}
	Let $n\in \N$, $d\in \N$, $x \in \R$ and $\tun_n^d \in \neu$. For all $n\in \N$, $d\in \N$, and $x\in \R$, it is then the case that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \tun_n^d \rp \in C \lp \R, \R \rp$
		\item $\dep \lp \tun_n^d \rp =n$
		\item $\lp \real_{\rect} \lp \tun_n^d \rp \rp \lp x \rp  = x$
 		\item $\param \lp \tun_n^d \rp = \begin{cases}
 			8d^2+5d &:n=1 \\
 			4d^2+3d+ (n-1)\lp 4d^2+2d\rp &:n \in \N \cap [2,\infty)
 		\end{cases}$ 
 		\item $\lay \lp \tun_n^d \rp = \lp l_0, l_1,...,l_{L-1}, l_L \rp = \lp d,2d,...,2d,d \rp$ 
	\end{enumerate}
\end{lemma}
\begin{proof}
	Note that Items (i)\textendash(iii) are consequences of Lemma \ref{idprop} and Lemma \ref{comp_prop} respectively. Note now that by observation $\param \lp \tun^d_1\rp = d^2+d$. Next Lemma $\ref{id_param}$ tells us that $\param\lp \tun^d_2\rp = 4d^2+3d$
	Note also that by definition of neural network composition, we have the following:
	\begin{align}
		&\param\lp \tun_3^d\rp \\ &= \param \lb \lp \lp \begin{bmatrix}
			1 \\ -1 \\ &\ddots \\& & 1 \\& & -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0 \\ \vdots \\ 0 \\0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1 \\ & &\ddots \\ & & & 1 & -1 
		\end{bmatrix}, \begin{bmatrix}
			0  \\ \vdots \\ 0
		\end{bmatrix}\rp \rp \bullet \right.\\ &\left. \lp \lp \begin{bmatrix}
			1 \\ -1 \\ & \ddots \\ & & 1 \\ & & -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0 \\ \vdots \\ 0 \\0
		\end{bmatrix}\rp, \lp \begin{bmatrix}
			1 & -1\\ & &\ddots \\ & & & 1 & -1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ \vdots \\ 0
		\end{bmatrix}\rp \rp \rb =\nonumber \\
		 &\param \lb \lp \lp \begin{bmatrix}
			1 \\ -1 \\ &  \ddots \\ & & 1 \\ & &-1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0 \\\vdots \\ 0\\0 
		\end{bmatrix} \rp, \lp \begin{bmatrix}
			1 & -1 \\ -1 & 1 \\ & & \ddots \\ & & & 1 & -1 \\ & & & -1 & 1
		\end{bmatrix}, \begin{bmatrix}
			0 \\ 0 \\ \vdots \\ 0 \\ 0
		\end{bmatrix}\rp, \right. \right.\\ & \left.\left. \lp \begin{bmatrix}
			1 &-1 \\ & &\ddots \\ & & & 1 & -1
		\end{bmatrix},\begin{bmatrix}
			0 \\ \vdots \\ 0
		\end{bmatrix}\rp \rp \rb \nonumber \\
		&=2d \times d + 2d + 2d\times 2d +2d+2d\times d + d \nonumber \\
		&=2d^2+2d+4d^2+2d+2d^2 +d \nonumber \\
		&= 8d^2+5d
	\end{align}
	Suppose now that for all naturals up to and including $n$, it is the case that $\param\lp \tun_n^d\rp = 4d^2+3d + \lp n-2 \rp \lp 4d^2+2d\rp$. For the inductive step, we have the following:

		\begin{align}
    & \param\lp \tun^d_{n+1}\rp = \param \lp \tun_n^d \bullet \id_d\rp \nonumber \\
    = &\param \lb \lp \begin{bmatrix}
        1 \\ -1 \\ & \ddots \\ & & 1 \\ & &-1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ 0 \\ \vdots \\ 0 \\ 0 
    \end{bmatrix} \rp, \lp \begin{bmatrix}
        1 & -1 \\ -1 & 1 \\ & \ddots \\ & & 1 & -1 \\ & & -1 & 1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ 0 \\ \vdots \\ 0 \\ 0
    \end{bmatrix} \rp, \hdots, \right. \\ &\left. \lp \begin{bmatrix}
        1 &-1 \\ & \ddots \\ & & 1 & -1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ \vdots \\ 0
    \end{bmatrix}\rp \bullet \id_d \rb \nonumber\\
    = &\param \lb \lp \begin{bmatrix}
        1 \\ -1 \\ & \ddots \\ & & 1 \\ & &-1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ 0 \\ \vdots \\ 0 \\ 0 
    \end{bmatrix} \rp, \lp \begin{bmatrix}
        1 & -1 \\ -1 & 1 \\ & \ddots \\ & & 1 & -1 \\ & & -1 & 1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ 0 \\ \vdots \\ 0 \\ 0
    \end{bmatrix} \rp, \hdots, \right.\\ &\left.\lp \begin{bmatrix}
        1 & -1 \\ -1 & 1 \\ & \ddots \\ & & 1 & -1 \\ & & -1 & 1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ 0 \\ \vdots \\ 0 \\ 0
    \end{bmatrix} \rp, \lp \begin{bmatrix}
        1 &-1 \\ & \ddots \\ & & 1 & -1
    \end{bmatrix}, \begin{bmatrix}
        0 \\ \vdots \nonumber\\ 0
    \end{bmatrix}\rp \rb \nonumber\\
    &= 4d^2+3d+ (n-2)\lp 4d^2+2d\rp + 4d^2+2d \nonumber \\
    &=4d^2+3d+\lp n-1\rp\lp 4d^2+2d\rp \nonumber
\end{align}
This proves Item (iv). Finally, Item (v) is a consequence of Lemma \ref{5.3.2}
\end{proof}







\subsection{The $\pwr_n^{q,\ve}$ Neural Networks and Their Properties}

\begin{definition}[R\textemdash, 2023, The Power Neural Network]\label{def:pwr}
	Let $n\in \N$. Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$, satisfy that $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. We define the power neural networks $\pwr_n^{q,\ve} \in \neu$, denoted for $n\in \N_0$ as:
	\begin{align}
		\pwr_n^{q,\ve} = \begin{cases}
			\aff_{0,1} & :n=0\\
			\prd^{q,\ve} \bullet \lb \tun_{\dep(\pwr_{n-1}^{q,\ve})} \boxminus \pwr_{n-1}^{q,\ve} \rb \bullet \cpy_{2,1} & :n \in \N
		\end{cases} \nonumber 
	\end{align}
Diagrammatically, this can be represented as:
\begin{figure}
\begin{center}
	\begin{tikzpicture}
  % Define nodes
  \node[draw, rectangle] (top) at (0, 2) {$\pwr_{n-1}^{q,\ve}$};
  \node[draw, rectangle] (right) at (2, 0) {$\cpy_{2,1}$};
  \node[draw, rectangle] (bottom) at (0, -2) {$\tun_{\dep(\pwr_{n-1}^{q,\ve})}$};
  \node[draw, rectangle] (left) at (-2, 0) {$\prd^{q,\ve} $};

  % Arrows with labels
  \draw[->] (right) -- node[midway, above] {$x$} (top);
  \draw[<-] (right) -- node[midway, above] {$x$} (4,0)(right);
  \draw[->] (right) -- node[midway, right] {$x$} (bottom);
  \draw[->] (top) -- node[midway, left] {$\lp \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp \rp \lp x \rp $} (left);
  \draw[->] (bottom) -- node[midway, left] {$x$} (left);
  \draw[->] (left) -- node[midway, above] {} (-5.5,0);
%  \draw[->] (-3,0) -- node[midway, above] {Arrow 6} (left);
\end{tikzpicture}
\end{center}
\caption{A representation of a typical $\pwr^{q,\ve}_n$ network.}
\end{figure}
\begin{remark}
	For an \texttt{R} implementation see Listing \ref{Pwr}
\end{remark}
\begin{remark}
	Note that for all $i \in \N$, $q\in \lp 2,\infty\rp$, $\ve \in \lp 0, \infty \rp$, each $\pwr_i^{q,\ve}$ differs from $\pwr_{i+1}^{q,\ve}$ by atleast one $\prd^{q,\ve}$ network. 
\end{remark}	
\end{definition}
\begin{lemma}\label{6.2.4}
	Let $x,y \in \R$, $\ve \in \lp 0,\infty \rp$ and $q \in \lp 2,\infty \rp$. It is then the case for all $x,y \in \R$ that:
	\begin{align}
		\ve \max \left\{ 1,|x|^q,|y|^q\right\} \les \ve + \ve |x|^q+\ve |y|^q.
	\end{align}
\end{lemma}
\begin{proof}
	We will do this in the following cases:
	
	For the case that $|x| \les 1$ and $|y| \les 1$ we then have:
	\begin{align}
		\ve \max \left\{ 1,|x|^q,|y|^q \right\} = \ve \les \ve + \ve |x|^q+\ve |y|^q
	\end{align}
	For the case that $|x| \les 1$ and $|y| \ges 1$, without loss of generality we have then:
	\begin{align}
		\ve \max \left\{1,|x|^q,|y|^q \right\} \les \ve | y|^q \les \ve + \ve |x|^q+\ve |y|^q:
	\end{align}
	For the case that $|x| \ges 1$ and $|y| \ges 1$, and without loss of generality that $|x| \ges |y|$  we have that:
	\begin{align}
		\ve \max\{ 1, |x|^q,|y|^q \} = \ve |x|^q \les \ve + \ve |x|^q+\ve |y|^q
	\end{align}
\end{proof}
\begin{lemma}
Let $\mathfrak{p}_i$ for $i \in \{1,2,...\}$ be the set of functions defined for $\ve \in \lp 0,\infty\rp$, and $x \in \R$ as follows:
	\begin{align}
		\mathfrak{p}_1 &= \ve+2+2|x|^2 \nonumber\\
		\mathfrak{p}_i &= \ve +2\lp \mathfrak{p}_{i-1} \rp^2+2|x|^2 \text{ for } i \ges 2
	\end{align}
	 For all $n\in \N$ and $\ve \in (0,\infty)$ and $q\in (2,\infty)$ it holds for all $x\in \R$ that:
	\begin{align}
		\left| \real_{\rect} \lp  \pwr^{q,\ve}_n \rp \lp x \rp\right| \les \mathfrak{p}_n
	\end{align}
	\end{lemma}
	\begin{proof}
	Note that by Corollary \ref{cor_prd} it is the case that:
	\begin{align}\label{(6.2.31)}
		\left|\real_{\rect} \lp \pwr^{q,\ve}_1 \rp \lp x \rp \right| =\left| \real_{\rect}\lp  \prd^{q,\ve}\rp \lp1,x \rp \right| \les \mathfrak{p}_1
	\end{align}
	and applying (\ref{(6.2.31)}) twice, it is the case that:
	\begin{align}
		\left| \real_{\rect} \lp \pwr_2^{q,\ve}\rp \lp x \rp \right| &= \left| \real_{\rect} \lp \prd^{q,\ve} \rp \lp \real_{\rect} \lp \prd ^{q,\ve}\lp 1,x \rp\rp,x\rp \right| \nonumber \\
		&\les \ve + 2\left| \real_{\rect} \lp \prd^{q,\ve}\rp\lp 1,x\rp \right|^2 + 2|x|^2 \nonumber \\
		&\les \ve + 2\mathfrak{p}_1^2 +2|x|^2 = \mathfrak{p}_2
	\end{align}
	Let's assume this holds for all cases up to and including $n$. For the inductive step, Corollary \ref{cor_prd} tells us that:
	\begin{align}
		\left| \real_{\rect} \lp \pwr_{n+1}^{q,\ve}\rp \lp x\rp \right| &\les \left| \real_{\rect} \lp \prd^{q,\ve} \lp \real_{\rect} \lp \prd^{q,\ve} \lp \real_{\rect}\cdots \lp 1,x\rp,x \rp ,x\rp \cdots \rp \rp \right| \nonumber \\
		&\les \real_{\rect} \lb \prd^{q,\ve} \lp \pwr^{q,\ve}_n \lp x\rp,x \rp\rb \nonumber \\
		&\les \ve + 2\mathfrak{p}_n^2 + 2|x|^2 = \mathfrak{p}_{n+1}
	\end{align}
	This completes the proof of the lemma.
	\end{proof}
	\begin{remark}
		Note that since any instance of $\mathfrak{p}_i$ contains an instance of $\mathfrak{p}_{i-1}$ for $i \in \N \cap \lb 2,\infty\rp$, we have that $\mathfrak{p}_n \in \mathcal{O}\lp \ve^{2(n-1)}\rp$
	\end{remark}
	\begin{lemma}\label{param_pwr_geq_param_tun}
		For all $n \in \N$, $q\in \lp 2,\infty\rp$, and $\ve \in \lp 0,\infty\rp$, it is the case that $\param \lp \tun_{\dep\lp\pwr^{q,\ve}_n\rp}\rp \les \param \lp \pwr^{q,\ve}_n\rp$.
	\end{lemma}
	\begin{proof}
		Note that for all $n \in \N$ it is straightforwardly the case that $\param\lp \pwr_n^{q,\ve}\rp \ges \param \lp \tun_{\dep\lp \pwr^{q,\ve}_{n-1}\rp}\rp$ because for all $n\in \N$, a $\pwr^{q,\ve}_n$ network contains a $\tun_{\dep\lp \pwr^{q,\ve}_{n-1}\rp}$ network. Note now that for all $i \in \N$ we have from Lemma \ref{tun_1} that $5 \les \param\lp \tun_{i+1}\rp - \param\lp \tun_i\rp \les 6$. Recall from Corollary \ref{cor:phi_network} that every  instance of the $\Phi$ network contains atleast one $\mathfrak{i}_4$ network, which by Lemma \ref{lem:mathfrak_i} has $40$ parameters, whence the $\prd^{q,\ve}$ network has atleast $40$ parameters for all $\ve \in \lp 0,\infty \rp$ and $q \in \lp 2,\infty\rp$. Note now that for all $i\in \N$, $\pwr^{q,\ve}_{i}$ and $\pwr^{q,\ve}_{i+1}$ differ by atleast as many parameters as there are in $\prd^{q,\ve}$, since, indeed, they differ by atleast one more $\prd^{q,\ve}$. Thus for every increment in $i$, $\pwr_i^{q,\ve}$ outstrips $\tun_i$ by at-least $40-6 = 34$ parameters. This is true for all $i\in \N$. Whence it is the case that for all $i \in \N$, it is the case that $\param\lp \tun_i\rp \les \param \lp \pwr^{q,\ve}_i\rp$.
	\end{proof}
\begin{lemma}[R\textemdash,2023]\label{power_prop}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$, and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. Let $n \in \N_0$, and $\pwr_n \in \neu$. It is then the case for all $n \in \N_0$, and $x \in \R$ that:
	\begin{enumerate}[label = (\roman*)]
		\item $\lp \real_{\rect} \lp \pwr_n^{q,\ve} \rp \rp \lp x \rp \in C \lp \R, \R \rp $
		\item $\dep(\pwr_n^{q,\ve}) \les \begin{cases}
			1 & :n=0\\
			n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 & :n \in \N
		\end{cases}$
		\item $\wid_1 \lp \pwr^{q,\ve}_{n}\rp = \begin{cases}
			1 & :n=0 \\
			24+2\lp n-1 \rp & :n \in \N 
		\end{cases}$
		\item $\param(\pwr_n^{q,\ve}) \les \begin{cases}
			2 & :n=0 \\
			4^{n+\frac{3}{2}} + \lp \frac{4^{n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp &: n\in \N		
			\end{cases}$\\~\\
		\item $\left|x^n -\lp \real_{\rect} \lp \pwr^{q,\ve}_n \rp \rp \lp x \rp \right| \les \begin{cases}
			0 & :n=0 \\
			\left| x \lp x^{n-1} - \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{n-1}^q & :n\in \N
		\end{cases}$ \\~\\
		Where we let $\mathfrak{p}_i$ for $i \in \{1,2,...\}$ be the set of functions defined as follows:
	\begin{align}
		\mathfrak{p}_1 &= \ve + 2 + 2|x|^2 \nonumber\\
		\mathfrak{p}_i &= \ve + 2\lp \mathfrak{p}_{i-1} \rp^2+2|x|^2
	\end{align}
	And whence we get that:
	\begin{align}
		\left| x^{n} - \real_{\rect} \lp \pwr^{q,\ve}_n\rp \lp x\rp\right| \in \mathcal{O} \lp \ve^{2q\lp n-1\rp} \rp &\text{ for } n \ges 2
	\end{align}
	\item $\wid_{\hid \lp \pwr_n^{q,\ve}\rp}\lp \pwr^{q,\ve}_n\rp = \begin{cases}
		1 & n=0 \\
		24 & n \in \N
	\end{cases}$
	\end{enumerate}
\end{lemma}
\begin{proof} 
	Note that Item (ii) of Lemma \ref{5.3.2} ensures that $\real_{\rect} \lp \pwr_0 \rp = \aff_{1,0} \in C \lp \R, \R \rp$. Note next that by Item (v) of Lemma \ref{comp_prop}, with $\Phi_1 \curvearrowleft \nu_1, \Phi_2 \curvearrowleft \nu_2, a \curvearrowleft \rect$, we have that:
	\begin{align}
		\lp \real_{\rect} \lp \nu_1 \bullet \nu_2 \rp\rp \lp x \rp = \lp\lp \real_{\rect}\lp \nu_1 \rp \rp \circ \lp \real_{\rect}\lp \nu_2 \rp \rp \rp  \lp x \rp
	\end{align}
	This, with the fact that the composition of continuous functions is continuous, the fact the stacking of continuous instantiated neural networks is continuous tells us that $\lp \real_{\rect} \pwr_n \rp \in C \lp \R, \R \rp$ for $n \in \N \cap \lb 2,\infty \rp$. This establishes Item (i).
	
	Note next that by observation $\dep \lp \pwr_0^{q,\ve} \rp=1$ and by Item (iv) of Lemma \ref{idprop}, it is the case that $\dep\lp \id_1 \rp  = 2$. By Lemmas $\ref{dep_cpy}$ and $\ref{depthofcomposition}$ it is also the case that\\ $\dep\lp \prd^{q,\ve} \bullet  \lb \tun_{\dep(\pwr^{q,\ve}_{n-1})} \boxminus \pwr^{q,\ve}_{n-1} \rb \bullet \cpy \rp = \dep \lp \prd^{q,\ve} \bullet \lb \tun_{\dep(\pwr^{q,\ve}_{n-1})} \boxminus \pwr^{q,\ve}_{n-1} \rb\rp $. Note also that by Lemma we have that $\dep \lp \tun_{\dep \lp \pwr^{q,\ve}_{n-1}\rp} \boxminus \pwr^{q,\ve}_{n-1}\rp = \dep \lp \pwr^{q,\ve}_{n-1} \rp$.
	This with Lemma \ref{comp_prop} then yields for $n \in \N$ that:
	\begin{align}
		\dep \lp \pwr^{q,\ve}_n  \rp &= \dep \lp \prd \bullet \lb \tun_{\mathcal{D} \lp \pwr^{q,\ve}_{n-1} \rp } \boxminus \pwr^{q,\ve}_{n-1} \rb \bullet \cpy_{2,1} \rp \nonumber \\
		&= \dep \lp \prd \bullet \lb \tun_{\dep \lp \pwr^{q,\ve}_{n-1} \rp } \boxminus \pwr^{q,\ve}_{n-1} \rb \rp \nonumber \\
		&= \dep \lp \prd \rp + \dep \lp \tun_{\dep \lp \pwr^{q,\ve}_{n-1} \rp} \rp -1 \nonumber \\
		&\les \frac{q}{q-2} \lb \log_2 \lp \ve^{-1}\rp +q \rb + \dep \lp \tun_{\dep\lp \pwr^{q,\ve}_{n-1} \rp} \rp - 1 \nonumber \\ 
		&= \frac{q}{q-2}\lb \log_2 \lp\ve^{-1} \rp + q\rb + \dep \lp \pwr^{q,\ve}_{n-1}\rp - 1 
	\end{align}
	And hence for all $n \in \N$ it is the case that:
	\begin{align}
		\dep\lp \pwr^{q,\ve}_n\rp - \dep \lp \pwr^{q,\ve}_{n-1}\rp \les \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1
	\end{align}
	This, in turn, indicates that:
	\begin{align}
		\dep \lp \pwr^{q,\ve}_n\rp &\les n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 \nonumber \\
		&\les n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1
	\end{align}
	This proves Item (ii).
	
	Note now that $\wid_1 \lp \pwr^{q,\ve}_0\rp = \wid_1 \lp \aff_{0,1}\rp = 1$. Further Lemma \ref{comp_prop}, Remark \ref{5.3.2}, tells us that for all $i,k \in \N$ it is the case that $\wid_i \lp \tun_k\rp \les 2$. Observe that since $\cpy_{2,1}, \pwr_0^{q,\ve}$, and $\tun_{\dep \lp \pwr_0^{q,\ve}\rp}$ are all affine neural networks, Lemma \ref{aff_effect_on_layer_architecture}, Corollary \ref{affcor}, and Lemma \ref{prd_network} tells us that: 	
	\begin{align}
		\wid_1 \lp \pwr_1^{q,\ve} \rp &= \wid_1 \lp \prd^{q,\ve} \bullet \lb \tun_{\dep(\pwr_{0}^{q,\ve})} \boxminus \pwr_{0}^{q,\ve} \rb \bullet \cpy_{2,1} \rp \nonumber \\
		&= \wid_1 \lp \prd^{q,\ve}\rp = 24
	\end{align}
	And that:
	\begin{align}
		\wid_1 \lp \pwr_2^{q,\ve} \rp &= \wid_1 \lp \prd^{q,\ve} \bullet \lb \tun_{\dep(\pwr_{1}^{q,\ve})} \boxminus \pwr_{1}^{q,\ve} \rb \bullet \cpy_{2,1} \rp \nonumber \\
		&= \wid_1 \lp \lb \tun_{\dep \lp \pwr^{q,\ve}_1 \rp} \boxminus \pwr_{1}^{q,\ve} \rb \rp \nonumber\\
		&= 24+2 = 26 \nonumber
	\end{align} 
	This completes the base case. For the inductive case, assume that for all $i$ up to and including $k\in \N$ it is the case that $\wid_1 \lp \pwr_i^{q,\ve}\rp \les  \begin{cases}
		1 & :i=0 \\
		24+2(i-1) & :i \in \N
	\end{cases}$. For the case of $k+1$, we get that:
	\begin{align}
		\wid_1 \lp \pwr_{k+1}^{q,\ve} \rp &= \wid_1 \lp \prd^{q,\ve} \bullet \lb \tun_{\dep(\pwr_{k}^{q,\ve})} \boxminus \pwr_{k}^{q,\ve} \rb \bullet \cpy_{2,1} \rp \nonumber \\
		&=\wid_1 \lp \lb \tun_{\dep(\pwr_{k}^{q,\ve})} \boxminus \pwr_{k}^{q,\ve} \rb  \rp \nonumber \\
		&=\wid_1 \lp \tun_{\dep \lp \pwr^{q,\ve}_{k}\rp}\rp + \wid_1 \lp \pwr^{q,\ve}_k\rp \nonumber \\
		&\les \begin{cases}
			2 & :k=0 \\
			24 +2 k & :k\in \N 
		\end{cases}
	\end{align}
	This establishes Item (iii).
	
	For Item (iv), we will prove this in cases.
	
	\textbf{Case 1: $\pwr_0^{q,\ve}:$}
	
	Note that by Lemma \ref{5.3.2} we have that:
	\begin{align}
		\param\lp \pwr_0^{q,\ve} \rp = \param \lp \aff_{0,1} \rp =2
	\end{align}
	This completes Case 1.
	
%	\textbf{Case 2: $\pwr_1^{q,\ve}:$}
%	
%	For this case, Lemma \ref{paramofparallel} tells us that we have:
%	\begin{align}
%		\param \lp \pwr_{0}^{q,\ve} \boxminus \tun_{\dep \lp \pwr_{0}^{q,\ve}\rp }\rp &= \frac{1}{2} \lp \param \lp \pwr^{q,\ve}_{0}\rp + \param \lp \tun_{ 1 } \rp\rp^2 \nonumber\\
%		&= \frac{1}{2} \lp 2+2\rp^2 \nonumber \\
%		&=8
%	\end{align}
%	Notice now that by Corollary \ref{affcor}, we have that:
%	\begin{align}
%		\param \lp\lb \pwr_{0}^{q,\ve} \boxminus \tun_{\dep \lp \pwr_{0}^{q,\ve}\rp } \rb \bullet \cpy_{2,1}\rp &= \param \lp \pwr_{0}^{q,\ve} \boxminus \tun_{\dep \lp \pwr_{0}^{q,\ve}\rp }\rp \nonumber \\
%		&=8
%	\end{align}
%	This now, coupled with Lemma \ref{comp_prop} and Lemma \ref{prd_network} tells us that:
%	\begin{align}\label{(6.2.19)}
%		\param \lp \prd^{q,\ve} \bullet \lb \pwr_{0}^{q,\ve} \boxminus \tun_{\dep \lp \pwr_{0}^{q,\ve}\rp } \rb \bullet \cpy_{2,1}\rp &= \param \lp \prd^{q,\ve}\bullet \lb \pwr_{0}^{q,\ve} \boxminus \tun_{\dep \lp \pwr_{0}^{q,\ve}\rp } \rb \rp\nonumber\\
%		&\les \param \lp \prd^{q,\ve}\rp + 8 + \wid_1 \lp \prd^{q,\ve} \rp \cdot \wid_0 \lp \tun_1\rp \nonumber \\
%		&=\param \lp \prd^{q,\ve}\rp + 32 \nonumber\\
%		&\les \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb -220
%	\end{align}
	
	\textbf{Case 2: $\pwr_n^{q,\ve}$ where $n\in \N$:}
	
	Note that Lemma \ref{paramofparallel}, Lemma \ref{param_pwr_geq_param_tun}, Corollary \ref{cor:sameparal}, Lemma \ref{lem:paramparal_geq_param_sum}, and Corollary \ref{cor:bigger_is_better}, tells us it is the case that: 
	\begin{align}
		\param \lp \pwr_{n-1}^{q,\ve} \boxminus \tun_{\dep \lp \pwr_{n-1}^{q,\ve}\rp }\rp &\les \param \lp \pwr^{q,\ve}_{n-1} \boxminus \pwr^{q,\ve}_{n-1}\rp \nonumber\\
		&\les 4\param\lp \pwr^{q,\ve}_{n-1}\rp	
	\end{align}
	Then Lemma \ref{comp_prop} and Corollary \ref{affcor} tells us that: 
	\begin{align}\label{(6.2.34)}
		&\param \lp  \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb \bullet \cpy_{2,1}\rp \nonumber\\&= \param \lp  \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb \rp \nonumber\\
		&\les 4\param \lp \pwr^{q,\ve}_{n-1}\rp
	\end{align}
	Note next that by definition for all $q\in \lp 2,\infty\rp$, and $\ve \in \lp 0,\infty\rp$ it is case that $\wid_{\hid\lp \pwr_0^{q,\ve}\rp}\pwr_0^{q,\ve} = \wid_{\hid \lp \aff_{0,1}\rp} = 1$. Now, by Lemma \ref{prd_network}, and by construction of $\pwr_i^{q,\ve}$ we may say that for $i\in \N$ it is the case that:
	\begin{align}
		\wid_{\hid \lp \pwr^{q,\ve}_i\rp} = \wid _{\hid \lp \prd^{q,\ve}\rp} = 24
	\end{align}

	Note also that by Lemma \ref{6.2.2} it is the case that:
	\begin{align}
		\wid_{\hid \lp \tun_{\dep \lp \pwr_{i-1}^{q,\ve}\rp}\rp} \lp \tun_{\dep \lp \pwr^{q,\ve}_{i-1}\rp} \rp = 2	\end{align}
	Furthermore, note that for $n\in \lb 2, \infty \rp \cap \N$ Lemma \ref{prd_network} tells us that:
	\begin{align}
		 \wid_{\hid \lp \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb\rp} \lp \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb\rp = 24+2=26
	\end{align}
	
	Finally Lemma \ref{comp_prop}, (\ref{(6.2.34)}), a geometric series argument, and Corollary \ref{cor:sameparal}, also tells us that:
	\begin{align}
		&\param \lp \pwr_{n}^{q,\ve}\rp\\ &= \param \lp  \prd^{q,\ve} \bullet\lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb \bullet \cpy_{2,1}\rp \nonumber \\
		&= \param \lp \prd^{q,\ve} \bullet \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb\rp \nonumber \\
		&\les \param \lp \prd^{q,\ve} \rp +  4\param \lp \pwr_{n-1}^{q,\ve}\rp+\nonumber\\
		&+ \wid_1 \lp \prd^{q,\ve} \rp\ \cdot \wid_{\hid \lp \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb\rp} \lp \lb \pwr^{q,\ve}_{n-1} \boxminus\tun_{\dep \lp \pwr_{n-1}^{q,\ve} \rp}\rb\rp \nonumber \\
		&= \param\lp \prd^{q,\ve}\rp + 4\param\lp \pwr^{q,\ve}_{n-1}\rp + 624 \nonumber\\
		&= 4^{n+1}\param\lp \pwr^{q,\ve}_0\rp + \lp \frac{4^{n+1}-1}{3}\rp \lp \param\lp \prd^{q,\ve}\rp + 624\rp \nonumber\\
		&= 4^{n+\frac{3}{2}} + \lp \frac{4^{n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp
	\end{align}
	
		
	Next note that $\lp \real_{\rect} \lp \pwr_{0,1} \rp\rp \lp x \rp$  is exactly $1$, which implies that for all $x\in \R$ we have that $|x^0-\lp \real_{\rect} \lp \pwr_{0.1}\rp\lp x \rp\rp |=0$. Note also that the instantiations of $\tun_n$ and $\cpy_{2,1}$ are exact. Note next that since $\tun_n$ and $\cpy_{2,1}$ are exact, the only sources of error for $\pwr^{q,\ve}_n$ are $n$ compounding applications of $\prd^{q,\ve}$.
	
	Note also that by definition, it is the case that:
	\begin{align}
		\real_{\rect}\lp \pwr_n^{q,\ve} \rp = \real_{\rect} \lb \underbrace{\prd^{q,\ve}  \lp \inst_{\rect} \lb \prd^{q,\ve}\lp\cdots \inst_{\rect}\lb \prd^{q,\ve} \lp  1,x\rp \rb, \cdots x\rp \rb, x \rp}_{n-copies } 	\rb 
	\end{align}
	Lemma \ref{prd_network} tells us that:
	\begin{align}
		\left|x-\real_{\rect}\lp \prd^{q,\ve} \lp 1,x \rp \rp \right| \les \ve \max\{ 1,|x|^q\} \les \ve + \left| x\right|^q
	\end{align}
	The triangle inequality, Lemma \ref{6.2.4}, Lemma \ref{prd_network}, and Corollary \ref{cor_prd} then tells us that:
	\begin{align}
		&\left| x^2 - \real_{\rect} \lp \pwr^{q,\ve}_2 \rp \lp x \rp \right| \nonumber\\
		&=\left| x\cdot x-\real_{\rect}\lp \prd^{q,\ve}\lp \inst_{\rect}\lp \prd^{q,\ve} \lp 1,x \rp \rp,x\rp \rp\right| \nonumber\\
		&\les \left| x\cdot x - x \cdot \inst_{\rect} \lp \prd^{q,\ve}\lp 1,x\rp \rp \right| + \left| x\cdot \inst_{\rect}\lp \prd^{q,\ve} \lp 1,x \rp\rp -\inst_{\rect}\lp \prd^{q,\ve} \lp \inst_{\rect}\lp \prd^{q,\ve}\lp 1,x\rp \rp,x \rp \rp \right| \nonumber\\
		&=\left| x\lp x-\inst_{\rect}\lp \prd^{q,\ve}\lp 1,x\rp\rp\rp\right|+ \ve + \ve\left| x\right|^q+\ve \left| \inst_{\rect}\lp \prd^{q,\ve}\lp 1,x\rp\rp\right|^q \nonumber\\
		&\les \left|x\ve + x\ve\left|x\right|^q \right| + \ve + \ve\left|x\right|^q+\ve \left|\ve + 2+x^2 \right|^q \nonumber\\
		&= \left| x\ve + x\ve \left| x\right|^q\right| + \ve + \ve\left| x\right|^q + \ve \mathfrak{p}_{1}^q
	\end{align}
	
	Note that this takes care of our base case. Assume now that for all integers up to and including $n$, it is the case that:
	\begin{align}\label{(6.2.39)}
		\left| x^n - \real_{\rect}\lp \pwr_n^{q,\ve}\rp \lp x \rp \right| &\les \left| x\cdot x^{n-1}-x \cdot \real_{\rect}\lp \pwr_{n-1}^{q,\ve}\rp \lp x\rp\right| + \left| x \cdot \real_{\rect}\lp \pwr_{n-1}^{q,\ve}\rp \lp x\rp -\real_{\rect} \lp \pwr_n^{q,\ve} \rp  \lp x \rp \right| \nonumber \\
		&\les \left| x\lp x^{n-1}-\real_{\rect} \lp \pwr^{q,\ve}_{n-1}\rp \lp x\rp\rp\right| + \ve + \ve|x|^q + \ve\left| \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp \lp x \rp \right| ^q\nonumber \\
		&\les \left| x \lp x^{n-1} - \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp\lp x\rp\rp\right| + \ve + \ve|x|^q + \ve\mathfrak{p}_{n-1}^q
	\end{align}
	For the inductive case, we see that:
	\begin{align}
		\left|x^{n+1}-\real_{\rect}\lp \pwr_{n+1}^{q,\ve}\rp\lp x\rp \right| &\les \left| x^{n+1}-x\cdot \real_{\rect}\lp \pwr_{n}^{q,\ve}\rp \lp x \rp\right| + \left| x\cdot \real_{\rect}\lp \pwr^{q,\ve}_n\rp \lp x \rp  - \real_{\rect} \lp \pwr^{q,\ve}_{n+1}\rp\right| \nonumber \\
		&\les \left|x\lp x^n-\real_{\rect} \lp \pwr^{q,\ve}_n\rp \lp x\rp\rp \right| + \ve + \ve|x|^q+\ve\left| \real_{\rect} \lp \pwr^{q,\ve}_{n}\rp \lp x \rp\right|^q \nonumber \\
		&\les \left|x\lp x^n-\real_{\rect} \lp \pwr^{q,\ve}_n\rp \lp x\rp\rp \right| + \ve + \ve|x|^q + \ve\mathfrak{p}^q_n
	\end{align}
	Note that since $\mathfrak{p}_n \in \mathcal{O} \lp \ve^{2(n-1)}\rp$ for $n\in \N \cap \lb 2,\infty \rp$, it is the case for all $x\in \R$ then that $\left| x^{n} - \real_{\rect} \lp \pwr^{q,\ve}_n\rp \lp x\rp\right| \in \mathcal{O} \lp \ve^{2q(n-1)} \rp$ for $n \ges 2$.
	
	Finally note that $\wid_{\hid \lp \pwr^{q,\ve}_0\rp}\lp \pwr^{q,\ve}_0\rp = 1$ from observation. For $n\in \N$, note that the second to last layer is the second to last layer of the $\prd^{q,\ve}$ network. Thus Lemma \ref{prd_network} tells us that:
	\begin{align}
		\wid_{\hid\lp \pwr^{q,\ve}_m\rp} \lp \pwr^{q,\ve}_n\rp = \begin{cases}
			1 & n=0 \\
			24 & n\in \N 
		\end{cases}
	\end{align}
	This completes the proof of the lemma.
\end{proof}
\begin{remark}\label{rem:pwr_gets_deeper}
	Note each power network $\pwr_n^{q,\ve}$ is at least as deep and parameter-rich as the previous power network $\pwr_{n-1}^{q,\ve}$, one differs from the next by one $\prd^{q, \ve}$ network.
\end{remark}

\begin{figure}[h]
	\centering
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Pwr_3_properties/experimental_deps.png}
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Pwr_3_properties/dep_theoretical_upper_limits.png}
		\caption{Left: $\log_{10}$ of depths for a simulation of $\pwr_3^{q,\ve}$ with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points. Right: The theoretical upper limits over the same range of values}
\end{figure}

\begin{figure}[h]
	\centering
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Pwr_3_properties/experimental_params.png}
		\includegraphics[width = 0.45\linewidth]{/Users/shakilrafi/R-simulations/Pwr_3_properties/param_theoretical_upper_limits.png}
		\caption{Left: $\log_{10}$ of params for a simulation of $\pwr_3^{q,\ve}$ with $q \in \lb 2.1, 4 \rb $, $\ve \in \lp 0.1, 2 \rb$, and $x \in \lb -5,5 \rb$, all with $50$ mesh-points. Right: The theoretical upper limits over the same range of values}
\end{figure}

\begin{figure}[h]
	\centering
		\includegraphics[width = \linewidth]{/Users/shakilrafi/R-simulations/Pwr_3_properties/isosurface.png}
		\caption{Isosurface plot showing $|x^3 - \pwr^{q,\ve}_3|$ for $q \in [2.1,4]$, $\ve \in [0.01,2]$, and $x \in [-5,5]$ with 50 mesh-points in each.}
\end{figure}


\subsection{$\pnm_{n,C}^{q,\ve}$ and Neural Network Polynomials.}

\begin{definition}[Neural Network Polynomials]
		Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. For fixed $q,\ve$, fixed $n \in \N_0$, and for $C = \{c_0,c_1,\hdots, c_n \} \in \R^{n+1}$ (the set of coefficients), we will define the following objects as neural network polynomials:
	\begin{align}
		\pnm^{q,\ve}_{n,C} \coloneqq  \bigoplus^n_{i=0} \lp c_i \triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb \rp  
	\end{align} 
\end{definition}
\begin{remark}
	Diagrammatically, these can be represented as
\end{remark}
\begin{figure}[h]
\begin{center}
	

\tikzset{every picture/.style={line width=0.75pt}} %set default line width to 0.75pt        

\begin{tikzpicture}[x=0.75pt,y=0.75pt,yscale=-1,xscale=1]
%uncomment if require: \path (0,475); %set diagram left start at 0, and has height of 475

%Shape: Rectangle [id:dp8950407412127579] 
\draw   (390,52) -- (455.33,52) -- (455.33,85) -- (390,85) -- cycle ;
%Shape: Rectangle [id:dp6602004057057332] 
\draw   (359.33,108.67) -- (454,108.67) -- (454,141.67) -- (359.33,141.67) -- cycle ;
%Shape: Rectangle [id:dp6567335394697266] 
\draw   (300,168.67) -- (455.33,168.67) -- (455.33,201.67) -- (300,201.67) -- cycle ;
%Shape: Rectangle [id:dp40847692689766735] 
\draw   (200,255.33) -- (456,255.33) -- (456,288.33) -- (200,288.33) -- cycle ;
%Shape: Rectangle [id:dp9479406055744195] 
\draw   (200.67,51.33) -- (358.67,51.33) -- (358.67,84.33) -- (200.67,84.33) -- cycle ;
%Shape: Rectangle [id:dp8579663805783284] 
\draw   (199.33,108) -- (330,108) -- (330,141) -- (199.33,141) -- cycle ;
%Shape: Rectangle [id:dp41506308397634806] 
\draw   (200.67,168.67) -- (268.67,168.67) -- (268.67,201.67) -- (200.67,201.67) -- cycle ;
%Straight Lines [id:da4565055641527326] 
\draw    (390.67,68.33) -- (361.33,68.33) ;
\draw [shift={(359.33,68.33)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da26211042309965304] 
\draw    (358,123.67) -- (332.67,123.67) ;
\draw [shift={(330.67,123.67)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da19391185534075384] 
\draw    (298,185) -- (272,185) ;
\draw [shift={(270,185)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Shape: Rectangle [id:dp5947036121491401] 
\draw   (518.67,155.33) -- (584,155.33) -- (584,188.33) -- (518.67,188.33) -- cycle ;
%Straight Lines [id:da9888083048478233] 
\draw    (518.67,155.33) -- (457.85,71.95) ;
\draw [shift={(456.67,70.33)}, rotate = 53.89] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da8782251261566656] 
\draw    (517.33,166) -- (457.03,128.72) ;
\draw [shift={(455.33,127.67)}, rotate = 31.73] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da09841373540031018] 
\draw    (518.67,178) -- (459.33,178.32) ;
\draw [shift={(457.33,178.33)}, rotate = 359.69] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da1515899374288483] 
\draw    (518.67,188.33) -- (458.51,271.38) ;
\draw [shift={(457.33,273)}, rotate = 305.92] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Shape: Rectangle [id:dp031165707162986944] 
\draw   (78.67,154.67) -- (144,154.67) -- (144,187.67) -- (78.67,187.67) -- cycle ;
%Straight Lines [id:da9492662023556374] 
\draw    (200,68.33) -- (145.09,152.99) ;
\draw [shift={(144,154.67)}, rotate = 302.97] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da028520602639475978] 
\draw    (198.67,123) -- (146.92,162.45) ;
\draw [shift={(145.33,163.67)}, rotate = 322.67] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da6814861591796668] 
\draw    (200,185) -- (147.29,174.07) ;
\draw [shift={(145.33,173.67)}, rotate = 11.71] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da019305885926265143] 
\draw    (198.67,271) -- (145.1,189.34) ;
\draw [shift={(144,187.67)}, rotate = 56.74] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da8585029210721031] 
\draw    (616,172.33) -- (586.67,172.33) ;
\draw [shift={(584.67,172.33)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;
%Straight Lines [id:da9805678030848519] 
\draw    (78.67,169.67) -- (49.33,169.67) ;
\draw [shift={(47.33,169.67)}, rotate = 360] [color={rgb, 255:red, 0; green, 0; blue, 0 }  ][line width=0.75]    (10.93,-3.29) .. controls (6.95,-1.4) and (3.31,-0.3) .. (0,0) .. controls (3.31,0.3) and (6.95,1.4) .. (10.93,3.29)   ;

% Text Node
\draw (412,217.73) node [anchor=north west][inner sep=0.75pt]    {$\vdots $};
% Text Node
\draw (406,61.4) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Pwr}^{q,\ve}_{0}$};
% Text Node
\draw (406,118.07) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Pwr}^{q,\ve}_{1}$};
% Text Node
\draw (403.33,177.07) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Pwr}^{q,\ve}_{2}$};
% Text Node
\draw (265.33,58.4) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Tun}$};
% Text Node
\draw (404,263.07) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Pwr}^{q,\ve}_{n}$};
% Text Node
\draw (249.33,115.73) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Tun}$};
% Text Node
\draw (222,176.4) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Tun}$};
% Text Node
\draw (525,162.4) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Cpy}_{n+1,1}$};
% Text Node
\draw (471.33,198.4) node [anchor=north west][inner sep=0.75pt]    {$\vdots$};
% Text Node
\draw (83,163.73) node [anchor=north west][inner sep=0.75pt]    {$\mathsf{Sum}_{n+1,1}$};
% Text Node
\draw (230.67,217.7) node [anchor=north west][inner sep=0.75pt]    {$\vdots$};
% Text Node
\draw (172,198.4) node [anchor=north west][inner sep=0.75pt]    {$\vdots$};


\end{tikzpicture}
\end{center}
\caption{Neural network diagram for an elementary neural network polynomial, with all coefficients being uniformly $1$.}
\end{figure}

\begin{lemma}[R\textemdash,2023]\label{6.2.9}\label{nn_poly}\label{mnm_prop}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. It is then the case for all $n\in\N_0$ and $x\in \R$ that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \pnm_{n,C}^{q,\ve}\rp \in C \lp \R, \R \rp $
		\item $\dep \lp \pnm_{n,C}^{q,\ve} \rp \les \begin{cases}
			1 & :n=0\\
			n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 &:n\in \N
		\end{cases}$
		\item $\param \lp \pnm_{n,C}^{q,\ve} \rp \les \begin{cases}
			2 & :n =0 \\
			\lp n+1\rp\lb 4^{n+\frac{3}{2}} + \lp \frac{4^{n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp\rb &:n\in \N
		\end{cases}$ \\~\\
		\item $\left|\sum^n_{i=0} c_ix^i - \real_{\rect} \lp \pnm_{n,C}^{q,\ve} \rp \lp x \rp \right| \les \sum^n_{i=1} c_i\lp \left| x \lp x^{i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp  $\\~\\
		Where $\mathfrak{p}_i$ are the set of functions defined for $i \in \N$ as such:
		\begin{align}
		\mathfrak{p}_1 &= \ve+1+|x|^2 \nonumber\\
		\mathfrak{p}_i &= \ve +\lp \mathfrak{p}_{i-1} \rp^2+|x|^2
		\end{align}
		Whence it is the case that:
		\begin{align}
		\left|\sum^n_{i=0} c_ix^i - \real_{\rect} \lp \pnm_{n,C}^{q,\ve} \rp \lp x \rp \right| \in \mathcal{O} \lp \ve^{2q(n-1)}\rp
		\end{align}
		\item $\wid_1 \lp \pnm_{n,C}^{q,\ve} \rp =  2+23n+n^2 $
		\item $\wid_{\hid \lp \pnm_{n,C}^{q,\ve}\rp} \lp \pnm_{n,C}^{q,\ve}\rp \les\begin{cases}
			1 &:n=0 \\
			24 + 2n &:n\in \N \end{cases}$
	\end{enumerate}
\end{lemma}
\begin{proof}
	Note that by Lemma \ref{5.6.3}, Lemma \ref{power_prop}, and Lemma \ref{comp_prop} indicate for all $n\in \N_0$ it is the case that:
	\begin{align}
		\real_{\rect}\lp \pnm_{n,C}^{q,\ve} \rp &= \real_{\rect} \lp  \bigoplus^n_{i=0} \lb c_i \triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb \rb \rp \nonumber\\
		&= \sum^n_{i=1}c_i \real_{\rect}\lp \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve} \rp \nonumber\\
		&= \sum^n_{i=1}c_i\real_{\rect}\lp \pwr^{q,\ve}_i \rp\nonumber
	\end{align}
	Since Lemma \ref{power_prop} tells us that $\lp \real_{\rect} \lp \pwr_n^{q,\ve} \rp \rp \lp x \rp \in C \lp \R, \R \rp$, for all $n\in \N_0$ and since the finite sum of continuous functions is continuous, this proves Item (i).
	
	Note that $\pnm_n^{q,\ve}$ is only as deep as the deepest of the $\pwr^{q,\ve}_i$ networks, which from the definition is $\pwr_n^{q,\ve}$, which in turn also has the largest bound. Therefore,  by Lemma \ref{comp_prop}, Lemma $\ref{5.3.3}$, Lemma $\ref{depth_prop}$, and Lemma \ref{power_prop}, we have that:
	\begin{align}
		\dep \lp \pnm_{n,C}^{q,\ve} \rp &\les \dep \lp \pwr_n^{q,\ve}\rp \nonumber\\
		&\les \begin{cases}
			1 & :n=0\\
			n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 &:n\in \N
		\end{cases} \nonumber
	\end{align}
	This proves Item (ii).
	
	Note next that for the case of $n=0$, we have that:
	\begin{align}
		\pnm_n^{q,\ve} = c_i \triangleright\pwr_0^{q,\ve}
	\end{align}
	This then yields us $2$ parameters.
	
	Note that each neural network summand in $\pnm_n^{q,\ve}$ consists of a combination of $\tun_k$ and $\pwr_k$ for some $k\in \N$. Each $\pwr_k$ has at least as many parameters as a tunneling neural network of that depth, as Lemma \ref{param_pwr_geq_param_tun} tells us. This, finally, with Lemma \ref{aff_effect_on_layer_architecture}, Corollary \ref{affcor}, and Lemma \ref{power_prop} then implies that: \\
	\begin{align}
		\param\lp \pnm^{q,\ve}_{n,C} \rp &= \param  \lp  \bigoplus^n_{i=0} \lb c_i \triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb \rb \rp\nonumber \\
		&\les \lp n+1 \rp \cdot \param \lp c_i \triangleright \lb \tun_1 \bullet \pwr_n^{q,\ve} \rb\rp \nonumber\\
		&\les \lp n+1 \rp \cdot \param \lp \pwr_n^{q,\ve} \rp \nonumber \\
		&\les \begin{cases}
			2 & :n =0 \\
			\lp n+1\rp\lb 4^{n+\frac{3}{2}} + \lp \frac{4^{n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp\rb &:n\in \N
		\end{cases} \nonumber
	\end{align}
	This proves Item (iii).
	
	Finally, note that for all $i\in \N$, Lemma \ref{power_prop}, and the triangle inequality then tells us that it is the case for all $i \in \N$ that:
	\begin{align}
		\left| x^i - \real_{\rect}\lp \pwr_i^{q,\ve}\rp \lp x \rp \right| &\les \left| x^i-x \cdot \real_{\rect}\lp \pwr_{i-1}^{q,\ve}\rp \lp x\rp\right| + \left| x \cdot \real_{\rect}\lp \pwr_{i-1}^{q,\ve}\rp \lp x\rp -\real_{\rect} \lp \pwr_i^{q,\ve} \rp  \lp x \rp \right| \nonumber \\
	\end{align}	
	This, Lemma \ref{6.2.9}, and the fact that instantiation of the tunneling neural network leads to the identity function (Lemma \ref{6.2.2} and Lemma \ref{comp_prop}), together with Lemma \ref{scalar_left_mult_distribution}, and the absolute homogeneity condition of norms, then tells us that for all $x\in \R$, and $c_0,c_1,\hdots, c_n \in \R$ it is the case that:
	\begin{align}
		&\left|\sum^n_{i=0} c_ix^i - \real_{\rect} \lp \pnm^{q,\ve}_{n,C} \lp x\rp \rp \right| \nonumber\\
		&= \left| \sum^n_{i=0}  c_ix^i - \real_{\rect} \lb \bigoplus^n_{i=0} \lb c_i \triangleright \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve} \rb \rb\lp x \rp\right| \nonumber \\
		&=\left| \sum^n_{i=1} c_ix^i-\sum_{i=0}^n c_i \lp \inst_{\rect}\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb\lp x\rp\rp\right| \nonumber\\
		&\les \sum_{i=1}^n \left|c_i\right| \cdot\left| x^i - \inst_{\rect}\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb\lp x\rp\right| \nonumber\\
		&\les \sum^n_{i=1} \left|c_i\right|\cdot\lp \left| x \lp x^{i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp \nonumber
	\end{align}
	Note however that since for all $x\in \R$ and $i \in \N \cap \lb 2, \infty\rp$, Lemma \ref{prd_network} tells us that $\left| x^{i} - \real_{\rect} \lp \pwr^{q,\ve}_i\rp \lp x\rp\right| \in \mathcal{O} \lp \ve^{2q\lp i-1\rp} \rp$, this, and the fact that $f+g \in \mathcal{O}\lp x^a \rp$ if $f \in \mathcal{O}\lp x^a\rp$, $g \in \mathcal{O}\lp x^b\rp$, and $a \ges b$, then implies that: 
	\begin{align}
		\sum^n_{i=1} \left| c_i\right|\cdot\lp \left| x \lp x^{i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp \in \mathcal{O} \lp \ve^{2q(n-1)}\rp
	\end{align}
	This proves Item (iv).
	
	Note next in our construction $\aff_{0,1}$ will require tunneling whenever $i\in \N$ in $\pwr_{i}^{q,\ve}$. Lemma \ref{aff_effect_on_layer_architecture} and Corollary \ref{affcor} then tell us that:
	\begin{align}
		\wid_1 \lp \pnm_n^{q,\ve} \rp &= \wid_1 \lp \bigoplus^n_{i=0} \lb c_i \triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb \rb\rp \nonumber\\
		&= \wid_1 \lp \bigoplus^n_{i=0}\pwr^{q,\ve}_i\rp \nonumber \\
		&\les \sum^n_{i=0}\wid_1 \lp \pwr^{q,\ve}_i\rp =2 + \frac{n}{2}\lp 24+24+2\lp n-1\rp\rp = 2+23n+n^2 \nonumber \\
	\end{align}
	This proves Item (v).
	
	Finally note that from the definition of the $\pnm_{n,C}^{q,\ve}$, it is evident that $\wid_{\hid\lp \pwr_{0,C}^{q,\ve}\rp}\lp \pwr_{0,C}^{q,\ve}\rp = 1$ since $\pwr_{0,C}^{q,\ve} = \aff_{0,1}$. Other than this network, for all $i \in \N$, $\pwr_{i,C}^{q,\ve}$ end in the $\prd^{q,\ve}$ network, and the deepest of the $\pwr_i^{q,\ve}$ networks is $\pwr^{q,\ve}_n$ inside $\pnm_{n,C}^{q,\ve}$. All other $\pwr_i^{q,\ve}$ must end in tunnels. Whence in the second to last layer, Lemma \ref{prd_network} tells us that:	
		\begin{align}
			\wid_{\hid\lp \pnm_{n,C}^{q,\ve}\rp} \les \begin{cases}
				1 &: n =0 \\
				24+2n &:n \in \N
			\end{cases}
		\end{align}
		This completes the proof of the Lemma.
\end{proof}
\subsection{$\xpn_n^{q,\ve}$, $\csn_n^{q,\ve}$, $\sne_n^{q,\ve}$, and ANN Approximations of $e^x$, $\cos(x)$, and $\sin(x)$.}
Once we have neural network polynomials, we may take the next leap to transcendental functions. For approximating them we will use Taylor expansions which will swiftly give us our approximations for our desired functions. Here, we will explore neural network approximations for three common transcendental functions: $e^x$, $\cos(x)$, and $\sin(x)$. 

\begin{lemma}
	Let $\nu_1,\nu_2 \in \neu$, $f,g \in C \lp \R, \R \rp$, and $\ve_1,\ve_2 \in \lp 0 ,\infty \rp$ such that for all $x\in \R$ it holds that $\left| f(x) - \real_{\rect} \lp \nu_1 \rp \right| \les \ve_1 $ and $\left| g(x) - \real_{\rect} \lp \nu_2 \rp \right| \les \ve_2$. It is then the case for all $x \in \R$ that:
	\begin{align}\label{6.2.14}
		\left| \lb f+g \rb \lp x \rp  - \real_{\rect} \lp \lb \nu_1 \oplus \nu_2 \rb \rp \lp x \rp\right| \les \ve_1 + \ve_2
	\end{align}
\end{lemma}
\begin{proof}
	Note that the triangle inequality tells us:
	\begin{align}
		\left| \lb f+g \rb \lp x \rp  - \real_{\rect} \lb \nu_1 \oplus \nu_2 \rb \lp x \rp \right| &= \left| f\lp x \rp +g\lp x \rp  -\real_{\rect}  \lp \nu_1\rp \lp x \rp -\real_{\rect} \lp \nu_2 \rp\lp x \rp  \right|\nonumber \\
		&\les \left| f\lp x \rp -\real_{\rect}\lp \nu_1 \rp \lp x \rp  \right| +  \left| g\lp x \rp  - \real_{\rect} \lp \nu_2 \rp \lp x \rp  \right| \nonumber\\
		&\les \ve_1 + \ve_2 \nonumber
	\end{align}
\end{proof}
\begin{lemma}\label{6.2.8}
	Let $n\in \N$. Let $\nu_1,\nu_2,...,\nu_n \in \neu$, $\ve_1,\ve_2,...,\ve_n \in \lp 0,\infty \rp$ and $f_1,f_2,...,f_n \in C\lp \R, \R \rp$ such that for all $i \in \{1,2,...,n\}$, and for all $x\in \R$, it is the case that,\\ $\left| f_i\lp x \rp  - \real_{\rect} \lp \nu_i \rp\lp x \rp \right| \les \ve_i$. It is then the case for all $x\in \R$, that:
	\begin{align}
		\left| \sum^n_{i=1} f_i \lp x \rp -\bigoplus^n_{i=1} \lp \real_{\rect}\lp \nu_i \rp \rp \lp x\rp\right| \les \sum_{i=1}^n \ve_i
	\end{align}
\end{lemma}
\begin{proof}
	This is a consequence of a finite number of applications of (\ref{6.2.14}). 
\end{proof}
\begin{definition}[R\textemdash 2023, $\xpn_n^{q,\ve}$ and the Neural Network Taylor Approximations for $e^x$ around $x=0$]
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$, and let $\pwr_n^{q,\ve} \subsetneq \neu$ be as in Lemma \ref{power_prop}. We define, for all $n\in \N_0$, the family of neural networks $\xpn_n^{q,\ve} as$: 
	\begin{align}
		\xpn_n^{q,\ve}\coloneqq \bigoplus^n_{i=0} \lb \frac{1}{i!} \triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb \rb
	\end{align}
\end{definition}

\begin{lemma}[R\textemdash,2023]\label{6.2.9}\label{tay_for_exp}\label{xpn_properties}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. It is then the case for all $n\in\N_0$ and $x\in \R$ that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \xpn_n^{q,\ve}\rp \lp x \rp\in C \lp \R, \R \rp $
		\item $\dep \lp \xpn_n^{q,\ve} \rp \les \begin{cases}
			1 & :n=0\\
			n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 &:n\in \N
		\end{cases}$
		\item $\param \lp \xpn_n^{q,\ve} \rp \les \begin{cases}
			2 & :n =0 \\
			\lp n+1\rp\lb 4^{n+\frac{3}{2}} + \lp \frac{4^{n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp\rb &:n\in \N
		\end{cases}$ \\~\\
		\item \begin{align*}\left|\sum^n_{i=0} \lb \frac{x^i}{i!} \rb- \real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp \right| \les \sum^n_{i=1} \frac{1}{i!}\lp \left| x \lp x^{i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp  \end{align*}\\~\\
		Where $\mathfrak{p}_i$ are the set of functions defined for $i \in \N$ as such:
		\begin{align}
		\mathfrak{p}_1 &= \ve+1+|x|^2 \nonumber\\
		\mathfrak{p}_i &= \ve +\lp \mathfrak{p}_{i-1} \rp^2+|x|^2
		\end{align}
		Whence it is the case that:
		\begin{align}
		\left|\sum^n_{i=0} \lb \frac{x^i}{i!} \rb- \real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp \right|\in \mathcal{O} \lp \ve^{2q(n-1)}\rp
		\end{align}
		\item $\wid_1 \lp \xpn_n^{q,\ve} \rp =  2+23n+n^2 $
		\item $\wid_{\hid \lp \xpn^n_{q,\ve} \rp}\lp \xpn_n^{q,\ve}\rp \les 24 + 2n$
	\end{enumerate}
\end{lemma}
\begin{proof}
	This follows straightforwardly from Lemma \ref{nn_poly} with $c_i \curvearrowleft \frac{1}{i!}$ for all $n \in \N$ and $i \in \{0,1,\hdots, n\}$. In particular, Item (iv) benefits from the fact that for all $i \in \N_0$, it is the case that $\frac{1}{i!} \ges 0$.
\end{proof}

\begin{lemma}[R\textemdash, 2023]

	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}.$ It is then the case for all $n\in\N_0$ and $x\in \lb a,b \rb\subsetneq \R$, where $0 \in \lb a,b\rb \subsetneq \R$ that:
	\begin{align}
		\left| e^x - \real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp  \right| \les \sum^n_{i=0} \frac{1}{i!}\lp \left| x \lp x^{n-1} - \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{n-1}^q	 \rp  + \frac{e^{b}\cdot |x|^{n+1}}{(n+1)!}
	\end{align}
\end{lemma}
\begin{proof}
	Note that Taylor's theorem states that for $x \in \lb a,b\rb \subsetneq \R$ it is the case that:
	\begin{align}
		e^x = \sum^n_{i=0} \lb \frac{x^i}{i!} \rb + \frac{e^{\xi}\cdot x^{n+1}}{(n+1)!}
	\end{align}
	Where $\xi$ is between $0$ and $x$ in the Lagrange form of the remainder. Note then, for all $n\in \N_0$, $x\in \lb a,b \rb \subsetneq \R$, and $\xi$ between $0$ and $x$, it is the case, by monotonicity of $e^x$ that the second summand is bounded by:
	\begin{align}
		\frac{e^\xi \cdot x^{n+1}}{(n+1)!} \les \frac{e^b\cdot |x|^{n+1}}{(n+1)!}
	\end{align}
	This, and the triangle inequality, then indicates that for all $x \in \lb a,b \rb \subsetneq \R$, and $\xi$ between $0$ and $x$ that:
	\begin{align}
		\left| e^x -\real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp \right| &=\left| \sum^n_{i=0} \lb \frac{x^i}{i!} \rb + \frac{e^{\xi}\cdot x^{n+1}}{(n+1)!}-\real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp\right| \nonumber\\
		&\les  \left| \sum^n_{i=0} \lb \frac{x^i}{i!} \rb - \real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp \right| + \frac{e^{b}\cdot |x|^{n+1}}{(n+1)!} \nonumber \\
		&\les \sum^n_{i=1} \frac{1}{i!}\lp \left| x \lp x^{n-1} - \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{n-1}^q	 \rp + \frac{e^{b}\cdot |x|^{n+1}}{(n+1)!} \nonumber
	\end{align}
	Whence we have that for fixed $n\in \N_0$ and $b \in \lb 0, \infty\rp$, the last summand is constant, whence it is the case that:
	\begin{align}
		\left| e^x -\real_{\rect} \lp \xpn_n^{q,\ve} \rp \lp x \rp \right|  \in \mathcal{O} \lp \ve^{2q(n-1)}\rp
	\end{align}
\end{proof}
\begin{definition}[The $\mathsf{Csn}_n^{q,\ve}$ Networks, and Neural Network Cosines]
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. Let $\pwr^{q,\ve}_n$ be a neural networks as defined in Definition \ref{def:pwr}. We will define the neural networks $\mathsf{Csn}_{n}^{q,\ve}$ as:
	\begin{align}
		\mathsf{Csn}_n^{q,\ve} \coloneqq \bigoplus^n_{i=0} \lb \frac{(-1)^i}{2i!}\triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_{2i}^{q,\ve}\rb \rb
	\end{align}

\end{definition}

\begin{lemma}[R\textemdash, 2023]\label{6.2.9}\label{csn_properties}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. It is then the case for all $n\in\N_0$ and $x\in \R$ that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \csn_n^{q,\ve}\rp \lp x\rp\in C \lp \R, \R \rp $
		\item $\dep \lp \csn_n^{q,\ve}\rp \les \begin{cases}
			1 & :n=0\\
			2n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 &:n\in \N
		\end{cases}$
		\item $\param \lp \csn_n^{q,\ve} \rp \les \begin{cases}
			2 & :n =0 \\
			\lp 2n+1\rp\lb 4^{2n+\frac{3}{2}} + \lp \frac{4^{2n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp\rb &:n\in \N
		\end{cases}$ \\~\\
		\item $\left|\sum^n_{i=0} \frac{(-1)^i}{2i!}x^{2i} - \real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp \right| \\ \les \sum^n_{i=1} \left| \frac{\lp -1\rp^i}{2i!}\right|\lp \left| x \lp x^{2i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{2i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{2i-1}^q	 \rp  $\\~\\
		Where $\mathfrak{p}_i$ are the set of functions defined for $i \in \N$ as such:
		\begin{align}
		\mathfrak{p}_1 &= \ve+1+|x|^2 \nonumber\\
		\mathfrak{p}_i &= \ve +\lp \mathfrak{p}_{i-1} \rp^2+|x|^2
		\end{align}
		Whence it is the case that:
		\begin{align}
		\left|\sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}x^{2i} - \real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp \right| \in \mathcal{O} \lp \ve^{2q(2n-1)}\rp
		\end{align}
	\end{enumerate}
	
\end{lemma}

\begin{proof}
	Item (i) derives straightforwardly from Lemma \ref{nn_poly}. This proves Item (i).
	
	Next, observe that since $\csn_n^{q,\ve}$ will contain, as the deepest network in the summand, $\pwr_{2n}^{q,\ve}$, we may then conclude that
	\begin{align}
		\dep \lp \csn_n^{q,\ve} \rp &\les \dep \lp \pwr_{2n}^{q,\ve}\rp \nonumber\\
		&\les \begin{cases}
			1 & :n=0\\
			2n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 &:n\in \N
		\end{cases} \nonumber
	\end{align}
	This proves Item (ii).
	
	A similar argument to the above, Lemma \ref{aff_effect_on_layer_architecture}, and Corollary \ref{affcor} reveals that:
	\begin{align}
		\param\lp \csn_n^{q,\ve} \rp &= \param  \lp  \bigoplus^n_{i=0} \lb \frac{\lp -1\rp^i}{2i!} \triangleright\lb \tun_{\max_i \left\{\dep \lp \pwr_i^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_i\rp} \bullet \pwr_i^{q,\ve}\rb \rb \rp\nonumber \\
		&\les \lp n+1 \rp \cdot \param \lp c_i \triangleright \lb \tun_1 \bullet \pwr_{2n}^{q,\ve} \rb\rp \nonumber\\
		&\les \lp n+1 \rp \cdot \param \lp \pwr_{2n}^{q,\ve} \rp \nonumber \\
		&\les \begin{cases}
			2 & :n =0 \\
			\lp n+1\rp\lb 4^{2n+\frac{3}{2}} + \lp \frac{4^{2n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp\rb &:n\in \N
		\end{cases} \nonumber
	\end{align}
	This proves Item (iii).
	
	In a similar vein, we may argue from Lemma \ref{nn_poly} and from the absolute homogeneity property of norms that:
	\begin{align}
		&\left|\sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}x^{2i} - \real_{\rect} \lp \csn_n^{q,\ve} \lp x\rp \rp \right| \nonumber\\
		&= \left| \sum^n_{i=0}  \frac{\lp -1\rp^i}{2i!}x^{2i} - \real_{\rect} \lb \bigoplus^n_{i=0} \lb \frac{\lp -1\rp^i}{2i!} \triangleright \tun_{\max_{2i} \left\{\dep \lp \pwr_{2i}^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_{2i}\rp} \bullet \pwr_{2i}^{q,\ve} \rb \rb\lp x \rp\right| \nonumber \\
		&=\left| \sum^n_{i=1} \frac{\lp -1\rp^i}{2i!}x^{2i}-\sum_{i=0}^n \frac{\lp -1 \rp^i}{2i!} \lp \inst_{\rect}\lb \tun_{\max_{2i} \left\{\dep \lp \pwr_{2i}^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_{2i}\rp} \bullet \pwr_{2i}^{q,\ve}\rb\lp x\rp\rp\right| \nonumber\\
		&\les \sum_{i=1}^n \left|\frac{\lp -1\rp^i}{2i!} \right|\cdot\left| x^{2i} - \inst_{\rect}\lb \tun_{\max_{2i} \left\{\dep \lp \pwr_{2i}^{q,\ve} \rp\right\} +1 - \dep \lp \pwr^{q,\ve}_{2i}\rp} \bullet \pwr_{2i}^{q,\ve}\rb\lp x\rp\right| \nonumber\\
		&\les \sum^n_{i=1} \left|\frac{\lp -1\rp^i}{2i!}\right|\cdot \left|\lp \left| x \lp x^{2i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{2i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{2i-1}^q	 \rp\right| \nonumber
	\end{align}
	Whence we have that:
	\begin{align}
		\left|\sum^n_{i=0} \lb \frac{\lp -1\rp^i x^{2i}}{2i!} \rb- \real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp \right|\in \mathcal{O} \lp \ve^{2q(2n-1)}\rp
		\end{align}
	This proves Item (iv).
\end{proof}

\begin{lemma}[R\textemdash, 2023]
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}.$ It is then the case for all $n\in\N_0$ and $x\in [a,b]\subseteq \lb 0,\infty \rp$ that:
	\begin{align}
		&\left| \cos\lp x\rp  - \real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp  \right|\\ &\les \sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}\lp \left| x \lp x^{n-1} - \real_{\rect}\lp \pwr^{q,\ve}_{n-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{n-1}^q	 \rp  + + \frac{|x|^{n+1}}{(n+1)!}\nonumber
	\end{align}
\end{lemma}

\begin{proof}
	Note that Taylor's theorem states that for all $x \in \lb a,b\rb \subsetneq \R$, where $0 \in \lb a,b\rb$, it is the case that:
	\begin{align}
		\cos\lp x \rp= \sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}x^i + \frac{\cos^{\lp n+1\rp}\lp \xi \rp \cdot x^{n+1}}{(n+1)!}
	\end{align}
	Note further that for all $n \in \N_0$, and $x \in \R$, it is the case that $\cos^{\lp n \rp} \lp x\rp \les 1$. Whence we may conclude that for all $n\in \N_0$, $x\in \lb a,b \rb \subseteq \R$, where $0 \in \lb a,b\rb$ and $\xi$ between $0$ and $x$, we may bound the second summand by:
	\begin{align}
		\frac{\cos^{\lp n+1\rp}\lp \xi \rp \cdot x^{n+1}}{(n+1)!} \les \frac{|x|^{n+1}}{\lp n+1\rp!}
	\end{align}
	This, and the triangle inequality, then indicates that for all $x \in \lb a,b \rb \subsetneq \lb 0,\infty\rp$ and $\xi \in \lb 0,x\rb$:
	\begin{align}
		\left| \cos \lp x \rp -\real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp \right| &=\left| \sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}x^i + \frac{\cos^{(n+1)}\lp \xi \rp \cdot x^{n+1}}{(n+1)!}-\real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp\right| \nonumber\\
		&\les  \left| \sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}x^i - \real_{\rect} \lp \csn_n^{q,\ve} \rp \lp x \rp \right| + \frac{|x|^{n+1}}{(n+1)!} \nonumber \\
		&\les \sum^n_{i=1} \left|\frac{\lp -1\rp^i}{2i!}\right|\cdot \left|\lp \left| x \lp x^{2i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{2i-1}\rp\lp x\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{2i-1}^q	 \rp\right| \nonumber\\&+ \frac{|x|^{n+1}}{(n+1)!} \nonumber
	\end{align}
	This completes the proof of the Lemma.
\end{proof}

\begin{definition}[R\textemdash, 2023, The $\mathsf{Sne}_n^{q,\ve}$ Newtorks and Neural Network Sines.].
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. Let $\pwr^{q,\ve}$ be a neural network defined in Definition \ref{def:pwr}. We will define the neural network $\mathsf{Csn}_{n,q,\ve}$ as:
	\begin{align}
		\mathsf{Sne}_n^{q,\ve} \coloneqq \csn^{q,\ve} \bullet \aff_{1, -\frac{\pi}{2}}
	\end{align}
\end{definition}

\begin{lemma}[R\textemdash, 2023]\label{6.2.9}\label{sne_properties}
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}$. It is then the case for all $n\in\N_0$ and $x\in \R$ that:
	\begin{enumerate}[label = (\roman*)]
		\item $\real_{\rect} \lp \sne_n^{q,\ve}\rp \in C \lp \R, \R \rp $
		\item $\dep \lp \sne_n^{q,\ve}\rp \les \begin{cases}
			1 & :n=0\\
			2n\lb \frac{q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q\rb -1 \rb +1 &:n\in \N
		\end{cases}$
		\item $\param \lp \sne_n^{q,\ve} \rp \les \begin{cases}
			2 & :n =0 \\
			\lp 2n+1\rp\lb 4^{2n+\frac{3}{2}} + \lp \frac{4^{2n+1}-1}{3}\rp \lp \frac{360q}{q-2} \lb \log_2 \lp \ve^{-1} \rp +q+1 \rb +372\rp\rb &:n\in \N
		\end{cases}$ \\~\\
		\item \begin{align}&\left|\sum^n_{i=0} \frac{(-1)^i}{2i!}{\lp x-\frac{\pi}{2}\rp}^{2i} - \real_{\rect} \lp \sne_n^{q,\ve} \rp \lp x \rp \right| \nonumber\\
		&= \left|\sum^n_{i=0} \frac{(-1)^i}{2i!}{\lp x-\frac{\pi}{2}\rp}^{2i} - \real_{\rect} \lp \csn_n^{q,\ve} \bullet \aff_{1,-\frac{\pi}{2}}\rp \lp x \rp \right|\nonumber\\
		&\les \sum^n_{i=1} \left| \frac{\lp -1\rp^i}{2i!}\right|\lp \left| \lp x -\frac{\pi}{2}\rp\lp \lp x -\frac{\pi}{2}\rp^{2i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x-\frac{\pi}{2}\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp \nonumber \end{align}\\~\\
		Where $\mathfrak{p}_i$ are the set of functions defined for $i \in \N$ as such:
		\begin{align}
		\mathfrak{p}_1 &= \ve+1+|x|^2 \nonumber\\
		\mathfrak{p}_i &= \ve +\lp \mathfrak{p}_{i-1} \rp^2+|x|^2
		\end{align}
		Whence it is the case that:
		\begin{align}
		\left|\sum^n_{i=0} \frac{\lp -1\rp^i}{2i!}\lp x-\frac{\pi}{2}\rp^{2i} - \real_{\rect} \lp \sne_n^{q,\ve} \rp \lp x \rp \right| \in \mathcal{O} \lp \ve^{2q(2n-1)}\rp
		\end{align}
	\end{enumerate}
\end{lemma}
\begin{proof}
	This follows straightforwardly from Lemma \ref{csn_properties}, and the fact that by Corollary \ref{affcor}, there is not a change to the parameter count, by Lemma \ref{comp_cont}, there is no change in depth, by Lemma \ref{aff_prop}, and Lemma \ref{csn_properties}, continuity is preserved, and the fact that $\aff_{1,-\frac{\pi}{2}}$ is exact and hence contributes nothing to the error, and finally by the fact that $\aff_{1,-\frac{\pi}{2}} \rightarrow \lp \cdot\rp -\frac{\pi}{2}$  under instantiation, assures us that the $\sne^{q,\ve}_n$ has the same error bounds as $\csn_n^{q,\ve}$. 
\end{proof}

\begin{lemma}[R\textemdash, 2023]
	Let $\delta,\ve \in \lp 0,\infty \rp $, $q\in \lp 2,\infty \rp$ and $\delta = \ve \lp 2^{q-1} +1\rp^{-1}.$ It is then the case for all $n\in\N_0$ and $x\in [a,b]\subseteq \lb 0,\infty \rp$ that:
	\begin{align}
		&\left| \sin\lp x\rp  - \real_{\rect} \lp \sne_n^{q,\ve} \rp \lp x \rp  \right|\nonumber \\ 
		&\les \sum^n_{i=1} \left| \frac{\lp -1\rp^i}{2i!}\right|\lp \left| \lp x -\frac{\pi}{2}\rp\lp \lp x -\frac{\pi}{2}\rp^{2i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x-\frac{\pi}{2}\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp  \nonumber\\ 
		&+\frac{|x|^{n+1}}{(n+1)!}\label{sin_diff}
	\end{align}
\end{lemma}
\begin{proof}
	Note that the fact that $\sin\lp x\rp = \cos\lp x-\frac{\pi}{2}\rp$, Lemma \ref{comp_prop}, and Lemma \ref{aff_prop} then renders (\ref{sin_diff}) as:
	\begin{align}
		&\left| \sin\lp x\rp - \inst_{\rect}\lp \sne_n^{q,\ve}\rp\right| \nonumber\\
		&= \left| \cos \lp x - \frac{\pi}{2}\rp - \inst_{\rect}\lp \csn_n^{q,\ve}\bullet \aff_{1,-\frac{\pi}{2}}\rp\lp x\rp\right| \nonumber\\
		&=\left| \cos \lp x-\frac{x}{2}\rp - \inst_{\rect}\csn_n^{q,\ve}\lp x-\frac{\pi}{2} \rp\right| \nonumber \\
		&\les \sum^n_{i=1} \left| \frac{\lp -1\rp^i}{2i!}\right|\lp \left| \lp x -\frac{\pi}{2}\rp\lp \lp x -\frac{\pi}{2}\rp^{2i-1} - \real_{\rect}\lp \pwr^{q,\ve}_{i-1}\rp\lp x-\frac{\pi}{2}\rp\rp\right| + \ve + |x|^q + \mathfrak{p}_{i-1}^q	 \rp \nonumber\\&+ \frac{|x|^{n+1}}{(n+1)!}\nonumber
	\end{align}
\end{proof}

\begin{remark}\label{rem:pyth_idt}
	Note that under these neural network architectures the famous Pythagorean identity $\sin^2\lp x\rp + \cos^2 \lp x\rp = 1$, may be rendered approximately, for appropriately fixed $n,q,\ve$ as: $\lb \sqr^{q,\ve}\bullet \csn^{q,\ve}_n \rb \oplus\lb \sqr^{q,\ve}\bullet \sne^{q,\ve}_n\rb \approx 1$. On a similar note, it is the case, with appropriate $n,q,\ve$ that $\real_{\rect}\lp \xpn^{q,\ve}_n \triangleleft \:i \rp\lp \pi \rp \approx -1$
	
	A full discussion of the associated parameter, depth, and accuracy bounds are beyond the scope of this dissertation, and may be appropriate for future work.
\end{remark}