{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.optimizers import SGD\n",
    "from keras import optimizers\n",
    "from keras.regularizers import l2, l1\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load heart data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>ChestPain</th>\n",
       "      <th>RestBP</th>\n",
       "      <th>Chol</th>\n",
       "      <th>Fbs</th>\n",
       "      <th>RestECG</th>\n",
       "      <th>MaxHR</th>\n",
       "      <th>ExAng</th>\n",
       "      <th>Oldpeak</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Ca</th>\n",
       "      <th>Thal</th>\n",
       "      <th>AHD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>typical</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>fixed</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>asymptomatic</td>\n",
       "      <td>160</td>\n",
       "      <td>286</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>108</td>\n",
       "      <td>1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>normal</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>asymptomatic</td>\n",
       "      <td>120</td>\n",
       "      <td>229</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>129</td>\n",
       "      <td>1</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>reversable</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>nonanginal</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>nontypical</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Age  Sex     ChestPain  RestBP  Chol  Fbs  RestECG  MaxHR  \\\n",
       "0           1   63    1       typical     145   233    1        2    150   \n",
       "1           2   67    1  asymptomatic     160   286    0        2    108   \n",
       "2           3   67    1  asymptomatic     120   229    0        2    129   \n",
       "3           4   37    1    nonanginal     130   250    0        0    187   \n",
       "4           5   41    0    nontypical     130   204    0        2    172   \n",
       "\n",
       "   ExAng  Oldpeak  Slope   Ca        Thal  AHD  \n",
       "0      0      2.3      3  0.0       fixed   No  \n",
       "1      1      1.5      2  3.0      normal  Yes  \n",
       "2      1      2.6      2  2.0  reversable  Yes  \n",
       "3      0      3.5      3  0.0      normal   No  \n",
       "4      0      1.4      1  0.0      normal   No  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('../data/heart.txt')\n",
    "data = data.dropna()\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hot encoding of categorical attributes\n",
    "x = data.ix[:,1:-1]\n",
    "x = pd.get_dummies(x).as_matrix()\n",
    "\n",
    "# Obtain labels and encode as {0, 1}\n",
    "y = data.ix[:,-1]\n",
    "y = (y=='Yes').as_matrix()*1.0\n",
    "\n",
    "# Normalize features\n",
    "x = (x - x.mean(axis=0)) / x.std(axis=0)\n",
    "\n",
    "# Split train-test sets\n",
    "n = x.shape[0]\n",
    "perm = np.random.permutation(n)\n",
    "n_tr = int(np.round(n*0.5))\n",
    "\n",
    "x_train = x[perm[0:n_tr],:]\n",
    "y_train = y[perm[0:n_tr]]\n",
    "\n",
    "x_test = x[perm[n_tr:],:]\n",
    "y_test = y[perm[n_tr:]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train neural network with 1 hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn(x_train, y_train, num_hidden_nodes):\n",
    "    # Construct neural network model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(num_hidden_nodes, activation='relu', input_shape=(x_train.shape[1],))) # hidden layer\n",
    "    model.add(Dense(1, activation='sigmoid')) # output layer\n",
    "    \n",
    "    # Optimizer: SGD (loss = binary cross entropy)\n",
    "    sgd = SGD(lr=1, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    \n",
    "    # Fit neural network\n",
    "    model.fit(x_train, y_train, epochs=100, batch_size=100, verbose=0)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain neural network\n",
      "Train accuracy: 1.0\n",
      "Test accuracy: 0.785234901729\n"
     ]
    }
   ],
   "source": [
    "model = train_nn(x_train, y_train, 100)\n",
    "\n",
    "# Evaluate train and test accuracy\n",
    "tr_acc = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "ts_acc = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "print('Plain neural network')\n",
    "print('Train accuracy: ' + str(tr_acc))\n",
    "print('Test accuracy: ' + str(ts_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train neural network with L2 regularization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn_l2(x_train, y_train, num_hidden_nodes, reg_param):\n",
    "    # Construct neural network model\n",
    "    model = Sequential()\n",
    "    reg = l2(reg_param)\n",
    "    \n",
    "    model.add(Dense(num_hidden_nodes, activation='relu', input_shape=(x_train.shape[1],),\\\n",
    "                        kernel_regularizer = reg)) # hidden layer, with L2 regularizer\n",
    "    model.add(Dense(1, activation='sigmoid',\\\n",
    "                        kernel_regularizer = reg)) # output layer, with L2 regularizer\n",
    "    \n",
    "    # Optimizer: SGD (loss = binary cross entropy)\n",
    "    sgd = SGD(lr=1, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    \n",
    "    # Fit neural network\n",
    "    model.fit(x_train, y_train, epochs=100, batch_size=100, verbose=0)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x135544810>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAETCAYAAAD3WTuEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FPX9x/HXBwQEuUFQueTwvqGIKGrEqmitWOsB1LNq\n1Yryo2rRtmjwKNVaW63VqqUeFbDWtl54oGC0alUUFKscCoiAKArIIcqRfH5/fCewhmyy2ezu7Cbv\n5+Oxj8zOfHfmkwzhnZnvzHfM3REREUlHg7gLEBGRwqUQERGRtClEREQkbQoRERFJm0JERETSphAR\nEZG0ZTVEzGycmX1mZjOraHObmX1gZm+b2QEJ80vNbLqZzTCzR7NZp4iIpCfbRyL3AsckW2hmxwI9\n3X0X4ALgzoTFX7l7b3c/wN1PzHKdIiKShqyGiLu/DKysoslg4IGo7etAKzPrGC2zbNYmIiK1F3ef\nSCdgUcL7JdE8gCZm9oaZvWpmg3NfmoiIVGebmLdf2dFG+TgsXd39UzPrDkw1s5nuvmCrFZhp3BYR\nkTS4e63P+MR9JLIY6JLwvjPwCYC7fxp9XQCUAAdU/HA5d4/9dc011+TF+lL9XCrtqmuTbHlN5mf6\n55YP+y/b+y4T+y+dZfm6/wrtdy+Vtrn43cuUXISIkbx/43HgTAAzOwj40t0/M7PWZtY4mt8eOBh4\nPwe1pq2oqCgv1pfq51JpV12bZMtrOj8fZLK2bO+7VNtW1SadZfm6/wrtdy+VtoX0u2eZTKStVm42\nASgC2gGfAdcAjQF397ujNrcDg4CvgLPdfYaZ9QfuAkoJQfd7d78vyTY8m9+DZFdxcTHFxcVxlyFp\n0v4rXGaGZ+B0Vlb7RNx9WApthlcy77/AvlkpSvJKvv51K6nR/pOsHonkgo5ERERqriCOREREsmXn\nnXdm4cKFcZeR97p168ZHH32UtfXrSEREClL0l3TcZeS9ZD+nTB2JxH2Jr4iIFDCFiIiIpE0hIiIi\naVOIiIhI2hQiIiJ5rqysjBYtWrB48eK4S9mKQkREJMNatGhBy5YtadmyJQ0bNqRZs2ab502cOLHG\n62vQoAFr1qyhc+fOWai2dnSJr4gUpEK5xLdHjx6MGzeOI444Immb0tJSGjZsmJXt6xJfEZECVtmo\nuaNHj2bIkCEMGzaMVq1aMX78eF577TX69+9PmzZt6NSpEyNGjKC0tBQIIdOgQQM+/vhjAM444wxG\njBjBcccdR8uWLTnkkENiu/FSISIiEoNHH32U008/nVWrVnHaaafRqFEjbrvtNlasWMErr7zCs88+\ny1133bW5vdm3DxomTpzIDTfcwMqVK+nSpQujR4/O9bcAKEREpA4rLgazrV/JBh6u2D6bAxQPGDCA\n4447DoAmTZrQp08f+vbti5mx8847c/755/Piiy9ubl/xaObkk0/mgAMOoGHDhvzoRz/i7bffzl6x\nVdDYWSJSZxUX1ywIatq+Nrp06fKt93PmzOGyyy7jrbfeYt26dZSWltKvX7+kn99hhx02Tzdr1oy1\na9dmrdaq6EhERCQGFU9PXXDBBeyzzz7Mnz+fVatWMWbMmIK4cEAhIiKSB9asWUOrVq1o2rQps2bN\n+lZ/SD5TiIiIZFHFI45kfve733HffffRsmVLLrroIoYMGZJ0PamuMxd0n4iIFKRCuU8kbrpPRERE\n8pZCRERE0qYQERGRtClEREQkbQoRERFJm0JERETSphAREZG0KURERCRtChEREUmbQkREJMMy/Xjc\ncv3792fChAkZrLT2shoiZjbOzD4zs5lVtLnNzD4ws7fNbP+E+WeZ2Vwzm2NmZ2azThGRTFqzZg2r\nV69m9erVdOvWjUmTJm2eN3To0LjLy6hsH4ncCxyTbKGZHQv0dPddgAuAP0fz2wBXA32BfsA1ZtYq\ny7WKiGRcZY/HLSsr47rrrqNnz5506NCBM844g9WrVwOwbt06hg4dSrt27WjTpg39+/dn1apVXH75\n5UybNo3zzjuPli1bcsUVV8Tx7WwlqyHi7i8DK6toMhh4IGr7OtDKzDoSgmeyu69y9y+BycCgbNYq\nIpIrN910E88//zyvvvoqixcvplGjRowcORKAv/zlL5SWlrJ06VKWL1/O7bffTuPGjbn55pvp27cv\n48aNY/Xq1fz2t7+N+bsI4n6yYSdgUcL7xdG8ivOXRPMqtWHD1vMaNgyvijZtgrIytc+X9o0ahceQ\nimRFJv5xZWGk4Lvvvpvx48fTsWNHAEaPHs3ee+/NuHHjaNSoEZ9//jkffPABe+21F3369KlQTn6N\nXBx3x3rFPWyAVzKfaH6lmjYt3vxq1qyE5s3httsqb/uLX0Dz5lu/1D737bfbDnbbDR54IISLSMa5\n1/6VBYsWLeK4446jbdu2tG3blt69ewOwYsUKzj33XA477DBOPvlkunbtyi9/+cuMBEdJSQnFxcWb\nXxlTfr4uWy+gGzAzybI/A6clvJ8NdASGAH9O1q7COlwKU1mZ+5Qp7ocd5v7663FXI4WmUH73d955\nZ58yZcpW86ZPn17tZxcsWOC77LKLT5gwwd3d+/fv7+PHj6/R9pP9nKL5tf4/PhdHIkblRxYAjwNn\nApjZQcCX7v4Z8CxwlJm1ijrZj4rmSR1iBgMHwosvwoEHxl2NSO5ccMEFjBo1isWLFwOwbNkynnzy\nSQCmTJnCrFmzcHeaN2/ONttswzbbhJ6Hjh07Mn/+/Njqrky2L/GdALwK7GpmH5vZOWZ2gZn9BMDd\nnwIWmNmHwF3AT6P5K4HrgDeB14ExHjrYpR5Zvbry/i6RQlLZo2xHjRrFUUcdxcCBA2nVqhUDBgxg\nxowZACxZsoTBgwfTsmVL9t13X44//nhOOeUUAEaOHMn9999Pu3btuPLKK3P6fSSjx+NK3rr9drj5\nZrjqKjjnHGjcOO6KJJ/o8bip0eNxpd4aPhwmToRHH4VeveDOO2H9+rirEpFEOhKRgvD663DttfDe\ne/D++9CsWdwVSdx0JJKabB+JKESkoCxYAN27x12F5AOFSGoUItVQiIjUTwqR1KhPRCQFl14Kt9wC\nX30VdyUi9YtCROqEc8+FV1+Fnj3ht79VmIjkikJE6oT99oNHHoHnnoNp06BHj+RDr0jd0K1bN8xM\nr2pe3bp1y+p+UJ+I1EnvvQczZsDpp8ddiUh+Usd6RCEiIlJz6lgXSdO998KXGkRHJCMUIlKvrF8f\nBnzs1QuuuQZWVvXINBGplkJE6pUmTeC+++C112DRohAmo0fDihVxVyZSmBQiUi/16gV//Wu4kmvp\n0hAsIlJz6lgXEamH1LEukmVlZfD553FXIZLfFCIiScycGZ4Bf9ll8OmncVcjkp8UIiJJ7L9/CJKN\nG2HPPWHkyNB/IiJbKEREqtC5cxg+5X//A3fYa69wZZeIBOpYF6mBpUuhfXto1CjuSkRqR8OeRBQi\nIiI1p6uzRPLIuHFw4YWwcGHclYjklkJEJANOOAHatIHeveEnP4GPPoq7IpHcUIiIZMD228PYsTB3\nbpju0wfOO08Px5K6T30iIlmwYgWMHw/Dh4PV+qyzSOapYz2iEBERqTl1rIsUqEcfhdmz465CJDMU\nIiI5tnAhHHooDBsGs2bFXY1I7ShERHJsxAiYNw/23hsOPxyGDAnPhBcpROoTEYnRmjVwxx1hWJW/\n/S3uaqQ+KZiOdTMbBPyBcNQzzt1vrLC8K/BXYHtgOXC6u38SLSsF3gEMWOjuJ1ayfoWIiEgNFUSI\nmFkDYC5wJPAJMA0Y4u6zE9o8DDzu7g+aWRHwY3c/M1q22t1bVrMNhYjUWUuXwo47xl2F1EWFcnXW\ngcAH7r7Q3TcCDwGDK7TZE5gK4O4lFZbrCnuptzZsgEMOgcGD4a234q5GpHLZDpFOwKKE94ujeYne\nBn4IYGYnAc3NrE20rImZvWFmr5pZxfARqdMaNw4d7t/9bgiS448Pz4QXySfbZHn9lR1JVDz3dAVw\nu5mdDbwELAE2Rcu6uvunZtYdmGpmM919QcUVFhcXb54uKiqiqKio9pWL5IGmTeGSS+D888Mgjyed\nBP/3f+FpiyI1UVJSQklJScbXm+0+kYOAYncfFL2/EvCKnesJ7bcDZrl710qW3Qs84e7/qjBffSJS\nb6xfD2vXQrt2cVciha5Q+kSmAb3MrJuZNQaGAI8nNjCzdmabRxe6inClFmbWOvoMZtYeOBh4P8v1\niuS1Jk0UIJJfshoi7l4KDAcmA+8BD7n7LDMbY2bHR82KgDlmNhvoANwQzd8DeNPMZgBTgLGJV3WJ\nyBbvvw9HHQUvvRR3JVLf6GZDkTpg48Zws+INN0DXrnDNNaCuQalKQdwnkgsKEZEtNm6ECRPg+uth\np53grrtg993jrkrykUIkohAR2dqmTTBxIhx9NHTsGHc1ko8UIhGFiIhIzRXK1VkikmfefBOefhr0\nt5dkgkJEpJ5ZuxauuAL69YNJkxQmUjs6nSVSD5WVwb/+BddeG4ZXufpq+P739Tz4+kR9IhGFiEj6\nysrgscfgT38Kj+1t3jzuiiRXFCIRhYiISM2pY11Esm7JknC0IpKMQkREkvrFL2DffeHvf4fS0rir\nkXyk01kikpQ7PPMMjBkDq1fD6NFw6qnQsGHclUltqU8kohARyT53eO65ECatWsFTT8VdkdRWzkLE\nzIYD4919ZW03lg0KEZHccYdlyzSUSl2Qy471HYBpZvawmQ1KePaHiNQzZgoQ+baUTmdFwXE0cA7w\nHeBhYJy7z8tuedXTkYhI/DZsgAYNYJtsP3BbMianl/hG/0t/Gr02AW2AR8zsptoWICKF789/hmOO\ngeXL465Ecq3aEDGzS83sLeAm4BVgH3e/COgD/DDL9YlIAbj4YujTB/r2hXffjbsayaVUDj7bAye5\n+8LEme5elvCIWxGpxxo2hJtugv33h4EDw5HJD/UnZr2QSog8Bawof2NmLYA93f11d5+VtcpEpOAM\nGwa77QYnnRQejHXaaXFXJNmWyiW+M4De5b3XZtYAeNPde+egvmqpY10k/yxbBk2bQosWcVciyWSq\nYz2VI5Fv/S8dncbSNRgiklSHDnFXILmSytVZ86PO9UbRawQwP9uFiYhI/kslRC4EDgaWAIuBfsBP\nslmUiNQ9X38N992nJynWNRo7S0Ry4tNP4Xvfgz32gHvuCX0mEp9cjp21LXAusBewbfl8d/9xbTee\nCQoRkcKxbh2cfz7MmQP//jd06RJ3RfVXLu9Y/xth/KxjgBeBzsCa2m5YROqfZs3gwQfDpb/9+sHL\nL8ddkdRWSpf4uvsBZjbT3fc1s0bAf9z9oNyUWDUdiYgUpmeeCUPK33Zb3JXUT7m8xHdj9PVLM9ub\nMH6WLuATkVoZNCi8pLClcjrrbjNrA/wKeBx4H7gx1Q1Ew8fPNrO5ZjaqkuVdzex5M3vHzKaa2U4J\ny86KPjfHzM5MdZsiIpIbVZ7Oiu5OP9ndH05r5eHzc4EjgU+AacAQd5+d0OZh4HF3f9DMioAfu/uZ\nUXC9CfQGDHiLcOf8qgrb0OkskTqkrCwMKy/ZlZOOdXcvA35ei/UfCHzg7gvdfSPwEDC4Qps9ganR\n9koSlh8DTHb3Ve7+JTAZ0MGvSB3mDkcdBRMnxl2JpCqVvH/ezC43sy5m1rb8leL6OwGLEt4vjuYl\neptoSHkzOwloHh2FVPzskko+KyJ1iBnccgv88pcwahSUlsZdkVQnlY718nE4L06Y50CPFD5b2aFS\nxXNPVwC3m9nZwEuEsNiU4mcBKC4u3jxdVFREUVFRCqWJSD7abz944w049VQ4/vhwVNK6ddxVFb6S\nkhJKSkoyvt6s3rFuZgcBxe4+KHp/JeFBiZV2zJvZdsAsd+9qZkOAIne/MFr2Z+AFd/97hc+oT0Sk\nDtq4ES6/HJ5+Gl57Ddqmev5DUpLLO9YrvSrK3R+oduVmDYE5hI71pcAbwNDE55CYWTtghbu7mV0P\nbHL34god6w2i6T5R/0jiNhQiInXYf/4DAwaEU12SObm8T6RvwvS2hECYDlQbIu5eambDCZ3iDYBx\n7j7LzMYA09z9SaAIGGtmZYTTWRdHn11pZtcRwsOBMRUDRETqvkMPjbsCqUqNT2eZWSvg7+WnqOKm\nIxERkZrL5dhZFa0Dutd2wyIi6ZozBxYsiLsKgRRCxMyeMLPHo9eThD6Of2e/NBGRyr3xBvTvD1On\nxl2JpNKxfnjC203AQndfnNWqakCns0Tqp6lTYdiwcE/J8OHqeK+pXF6d1R1Y6u7fRO+bAh3d/aPa\nbjwTFCIi9df8+XDiifCd78Cdd0KTJnFXVDhy2SfyD6As4X1pNE9EJFY9esCrr8Lq1TBpUtzV1E+p\nHIm87e77V5j3jrvvl9XKUqQjERFx1+msmsrlkcjnZnZCwoYHA1/UdsMiIpmiAIlPKkciPYHxQPlz\nPhYDZ7r7h1muLSU6EhERqbmcHYm4+7zoUbh7Anu5+8H5EiAiIsnMnQsnnADLl8ddSd2Wyn0ivzaz\n1u6+1t3XmFmbaIwrEZG81bMn7L479O0L774bdzV1Vyp9Iscmjlnl7iuB47JXkohI7TVsCDfdBNdf\nDwMHwj//GXdFdVMqAzA2NLMm7r4eNt8noquxRaQgDBsGu+0GJ50Es2eHmxMlc1IJkQeBKWZ2b/T+\nHOD+7JUkIpJZffrAtGkwY0bcldQ9KY3ia2aDgO8Snja4EtjR3S+u+lO5oauzRERqLtej+H5KuGv9\nh4TnicyqurmIiNQHSU9nmdmuwBBgKLAc+DvhyOWIHNUmIpJ1770He+6pGxbTlfR0VvSkwf8A55bf\nF2Jm8929Rw7rq5ZOZ4lIusrK4LDDoFs3+MtfoGnTuCvKnVyczvoh4TTWC2Z2j5kdSegTERGpExo0\ngMmTw/Shh8KiRfHWU4iShoi7/9vdTwN2B0qAkUBHM7vTzI7OUX0iIlnVrBk8+CAMGQL9+sHLL8dd\nUWGp0TPWzawtcApwmrsPzFpVNaDTWSKSKc88AxdfDG+/DS1axF1NduXsoVT5TiEiIpm0fn39eLhV\nri/xFRGpF+pDgGSSQkRERNKmEBERqca998LEiXFXkZ8UIiIi1ejdOwzcOGoUlJbGXU1+Uce6iEgK\nvvgCTj019JlMmABt2sRdUe2oY11EJIfat4dnn4Vddw33k8yeHXdF+SGVoeBFRARo1AhuvTWc3mqg\nP8GBHJzOioaR/wPhqGecu99YYXkXwvNJWkdtrnL3p82sG2G04PK8f83df1rJ+nU6S0SkhgriZkMz\nawDMJQwf/wkwDRji7rMT2twFTHf3u8xsD+Apd+8ehcgT7r5vNdtQiIiI1FCh9IkcCHzg7gvdfSPw\nEDC4QpsyoGU03RpYkrBMAz6KSEFwh2XL4q4i97IdIp2AxHExF0fzEo0BzjCzRcCTwCUJy3Y2s7fM\n7AUzG5DdUkVE0vfOO7DvvvDCC3FXklvZ7liv7Eii4rmnocC97v57MzuI8Ez3vYClQFd3X2lmvYFH\nzWxPd19bcYXFxcWbp4uKiigqKspQ+SIiqdl//3Dp79Ch4Z6S4cPz60FXJSUllJSUZHy92e4TOQgo\ndvdB0fsrAU/sXDez/wHHuPuS6P08oJ+7f1FhXS8Al7n79Arz1SciInlj/nw48UTo2xfuuCN/x+Iq\nlD6RaUAvM+tmZo0Jj9t9vEKbhcB3AaKO9Sbu/oWZtY865jGzHkAvYH6W6xURqZUePeDVV2HVqjCs\nfF2Xq0t8b2XLJb6/MbMxwDR3fzIKjnuA5oRO9ivcfYqZnQRcC2wESoGr3f2pStavIxERyTtlZfDl\nl9C2bdyVVK4gLvHNBYWIiEjNFcrpLBERqcMUIiIiOeIOY8fC8uVxV5I5ChERkRwp7yfp2xfefTfu\najJDISIikiMNG8KNN8L118PAgfDPf8ZdUe2pY11EJAZvvQU/+AGcfTYUF+d+VGB1rIuIFLA+fWDa\nNFixAjZsiLua9OlIRESkHtKRiIiIxE4hIiKSZ775JlwOXAgUIiIieWb0aDj9dPj667grqZ5CREQk\nz4wZE45EBgyARYuqbx8nhYiISJ5p1gzGj4chQ6BfP3j55bgrSk5XZ4lIzWzcGJ4D++WXsMceub/B\noZ555hk480x45BE47LDMrVej+EYUIiIZ8M038Nln4bVs2Zbpyl6rV0O7drDttrDddvCrX8Gpp4bb\nsSUrFiyATp2gcePMrVMhElGIiCSxdm3qwfD119ChA3Ts+O1XZfPatQuB4Q6TJ4cT+MuXhzAZOhS2\nyfZTtyUTFCIRhYjUG+7hcXmpBkNZ2dYBkCwc2rRJ/4Hg7jB1agiTTz4JDxg//XRo1Ciz379klEIk\nohCRglZWFsa9SCUYli0L/zGnGgwtWqQfDOl68UW49trwoPFf/ALOOiuz52BksxUroKQETjopvc8r\nRCIKEck7paXw+eepBcPnn0Pz5qkHQ7NmcX93qXn5ZbjuOpg9G668En78Y2jSJO6q6pQ5c+DYY+GU\nU+DXv655l5RCJKIQkZzYsOHbYVBVMKxYEU4PJQuGxHDo0KFu/+f62mshTN55B0aNgvPOg6ZN466q\nzvjii3BNQ5MmMGFC+GeXKoVIRCEiafv669T7F9asgfbtUwuG7bdX53JFb74ZwmTaNLjiCrjggsI5\nqspzGzfC5ZfD00/DY4+Fq65ToRCJKERkM/eaXZG0fn3lVx9VdiqpXTvdD5EJM2aEJzK98gpcdhlc\ndFE4nSe1du+98NFH4fqGVChEIgqROs493NSWajBA1UcLieHQunXuO54lePfdECYvvAAjR8LFF0PL\nlnFXVa8oRCIKkQK1YUP4s2nRouqvSGrSJPVgaN5cwVBI3n8fbrgh3G9y6aVwySUh3CXrFCIRhUie\ncg8dzPPnw7x54Wvi9KefQufO0KVL9cGgjti6b+7cECaTJoWjkhEjoG3buKuqE9wr/7tKIRJRiMRo\n48ZwJFFZSMyfH/719uwJPXps+Vo+3aWLbkaTrX34IYwdC48+ChdeGE51tW8fd1UFa8MGOPRQ+M1v\n4Igjvr1MIRJRiGTZqlWVh8S8ebBkCeyww9YBUT7dtq1OLUl6FiwI//P94x9w/vmhE75Dh7irKkhT\np8KwYWEggeHDt/xKKkQiCpFaKi2FxYsrD4n588MVTMlColu3un2Pg8Tv44/hxhth4kQ455xwLeuO\nO8ZdVcFZsAAGD4a+feGOO8KvrUIkohBJwdq1yfsmPv44nC6o7JRTjx7hngcdTUjcliyBm26Cv/0N\nzjgDfv7zMKytpGztWjj77DC8WUkJNGlSICFiZoOAPxAegDXO3W+ssLwLcD/QOmpzlbs/HS27Cvgx\nsAkY4e6TK1m/QqSsDJYuTd43sWYNdO9eeUjsvLM6rqVwLF0KN98cbooYOjTcBd+1a9xVFQz3ECBH\nHFEgRyJm1gCYCxwJfAJMA4a4++yENncB0939LjPbA3jK3bub2Z7AeKAv0Bl4HtilYmLUmxBZty4c\nk1Z22umjj6BVq+Sd2DvsoKMJqVuWLYNbboF77oGTT4arrgp/EEnKMhUi2R6b4UDgA3dfCGBmDwGD\ngdkJbcqA8ruMWgNLoukTgIfcfRPwkZl9EK3v9SzXHA/3cF9EstNOK1aEX5LEkDjyyDDdvXt4OJBI\nfdGhQ+h4v/xy+MMfoE8fOPHEMHJwz55xV1evZDtEOgGJj5lfTAiCRGOAyWZ2KdAM+G7CZ/+b0G5J\nNK9wrV8fjhqSnXZq1uzbIVFUFEY/7dEjnP/VsBsi39a+fbjz/Wc/g1tvDQ8k/973wqVIu+4ad3X1\nQrZDpLJDpYrnnoYC97r7783sIOBBYK8UPwtAcXHx5umioiKKiorSqbX23MMT3ipe4VQ+vWxZOH+b\neKppwIAt7zXsg0h62rYNg0aNHAl//CMccggcfXQIkz33jLu6vFBSUkJJSUnG15vtPpGDgGJ3HxS9\nvxLwxM51M/sfcIy7L4nezwP6AecRGv8mmv8McI27v15hG7ntE9m4ERYuTH7aaZtttu68Lv/aubNG\ndxXJhdWr4U9/gt//PvQi/+pXsM8+cVeVVwqlY70hMIfQsb4UeAMY6u6zEtpMAh529/ujjvXn3L1z\nQsd6P8JprOfIVcf6ypXJQ+KTT2CnnZLfO1GTAf1FJLvWroU774Tf/S4cnYweDfvvH3dVeaEgQgQ2\nX+J7K1su8f2NmY0Bprn7k1Fw3AM0J3SyX+HuU6LPXgWcC2wkk5f4btoUbrBLdid2aWnlIdGzZzgd\npeE6RArLunVw993hXpO+fUOYfOc7cVcVq4IJkWxLGiKrV1feeT1vXhjvqWPH5DfYtWunS2JF6qKv\nv4Zx48KVXfvtF8LkoIPirioWCpGImbmPG7d1YKxblzwkunWDbbeNu3QRicv69eGGxbFjYffd4eqr\nw+muekQhEjEz9zPP3DosOnTQ0YSIVG3DBnjgAfj1r8P9VldfDYcfHndVOaEQidSbO9ZFJHs2boQH\nHwzPNOnUKYTJwIF1+g9RhUhEISIiGbNpEzz0ULiBsV27ECZHH10nw0QhElGIiEjGlZaGZ5lcd114\n5PLVV8Nxx9WpMFGIRBQiIpI1ZWXwr3/BtdeGS/uvvhpOOKFOhIlCJKIQEZGsKyuDxx8PYVJWFi4N\n/sEPCno8O4VIRCEiIjnjDpMmhXG6vv46hMnJJ0PDhnFXVmMKkYhCRERyzh2efTaEyZdfhrG5Tjut\noMbGU4hEFCIiEht3mDIlhMlnn4VRg3/0o4IIE4VIRCEiIrFzhxdfDH0mCxeGh2OdcQY0bhx3ZUkp\nRCIKERHJK//5T7g0eO7c8Njes8+GJk3irmormQqRwr20QEQkHx16KEyeDBMnwmOPQa9e4dkm33wT\nd2VZoRAREcmG/v3hqafCfSbPPhvG9Lv11jA4bB2iEBERyaa+fcM9Jk88EfpNevYMD8n66qu4K8sI\nhYiISC707r3lqOT118OI4zfeCGvWxF1ZrShERERyad994eGHYepUeOedcGRyww2walXclaVFISIi\nEoe99oIJE+Cll2DOnNABP2YMrFwZd2U1ohAREYnT7ruHB2P997/hHpNddgnDqSxfHndlKVGIiIjk\ng1694K+I0W0+AAAJ9ElEQVR/hTfeCHe/77pruM/k88/jrqxKChERkXzSowfcfTdMnx76SXbbDa64\nIgRLHlKIiIjko27d4I47YOZMWL8e9tgDRo6ETz6Ju7JvUYiIiOSzzp3httvgvffCw7D23hsuuQQW\nL467MkAhIiJSGHbcEW65BWbNgqZNw6XCF10UOuNjpBARESkkHTvCTTeFy4LbtAk3MZ5/PsyfH0s5\nChERkUK0/fbw61/DBx+Eo5QDD4Rzzgnvc0ghIiJSyNq2Dc8x+fBD6N4dDj44PMtk9uycbF4hIiJS\nF7RuDVdfDfPmhSu5DjsMhg4NHfJZlPUQMbNBZjbbzOaa2ahKlt9iZjPMbLqZzTGzFQnLSqP5M8zs\n0WzXKrlXUlISdwlSC9p/eahly/BkxXnz4IAD4Mgj4ZRTwjhdWZDVEDGzBsDtwDHAXsBQM9s9sY27\n/8zdD3D33sAfgX8lLP7K3XtHy0/MZq0SD/0nVNi0//JYixbw85+HMOnfHwYNgh/8INzEmEHZPhI5\nEPjA3Re6+0bgIWBwFe2HAhMT3tf60Y25kulfpnTXl+rnUmlXXZtky2s6Px9ksrZs77tU21bVJp1l\n+br/Cu13L5W2Gf3d2247Snr3DldvHXEEnHACfP/7KdWZimyHSCdgUcL7xdG8rZhZV2BnYGrC7CZm\n9oaZvWpmVYVP7ArtH7JC5NsUItUvy9f9V2i/e6m0zcrvXtOmcOmloQP+2GOrrTFV5u4ZW9lWKzc7\nGTja3X8SvT8d6OvuIypp+3OgU+IyM9vB3T81s+6EcBno7gsqfC5734CISB3m7rU+27NNJgqpwmKg\na8L7zkCygV+GAD9NnOHun0ZfF5hZCXAAsKBCm4I55SUiUtdk+3TWNKCXmXUzs8aEoHi8YiMz2w1o\n7e6vJcxrHX0GM2sPHAy8n+V6RUSkBrJ6JOLupWY2HJhMCKxx7j7LzMYA09z9yajpEEKne6I9gLvM\nrDT67Fh3z83dMyIikpKs9omIiEjdpjvWRUQkbQoRERFJW50OETNrZmZvmtlxcdciqTOz3c3sTjN7\n2MwujLseqRkzG2xmd5vZRDM7Ku56JHVm1t3M/mJmD6f8mbrcJxJ14K8F3nP3p+KuR2rGzAy4393P\njLsWqTkzaw381t3Pj7sWqRkze9jdT02lbd4fiZjZODP7zMxmVphf3cCORxIuCV5GAQ2fUpeku++i\nNt8HngQU/jGpzf6L/Ar4U3arlMpkYN+lLO9DBLiXMIDjZlUN7GhmZ5jZ74FhQL/o63k5rVjKpbPv\nbjGzHd39CXf/HnB6rouWzdLdfzuZ2W+Ap9z97VwXLUAtfvfKm6e6oWzfsV5r7v6ymXWrMHvzwI4A\nZlY+sONsd/8b8LfyhmZ2JvBFruqVLdLdd2Z2uJldCTQBJuW0aNmsFvvvEuBIoKWZ9XL3u3NauNRm\n37U1szuB/c1slLvfWN228j5EkqhsYMcDK2vo7g/kpCJJVbX7zt1fBF7MZVGSslT23x8Jj3WQ/JLK\nvlsBXFSTlRbC6azKVHaoVXevEKhbtO8Km/Zf4crKvivUEKnJwI6SX7TvCpv2X+HKyr4rlBAxvp2i\nKQ3sKHlB+66waf8Vrpzsu7wPETObALwK7GpmH5vZOe5eClxCGNjxPeAhd58VZ52yNe27wqb9V7hy\nue/q9M2GIiKSXXl/JCIiIvlLISIiImlTiIiISNoUIiIikjaFiIiIpE0hIiIiaVOIiIhI2hQiknVm\nVmpm083sXTN7zMxaZmEbh5vZEzX8zI41eYJbwudamdlFCe/TWk8hiX6+/eOuQ/KPQkRy4St37+3u\n+wArgYuztJ2U75w1s4buvjTVp7dV0Ab46eaNpr+ejDKzhllcfRFwcE0+kOV6JE8oRCTX/ksYkhoA\nM7vczN4ws7fN7JqE+aOjJ7C9ZGYTzOxn0fwXzKx3NN3OzBZU3ICZ9TWzV8zsLTN72cx2ieafFR0J\nTQGej8YQejdado+ZzYhey6Ltb2dmz5vZm2b2TvS0RYCxQI/o6OrGCutpYmZ/NbOZ0faLErb9TzN7\n2szmmFmlz2kwswXROmea2Wtm1iOaf3z0/i0zm2xm20fzrzGzB8zsZeCBqJaXoprfNLODonaHm1mJ\nmT1qZh+a2VgzG2Zmr0ffW/eoXXszeySa/7qZ9Y+eS3Eh8H/R93xIZe0qqyeNfx9SaNxdL72y+gLW\nRF8bAg8DR0fvjwLuiqYNeAIYAPQBpgONgebAXOBnUbsXgN7RdDtgfjR9OPB4NN0caBBNHwk8Ek2f\nBXwMtIredwNmVqi1K+Gxyl0If2Q1T9jWB5V9LvE98DNgXDS9G7Aw+j7OAj6MamsCfAR0quRntQC4\nMpo+A3gimm6V0OZcwrPLAa4hDKzXOHq/bcJ0L2Baws9nBdAhqmcxcE207FLglmh6PHBwNN0FeD9h\nOz9LqKGqdpvr0avuvwr1oVRSWJqa2XTC0NPvA89F848GjoqWGbAdsAvQEnjM3TcAG2ra1wG0JvxV\nvgvhFFfiv/Pn3H1VZR8ys22BfwAXu/siM9sGGGtmhwFlwE5m1qGabQ8AbgNw9zlm9hGwa7Rsiruv\njbb1PiF8llSyjoeirxOB30fTXaJ+lx2BRoSwKfd49LOCEBC3m9n+QCnh51lumrsvi7Y/jzAQH8C7\nhNNVAN8F9jCz8tFfm5vZdpXUWFW7xHqkjlOISC6sc/fe0X/SzxL6RG4nBMdYd78nsbGZ/V8V69rE\nltOw2yZpcx0w1d1Pik7FvJCw7Ksq1n0n4ailvP2PgPbAAe5eFp06S7bNzeVX8X59wnQpyX//Evt2\nyqKvfwRudvdJZnY44S/+conf00jgU3ffN+qT+DrJ9ssS3pcl1GLAQRVDYEtWbJlVRbuqfsZSx6hP\nRHLBANz9G2AEcEX0H9yzwI/L/4I1s52ic/0vA9+P+heaA8cnrOsj4DvR9ClJtteKLX/hn5NSgWYX\nE05d/bbCepZFAXIE4cgBYA3QIsmqXiKED2a2K+FUz5xUakhwWvR1CKEPCcLRWfkDhM6q4rOtgKXR\n9JmEU4g1MZlwegsAM9svmlwT1VBdO6lnFCKSC5v/snb3t4G3gSHu/hzhlM1/zWwm4VRSc3d/k/Cw\nnHeAScBMoPwU1M3ARWb2FtA2yfZuAn4TtUn13/hlwD5Rx/p0M/sJ4bx/XzN7BzgdmBV9DyuAV6LO\n74od5HcA20Tfz0TgLHffWNXPpBJtom1eQjiyABgDPGJm04DPq/jsHcDZZjaDcBot2VFBsu2PAL4T\ndbb/D7ggmv8E8IPyjnVCgFTWTuoZPU9E8pKZbefuX5lZU8Jf9+dHAVSnRafM+kRBJZL31Cci+epu\nM9uTcCXTffUhQCL6q04Kio5EREQkbeoTERGRtClEREQkbQoRERFJm0JERETSphAREZG0/T/FMdJR\nX5CwQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1357b6150>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# List of regularizer parameters\n",
    "reg_params = [0.0001, 0.0001, 0.001, 0.01, 0.1]\n",
    "num_params = len(reg_params)\n",
    "\n",
    "# Iterate over parameters\n",
    "tr_acc = np.zeros(num_params,)\n",
    "ts_acc = np.zeros(num_params,)\n",
    "\n",
    "for i in range(num_params):\n",
    "    # Train neural network with L2 regularization\n",
    "    model = train_nn_l2(x_train, y_train, 100, reg_params[i])\n",
    "    \n",
    "    # Evaluate train and test accuracy\n",
    "    tr_acc[i] = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "    ts_acc[i] = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "# Plot train and test accuracy as a function of reg parameter\n",
    "plt.semilogx(reg_params, tr_acc, '--b', label='Train')\n",
    "plt.semilogx(reg_params, ts_acc, 'r', label='Test')\n",
    "plt.xlabel('Regularization parameter')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('L2 regularization')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train neural network with L1 regularization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn_l1(x_train, y_train, num_hidden_nodes, reg_param):\n",
    "    # Construct neural network model\n",
    "    model = Sequential()\n",
    "    reg = l1(reg_param)\n",
    "    \n",
    "    model.add(Dense(num_hidden_nodes, activation='relu', input_shape=(x_train.shape[1],),\\\n",
    "                        kernel_regularizer = reg)) # hidden layer, with L2 regularizer\n",
    "    model.add(Dense(1, activation='sigmoid',\\\n",
    "                        kernel_regularizer = reg)) # output layer, with L2 regularizer\n",
    "    \n",
    "    # Optimizer: SGD (loss = binary cross entropy)\n",
    "    sgd = SGD(lr=1, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    \n",
    "    # Fit neural network\n",
    "    model.fit(x_train, y_train, epochs=100, batch_size=100, verbose=0)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x142c2fd10>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEdCAYAAADNU1r0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmclXXZx/HPFwQUEdzScgE0NRU1BTcMFbdEs6jUBPcl\nxTJFzfUpHyHzSS21zDJccimU0Mq93HBcMldATMElFUHFDQUUF5i5nj9+9+BxnGHOLOfc58x836/X\neXHOff/OfV8zh5lrfrsiAjMzs9bokncAZmZWvZxEzMys1ZxEzMys1ZxEzMys1ZxEzMys1ZxEzMys\n1ZxEzNqJpH6S6iS16udK0hBJ00sQ1/6S/tne1zUDJxGrUJJekrRzI8e7Sbo+O18naYc84luKVk+8\niogHI2Kjtty8sUQWEddGxLC2XNesKU4iVo0eAA4AXm/pGyWp/cNpO0ld2+tSpERWkV+ndTxOIlZV\nImJRRFwUEQ8Bdc2Vl3SvpJ9LelDSB8A6knpLukLSa5JmSTqrPrlI6iLpfElvSfqvpGMK/7JvWEOS\ndKakPzVx70MlPSNpvqQXJB1VcG7H7N6nSHod+GP9sez89yQtyN47X9JHkiZl5/aUNFnSPEkzJZ1Z\ncNv7sn/fy963jaRDJD1QcO/tJD0q6V1Jj0ga3OD79bPs+zVf0j8lrdzsB2OdlpOIdQYHAt8HVgBe\nAa4BPgbWBbYAdsvOAxwF7A5sBgwEvk3zTVRNnX8D2DMiegOHARdK2rzg/BeBFYG+2X2XXCsiJkbE\nCtl71wReBK7NyrwPHBQRfYBvAEdL+lZ2rr55r3dE9I6IRwqvK2kl4Fbg18AqwIXAbdnxeiOBQ4Av\nAD2Ak5r5+q0TcxKxzuCqiJgREXXAysAw4ISI+Cgi3ib9Qh2Rld0X+E1EvB4R84BzWnvTiPhHRLyc\nPX8AuBPYvqBILXBmVrv6uLFrZDWk64BJEXF5dq37I+Lp7Pl/gAnAjg3f2kRY3wCey/pJ6iJiAjAD\n+GZBmSsj4r9ZTBOBzRu7kBnAMnkHYFYGswqe9wO6Aa/Xt2Blj1ey82s0KF/4vEUk7QH8L7AB6Q+2\n5YBpBUXeiohFzVzm/4DlgdEF192alNw2Abpnj+uLDGsNYGaDYzNJtZ16cwqeLwR6FXlt64RcE7HO\noLC5aRbwEbBKRKwcEStFxIoRsVl2/nVgrYLyfRtc6wOgZ8HrLzZ2Q0ndgRuA84AvRMRKwD/4bA1h\nqc1kkkYA+wF7R0RtwalrgRuBNSNiRWBcwXWba3p7Dejf4Fhf4NVm3mfWKCcRq2TdJfUoeHSF9Ata\n0rJZmR6SehR7wYiYQ2pWulDSCkrWLRgqPBEYLWkNSSsCpzS4xFRghKRlJG0J7NPgfP0v8/oawtsR\nUZfVSr5ebJyStgAuAr4dEXMbnO4FvBsRi7Jayf4F594iDTj4chOXvh1YX9IISV0l7QdsBNxSbGxm\nhZxErJLdRmpO+TD7t34U0rOkGsEawD+BhZIa1hjqNfaX+cGkX/DPAHNJTUH1NYrLSElmGvBEFsPi\nrD8F4Axgvex9ZwLjG7tfRLwPHAdcL2kuqc/lpmK+6My3SJ3uDxaM0rotO3cMcJakecBPgb8suXnE\nh8DZwL8kzc2SDAXn5wJ7kTrL387+/UZEvFsYv1mxVMpNqSRdQfoP+0ZBc0HDMhcBe5B+KRwWEVOy\n47XAk6S/7GZGxLdLFqhZEyQNAy6JiHXyjsWsEpW6JnIlabhko7Iq/pcjYn1gFHBJwekPImJgRGzh\nBGLlImlZSXtkTT1rkmobf8s7LrNKVdIkEhEPAu8upchw0ph9svHsfSStnp3zjFvLg4CxpOaqJ4Cn\n+bQZzcwayHuI75p8dgjlq9mxN0gdpo8Ci4FzI6Il7clmrZL1KWzdbEEzA/JPIo3VNuo7afpGxBxJ\n6wCTJE2LiJc+dwHJHYFmZq0QEW1u8cl7dNZsYO2C12uRxrHXD8UkSxw1pOUpGhURuT/OPPPMirhe\nse8rplxzZZo635Lj7f19q4TPr9SfXXt8fq05V6mfX7X97BVTthw/e+2lHEmkfkZwY24mDbdE0rbA\nexHxhqQVs8laSFoV2I40HLNiDR06tCKuV+z7iinXXJmmzrf0eCVoz9hK/dkVW3ZpZVpzrlI/v2r7\n2SumbDX97JV6iO+1wFDSQm9vkDoouwMREZdmZS4mrWX0AXBoREzJVhUdR1pbqAtwYURc1cQ9opRf\ng5XWmDFjGDNmTN5hWCv586tekoh2aM4qaZ9IROxfRJkfNXLs36RVVK2Dq9S/bq04/vyspDWRcnBN\nxMys5aqiJmJmVir9+/dn5syGCxJbQ/369ePll18u2fVdEzGzqpT9JZ13GBWvqe9Te9VE8h7ia2Zm\nVcxJxMzMWs1JxMzMWs1JxMzMWs1JxMyswtXV1bHCCiswe/bsvEP5HCcRM7N2tsIKK9C7d2969+5N\n165d6dmz55Jj1113XYuv16VLFxYsWMBaa61VgmjbxkN8zawqVcsQ33XXXZcrrriCnXbaqckytbW1\ndO3atST39xBfM7Mq1tiquWeccQYjRoxg//33p0+fPowfP56HH36YwYMHs9JKK7HmmmsyevRoamtr\ngZRkunTpwiuvvALAQQcdxOjRo9lzzz3p3bs3X/va13KbeOkkYmaWgxtvvJEDDzyQefPmsd9++9Gt\nWzcuuugi5s6dy7/+9S/uuOMOxo0bt6S89NlKw3XXXcfZZ5/Nu+++y9prr80ZZ5xR7i8BcBIxsw5s\nzBiQPv9oauHhhuVLuUDxkCFD2HPPPQHo0aMHgwYNYquttkIS/fv358gjj+S+++5bUr5hbWafffZh\niy22oGvXrhxwwAFMnTq1dMEuhdfOMrMOa8yYliWClpZvi7XXXvszr5999ll+/OMf88QTT7Bw4UJq\na2vZZpttmnz/F7/4xSXPe/bsyfvvv1+yWJfGNREzsxw0bJ4aNWoUm266KS+++CLz5s1j7NixVTFw\nwEnEzKwCLFiwgD59+rDccssxffr0z/SHVDInETOzEmpY42jK+eefz1VXXUXv3r35wQ9+wIgRI5q8\nTrHXLAfPEzGzqlQt80Ty5nkiZmZWsZxEzMys1ZxEzMys1ZxEzMys1ZxEzMys1ZxEzMys1ZxEzMys\n1ZxEzMys1ZxEzMys1ZxEzMzaWXtvj1tv8ODBXHvtte0YaduVNIlIukLSG5KmLaXMRZKelzRV0uYF\nxw+R9JykZyUdXMo4zcza04IFC5g/fz7z58+nX79+3HbbbUuOjRw5Mu/w2lWpayJXArs3dVLSHsCX\nI2J9YBTwh+z4SsD/AlsB2wBnSupT4ljNzNpdY9vj1tXVcdZZZ/HlL3+Z1VZbjYMOOoj58+cDsHDh\nQkaOHMkqq6zCSiutxODBg5k3bx4nnXQSjz32GN///vfp3bs3J598ch5fzueUNIlExIPAu0spMhy4\nJiv7CNBH0uqkxHNnRMyLiPeAO4FhpYzVzKxczjvvPO6++24eeughZs+eTbdu3TjhhBMAuPzyy6mt\nreX111/nnXfe4eKLL6Z79+786le/YquttuKKK65g/vz5/PKXv8z5q0jy3tlwTWBWwevZ2bGGx1/N\njjXqk08+f6xr1/RoaPFiqKtz+Uop361b2obUrCTa4z9XCVYKvvTSSxk/fjyrr746AGeccQabbLIJ\nV1xxBd26deOtt97i+eefZ8CAAQwaNKhBOJW1cnHeHesNP2EB0chxsuONWm65MUsePXvW0KsXXHRR\n42X/53+gV6/PP1y+/OWXXx423RQmTWq8rFmbRbT9UQKzZs1izz33ZOWVV2bllVdm4MCBAMydO5cj\njjiCHXbYgX322Ye+ffvyk5/8pF0SR01NDWPGjFnyaDf17XWlegD9gGlNnPsDsF/B6xnA6sAI4A9N\nlWtwjbDqVFcX8de/RqyzTsTTT+cdjVWbavnZ79+/f9xzzz2fOzZ58uRm3/vSSy/F+uuvH9dee21E\nRAwePDjGjx/fovs39X3Kjrf5d3w5aiKi8ZoFwM3AwQCStgXei4g3gDuA3ST1yTrZd8uOWQciwXe/\nC88+CxtvnHc0ZuUzatQoTj31VGbPng3Am2++ya233grAPffcw/Tp04kIevXqxTLLLMMyy6Seh9VX\nX50XX3wxt7gbU+ohvtcCDwEbSHpF0mGSRkk6CiAibgdekvQCMA74YXb8XeAs4HHgEWBspA5264C6\ndcs7ArPSaWwr21NPPZXddtuNnXfemT59+jBkyBCmTJkCwKuvvsrw4cPp3bs3m222GXvttRf77rsv\nACeccAJXX301q6yyCqeddlpZv46meHtcq1iXXALbbw+bbJJ3JFaJvD1ucbw9rnVaEuy8Mxx3HLy7\ntIHiZpYbJxGrWEcfDc88A4sWwYYbwqWXQm1t3lGZWSE3Z1lVmDIl1Uh22gl+9rO8o7FK4Oas4pS6\nOctJxKpGBHz4IfTsmXckVgmcRIrjPhGzjOQEYlZpnESs6j35JNxyS8kmF5vZUjiJWNVbuBBOOQX2\n2ANmzMg7GiuXfv36IcmPZh79+vUr6efgPhHrEBYtgosvhrPPhkMPhf/9X+jdO++ozCqX+0TMCnTr\nBiecAE8/neaUbLuthwOblYNrItYhzZ0LK6+cdxRmlctDfDNOImZmLefmLLMWqquDq69ufBMzM2sd\nJxHrNObPhwkT4KtfhTvvzDsas47BzVnWqUTAbbfB8cen1YEvuADWXTfvqMzKz81ZZq0gwV57pVFc\n22wDW28Njz6ad1Rm1cs1EevUXnsNVl8dunbNOxKz8vLorIyTiJlZy7k5y6yEHngA3n477yjMKp+T\niFkj7r4bNt4Yfvc7WLw472jMKpebs8ya8NRTMHp0qpFcdBEMHZp3RGbtx30iGScRK6UI+Otf4aST\n4JBDYOzYvCMyax9OIhknESuHhQvhzTehf/+8IzFrH04iGScRM7OW8+gsswrwzjtp4qJZZ+UkYtYG\n06alDvfjj4f33ss7GrPycxIxa4OddoJnnoEPP4QNN4TLLvNmWNa5uE/ErJ1MngzHHQcffwwPPgg9\neuQdkVnT3LGecRKxShIBjzyStuc1q2ROIhknETOzlqua0VmShkmaIek5Sac2cr6vpLslPSlpkqQ1\nCs7VSposaYqkG0sdq1kpPfVU3hGYtb+S1kQkdQGeA3YBXgMeA0ZExIyCMhOBmyPiz5KGAodHxMHZ\nufkR0buZe7gmYhVv4UIYNAi+/GW48EJYf/28I7LOrlpqIlsDz0fEzIhYBEwAhjcoszEwCSAiahqc\nb/MXaFYJevaEJ59Mw4EHD4bTToMFC/KOyqztSp1E1gRmFbyenR0rNBXYG0DSd4FeklbKzvWQ9Kik\nhyQ1TD5mVaV797QG11NPwZw5aUjwww/nHZVZ2yxT4us3VpNo2PZ0MnCxpEOB+4FXgfrFt/tGxBxJ\n6wCTJE2LiJcaXnDMmDFLng8dOpShXm7VKtiXvgRXXZUSiJu1rFxqamqoqalp9+uWuk9kW2BMRAzL\nXp8GRESc20T55YHpEdG3kXNXArdExN8aHHefiJlZC1VLn8hjwHqS+knqDowAbi4sIGkVSfVfyOnA\nH7PjK2bvQdKqwHbAMyWO1yx3s2fDokV5R2FWnJImkYioBX4E3Ak8DUyIiOmSxkraKys2FHhW0gxg\nNeDs7PhGwOOSpgD3AL8oHNVl1lGdfz589atpd0WzSufJhmYVJgJuvhlOOAE23zwllXXWyTsq62iq\npTnLzFpIguHD08KOgwbBllumRGJWiVwTMatws2fDc8/BzjvnHYl1JF47K+MkYmbWcm7OMuvkamth\n7ty8o7DOzknErEr9619p1vsll3gjLMuPk4hZldphhzQM+C9/SR3w99+fd0TWGblPxKzKRcD116d1\nubbbLm3Ru8IKeUdllc59ImYGpCHB3/sezJgBO+4Iyy+fd0TWmbgmYmbWCbkmYmZFe//9vCOwjspJ\nxKyDW7wYttgCTjwR5s3LOxrraJxEzDq4ZZZJw4EXLEhDgv/4R6iryzsq6yia7ROR9CNgfES8W56Q\nWsZ9ImbFe/xxOO64VDu57LK0WrB1TuXsE/ki8JikiZKGFez9YWZVZsst4cEH4dhj09Bgs7YqanRW\nlji+DhwGbAlMBK6IiP+WNrzmuSZiZtZyZR2dlf2WnpM9FgMrATdIOq+tAZhZ5fjkk7wjsGrTbBKR\ndJykJ4DzgH8Bm0bED4BBwN4ljs/MyugnP4FvfQteeCHvSKxaFFMTWRX4bkTsHhHXR8QigIioA/Za\n+lvNrJr8/OcwZAhsuy2cfrrnl1jzikkitwNLFpyWtIKkbQAiYnqpAjOz8uvRA045BaZNg1dfTUOC\nr7su76iskhUzxHcKMLC+91pSF+DxiBhYhvia5Y51s9L597/hkUfg+OPzjsTaW9l2NpQ0NSI2b3Bs\nWkRs1tabtwcnETOzlivn6KwXs871btljNPBiW29sZtXNs94NiksiRwPbAa8Cs4FtgKNKGZSZVbap\nU9Nsd4/iMi8Fb2YtFgHjxsHZZ8OkSbD++nlHZC3VXs1ZyxRxo2WBI4ABwLL1xyPi8Lbe3MyqkwRH\nHw3du8NOO8E998BXvpJ3VJaHYpqz/kRaP2t34D5gLWBBKYMys+pw+OFpbskuu6SdFa3zabYmAqwX\nEftKGh4RV0u6Fnig1IGZWXU49NC03Pz8+XlHYnkoJoksyv59T9ImpPWzVitdSGZWbQ48MO8ILC/F\nNGddKmkl4KfAzcAzwLnF3iBbPn6GpOckndrI+b6S7pb0pKRJktYoOHdI9r5nJR1c7D3NzKw8ljo6\nK5udvk9ETGzVxdP7nwN2AV4DHgNGRMSMgjITgZsj4s+ShgKHR8TBWeJ6HBgICHiCNHN+XoN7eHSW\nmVkLlWWyYbbI4iltuP7WwPMRMTNbuHECMLxBmY2BSdn9agrO7w7cGRHzIuI94E5gWBtiMbMyGj8e\nnnwy7yis1Ippzrpb0kmS1pa0cv2jyOuvCcwqeD07O1ZoKtmS8pK+C/TKaiEN3/tqI+81swrVowfs\nvjtMmZJ3JFZKxXSs75f9e0zBsQDWLeK9jVWVGrY9nQxcLOlQ4H5Sslhc5HsBGDNmzJLnQ4cOZejQ\noUWEZmaltM8+aT7JsGHwj3/AwIpYsrXzqqmpoaampt2vW9IZ65K2BcZExLDs9WmkjRIb7ZiXtDww\nPSL6ShoBDI2Io7NzfwDujYi/NHiP+0TMKtjf/54mJt5+OwwalHc0Vq+cq/g2OioqIq5p9uJSV+BZ\nUsf668CjwMjCfUgkrQLMjYiQ9HNgcUSMadCx3iV7PijrHym8h5OIWYW76SY47bS0T0m3bnlHY1DG\nZU+ArQqeL0tKCJOBZpNIRNRK+hGpU7wLcEVETJc0FngsIm4FhgK/kFRHas46Jnvvu5LOIiWPAMY2\nTCBmVh2GD4ddd3UC6Yha3JwlqQ/wl/omqry5JmJm1nLl3E+koYXAOm29sZmZVb9iVvG9hU9HRXUh\nzeto1eRDM7NCb74Jq3kRpapWTMf6jgUvFwMzI2J2SaNqATdnmVWn116DLbaAG26A7bfPO5rOp5yj\ns9YBXo+Ij7LXywGrR8TLbb15e3ASMated98N+++fEskOO+QdTedSzj6R64HC3ZRrs2NmZm2y664w\nYUKamFiCeXBWBsUkkWUi4pP6F9nz7qULycw6k513hokT4Xvfg3vvzTsaa6likshbkr5V/0LScODt\n0oVkZp3N0KGpSWvVVfOOxFqqmD6RLwPjgfp9PmYDB0fECyWOrSjuEzEza7mydawX3LBXVr6i9ld3\nEjEza7mydaxL+j9JK0bE+xGxQNJK2RpXZmbWyRXTJ7JH4ZpVEfEusGfpQjIzS8aNS6v/WuUqJol0\nldSj/kU2T6THUsqbmbWLr34VDj0Ubr0170isKcUkkT8D90g6QtIRwF3A1aUNy8wMtt02JZAjjoBb\nbsk7GmtMUR3rkoYBu5J2G3wX+FJEHLP0d5WHO9bNOr7HH4dvfAMuvTQtK29tV+5VfOeQZq3vTdpP\nZPrSi5uZtZ8tt0x9I2eeCR99lHc0VqjJmoikDYARwEjgHeAvwEkR0a984TXPNRGzzqO2Frp2zTuK\njqHk80SynQYfAI6on1go6cWIWLetN21PTiJmZi1XjuasvUnNWPdKukzSLqQ+ETMzM6C4ZU+WB75N\natbamTQy6+8RcWfpw2ueayJmnduMGbDhhnlHUX3K1rEeER9ExPiI2AtYC5gKnNbWG5uZtdU776RV\ngMePzzuSzqvotbMqlWsiZp3b00/DbrvBuefCQQflHU31aK+aSLN7rJuZVbIBA9IOibvtBnV1cMgh\neUfUuTiJmFnV23hjuOeetFNit25py10rDzdnmVmH8eyzaR7JeuvlHUnlK/t+IpXKScTMrOXKveyJ\nmZnZ5ziJmJlZqzmJmFmH9tvfwu9+l3cUHVfJk4ikYZJmSHpO0qmNnF9b0iRJkyVNlbRHdryfpIXZ\n8cmSfl/qWM2s49lrL/jVr1IysfZX0iG+kroAF5OWj38NeEzSTRExo6DYT4G/RMQ4SRsBtwPrZOde\niIiBpYzRzDq2ddaBe+9NM9vr6mD06Lwj6lhKPU9ka+D5iJgJIGkCMBwoTCJ1QO/s+YrAqwXnvOCj\nmbVZ//4pkey0U0okJ5yQd0QdR6mbs9YEZhW8np0dKzQWOEjSLOBW4NiCc/0lPSHpXklDShuqmXVk\n/fpBTU3abnfBgryj6ThKXRNprCbRcFLHSODKiLhQ0rakPd0HAK8DfSPiXUkDgRslbRwR7ze84Jgx\nY5Y8Hzp0KEOHDm2n8M2sI+nbN81s74xqamqoqalp9+uWdLJhlhTGRMSw7PVpQETEuQVl/gPsHhGv\nZq//C2wTEW83uNa9wI8jYnKD455saGbWQtUy2fAxYL1spFV30na7NzcoMxPYFSDrWO8REW9LWjXr\nmEfSusB6wIsljtfMijVnDvgPuE6vpEkkImqBHwF3Ak8DEyJiuqSxkvbKip0EHClpKjAeqF+Dcwdg\nmqQpwERgVES8V8p4zawZH30Ef/4zbL996q3efnuYPLnZt1WyiNTpbq3jtbPMrHnPPguXXgrXXAMD\nB8KoUfCNb6TXZ5wB3/wmnH02rLZa3pG22IIFsM02sN9+cOaZeUdTPtXSnGVm1erjj2HChDQudscd\noXt3eOQRuOMO+O53oUcPOPLItD9tr15pY48LL4RFi/KOvEVWWCHVRCZOTEnEf5O2jGsiZvZZL7yQ\nah1XXw2bbJJqHd/+dkoiSzN9epqAMXMm/PrXsPvu5Ym3nbz5ZpqQ+J3vwM9+Burgs9S8FHzGScSs\nHXzyCdx0E4wbB9Ompe0BjzoK1l+/ZdeJSBMxTjgh7RR1wQVVtbnHW2/BLrvAoYfCiSfmHU1pOYlk\nnETM2uDFF+Gyy+DKK2HDDVOto76pqi0+/jjVRn75SzjiCPjpT1O7URV4+2348ENYe+28Iykt94mY\nWessWgR//zsMGwZbb51+4dfUpMfIkW1PIJCuceqpqVYzZ05KUFdfndYcqXCrrtrxE0h7ck3ErLOY\nORMuvxz++Me0KuGoUbDPPrDccqW/98MPw3HHQZcucNFFKXlZrlwTMbPmLV4MN9+chuMOHAjz5sGd\nd8KDD8JBB5UngQBsu21KJEcfnTrpDz0UXn+9PPduBxEetdUUJxGzjmj2bBg7NtU4fvEL2HdfmDUr\n1QIGDMgnpi5dUvKYMSPNJ9l0UzjvvNScVuEuvhiOP96JpDFOImYdRW0t3H47DB8Om22Wxqzedhv8\n+9/pl3fPnnlHmPTunZLHQw/BAw+kYcS33FLRv6EPOujTFrkKDjMX7hMxq3avvZb6OS67DFZfPfV1\njBgByy+fd2TF+ec/05/5/funEV0bbph3RI2aNy9NfRk0KNVMqn0eiftEzDqzurpPZ44PGJCaqv7+\nd3j00TSktloSCKRRYtOmwde/DkOGpAka8+blHdXn9OmTupOmTIFjjqmKgWZl4SRiVk3eeCP1cay3\nHpx+evrT+JVX0iTBgVW8k3T37il5PP00zJ8PX/lKGklWW5t3ZJ/Ru3eqOC1cCO95OVjAzVlmla+u\nLi3uNG4c3HUX7L13arLacsvqb1NpyhNPpA6Ijz5KgwG+9rW8I+pwPGM94yRiHdZbb8FVV6V1rHr2\nTInjgANSu0pnEAHXXpsmLe64I5x7Lqy1Vt5RdRjuEzHriCI+nTm+wQbwzDPwpz/B1Knwwx92ngQC\nqZZ1wAFpSHD//vDVr6bl5j/6KO/IrICTiFkleOedtFjhRhulXtvBg9O6VldemSbqddRmq2L06pWS\nx2OPweOPp4Ud//73ihprGwHjx1dcF05ZuDnLLC8Raeb4uHFp5dtvfjM1WX3ta507aTTn7rvTkODV\nV4ff/CbNM8nZxx/DnnvCGmukFsiuXfOOqHluzjKrVu++mzqLN9kkbeo0aBD897+p2WrIECeQ5uy6\na2re+/a304ZZxx4Lc+fmGlKPHmm+5Jw5cPDBabWZzsJJxKwcIj6dOb7uumn68+9//+lGTquskneE\n1WWZZVLymD49/cbeaCO45JJc25N69kzLlL31Vprh3lkSiZuzzEpp3jz4859Tk9WHH6bmqkMOgS98\nIe/IOpYnn0xDgt97L9Xydtwxt1A+/DDtjrj55nDOObmF0SwP8c04iVjFiUidwOPGwd/+BrvtlpLH\nTjulRQitNCLg+uvh5JNhm23Shlj9+uUSykcfwYIFlf23gvtEzCrNggWfzhwfMSJtLTtjBkycmPZc\ndQIpLQm+973UxDVgQPocxoxJ08vLbNllKzuBtCfXRMzaavLklDyuvz7VNkaNSp2/Thr5euWVVCt5\n+OFUK9l3Xw9aKODmrIyTiOXi/fdhwoSUPN58M42yOvzwNMbTKst998Ho0Wmi5m9+kzorclBXlzrb\nu3fP5faf4+Ysszw8+WSaOd63bxrTOXZsmhT40586gVSqHXdMa3GNHJkWrDz6aHj77bKHcfnlaTfi\nKtiDq0WcRMyas3BhmkE2eDDstVea5DZtGtx0U5phVg0zyzq7rl1T8pg+PVUFNtoojeJatKhsIRx2\nWLr13nsYAsSTAAAQHklEQVR3rJVb3Jxl1pSnn07NVePHpwQyahTssUeao2DV7T//SbPeX389NXHt\numtZbrtoEey/P3zwQRq4t+yyZblto9wnknESsXb14Ydwww0pebz0Utrg6fvfT81X1rFEpNrkiSem\n7YQvuCBNBC2xRYvgwAPTFKIbb8wvkVRNn4ikYZJmSHpO0qmNnF9b0iRJkyVNlbRHwbnTJT0vabqk\nr5c6VuvEZsxIM8f79k01j5NOgpkz4Wc/cwLpqKS0dMozz8DWW8NWW8H//E8aNFFC3bql/2JbbZXL\n6ON2V9KaiKQuwHPALsBrwGPAiIiYUVBmHDA5IsZJ2gi4PSLWkbQxMB7YClgLuBtYv2G1wzURa7WP\nP4a//jXVOp57LjVaH3kkrLNO3pFZHl59Ne1dUlOTppofcECHHhJcLTWRrYHnI2JmRCwCJgDDG5Sp\nA3pnz1cEXs2efwuYEBGLI+Jl4PnsemZt8/zzaf7A2munpdaPPTbNKfi//3MC6czWXDMtUTNxIvz6\n12k15ccfzzuqilfqJLImMKvg9ezsWKGxwEGSZgG3Asc28d5XG3mvWXE++eTTmeNDhqSJgA89lLab\n3Wef1MZgBrDddvDoo6k/7JvfTP++8UbeUVWsUg8zaayq1LDtaSRwZURcKGlb4M/AgCLfC8CYMWOW\nPB86dChDhw5tTazWEb34Ytpe9qqr0mZGo0aldvAePfKOzCpZly4peeyzT+oXGzAATj891VpLNFuw\nthbOPz9NQ+rVq/2vX1NTQ01NTbtft9R9ItsCYyJiWPb6NCAi4tyCMv8Bdo+IV7PX/wW2Ab5PKnxO\ndvyfwJkR8UiDe7hPxD5r0aK0Jve4cTBlSlo196ij0nazZq1RP/DixRfhwgvT/KB2VleX/ps+9xzc\nfntpEkmhqhjiK6kr8CypY/114FFgZERMLyhzGzAxIq7OOtbvioi1CjrWtyE1Y92FO9ZtaWbOhMsu\ngz/+EdZbL9U69t4738H41nFEwG23pWSywQYpmbTzHyZ1dem/7YwZKZGssEK7Xv4zqqJjPSJqgR8B\ndwJPkzrKp0saK2mvrNhJwJGSppKSxiHZe58BJgLPALcDP3S2sM9ZvPjTmeODBqXhmXffDfffn0bX\nOIFYe5HSigX/+Q8MHZr6Tk4+GebPb7dbdOmSKtADBsCwYe166ZLxZEOrTrNmpcWIrrgi7RkxalRa\npXW55fKOzDqLOXNSP8kdd8DZZ6dm03ZaubmuDn70o1T5ueSSdrnk51RFc1Y5OIl0IrW18M9/wh/+\nkEZW7b9/akTedNO8I7PO7NFH066KdXVpPa5tt22Xy0ak5VFK1TfiJJJxEukEXnst1Tguvxy++MVU\n69hvP1h++bwjM0vq6tIck9NPT8PIzzmn4ld1roo+EbNWq6tLtY7vfAc22SQlkptugkceSft2OIFY\nJenSBQ4+OPWIr7FGWovrnHM63rrvjXBNxCrLnDlpdNVll8Eqq6Rax8iRpR/vaNaeXngBfvzjtBL0\nBRekSYvtsITK4sWpiatPn7aH6JqIdRx1dZ/OHN9oI3j55bSS7uOPp7WsnECs2qy3Xqo5/+53aT2u\nYcPSQo9tNHFi2oF57tx2iLGddIyayP335x2GtUZE2v/60ktTohg1Kg3L7d27+feaVYtFi1IyOfvs\n9P97zBhYccVWXSoi5aS77koj2VdZpfVhuWM9IyliyJC8w7DW2mCDNMJq66079IqpZrz5ZtpG+eab\n01IqRxzRql0xI9KK9bffDvfcA6uu2rpwnEQy7hMxs6oyeTKMHp06N37zG9h++xZfIiLlo1tuSYnk\nC19oeRhOIhknETOrOhHwl7/AKaekJefPOy9tTdDCS1x+eRrt3poWYHesm5lVKwlGjIDp02H99WHz\nzeGss9L2zC24xJFH5t+F6CRiZpaX5ZdP/SNPPAFPPplGJ95wQ6pmVAk3Z5mZVYpJk1J/yRe+kPpL\nSrikj5uzzMw6mp13Tnvg7L13Wj7lmGPgnXeKfntdHZx4YtouvlycRMzMKskyy6TkMT3bdmmjjdI8\nk8WLm31rly6w2mpppfrZs0sbZj03Z5mZVbJp01IT19tvpyaunXdu9i3nnZfm8N57b9ODvjzEN+Mk\nYmYdXgT89a9w0klp87Xzz4f+/Zf6lvPPh9//PiWSvn0/f959ImZmnYWU1pabPj0NBx40CM44I01Y\nbMKPf5w2tjrjjBKHVu1/xbsmYmadzqxZaaLigw+mtqsRI5pcNmjx4tTN0pCbszJOImbWaT3wQNpV\nsVevtKviFlsU/VY3Z5mZdXbbb5+2TDjoINhjj7SY6VtvlTUEJxEzs2rWtWtKHtOnQ8+esPHG8Otf\npyXoG/Hxx/DKK+13eycRM7OOYKWVUvK47760Tvxmm8Edd3yu2KRJ0J67Z7hPxMyso4lI+5aceCIM\nGJC26F1vvSWnr7wSDj/cfSJmZtYYCYYPT1vybrcdbLMNnHYaLFgAwGGHtd+tnETMzDqqHj1S8njq\nKXjtNdhwQ7jmmrTIVjtxc5aZWWfx73+nJVS6dEGPPOJ5IuAkYmbWInV1cPXV6PDDnUTAScTMrDWq\nZrKhpGGSZkh6TtKpjZy/QNIUSZMlPStpbsG52uz4FEk3ljpWK7+ampq8Q7A28OdnJU0ikroAFwO7\nAwOAkZI2LCwTESdGxBYRMRD4LfC3gtMfRMTA7Py3Sxmr5cO/hKqbPz8rdU1ka+D5iJgZEYuACcDw\npZQfCVxX8LrNVa1yae8fptZer9j3FVOuuTJNnW/p8UrQnrGV+rMrtuzSyrTmXKV+ftX2s1dM2Wr6\n2St1ElkTmFXwenZ27HMk9QX6A5MKDveQ9KikhyQtLfnkrtr+IzuJfJaTSPPnKvXzq7afvWLKVtPP\nXkk71iXtA3w9Io7KXh8IbBURoxspewqwZuE5SV+MiDmS1iEll50j4qUG73OvuplZK7RHx3ojq8y3\nq9lA4Z5aawGvNVF2BPDDwgMRMSf79yVJNcAWwEsNylRNk5eZWUdT6uasx4D1JPWT1J2UKG5uWEjS\nV4AVI+LhgmMrZu9B0qrAdsAzJY7XzMxaoKQ1kYiolfQj4E5SwroiIqZLGgs8FhG3ZkVHkDrdC20E\njJNUm733FxExo5TxmplZy1T9ZEMzM8uPF2A0M7NWcxIxM7NW69BJRFJPSY9L2jPvWKx4kjaUdImk\niZKOzjseaxlJwyVdKuk6SbvlHY8VT9I6ki6XNLHo93TkPpGsA/994OmIuD3veKxlJAm4OiIOzjsW\nazlJKwK/jIgj847FWkbSxIj4XjFlK74mIukKSW9ImtbgeHMLO+5CGhL8JlW0fEpH0trPLivzTeBW\nwMk/J235/DI/BX5X2iitMe3w2RWt4pMIcCVpAccllrawo6SDJF0I7A9sk/37/bJGbPVa89ldIOlL\nEXFLRHwDOLDcQdsSrf381pB0DnB7REwtd9AGtOFnr754sTcq9Yz1NouIByX1a3B4ycKOAJLqF3ac\nERF/Av5UX1DSwcDb5YrXPtXaz07SjpJOA3oAt5U1aFuiDZ/fscAuQG9J60XEpWUN3Nry2a0s6RJg\nc0mnRsS5zd2r4pNIExpb2HHrxgpGxDVliciK1exnFxH3AfeVMygrWjGf329J2zpYZSnms5sL/KAl\nF62G5qzGNFbV6rgjBDoWf3bVzZ9f9SrJZ1etSaQlCztaZfFnV938+VWvknx21ZJExGezaFELO1pF\n8GdX3fz5Va+yfHYVn0QkXQs8BGwg6RVJh0VELXAsaWHHp4EJETE9zzjt8/zZVTd/ftWrnJ9dh55s\naGZmpVXxNREzM6tcTiJmZtZqTiJmZtZqTiJmZtZqTiJmZtZqTiJmZtZqTiJmZtZqTiJWcpJqJU2W\n9JSkmyT1LsE9dpR0Swvf86WW7OBW8L4+kn5Q8LpV16km2fd3cN5xWOVxErFy+CAiBkbEpsC7wDEl\nuk/RM2cldY2I14vdva2BlYAfLrlp66/TriR1LeHlhwLbteQNJY7HKoSTiJXbv0lLUgMg6SRJj0qa\nKunMguNnZDuw3S/pWkknZsfvlTQwe76KpJca3kDSVpL+JekJSQ9KWj87fkhWE7oHuDtbQ+ip7Nxl\nkqZkjzez+y8v6W5Jj0t6MtttEeAXwLpZ7ercBtfpIemPkqZl9x9acO+/SvqHpGclNbpPg6SXsmtO\nk/SwpHWz43tlr5+QdKekL2THz5R0jaQHgWuyWO7PYn5c0rZZuR0l1Ui6UdILkn4haX9Jj2Rf2zpZ\nuVUl3ZAdf0TS4GxfiqOB47Ov+WuNlWssnlb8/7BqExF++FHSB7Ag+7crMBH4evZ6N2Bc9lzALcAQ\nYBAwGegO9AKeA07Myt0LDMyerwK8mD3fEbg5e94L6JI93wW4IXt+CPAK0Cd73Q+Y1iDWvqRtldcm\n/ZHVq+Bezzf2vsLXwInAFdnzrwAzs6/jEOCFLLYewMvAmo18r14CTsueHwTckj3vU1DmCNLe5QBn\nkhbW6569Xrbg+XrAYwXfn7nAalk8s4Ezs3PHARdkz8cD22XP1waeKbjPiQUxLK3cknj86PiPat2U\nyqrLcpImk5aefga4Kzv+dWC37JyA5YH1gd7ATRHxCfBJS/s6gBVJf5WvT2riKvx/fldEzGvsTZKW\nBa4HjomIWZKWAX4haQegDlhD0mrN3HsIcBFARDwr6WVgg+zcPRHxfnavZ0jJ59VGrjEh+/c64MLs\n+dpZv8uXgG6kZFPv5ux7BSlBXCxpc6CW9P2s91hEvJnd/7+khfgAniI1VwHsCmwkqX71116Slm8k\nxqWVK4zHOjgnESuHhRExMPslfQepT+RiUuL4RURcVlhY0vFLudZiPm2GXbaJMmcBkyLiu1lTzL0F\n5z5YyrUvIdVa6ssfAKwKbBERdVnTWVP3XBL+Ul5/XPC8lqZ//gr7duqyf38L/CoibpO0I+kv/nqF\nX9MJwJyI2Czrk/iwifvXFbyuK4hFwLYNk8CnueLTQ0spt7TvsXUw7hOxchBARHwEjAZOzn7B3QEc\nXv8XrKQ1srb+B4FvZv0LvYC9Cq71MrBl9nzfJu7Xh0//wj+sqAClY0hNV79scJ03swSyE6nmALAA\nWKGJS91PSj5I2oDU1PNsMTEU2C/7dwSpDwlS7ax+A6FDlvLePsDr2fODSU2ILXEnqXkLAElfzZ4u\nyGJorpx1Mk4iVg5L/rKOiKnAVGBERNxFarL5t6RppKakXhHxOGmznCeB24BpQH0T1K+AH0h6Ali5\nifudB5yTlSn2//iPgU2zjvXJko4itftvJelJ4EBgevY1zAX+lXV+N+wg/z2wTPb1XAccEhGLlvY9\nacRK2T2PJdUsAMYCN0h6DHhrKe/9PXCopCmkZrSmagVN3X80sGXW2f4fYFR2/BbgO/Ud66QE0lg5\n62S8n4hVJEnLR8QHkpYj/XV/ZJaAOrSsyWxQlqjMKp77RKxSXSppY9JIpqs6QwLJ+K86qyquiZiZ\nWau5T8TMzFrNScTMzFrNScTMzFrNScTMzFrNScTMzFrt/wEMdMQN2Huj+QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13f7d1fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# List of regularizer parameters\n",
    "reg_params = [0.0001, 0.0001, 0.001, 0.01, 0.1]\n",
    "num_params = len(reg_params)\n",
    "\n",
    "# Iterate over parameters\n",
    "tr_acc = np.zeros(num_params,)\n",
    "ts_acc = np.zeros(num_params,)\n",
    "\n",
    "for i in range(num_params):\n",
    "    # Train neural network with L1 regularization\n",
    "    model = train_nn_l1(x_train, y_train, 100, reg_params[i])\n",
    "    \n",
    "    # Evaluate train and test accuracy\n",
    "    tr_acc[i] = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "    ts_acc[i] = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "# Plot train and test accuracy as a function of reg parameter\n",
    "plt.semilogx(reg_params, tr_acc, '--b', label='Train')\n",
    "plt.semilogx(reg_params, ts_acc, 'r', label='Test')\n",
    "plt.xlabel('Regularization parameter')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('L1 regularization')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train neural network with early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn_earlystop(x_train, y_train, num_hidden_nodes, x_val, y_val):\n",
    "    # Construct neural network model\n",
    "    model = Sequential()    \n",
    "    model.add(Dense(num_hidden_nodes, activation='relu', input_shape=(x_train.shape[1],))) # hidden layer\n",
    "    model.add(Dense(1, activation='sigmoid')) # output layer\n",
    "    \n",
    "    # Early stopping call back\n",
    "    e_stop = EarlyStopping(monitor='val_loss', patience=0, verbose=0, mode='auto')\n",
    "    \n",
    "    # Optimizer: SGD (loss = binary cross entropy)\n",
    "    sgd = SGD(lr=1, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    \n",
    "    # Fit neural network\n",
    "    model.fit(x_train, y_train, epochs=100, batch_size=100, verbose=0,\\\n",
    "              callbacks=[e_stop], validation_data=(x_val, y_val))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain neural network\n",
      "Train accuracy: 1.0\n",
      "Test accuracy: 0.744966444953\n",
      "\n",
      "Neural network with early stopping\n",
      "Train accuracy: 0.89189188867\n",
      "Test accuracy: 0.818791947509\n"
     ]
    }
   ],
   "source": [
    "model = train_nn(x_train, y_train, 100)\n",
    "\n",
    "# Evaluate train and test accuracy\n",
    "tr_acc = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "ts_acc = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "print('Plain neural network')\n",
    "print('Train accuracy: ' + str(tr_score))\n",
    "print('Test accuracy: ' + str(ts_score))\n",
    "\n",
    "print\n",
    "\n",
    "model = train_nn_earlystop(x_train, y_train, 100, x_test, y_test)\n",
    "\n",
    "# Evaluate train and test accuracy\n",
    "tr_acc = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "ts_acc = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "print('Neural network with early stopping')\n",
    "print('Train accuracy: ' + str(tr_acc))\n",
    "print('Test accuracy: ' + str(ts_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train neural network with dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn_dropout(x_train, y_train, num_hidden_nodes, do_rate):\n",
    "    # Construct neural network model\n",
    "    model = Sequential() \n",
    "    model.add(Dropout(do_rate, input_shape=(x.shape[1],))) # drop out on input nodes\n",
    "    model.add(Dense(num_hidden_nodes, activation='relu')) # hidden layer\n",
    "    model.add(Dropout(do_rate)) # drop out on hidden nodes\n",
    "    model.add(Dense(1, activation='sigmoid')) # output layer\n",
    "    \n",
    "    # Optimizer: SGD (loss = binary cross entropy)\n",
    "    sgd = SGD(lr=1, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    \n",
    "    # Fit neural network\n",
    "    model.fit(x_train, y_train, epochs=100, batch_size=100, verbose=0)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x157d9cb90>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEZCAYAAABWwhjiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VPXZ//H3zSYqhM26sbpgxbpUFIFWaypVkVqxLhWs\nQBUtjxuWqj+0jwtoba3WpxWtFhUFlUVc6l530SqVRUCrDasIREXRsERABXL//viewDgkMElm5sxM\nPq/rysWZs82dQ5J7vru5OyIiIrXRIO4AREQkfymJiIhIrSmJiIhIrSmJiIhIrSmJiIhIrSmJiIhI\nrSmJiIhIrSmJSL1nZh+a2TozW21mZWb2hpkNMTOLO7ZkZrbYzI6JOw6RSkoiIuDAT929BdARuBEY\nDoyp6mQz0++NSES/DCKBAbh7ubs/DZwBDDSzA8zsPjO7w8yeMbNyoNjMiszsfjP7LCod/O/mG5kN\nikozo8xslZn9N7H0YGZ7mNkTZvaFmc03s3MTjt1nZtclvD7azJZF2/cDHYCnzGyNmV2W8acish2N\n4g5AJBe5+wwz+wg4KtrVHzjB3d8ysx2Au4HmQCfgO8ALZvaxu98Xnd8dmAy0AU4FHjOzTu6+CpgE\nvAvsDhwAvGhmi9z91erCiWIaaGZHAeds41yRrFJJRKR6HwOto+0n3P2taHsD8AvgCndf5+5LgFuA\nAQnXfuruo9x9k7tPBuYBPzWzdsAPgOHuvsHd3wHuSbp2e3KurUbqLyURkeq1Bcqi7WUJ+3cBGgNL\nE/Ytic6v9FHSvZYAe0ZfZe6+bhvXiuQNJRGRKphZN8If/DeiXYnTXX9OKI10TNjXkW8njuSk0IFQ\nsvkYaG1mOycdq7x2LbBTwrE9ku6jabclpyiJiCQws+ZmdiIwEXjA3d9PPsfdKwjtHTeYWTMz6wgM\nAx5IOG1XM7vYzBqZ2enA/sAz7l4KTAX+aGY7mNnBwGDgwei6OUAfM2tlZrsDlyS9/XJg7/R9xyJ1\noyQiEjxlZqsJVVRXAn8GztnG+UOBdcAHwOvAgwmN6gDTgM6EUsv1wKlRozqERvq9CKWSR4Gr3f2V\n6NgDhEb3D4HnCI3wiW4Ero7Gs/y2Ft+nSFpZJhelMrMxwImERsaDqzlnFHACoRj/K3efE+0fBPwv\nofh+g7vfn7FARdIo+tkd7O4/ijsWkUzLdEnkPuD46g6a2QnAPu7eGRgC/D3a3wq4BuhG6Cp5rZm1\nyHCsIiJSQxlNIu7+BrByG6f0Be6Pzp0GtDCz3QiJ5wV3Xx1VAbwA9M5krCIiUnNxt4m05dtdJ0uj\nfcn7P0JdICVPuPs4VWVJfRF3EkkeNGWENpCqBlOpa6OISI6Je9qTUqB9wut2hB4rpUBx0v4qp3kw\nMyUXEZFacPc6z36QjZKIUf00DU8CAwHMrAewyt0/BZ4HjjWzFlEj+7HRviq5e85/XXvttbHHoDgV\np+JUjJVf6ZLRkoiZTSCUKNqY2VLgWqAJ4O5+l7s/a2Z9zGwhoYvv2YSDK83semAmoRprpG/pYy8i\nIjkio0nE3c9M4ZyLqtk/Fhib5pBERCSN4m5YrzeKi4vjDiElijO9FGd65UOc+RBjOmV0xHo2mJnn\n+/cgIpJtZoanoWE97t5ZIiK10qlTJ5YsWRJ3GDmvY8eOfPjhhxm7v0oiIpKXok/ScYeR86p7Tukq\niahNREREak1JREREak1JREREak1JREQkx1VUVNC8eXNKS0vjDmUrSiIiImnWvHlzioqKKCoqomHD\nhuy0006b902cOLHG92vQoAHl5eW0a9cuA9HWjXpniUheypfeWXvvvTdjxozhxz/+cbXnbNq0iYYN\nG2bk/dU7S0Qkj1U14eHVV19Nv379OPPMM2nRogXjx4/nrbfeomfPnrRq1Yq2bdtyySWXsGnTJiAk\nmQYNGrB06VIABgwYwCWXXEKfPn0oKirihz/8YWxjZpRERERi8Pjjj3PWWWexevVqzjjjDBo3bsyo\nUaMoKyvjzTff5Pnnn2f06NGbzzf7dqFh4sSJ3HDDDaxcuZL27dtz9dVXZ/tbAJRERKSAjRgBZlt/\njRiR2vnVnZcORx55JH369AFghx124LDDDqNbt26YGZ06deK8887jtdde23x+cmnmtNNO49BDD6Vh\nw4b88pe/ZM6cOZkLdhs07YmIFKwRI2qWCGp6fl20b9/+W6/nzZvHpZdeyttvv826devYtGkT3bt3\nr/b63XffffP2TjvtxJdffpmxWLdFJRERkRgkV08NGTKEgw46iA8++IDVq1czcuTIvOg4oCQiIpID\nysvLadGiBTvuuCMlJSXfag/JZUoiIiIZlFziqM4tt9zC2LFjKSoq4vzzz6dfv37V3ifVe2aDxomI\nSF7Kl3EicdM4ERERyVkFkUTWro07AhGR+qkgkshll8UdgYhI/VQQSeSf/4Rnn407ChGR+qcgksi4\ncXDeefD553FHIiJSvxRM76zLL4cPPoBHHgnTFYhIYVPvrNSod1aKrr8eNm6EFSvijkREpP4omJKI\niNQvKomkRiURERHJWUoiIiJplu7lcSv17NmTCRMmpDHSutNU8CIiaVZeXr55O5XlcfNZwZZENm6E\nm26Cr7+OOxIRqc+qWh63oqKC66+/nn322Yddd92VAQMGsGbNGgDWrVtH//79adOmDa1ataJnz56s\nXr2ayy67jBkzZnDuuedSVFTE5ZdfHse3s5WCTSING8Kbb2ZvgRkRkVTddNNNvPTSS0ydOpXS0lIa\nN27MsGHDALjnnnvYtGkTn3zyCV988QW33347TZo04c9//jPdunVjzJgxrFmzhptvvjnm7yIo2CRi\nBnffDWPHwhtvxB2NiMSiqrVxa/qVAXfddRc33ngju+22G02aNOHqq69m0qRJADRu3JgVK1awYMEC\nGjRowGGHHcaOO+64+dpc65FW0G0iu+4Ko0fDwIHwzjvQvHncEYlIVuXYH9xKy5Yto0+fPpvXBalM\nDGVlZQwePJjly5dz2mmnsXbtWgYMGMDvf//7nFpDJFHBlkQqnXQS9OoFUUlRRCR27dq145VXXqGs\nrIyysjJWrlzJ2rVrad26NU2aNGHkyJGUlJTw+uuv8/DDD28upeRiIin4JALwf/8HO+0UGttFROI2\nZMgQhg8fTmlpKQCfffYZTz/9NAAvv/wyJSUluDvNmjWjUaNGNGoUKo122203Pvjgg9jirkq9SCLN\nm8OoUdCooCvvRCQXVVV6GD58OMceeyzHHHMMLVq04Mgjj2T27NkAfPTRR/Tt25eioiIOPvhgTjzx\nRE4//XQAhg0bxrhx42jTpg1XXHFFVr+P6mR82hMz6w38lZCwxrj7n5KOdwDuBb4DfAGc5e4fR8c2\nAe8ABixx95OruL+mPRGphzTtSWoyPe1JRpOImTUA5gO9gI+BGUA/d5+bcM5k4El3f9DMioFz3H1g\ndGyNuxdt5z2URETqISWR1OT73FlHAAvcfYm7bwAmAX2TzjkAeAXA3ackHc+9ViQREdks00mkLbAs\n4XVptC/RHOBUADM7BWhmZq2iYzuY2XQzm2pmycmn1j7/HC64ADZtStcdRUTqp0w3NVdVkkguV10O\n3G5mvwJeBz4CKvtRdXD35Wa2F/CKmb3r7ouTbzgiYVh6cXExxcXF2wyqdWsoKYGbb4YcaZsSEcmo\nKVOmMGXKlLTfN9NtIj2AEe7eO3p9BeDJjesJ5+8MlLh7hyqO3Qc85e6PJe2vVZvIkiVw+OHw4ovw\n/e/X+HIRiZnaRFKT720iM4B9zayjmTUB+gFPJp5gZm1sSx+4Kwk9tTCzltE1mNkuwA+A/6YrsI4d\nw/iRs86Cr75K111FROqXjCYRd98EXAS8ALwPTHL3EjMbaWYnRqcVA/PMbC6wK3BDtL8LMNPMZgMv\nA39M7NWVDmedBfvvD1ddlc67ikg2dOzYETPT13a+OnbsmNH/h3q/PO7nn8Pjj8O556YxKBGRHJcX\n40SyQeNERERqLl/aREREpIApiYiISK0piYiISK0piSSZPRv698/ZtWxERHKKkkiSAw+EBQvCiogi\nIrJt6p1VhZIS+NGPYOpU6Nw5rbcWEckJ6p2VQV26wDXXwIABWg1RRGRblESqceGFYUXEG2+MOxIR\nkdyl6qxtKC0NI9o1QaOIFBqNWI9oxLqISM2pTURERGKnJCIiIrWmJFJD6q0lIrKFkkgNTJgAAwfG\nHYWISO5QEqmBk0+GWbNg0qS4IxERyQ3qnVVDM2dCnz5hjq22bbP2tiIiaaXeWTE5/HC4+GI4+2yo\nqIg7GhGReCmJ1MKVV8KaNXDffXFHIiISL1Vn1VJpKbRsCc2aZf2tRUTqTCPWIxqxLiJSc2oTERGR\n2CmJiIhIrSmJpMn69bBuXdxRiIhkl5JImowcCZddFncUIiLZpYb1NFm9Gg4+GP7+dzjhhLijERHZ\nNjWs55gWLWDsWDj3XPjii7ijERHJDpVE0uzSS2HpUpg8GazOOV5EJDNUEslRN9wAJSXw+utxRyIi\nknkqiWTAypVhNLtKIiKSqzRiPZKLSUREJNepOktERGKnJCIiIrWmJJIF8+fDN9/EHYWISPopiWTB\nVVfBiBFxRyEikn4ZTyJm1tvM5prZfDMbXsXxDmb2kpm9Y2avmNmeCccGRdfNM7OBmY41U267LSxg\n9eabcUciIpJeGe2dZWYNgPlAL+BjYAbQz93nJpwzGXjS3R80s2LgHHcfaGatgJlAV8CAt4Gu7r46\n6T3yonfW44+HgYhz5kDz5nFHIyL1Xb70zjoCWODuS9x9AzAJ6Jt0zgHAKwDuPiXh+PHAC+6+2t1X\nAS8AvTMcb8acfDIUF8Nvfxt3JCIi6ZPpJNIWWJbwujTal2gOcCqAmZ0CNItKIcnXflTFtXnlL3+B\nt94KS+uKiBSCTCeRqopKyXVPlwPFZvY2cBQhWWxM8dq8UlQUqrPatYs7EhGR9GiU4fuXAh0SXrcj\ntI1s5u6fsKUksjNwqruXm1kpUJx07atVvcmIhK5PxcXFFBcXV3VaTmjYMO4IRKQ+mjJlClOmTEn7\nfTPdsN4QmEdoWP8EmA70d/eShHPaAGXu7mb2e2Cju49IalhvEG0fFrWPJL5HXjSsi4jkkqw1rJvZ\nRdEf9Bpz903ARYRG8feBSe5eYmYjzezE6LRiYJ6ZzQV2BW6Irl0JXE9IHtOAkckJRERE4rXdkkhU\nOugHzALuBZ7PpY/++VwScYennoKf/lTVXCKSXVmdxdfMDDgOOBs4HJgMjHH3RXUNoK7yOYlUVMAx\nx0CfPvD//l/c0YhIfZLVcSLRX+nl0ddGoBXwiJndVNcA6rMGDWDcOLj5ZnjnnbijERGpuVSqs4YC\ng4DPgXuAx919QzQafYG775P5MLcZX96WRCqNGwe33ALTp0PTpnFHIyL1Qdaqs8zsOkLV1ZIqjnVJ\n7GkVh0JIIu5w2mmw996hVCIikmnZrM56FihLeOPmZtYdIO4EUijMYPRo+M9/4Ouv445GRCR1qZRE\nZhMmPvTodQNgprt3zUJ821UIJRERkWzLZknkW3+l3b2CzI90FxGRPJBKEvnAzIaaWePo6xLgg0wH\nJiIiuS+VJPI/wA8IEyOWAt2BX2cyKBERyQ8ZnTsrGwq1TWTtWrj7brjkktDwLiKSTulqE9lu24aZ\nNQUGA98DNo9icPdz6vrmUr0mTeCBB2CnneDXKveJSI5KpTrrAWB3wkqDrxGmZC/PZFACjRuHJPK/\n/wsLF8YdjYhI1VLq4uvuh5rZu+5+sJk1Bv7l7j2yE+K2FWp1VqVbb4WHHoLXX4dG6hMnImmSzS6+\nG6J/V5nZgUALwpTtkgUXXxyqtP70p7gjERHZWipJ5K5oPZGrgCeB/wL6k5YlDRrA2LGwSiupiEgO\n2mZ1VjQ6/TR3n5y9kGqm0KuzREQyISvVWdHodK10ISIiVUqlYf1GwjTwDwFrK/e7e1m1F2WRSiIi\nIjWXzangF1ex291977q+eTooiYiI1FxWl8fNZfUxiSxaBPfeCzfcEHckIpKvsjlifWBV+939/rq+\nudTOHnvAo4/CwQfDGWfEHY2I1GepVGfdlvCyKdALmOXup2UysFTVx5IIwIwZcOKJMGsWtG0bdzQi\nkm9iq84ysxbAQ+7eu65vng71NYkAjBwJU6fCc89pkkYRqZlsjlhPtg7Yq65vLHX3u9+FQYh33BF3\nJCJSX6XSJvIUUPlRvwFwAJCzgw/rk8pJGufMiTsSEamvUmkTOTrh5UZgibuXZjSqGqjP1VkiIrWV\ntd5ZwFLgE3f/KnrjHc2sk7t/WNc3FxGR/JZKm8jDQEXC603RPhERqedSSSKN3P2byhfRdpPMhSQi\nIvkilSSywsxOqnxhZn0Jc2lJDnr+ebjssrijEJH6IpWG9X2A8cCe0a5SYKC758SirWpY/7ZVq8JI\n9rvvhuOPjzsaEclVWR9saGbNovNzan11JZGtvfwyDBoE77wDbdrEHY2I5KKsDTY0sz+YWUt3/9Ld\ny82slZn9vq5vLJnTqxecfjqcfz4ov4pIJqXSJnKCu29enNXdVwJ9MheSpMMf/gDvvQcTJsQdiYgU\nslTGiTQ0sx3c/WsI40SAHTIbltTVjjvCQw9B06ZxRyIihSyVJPIg8LKZ3Re9PhsYl7mQJF0OOiju\nCESk0G23OsvdbwJ+D3QhzJv1HNAx1Tcws95mNtfM5pvZ8CqOtzezV8xslpnNMbMTov0dzWxdtH+W\nmWmaQRGRHJNKSQRgOWHU+i+AxcCjqVxkZg2A2wlrkHwMzDCzJ9x9bsJpVxGmlh9tZl2AZ9kyS/BC\nd++aYowiIpJl1SYRM9sP6Af0B74AHiJ08f1xDe5/BLDA3ZdE95wE9AUSk0gFUBRttwQ+SgyjBu8l\n2+EOFRXQsGHckYhIodhWddZcQgniZ+5+pLvfRpg3qybaAssSXpdG+xKNBAaY2TLgaeDihGOdzOxt\nM3vVzI6s4XtLkptugmuvjTsKESkk20oipxKqsV41s7vNrBc1LxlUdX7yyIX+wH3u3h74KaEhH+AT\noIO7HwZcCkyIBjxKLQ0aBPfcE1ZDFBFJh2qrs9z9H8A/zGxn4GRgGLCbmd0J/MPdX0jh/qVAh4TX\n7QhtI4kGA8dH7/mWmTU1s13c/XPgm2j/LDNbBOwHzEp+kxEjRmzeLi4upri4OIXQ6p/dd4c774SB\nA8NCVs2UkkUK1qZN8OSTcNttMGYMLFkyhSlTpqT9fWq0xrqZtQZOB85w92NSOL8hMI9QLfYJMB3o\n7+4lCec8A0x293FRw/qL7t7OzHYByty9wsz2Bl4DDkoc+Bhdr2lPaujss6FJExg9Ou5IRCTdVq0K\nSeP228MHx0sugVNPDSuhJsr63Fm1fgOz3sCthKqzMe5+o5mNBGa4+9NR4rgbaEZoZL/c3V82s1OA\n64ANhLaYa9z92SruryRSQ2vWhEka77kHfvKTuKMRkXR55BE47zzo0yckjyOOqP7cvEkimaYkUjvz\n50PHjrCD5h4QKRjLl4cemHvuuf1zlUQiSiIiUt+sWwc77VS3e2RtFl8REckNixbBb34D7drBBx/E\nHU2gJCIiksPcwxpBJ50E3buHKug5c2DvveOOLEh12hMpcGvWwM47azS7SK75+9/hb3+DoUNh0qS6\nV2Olm9pEBIAzz4SuXbU+e1p9+WX4jW+gAr/U3jffhO65luZJoNSwHlESSY8PP4Ru3cLgpK5d1Wsr\nZeXlsGABLFwY/k3cXrUqdIG76CL41a+gqGi7t5P6yR3eeitUV2XrM4eSSERJJH0mToRhw6CsLHzy\nefRR6N176/OefBK++AJatw5ruLduHb522QUaFWIFaXl51Uli4cJQD7jvvtC585Z/K7f32AP+/W8Y\nNQpefBHOOisklP32i/s7khzx9dehiurWW0PB9aWXoEOH7V+XDkoiESWR9HMPP9BNmlRdIrnzTpg2\nLSSSsrIt/06cGNZ3T3b77aH/emXCqfz3oIOgefPMfz8pqUwUVSWLykRRVbLYY4/UPjqWloYHd/fd\ncPjhoYL7uONU1VVPffJJ+HG46y445JAwMLB37+z+OCiJRJREct8jj0BJyZZkU5l47rgDDj106/Mv\nvRQ+/njrpNOnT9iutS+//HYpIjFZrF4N++xTdYlizz3T99u9fn346DlqVOjsf/HFYWbMnMmmkg3j\nx8Obb4b//i5d4olBSSSiJFJ43ngDliz5dimnrAxGjgx/55P9/Och6bRuDXsWfcl+DRay16aFnNB5\nAc2XJySLVatgn33wzp2x5GSRzkSRCvfwjd52W+i/OWBAqOrad9/sxSD1mpJIREmkHlq79lslijWz\nQpJosnQhjcpXsqrNPqwo2pcOvTqz8/cTkkXbttCgAQceuHX1Wps2cOONoXYq2ccfhxmPmzdPfw8Z\nAJYtC3Ub99wTJjsaOhSOPTZDbybZsmIF3Hdf+O9s2jTuaLamJBJREilQa9eG4bnJ7RMLFsDKlWGk\nVWKVU+V2lCi2ZePGcIvkNp2f/7zqDlQ//CG8+y589dWWTgStW4eOB7vvvvX5U6fCjjuGc9q1q8HY\nm/XrQ8PSrbeGfp0XXxzm7dec/XnlnXdCbeVjj8Epp4QPJ9/5TtxRbU1JJKIkksfWrau+MbusLCSK\n5PaJzp3DX+YYGqS/+ebbbTpHHFF1x4NTTglTUnz+eWhqOfxw6NEDrrkmJJftcod//Sv8JXr11dBm\ncuGFVdflSc544w246qrwI3zBBWE23VxMHpWURCJKIjlu3botJYrkZPHFF7DXXtWXKApg+HxZGcyY\nATNnwu9+t3UNlXsogFQ7CnnJklDVNWYM9OwZ6kZ69VJVVw56881QM1nV2h25SEkkoiSSA9avr77q\nqTJRVFeiKIBEURdLl4beOfvvHwaa9egR/u3cOamwtW4dTJgQSicbN4ZkMmBAmKtGpBaURCJKIlmS\nmCiSSxQrVlRdoth3X2jfvt4niu356iuYPTuMWJ42Lfy7337wQlULULvDa6+FZPLaa2Ek/IUX5s5s\nfAWsogKeey4UDMeOrWN38xygJBJREkmjioowoKOqEkVloqiqRKFEkXbVrRfx9tthEHz37nBIiw9p\ncs8dcO+9ofV/6FA45hhVdaVZeXlIGrfdFvo4XHIJ9OuX/1MDKYlElETSZNmyMC3H0qVw4IFbJ4sO\nHZQocsDMmTB6dCixLFoURjsf1XUtQ5qPZ++nRoWTLr44/F+qqqvOJkwIj/OYY0Ly+OEPCydHK4lE\nlETS4OGHw0C33/42TOOrZJEXystDUpk2LYz8P/44D725Ro0KXYXOPpuK8y+kwd6d4g41by1eHH4d\nsjWfVTYpiUSUROrgyy9DFci//hU+cnXrFndEki6LF8Pf/kb5bWOZvsNRzDpyKC1PLqZ7D+N739Pn\nhGQbNuRHj6p00vK4UjczZoSPr2ahVVcJpLDstRf8+c80/XQJ+17Um8GzL6L38IOZfOzd7NFiHbNn\nxx1gbli6FIYPDx0FP/007mjyk5JIfbNpUxhC+9Ofwh/+EMYfaER0wWrccmc6/mEIrT9+j/aP/JXf\n93ia5U07cvD44WEMSpJ77w01YevXxxBsllROW3b66eFz1DffhFkGdtst7sjyk5JIfVJaCj/5Cfzz\nn6Ey/fTT445IssUsDFJ84gkaTJ9GQ98YVh879dTQVdgd9zBlx7BhYW2Yww4LI6/HjQsd9wrFLbfA\nOefA0UeHxdj+8hdNBlAXahOpLx57DM4/P3QxGT5cleIS2sQeeCA0xDdpEtrHzjwTdtyR9etDLee0\naTB/fhgbUSjWrg3Tz9T3pVzUsB5REtmOtWvhN78JvXbGjw8DDEQSuYcl9UaNCllj8ODwgWM7XZJm\nzYIzztgyyr5799DluEmTLMW9He+9F3qrS9XUsC7b9/bbocpiw4bwsVIJRKpiFqaef+qp0Djw1Veh\nseD00+H110OSqcIhh8A//gHFxWGW48GDoVWrUB0Wlw0bwppfP/gBnHhimARTMkslkUJUUREqfm++\nOXy67Ncv7ogk35SXb6nqato0VHX177/daYjLy8N0aZ06bX3svfdCD6hu3aqecr8uVqwIS83eeWcY\nHzt0KJx0kmptt0XVWRElkSQffRSmDv/6a3jwQejYMe6IJJ9VVGyp6po+Hc49N1R1tW9f41s9/nj4\nbDN7dkgylRNO9u5dq9t9y6hRoVPA0KGhhCTbpyQSURJJ8MQTMGRImJDvyiuhUaO4I5JCsmAB/O1v\noYTSq1f4i12LeUA2bID//GfLhJO/+EXocS7ZpSQSURIhzNb329+GaV/Hjw/rTohkSnl56Pd7221h\nfq6hQ0OVaZrXgB00KPxoV5ZY9tknlGZ+/WtVU6WDGtYlmD07dOhfuxbmzFECkcxr3jzMtVZSEgas\nTp4cqk2vuipUp6bJNddA375hLMewYaGt4403Qg6T3KGSSL6qqIC//hX++Mfw7y9/GXdEUp/Nnw+3\n3x7a4Y47LpROevYsnClvC5CqsyL1Mol88klYjKi8PFRf7bVX3BGJBGvWbFl8o0WLkEzOOCP/F98o\nQKrOqq+eeir04e/ZM/ThVwKRXFJUFBLHvHlw3XVhduiOHUPd1Mcfxx2dZIBKIvli/fqw1sezz4be\nMUceGXdEIqmZOzdUdU2YEPrzDh0aWstV1RUrlUTqk3ffhcMPh7Ky0JCuBCL5ZP/9QxJZvBiOOCK0\n33XvHtpPvv467uikjjKeRMyst5nNNbP5Zja8iuPtzewVM5tlZnPM7ISEY1ea2QIzKzGz4zIda85x\nh1tvDX3yhw8Pn+Ratow7KpHaadEizOM2f36o3rr//jDq8NprQzuf5KWMVmeZWQNgPtAL+BiYAfRz\n97kJ54wGZrn7aDPrAjzr7nuZ2QHAeKAb0A54CeicXHdVsNVZn34aGs9XrgyN55qrWgpRSUkopUyc\nCCecsKWqSzIuX6qzjgAWuPsSd98ATAL6Jp1TAVTOpNMSqOxofhIwyd03uvuHwILofoXvmWfg+98P\nVVj/+pcSiBSuLl3CKPgPPgg/7/37hyQyfnxYLUpyXqaTSFtgWcLr0mhfopHAADNbBjwNXFzNtR9V\ncW1h+epB+6LCAAAMoUlEQVSr8EnsggvgoYfg+uvr38LPUj+1bBlGFC5YEAYtjh0benWNHAnLl8cd\nnWxDpidXqqqolFz31B+4z93/YmY9gAeB76V4LQAjRozYvF1cXExxcXFtYo3Xe++FT2EHHBBGnrdq\nFXdEItnXsCH87Gfh6/33Q1VXly5hXvchQ2D33eOOMG9NmTaNKdOmpf2+mW4T6QGMcPfe0esrAHf3\nPyWc8x5wvLt/FL1eBHQHziWcfGO0/zngWneflvQe+d0m4h6K8yNHwk03hXYQdX0U2WLlyrD4+/33\nh+l9JC1s0aLcH7FuZg2BeYSG9U+A6UB/dy9JOOcZYLK7j4sa1l9093YJDevdCdVYL1JoDeuffRYW\ne/7009DzqnPnuCMSkXoiLxrW3X0TcBHwAvA+oaG8xMxGmtmJ0WmXAeeZ2RxC0hgUXftfYDLwX+BZ\n4IL8zRZVeO650Hh+8MFhNTklEBHJQxqxnm1ffRXW+nj00VA8z8f2GxHJe+kqiWjVomz6739D43nn\nzqHxvHXruCMSEakTTXuSDe5h8eejjw5deB9+WAlERAqCSiKZtmJFWJe6tDSsqPPd78YdkYhI2qgk\nkkkvvhgaz/ffH/79byUQESk4Kolkwtdfw+9+F5YNfeABOOaYuCMSEckIJZF0mzs3NJ536hQaz9u0\niTsiEZGMUXVWurjD6NFw1FFw/vnw2GNKICJS8FQSSYfPPw+N50uWhFl3998/7ohERLJCJZG6evnl\nsOb5vvvCW28pgYhIvaKSSG19802Ysnr8+DBt9bHHxh2RiEjWKYnUxrx5YZ3oPfcMjeff+U7cEYmI\nxELVWTXhDvfcA0ceCYMHwxNPKIGISL2mkkiqysrgvPNg4UJ47bWweJSISD2nkkgqXn0VDjkEOnSA\nadOUQEREIiqJbMuGDXDNNTBuXFhZrXfvuCMSEckpSiLVWbAAzjwTdt01NJ7vumvcEYmI5BxVZyVz\nD6WOnj1h0CB4+mklEBGRaqgkkmjlShgyBEpKQjvIQQfFHZGISE5TSaTS66+Hadt33x2mT1cCERFJ\ngUoiGzbAiBGhCmvMGOjTJ+6IRETyRv1OIosWhcbz1q1h9uxQChERkZTVz+os99Btt0ePkESeeUYJ\nRESkFupfSWTVqrDex7vvwksvhUGEIiJSK/WrJPLGG6HxvHVrmDlTCUREpI7qR0lk40a47jq4667w\nddJJcUckIlIQCj+JLF4cpm1v1iw0nu+xR9wRiYgUjMKuznrwQTjiCDjtNHjuOSUQEZE0K8ySyOrV\ncOGF8Pbb8MILYflaERFJu8IriUydGpJGs2YhiSiBiIhkTOGURDZuhBtugDvugNGj4eST445IRKTg\nFUYS+fBDOOssaNo0NJ7vuWfcEYmI1AuFUZ11xBHQt29o/1ACERHJGnP3uGOoEzNznzkTDjss7lBE\nRPKGmeHuVuf7FEQSyfPvQUQk29KVRAqjOktERGKR8SRiZr3NbK6ZzTez4VUc/z8zm21ms8xsnpmV\nJRzbFO2fbWaPZzpWERGpmYwmETNrANwOHA98D+hvZvsnnuPuv3X3Q929K3Ab8FjC4bXu3jU6ntd9\ndqdMmRJ3CClRnOmlONMrH+LMhxjTKdMlkSOABe6+xN03AJOAvts4vz8wMeF1nevrckW+/GApzvRS\nnOmVD3HmQ4zplOkk0hZYlvC6NNq3FTPrAHQCXknYvYOZTTezqWa2reQjIiIxyPRgw6pKEtV1peoH\nPJLU1aqDuy83s72AV8zsXXdfnPYoRUSkVjLaxdfMegAj3L139PoKwN39T1WcOwu4wN3fquZe9wFP\nuftjSfvVv1dEpBbS0cU30yWRGcC+ZtYR+IRQ2uiffJKZfRdomZhAzKwlsM7dvzGzXYAfAFsln3Q8\nBBERqZ2MJhF332RmFwEvENpfxrh7iZmNBGa4+9PRqf0Ije6JugCjzWxTdO0f3X1uJuMVEZGayfsR\n6yIiEp+cHrGewkDFo8zsbTPbYGanJB0bFF03z8wG5nCcWRtQmUKcw8zsfTObY2Yvmln7hGO59Dy3\nFWdWnmcKMQ4xs3ejOF5PHB9lZlea2QIzKzGz4zIVY13iNLOOZrYuepazzOyOOONMOO80M6sws64J\n+3LmeVYXZ649z+j3+bOEeM5JOpb677q75+QXIcEtBDoCjYE5wP5J53QADgTGAqck7G8FLAJaAC0r\nt3MtzujYmhx6nkcDTaPt/wEm5ejzrDLObD3PFGNslrD9M+Cf0fYBwGxCVXKn6D6Wg3F2BN7NlZ/N\nyliB14CpQNdoX5dcep7biDOnnicwCBhVxbU1/l3P5ZLIdgcquvtSd3+PrbsNHw+84O6r3X0VoU2m\ndw7GCdkbUJlKnK+5+1fRy7fYMqYn155ndXFCdp5nKjF+mfCyGVARbZ9ESHob3f1DYEF0v1yLE3Lo\nZzNyPaFzzdcJ+/qSQ89zG3FC7j3PquKp8e96LieRlAcqpnDtRzW4tqbqEidkb0BlTeMcDPyzmmtz\n6XkmxgnZeZ4pxWhmF5jZQuBGYGg118b+LKuJE6BTVA37qpkdmaEYU4rTzL4PtHP3Z7dzbazPcxtx\nQg49z8gpUZXwZDOrPF7j55nLKxvWZKBiOq+tqbq+V7YGVKYcp5mdBRxGqDaq0bVpUJc4ITvPM6UY\n3f0O4A4z6wdcDfwq1WvTpC5xfkJ4liujev3HzeyApJJLVuI0MwP+QqiCqdG1aVabOCuvyZnnGXkS\nmODuG8xsCHA/0CvFa78ll0sipYS2hErtgI+zcG1N1em93H159O9iYApwaDqDS5BSnGb2E+BK4GdR\nUTjla3Mgzmw9z5o+j4eAyglES4H2Ccdif5YJNsfp7t+4+8poexahbny/mOJsTpjAdYqZLQZ6AE9G\nf4xz6WezqjifMLOuOfY8cfeVCb83dwNdU712K9lo6Kll41BDtjQONSE0DnWp5tz7gFOraRyq3G6Z\ng3G2BJpE27sA86iioS5bcRL+4C4E9knan1PPcxtxZuV5phjjvgnbPwOmR9uVDetNgL3IbENwXeLc\nBWgQbe9NqOKI/XcoOv9V4NBcfJ7biDOnniewe8L2z4Gp0XaNf9fT/g2k+WH0jv4QLACuiPaNBE6M\ntg+P/jPKgRXAfxKu/VV03XxgYC7GCfQE3o1+Cd4BfhVznC8Sit2zopgez9HnWWWc2XyeKcT4V+C9\nKMaXE3+JCSWohUAJcFzMz7LKOIFTov2zgZlAnzjjTDr3FaJeT7n2PKuLM9eeJ/CHhHheBvZLuLZG\nv+sabCgiIrWWy20iIiKS45RERESk1pRERESk1pRERESk1pRERESk1pRERESk1pREpOAlTA//XjTl\n+bBoioq44umbODV8Le9xiJmdkK6YRGpLSUTqg7Xu3tXdDwSOBfoA1yafZGYNsxTPyYTpMbZpO/F8\nn/B9iMRKgw2l4JnZGncvSni9F2F55l3MbBBhNHEzwrQUPzazmwkjfiuAG9x9spkdDVxHmHVgX+AV\nd78gul9/wqhpgGfd/Ypof7m7N4+2TwVOBO4CngZWAasJ0+BsniDSzO4DviJM7fIGYT6rvwJNgfXA\n2cCHhBHaTQmzrP4ReAa4jbBuTSNghLs/lZ4nKFK9XJ7FVyQj3H2xBd+Jdh0KHOTuqy2sPHmwux9k\nZrsCM8zstei8boRFkJYCz0fn/pswhfqhhMTwopmd5O5PsvXsp+7u/zazJ4Gn3P2xakJs6+49AMys\nGXCUu1eYWS/gj+5+mpldAxzm7kOj824AXnb3wWbWAphuZi+5+/q6Pi+RbVESkfoqsSr3RXdfHW0f\nCUwEcPfPzGwKIXmUEyYnXAJgZhOjczcCr7p7WbR/PPAjwlTbtW13eThhuyVwv5l1JiSl6n5njwN+\nZmaXR6+bEGZjnVfLGERSoiQi9Y6Z7Q1sdPcVUfv62sTDyadv41a+nXMSSyJNaxBiYjzXE6rOTjGz\njoSZYatzqrsvqMH7iNSZGtalPtj8Rz6qwrqT0H5QldeBM8ysQXTuUcD06Fg3M+toZg2AMwhtFtOB\nH5lZ66ghvD9hHROA5Wb23ej8nye8RzlQRGqKCO0eENpDqrvH8ySsShitsCeScUoiUh80reziS1gz\n+jl3v66qE939H4Tp5N8BXgIud/fPosMzgduB94FF7v4PD4tgXUlIHLOBme7+dHT+lYQG7zf49sI+\nk4DLo6VS90oOIen1zcCNZvY23/59fRU4IPq+TieUWBqb2btm9i6hE4BIxql3lkgKot5Zl7r7SXHH\nIpJLVBIREZFaU0lERERqTSURERGpNSURERGpNSURERGpNSURERGpNSURERGpNSURERGptf8P1s6Y\nubdXXAcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15490ac50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# List of regularizer parameters\n",
    "do_params = [0.1,0.2,0.3,0.4,0.5]\n",
    "num_params = len(do_params)\n",
    "\n",
    "# Iterate over parameters\n",
    "tr_acc = np.zeros(num_params,)\n",
    "ts_acc = np.zeros(num_params,)\n",
    "\n",
    "for i in range(num_params):\n",
    "    # Train neural network with L2 regularization\n",
    "    model = train_nn_dropout(x_train, y_train, 100, do_params[i])\n",
    "    \n",
    "    # Evaluate train and test accuracy\n",
    "    tr_acc[i] = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "    ts_acc[i] = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "# Plot train and test accuracy as a function of reg parameter\n",
    "plt.plot(do_params, tr_acc, '--b', label='Train')\n",
    "plt.plot(do_params, ts_acc, 'r', label='Test')\n",
    "plt.xlabel('Dropout rate')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('Dropout')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Does the network overfit? [remove]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x134432d50>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEdCAYAAAAikTHKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVWW9x/HPFwTUZBBRobh5q0yLFBPBPDmZpqhFqSWI\nZmVlnWP5MvXoqcih8mRZr05FF01SSxS1czQxT5LieDcveOkkoAUikJoIIjKvDIbf+WOtgc1mz7D3\nmlmz9575vl+vebH3s5611rNnMfOd53nWRRGBmZlZpfpUuwFmZlafHCBmZpaJA8TMzDJxgJiZWSYO\nEDMzy8QBYmZmmThAzLqYpEMlPSPpNUkflnSbpNMq3Mb/SXpfXm006wrydSDWG0j6JPBlYG9gDXAz\n8B8RsSaHfd0B3BwRM0osOx34TET8S0HZlcCyiPh6V7fFLE/ugViPJ+lc4NvAuUADMB4YDfxB0nZd\nuJ++6cvRwNPtVQP8V5v1CA4Q69EkDQSagLMi4g8R0RoRzwMfB0YB50tqkbRzwToHSnq5LRAkfVrS\n05JekfS/kkYV1N0o6V8lPQM8I+kvwF7ArekQVj9Jd6Xb2Bf4GTBB0lpJqyR9FpgK/Hta/7fpdpdI\nOiJ9fZGk6yVdndb5k6SxBW0YK2m+pDWSbpA0W9I3cv3GmuEAsZ7vUGAAcFNhYUSsA34PvAt4ADix\nYPEU4MaIaJX0EeBC4CPAbsC9wHVF+5gEjAP2i4h9gOeB4yKiISLWF+xzIfB54MGIGBgRu0TEL4BZ\nwHfT+pPa+RwfAq4FBgFzgJ8ASOoH/A/wS2CXtG0fLfebY9YZDhDr6XYFVkbExhLLXgCGkPxiPqWg\nfDLJL3WAzwHfjohn0m1cAhwgaWRB/f+MiFcj4o2CMnXZJ0jcFxG3RzJp+WtgTFo+AegbETPS3tVN\nwMNdvG+zkhwg1tOtBHaVVOr/+pvT5f9NMqw0TNLhwMaIuD+tMxr4YTrctAp4hWQOY3jBdpbn1/xN\nXix43QJsn36mNwMriuou64b2mDlArMd7EHgDOKGwUNKbgInAHemZWLcDJ5MMXxUOUT0PnJkON+0S\nEYMjYqeIeKigTiWT4qXqdmZS/QW2DDOAkaUqmnU1B4j1aBHxGvAN4MeSjpa0naQ9gBtIwuGatOp1\nwCdIgubagk1cBnxF0n4AkgZJOqkTTXoJGJHOXRSW7VXhdtqGyB4EWiX9m6S+ktrmY8xy5wCxHi8i\nLgW+AnyP5BqQB4GlwJEFk9y3AG8FXoyIPxWsezPJvMdsSa8CTwHHFG6+1C47eD8P+DPwoqS/p2Uz\ngf3TYbL/6WC7W20zbf8JwGeA1SRzOXNIel1mucr1QkJJM4HjgZciYkw7dX5EMpSwDvhURDyelrcC\nT5L8pbU0Ij6SW0PNehBJDwE/i4irq90W69ny7oFcCRzd3kJJE4G9I+KtwJkk58i3WRcRYyPiQIeH\nWfskvU/S0HQI63SSU5N/X+12Wc+Xa4BExH0k3er2TAJ+ldb9IzBI0tB0WVefBmnWU72dpLf+KnAO\ncGJEvFTdJllvUO05kOFsecrhCjafUTJA0sOSHkgnBs2shIj4RUQMSy9OPCAi3PuwbtFl9wHKqFQv\no21SZlREvChpT2CepKciYslWG5B8XyEzswwiolMjPdXugSxny3PWRwB/A4iIF9N/lwDNwIHtbSQi\nqv510UUX1cT2yl2vnHrbqtPe8krKu/r7VgvHL+9j1xXHL8uyWj1+9fazV07d7vjZ6wrdESCi/fmM\nW0jOvUfSeODViHhJ0s6S+qflu5Lcz6i9u5vWhMbGxprYXrnrlVNvW3XaW15peS3oyrblfezKrdtR\nnSzLavX41dvPXjl16+VnL+/TeK8FGknuN/QScBHQH4iIuDytM4PkvPp1wCcj4nFJE0gu4GolCbkf\nRMRV7ewj8vwMlq+mpiaampqq3QzLyMevfkkiOjmElescSEScUkads0qUPcjmm8VZD1arf9VaeXz8\nere6fyKheyBmZpWr+R6ImVle9thjD5YuXVrtZtS80aNH89xzz+WybfdAzKwupX9BV7sZNa+971NX\n9ECqfRqvmZnVKQeImZll4gAxM7NMHCBmZpaJA8TMrMZt3LiRgQMHsnz58mo3ZQsOEDOzLjZw4EAa\nGhpoaGigb9++7LjjjpvKrrvuuoq316dPH9auXcuIESNyaG12Po3XzOpSvZzGu9deezFz5kze//73\nt1untbWVvn375rJ/n8ZrZlanSt39dtq0aUyePJlTTjmFQYMGMWvWLB566CEmTJjA4MGDGT58OGef\nfTatra1AEjB9+vTh+eefB+C0007j7LPP5thjj6WhoYH3vve9Vbmo0gFiZlYFN998M6eeeipr1qzh\n5JNPpl+/fvzoRz9i1apV3H///dx+++1cdtllm+pLW3YWrrvuOi6++GJWr17NyJEjmTZtWnd/BAeI\nmfVcTU0gbf3V3g2Ei+vneaPhww47jGOPPRaAAQMGcNBBB3HwwQcjiT322IPPfvaz3H333ZvqF/di\nTjrpJA488ED69u3L1KlTeeKJJ/JrbDt8Lywz67GamioLgUrrd8bIkSO3eL9o0SLOPfdcHnvsMVpa\nWmhtbeWQQw5pd/1hw4Zter3jjjvy+uuv59bW9rgHYmZWBcVDUmeeeSbvete7WLx4MWvWrGH69Ok1\nf5KAA8TMrAasXbuWQYMGscMOO7BgwYIt5j9qlQPEzCxHxT2N9nz/+9/nqquuoqGhgS984QtMnjy5\n3e2Uu828+ToQM6tL9XIdSLX5OhAzM6s5DhAzM8vEAWJmZpk4QMzMLBMHiJmZZeIAMTOzTBwgZmaW\niQPEzMwycYCYmVkmDhAzsy7W1Y+0bTNhwgSuvfbaLmxp5/h27mZmXWzt2rWbXpfzSNt65R6ImVmO\nSj3SduPGjXzzm99k7733Zvfdd+e0007jtddeA6ClpYUpU6YwZMgQBg8ezIQJE1izZg3nnXcejzzy\nCJ/5zGdoaGjg/PPPr8bH2YIDxMysm333u9/ljjvu4IEHHmD58uX069ePc845B4ArrriC1tZWXnjh\nBV555RVmzJhB//79+d73vsfBBx/MzJkzee2117j00kur/CkcIGbWk5V6nm2lXzm4/PLLueSSSxg6\ndCj9+/dn2rRpzJ49G4B+/frx8ssv8+yzz9KnTx8OOuggdthhh03r1tIdiD0HYmY9Vw39si20bNky\njj322E3P9WgLhVWrVnHGGWfw4osvctJJJ7Fu3TpOO+00vvWtb9XMM0AKuQdiZtbNRowYwbx581i1\nahWrVq1i9erVrFu3jl122YX+/fszffp0FixYwD333MONN964qXdSayHiADEz62ZnnnkmF1xwAcuX\nLwfg73//O7feeisAd955JwsWLCAi2Gmnndhuu+3YbrtksGjo0KEsXry4au0u5gAxM8tRqV7DBRdc\nwFFHHcURRxzBoEGDOOyww3j88ccBWLFiBZMmTaKhoYExY8Zw/PHH87GPfQyAc845h6uvvpohQ4Zw\n4YUXduvnKCXXR9pKmgkcD7wUEWPaqfMjYCKwDvhkRDyRlp8OfBUI4OKI+FU76/uRtma9kB9pW556\nfqTtlcDR7S2UNBHYOyLeCpwJ/DwtHwx8HTgYOAS4SNKgnNtqZmYVyDVAIuI+YHUHVSYBv0rr/hEY\nJGkoSejMjYg1EfEqMBc4Js+2mplZZap9Gu9wYFnB++VpWXH5irTMepA5c2DFCvj857de9uSTcNll\nW5ePGeP6rm+1otoBUjz+JpI5j1Ljcu0OdjY1NW163djYSGNjYxc0zfLS0gJf/jLMnQs//GHpOgMH\nwjvfuXX56NGu7/qWRXNzM83NzV26zVwn0QEkjQbmlJpEl/Rz4K6IuD59vxA4HHg/0BgRny9Vr2gb\nnkSvI088AaecAmPHwk9+AoM8s2UZeRK9PPU8iQ5Jb6K9Rt4CfAJA0njg1Yh4CbgdOErSoHRC/ai0\nzOrYnDlw1FHwla/ANdc4PMzqXa5DWJKuBRqBIZKeBy4C+gMREZdHxG2SjpX0F5LTeD9FsnC1pG8C\nj5IMXU1PJ9Otjo0fDw89BHvvXe2WWE8wevTomrsyuxaNbm9csAvkPoSVNw9hmZlVrl6GsMzMrAdy\ngFiXe/ppOOMM2LCh2i0xszw5QKzLRMDPfgaHHw6HHgp9+1a7RWaWp2pfB2I9xMqVSa9j+XK47z54\n+9ur3SIzy5t7INZpS5fCAQckofHggw4Ps97CZ2FZp23cCI8+CuPGVbslZlaurjgLywFiZtYL+TRe\nMzOrGgeIlW316mSifNmybdc1s57PAWJluffeZKL8TW+C3XardmvMrBb4NF7r0Pr18I1vwBVXJF/H\nHVftFplZrXCAWLtaW+HII2H77eHxx2HYsGq3yMxqic/Csg49/DC85z3Qx4OdZj2KT+PFAWJmloVP\n4zUzs6pxgBitrXDxxXDrrdVuiZnVEwdIL7dsGRxxBNxxR3KarplZuRwgvdhvfpNMkE+cmATIiBHV\nbpGZ1ROfxttLTZsG110Hc+b4Johmlo3Pwuqlnn4aRo6EgQOr3RIzqwafxosDxMwsC5/Ga2ZmVeMA\nMTOzTBwgZmaWiQPEzMwycYCYmVkmDhAzM8vEAWJmZpk4QMzMLBMHiJmZZeIAMTOzTBwgZmaWiQPE\nzMwycYCYmVkmDhAzM8sk9wCRdIykhZKekXRBieWjJN0h6UlJ8yS9pWBZq6T5kh6XdHPebTUzs/Ll\n+jwQSX2AZ4APAH8DHgEmR8TCgjo3ALdExDWSGoFPR8Qn0mWvRUTDNvbh54GYmVWoHp4HMg54NiKW\nRsR6YDYwqajOfsA8gIhoLlreqQ9nZmb5yTtAhgPLCt4vT8sKPQGcCCDpBGAnSYPTZQMkPSzpAUnF\nwWNmZlWUd4CU6kEUjzedDzRKegz4F2AFsCFdNioixgFTgf+StGduLTUzs4psl/P2lwOjCt6PIJkL\n2SQiXmBzD+RNwIkRsTZd9mL67xJJzcCBwJLinTQ1NW163djYSGNjYxd+BDOz+tfc3Exzc3OXbjPv\nSfS+wCKSSfQXgIeBKRGxoKDOEGBVRISkbwEbIqJJ0s5AS0T8U9KuwP3ApMIJ+HR9T6KbmVWoWybR\nJZ1VMCdRkYhoBc4C5gJ/BmZHxAJJ0yUdn1ZrBBZJWgjsDlyclr8DeFTS48CdwLeLw8PMzKpnmz2Q\ntFcwGZgP/BK4vZb+5HcPxMyscl3RAylrCEuSgA8CnwLeA9wAzIyIv3Zm513BAWJmVrluuw4k/Q39\nYvq1ARgM/EbSdzuzczMzq1/lDGF9CTgdWAlcAdwcEevTq8yfjYi9829mh+1zD8TMrEJd0QMp5zTe\nXYETImJpYWFEbCyYCDczs16mnCGs24BVbW8kDZR0CEDh6bhmZta7lDOE9Tgwtm2cKB26ejQixnZD\n+7bJQ1hmZpXrrkn0LX5DR8RG8r+C3czMalw5AbJY0pck9Uu/zgYW590wMzOrbeUEyOeBQ0lucrgc\nOAT4XJ6NMjOz2pfrvbC6g+dAzMwq1y2n8UraHjgD2B/Yvq08Ij7dmR2bmVl9K2cI69fAMOBo4G6S\nW7KvzbNRZmZW+8o6jTciDpT0VESMkdQPuDcixndPEzvmISwzs8p112m869N/X5X0TmAQyW3Xzcys\nFyvneo7L0+eBfA24BdgJmJZrq8zMrOZ1GCDpVeevRcRq4B5gr25plZmZ1bwOh7DSq87/vZvaYmZm\ndaScSfRLSG7lfj2wrq08Ila1u1I38iS6mVnluuWJhJKWlCiOiKiJ4SwHiJlZ5brtkba1zAFiZla5\n7roS/ROlyiPiV53ZsZmZ1bdyTuM9uOD19sAHgPmAA8TMrBereAhL0iDg+og4Jp8mVcZDWGZmleuu\nK9GLtQB7dmanZmZW/8qZA5kDtP2J3wfYD7ghz0aZmVntK+c03sML3m4AlkbE8lxbVQEPYZmZVa5b\nzsICngdeiIh/pDvdQdIeEfFcZ3ZsZmb1rZw5kBuBjQXvW9MyMzPrxcoJkO0i4p9tb9LX/fNrkpmZ\n1YNyAuRlSR9ueyNpEsm9sczMrBcrZxJ9b2AW8Ja0aDnwiYj4S85tK4sn0c3MKtet98KStFNav6ae\nh+4AMTOrXLdcSCjpPyXtHBGvR8RaSYMlfaszOzUzs/pXzhzIxIh4te1N+nTCY/NrkpmZ1YNyAqSv\npAFtbyTtAAzooL6ZmfUC5VxIeA1wp6Qr0/efAq7Or0lmZlYPttkDiYjvAt8C3kFyH6zfA6PL3YGk\nYyQtlPSMpAtKLB8l6Q5JT0qaJ+ktBctOT9db1N5zSczMrDrKvRvviyRXo59I8jyQBeWsJKkPMAM4\nGtgfmCJp36Jq3wOuioh3A98ALknXHQx8neR5JIcAF6W3kjczsxrQboBIepukr0taQBICy0hO431/\nRMwoc/vjgGcjYmlErAdmA5OK6uwHzAOIiOaC5UcDcyNiTTqJPxeoiWeQmJlZxz2QhSS9jQ9FxGER\n8WOS+2BVYjhJ8LRZnpYVeoKkZ4OkE4Cd0t5H8borSqxrZmZV0tEk+onAZOAuSb8n6T1UetFJqfrF\nV/2dD8yQ9EngHpKg2FDmugA0NTVtet3Y2EhjY2OFzTQz69mam5tpbm7u0m2WcyuTNwEfAaYAR5Cc\ngXVTRMzd5sal8UBT2+NvJV0IRER8p4N9LYiIUZImA40R8fl02c+BuyLi+qJ1fCW6mVmFuvVWJukO\ndwE+BpwcEUeUUb8vsIhkKOwF4GFgSkQsKKgzBFgVEZFe4b4hIprSYaxHgbEkQ22PAgcVXtSYru8A\nMTOrULc/Ez0iVkXEZeWER1q/FTiLZAL8z8DsiFggabqk49NqjcAiSQuB3YGL03VXA98kCY4/AtOL\nw8PMzKqnoh5ILXIPxMysct3eAzEzM2vjADEzs0wcIGZmlokDxMzMMnGAmJlZJg4QMzPLxAFiZmaZ\nOEDMzCwTB4iZmWXiADEzs0wcIGZmlokDxMzMMnGAmJlZJg4QMzPLxAFiZmaZOEDMzCwTB4iZmWXi\nADEzs0wcIGZmlokDxMzMMnGAmJlZJg4QMzPLxAFiZmaZOEDMzCwTB4iZmWXiADEzs0wcIGZmlokD\nxMzMMnGAmJlZJg4QMzPLxAFiZmaZOEDMzCwTB4iZmWXiADEzs0wcIGZmlokDxMzMMsk9QCQdI2mh\npGckXVBi+UhJ8yTNl/SEpIlp+WhJLWn5fEk/zbutZmZWPkVEfhuX+gDPAB8A/gY8AkyOiIUFdS4D\n5kfEZZLeAdwWEXtKGg3MiYgx29hH5PkZzMx6IklEhDqzjbx7IOOAZyNiaUSsB2YDk4rqbAQa0tc7\nAysKlnXqw5mZWX7yDpDhwLKC98vTskLTgdMkLQNuBb5YsGwPSY9JukvSYfk21czMKrFdztsv1YMo\nHm+aAlwZET+QNB64BtgfeAEYFRGrJY0Fbpa0X0S8XrzBpqamTa8bGxtpbGzsouabmfUMzc3NNDc3\nd+k2854DGQ80RcQx6fsLgYiI7xTU+T/g6IhYkb7/K3BIRKws2tZdwLkRMb+o3HMgZmYVqoc5kEeA\nfdIzqvoDk4FbiuosBY4ESCfRB0TESkm7ppPwSNoL2AdYnHN7zcysTLkOYUVEq6SzgLkkYTUzIhZI\nmg48EhG3AucBv5B0DsmE+unp6u8DviFpPdAKnBkRr+bZXjOzLrVhAzz3HCxaBAsXJv8uWgS//jWM\nGlXt1nVarkNY3cFDWGZWdatXbw6HwqBYvBiGDoV994W3vz352ndfGD8edtyxqk3uiiEsB4iZWTk2\nbIAlS0oHRUvL5nAoDIq3vhV22KHaLS/JAYIDxMy62KpVpUNiyRIYNqx0ULz5zaD6umzNAYIDxMwy\n2LAhGV4qFRT/+EfpkNhnn5rtTWThAMEBYmYdWLVqy3Boe71kCbzlLaWDYtiwuutNZOEAwQFi1uut\nX58EQqmg+Oc/twyHttc9rDeRhQMEB4hZr/HKK1sPNy1cmJwmO3x46aDoJb2JLBwgOEDMepT16zfP\nTRQHxYYNm4OhMCj22Qe2377aLa87DhAcIGZ1aeXKrUNi0aKkNzFiROnexNCh7k10IQcIDhCzmrV+\nPfz1r6V7E62tW09et/UmBgyodst7BQcIDhCzqorY3JsoDoqlS5PeRKmg2H139yaqzAGCA8Ss261c\nCTfeCDfcAE89BRs3lj4ddu+93ZuoYQ4QHCBm3aKlBW65BWbNgnvvhYkTYcqU5J5Ou+3m3kQdcoDg\nADHLTWsrzJsH11yThMe4cTB1Knz0ozBwYLVbZ53kAMEBYtalImD+/CQ0Zs9O5jCmToXJk5NrKqzH\n6IoAyfuRtmZWDxYvToanZs1Kzp6aOhWam5P5DLN2OEDMequXX04mwmfNgr/8BU4+Ga66Cg45xHMa\nVhYPYfVGLS1w+OHVbkVi++1hyBDYddfkq73XO+8MffJ+AnMv0NICv/1tEhr33QfHHZf0No46Cvr1\nq3brrBt5DgQHSCatrck4dy1oaUnucfTKK8npoStXln69di0MHtxxyBS/HjzYoQPJLUDuvDMJjTlz\nkjOnpk6Fj3wEdtqp2q2zKnGA4ADpNTZsSG7N3V7AlHr92mtJiJQbOLvu2nNCJwIefTQJjeuvh5Ej\n4dRTk2GqoUOr3TqrAQ4QHCDWgbbQKTdwVq5MQmfnncsPnLbhtb59q/1pE3/96+bJ8I0bk57GKafA\n295W7ZZZjXGA4ACxLrZhA6xeXX7gvPIKrFmzOXQqGV7rqtB5+eWklzFrVnI21cknJ8Exbpwnw61d\nDhAcIFYD2kKnkp5OW+i0BUs5wVMYOuvWbZ4Mv/9+OP74JDSOPNKT4VYWBwgOEKtTra2V9XTaQmfQ\noCRMXnoJDj00CY1JkzwZbhVzgOAAsV6kMHSGDEnuQWWWkQMEB4iZWRZdESA94HxFMzOrBgeImZll\n4gAxM7NMHCBmZpaJA8TMzDJxgJiZWSYOEDMzy8QBYmZmmThAzMwsEweImZllknuASDpG0kJJz0i6\noMTykZLmSZov6QlJEwuW/YekZyUtkPTBvNtq3a+5ubnaTbBO8PHr3XINEEl9gBnA0cD+wBRJ+xZV\n+xpwfUSMBaYAP03X3Q/4OPAOYCLwU8kPN+hp/Auovvn49W5590DGAc9GxNKIWA/MBiYV1dkINKSv\ndwZWpK8/DMyOiA0R8RzwbLq9mtTVP0hZt1fueuXU21ad9pZXWl4LurJteR+7cut2VCfLslo9fvX2\ns1dO3Xr52cs7QIYDywreL0/LCk0HTpO0DLgV+GI7664osW7NqLf/xA6QLTlAtr2sVo9fvf3slVO3\nXn72cr2du6STgA9GxOfS96cCB0fE2QV1zgGIiB9IGg/MjIj9Jc0AHoiIa9N6VwC/i4ibivbhe7mb\nmWXQ2du5b9dVDWnHcmBUwfsRwN+K6pxBMkdCRDwkaXtJu5a5bqe/AWZmlk3eQ1iPAPtIGi2pPzAZ\nuKWozlLgSABJ7wAGRMTKtN7JkvpL2hPYB3g45/aamVmZcu2BRESrpLOAuSRhNTMiFkiaDjwSEbcC\n5wG/SIeyNgKnp+s+LekG4GlgPfCvfvSgmVntqPtH2pqZWXX4SnQzM8vEAWJmZpnkfRZWVUjakeSK\n9jeAu9tOBbb6kJ408VWgISI+Xu32WPkkTQKOAwYCv4yIP1S5SVaB9E4hZwNDgHkR8fMO6/fEOZD0\nepPVEfE7SbMjYnK122SVk3SDA6Q+SdoZuDQiPlvttljl0ttGXR0Rn+ioXl0MYUmaKeklSU8Vlbd3\no8YRbL6KvbXbGmolZTh+ViM6cey+Bvyke1pp7cly/CR9iOSuILdta/t1ESDAlaQXG7bZxo0al5GE\nCIAvNKy+So/fpmrd0zzrQMXHTtIlwG0R8UR3NtRKqvj4RcSciDgOOHVbG6+LAImI+4DVRcUd3ajx\nJuAkST8B5nRfS62USo+fpF0k/Qw4wD2T6spw7L4IfIDk5+9z3dpY20qG43e4pB9K+jnwu21tv54n\n0UvdqHEcQES0AJ+uRqOsbB0dv1XAF6rRKCtLR8fux8CPq9EoK1tHx+9u4O5yN1QXPZB2lBre6Hln\nBPRcPn71y8euvnXZ8avnACnrZotWs3z86pePXX3rsuNXTwEitkzOcm7UaLXDx69++djVt9yOX10E\niKRrgQeAt0l6XtKnIqKV5OFTc4E/kzy9cEE122ml+fjVLx+7+pb38euRFxKamVn+6qIHYmZmtccB\nYmZmmThAzMwsEweImZll4gAxM7NMHCBmZpaJA8TMzDJxgFjNkrRR0qUF78+V9PUc9nOppD9J+k5R\n+UWSvtzOOve1U36lpBNKlB8uqSbuDC1piaRdqt0Oq3/1fDde6/neAE6Q9O30Dr15+SwwOCq4qjYi\nDsuwn1q5ardW2mF1zj0Qq2UbgMuBrXoBkkZJukPSE5L+IGnE1qtvtU5bT+NJSR9Ly34L7AQ81lZW\nZH9Jd0n6S/qsi7ZtrS14PUPSAklzgd0Lyo9Jyx8FTigo3zF9UtwfJT2WPgEOSadL+m9J/ytpUXGP\nqGD9JZKa0nWflPS2tHywpJvSsgckvSst30XS7eln/wUF90WSNDVtx3xJP1OiT9qTeird1tnb+t5a\n7+QAsVoWJI9FnSppYNGyGcBVEXEAcC3beAZFOqw0JiLeBRwFfE/S0IiYBLRExNiIuLHEqm9P6x8C\nXCSpb0Hb2rb71oh4B3A6cGhaPoAk/I6LiPcAwwq2+VXgzog4BDgibcsO6bJ3Ax8DxgAnSxrezkf6\ne0QcBPwcOC8tmw7Mj4h3p/v4VVp+EXBv+tlvIr0Ta/oUupOBQyNiLLARmAocAAyPiDHptq5spw3W\nyzlArKZFxOvA1UDxX8ETgOvS178GtjWkdFhb/Yj4O9AMHJwu6+jRub+LiA0R8QrwEjC0aPm/FGz3\nBeDOtHxfYHFELE7fX1OwzgeBCyU9nrajP5tvr31nRLweEW8ATwOj22nXTem/jwF7FHzGX6dtuQvY\nRVID8L62/UfEbWx+Qt0HgLHAI2lbjgD2AhYDe6ZPpjsa2NTbMivkORCrBz8E5rPlX8LF4/gbt7GN\n4pAofN/gzs2HAAABf0lEQVTRnMAbRfso9TNTuH45z3EXcGJEPLtFoTS+aH+t7eyvsF2FdUrtu+37\nUqqNAq6OiK9u1UDp3STPzD4T+DhwRjvtsF7MPRCrZQKIiNXADWz5S+wBYEr6+lSg5FlRBe4hGRLq\nI2k3kp7DHwv3k6Vt6XYnp9t9M/D+tHwhsIekPdP3UwrWvR340qYNSQdk2H8p95B8L5DUCKxMe3CF\n5ROBndP6d5I8u3y3dNngdG5pCNA3Im4CpgEHdlH7rIdxD8RqWeFfzd8H/q2g7Gzgl5LOA14GPgWQ\nTkgfFBFNW2wo4qb0L/wnSf4qPz8iXi6xn3LbEwXbPYLkuQrPkwQbEfGGpDOB2yStA+4lmawH+Cbw\nX5KeIgmiJcCHt7G/csqbgCslPQmsI5mTgWRu5DpJk9P2PZ+2cYGkrwFzJfUB/knyPf5Hup0+6b4u\nbGd/1sv5eSBmZpaJh7DMzCwTB4iZmWXiADEzs0wcIGZmlokDxMzMMnGAmJlZJg4QMzPL5P8Bba+A\n1OCYUm8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12c9969d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# No. of hidden nodes\n",
    "num_hidden = [5,10,50,100,500]\n",
    "num_params = len(num_hidden)\n",
    "\n",
    "# Iterate over parameters\n",
    "tr_acc = np.zeros(num_params,)\n",
    "ts_acc = np.zeros(num_params,)\n",
    "\n",
    "for i in range(num_params):\n",
    "    # Train neural network with num_hidden[i] hidden nodes\n",
    "    model = train_nn(x_train, y_train, num_hidden[i])\n",
    "\n",
    "    # Evaluate train and test accuracy\n",
    "    tr_acc[i] = model.evaluate(x_train, y_train, verbose=0)[1]\n",
    "    ts_acc[i] = model.evaluate(x_test, y_test, verbose=0)[1]\n",
    "    \n",
    "# Plot train and test accuracy as a function of reg parameter\n",
    "plt.semilogx(num_hidden, tr_acc, '--b', label='Train')\n",
    "plt.semilogx(num_hidden, ts_acc, 'r', label='Test')\n",
    "plt.xlabel('No. of hidden nodes')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('Overfitting')\n",
    "plt.legend()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}