{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Net Example\n",
    "\n",
    "Partially based on [Stanford CS231n neural network case study](http://cs231n.github.io/neural-networks-case-study/).\n",
    "\n",
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "# A bit of setup\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "# for auto-reloading extenrnal modules\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-1, 1)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXd4W9X9uN+j6Rk7znTiODtkk5ANYYcRCLNAgVIohVLa0kVLSwctu7TlB3TQllHGF1qgpBTCCCGQMJKQvXdiZzlx4p14yBr3nt8fR4ol68qWbXmf93n02L5LR/K953M+W0gp0Wg0Go0mHmztPQCNRqPRdB600NBoNBpN3GihodFoNJq40UJDo9FoNHGjhYZGo9Fo4kYLDY1Go9HETUKEhhDiBSFEkRBia4z9QgjxZyHEXiHEZiHEaWH7bhFC7Am+bknEeDQajUbTOiRK03gJuLiB/XOBkcHXHcDfAYQQWcBvgRnAdOC3QoieCRqTRqPRaBJMQoSGlPJzoKyBQ64A/k8qVgKZQohs4CJgsZSyTEpZDiymYeGj0Wg0mnbE0UbvMxA4FPZ3QXBbrO1RCCHuQGkppKamThk9enTrjFSj0Wi6KOvWrSuRUvZpyTXaSmgIi22yge3RG6V8FngWYOrUqXLt2rWJG51Go9F0A4QQB1p6jbaKnioABoX9nQMcaWC7RqPRaDogbSU0FgA3B6OoZgLHpZSFwCLgQiFEz6AD/MLgNo1Go9F0QBJinhJCvAacA/QWQhSgIqKcAFLKfwAfAJcAe4Ea4NbgvjIhxEPAmuClHpRSNuRQ12g0Gk07khChIaW8oZH9EvhejH0vAC8kYhwajUajaV10RrhGo9Fo4kYLDY1Go9HEjRYaGo1Go4mbtsrT0HQCaj1+tmw4gs9nMH5SNhmZye09JI1G08HQQkMDwNqVB3nmyWXYbAIkGIbJZddO4IrrJrb30DQaTQdCm6c0lJVU88wTy/B5DWo9AWprA/j9Ju//dxvbNhW26Np+v0HVCS8qgE6j0XR2tKahYfnSfEwzelL3egMsencH407NbvI1vbV+XnluDSs/34eUkJbu5vpbpzDrrKGJGLJGo2kntNDoxnhqfKxbeYjN6w8TCJiWx1SUeZp17T/97lN2by/C71fXrSj38MJfv8TtdnDajEGNnK3RaDoqWmh0U7ZuPMKff/cZCPD7jJjHlRRVsnPrMUaP7xf3tQ8fqmDPjuKTAiOEz2fw5qsbtNDQaDox2qfRDfF4/Pz5sc/wegN4awOWpqkQ1VV+/t9Dn1BwoDzu6xccqMBmtypgDMcKK5s8Xo1G03HQQqMbsmH1ocYPCiPgN3nvv9viPr533zRi+b0ze+owXo2mM6OFRjekptrfoHZRH9OUHMiPv47ksJG96NMvTYXvhuFyO7jsmvFxX0ej0XQ8tNDohoyZ0M+y+xUQNdEDIKD/wB5xX18IwT33z2HYyF44XXaSU5y4XHbmXjGGcy4c2bxBazSaDoF2hHdDBg7KZMrMXNatOojPq5zgNhu4k5wAeGr8Ece7XHbmfWVck94js2cy9/1+LsXHKjleUcvA3EySk52J+QAajabd0EKjm3LHj87gs8V9+ei9ndRU+xg3MZsrr5+IzxvgL7//nLLSamw2gd1u45Y7ZzB8VPPaCvfpl06ffukJHr1Go2kvRGfM1NU9wlsXKSVHj5zA5zXIGZyJ3a6tmBpNV0AIsU5KObUl19CaRgfHNCXLlubxyQe7qPX4mTwth0uuGofXG2B/XhkZPZMZOboPQsTyUjQdIQTZAzMSdj2NRtN10EKjg/PMk8tYv/rQSd/DR+/v4uMPdiElOJx2kJL0jCTuuX8O/bK1GUij0bQuCbE7CCEuFkLsEkLsFULca7H/SSHExuBrtxCiImyfEbZvQSLG01XYn1caITAAjICJ328SCJjUevzU1gYoKari8Qc+1kUBNRpNq9NiTUMIYQeeBi4ACoA1QogFUsrtoWOklD8OO/77wOSwS3iklJNaOo6uyLZNhTFrQoUjJRyvqCVvVwkjRjfPYa3RaDTxkAhNYzqwV0qZL6X0Aa8DVzRw/A3Aawl43y6Py+2I2wltE4KKiuYVF9RoNJp4SYTQGAiE16UoCG6LQggxGBgKLAnbnCSEWCuEWCmEuDIB4+kyTD9jcNzHBgIGQ4f3asXRaDQaTWKEhlXYTizj+vXAfClleFnV3GAI2I3AU0KI4ZZvIsQdQeGytri4uGUj7iRkZCbzze/NxOmy43TaEQKcThv1A6VcbjszZg+hV5/U9hmoRqPpNiQieqoACK91nQMciXHs9cD3wjdIKY8Ef+YLIT5F+Tvy6p8opXwWeBZUnkaLR91JOP3sYYyd0J/Vyw/gqfEzblI23lo/b7y8gYID5aSmubnostFcclXTMrY1Go2mObQ4uU8I4QB2A+cDh4E1wI1Sym31jjsFWAQMlcE3FUL0BGqklF4hRG/gS+CKcCe6FTq5T6PRaJpOh0juk1IGhBB3oQSCHXhBSrlNCPEgsFZKGQqjvQF4XUZKqTHAM0IIE2Uqe6wxgaGpQ0rJ5vVH+GzxHmprA0w/YzCnnz0Ml8ve3kPTaJqJCZQAtUAa0BNrC7imvdBlRDoxL/19JSs+3YfXGwCUb6Nfdjr3/X4ubrfO2+z4HEdZcn1ALyAbte7qrtQA61GCI7SOdAOnAbrYZSLoEJqGpmUcKzzBO//Zwq5tx8jITGbulWOZOiu30bIg+/NKWf5pfkTin89rcOxIJUsW7mLuldrH0bE5AOxHTY4AFSj34BS65wQpgS1AeIVlA/AAuwDdh6WjoIVGO1JwsIKHfr4QnzeAaUJJUTXP/Wk5B/LLuOamyQ2eu37VIcve3j6fwYrP9mmh0aGpJVJgEPy9FiVMRgCVwD6gCkgCBqO0kY6AiYqyP4ya2HsCw4CUFlyzBvX56yNR5qqQ5qFpb/R/oR157YW11HqUwAjh9Rp8+M52jjeSqGeziZjaiGUjJU07UQVsR8WH7ERNjqUxjpVAEVCOMtOUAl6UGWsrShPpCGxFCT0vEACKgbUoraC5BGjYd9F4ZQRN26CFRjuyc9sxy+12h41d24oaPHfaGYOxO6L/fS63nbPmjEjI+DQtpQRYBxxDCY9ClPBobHLdRfQkaQL5qJV9e3ICJdTqj89AmZfKiZ2m1RBpDZyXhDaKdBy00GhHXK5YD4IgKbnhh2TgoEwuuXIsLrf9ZLKfO8nBkOG9OFsLjQ6ARGkWVpN/rORUAfTB2kwToqrlQ2sRFcSe3KtRgmMDjQs3I3h8yIdhB4YTPSXZgFHNGqmmddDiux058/zhLFm4C78/cmKx2QRjJ/Rv9Pyrb5zEpGk5fLEkD6/Hz9RZg5k0dSA23TSpA1BNbJOKF+WjOBh2jA21os5F+QqskEQ/shJl2rHTNmtAJ0q4xRIcBsofsx8lBOojUb6aQ8HrmEBvVPR9Duo7OEBdyO1QINSf3kBFm4U09OzgS9/vbYkWGu3IV248lT07izl8sAKfN4ArGCb7o1+eg8NpR0rJ9s1H2bn1GKnpLmbOHkJmVqSzcdjI3gwb2bs9hq9pkNCEGIsc1GR5BLXa7gX0RU3+vVD+jPoTs5tIZ3MhymTlD75fNsqJ3pqTaB9ULm9DmMGxWQmNgyiBEf7dlADbgImo78TqfjZRGky4MK5GCZBJaMHRdug8jXZGSsmOLUfZu6uEjMwkpp8xmOQUF36/weMPfMK+vaV4awM4nTYQgm//6AymnR5/IUNNW1OJMuEcIDJ8NJwMVO5BLHyoCdKLWl2HtIjJQKi+2FGifR82lMBp7fDUcpQZShJbMNqBs+ptk8AylGZUHxswA3ChBE5h8Np9UFWKilHCqv772YCxweM0jaHzNLoAQgjGTsxm7MTsiO0L395O3u6Sk2G1IRPWs08tZ+zEbFLTXG0+Vk1DBIBNKJ9DQxqGE2WKaQgXquNAGXUht32IXE3nW7yPidJQaoPntBY9gTNQE/terE1VPS22mcT2dQhUZNkulNANfTYPSkAmY/29mqiIMy002gqt03VQPlu8xzIPQ9gEG1YfsjhD0zI8KFNHGc2L/tmB0jIaCw2djpoAG0OgtIbBQD8iH1WJ0kJinVcdx/Vbih1lYuthsc+GtWnKRuyMd4nSzMIFBsHffcT+vKGxxIsf5TPKQ2kvOpS3qWhNo53ZvaOIpR/u5sTxWiZNy+HM84aTlOzEZyEwAExT4q21Uu81zcNERTkVU5cnYEfZyeMtNR/A2gdRH0Fi6igJ1KNrdR9IYmsZBkpDCZl+MoCRKIezFSZqgi0MXrcvylQUmjaOo0JwrcZQf2xG8DpOi302IIvYQtcMXtNmsd+G8uXEw3GUNhgyq9lRWl13zcJvHlpotCPvzt/Kgjc34/cZSKkEyKIFO7j/8UuYNDWHZUvyMM3oiWj85AHtMNquhkQ5ofOINpkYwEbgdOKb5EOJaY0JDReJe+Ryic4qFygBYCXsJOozhZvPKlBJhFMszgkdHz6RH0RpY1NRnyMkTKze6wh1WkgAlfzntRhvKMz4FJQfKNb36ESZvI6GXUOgggOOBcfTkJAPlSkJ/18bKFPeHpRfRBMP2jzVTpSVVPPOG5vweZXAAFU7qry0hnfnb+Gq6yeSmubCEZbA53Y7OGvOCPplp7fTqLsS+1H2+Fg2dgPl8I0HN42bSGyoVX08QugEaoJbhcq+rrQ4JhcYELxeyFHeA5gQ45oVWPtbDNR3UZ9Si+NN1MRfGPzb18BnCA8CCIXQWmkRU1ATth3oj/X3Y0OZwk5BBRDkUPd9e1Da0NqwcVlxPMb7h7LwO19AUHuhNY12YtO6wwiLch+BgMnKL/Zzw61TefhPl7Hw7W1sWneY9HQ3F8wbw7TTc9thtF2NAJE5ErGoRU1EBpBJbDOOQNnwY0X3pKFqM1k5h+tTjCo7ErpOqOzIeCJrTwmUEBqC8mG4adhXcsJibCGOW2wrwVqghpITBwXHY5UdbiMybLahSbmcuu81BfU95QePD5mkeqNMYwDpKCERMlmFj2s3SmuxmtYaSjbUAqMpaKHRTjRUxTa0L7NnMjfcOpUbbm1RhJwmihoaX/GHJqHw43qjVsVW52ajHqd8lLBxoyb0xpM065DELiGyC5hl8d5OlEBrDBdqdW41eVpF4jU0NYT29UflXNRSN/EKlE+lr8V59bHy8QxCfc9FqM/di2hnezHWE71ABTJYvXdGjHNC+3S9tnjR5ql2YtK0HKSFv8LptHHGOUPbYUTdCReNaxmhla4Z9iohdidjUKvcGcDZwEyaJjBACbNY4/LTcARRY8QKSbWhTF316Y/19GBDmcVACaGpqIneHXwNQpmcws11scxOEqU55AHLUTkcO4PvMZjIbPD658Ui1j4HSoup/5nsKI1NEy9aaLQTmT2T+eo3puBy2bEF/wsutx2ny8HyT/fxqx++y9JFuzENHRLYMqxyA5JoPOzVapIzSUyl2QDKX1A/+c9O7EkvZKppLg7g1OBPO3V+kEFYC5RQCY/67xkSnqHv1IEyzZ0efA0nWkvJRZme6l9Lohzxh1D+ET91RR0b8pfEEoASFYVlRTXK/5GK+t8nB68zEGWea4lA7l7ojPAE4PUGWLpoDyu/2IfDbuOsC0ZwxjnDsMdRA+rgvjI+W7yXkqIqtm89qiKpgnLC5bYzedogvvvTM1v5E3RFfCiTTigUNhXlSM0I7i9Cla6wIiQwrJ4NFyqxrTlIlPP9CHVRQqHIodDKfA3WRQl7oFbwLcVEOcUDKLNWY0mih1CaQPh3EQqRjeV0t8IAviR2lnw4AiVohgX/Lg6Ow4v6/w1A/e8CRNbuGoESAvU5RmTxSBuR5rQQw1FO9q6LzgjvAPi8AR76+UKOHak8mVtxcF85q5cd4O77zmu0t0Xu0Cy+fsd0XvzblwR85kmBoa5tsGH1IQ7uLyd3SDxOVI3CD6wmcoKqRoWQTkUJkN5Y5zqEHK8lWAuNWCvZeMhHCYxw7TFknw81zRqHWn0b1OUS2ElcSGhowo+XAqK/BxPlO/AQX6Ii1JVEiQeJEvbDUMUNw4MWalH/m1NRzv0ylFlsIMrUVR+D6GrD4b+Hf7Y8VLBCvPk53ZOECA0hxMXAn1B39/NSysfq7f8G8Efqynf+VUr5fHDfLcCvg9sfllK+nIgxtQbeWj/z/7WRZUvy8fsMRo/vx9CRvThWWBmRjOf1Bti9o4htmwqZEGdOxeZ1RyxzMkxTsn1ToRYaceNFCQyrxLeQQzkbNTmcihIkoX021IQxGpXhXUrkBONAObebQ8i0FatUug+16k9BObyLUD6OVJQ20h69wyWxy7SHMs/jFRpNtWi4UELfKsotlKQ4GWVea4gK4ndyhwot1m8tEN6pMKShDae7CpcWCw0hhB14GrgA9VSsEUIskFJur3foG1LKu+qdmwX8FrX8k8C64LnxBsi3GaYpeey+xRzaX36yDtSWjUfYtqnQcrL31gZY++XBuIWGO0b/DLvdRlKKzlaNnz1YC4wQx6nLexiIssOXoCbtHtRF0oxFTRLhLU2HEv8kWZ+GzDI21OQcMhXZiT/LuTURqOgsq7E3lHluRQpqumnIVxEilJfR0IRvFSYsUaaoQ6gxZ1FnjowXq3tnO5ELiNLg2KySIrs+iXCETwf2SinzpZQ+4HXgijjPvQhYLKUsCwqKxcDFCRhTwtmx5SiHDx2P7H0hsRQYAEIon0S8zJl7iuXxUkqmzdK5GfER6ifdGKFoqMOoPIH+KBt6JmqSMlArfR9Ks5iJEiLNFRhQ14ci1nhacu3WJJfoaUKgJstYeStWCFShRlu9baFXyDEfcs73omHtymrq2o3SJKuoS0LcTfxaTqgsfThVRGucUKftdD8SITQGokR7iAKsvVFfEUJsFkLMF0KEdMp4z0UIcYcQYq0QYm1xcazOZ63H3l3F+Lzx13xyOu2ccY5y5FWd8LI/r5SqytgRGudePIoJkwfgctux2wVOpw2n0853fjKb1DR3i8ffPfDQNDNIyOwQTg3KYbsbZRrZHfy7poVjC02GVp3p+tFxax8NQj2SoWKDoczzic24VhYwDeXIzghedwYqsGAUKvR1BnUO8EyspyhBdDhzqBqulfkvHuFmQ2lD9YWGlUYToiKO63Y9EuHTiBWAHc67wGtSSq8Q4k7gZeC8OM9VG6V8FngWVPRU84fbPDIyk3G5HHgtBEd6DzdebwCf18BmEzgcNuZeOZaBuZk8+9RyVi3fj9Npx+83mHnmUG79zgwczshVlN1u4/s/P5sX/volyz/NByGQUrLwnR2MOKUPPTI76kq0I+BFld1oTnXX+qaX7fW2GcHXdpQVtSUMCf48RN1t3p+OnScgUDb+wcSXed4YKahosfpY5bTYUBFa9YsMJlMnWEI0ZNGuCb5vLMHvQAlHK6HekIbYUQV965IIoVFApDcqh3oZUFLK0rA/nwN+H3buOfXO/TQBY0o4004fzL9fiB3m+83vzeTgvnLsDjszZw8mZ3BPnv/LClavOEDAbxIImrVWL9uP02HjG9+dGXWNJQt3sXLZfgxDYhjKsZ63q5g/PvAJDz5xaYNZ5N2XUGG95mgCgsjSHl5i9+AOmTzCtT4DpY2ECvf1Rvk9YoWxiuD+wSjTl5P2cXA3h3gzzxNNBsrvFDIX9kD9z+o/C3aLbeH7BmDdg8SGqmcVyzdRX/MIP69rh+fGIhHmqTXASCHEUCGEC7geWBB+gBAi3Kt3OSo0BWARcKEQoqcQoidwYXBbhyM1zcXd951n6XeoPOHlX8+v5eobJ3HN1yaRM7gnNdU+Vn6+L6onhs9nsOzTfGo90c7Fd+dvxeeNPN4wJMeOVHIgvyyxH6jL0FBilkCVlDgNNdnUn1TCK92GVrKxJp767VtD7UcPBt/fR11iWmO5CKF+4J1FYLQ3DtSkPwRl4rL6H8VqeRzKYB+AMlPZ6u3LoWFnth0VZReeEGmjLjGw+9FioSGlDAB3oSb7HcB/pJTbhBAPCiEuDx72AyHENiHEJuAHwDeC55YBD6GetDXAg8FtHZLR4/oxIMc6GsPvN9i4pi5buKLcg91h/fXabILjFZ6o7ccrrMMbhU1QfCzWCri7EyskFNTkMg61Wp2EtflBohTefNREHkv5dhAZLVRKdNmPUB+JRGSNa5qGHVXUMTSpE/yZgXLm21EhuqNRE342ShhYNYuqTwbK7zIGZUqcRuwaZF2fhORpSCk/AD6ot+03Yb//AvhFjHNfAF5IxDjagrJSazNIwG9SWlJnU+/VOwXTsHa9SCnpmZUStb1PvzSOFUaXwTYMk5zB7WEa6AykEdv5Hf4d21FmIasJPZRDMQQ1qWwluvf2KUROEmXErgJbijJDadqWLJQpqxil7WUSqWGGgg76NePadprfUrYUlaQYKmMyhJYlibYvuvZUExkyPMtygWF32Bg8tO5GcCc5OW/uqChzlstt54JLR+NyR8vra78+GZcr8nin084pY/uSPbCp8ebdhTSsTU82op2lXovjwvGhbNinoSaIlODP04g2f2gHacfEiTJFDaZjVK89Ql1PlADKnLoFlU/SOdFlRJrI1TdMYufWYxG+B4fTRv8BPRg9PnIF89WbT8PhsLH4/V1IUyJsgosuG81VN0yyvPa00wfjrQ3wn/9bT3W1D4Fg1tlDuOlb01v1M3UcfKiGQKHWq/1RD39jtv8JqNDYouDfNtTkcSz4M1QptbHqtiEHdjrK1NEQ2URGQYXovg5STX1MlL/MKgx4D8rf1t5CrenogoXNYOfWY7zy3GoOHzyO3S6YceYQbrp9Gimp1lEzAb9BZaWX9HR3VKitFaYpqa7y4k5yRmkeXRc/ddVNw4vJpaIyb+NRiquoq9sUwobKAQjFYuygrldD+DHZweOaQih5LJwc4rOTa7o+1cA6rM2YNlROSlOy6luOLljYTowe349H/nQZfr+B3SawBavZ+v0GxUerSM9wk95D3QzVVT4+WbiL9asOkZLmYs7cU5g8PafB8FmbTZw8v/twGCU4whcxkjpBMJHGK7Ja9fsONVPqi9JYQjkCx6irdtqf6HpD8ZCNMmeFMoaz6LiZ3Zq2RxK7SKOks0bPaaHRApxBrUFKyYfvbOft1zcDEDBMxk3M5mu3T+Wx+xZTecJ7MvR2745iZp8/nJvv6C4mp3ixKtUQohK1YptBbI1DEjvBS6Cyd3sFzx+DEhKhvIuW+CBcdIw6UZqOx94G9vWks/q+tCM8AXyxJI+3XttEbW2A2toAAb/Jtk2FPPrLjzheURuRq+H1Bvj8470cKWioPEF3pLEHyI8SLH6URrEy+NpNfIl9Vm1S0+J4X42mOXhpuARJU02hHQetaSSAd97YHJWUFwiYVJRH52KA0kw2rzscM+eje5KD0hRiaRsGqn/CXtQDGTJjhSrRZqA0iVgFC3XIsqYtaShST9CZ1+udd+QdiLIS65VuLLeFECIuh3j3IouGeyPYUXHu4Y7ycI4H97upu61DD+c49K2uaVtSiJ0/ZKdx/1zHRT9JCaBXH+syBKHihVFIydSZjTWP6Y4MQ/kbrKStQK3eGgqZ9aAExCjqwnVnELt+kEbTWjhQ2rNVVeOhdMZQ2xBaaCSAq66fGJXE53DaGDayN8NG9cadpKyAdrvA6bLztdunkWmREa6Buqqv4bV+klAlIBrzPwiUzyMbJXyG0tYhjRpNHcNQ92DovnWjFjSdO49H+zQSwBnnDqem2s9br20MVqg1OXXKQG7//ukkJTvZsuEIW9YfJiXNzRnnDKVfdo/GL9qtGYgSHpWoWzQVJRAG0LDfQ9IdO6lpOioCVfcqF3Vvdl7tIhyd3JdAAgGTspJqUtPcpKZ1Xptlx0SihMg+lOCwysTuSfOaA2k0HQ2JqowQKjfSH1XKpmWCRyf3dTAcDht9+6e39zC6IF5UIx4PdQ+NExVRZVJXiK4jNzPSaOJFoupThWvVZahF0QTaW2PRQkPTCbDqymegfBdDULexds9pOguSumi/FKILbpYQbYY1g9tKid07pG3QQkPTwanGuo2riar9NJL2XnlpNNZIVN/yUKOudFRYeR6RuUYpqH4vIYf5Maz9dmZwnxYaGk0D+IgtFEzq+kZrNB2NfFSflpAAqAi+6lONKqTZOfxxWqfXdEAkkauwWEXfwru0aTQdCT+RAqMhJMpnEQj+3R/r+9oW3Ne+aE1D04HwovoMlKAepCxUmG0sTLpSKKOmK3GCpt2XAiU0HKhk1PDqyaAERi86Qse/hCzThBAXCyF2CSH2CiHutdh/txBiuxBisxDiEyHE4LB9hhBiY/C1IBHj0XQGalAhhSF/hYGqZFtMnZZRhlLbG6KhHuEaTXvR1EKYdlTyH9T1th9PXXva8cFt7b9AarGmIYSwA08DF6D0sTVCiAVSyu1hh20Apkopa4QQ3wH+AHw1uM8jpbRuZddOGIbJF0vy+OyjPfj9JjPPHML5l5xCcrKuiNp8DFTzo3JU5IiXun4W6ah+F36L8xpT731E9gLXaDoC6dSFhTeGDVWqP1wgCOo0jo5FIsxT04G9Usp8ACHE68AVwEmhIaVcGnb8SuCmBLxvq2CakicfXsru7UV4vcrGePTICZYtyeOB/3cJ7qTECA7DMNmzoxi/32DUmD4Ju27HxAusRanf4UIg9ECdQIUfWgmIhpJPBbpMiKZjIoBTUetlg7r7uAcqVPwgSktORpUaad+IqKaQCKExENUsOUQBqkpcLG4DFob9nSSECM0oj0kp37Y6SQhxB3AHQG5ubosG3BDbNhWye0edwADw+wxKi6v5dPFeLrpsTIvfY8eWo/zlD59hBCRCKAFy0+3TOPuCrpqctgulEcRCNrI/FWXOqi9AstBCQ9NxSQFmobRrL6p/S6iEUPs7tJtLInwaVkY2y+WhEOImYCrwx7DNucG09huBp4QQlg2WpZTPSimnSimn9unTp6VjjsmGNQV4awNR230+g9XLDrT4+scrPDz5yFKqK33Uevx4avz4vAavPr+GvTuLW3z9joeJ8k00hg3rW8mGatEaqhhqDx7XGxiboDF2X6RpYvqj73dNogg5sAdQJzA6N4nQNAqIbISQAxypf5AQYg7wK+BsKaU3tF1KeSR3nEuWAAAgAElEQVT4M18I8SmqnGleAsbVLNxuBzabwDSj5Z47ueVf17IleZhG9LV9PoOFC7bz/dFnt/g9Oh7x1DcTKCGwg7qoKFDNk9yo2yoXpZG46Kz9CCq272fvK4vxV3nInTeLARdMQdiavnaTUlK+OR/THyBr0ghsjqblqviOV7Hqh0+T/8ZSdY2Jw5j55+/Tb/aEJo9F071IhNBYA4wUQgxFtVC7HqU1nEQIMRl4BrhYSlkUtr0nUCOl9AohegNnoJzk7cbpZw9l8fs7MX2RDix3koNzL2y5+aikqAq/38I5JqHkWFWLr9/xsKFWWCcaOSYXtSJLJjIDvAz4krqGSrmoPhkhfKh1SynK8ZhDR7UPb3n8DTb89iVMv4EMGOx9eRF9Z47jgvcfxeaM/1EsXrWDpdc9gLesEmGzIRx2Zv/zpwy+cnZc50vTZOG5d1Ox/SCmTwUflG3MY9HFP+fSZX+m16QRzfp8mu5Bi81TUsoAcBewCLVM/I+UcpsQ4kEhxOXBw/6IMui9WS+0dgywVgixCViK8mlspx0ZNKQnV351Ik6XHbtdIAS43Hamzsxl6qw6X0ogYLJz2zG2bSrE541fvR9xSt+T/TXCsTtsjBrbNyGfoeNxCnVmpfq4UCGFHmAbSmBYaSYS5VA8EHyBshOvRjkVq1C2422olrAdixN5R9jwm5cwPD5kQC0aAlW1FK3Yyu4XFjZydh21xRV8eME9VB8qJlBdi7+yBl95JZ/d9Chlm+JT0AuXbuTE3iMnBUYIw+Njw/0vx/+hNN2ShCT3SSk/AD6ot+03Yb/PiXHeClTZxg7FvK+MZ+rMXFYt30/AZzB5xiCGjaxbvW7bVMjTf/wcw1CObNOUfOM7Mzj97GGNXnvaGYN567WNBPwGRshMJcDltHPx5V3VRp+GCrI7hAq3TUZZNFOA9ahQ3HhCE0GZrg6iNI59qPgJWW//YZTG0XGc5Pvf/AxpREeHBWq87H7ufUZ/+7K4rrPn5UVII/q7Mmv9bH3yTc56KSpNKorS9XswvBaBB1JSunZXXOPQdF90RngM+g/swRXXRdeCKSup5qlHl+LzRj64Lz69kgE5GQwZ3nBctctl5zd/mMurz61h3cqDmKZk9Pj+fP2OaTHbxnYNkoguXb4X66ioxjBRwiKUOW5FKSqwr2Ng+PyYFkIjtA+Un2Lf60vZ+sSb1BZX0P/sU5l039fpMaLuc5RtysPwRE/40jQ5vvNQ1HYrUgf1we52EfB7oval5LRekImma6CFRhz4/QZrVhxg09rDFB2txLB4+P0Bk4/e28kdPzyj0etlZCbzvXvOQkqJlKqXePfkKE0XGKDMXKFWsLHoWDWpBl06ky1/eB2jxhux3Z7kYtj15wGw+qd/Z/ez7xOoVlnu+f/+hINvL2feyr+SOUb5ccq37Y/5Hn2mj45rLLlXzmbl9/9CoCpSaDhS3Ez46VeRptks57yme6DvjEbwePzc/5MPeOnvq1j5xX7y95RiBKInOmlKio5WAlhGXlkhhOjGAgMaT9yLhYnyXcQq7AYdzRnee8oohl53Do7UOpOZPdlN6qA+jLnrSqoLitn5twUnBQaANEz8VR7W3vscoDSSiq37Y77HgAtjN2QzfH6KV+2gdONe7G4nFy/5f6Tm9sWRloyzRyo2txNnZhpLr3+Q/0uZy+e3PIa3oisGZnQFfKjEwPbpuqo1jUZ4/61tHCs8gd/fcDkLh9NGRmYS93znbYoKK0lOcXLBpaO58vqJ2O1aNlvTG6VtWJGFipyK9WCUozJrQ4l/BkrQCJTjveNl2M/+5z0MunQmu559D3+VhyFfOYtT7piHMy2ZvS9/hOmzCKiQksOL1rD9L/8j59KZEKM9sz3VjTszLWp7oNbHmp/+nd3PL0TYBMJuw9UznfPm38+1+/5N6brdlKzfzeof/w3PkVIATF+AfW8spWxTHlesf0ZrHR2GWlShjVAxRAfqXm/bBZLuEd4Id3/rLUqLrZoA1SEEOF12pCkjhIvLbWfKjFzuvDu+UMjuhwdVVcYKN2pF1dD9mY16aEpRQsSJ0j46jgM8Xv475huc2BXbJ2FPUnkpSX0zqT5YZLHfyXUF/yEpqy6BrOpQEQum3om3+HjU8c70FK7d9y/cWT1Ycu39HHhrWZRAcqQlc978+xnYgAajaStM1LPirbfdhmrglBHXVRLRI1wvIRpBxjA1CRF82QRjJvSnRw93lDbi8xqs+fJAo0Kn++Ij9i3opfGKnjaUQ1ygwnYH0xkERk1hKWt+/iwLpt7Jh+f/hPmjb25QYAAYtT6MWh+eo2XYk92IetqrlDB/6NfY9Oi/KFy6AdMwWH7b45YCA8A0DPa++jEAxat2WmowRq2P0g17mvkpNYmlEOtSOyZ1IehtgzZPNcL02YP5+P1dBAKRAsHhsPO7v15Or94pCJvgG1e9anm+02nn4L6yLh4Z1VxC5qTmYEOZpFaEXcOJ6n7Wcb/ryn2FLJj2HQJVtVF5EvEgbDbG/fBqDr77JRXb959UxEyvH9PrZ/2vX1B+ivQUaoutusQpjBovlXsPA5CW25eagugSNvYkF6mDumruUGfCh4o0jKV1t+2iVAuNRrjsmgmsW3mI4+UefD5D5VS47My7ejx9+tXZkFPTXFRXRa8ETFOS1bvjTmLtSzoqCipWjsYw1Cqqfi6GQJUXKSK6au4G4HQ6khLtKSon718fU3O4hKLl2/BXVMXUYBvD8PlxpCbjOVoWcw4JVHmiIqPqY09202vKKABGfetSilZuh3pjsjkdDL5KmVZLN+6leOUOkrOzyJk7HburcZ+RlJKiL7dTtnEvabl9GXjx9CaXO9FA4x0A27Y1gBYajZCW7ubhp+bx+Sd5bFxziLT0JM6bO4rR4/pFHHfhZWN4/62tEfkbNpugd99Ucof2bOthdxJC9aY2xth/CFUltAKoRDkCHShTVKhGVX1CBRI7RvTU4cVr+eSK+zADxslM8BZhSrImD4+IsoqJEDEd567MVIZedw5SSjY/9lq0ABIw6b6bEDbBR3Pv5egXm0GCzWHH5nZy8eI/knWqZW1RAKoLilh4zt1UFxSDENhdDpzpKVy89AkyRuY04QNrGi74KYgsq9P6aEd4Aji4r4yD+WWsXnGAbZsKcbocmIakT/80fvKb88nqpZsExcYDrMJ62WxDZZInW+z7grqeyvXPGUF7JvYZPj8bH36VHX97B39ZZWIvLgRj7rqSim37KVyyocFD7SkupCFVFdswLaLnxGHMefcR0gb1pfDTjXx8+a8tNRN7sps+M8ZQvHI7Rm2kFp3cP4vrDr2OzR6tORR8uJqPL/tVdAa8EPQYOZCrd7yEEN051LypbEYFe1jRm6YU1UiEI1xrGi2g1uPnyYeXkr+3BCEEAsjqncoV100kd2hPcoe2fz/fjk+oJHqselOHUZqGl7ouf0OCP8tjXDM69LSt8B2v4n8Tb6fmUMvK3Luy0vGVV0Z/LVJSsHA1575xHx+c9SMCNd6Y2gQmnP/2Q+x7Yyml63aTNjSb8XdfS/+z6iodnNhzGGnGyFT3eDn6qbUWGKiu5djnm8k+d3LE9pojJXx8+a8tS6YgJTWHSyjfuo+sCY2X3NGEyEHd6/W/U4FaILUtWmi0gFeeXc3e3cUEwqKmfEXVfPzBTu5//NJ2HFlnwk10ZdsQEmXPDZ8Uy1BCZCiqjlX4gySIbHTTthheH+9O/27LBIaAYV+bQ88Jw9h4/8sYnvohluDOTKPX5JHM+/KvrP/Nixz9bBO+iuoI4WFPcTNo3ixyLppGzkXTYr5d5pjc5uVhCPCWRlYuDtT6eG/WXQ2a4YTDHnWepjGyUCaoA9QFfUhgNNZaeOuihUYzCfgNVi7bHyEwQDm+Dx88zrHCE/TL7hpNV1qfcajChWbwFeodHnrVx0S1XKnfU7k36iHaD/ShrTQOf7WHVT98mrxXFreooZFw2Dn9Hz9m1Dfn4ikqZ6NFxVlHahJjvn8VAD3HD+X8tx4E4NiyLaz52TOUrt+DKzOdMXddwcR7b4w6vz59zxhP+rBsju842KSxBzxeaktP4DtRjatHKmbAYMNvX6KmMJYZRWF6/fQ6rat2qGxNhqDyksqoa+wUPn17UELlOGohlosSNolHC41m4vUGYloF7A4bJ47XaqERN6koh/dRlMaRCsSTHyDr/R6+wj+I6pY2guaH9cbHx/N+RdHK7S3ugCfsNnIvmwVAct+enPXKL/j8679D2ASmYSKEYOh15zD8puii0f1mT2Deir/GvLbh9XHovZVUFxTTe8oo+p4xXplUhWDKo7ez4s4nLcNuG2Ltz55h9d1/Z/xPrmXnP97FV15pbZYKfT6HnYm//BquHjqasHm4UYKjPlWoRVdIw6tBCY9hRPbHSwxaaDSTlFQXPTKSKC+tidpnBExyBvdESklZSQ0ut530Hh0/6ax9caBst6AEQENx6fFgohpI9iJ6xeVBRWO5UJm0zRcqZZvyKF6zE9MbZ86Fw8bAOVMoXrkDf5XnpCnHkZrExHtvIKlP5slDh1x9JtnnTuLA/5YRqPIw4IIpJwsXNoXybftZeO7dmF4/hs+PzWEnc/wQLv7oj2z+w+tsf3K+8o3EwJ7kosfIgVTuP0qgqhakRPoN/MEquZsets5Rqs/0J77D2LuuavL4NY2xh+iwdRPIRwmZxE7zWmg0EyEEN942leeeWq7yN4K43HYuuWocu7cX8dLfVlJZ6UWakqEje3Hnj2fTu2/7OWk7DwI10Tds6mgcE9iKimPPQEVU5QevG54QOInm2obLNuUh4i06KWD6H+5k7A+vpuZwCVv++AaFSzaQkt2LcXdfQ87F06NOcfdMZ9Q35zZrbKByJRbP+yXekrrMcNPrp2xjHstu+yOH3l9pWWo9hCM1iQEXTOHcN3/LZzc+wv75nzd9EHYb5//3AXIvP705H0HTKLGSOEVwX2LDz7XQaAHTTx9MUpKDN1/ZwNHDJ8jMSuGya8eTk5vJ7379UUTORt6uEh6690Mef+YqnE6d4NQ4o4C1RCf2NRUDpVVUopzqVvs3ofwqPpQfxB331dMG91P5EHGQnN2LsT+8GiEEqTl9mPmnu+J+n+ZSsmaXpePZ9Po58PbyBpUs4bBzyh3zmPb4nQghlL+iGSH6Y79/lRYYrYqN2Ml/iZ9rtNBoIRNPG8jE0yJzAv70u08jtA9QDvJaj5/1qw4xY/aQNhxhZyUJmIESHHEksrUID7COuocvAyU8klCJhK6YZ/Y7cwKOZDfmwCyqz5pCJUkEjp5AbNpL7yMHcfrVKl4CPb42j8Xv7yQjM5nJ0wfhcrX+4sFXURVVpyqEDBiIBhYwMmBQue/oyZyKgRdOpXTt7qicjYYQTgfO9LaP8Ole9AWOYZmhGWchw6aQEKEhhLgY+BNKrD0vpXys3n438H/AFJRt4KtSyv3Bfb8AbkMt+X4gpVyUiDG1JwUHKywXx7WeAEcOWReQ01jhRE3crS00oK4HOSiVvgL10OWjtJ5QJV0bysHeF/AgpRt5wxl8XtCHwMkq7ylwan92TTDJKjpM9oGdFIyYgGefAzN/PQ6HDfs/VvGzB+Y02umxpfSedkpMf0vG6EFUHTiG4bcOkRV2GykD6sY3+juXs+Ovb2P6Aycd3sLpANOM6QCX/gBHFq/ntAdubeEn0cRmBKpcuhd1D4dynybQGuV0WnxFIYQdeBqYi6oJcYMQon6z69uAcinlCOBJ4PfBc8cC16NsAxcDfwter1MzICfDUu13JznoNyC97QfUqcnG+jYNTd6tiURpHjtRkVkBlAlrP7Aa2EZV1SqWHO5DwGpettko65fDtulzOJ7VD5/PIOA3qfUEqK7y8eivPmLtigNRxTATibtnOhPuvQFHSlgghhDYU9yc/szdjP3hV06WXY8avsvJKXfU5Rsl9crg8nX/YPjX5uDMSMWd1YPR357HFZuep8eoGKVBhCA1x9qmbvoD5L26mI8uuZePL/8V+9/6Ak9ROcd3HTrZAlcTD05gGjAGFS01HBWNmNnQSc0mEZrGdGCvlDIfQAjxOnAFqltIiCuA+4O/zwf+KpTOewXwupTSC+wTQuwNXu/LBIyr3Zj3lXFs31wY4dMQAtxuB1NntW2dmM5PSPWuoM5ua0MptbEaOLUVJhvX1Aa7L8aw9Tfg7/DWBnjmqeUkpzj55SMX0X9g64RoT/7NzWSOzmXzY/+m5kgpvaeMYvKDt9J7yij6nzmRnLnTWXPPM5Ss3YXN5cTmsCMDBjP+9D2yJkbWl0od2IczX/o5Z9Z7j4sW/YG3xt4alYxoT3Yx9gdXR43J9Af4cM5PKV2/52QdrYKFq5FS4khJQtgEkx+6lXHfjz5XY4UNlZvU+j3eEyE0BqIqy4UoQBmjLY+RUgaEEMdRsZADiezCU0CMokFCiDuAOwByc3MTMOzE88Unebz12kbKSmpISXWdLI4hTRgwKIPv3XNmm9ixuxY2VLnzUlRV25DqfRTrsgptW0stEJDN8Q2fxOcz8PsNnnxkKY89fXmr1WQaet05DL3uHMt9/c+cyGUrn6a29DiHP1wDQpBz8TTcWfELsbTB/Tjr1V/yxc2/Uz4UqQTDlEduo9/s6NpI+a8vjRAYwEkTV6gO1rpfPE9SrwyG33h+Ez6pprVJhNCwusutPDJWx8Rzrtoo5bPAs6AKFjZlgG3B4vd28p9X1p/ULmqqfbjcds6eM5K5V47V/TRaRCjbO2TmWEXsaJG2FRwTT0vi3/+M3bciHqSEo0dO8Na/N3L1jZParZhfUq8Mhn8tOnHQCmmaHPl4PWWb80kf0o9Bl5/OkKtmM/DofI4sXofpNxhw/uQowVNztIx9byxlx1/fbrRSr1HjZeP9L3dzoeFD5RtVoZJeBxAd3XcCtaASKM28dU3giRAaBUSmHeagPqXVMQVCCAfKpV8W57kdHsMweeu1TRHmKFCd+z7/eC/X3Tw5xpmaxBNqztQ29O7r4KLL0/no3Up88QcVWfLB29uprQ3wtdti14rqCHjLTvDB2T+m6sAxTK8fe5ILe7KLSz57ioxTBjH4Suv2xvvf+oLPv/4oSOKOwKo6FN3atvsQyvQO+dZKUQabU1FTqAR2ocy3oUVUAUqwtF6plkS41tcAI4UQQ4UQLpRje0G9YxYAtwR/vwZYIlVN9gXA9UIItxBiKOqTrk7AmNqU8tIaAjGKtAkbFB1V5bGllKz4LJ9f/+g9fnDrm/zl95+pSCtNE+mH9a0rULdQS2/rpq30r/16Bj/4RW/GTnTjjj/FI4qA32TJwl0cr2i4gVJ7s+I7T3FidwGBKg+mP4C/sobaogrePf0uvBXWpeC95ZV8/vXfYXh8TQrZTcvt1/hBXZbtqAVQSCCYwb+3owRGGZECI3TMEWIn/LWcFmsaQR/FXcAilHfyBSnlNiHEg8BaKeUC4J/AK0FHdxlKsBA87j+obyEAfE9K2XbLxASRmu7GjNGJLRAw6ZGhIlfeeHk9nyzcjc+rahStW3mQLRuO8MtHLmz10MuuxSCUOu4h0jmeg4q2SkMVbwup9OnBv+OJUhKoYD4TtYqL73acMDmJCZOTgF7k78nmnf9sZceWo3hrm1aPKhCQPPnwEqorfdTW+vF6DXpkJnHhvDHMmTsKW4yci7bC8Po4+M5yyzpb/vIqFky5k6u3v4jdHRmRdfDt5fFnzgexp7iZ/MAtjR/YJfGiakhZ4UPd+4XEbkR2lI4cPYWU8gPgg3rbfhP2ey1wbYxzHwEeScQ42ovkZCdTZ+ayduXBiKq3DoeNsROz6ZGZTEW5h8Xv74zYL6WKoHntxXX84uEL22PonRQ7KuWnCBUK60Cp5KGHJB0YH3Z8SLWvonHBYUOVFHEQv29EBN9/EJDMsJHw41+dC0BpSTXLluRRfKyS4xW1bNlwBNnIEPbtjezUVny0ijdfWc/+vSXc8SNr009bYXj9Dbaq9RwtZ/9/v4jyQ/iraxssZjjoqjPwHqugZN1u1R3QYee0R25j2PXnJWzsnY/G/HMN3Uitt/bWGeEJ4tbvzqTyhJfdO4pwOGwYAZPBw7O488fqId+zowiHwx5VSh1gz86WNezpnthRWoVV1c/62FD1pY6gVmCSSC0lHIGqVWVDORXr9yEHpb3YUavBVFRvD+tIo17Bplwh9ueXcf9P3m9yxJXPa7B6xUEuv+4E/Qe0X/VkV49U0odlc2K3VUkW1bjpyMfrooTGwAumsKaB65oeH5cu+zPeshN4yypJG9wPm7M7T09uVGKrlbbhRC1s+mLdnCkUfts6dOf/SkJJSnbyswfmUHj4OIUFJ3AnO9m/t4QFb25mzPj+JCU7ibVqcLv1v6H1saM0gVDcRRHRfcZtqMSokAloNEooFAB+lDAZSksKwA0ZlkVKmovqyqZ7zW0Cdu8oalehATDr6R+y6KKfg0XHP+F0kNw/uo9DximDyD73VA4vtBYdRxavQ0qJO6tHk0J9uzZjgI2oezQUbCpQOdShSKnDRGrQNpSmndgiheHo2SrBZA/M4PCh4zz18BJMKQn4TZYu2kP2wB44HNH2aKfTxtlzhltcSdO69EXVlNqH6uGRjGp0E+5bEqhmNonNCxo2sjdb1jc9SNCUkrS0FnjaE8SA809j1t9+yMrv/SnK5GRz2GNW5R3342s5+ukmy6q6sepjdW96oHKdD6MKbqah0thCtbxswGSUBn0suC0b6E9rlA8JoYVGgvHU+HjmyWURBQu9tQEOHzzO9NmD2bDqEKYpCQRM7HYbQ4ZncfWNk9pxxN2ZTNRD17Zcdf2p7Nx6FL+vaeVDDEMy4bTWLp0SH6PvmIe/oooN97+McNhBqAKHZzx3Nz1GWObn0v+sidhcziihIRx2cq+c3W75KR2bJJT2G4tQAEiMMi6tgBYaCWbTusPYLG5+v99g64YjPPXiNaxfdYjjFR5GjOqDzS5Y8fk+evVOZdzE/u0eHaNpfYaP6s33f3Y2Tz6ytEm+jT590zpUWf0JP7ueEbdcyOFFa7E5HeRcMh1XRux+MXaXk3Ne+zVLrrkfaZiYXj+O1CTcWenMeOp7bThyTUvQQiPB+HxGzHgHv9/E7XYw66yheGv9/L8Hl7AvTzUastkEKaku7n3oQvpl66KGXZ1Tp+Zw1Q2n8t78rVFl9GMxYlRvPl28h8yeyUyYPAB7B1hgJPfLYsTN8Uf+5Vw8na/sfJnd//yAqn1H6XfWRIZdfy6OlCRK1u1m7S+eo2T1Tty9ejDuR9cw5ntXIGzt/zk1dQjZksI57cTUqVPl2rVr23sYlpSVVPOz77yNv16UlM0GM88cyreD0VQvPP0lyz/Nj4imEgL6D+zBg09cyobVBZQUVTNoSCbjJw0IFsXTdCWklHzwv22899+teL0BjEDDz6LTZUcIsNlsuNx2fv7gBeTktk4sfltTvHonH553d0TbWUeKm6E3nMfs537ajiPrWggh1kkpp7boGlpoJJ75r27go3d34g0m8TkcNpKSnTz4xKX06pOKaZjccf3r+C36GDhddpwuG6Yh8fsMnE47Wb1T+dWjF5HWo/2doJrEYxomb76ygUXv7WhUcITTs1cKTzx3dZdYUHxw9o849sWWqO32JBdX73hJdUjUtJhECA2t97UC19w0me/ecybjJmUzaEhPLrxsDI/8+bKTRQsDhozZQyHgN6ip8lPrCWAYktraAMcKK3nx7ystj9d0fmx2G9s2FTZJYIAKutizo2vUZipevdNyu3DaKfpyWxuPRtMQ2qfRSkyamsOkqdYRDS6XnX4D0jl6OLp3s5XiZxgmG9YU4PcbHcoRqomffXtLOZBfRlbvFMafmh0V8JDajFBaI2Cy6N0d1HoCTJgcfc3OQNWBY5Rv248zNQmvRYdB0xeg6MvtZJ87ieR+0fkfGlDZ3wWosFuBCrsdQGvpBFpotBM33T6NP9frJe502jGlab3ilJKAFhqdDq83wBMPLSF/TwmgAh6Sk53c+/CFEUl65196Cnm7S06aNOPB7zdZt/IQ2zYW0n9gD37x8IXBJNKOj+H18dlNj1Lw3kpsSS5VJl2IqFWT6fWz+/kP2P38+5z+jx8z4uu63E4kBqq/fXiFgzxU8upkmlp8Mx4639KkgyKlZM/OIpZ/ms+B/LJGj58weQA/+e35jBzdh6RkJ9kDe3DDN6fENFH06pNKcop1W05Nx+X1F9eRt6sEn9fA5zWo9QSoKPfwxENLCPcnTpkxiNnnDWuWf6K2NsDhgxW8/fqmRA69VVn902co+GAVhteP/3g1MlQl2iawuSIFn+HxYnh8rPj2k928VLoVR4kuiWOissRLW+UdtaaRACrKavj9bz+mtLgagcrcHTK8F3ffdx7JDaz8Ro/rx68fu/jk36uW7cfhsFn6O0aP047AzoaUki+W5EUFPEgJFeUeDuSXnaxuLITg5m/P4IzzhvHoLz+yrFHWEH6/yRdL87n+1hb5ONsE0x9gzwsLozPDpcTmdJA8IIvq/ceizpOmZN/rS5hwz/VtNNLOgFVtNFAaSDGtUU5EaxoJ4C9/+Jyjh0/grQ1QWxvA5zXI313Cy39f1aTrbN1YGNNBXnj4BDu3HWPNigOUFFUlYtiaVsYwlEnRCptNUFWpwkullOzPK2Xj2gJ6ZqXy2z9e0uB1YyVOB+LM92hv/JU1MSve2pPcMfMyTJ8fb7m+9yNpaN3fOqZsrWm0kJKiKg7kl0X10wgETNZ8eYBvemfiirMgoVVtqhD788t48uElgMAIGMyYPYTb7prVKZ2f3QWHw0Z2TgZHDh2P2hfwGwwZ3ovS4moef/ATSouqsdkFAb/BzLOGMuG0bLasL4z7vYRNMGGKdfmO9sRf7WHf60spXrOLjFE5jLj5QtxZ6bh6plF7rDz6+BPVJA/IwuZyYPoi/TuOtCQGXjClrYbeSRiAalFkVek2ngrQTUfPOC2k8oQXRwMTd60nOiIkFqsuPKUAACAASURBVKXF1TH3+X3KHl7r8eP3m6xefoAP3t7epLFq2p6bbp+GyxW54nO5HVx0+VhS01w8/sDHSkv1BvDUqP/tl5/vIyc3E7vFIsLlsjP73OERlZHtdhvJyU6u+3rb1tGSpsmhD1bxxTd+z7JvPU7hpxsj/DRVB47x35E3s+pHT7P72fdYf9+LvDn8a5Ss2cWUR2/DnmIdMXZidwGmP4AIC/qwp7jpM3Ms/c/RddoiycK6J3ifGNtbjtY0WsiAnB4YFiWiQYVRpvVIivtau5sQc+/zGXz03k7mfWV84wdr2o1xp2ZzzwNzmP/qBg7uKyezZzKXXj2O2ecNZ39eGaUlNdFaqt/kw3d2cPsPTuf1F9fh8xoIm/KF3HT7NM6aM4Kps3JZtGAHxys8jDs1m7lXjSOrV0qbfS7TMFjylfsp/GT9ycinfa8vZdiN53P6P36MEIJl33qc2uKKk6Yow6PMcUuve4Br97+Gzelg7c+exXO0XuCIKcEmSB+WjeHx4UhxM+pb8xh715W6qGEUlcFXfYpR/V4SnxCshUYLcSc5ufyaCSyYvwWft86m7HLbueHWKU2KhnE67XiIXzOprvQ2fpCm3Rk1pi+/fOSiqO3lpTUx/RNSwusvreOJ567m4L5y/H6D4aN6nzR1TpqWw6RpbVfZtD4H/vtFncAIDjhQXUv+vz9h+E1z6D1lFMc+22Tpu/CWV1G+OZ8RN12Ar6KatT97JrpvuCnxlVdxw9H/tsGn6cwcInYHv0JUuf/E0iLzlBAiSwixWAixJ/izp8Uxk4QQXwohtgkhNgshvhq27yUhxD4hxMbgq1PqnvOuGc8t355B3/7puFx2cgZn8r2fnsWss4Y26TqzzxuO0xn/v2TwMJ3s1JkZNKQntZ7YeRmeGj9ffJLHiNF9GDOhf9y+sbZgz8uL6gRGGIEaL/n//gRpmDELdwohMIKJfO7MVFVa3QJXRmqihtuF8cTYbgLR/59E0NK78F7gEynlY0KIe4N//7zeMTXAzVLKPUKIAcA6IcQiKWVFcP89Usr5LRxHuyKEYPZ5w5l9XsuaKV351Qls31xIYTASy+m0IYRgyIgs9u8ti0gEdLnsXHfLaS0duqadKC2u5omHljR4TMBv8q9/rmHvrmKuuWkyn328h8KCEwwb2Yszzx9BWnr71SKL2e9bSsyAiTM9hZ7jhlC2MS/qEGG30WvyCAByrziDFd95KuoYe4qbMXddmdAxd00yUTkZ9UW0HcholXdsqdC4Ajgn+PvLwKfUExpSyt1hvx8RQhShvDQVaCJwJzn57R/msnnDEXZtK6JHhptZZw8jvYeb9/67lcXv7aSm2kfu0Cyuv3VKk3M3PB4/fm+A9IwkbRtuRwIBk4d/8SFlJVb9nyMxDMnqFQdYvfwACCVINq4p4N35W/nN7+fSf2D7tEYdcdMcipZvjdI2HKlJDPvqOQCc8exPWHjeTzC8PqTfACEQDhuDLj+d6sMlpA/pjzM9hfP/9yCfXPUbEAIzYCCEYNAlMxj93Sva4ZN1NgahOvfVD7d2oLpTJp4WVbkVQlRIKTPD/i6XUkaZqML2T0cJl3FSSlMI8RIwC+Wx+QS4V0ppaagXQtwB3AGQm5s75cCBA80ed1ekvKyG9+ZvYdO6I6SkOJkzbzSzzx2OzSY4XuHh+b+sYNumowgBmT1TuOXO6Uw8reOFaHYH1qw4wPN/WdGgaaoxhIARp/SJSA5tS0x/gEUX/oyStbtOCg5HahI5c6dzzhu/Obkoqcw/wtYn57N//ud4S44rwWFXGvSkB77BxHuUtdpfWcPBd5bjLa+i/zmnkjVhWLt8rs5JNbAHKEeVDekNjEK1M46kTUqjCyE+RjWdrc+vgJfjFRpCiGyUJnKLlHJl2LajqE/3LJAnpXywsUF39NLobU1ZSTX3/fh9PDU+DEP9P91uB1Nn5XL792fx87sWUFJUhWnU/a9dLjv3Pnwhw0e1XgN6jTUL3tzCW//e2KSufVbY7YKnX7mu3crLmP4A+978jLx/fYzN6WDkLReRe8XpUcl5hxevZcnVv43SSuzJbuat+AtZp7bMrKsJEbqhYlsREiE0GjVPSSnnNDCAY0KIbCllYVAAWMaMCiF6AO8Dvw4JjOC1Q9lLXiHEi4DuttIM3vnPFmpqfBFCwesNsGbFAYad0psT5Z6IfQA+v8E7b2zm7vvOa+vhdnv6ZafjdjuorW2+phGiPdvh2JwOht94PsNvPL/B43Y9+56l09z0+dn94ofM1K1eE0TbmJxbmty3ALgl+PstwDv1DxBCuID/Af8npXyz3r7s4E8BXAlsbeF4uiWb1h2OEgqgamBtWXfYup2ohEP7ozNyNa3PlBmDElKNdtCQnqSkdvwilr4K69If0jA59N6XHN9T0MYj0rSElgqNx4ALhBB7gAuCfyOEmCqEeD54zHXAWcA3LEJr/yWE2AJsQRniHm7heLol7iRrhdFus9GzVwout3VIY1/di7xdcDjtnHPhiJj7hYCMzCRS01xMmTGIW787E5fbjs0uguerTpC33TWrrYbcIgZfOTtm9nfV/qMsmHonZVvy23hUmubSougpKWUpEKWbSinXArcHf38VeDXG+do2kgDOv3gUb766ISK5EFQhvCu/OpH1qw7hrQ1EmDJcbjuXXzuhwetu3XiEhe9sp7ykhtHj+3HJVePo3TetNT5Cl6D4WBX780rJyExmxOg+MRM7S4ureXd+bKVaSnj0r5eTFtaYafS4fix+fwdHCk4wbGRv5lx6Cj2z2i4DvCWM+P/tnXd4VFXawH/n3jslDRJaCCX0EoqAVEGKUhQbiAV7WV0/y+ruura17Gddce2uqyu69v0EEVHAgnRp0kvoobeQBEhIn3q+P2YSJpk7YUImDc7vefJk5txz7n3nzJ3z3nPOW+64hK3/nEHevqPIcvGk8ErceUWsfvQDLvn5ldoRUFEpVI7wswCPx8s7Ly9ia+pR3C5vaeDD/3n4QvoNSubokVz++cpiMtPz0HSBEIKb7uzLsNGdQp7zhxlb+G7KxlJFpOsCw6Jz0SWdaBgfRd9BySTW85mKw+HmxLECGsZHVWmZx+Px8tE7y1m9fD+6oQOS2Dgbjz03isSkYJPYWd+k8u3/bQwKHxJIfKMonn/jchrGR52xXHUJZ24By+5+jX3Tl5huxGg2C7cX/VwLkp1b1Ij1VF1EKY1gpJTs3nmMbZuOEh1jpf+QNjRoWDbuVUZ6HsVFLlq2bohRQQbA/DwHf/rdN7hC5HQwDJ/J5Ljre3LlaWYrdRGvVzL9vxv4ZfY2NCHweLwMGNKWO+4fFBRcEHwRadeuPMjuncdo0jSGwcPbE9vg1Cxg5rRUZpULIyOEL3HWax9cHeQT89Una/n5+4qDTeq6YNiojlx4cQdOZhfRrmNjGjWp3x7Sh35exaIbXsCVG+yfYsRGcfWWj4ltXT2+BQofNWI9pagfCCHo2KUpHbs0DVknMSmOY5n5zPpmM9knCul2XnP6DUoOUiDbt2SgG3pIpVGS82PmN6n0PL9FaSKh+sKsaan8MntbmUF+1fL9uFweHnh0WJm6uSeLeeHxnziZU+zz0rfqfPPfDTzyt5F07uYb4ObO3m6yNAj5uQ7StmWV1iuhR+8kFs7ZiaMC6ymPR7LolzSWL96LpglcLg8XDGvH7+4fVG/D4Sdd3Aehmz+seJ1uvu1yO0kX92HElGewxJ4dM6yzEaU0ziFWL9/P5LeW4fVK3G4vS+bv5n3vUnRd0LtfK26+uz+Nm8aUCbtdES6Xl1/n7TpjpZGRnsePMzaTtj0Li0XH7fbgcnnpfl4SV1zTg8ZNI/9k7fF4+en7rUGDvMvpYf2qQ+RkFxGfcGrA+vLD1RzPKij1f3H5LdHembSIdz65Fk3XKMgPEThS+Bwry9O9VxLJ7RLYt/tE6fnMkJIyimXl0n00b9GgXkY2zlq1nS1vTye6ZRNc+YVoFgNP4al+8zp9saiOzF/Hr7f+nZEzXqgtURWnQSmNc4TiIheT315Wxvy2ZE3d45GsW3WQndszmfTuOFJ6JIaMvhqI9EoKC8OPyhvI/j0neOnJObicnqC1/WMZ+fy2ZC/PvnaZ6Z5AVSgqcAWlXy1BNwTHMvKJijJYtmgPqRvSWbfyYHBYH3wKM217Fl26J9KqTYJpXniP22uqUDVN8Nhzo5kzcysL56RVmEclEKfDw5xZ2+qF0ig+fpLVj3zA3mmL8DicCCl8uTakRLP5zI2FoZ/KDe7H63BxeM4aijJOYG+WwL5vfiX1H1MoSj9O00Ep9Hn2DhJ6VC4QqCKy1M95rqLSpK4/ghYijSb4nmqLi9wsnpuGYdF56Inh2GwGFpM1/hJsdoO+A1uHLYPX42X9qoN8PnkVb720EEex23Qz2OORFBW6+PqL9WGfO1yiYixYQuznFBe5ycku4q8PzuKrT9ay7jdzhQG+PYsSBXzjnX2DEy1ZdfpdkEzTRHNrM6tV58pre/LGhxO4+8ELsFh19ACT2lBKuz6Ew/c4nMwe9Af2/N9832zCI5Feb+kGuNfh8uXWCGEIoFktFBw+xvrnPmPp7/7B8bU7KTxynP3fLWP2oD+QtXp7TX4cRTnUTOMcwbcPUbHRg8vpYeumo1w+oQfdzkvi9Q+vZsWvezmWmc+qZQfIO1lcup9hseq0bB3P+WEqjcICB688M5cjh3ODlobMkBI2rz8S1rkrg65rjB3XjRlTNyJNtmw+9M/GKrJsAt8solNX3/5RSs/mPPzMxXz1yVoO7c8mOsbK6Cu6hj0jGDqyI526NuPX+bvIPVlMcrsEvv5svemMqFWbeJMz1C32f7uUooxsvK4KPN4lSLMvAN9SlS0hls3/mFo2z4ZX4i4sZtWf3+Pype9EWGpFuCilcY7QvVdS6YAfCk0TZfww4hrYGXNFCgBX39ibX2ZuY8WSvWhCMPTiDoy8vCv6aTZli4tcfDF5FcsX7z3tQFyeimY5VWHs+G7MmLLR9Fg4oT0sVo3e/Vvx6rPzsVg1RozpxIAhbXn+jcvPWKbmLRtw/W2nQt3v2n6M9asOBoXDn3h73c+RfXTJJtz5ofI8VIwebaPz3ZeTvXkfmtUITs4EZP52rqY5Pg7sxZdtwg60wxcwvGY5J5WGM7eA3V/O4/janTRMSabTHZdib1I9sefrCg0a2rnm5t7M+GpjyCd9w9AYdVkX02NRURbGTTyPcRPPq9R1X3t+Pnt3Ha+8wrBoDBsZ2mu6KkgpEUJQWXNzm02nU9emZBzNZ8PqQ6UD+p6dx1mz4gAPPDosYiHn7/nTEGZOS2XeD75w+K3aJHDjnX3p3ispIuevTjSbxbd+V8n+FYZGj79cT++/3Urmsi0hJ8ZGVO3lEak9jgI7OJWlrwDYCnQAajaD4zmnNHJ3HWb24AfxFDlwFxSjR9nY+MKXXDr/NZr0Mx8wzxYuG9+d9p2aMHfWdg4eyOZYRj4Wq+7LpObxcsd9g2jdNmRk+0qzd9dx9u85gTuE6W4obHaDVsnxlVZQ4eKV0KhJNMcyw9uALsHjlWRlFXAyu6jMDMDhcLNp7RF2bs2kSyVznITCMDQm3NiLCTf2isj5aoq8fUdJ+8+PZxRJUTN0Uu6/Ck3XaTakO0aMDVdeWZ8OzWah4x3BqXPPbiSwi+C0rl5gD9CCmtyePueUxpI7XsFxIrd0E85T5MADLLz+ea7d/eVZn5yoa/fE0uRNToebbZszkFKS0iMRm73qQfRKKCp0snr5ftN9g0A0zZf1sHmLOBACR7GLzinN6HZeUqXyq4fL3B+28/Vn684oIKjb5SUzPc90PHQ43axdeTBiSqO+kvrKFDzFZ2ZRp9ms5O09SlRiIzRdZ9SsvzNn9KN43R48Die61UJ897b0e/n3EZa6ruMgOMlSIIVAzYX3OaeUhiMnn2Ord5habRRn5XBy+wHiU9rUgmS1g9Vm0Ktv+ImYPB4vc2ZuY/5POygucpFyXhLX3tyb5i1OmcVKKZk5LZXZ32xGCBHSvLUETRfc+vv+NZL/esvGdL7+fJ151N9wCbHsoglRqfzuZysZSzYFmdGGi9fhpEGHFqXvm/TtzMRDU9n/3TIKjxynaf8uJA4776x/sAvGILQRiwQi97AXDueU0pBuT+gnTCHwlg+mVk/Jyshj/epDCCE4f0DriDnJvfvKYjZvSC8ddNcs38/mdUd47o1T/hS//bqP2dM3hz0wG7rGulUHGTS0rO29x+MtXdMvyHfSMjmeG+7oS88+LUKcKRiHw42j2E1cAxtCCH6csSUsyy0Ae5RhmllP132Z58orQ8PQuGCY8h+Ibt2UnK0mWTU1gdAEMoQxhjB02l43AnvTstZhRrT9tPk6zn4MoBFwgmDlEQfU7B7POaU07E0a0qBTK3K27As6ZkTZiO/RtsZlijQzpmzkh+n+CKpCMPXTtVx9U28uv7p7pc+VlZHPof3ZNG4Wi9fjZfPG9DLKQErfwDzjq03c+/CFgC+0SLgDM/hMgU9mByfo+fDt5az97UDp9Q7tz+GdlxfxxydH0KN3xYqjIN/BJ++tZP2qgwDENbRzy+/7cyxMJzohYMyVKfxcznNc0wVNm8UiNMGRgyfLtBk6qgOt2kRuP6i+0vMv15O5JBV3YTl/Eq9EVjBD0K0WBrx5P/n7M4hu0RjNck4NTWGQAqwHivEpDoEv4Wnlf9dV5Zz7Zi786BF+Hv0IHocL6fIgdA3NZuHCjx9FCxEXp76wc2smP87YEhQz6rspG+nWszntOoYX7sPl8vDvN5aycc1hDIuGx+MlNtaGxxP8lOj1SramHi19n308OBgdhDam0XUtKOVsVkY+a1YcCHqadzo9TPl0LS++FVppSCn5x//O49D+nFIT4+zjhXzwxlK6dGtGZnreaS25Lh7bhQk39kIAP363FcPQ8Li9tEyO5+Kxnfli8qqgz7Z62QFuurNfhYEgzwVajOpLn+fvZN3TH+NxusouBZvcPyW4C4uZ2vJ6NF1Hs+j0fu52uj90TQ1IXF+wAP2Bk/j2MKKAeGoqW18g55zSaDowhfEbPmTLW9M5tmYn8SnJdP/ztWdFaIKFc3aaLgu5XF4Wz0sLW2lM/XQdm9YexuXylA7c2a7CkMuqMbGnwoq3ahNP2rasoDqGRcfj9pYZsDUN2rRPoEOXskpj/54TGEbwEhDA4QMng8oCSdueRfrh3CCfFKfTQ25uMYZFM50JaRogBGOu6MqNd/qCgE64qTdjx3fj4P4cGjS007xFA55/7CfT4IROp5v1qw/Rf/C5sycWih4PX0fisPP4cegf8TrC3xT3Olx48dVf9+THGNF2utx95r4vZx8Cn6KoXQfPc05pAMS1b8Ggdx6sbTEiTn6ew3Rgl15JQW544Sc8Hp+CKa98QllB2WwGY67oWvr+2lv68Ppz88u0NywaMTEWcsotQ0l84cPLb2zGJ0ThDWGyGaigzDh8IAcZYiaReTSPRo1jyEjPLZ31CCE4f2ArUno2p0//VkFJpqKirXROORWl9liWeepSl8sbdgypcwFN19CtlkopjUDchcVsePYzpTTqIMrc4yzi/IGtTVO72uwGvQeEF+7D6XDjCbFZabPpWK069igDq1XH4o+vNDwgmVPX7on84fHhNEuKQ9MEhkWjY9emnMwJ3reQXliz4gDHMssOxB26NKFhQlRQ/CWrTWfMlSkVyt+kWWxpWtTyWCw6x7MKyiyTSSnZtPYIA4e0CSsrYXLbRubnNrSI+rjUdxJ6tEOE+B70KBu63YqlQTSaxUCzmlv/FB457otZpahTVGmmIYRoBEwF2gL7gOullNkm9Tz48oADHJBSXuUvbwdMwWcasA64VUoZHDdAERaDR7Tn5++3cSwrv9ShzmLRaZoYy4Ah4S2b2KMsxDW0k3MiOAyE1wv/eP8qdm7LorDARdceibRoFexJ36tvS3r1bUlxkQuLVefJh2aF9PXSNI29u46XGbCFEDz27EhefXY+OdlFpfkkBgxuwxUTKt74635ec2LjbDgdZeNHWW06XonpkpemwbpVhxgxJnQmwxKuvvE8dmzNKLNEZRgajZvFktKz+WnbnytoFoNB7z7EsnveOBUCXRPodiujvn+Rhl1bU5R+AgT8OOzPpuewJyYgKgiyeXYggRygiNrcp6gMVV2eegKYL6WcJIR4wv/+cZN6RVLK3iblrwBvSimnCCH+DdwFvF9Fmc5ZbDaD/311LD/O2MKKX/cihGDIRe0ZO65byMiu5RFCcP1t5/Ppe7+VjXtk0xk+uhONmsQyaGh4jkT2KN8TZGZ6Xsg6Xq+XhgnBCXeaJsbxynvj2JN2jJwTRbTt0Dgs02FN13jypUt4Z9IiDh88ia5rSK9kws29+Pk785hFUhJydlWeDp2b8qcnL+LzD1aSlZHvX95qze33DqwWZ8T6TIebRhGbnMimSV+Rm3aIRr060OupW2jUqwMAMS19cZOaDuhK5oqtpTk1AIwYO72evLlW5K45nPgsohycsoiyAX3wWUbVTaqU7lUIsQMYIaVMF0IkAYuklEGxOIQQ+VLK2HJlAsgCmksp3UKIC4BnpZSnjRGg0r1WPyuX7mPaF+vJysgnroGNseO7MXZ89zMaGP901/SQVlUNE+y8/fG11eKwlXk0z+fj0bohVpvBlx+tZuHPO4M2yS0WjUn/GhfW8lQgRYVODIsetkJWmOM8mc+vt03i8C9r0K0WpMdDj8duoPczt57ljnwb8M0yAsfgks1us2fsqlPrOcKFEDlSyviA99lSyqCFXSGEG18PuYFJUsrvhBBNgN+klB39dVoDP0kpTeNJCyHuAe4BSE5O7rt/v4kDkSLilAT3qwrzftjO1M/XBVkd6brg7/+8kuYtaiZYZF5uMX97+Afych2lGfOsNp1Lr0rhmpv71IgMitAUZWZTnJlDXPskjGj76RvUa5zAcsxNEgUwmOqYbdRIjnAhxDzAbLH2qUpcJ1lKeUQI0R5YIIRIBXJN6oXUYFLKycBk8M00KnFtRRWIxJPeyMu6kJNdxM8zt6HrApfLS1KLBjzy7EgSGkVHQMrwiGtg56W3r2ThL2lsWH2IuAY2Ro7tUi8ix9ZVTmzazaGfVqHbrbS9dljpktOZENUsgahm54oxgRufcgilNNzU1SWqGlmeKtfmU2A2MB21PHVOUVTo5PDBkzSMt9M0Ma62xVFUASkly+5+jT1TF+J1utEM3xLdwLcfoMvvr6hl6eoDXmAZPuVQHgMYQnUYt0ZiplFVqWYCt/tf3w58X76CECJBCGHzv26Crze2Sp+2WghcW1F7xdlDVLSVjl2aKoVRT5BSkrFsM7899E9W/OFtji7eWJqDZN+0xez9ehGeQgfS7cFT7MRT7GTln/5F3p7IZ1w8+9CAjgQPwRq+HBl112qsqtZTk4CvhRB3AQeA6wCEEP2Ae6WUd+MLmvKBEMKLrycmSSlLzFgeB6YIIV7EZ0bwnyrKo1AoIoCUkuX3vcme/84vjSO167NfSL76QoZ99gTb35+Ju8DE98bjZff/zaf307fWtMj1kCR84UH2csrkth3QpKJGtU6VlIaU8jgQFIJSSrkGuNv/ejnQM0T7PcCAqsigUCgiz9FFG3wKI0AxuAuKOTBjKYduWIUz19z73et0U3zcbLtSYU4T6rqSKE/dnQPVQTxOF4d+WsneaYspyjhR2+IoFNXGrs/nms4k3AXFpH3yM8njhiCs5s+cGQHLWIqzj3My9tSZcPTXTcwf/4wvrIEEr8tNj0cncv5zd9S2aApFxPE4Q8eM8jpcdHvwalL/MQWPSQ6a3LTDZK7YSuLgmg/brah+1EwjDBw5+cy94kmcOfm4cgtx5RXiKXay5Y1p7P9uaW2Lp1BEnHbXj8CIDfaVMGLstL/pYmwJcVjjzZ0hPU4Xmcs2V7eIilpCKY0w2Pf1ItNkEO6CYja/Pq3mBVIoqpnWVwwicUhPjJhTisOIsdO4b2faXjMMgKhEc58K3WrB3rRmHDYVNY9angqDoozs4ExkJcfSj9ewNApF9aPpOqNmvcTeKQtI+/wXpMdLx1tH0+HmUaVZ9Xr8+VqW3/dW8N6HEKWKRXH2oZRGGDQdlIIRY8edXzbyq9A1mg/vVUtSKRSnR0pJduoenDkFND6/E5bY4OCQodAMnQ63jKbDLaNNj7e/eRRZq7az86MfEYbuix6gCUZ9/yKWuJrz9FfULEpphEGLkefTsGtrslP3nkoqIwRGtJ3zwozEWZSZzebXvubg7BVY42NJeWA87W/yWStnb9qDM7eAJn07nwMxdxQ1xckdB5l31dMUHjmG0HW8Ljd9nr2dno9OjMj5hRAMeudBejxyPUcXb8LSIJqWl/THsNfN8BeKyFClMCK1RW2EEXHlF7HumY/Z9dkvuIscJI3oTf/X7iWhe9sK20mvl6LMHGb2uQdHdh5ev7WJEWOnxei+ZKfupejoCYSuIT1e+v79bro9eDW5uw6zb/qvSI+X5KsGnxXpaBU1h8fpYlqbGynKzCmzH6dFWRn+xZO0nTC0FqVT1Ba1HuW2tqjrsaeklGx9azqbJn1FcVaOb2mryAEh0pAGYkTbaXPNUPZNW4z0eJFeiWY16HzXWAa+/YezPFS0IlLs/24pS25/BVdecEh6a0IcEw9/HdaMQHq9IIS6784S6kLsKYUJ6575hHXPfEJxVg7gs7IKR2GALzfy7i/n4Sl24nW5kR4PniIHaZ/8zOE5qysti/R6OTx3Deue+Zgtb01XTonnCPn7M0L6Wjiz8/jtgbcrbH/o51V82/1OPjVG89/4q1j9+OQKfTcU5w5KaUQYV34RW978BndhsDdt2IQw793x4Q+VOo27yMGPw//Mgmv+l40v/Ze1T37EtPa3cGDW8jOXTVEvaNy7I8IInRxqz1cLQoYCOTJvLQuueZaT2w4A4MorZNu7M1h0wwvVIquifqGURoTJTTuEVk2Z3FwnzX/kospNkwAAFt1JREFUoUh95SuOr92JO9+nwDzFTjxFDhbd+KLpsoXi7CFx2HnEtUkMeVwYOkUZ2abHVj/+IZ6isibmniInh+es5uTOgxGVU1H/UEojwkQ1b1S62R0KzWZBt1uJatG41Oa9zLEoW1AbPdpGm0ravu/8z094ip1B5ULXODBrRaXOpag53IXFpL42le/Pv4eZA+5j2/szTZeG9n+/zLeEZB3D1NYT2frPGaUxn4QQXDL3VYRu/hOXXi8xrcwTJuVs2WtarhkGx9elneGnUpwtKJPbCBOd1JjmI3qRvmB9GeWh2600v6g3Tfp3wdoghnYTR2BE2VjxwNvsn7EU6fHSsGtrBr79IOv/9gnHN+zC43co1CwGRowdS2wUHqcL3WqpUIYTm3az7d3vQj5J4pF4qrJ8pqg23MVOZg9+kNy0w6VP+zlb97Pv60VcMu9VNN03i9379UKW/O7V0nuk8PAx1v71IwoPZ9Fv0j2A717s8ehEtr3zbRnnVD3aRvc/XoNh8nACYG8ST+GRY0HlUkqiW9aviKyKyKOsp6oBR04+88c9zbE1O9GsBl6HixajzmfElGdM/TC8LjcepwtLTBTuYie7v5zLlremU3AoC3d+MZqh4XV5MGLtWGKjuXzZO8S1NcvAC7u+nMvye9/E63AhPV7TOrrdyoRtnxJbwfKFonbYMXk2qx5+P2hPzIi1M/zLp0i+ajBSSr5ucwOFh4IHdt1uZeKRadj8caGk18uGF79ky+vTSh84uv/lOno/fQtCM5+FbH7zG9Y983GpQgJAE8S1bc41aV8oS6p6TI3kCFdUHlt8LJctfouc7QfI232E+G5tiGsXOg+1ZjHQLAaugiJ+uPCP5O06XCY0g9fpG/zdeUV4ChwsuvEFrlzxr6DzuAqKWHHvm2V/7OUwYux0vufyChVGyYNEcWY2a578iP3fLkUIQdvrhtP3pbuwN1FxhcojpSRjaSpHF27AGh9Lu4kjiEpsVOnz7J222NSIwp1fzP4ZS0i+ajCuvEKKQ8wiNZuFnM17SbzQl8JGaBp9/nYbvZ68GWdOPtb42NLUrKHo/scJ5O0+zM7//IRusyK9HqJbNGHMT5OUwlAopVGdxHdNJr5rctj1t737Hbk7D+IpCt6HKEF6vWRv2E1h+nGikxqXOXZ08abQFjMCEi/sSbeHJtAmhGPXidQ9rHzoXY4u2YRmGAhd862l+2csuz6dw5Ff1jB+83+wxIQfjuJsx+N0Me+Kp8hcsQV3YTG6zcqav37IsDNwogsV5kNoGob/mBFtR+g6uDxB9Vz5RT7finJohh62sheaxgXv/pFeT9/K8XVpRCUm0Pj8TkphKAC1EV6n2P3lvAoVRgnC0HGVi4MFILTQP2rNauGyxW/R9pphpj/+vH1H+fHCP3J08UbwSrxOl29NPWCJy+tyU3zsJLu/nBfmJzo32Pr2t2Qs2+ybHcoSKzUnv976dxzZeaX18vdnkPrqVNY/9xlZK7eZJirq/PvLy0SWLUGzW+h0+yW+14ZOxzsuQTdzzvN4mXvFU5zYtLvKnyu6eSNaXzaQJn07K4WhKKVKSkMI0UgIMVcIkeb/HxQrWQhxkRBiQ8BfsRBivP/Yp0KIvQHHeldFnvpORYN+IEaMnQYdWgSVNx/eC2nmRKgJmo/ohdcV2qpr86tTfF7rp8FdUMyRuWvDkvNcYcfk2UEmquB7Yj/w3TIAtn8wi29T7mDdMx+z4YUv+HnUIyya+HzQrKDV2AG0v3kkerQNNA1h6OhRVnr85Tqa9OtSWm/A6/eROOw8U3nc+UUs/d2rEfyECsUpqjrTeAKYL6XsBMz3vy+DlHKhlLK3lLI3cDFQCPwSUOXRkuNSyg1VlKde0+G2MabmtoHo0TYueO9PppuYRpSNoZ8+jh5lQ5T4imgCvJLMpZv5KvEadn051/S8R5dsRrqDlzvKI3SN6BaNT1vvXMIsLSqA9Hhx5ReRt+8oqx5+z+fl73SDV+IuKObQT6uCZm1CCIb8+2EuW/wWvZ68id5P38JVq9/n/OfuLFPPiLJx3mM3QIjnjOPr0pQHt6JaqKrSGAd85n/9GTD+NPWvBX6SUirPMhNS7h9HQo+2p5YnhEC3W4nr2JLYts1pNXYAl859tcJ18rYThjJ+44d0+8PVvuUL/8TDXVCMMyef5fe+SfrC9UHtwrWk0qwWuvzPlZX+bGczrS4fFHIvqeWYfuydutDUks1dUMz2f88sU+Zxutjw4hfMH/c0W9/5luPrd0EIK6e8veml368ZFc0sFYozpaob4YlSynQAKWW6EKLZaerfALxRruwlIcTf8M9UpJSnXyM5SzGibFy+9B32fbuE/d8uwdowhk53XkqzCyqXa7lBx5Y0G9KDHR/9AOWc+zyFDja8+CVJF/UpU97jL9eRvnC9qeWVHmtHEwKv28MF/3rotJF9I4X0etn+/ky2vDUdx4k8mg3pTt+X7qJRz/Y1cv1w6fO/t3Hg+6W4ThaWDtS+tKgjadilNe78opAOn4F7U1JK5l31NBlLUkuXuw7MXE76gvVcueZ9GnZqVaZtg86tfDMNM8UhCGlSq1BUhdMqDSHEPMDMKeCpylxICJEE9ATmBBT/FTgKWIHJwOPA8yHa3wPcA5CcHL5FUn1Dsxi0n3gR7SdeVKXz5O06HHJTPXfnoaCypBG96f/KPax+bLIvDIoEzWow9NPHfSFHhKDVpf2xNjTPC10dLL3rVfZOW1yqyA79sJKjCzdw+dJ3aNSrQ43JcTpiWjVl/MaPSH11Kod+XIk1IY5uD15N+xsvpjgrh4wVW03baXZLmQx3x1bvIHPZ5rL7I1LiLixmw3OfM/zLJ8u0bzaoG0ZMVFByMIDGfTuHdN5TKKrCaZWGlHJUqGNCiAwhRJJ/lpEEZFZwquuBGVLK0oXWklkK4BBCfAI8UoEck/EpFvr161f/PBJrmIYpyRhRVlx5wQNKfIiZQsoD4+l42xgylm3GiLLRbEiP09r0Vxe5u4+wd+qismFQpG8vYPXjH9Dl91ew56sFaBadjreNoeWlA6rNwqfgcBarHn6fAzN9gR5bXzGIAW/cT2zrUxPr6KTGDHzjfga+cX9pmdflZvbgB8k/kBF0TmExiGqWQLeHJpSWZSxNNV1Skh4v6YuCt/s0i0+p/3rry3gcTvBKhKFjRNsY+vFjVfrMCkUoqro8NRO4HZjk//99BXVvxDezKCVA4Qh8+yGbqyiPwk+rsQOxNYnHXeQss8GtR9no/bfbQrazxEXT6tIBNSFihWQsTUUY5ssrR+avJ3PZltIN6IOzfyN5/BCGff7XiCsO58l8ZvW/n+KsnNJ9iQPfLSNjSSoTtn9W6nltxoGZyynKzEaa+FPEd0tm7II3sCXElZbZmzQMGSvKEmfuv9F2wlAadm7Flrenk5t2mGYXdCPlD+OJaWkeV0qhqCpVXfScBIwWQqQBo/3vEUL0E0J8VFJJCNEWaA0sLtf+v0KIVCAVaAK8WEV5FH40Q+fypW/TfHgvNKuBbrcS3aIxI756msTBldsjqQ3sjRsgRIjb0+stY7HkLijmwHfLOGryNF5V0j6dgyu3oMxGdolV1M7//Fhh22Nrd+I2mekBeB3uMgoDIHncYNMAk1Dx/kRCj3Zc+OEjXLboTfq9/HulMBTVSpVmGlLK48BIk/I1wN0B7/cBLU3qXVyV6ysqJjqpMZfOfRVHdh7ugmKiWzapMSctR04+QIVP4iVIr5f0Bes5ueMgDTq3osXI82kxpt8ps+FA/CbE5XEXOtjz1YKgDf6qkr5wQ5lgfyV4Ch0cXbiBnn+5PmTb2OREjGi7aViQ2HbB24S5aYfR7TZTn4/ctMO4i50q/7ai1lFhRM4BbAlxQU+11cWJ1D0s/d2rZG/aA0CjPh0Z+vGjxHdra1q/8OgJfhrxZwqPHMfr9qBZdKKbN2Ls4rcY8+Mkfrn0caTHi9fjASmxxEVTnJljfvFqUIixbRIRhh7kwyJ0jdi2FZspt7vhItY8MTmoXI+20fPRiRxbu5MNz33O8fVpxLVPos2EoWiGTkhvGZPwIApFTaOi3CoiRmH6cb5NuQNXboAbjhBYG8Zwzc7PTWMf/TTyL2QsSS0zKAtDJ/HCHoxd8AYeh5ODP6ykOCuHxCE9yE7dy7J7Xg9yqDNi7Iya9RJJIyIbVCBn+wFm9r036Olfj7Zx5cr3Tmt+nLVqO/Ov/huuvEKEJpBuD/1fv48GHVsyb9zTPgs3/29Qj7aBlKZWb00HpXDF8ncj9rkU5yYqyq2iTrHjg1l4HOW8kKXE43Cy88MfOO+vN5U5VJyVQ+byLUFP8dLtIXPFVooys4lqllDGmbFhSjK7vviFjCWppYrDiLHTZsJQmg/vFfHPFN81maGfPMbSu04lNJJuL4M//EtY/ipNB3Rl4sEpHFu9A3dhMU0GdMWItvNt1zuCfGI8hQ50uxU9yorX6UZ6vGhWC7rdwuAPHo74Z1MozgSlNBQRI2vldrzllQa+VKFZq3cElTtPFqAZumkbzdBxniwgqlnZcGaarjNq1kscnLXCb3Jr0PHW0b49kGrar2l3/QhaXzGI9EUbQUqSLuptmhclFELTaDowpfS982Q++fvSTetqVoML3vsTRxdt4OTOQzQb1I2UB5U1lKLuoJSGImLEd2vDkQXrkeV8DTSrhfiUYIfM2HbN0e1W09hNut0aMgeJpuu0GX8hbcZfGHQse8s+1j/7KZnLt2BvlkDPR66n/U0jq6xQjGg7rS8bWKVzlKDZrCH3X6TbS+PeHehwU5B9iUJRJ1BxBhQRI+WBceiW4OcQzaLT5d7geFWarjPgzft9a/kB6NE2+r9+b6UdC4+vT2P2oAfYP2MpReknyN64m+X/8yarH/ugch+kmjHsVlpdPigoPzxCEN2qCQ1T2tSOYApFGCiloYgYce1bMPL7F7AnJmDERmHE2olu0ZjRP7xcxns6kI63jGbk9OdoOjAFW+MGNB2Ywsjpz9Hptksqff1Vj/zbN2sJMMl1Fxaz7d3vKEw/fsafqzoY8sHDxHVIwoiN8nlxx0Vhb9KQkd+9oHJXKOo0ynpKEXGk10t26l4QPsezmgqc95n9EtPAgEZcNBd+9AjtrhteI3KEi9fj4cgva8hO3Uts2+YkjxuMblN+GIrqQ1lPKeokQtNqJaCgEWPH6cwPlgewxsfUuDynQ9N1Wo0dSKuxkdkrUShqArU8pThr6HzXZaYpUDWbEXFPcYXiXEUpDcVZQ5/n7qDJwBSMGDua1cCIjcLaMIbRs1+utWi9CsXZhlqeUpw1GFE2xi54nazftpL12zbsiQm0GT+kUj4VCoWiYpTSUJxVCCFodkH3Smc7VCgU4aGWpxQKhUIRNkppKBQKhSJslNJQKBQKRdgopaFQKBSKsFFKQ6FQKBRho5SGQqFQKMKmSkpDCHGdEGKLEMIrhAgZz0QIcakQYocQYpcQ4omA8nZCiJVCiDQhxFQhhAq8o1AoFHWYqs40NgMTgF9DVRBC6MC/gLFAN+BGIUQ3/+FXgDellJ2AbOCuKsqjUCgUimqkSkpDSrlNShmckq0sA4BdUso9UkonMAUYJ3zxny8GvvHX+wwYXxV5FAqFQlG91IRHeEvgYMD7Q8BAoDGQI6V0B5S3DHUSIcQ9wD3+tw4hxOZqkDXSNAGO1bYQYVAf5KwPMoKSM9IoOSNLl6qe4LRKQwgxD2hucugpKeX3YVzDLKOMrKDcFCnlZGCyX6Y1VY0JXxMoOSNHfZARlJyRRskZWYQQVU5EdFqlIaUcVcVrHAJaB7xvBRzBp5XjhRCGf7ZRUq5QKBSKOkpNmNyuBjr5LaWswA3ATOlLGbgQuNZf73YgnJmLQqFQKGqJqprcXi2EOARcAPwghJjjL28hhPgRwD+L+AMwB9gGfC2l3OI/xePAw0KIXfj2OP4T5qUnV0XuGkTJGTnqg4yg5Iw0Ss7IUmU562WOcIVCoVDUDsojXKFQKBRho5SGQqFQKMKmziqN+hCiRAjRSAgx13+NuUKIBJM6FwkhNgT8FQshxvuPfSqE2BtwrHekZQxXTn89T4AsMwPKayTcS5j92VsIscJ/b2wSQkwMOFat/RnqXgs4bvP3zy5/f7UNOPZXf/kOIcQlkZTrDOR8WAix1d9/84UQbQKOmd4DtSDjHUKIrABZ7g44drv/HkkTQtxeXTKGKeebATLuFELkBByrkb70X+tjIUSmCOG/Jny84/8cm4QQ5wccq1x/Sinr5B+Qgs8RZRHQL0QdHdgNtAeswEagm//Y18AN/tf/Bu6rBhn/ATzhf/0E8Mpp6jcCTgDR/vefAtfWQF+GJSeQH6K82vsyXDmBzkAn/+sWQDoQX939WdG9FlDnfuDf/tc3AFP9r7v569uAdv7z6LUo50UB9+B9JXJWdA/Ugox3AO+atG0E7PH/T/C/TqgtOcvVfxD4uCb7MuBaw4Dzgc0hjl8G/ITPP24QsPJM+7POzjRk/QhRMs5/7nCvcS3wk5SysBpkqYjKyllKDfYlhCGnlHKnlDLN//oIkAk0rSZ5AjG918rVCZT/G2Ckv//GAVOklA4p5V5gl/98tSKnlHJhwD34Gz4fqZoknL4MxSXAXCnlCSllNjAXuLSOyHkj8FU1yVIhUspf8T2QhmIc8Ln08Rs+H7kkzqA/66zSCBOzECUtqWSIkiqQKKVMB/D/b3aa+jcQfFO95J8uvimEsFWDjBC+nHYhxBohxG8lS2jUXF9WRk4AhBAD8D0B7g4orq7+DHWvmdbx99dJfP0XTtualDOQu/A9gZZgdg9EmnBlvMb/XX4jhChxEK6Tfelf4msHLAgorom+DJdQn6XS/VkTsadCIupIiJIKL1CBjJU8TxLQE5+/Sgl/BY7iG/gm4/Nbeb4W5UyWUh4RQrQHFgghUoFck3pnbKcd4f78ArhdSun1F0esP80uaVJWvh+q/X4Mg7CvJYS4BegHDA8oDroHpJS7zdpXs4yzgK+klA4hxL34ZnAXh9k2UlTmWjcA30gpPQFlNdGX4RKxe7NWlYasByFKKpJRCJEhhEiSUqb7B7HMCk51PTBDSukKOHe6/6VDCPEJ8MiZyBgpOf3LPUgp9wghFgF9gOlEMNxLJOQUQjQAfgCe9k+1S84dsf40IdS9ZlbnkBDCABriWzIIp21NyokQYhQ+RT1cSukoKQ9xD0R6oDutjFLK4wFvP8SXRqGk7YhybRdFWL4SKvO93QA8EFhQQ30ZLqE+S6X7s74vT9V2iJKZ/nOHc42g9U7/wFiybzAeX36S6uC0cgohEkqWc4QQTYAhwNYa7Mtw5bQCM/Ctz04rd6w6+9P0XqtA/muBBf7+mwncIHzWVe2ATsCqCMpWKTmFEH2AD4CrpJSZAeWm90AtyZgU8PYqfNEkwDdTH+OXNQEYQ9nZe43K6Ze1C75N5BUBZTXVl+EyE7jNb0U1CDjpf8iqfH/W1O5+Zf+Aq/FpQQeQAczxl7cAfgyodxmwE58GfyqgvD2+H+YuYBpgqwYZGwPzgTT//0b+8n7ARwH12gKHAa1c+wVAKr7B7Usgtpr68rRyAoP9smz0/7+rJvuyEnLeAriADQF/vWuiP83uNXzLX1f5X9v9/bPL31/tA9o+5W+3Axhbzb+d08k5z/+bKum/mae7B2pBxpeBLX5ZFgJdA9r+zt/Hu4A7a7Mv/e+fBSaVa1djfem/3lf4LAld+MbNu4B7gXv9xwW+ZHi7/fL0C2hbqf5UYUQUCoVCETb1fXlKoVAoFDWIUhoKhUKhCBulNBQKhUIRNkppKBQKhSJslNJQKBQKRdgopaFQKBSKsFFKQ6FQKBRh8/8cOh4/6n1p3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "\n",
    "n = 2 # dimensionality\n",
    "points_per_class = 100 # number of points per class\n",
    "num_classes = 3 # number of classes\n",
    "\n",
    "m = points_per_class*num_classes\n",
    "\n",
    "X = np.zeros((m,n))\n",
    "y = np.zeros(m, dtype='uint8')\n",
    "\n",
    "for j in range(num_classes):\n",
    "    \n",
    "    inds = range(points_per_class*j, points_per_class*(j+1))\n",
    "\n",
    "    # Generate radius and angle for each point\n",
    "    r = np.linspace(0.0, 1, points_per_class) # radius\n",
    "    t = np.linspace(j*4,(j+1)*4,points_per_class) + np.random.randn(points_per_class)*0.2 # theta\n",
    "\n",
    "    X[inds] = np.c_[r*np.sin(t), r*np.cos(t)]\n",
    "    y[inds] = j  # class label\n",
    "    \n",
    "fig = plt.figure()\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral)\n",
    "plt.xlim([-1,1])\n",
    "plt.ylim([-1,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = 0.05\n",
    "x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1\n",
    "y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "\n",
    "X_test = np.c_[xx.ravel(), yy.ravel()]\n",
    "\n",
    "def plot_model(scores):\n",
    "    # Put the result into a color plot\n",
    "    Z = scores.reshape(xx.shape)\n",
    "    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)\n",
    "\n",
    "    # Plot also the training points\n",
    "    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)\n",
    "    plt.xlabel('x1')\n",
    "    plt.ylabel('x2')\n",
    "    plt.xlim(xx.min(), xx.max())\n",
    "    plt.ylim(yy.min(), yy.max())\n",
    "    plt.xticks(())\n",
    "    plt.yticks(())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions for Non-Linearities\n",
    "\n",
    "First, setup some functions for non-linear functions we may use in the neural network, together with their derivatives."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# ReLU: \"rectified linear unit\" nonlinearity\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "# Derivative of relu wrt its input (for backprop)\n",
    "def relu_prime(z):\n",
    "    d = np.zeros_like(z)\n",
    "    d[z > 0] = 1.\n",
    "    return d\n",
    "\n",
    "# Logistic function (aka \"sigmoid\") nonlinearity\n",
    "def logistic(z):\n",
    "    return 1./(1+np.exp(-z))\n",
    "\n",
    "# Derivative of logistic function wrt its input\n",
    "def logistic_prime(z):\n",
    "    p = logistic(z)\n",
    "    return p*(1-p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train a 2-Layer Neural Network Using Autograd\n",
    "\n",
    "Here we will show a complete example of training a feed-forward neural network with one hidden layer using autograd. First, we need to define a multiclass loss function for our model.\n",
    "\n",
    "### Cross-Entropy Loss Function\n",
    "\n",
    "We would like a multiclass loss function analogous to log loss for logistic regression. First, suppose we are operating on a single input $\\mathbf{x}$ with class $y$, and we have computed class scores $s_1, \\ldots, s_c$ for this example for each of $c$ classes. We first transform these scores to probabilities by the following trick, known as the \"softmax\" transformation:\n",
    "\n",
    "$$\n",
    "p_k = \\frac{\\exp(s_k)}{\\sum_{k'} \\exp(s_{k'})}\n",
    "$$\n",
    "\n",
    "We have first exponentiated the scores to make them positive, and then normalized them to sum to one. \n",
    "\n",
    "Then, the appropriate generalization of log loss is for multiple classes is:\n",
    "\n",
    "$$\n",
    "L = - \\sum_k \\mathbb{I}\\{y = k\\} \\log p_k\n",
    "$$\n",
    "\n",
    "Here the indicator function $\\mathbb{I}\\{y = k\\}$ is equal to one when the true class $y$ is equal to $k$ and zero otherwise. Note that, like log loss, this simply picks out the negative log predicted probability for the correct class. This loss function is usually called the \"cross-entropy\" loss function, for reasons we will not dive into.\n",
    "\n",
    "This is the loss on a single training example. Our overall loss function is an average of the cross-entropy loss over all training example:\n",
    "$$\n",
    "L = \\frac{1}{m} \\sum_{i=1}^m \\Bigg(\\sum_k -\\mathbb{I}\\{y^{(i)} = k\\} \\log p^{(i)}_k\\Bigg)\n",
    "$$\n",
    "\n",
    "To train a model with automatic differentiation, all we need to do is write a routine to compute the loss, and then we can automatically compute the derivatives of the loss function with respect to parameters for gradient descent. Here is a complete example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 0: loss 1.098531\n",
      "iteration 1000: loss 0.298401\n",
      "iteration 2000: loss 0.261372\n",
      "iteration 3000: loss 0.249408\n",
      "iteration 4000: loss 0.245247\n",
      "iteration 5000: loss 0.244474\n",
      "iteration 6000: loss 0.244194\n",
      "iteration 7000: loss 0.244081\n",
      "iteration 8000: loss 0.244031\n",
      "iteration 9000: loss 0.243955\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD5CAYAAAAHtt/AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt0pHd5H/Dv83sv885Fo/uuVrveXRub9QWIHTANYBIwgaYxFI4NgdoJ5Vb7tPVpexIgQJv0lJYS41P6R2hP4VDsOPWWEEhIioNpi1NDSAiE4BgMvq69911ppRlp7u/l9/SPd2Y0I71z1YykkZ7POT7HK72aeVdafeeZ3+X5ETNDCCHE6FDbfQNCCCF6I8EthBAjRoJbCCFGjAS3EEKMGAluIYQYMRLcQggxYiS4hRBixEhwCyHEiJHgFkKIEWMO40HHEzGeG08M46HFiPLdACWeweRMDI5/Cf6Su923JMSO8/jq6iVmnu103VCCe248gc+97w3DeGgxoi6czOKn/gfwjvccxZWZ+5F94NR235IQO87cw18/2c11MlQihBAjRoJbCCFGjAS3EEKMGAluIYQYMRLcQggxYiS4hRBixEhwCyHEiJHgFkKIESPBLYQQI0aCWwghRowEtxBCjBgJbiGEGDES3EIIMWIkuIUQYsRIcAshxIiR4BZCiBEjwS2EECNGglsIIUaMBLcQQowYCW4hhBgxEtxCCDFiJLiFEGLESHALIcSIkeAWQogRI8Ettswx+hz+8AvP4pn4uzDx7sPbfTtCjCwJbrEl5o5MQBkKx4wv4MvHz9TDWwJciN5JcIstUwvva+z78aM/+Dg++cc3SfUtRB8kuMWWmjsyUQ9wKBPkOBLeQvRIgltsr8R+kONs910IMVIkuIUQYsRIcAshxIiR4BZCiBEjwS2EECNGglsIIUaMBLcQQowYCW4hhBgxEtxCCDFiJLiFEGLESHALIcSIkeAWQogRI8EthBAjRoJbCCFGjAS3EEKMGAluIYQYMRLcQggxYiS4hRBixEhwCyHEiJHgFkKIESPBLYQQI0aCWwghRowEtxBCjBgJbiGEGDHmdt+AGG2sNYJKESCCEUuAiLb7loTY9SS4Rd/c/ApKl87V/0yKkNx/BEYsHnk9MwOsAZI3ekJshgS36IvvllFaPAuA6x/jAMhfeAHpy46BVHM4V1aXUcksgHUAkILJqaavFUJ0T0of0TOvmEPh7HOIDF6t4RZWmj7k5rIoL18IQxsAWMPEKvbhr4Z/s0LsQhLcoieB56K4cLrtNZXli+GwSFU5uwhwc8gTgDn6dtN1QojuSHCLnriryxtCeD3WGn4xt/bnwIu8zkAF0P5A70+IvUCCW/SkVQivuwrac+t/UpYdeZWHFKBkmkWIXklwCwCA9l2UMxdRXDyLysoS3HwWfqW0YSjDjKeATkv+SEHZsfofnam5DV/DAM7yG2X5oBB9kHJHwCvmwnHrakg31tTKiiF54CiUEf5TsZLjqKxcgva9lkMmyjDDgK+y4ikk9x9GefkiAs+FMi2U/CQyeOnQ/k5C7GYS3HscM4fL+lqEsPYqKC6cQerAUQAAKYXU/BWorCzBK6zWK2ntlgEiWIk0nOm5DZW0GU8hdXAtzAsns8P5CwmxB0hw73GBW+64siMoF8FBADIMAAApA87kPjiT+7biFoUQ68gY9x4XVsYdluQRwKy35H6EEJ1Jxb3HKSsGMkyw33q1CJEClLGFd7W1yFFQKRNB1gP8bV5XToB9ZRL2i5LQJY3Kj1YQLHezkkfsJRLcexDrAJXVDPziKkiZcMZnUM4shFV1xLAJ6wCV5QuIz8wP5PkvnMxCBxrQPnj55EAesy8GYeyX9iN2dQocMIgIhb9aQuWJHFTcgL/kAsFggtyYsUGWgn+xDLR680LA+K8chDnvQNkGOGAkXj6B3NcvovLTXIsvEnuRBPcew1ojf+4EtOehNkTil/Owx2dg2g68Yg5efuPEoZvPwh6fhmHFNnyuWxeqE5I60HgqeB/efvshXFX6IrIPnMKWTVUSYF+Vgn1VEuY+B+aUBTIVqPqbkHztDJI3TYN9BkAoPLKA8uOrfT+dMWUhfdtBqJQZvihqIPfQBbjPFTZcG7tmDOZ8HMoORzDJoPqLS+XZPODJLlMRkuDeY9xcJlzK1ziuzQx35RKcy47BL+Vbfq1fKvQd3LUquxbYt5a+CJSA7AOn+nq8vtQr2jAcmXnD6hdSBIBA1ZGh1Bv2Ich48E6X+nu+f3QZVMKoPm4ofds8it9bRvHRpfqPwZiy4NwwUQ/tRqwZ1mVxeCeKvd+D2JUkuPcYr5hrufSvvLLYZoUJQRmbG+d+iu/EO953FFdm7t/awK6KXb2uou1m849FSP7CDIIVD1CEyhOrcJ/dWC1HfunRBMiiptCuPW/iFVNQloHiXy1j/O3zMCZtwED0i4mtMH7bQSBglH+8ivwji9s/Fi+2lQT3HqMME0HUJ5jhriyj5QoTAsz42BDvbLisI3Gk/v6+yIq2HSKCecCBOe+AiGBfkYT7TB65r13o+LUq0fqFjgyC87I0rMNxGJMWyGh9X/XgV4TYS9JQYyZWv3Ku5fVi95Pg3mPs9BS84mqLqnvdx6oHHpBSSO4/vKHH9o6ggNi16XCCsaJRfmxlw7CG8/IJpH5hBmS1v/+oapeZmypmZSvErkqhdNCBf7YM63Ac5gEHetVH5Zl8vRKmhAFyDJDZ+jlZM4yJjaHdeB/r70lZCvaRBNS4Bb0SrjYx98cQ/3tTMKcteGfLKP71MvSKNO/azSS49xgj5oBMG+xVOl6b2H8YpAiGHd+RPUXMAzGMv/MQyFYgIjAzYlemUPirJZS+mwkvsqhjaHPA0KsedDmAOROrX8sBR+90sAj2lUmkXjcDY58DMgjsM1JvmEX2wdOwrgg/xwEAZjBHD8u0uqfGayO/7xSGtbviwb4igfTb5gEjHJIxpmOIXTuG7AOnZBnhLibBvcdUMotg3+18IQDT2blnSBqzNiZuv6ypoiUiwCYkXzsDihso/20WxpgVLvWzoh+HmZH/3wsoP76C5OtnYO6LhYFNAPsaIALZ674HAcPa58Dc79TDlwwCW4SxW+dhjjevVGmnr++vQUj94iwyJ4tIvWl/0wsAGQSQQvJ1s1j9IxlO2a0kuPcYN5fp2E8bwI4/+Df5mmnAiL4/UoTEyyeQuGEC+W8ubpgcbMQeo/yTVcSuGYNz/UTzsIWpwhMfNnwRYExbGypmUgRzKrqFbeR9Rnx/Ww7XrKvClWPAeeVEuMxw/eMqgnVZ9LmfYneQ4N6l/FIBbj4LgGElJ2DGk9XhhE5b1wkgQnzmwEDuo3EZINgFl8t9P5Z1NIHEKyehxkyotNX2hYUMBRhA8nUz8DNuOATSEPThwcUAfI2p9x0BDNowcUkGgQMN7TKgqy92ipB76AJSN8+2fu6IF4qoQI782ojQjrzOUogdSUa/sNQotN7sI0aaBPcuwczwS3m4uWzYOKphOMQrrMJKjiM+Mw8znoRfbL1W24gnkZg9WG/juhnr125fVfoisp/9Yl+bbVK/tA/Oy8abJu26CkMCCt9eQvKmaZjTNhjVQPY0yCCohAkkWgckB0D2wdNQyXAttnuqCHiM8v4Y4jdOQjVU3azDsXKVNCPHr2v3y0E1TWnjUsH114eXRbwQaAYljdaLgCzC2JsPIPdnF+BcOwbriiT0qofyY7KFfjeQ4N4lSpfOhm1WowKIGV5hBfbYJJypA8iXnwN0dCmm3crAQvun/gcA7TftkOxH7LqxptAGUJ+M7IgIOucj+3unwmV3CQMUN5B+y4EN4+ORLwS+RrBYQbDQ/OHid5dhX5EEJi2QrcAeAwFj5SvnMP6uQzDWD6MQgQOGd6kM/2IFpR9mMfmrhzvcepsXJQKM8dbvOshQiF2ZhPneI1ApM9xwFDDi109g9U/Pd70WXexMEty7gF8ptQ7tGmYUF87AHhtHYv9hFM+/EH3ZgM+AfMf7rtzchhuLkHrDbPSqjA7VNjODiwGChXAFTZDxgIyHxGumQFb013KgQYYKq2IN5L5+Mbqq9RjZB07BviJZXQ7oofJkDuwydM6Hkdz4q8W+Rv7hi/AvhPejV71w400P2lXhG64FYKTN+gtUfQv9L89h6Xef69gUUuxcEty7gF/KdzXhyIGHSnYJWF0GDBMINoY07aAzIMkmTLz7CMjpvGOz1bBJ6e+ymLzzKIwxE/6Si8L/uwS96oM93rBahD0N95kCVNxAsOqh9IMsgkttVuAw4D5X2NB3pPJ0Dsa03TSMUrveX1hbhln6YRaJm2b62hTU1XUmRV+rAGM2Vn9BE6Nn5/yWir4RKYSzVN2UUOFEGxlG5NUceNC+B2W2WD+3hZwbJsKKsd/VLT4j+erp+niztd/B+K3zWPmTc0DA4ThxdYyZNQM+kHv44qa3k5d/sALnunHQmBkOo9Sq94cvNk0Wln6QhXUoDvvyJBgcTqhSuNIl/JFS04Sq9nU4F2l2UW1rhi4FkZU/KQK7azeixkxYh+LQRR/eqZJU4iNAgnsXsFLjKC933oLdqOVp7UTwSwXYYxMDuLPumXMxkGPAP1cCu2FyxI6lOu52rIkM94iKkyyF5KunkT1+Gum3HIAxFb5A+Usucv/rwkB6gLCrkfm9k3CuS4d9tXM+Sn8bUb1rYPWPz8OYsWHOhcMtuhiAFCHIeRh/xyEY0zZQ3b3pnSzCPpLo/PzMqDyZg/tsHqlfmmuq6FkzgowLnQ1//snXzyL+s+Ph2nUAXNFY+eKZcFhJ7FgS3LuAMkyoWBy60ksHu1YVOm3p1nZj0sL4rxwEJcywmlRA/tFLKP9NFlyKnkDtdmldy+ecjiG45CJz30moVLgyQxciO7j0z2OUH1tB+bGVjpcGl9zIIZnsA6dg7o9BjVvwFyrQWQ/pW+dhH02s7e7Utc1CjNphRuUfrSL/f8LZVHN+BfHrG4K5pLFS3ZhjvzgF5/rxps1CbCmkbzuIzOdf2Pz3QAyNBPcuYSfHUe4huM1kGn5UzxJC0wnt/Wg6KKHDuu3xdx6CGjOblsWlbp6FMW6h/JNVWJfFN250aRHaTcvtdPiyFDUJqVfWQlLnBxzYA+ZfrAAX18aiV796DolXTYWbhUyCe6KA4neXYR2Mg2wF90Sh6UWg8M1FlL6XgXXQgS4ETX1cEq+a2rh2XRGMMRPGtI1gqbsdtmLrSXDvEvbYBCqrS2DfR1eDlMxwpg+gvHQetV0cRITEXP/NpOqBDcZTwfs7LgM05x2QoyLbnsZfPlFN3u52E7JmBFkX3rkyUNEoP74K6/JEOMbdOFTgaRS+vdT13+n57AloHtxKG0UmLp+4ov8H0EDxO8sofme56cPBYuuQ1TkflSfXrd1XgLmvRW91QtP3TOw8Ety7BCkDqfkrUMlegrvaOZj8Yg6J2XnYyTT8chFECsYmepNEHZSQ/Wz7k22UYwCt1iETtd8ViGoTKB1OMrLLWPnS2aaueP5CBdCMxKumQTEFnfdR+PPFrtcw10L7kx86grtuuKOrr2nnsz98EB+99ySeyzy9+QDfJOtgvGlytomi8Ig1sWNJcO8iyjDhTM52FdwgQAcBDMuGldhcn+3IHZJdrNv2zpaaVk30gj1G4ZFFwCDonBcuyYsYEi99P4vS97M9bf9uDGwAuOuGO2C5t/R1n43uvu4WLB9/qB7gz2dPbF94KwonYiMSwF9scy6m2BEkuHcbUmEV22ldN9FAl/z1c7oNVzQKf3EJyZ+f6anS166Gd6aE8t91nvhb+6KNH2o3DFKrsi33FmCAQ72WewvuugH45IfWqu/1tqIa986UIt/RaFej9L3wfRLFFRKvnkLsqjGwp1H6QbaryVYxfBLcuwwRwR6bgptbbh3eRHAm9++I7n+l72ag4gbir5is9rdqsfUca0vZSj/I9hbaEToNg9wFDKTKjmK5t9Sr7/Uaq3EAwwvwgJH72gWk/+GBsPpW4ce4HMA6FEew6iH9lgNhj5bqzsvU62dhzjvI/9nF4dyT6JoE9y7kTO0PjyLLZ+ofU6ZdHRqxEJuY3fTwyCAV/vwSKk/mMfaWufBEmBbtToOMi8znT276+Z7PnsDvfOgQAuaBDYP0I+p5a4H++ceO4yP3nhnq8+tCUO89ThxuhzfSFpzrx+H8TBpgAjVs9iFbwblmDMXvLNdP3xHbQ4J7F6JqW1Znah84CECmWd1duTPZL04h8crJtqHtX6gM/GCA7QztbaeA8XccDCeI16mddB+FA4Z1IIaKBPe2kuAeYcwaXmEVXiEHZRiw01MwbKf+eVIGSG3uZPZBo7hC/GcnEbs6BZU2AQ6bH7U7mxEAsl88DbiyF3tQrCOJ6GPZGkQOWREhyMl5lttNgntEMWsUzr+AwC3Xx7LdfBbO9AHExiZbfp1fKqCcWYD2XRi2A2dyH4xY/6el9LLZxjzoYPxXDoGsFs2P2jAmbQQXN98UqTa2HXTTEnYbBczQ7A9t5UmnF8ootbM5/bOyVHC77dz3z6Kt2oEJTROQzCgvnQe36LXt5ldQuHgSQaUIDnz4pTzy55+H39NW+dCFk9melwGm3zoPVT3Yt2eb7CHyfPbEhglJVf5lVHwNN9Bte3sHmuFrbnmNrxkF10eu4qPit3+sboQrT+7AJz90BAH7eC7zdH2yclC8U8W2hzgA4aqfYMUDexrsa3hnS8h+cbjj7qI7UnGPqNb9twluYRWWkwCZa2PGzBw2olr/NdWPpw5c3vVzR222QQltQ9uYsUGx/uuEiV+7DJWf5FD4f5eaOtt1Y31g//PKL6OYD1BpaGtLAMZiJsyGMNPMyFWCenVOABKWQsxcG34q+wGK3tr9eDpA2QfSsU10NcTaypPMkNZ9c0Uj/8hieARb9YT42nFu7IUtA1a+dAb+hQpU2gR7DC7t7PYAe4kE94hquS2dNcqXzqJMBGVYSOy/LBz3Zg2O6L8NAEGl97e+Pa/b5o4bITd+ScOhARQz4Lw0DfOAg+zv9X4ow6c+fBR3XX87LPcWuFqjEjS/gDGAXMVH3AzfEZgKyLlB/ajJ2jUFT8NQBFMpMHNTaNcEDFR8Dcfa/PxC47pvAPjovYPbdVl+bAXe2RKcl41DOQbck0Ug0OAKw31hbUOTXpUx7Z1GhkpGVCw91XK7OACAGdp3kT//PFgHaxtzItAAjirrJFhyoYt+T8MIRM1j4WQqGJM2rMObO8G84rfoOgig6GsUvAArlebQblSufr3f6gIAlWBwWw/D6vt4ffikNvY9CMGii8I3F5F76AIqP15F5ad5uCeid6GKnUMq7hFlxlOIjc+gkr2Etk2ltMbqqadhp6dgpSbh5TPNwyVEiE3MDP1+AWDlj89j4h8dAgyK7JXdDTIAczYWNvzv02anJb2Akav48NoEd8BA0QuQGEDVXTPM6rvjc1+eQOp1szCmbOi8h8JfLKHyRG7ozyuiScU9wpzJfbDTrVeQ1LGGu7IE7ZZhJSfCypsIIIXYxCzs1NYcmhAsVLD0X0+EJ8H0mZ4chBtxNsPuMCnX8R6AtqFdU/Y1vAFW3sBwq++Wz3lFAuO3HYS5LwYyCcaEjbE37Ydz/fhQn1e0JhX3iPPLxS6vZARuCc7UfsSn58CBvz0bczxG5Sc5JH9hJuzD3UPVzczQxQDu893+naNZBoH8rTmhq+gFcBiwjeZ3GJoZbsAAGJZSMHp8Mdmy6puA9FsPbGgGRrZC8rUz0rtkm0jFPeJUL+PTDARuCaQUlGVv625Kf9HtfaiEgcpzhf6rdTaQK3tYqQRbdqxiwEDBC5At+/UlhfmKh2zZR9ELV6SsVHyUvN5XbGxF9W1f1fr4OIqpTa0UEv2T7/qIi41Pt5+kbEQEZdrDvaEuVX662vN6Z1KE+DX99ViZsI8iv/p6eNu074YB5CvhWu+o1YwlX7ed7Gyntu77Ux8+uql7jGIfadOjXXPPSzPFYEhwjzgznoIzNVcfs26HlLHpY8kGpfJsvvNFEcjuvcrLvv5WvGn+36L3BYmDpdF+H5Hr77x10kHOC4+CW4eZAYMw86+uROqX94PiEiVbSb7bu0AsPYX04auRmD3Y9jprbDxcGrgTVBi6n54XJmHqzqOwrx1D6h/sR/q2eTgvS4crVSJceNmNyLz1A9Wqcfvb2LazEzfhV57IAev+ydTX1ysKOwZeO4aJXz2807+9u4oE9y4RjlvH2g6buCtLyJ1+pq8NN8OQ/78L4S69HhARKG4g/eY5OC9NI3ZlCqk37sP0P7sciddOwTwUBxoOCH7h5jcDtg2znEH6zHdgX3wSqG1EKmbD/3aIWB/9Q4ZN53ysfOUsdMGHruj6z6tpfb2hoJIG7CuT23Wbe07bmS0iSgOYZebn1n38Zcz8+FDvTPSsNuHI3KKq5nAVQ3HxDMYOXbml9xbFfaaAlT88i+TrZmAecLqerFx/HZkKZCokXjWNxKsAaECXAyBgvNL+Js49/Jc4+6NnEACwqv+xMlGrcfXUEVTe8lHwzNHw8dBc/SYtBdtQ8DTDDzRczS035/TLUoAigq8ZBrU+yX47eKdKWPovJ2DM2HBeNh4e5LwOWQrmTAzuM92d5yk2p+VLPBH9CoAnAXyFiJ4gohsbPn3/sG9M9I6IEJ891HGyUnsudIvt71vNO11C9vdPw79Q2XRzptpOSzIIRtKEkbZQ+PFf49yPn9l4rfZBOgDpAOrSCcTv+yew/vq/I5n8NiYcE2O2gZRtYNIxETON8GQhQyFhD2cFLREhW/axWvGRKftYLnnIVXwEg36F6BeHuyz9c+XICUn2GP7yAM94E221e2/2MQAvZ+brAbwXwO8T0a3Vz+2cckA0sRIppA6+CEZ8tN62rnzl7FAGeZ/7u+c7H79Z/c9+9H/C+Zf/FmPvvQmfffJXYRvRnQytTW7gieIGG2/S04zVig+9nS1oFRC7OoXkG2YRf8UE3NNFsKvBDS8oXD3yzH2mvwln0bt25YPBzOcBgJm/R0SvB/A1IjqEnTmPImqYEbTZmENKwc1lYY9N9LYOfIi4EKD8xCqcq8darhvuh1fpoQokQmnCwI9Tt+Hj7t9D4T2vQfb4xzackhO3DLiB39UvgUmb60jLAMpeMLRKvx2yCRN3HIaasKBsBe1pJH5+pv6GrvYOSa96yD54WvqbbKF2vyE5InpR7Q/VEH8dgLcCuG7I9yU2wc2t60eyDusAlewC8meeReBt7u3tM/F3YeLdhzf1GDX5byyg/NNcUzW3WROzvW3LJgJeZH4Vp/70o1i4PIVjf/hH+MwTtzddo4gw7phwTAWDwl2RCZPQ2JXEICAdMwaSZX6XFXfj4QuDEH/lFIzJMLQBQFkKygrnE0hRfWhKJU0YUztjf8Be0S64/ykARUTX1j7AzDkAvwTgA8O+MdG/Vu1bmy9isA5QXjrf8+PPHZnANebn8fjvfQxfPn5mcOEdMPJfv4jVP7sAPaCNHde9+pqeB/ZqrVxMquAnjz2Hf7N4K1LveQ08e+1UdkWEhGVg3LGQsk04lonxuIWpuIVJx8S4Y4FAA5nEVF1MVDYevjCo8Hau7fLdj0mI9bkxSvSn5U+Fmf+OmZ8B8CUi+k0KxQF8GsA/27I7FD0zE+mud1P6pf7GJeeOTEAZCseML9TDe/GuDw8kwN2f5hBkXeiGpYKsORxLbXMSTZTxmTR+4e2vwfhMuq97IQLOPfrf8H8O/TZ+/i+/taH6jv6awY6Bx83uOgzWwnv5+MfwiQ8e3PTJOT2985HB0y1FnX4JiCgJ4B4ALwcwBuBBAPcwc8uS6NiBSf7c+94wyPsUPWBmFM4/j6CrI8kI6aNXb6pvSdQRZlG6OnChxiIkbpxE7Lo0EDBKj63AP1+COR9H4lVTUDEVntyyLiSZm1+zGv99r2Yr+OG3n0X+0iLgdz9EpEEo8X6Ageve+VvI3XMnMhFj3+sxM7Ll1mPh65cdRn0+ZRuwjN5/Np79ED7/2HF8+FMv9N18Kn7jBBKvnYFqqLqjDhBmVyP7pTNyFuUAzD389R8w8ys6XddNcNsAPgHgjQBSAP4NM0f/ZlZJcG+/wKsgf+bZ7i5WBsx4ElZiDFYy3VeIN4Z3lG7OpOyaAuwrU9UOgwbIVGAY8Dzg7IlLOPyiCRgmIQgY2aUysssVLC2W4VbW1rfrwEfu1FNdP6UzfQBLlwAw4yn/vfj1tz2CO+Kncfd1x9t+nRdo5NzodfVxU6HU4lCHGlMRxmyj5yresx9C2jHx6e8+gI/ce6a/roEKSN86D+uyBIiq73o0oCwC62qHBQaKP8ig+OhS748vNug2uLuZqv4+gD8BcCOAaQCfJaK3M/PbN3mPYpiYw9+s1m+M1ugAfmEVfjGHSvYSUvNXtD4arYW5I+GmDHXy/oiH1/jy8ffh7be/C7gLmw9wDbhP55H4uSloBjLJq/E33y+jfOJh+B7j7OliGDRtahJlmLDGpuDllrt6yvLSeczsO4TlJcYxug8PfQ34xDt/C7n3vKZt9W0ZCkmLUVi3Q7S2qccNNCJWAtb5mlHaplUl0MDql8/BnIvBPOBA5324zxVAjoHY1SkYaROVp/Lwz1e2/t72uG7+Nbyfmf+m+v8XALyViH5tiPckBkBZMRBRxzXMTZihvQoqq0twJmb7et5agG+4n5P340d/UBtOeReuenfbN21deSb+Lnz5+BkcM74AVR1OqD1/u793JZdBJbsI9j10TPgGpcUzmN5/BFZiAudOLOOP/uc5/Mbdr8XtP3wQd92AluEdM436zksgXAdeq6DTMRN510e7nf/lgBHT3HPP7kHxL1TgX1gLZ+e6NBI3TQPMiL98EpWn8sh9/SLavgKJgeoY3A2h3fix3x/O7YhBYGa4+WzEBGWnUdWQm8v2HdytzB2ZwIWTWRzDF/Dl4+8DcNMAHnUttFu9YKxXWc2gvHx+Lax73NxSXDiF5NzR8IWiOgJidDGMEe683HgdEVVXjbS/j9WKj3HH7GqFyTDFrk4hedM0yF57RxZ7cQpcXREktsbO2H0hBqqSXURl5dKGULLTU3ALK2tNllpg34Wby8JKjQ90hUS74ZT+dR/azIxK5mLPYb3uQVDJLgLY/LFdfrX3yfoT5yOfFuG//4JkAAAWRElEQVQhx/EBnmHZj/irpppCGwj7lDjXjiH/fxewbQ3P9xgJ7l2GdRAZ2rXPpeaOonDhBXAQoF2VV1o6B79SRGJmfuD32G3QDlx17fpmBV44bHDM+O/4T199Lx542yP40e3/EZnj0cMlzOE4dS2gLQVo7n1HZbCdW9+rjGSLyGBAxQxob2f0wNntdl4fSbEp2nPRasdJUC7CsGMYu+zFMJxE+wdihpfPVh9vlyACqc1XrIYdx9yRCcxfMY1j1v146GsnMfabn4tc583MyFUClANG2JsRcHV/2+C7GZIZtiDnR6+jJ0AXJLS3igT3LkOm1XIoQFnhtmQigrKsbh4NfldrwUcDESE2Mdv9UW/RDwJncm38f/6KKSjTwBNf+vf49FdvxoOly5p2Wfqau96y3vZpsTP6dSsnuvEWiFoeZiEGT4ZKdhllmDATY/CLOTQNhdRCq8qMJeDlVzqM9zIqK5dQyS7AcJIgIniFVbAOYDpJOFNzMOzY0P4uw2CnpwCE8wC9Dpso20F85gAM22n6eH3o58T9OJX6bRD+ov65fs+RbGQqQtIyep6YXC37Tf1LBnEC/Prx7TpmkK3AO/D4td1o+1/CxcAlZg/CSo0DCJtukGEiPnMQZsPwiJUc7zxswAztlqE9F14uA3d1OeyDwgy/lEf+3Alo3xvuX2bAiAix8WmMXfbinitv00nCjHUYYlpns0v4wmZVZs+PY7m3DKV/iXuyGLkVXhcDcFFCe6tIxb0LkVJIzB4ETx8A6wBkmBtPjVEKqfkrUFq+UK3OsVZ9d7u2mTUqK0uIT89V/xiAmXdMq9h2ai9AvfBLOQBzPX2NpajLRZgbhVveN/e9DMMbWD4OfPaHD+Kj9z7d9xZ4ACh8awn25UnApHDHanWWNf8NWQq4lXb+b5joGynVdgekMi0k911W/zNXN+Bo30dx4XRXuy6DSgk68FFaPAu/VADAIMOEnZqAnZ6CMrsZS+9f4FbAOoBhOy3/rtr3UM4swC/lQUrBTk9X35H0hvp4QSIipGMmCm5QH+uuDQW3WwWYsBRiLQ5y6FVtpctdNwCf+vBxfOTeM30/ls56yHzhJOI3TsA6FEew7KH4vQyCBdk9uZUkuEUdEYUBaHRfjZJpo3D+BWhv7ReXAz8cG1+5BGf6AGLVceVB0r6HwsWTTatonKk52Kk0ypmFcPwegJlIwSvk6i9CHADl5Qu9H5hMhNj4TF/3aihC2jHrJ9koIpS8oG2fEj9gOObOnOzTOR+FRy5t923saRLcYoNwgjMFv5hHpzf5fqH9Kenl5Qsw40kYVvMkpvY9lJbOrw3TACArhsTMfNNYfBRmRuHCyYYXi/Aey0vnUM4sAHptWVotwNc9ALx8b6e7xyb3wUpsrud0bXKRmdFpyNrVjILrQxGBCLANte27JsXOIZOTI+7CySzOnVjGuRPdNUvqVmL2IMx4EuEEp+p/CR1zU3iy1ihlFpA7/XRTaAMAe5Wu2tFqtwzdqi2r7nItcS9/H6UQS093f30bzIy8G2xoOhWlEjBKvkbR08iWfVRkxYaokop7xAROGjo2BjO3gPPPXqy3GT1m3odzJ5agDGMgOxNJGUjOHYH2vXCXJRHy557ra7t4bcOGW1hBaeEsOlXx5ewikvtbH8igdTCAxv0MKzkOrxBRkW+4lKF9D4a1+eO53IDrzaZ6VfB0ywOMh8352XEkbpwCxRS8U0UUHr2EIDNaK4p2EwnuEaFNB5mf+8eozF4J+D5AhIuPnMTNVxMO33MnXnr3Tfj0V2/GMb4P504sY/6KwYwrK9MCTKu+zbtnRLCSaXjFfFehDQCB23782bTjXT1OO4btID57EGTZcLOL7S9mQA1gxyUAlDdZNXuaI5tVDVPyDbNwXjZeP3vSvioF60gCmS+chM7JbsntIME9AOEhAkN+G/v2DwIzVwKGFf4HYP5Nl+MzF38Xt93/HXzLfghv/MzHMfabn8MTf/DvB1p9AwAZFkgZHc6z3LjwzYynQAAKC6c2fK4VZZiRJ63UtBwm6ZIZTyExewhEhPjkPjjpaZQyF6sbktYPYRDMxBjIaB3ctZ//U957cUv+ftzxoSO4K+I6Zt5059OtbldCcQPx68dBDbs2SRFgEuI3Tsgk5TaR4N6kcyeW68MVv/G2Px/Kc5TNFB45ei20al5aFygbbzz4QQDhkq/8/bfgd564HQ+++Qj+01dvxtUDqL4ruQwqmQVw4LfdsGMlx2HEHJQzi03h55fyyK8by+4kcMvInXoKif2XwXSSAMITayorl+AXcpsKbmXZSM4dafoYGUbYTGtmHuVsuFO09iIUhnx0o63aqT9gxnXv/C0cvudO3BE/Ep6KE3GL/Q6RNLK2uNo2Z2ywz6B1SUGmgnUwvqX3ItZIcPepXmVXzyE8fM+dePTVHxvKcwVBCjrfovpcV4Ldfd1xfMt+CG8aQPW9vnc163CsW8USYM8Fax8gBcOy4RVXo8eL+ykRmcEcoHDhFNKHXwwAyJ87AfZ9bHaIpNNabGdiBrH0FLRXAZlWy81EjVX2r7/tEdyhP4G7jn8Md3c4h3IzktbWrywJVjxQxIsFBxr+0i5qQDZidk1wb8lwRSMGnvTfi9+o/tLeff93YA3p37HJjCKihyisiHVlravvjecCtgv0SnZhY/AyA75XD1SvsILi4tkhvYfnam8UXR2iGUCzpi420ZBSMGKdq8mngvfjN972Tfxa4gzuuv6OjocHR/2sOt4LAMcMjzkbxAk4H7j+dnzig93voNSrPtxTRdiHE6CGQ4OhgdL3Mpu+H9GfXRHcteGKWpBulcOf+Ti+9erhVllAuDEmYSoU123YIKBtY/3G6vuld288cabdZCYztxzPbhyqqKwuD2/gtdo/2y8VBvYcvI29VYgICUuh2MVSwBoGBhbajTsoP/mhB/HRe09uaD5l7osh/nOTMKdj8M6XUPxuBqt/ch6pN+6Dc80YAILOe8g9vIDgklTc22Uowe27AS6c7G2DQ7/WD1d8a0jDFVGsV9wztCp7PccyYChCydfQzLAUId5Fx7ha9f2tapvRRu0mM6nanCoqvJXZsCxumLNlRDCdZE89wePTBxG4JbgtDgFWVn/dDNf/ew7HtsMA7uWAA8c0UPY0uo9uIOzkPbghklb9S178My/B+G3zgEEgRTCmbcSuSSP7+6eQ/7OLyH9jAWQRuNzL3YthGEpwl3gGP3XfM4yHbsYazNiS4YqdwDIULKO/PVNRb+M7TWbGJvehvHS+OZyJ4EztX3vc5Hi4fK9FeBmxBGLj0ygtX2hb7apYHLrxcarLCI1YHDZReIZmh4BUtgM7PQFgAoFXRlAuYUNr2/HeNtKEG5sYTwXvb/6E1rj1jkN4/J5v4642p7wD4bsXTzMq1XdMlkFdHVdWM4wDFKKqb+cXJ5uGQ8ggQAHJ189i9Q/PAgGD5UDgHYEiT7PYpBuumOFHP/7mgT9ulB995i+Q6fCLI9rz7Icweft/rFffoLWx73BVySI48ECmBWdyP+yGBk2sNfLnnw+3nzeEbmxiFnZyvH54A7PG6sknW4bv2JFr4BdzcHOZ8GDdsQmYiXR9SaBXzKG0eLZ1D20iOJP768HMOkBx8Vx9d2bY2nYeViLV1fdk/eTz0cIDG665I346XEGyDjPDDRhuoEEUTiCvb0tiUHh8GQNQBDgGoRww1i88SVgKjjnccyY9+yEYOsAvnv5x5IuEdjWW/vOzQ70HEZp7+Os/YOZXdLpuKMF91UuP8n/+k98e+ONGkcAenM88cTseLF0WVt/mfeEKki4q/P2Hx+EVVuGXciDDgj02GbnLsLKaQXnp3IaPJ/YdhpXs3Aek1r0w8FyUFqubeZgBUjBjcSTmjmxY+81ag1mDlNH1jsPGJZ6//rZHcEf8NO664Y4N17U6XzLnBl0doJCOGTAbOhoyM8q+hqfDXiaOqZo+P0zEGq9f+BSMiDwIVjws/7fnt+Q+9rptDe4Xv+Rn+He//L8H/riiNTfQKHoBNIfVXNwyYPcxrNJYff/Rg2c6Dq0eM+5rqtA78Ut5lLOL0J4Lw47DmZ7rayu5Dnx4hRVo34cVT9ZP6NmM9VV27p47e343V/E1Cl53q5tsgzbdb3uQrl75BuZLj8NoWMEUuD6Kjy6j/LdbM2e113Ub3DvnX43oW8VvbloUMJB3A6Rs9BzejUsJP3LmZMcpsZfUttqju80+ZjyFVLy74Yp2lGEOrPETEFFl9zln4unuJ+4GcazZID2VfgNMLmNf+SloYrD2sfTXp3D6z58GGAM5+kwMhlTcu0Cm5EWucFYETDj9H2TgRaxEiRI1Pt7JoLbit9PtyqZ+qmzNDM0MRdS0sqfg+j1NPE7Fh3vQRD/soAAnWEHRnETJeaS68uTkpk7OEd2RoZI9gpmRKbfuHzIVt8DM8LVG0WMEzCCg3g/aVATbUGCEqxf6XS9cGx//9Fdv7mrl2jHzvoH2UlmvvrPRf2/nixn1seyoycamS5lR9IKmcLZUeMQYEcHXjNVKd42XCMDkDgzu9Tz7oabwBqT6HhYJ7j2CmZEt+9EVN4CYqdqetLKeSYRULHp9uGZGoBmGosjP18bHu7FWoYcToIMK8PX9Q3L33NnV13U7lt3q5BpTAelYGMIl10epi6rbMRUSbTZQ7TTrA1zCe/AkuPeQVmFiEeD18eO1FGEstjb9wcwoeAHcgOv9/2IGIWF1v1KjUS0AahX6Meu+roZXOtGBBsBr/UOqq0EGufKo1bAUAIzZBhjh/EIjS4XVtdvwI9rM9287NYY3AAnwAZPg3kNqy8jKvq7vsXNMQtnnvrt7TDprJ8PnK15T6NTETdV2y30nTStYjvd/gG2jW28/1NdqEGDt++gGGgDBMRUshfBjmsHcvltKuxfKlG3AVATN4TDVqB9DJtX3cMiqkj2EKNz+7jT0TGYAJb//JvcBM0wilDw/MrSBMNAag7vTUMp6lnsLMseB3/nhJ/CR0yf7vtdG/1ofabkapFakRFW5zIxcZe0kdiB8l7Gxw3hr7RYBVnwNO2Z2PGtyVNS2zdd2XT6X6a5plRgMqbh3qXZj391QFFbUnc5GHLMNlHyNQDdX97ZBSO6QoQBfMwpuUO8pEnVvbqA3DHH0ylLUsuf2+uGn3USq78HptuKWw4J3KSJqqsB7pRkodQhtAuq7BNfHlVs96Ha7aWbkKn5TIyg3CF/UMiUPyyUPq5XNH8RLCLentxLbxM9ipwur7zvwyQ8dgWYfz2WexvPZE9t9W7va7v3XJBC3DCQsVX97bhCQshSm4hZiXWzM6VStd/p8pY/gZub6f4NQG/ff8DxYu39fM3rotFpH1f9sgzDumDCUQsreOOZvG9RXL+5RYrm34O7rjmP5+MfqAS7hPTwS3LucYxqYcCxMxS2MOxbsasOibjZU9tmIsK6X6GVmlL0grITLPrJlf1MH69bCPxji7kTbIEzGLaRssz6mbxsKE46JhKUQNxXSMbO+xnsvaKy+A6m+h0aCe4/qtBXeVISktbkxWbOHKrPiaxQbqmMGUPQ0MiUPuYrf9fZwZkbBDcM/U+7+6/rR6k2BIoJjGohbRk/fg92iVn1npPoeGgnuPUoRIdliTNYxFcbs8OCGRJux2U6R1O3mEs284XSfGkZ4yO5qxYcXdB7PyLvNuxqH2Q1kqw/uHTVSfQ/P7pzmFl2JmWEHwdpKCEvRhrf0dsSRaTWOGX5tY1VLCAMtbobBzxweIFAONDSHVXjCCtuV+pqRd/0NPahbKXoa423eKQSaW67q6GVZXzfCsW2pezoJq+9bkDm+tvKkVXjLapTuSXDvcUQEu03lqKqrU8rrwtuo9ouOE1WXAjIM2hj8perGoJqwl0eAMTtckdKLTkeErT/xvvnvEb4D6PU5gXDJY61PNgDYipCwd8ZSx1HRuO4b2Hiqz4c/9cKG8y9FaxLcoqNEday2Uj3v0jYUHFPVgytsTBW9qWV94Nd027N6vVzFR8xUke8O2jXIMhXBVNRX5W3s4jXYW6lWfUd1nfzkhx7ER2QjT9fkX6Poim2onocG2g2B9Dtn6GmG5waRm2hU9d2DG9HgyTHDa1O2gbwbdB3ettHdLlDRvahWBFHDKRLerckgnRiadnnXbrHFmG0g1mHizw04cugkWd36X/tqUxHSMbNejVvV5XpJy+g4uWqp8MVBbA2ZzOyeBLcYGtVm/DxpGZHhbSqCZSgkbROTjgmnTZXvRVTWRGHXvcl4uHY9HTM3LMkjIsTMcJ11FFsRJhwTY7G9s/56p5ClhN2RoRIxVEnLgEKAcjVka5OElqGQVoRSrV0sAbHq2HkNEUEptOzetNlMjZkKGmgah99JPVb2svWTmR+9V8a+G0nFLYaKiJCoVs+TjonxmFkfK1dEYWUdtzDhWIhHBGa7cfXNLserV+eOiXTMxISzt3Y57nS16ruxD4pU3yGpuMWW6DcMFRHGIiYUU3b0KT393pspWb1jSfW9kVTcYserTSiO2QbG7LBCls0ve4tU382k4hYjgYhki7mQ6rtKyhYhxEiR6lsqbiHEiKpV38vHUd24s3eqb6m4hRAjy3Jv2XACz16ovqXiFkKMvL1WfUtwCyF2hVoPlPWnz6+3GwJdglsIsavUuhAuH9/YhXC3NLGS4BZC7EpRXQh3y3CKTE4KIfaM3TKZKRW3EGLPGfXJTKm4hRB7UqvqexQqcAluIcSeVtuJuXz8Y/jEBw+OxPCJBLcQQgAjVX1LcAshRNWoVN8S3EIIsc5Or74luIUQIsJOrr5lOaAQQrTR2AO8to1e0cbo3MqlhBLcQgjRQeM2+s/+8EEY647N+/CnXqhX41sR4BLcQgjRpVr1vd7aRp6t6YMiwS2EED2I6oECNHclHHb1LZOTQggxAFs5mSnBLYQQA7R+KeFzmacHHuAS3EIIMWCN1fcwuhBKcAshxJAMq/qW4BZCiCEaRvUtwS2EEFtgkNW3BLcQQmyRQVXfEtxCCLHFGqvvoI/qW4JbCCG2Qa36zjRU392S4BZCiG3UWH13S4JbCCG2Wa367pYEtxBCjBgJbiGEGDES3EIIMWIkuIUQYsRIcAshxIiR4BZCiBEjwS2EECNGglsIIUaMBLcQQowYCW4hhBgxEtxCCDFiJLiFEGLESHALIcSIkeAWQogRI8EthBAjRoJbCCFGjAS3EEKMGAluIYQYMRLcQggxYiS4hRBixEhwCyHEiJHgFkKIESPBLYQQI4aYefAPSrQI4OTAH1gIIXa3I8w82+mioQS3EEKI4ZGhEiGEGDES3EIIMWIkuMWeQ0QPE1GWiL623fciRD8kuMVedC+AX9vumxCiXxLcYtciohuJ6HEicogoSURPENFLmPmbAHLbfX9C9Mvc7hsQYliY+ftE9KcA/gOAOID/wcw/3ubbEmLTJLjFbvdxAN8HUAbwL7b5XoQYCBkqEbvdFIAUgDEAzjbfixADIcEtdrvPAfgtAA8CuGeb70WIgZChErFrEdG7AfjMfJyIDAB/SUQ3A/h3AK4GkCKiMwDez8zf2M57FaIXsuVdCCFGjAyVCCHEiJHgFkKIESPBLYQQI0aCWwghRowEtxBCjBgJbiGEGDES3EIIMWIkuIUQYsT8fzc2wi589OlKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import autograd.numpy as np  # Thinly wrapped version of numpy\n",
    "from autograd import grad\n",
    "\n",
    "# Initialize parameters randomly\n",
    "h  = 100 # size of hidden layer\n",
    "\n",
    "W1 = 0.01 * np.random.randn(n,h)\n",
    "b1 = np.zeros((1,h))\n",
    "W2 = 0.01 * np.random.randn(h,num_classes)\n",
    "b2 = np.zeros((1,num_classes))\n",
    "\n",
    "# Select hyperparameters\n",
    "iters      = 10000\n",
    "step_size  = 1e-0\n",
    "lambda_val = 1e-3 # regularization strength\n",
    "\n",
    "'''\n",
    "Do entire feed-forward computation and compute loss function\n",
    "'''\n",
    "def compute_loss(params):\n",
    "    W1, b1, W2, b2 = params\n",
    "    \n",
    "    # Compute scores\n",
    "    hidden = relu(np.dot(X, W1) + b1)\n",
    "    scores = np.dot(hidden, W2) + b2\n",
    "    \n",
    "    # Compute probabilities\n",
    "    exp_scores = np.exp(scores)\n",
    "    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)\n",
    "\n",
    "    # Compute cross-entropy loss    \n",
    "    logprob_correct_class = -np.log(probs[range(m),y])\n",
    "    data_loss = np.sum(logprob_correct_class)/m    # cross-entropy\n",
    "    \n",
    "    # Compute regularization loss\n",
    "    reg_loss = 0.5 * lambda_val * (np.sum(W1*W1) + np.sum(W2*W2))\n",
    "    \n",
    "    return data_loss + reg_loss\n",
    "\n",
    "\n",
    "# This is the gradient of the entire feedforward training\n",
    "gradient = grad(compute_loss)\n",
    "\n",
    "# Gradient descent loop\n",
    "for i in range(iters):\n",
    "  \n",
    "    # Print diagnostic\n",
    "    loss = compute_loss((W1, b1, W2, b2))    \n",
    "  \n",
    "    if i % 1000 == 0:\n",
    "        print (\"iteration %d: loss %f\" % (i, loss))\n",
    "        \n",
    "    dW1, db1, dW2, db2 = gradient((W1, b1, W2, b2))\n",
    "    \n",
    "    # perform a parameter update\n",
    "    W1 += -step_size * dW1\n",
    "    b1 += -step_size * db1\n",
    "    W2 += -step_size * dW2\n",
    "    b2 += -step_size * db2\n",
    "\n",
    "def predict(X):\n",
    "    hidden = relu(np.dot(X, W1) + b1)\n",
    "    scores = np.dot(hidden, W2) + b2\n",
    "    pred = np.argmax(scores, axis=1)\n",
    "    return pred\n",
    "\n",
    "plot_model(predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Train a 2-Layer Neural Network  Using Backprop\n",
    "\n",
    "Now, we will do the same thing but code the backpropagation ourselves. First, we \n",
    "\n",
    "### Derivative of Log-Loss with Respect to Scores\n",
    "\n",
    "It is possible to show using calculations similar to the derivations of the gradient of log-loss that:\n",
    "\n",
    "$$\n",
    "\\frac{dL}{ds_k} = \n",
    "\\begin{cases}\n",
    "p_k     & \\text{if } y \\neq k \\\\\n",
    "p_k - 1 & \\text{if } y = k\n",
    "\\end{cases}\n",
    "$$\n",
    "\n",
    "Computing these derivatives is the first step of backpropagation.\n",
    "\n",
    "### Backprop through One Layer Using Matrix Multiplication\n",
    "\n",
    "Our feed-forward computation is structured very efficiently through matrix multiplication. For example, one layer without any non-linearity looks like:\n",
    "\n",
    "``` .python\n",
    "S = np.dot(X, W) + b \n",
    "```\n",
    "\n",
    "Here\n",
    "\n",
    "* $X \\in \\mathbb{R}^{m \\times n}$ is a data matrix (with feature vectors in rows),\n",
    "* $W \\in \\mathbb{R}^{n \\times c}$ is a matrix of weight vectors (one in each column), \n",
    "* $b \\in \\mathbb{R}^{1 \\times c}$ is a row vector containing the biases/intercepts for each class. \n",
    "\n",
    "We have imagined that this is the first layer, so the inputs are the original inputs $X$. However, the same math holds true if we replace $X$ by a matrix $H$ where each row contains the outputs of the previous layer for a single input feature vector. \n",
    "Observe that we are simultaneously conducting feedforward for the entire batch of inputs represented by the rows of $X$. This makes it extremely efficient. Also, note that broadcasting is happening here with $b$, so the proper way to write this algebraically is:\n",
    "\n",
    "$$\n",
    "\\newcommand{\\del}{\\partial}\n",
    "S = X W + \\mathbf{1}b\n",
    "$$\n",
    "\n",
    "Here, $\\mathbf{1} \\in \\mathbb{R}^m$ is a vector of ones, so that the product $\\mathbf{1}b$ is an $m \\times c$ matrix (just like $XW$) that contains a copy of $b$ in each row. Now, suppose we have a loss function $L$ that depends on $S$, and we have already computed the partial derivative $\\frac{\\del L}{\\del S}$, which has the same dimensions as $S$ ($m \\times c$). Then, the backpropagated partial derivatives with respect to $X$, $W$, and $b$ are as follows:\n",
    "\n",
    "$$\n",
    "\\begin{aligned}\n",
    "\\frac{\\del L}{\\del W} &= X^T \\frac{\\del L}{\\del S} \\\\\n",
    "\\frac{\\del L}{\\del X} &= \\frac{\\del L}{\\del S} W^T \\\\\n",
    "\\frac{\\del L}{\\del b} &= \\mathbf{1}^T \\frac{\\del L}{\\del S} \n",
    "\\end{aligned}\n",
    "$$\n",
    "\n",
    "These follow from standard identities for derivatives with respect to matrices and vectors. Note that the left-multiplication $\\mathbf{1}^T \\frac{\\del L}{\\del S}$ by the ones vector simply adds along columns of $\\frac{\\del L}{\\del S}$. Therefore, in code we would write:\n",
    "\n",
    "``` .python\n",
    "# dS = array holding dL/dS\n",
    "dW = np.dot(X.T,  dS)       # dL/dW\n",
    "dX = np.dot( dS, W.T)       # dL/dX\n",
    "db = np.sum( dS, axis=0 )   # dL/db\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 0: loss 1.098765\n",
      "iteration 1000: loss 0.292278\n",
      "iteration 2000: loss 0.256588\n",
      "iteration 3000: loss 0.248217\n",
      "iteration 4000: loss 0.246182\n",
      "iteration 5000: loss 0.245667\n",
      "iteration 6000: loss 0.245438\n",
      "iteration 7000: loss 0.245280\n",
      "iteration 8000: loss 0.245174\n",
      "iteration 9000: loss 0.245097\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD5CAYAAAAHtt/AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmUZGd55/nv+94l1oxcKmtfERIlEGAwyN1sNstAe1ww+EhgaMlmQPZIZ3p0ZuZ0Awa67T5NN0ODTnv+MD3dcDDI8qgabHAbDzLY3eAGjI2hMRgQICQVqlLtVZkZmbHf7Z0/bkRWRmbsS2ZG5vM5R+eoMm/cuLn94onnvosyxiCEEGJy6K2+ACGEEP2R4BZCiAkjwS2EEBNGglsIISaMBLcQQkwYCW4hhJgwEtxCCDFhJLiFEGLCSHALIcSEscdx0ul0whyYTo/j1GJCBV5IxcwzO58gGVwnWPC2+pKE2Ha+t7Jy3Rizt9txYwnuA9NpPnbPa8ZxajGhLp/N86PgN3jz209w89KD5B86t9WXJMS2c+CLXzjby3HSKhFCiAkjwS2EEBNGglsIISaMBLcQQkwYCW6xaU6qj/FHn3iCx1NvZeZtx7b6coSYWGMZVSLEegeOz3D5bJ6TfILvfxo+E97Dm+47wi2VT8kIEyH6JBW32DQHjs9w6KY5tKU5aX2Cz5w+L9W3EAOQ4Bab7sDxGbSlQduoueM8Mft2CW8h+iDBLYQQE0aCWwghJowEtxBCTBgJbiGEmDAS3EIIMWEkuIUQYsJIcAshxISR4BZCiAkjwS2EEBNGglsIISaMBLcQQkwYCW4hhJgwEtxCCDFhJLiFEGLCSHALIcSEkeAWQogJI8EthBATRoJbCCEmjAS3EEJMGAluIYSYMBLcQggxYSS4hRBiwkhwCyHEhJHgFkKICSPBLYQQE0aCWwghJowEtxBCTBgJbiGEmDAS3EIIMWEkuIUQYsJIcAshxISR4BZCiAkjwS2EEBNGglsIISaMBLcQQkwYCW4hhJgwEtxCCDFhJLiFEGLCSHALIcSEkeAWQogJI8EthBATRoJbCCEmjAS3EEJMGAluIYSYMPZWX4CYbCaKCGtlUAorkUYptdWXJMSOJ8EtBuYVl6lcv7j6b6UVmf3HsRKplscbY8BEoOSNnhDDkOAWAwm8KpVrFwCz+jETQvHyU+SOnkTp5nCurSxSW7qKiUJQGttkmx4rhOidlD6ib365QOnCk7QM3ijCKy03fcgr5KkuXo5DG8BE2Kywj78Z/8UKsQNJcIu+hL5H+erTHY+pLV6J2yJ11fw1MM0hr4AD6mtNxwkheiPBLfrirSxuCOH1TBQRlAs3/h36LY+zqEEUjPT6hNgNJLhFX9qF8LqjiHxv9V/acVse5ZMFLbdZhOiXBLcAIAo8qktXKF+7QG15Aa+YJ6hVNrQy7FQWug35UxrtJlb/mZw7sOExBrhgXivDB4UYgJQ7Ar9ciPvW9ZBeW1NrJ0Hm4Am0Ff+qOJlpasvXiQK/bctEW3Yc8HVOKktm/zGqi1cIfQ9tO1SCDEs8b2xfkxA7mQT3LmeMiYf1tQnhyK9Rvnqe7METACityR66idryAn5pZbWSjrwqKIWTzpHcc2BDJW2nsmQP3wjz0tn8eL4gIXYBCe5dLvSqXUd2hNUyJgxRlgWA0hbJ2X0kZ/dtxiUKIdaRHvcuF1fGXYbkKTAm2pTrEUJ0JxX3LqedBMqyMUH70SJKadDWJl7V5lJJjc7ahHkfgi0eV67AvTmD+8wMUSWi9v1lwsVeRvKI3USCexcyUUhtZYmgvILSNsnpeapLV+OqukXbxEQhtcXLpOYPjeT5L5/NE4URRAFm8exIzjkQSzH1i/tJ3JrFhAalFKW/WaD2aAGdsggWPAhHE+TWvItyNMGVKrR786Jg+lcOYx9Kol0LExrSL5qh8IUr1H5UaPMgsRtJcO8yJoooXjxD5Ps0WiRBtYg7PY/tJvHLBfzixhuHXjGPO70Hy0ls+FyvLtdvSEZhxGPhPbzpriPcUvkU+YfOsWm3KhW4t2Rxb8lg70tizzkoW6PqfwmZV8yTefkeTGAARenLV6l+b2Xgp7PmHHJ3HkZn7fhFMYLCI5fxnixtODbx7CnsQym0G3cwlaVWX1xqTxTBl1mmIibBvct4haV4KN/avrYxeMvXSR49SVAptn1sUCkNHNyNKrsR2HdUPgUVyD90bqDzDWS1oo3D0RizYfSL0gpQqHpnKPuafYRLPv7TlcGe7x8fRaet+nljuTsPUf7mIuWvLKz+GKw5h+QLZ1ZDey0TGZyjKfwz5f6vQexIEty7jF8utB36V12+1mGEiUJbw/W5HzP38uZ7TnDz0oObG9h1iVvXVbS9TP5xFJlfmCdc9kErao+u4D2xsVpu+dATaZSjmkK78bzpF8+hHYvy3ywy/aZDWLMuWLR+MXE103cehtBQ/cEKxS9f2/pevNhSEty7jLZswlafMAZveZG2I0wU2KmpMV7ZeDnHU2T/0b6WFW0nSinsg0nsQ0mUUrg3ZfAeL1L4/OWuj9Xp9i90ylIkn5/DOZbCmnVQVvvrWg1+rUg8N4eesln57MW2x4udT4J7l3Fzc/jllTZV97qP1Tc8UFqT2X9swxrb24KGxHNy8Q3GWkT1u8sb2hrJF82Q/YV5lNP5+ltVu8aYpopZu5rELVkqh5MEF6o4x1LYB5NEKwG1x4urlbBKW6ikhbLbP6eJDNbMxtBeex3rr0k7Gvd4Gj3tEC3Ho03s/QlS/2AOe4+Df6FK+W8XiZZl8a6dTIJ7l7ESSZTtYvxa12PT+4+htMJyU9tyTRH7YILptxxBuRqlFMYYEjdnKf3NApVvLMUHOapraJvQEK34RNUQez6xeqwJTeuZDo7CvTlD9pXzWPuSKEthAkP2NXvJP/w0zk3x50wIGIMxrdsy7a5p7bEtv+8qDmtv2ce9KU3ulw+BFbdkrD0JEs+ZIv/QORlGuINJcO8ytaVrmMDrfiBgJ7fvHpLWXpeZu442VbRKKXAVmVfMo1IW1b/LY0058VA/p/V5jDEU/+Iq1e8tk3nVPPa+RBzYCkwQgVIod933IDQ4+5LY+5Or4asshXEUU3ccwp5uHqnSyUDfX0uR/R/2snS2TPZ1+5teAJSlQGkyr9zLyh9LO2WnkuDeZbzCUtf1tIFtv/Fv5mV7wGp9fUor0i+aIf3CGYpfurbh5uBaxjdUf7hC4tlTJF8w09y2sHW848OGB4G1x9lQMSutsOdaL2Hb8jpbfH/btmvWVeE6aZH8uZl4mOH682qFc7T1vp9iZ5Dg3qGCSgmvmAcMTmYGO5WptxO6TV1XoBSp+YMjuY61wwAxHqZaHfhczok06Z+bRU/Z6JzT8YVFWRosyLxynmDJi1sga4I+3rgYCCLm7jkOltpw41JZChNGRJ6BqP5ipxWFRy6TffXe9s/d4oWiVSC3fGyL0G55nKNJHM+0fmFp0LSf7CMmmgT3DmGMIagU8Qr5eOGoNe0Qv7SCk5kmNX8IO5UhKLcfq22lMqT3Hl5dxnUY68du31L5FPmPfmqgyTbZX9xH8vnTTTftegpDBaWvLZB5+R7sPS6GeiD7EcpS6LQN6fYBaULIP/w0OhOPxfbOlcE3VPcnSN0+i15TdZso7pXrjN2yf924XhPW01RtHCq4/vj4sBYvBJFBZaz2g4AcxdTrD1L4s8sknzOFc1OGaMWn+l2ZQr8TSHDvEJXrF+JlVlsFkDH4pWXcqVmScwcpVp+EqHUpFnm1kYX2j4LfgChomiE5iMRtU02hDazejOxKKaJCQP73z8XD7tIWKmWRe8PBDf3xli8EQUR4rUZ4tfnD5W8s4t6UgVkH5WqMbyA0LH/2ItNvPYK1vo2iFCY0+NerBFdqVL6TZ/ZXj3W59A4vSgqs6fbvOpSlSdycwX7HcXTWjicchYbUC2ZY+dNLPY9FF9uTBPcOENQq7UO7wRjKV8/jTk2T3n+M8qWnWh824j0g33zPzcNNuHEU2dfsbT0qo0u1bYzBlEPCq/EImnDJhyWf9MvmUE7rx5owQlk6roojKHzhSuuq1jfkHzqHe1OmPhzQp/bjAsYzRIUAK7PxT8sEEcUvXiG4HF9PtOLHE2/60KkK33AsYOXs1Reo1Sn0v3SAhd99suuikGL7kuDeAYJKsacbjib0qeUXYGURLBvCjSGtttEekMpVzLztOCrZfcZmu7ZJ5e/zzN57AmvKJljwKP2360QrAcY3G0aLGD/Ce7yETlmEKz6Vb+cJr3cYgWPAe7K0Yd2R2k8KWHvcpjZK4/jg6o1hmJXv5Em/fH6gSUE9HWer1sdqsPYmVl/QxOTZPn+lYmBKaeK7VL2UUPGNNmVZLY82oU8U+Gi7zfi5HjSt/jfEzcjkC2fiinHQ0S2BIfPSPav9Zmd/kuk7DrH8uYsQmrhPXO8xm8hAAIUvXhl6Onn128skb5tGTdlxG6VRvX/xStPNwsq38zhHUrjPyGAw8Q1VFY90iX+kqumGahRE8b1Iu4dqOzJElbBl5a+0wng3LkRP2ThHUkTlAP9cRSrxCSDBvQM42Wmqi92nYK/Vdrd2pQgqJdypmb6vo9Pqf93YBxKopEVwsYLx4uRInMx2ne1447JbhFmLilM5msxL95A//TS5NxzEmotfoIIFj8L/d3kka4AYL2Lp98+SvC0Xr6tdCKj8XYvqPYKV/3wJa97FPhC3W6JyiNKKsOAz/eYjWHtcqM/e9M+WcY+nuz+/MdR+XMB7okj2Fw80VfQmMoRLHlE+/vlnXrWX1M9Ox2PXAVOLWP7U+bitJLYtCe4dQFs2OpEiqvWzgl27Cl0NNLV90NX/rFmH6V85jErbcTWpofiV61T/ex5TaX0DtdehdW2fc0+C8LrH0ifPorPxyIyo1HIFl8H5hup3l6l+d7nroeF1r2VLJv/QOez9CfS0Q3C1RpT3yd1xCPdE+sbszqgxWcjQ2Myo+v0Viv8lvptqH1om9YI1wVyJWK5PzHGflSX5gummyULG0eTuPMzSx58a/nsgxkaCe4dwM9NU+whuO5MjaLVmiaJph/ZetBz21+PNyOm3HEFP2U3D4rKv3os17VD94QrO0dTGiS5tQrtpuF0Uvyy1ugkZLd8Iyag44sAeseBKDa7c6EWv/MlF0i+ZiycL2QrvTInyNxZxDqdQrsY7U2p6ESh96RqVby7hHE4SlcKmdVzSL5nbOHZdK6wpG2uPS7jQ2wxbsfkkuHcId2qG2soCJgjoqUlpDMk9B6kuXKIxi0MpRfrAYItJDbJkq30oiUrqlsuepl40U0/e3mYTmsgQ5j38i1WoRVS/t4LzjHTc417bKvAjSl9b6PvrW+un+TNEprfRN1rZPGPmpqGer0kE5a8vUv76YtOHw2vtQzYqBNR+vG7svgZ7X5u11RVN3zOx/Uhw7xBKW2QP3UQtfx1vpXswBeUC6b2HcDM5gmoZpTTWJq9NopMWtBuHrFTnWYHUF4GK4puMxjMs/+GFplXxgqs1iAzpl+xBJTRRMaD0l9cGHsPcCOwPvus4973w7q7Hf/Q7D/PeB87y0/yZ0Yb3CDiHU003Z5toFW+xJrYtCe4dRFs2ydm9PQU3CqIwxHJcnPTg62yvHUHSL/9CpWnURD+Mbyh9+RpYiqjgx0PyWrTEK9/KU/lWfujp3+tD2/FOdX3MfS+ED77rYd7zwFmeXPrJ6KvvYWgV34htkQDBtQ77YoptQYJ7p1E6rmK7jetWaqghf7Cxt93vRBtTiyj91XUyPz/fV6UfeRH++QrVv+9+4+/Gg3o7rF0bpBHY93unoMfWr+Od4v7bTrF0+pHV6vvJpZ90fdxmBLx/vtLyHU3kRVS+GY8OUilN+qVzJG6ZwvgRlW/ne7rZKsZPgnuHUUrhTs3hFRbbh7dSJGf3D9wWWa2yMTwW/vpQU9or31hCpyxSL56tr2/VZuo5N4ayVb6d7y+0e9CtDXIf9FRlt+J4p7jvhbB4uvuxa9srwPgCPDQUPn+Z3P90MK6+dfwxUw1xjqQIV3xybzgYr9FSn3mZfdVe7ENJin92ZTzXJHomwb0DJef2x1uRFZdWP6Ztt94acUjM7B2qPQL1m5FvP8EdSw+S/+hwu7SX/vI6tR8XmXrDgXhHmDbLnYZLHksfPzvEM7U2SBukX72e8/7bTrF4+hE+/t3TvOeB8yO/jrWiUri69rgy8XR4K+eQfME0yZ/JgVGoNZN9lKtJPnuK8tcXV3ffEVtDgnsHUvVlWZNz+zBhiLLt+uzK7cl9Vpb0z812DO3gcm3kGwOsD+xhquqJo2H6zYfjG8TrNHa6b8WEBudggpoE95aS4J5gxkT4pRX8UgFtWbi5OSw3ufp5pS2UHm5n9lFTKU3qZ2dJ3JpF52ww8eJHnfZmBMh/6mnwRjcXezOq7O3MOZ5uvS3bGi1bVkoRFmQ/y60mwT2hjIkoXXqK0Kuu9rK9Yp7knoMkpmbbPi6olKguXSUKPCw3SXJ2H1Zic3ZLsQ8nmf6VIyinzeJHHVizLuGV4RdFWh/Y/dxs3EyhMUQmGNtQwm4vlK009uYMLshQwa22fd8/i44aGyY03YA0hurCJUybtba94jKlK2cJa2VMGBBUihQv/ZSgr6nyg8u98RC6vrFv34ZcQ+Sn+TMbQltXf4laEOGFUce1vcPIEESm7TFBZCh5AYVaQC3ofK5exDcz7+aD7zpOaAKeXPrJ6s3KUfHPlTtu4gDxqJ9w2cf4ESaI8C9UyH9qvH130RupuCdU+/W3FV5pBSeZRtk3esbGmHghqvWPqX88e/AZY71ea95FJQavE2Z+7Si1HxYo/bfrTSvb9WJ9YP9vtV+iXAyprVnWVgFTCRt7TZhFxlCohYSNNbCBtKNJ2DfaT9UgpOzfuB4/CqkGkEsMsaohrYcSjrL6NrWI4pevxVuw1XeIb2znZvx4yYDlPzxPcLmGztkY32Aq23t5gN1EgntCtZ2WbiKq1y9QVQptOaT3H4373ibCtFh/GyCsbcJbX9N1IuTGh6zZNEAlLJLPy2EfTJL//f6HHX743Se47wV34Xin8KKIWtj8AmaAQi0gZcfvCGwNBS9c3WqycUzJj7C0wtYaY0xTaDeEBmpBRNIZ/v5CYyjhB9/1MADvfWB0E3mq313Gv1Ah+fxpdNLCO1uGMMLUDN5TNyY0RSvS095upFUyoRK5ubbTxQEwhijwKF76KSYKb0zMaUGNYKuybsIFj6gc9NVGUKq5F65sjTXr4hwbridfC9qsOgiUg4iSH7Jcaw7ttar1xwftDgBq4eimHsbV9+nV9kmj9z0K4TWP0peuUXjkMrUfrFD7URHvTOtZqGL7kIp7QtmpLInpeWr563RcVCqKWDn3E9zcHE52Fr+41NwuUYrEzPzYrxdg+T9fYuYfHwFLtVwruxfKAntvIl7wf0DDjk3xQ0OhFuB3CO7QQNkPSY+g6m4YZ/Xd9bmfkSb7yr1Ycy5R0af0VwvUHi2M/XlFa1JxT7Dk7D7cXPsRJKtMhLe8QORVcTIzceWtFChNYmYvbrb/TRMGEV6tsfD/nIl3ghkwPU0YT8QZhtvlplzXa4COod1QDSL8EVbe0Fx9f/jdJ0Z67rbPeVOa6TsPY+9LoGyFNeMy9br9JF8wvSnPLzaSinvCBdVyj0caQq9Ccm4/qT0HMGGwNRNzfEPthwUyvzAfr8PdR9VtjCEqh3g/7fVrbs2xFCrYnB26yn5I0oBrNb/DiIzBCw1gcLTGGvLFZGwU5N54cMNiYMrVZF4xL2uXbBEJ7gmnLbv3dqSB0KtgJ9Mo3d/u4qMWXPNI5Ppc5MpA7cnS4NW6sShUffxN3FMxNFDyQ8p+PGrFUlDyApoHxkSkbE1qhG2VUXFvab99nEpoVEJjatIQ32zSKplwiek9nW9SrqUU2t7awG6o/Wil7/HOSitSzx5sjZUZ9wTFlVdtamivZYBiLR7r3Wo0YyWIOt7sbGXtJJ1Rj/NucI93WKM9Mn0PzRSjIcE94exUluTcgdWedSdKW31vSzYutSeK3Q9qQbm67/Hg+VfdwesO/Uv6H5A4WhGd5xF5Qe/jpBu97sXT7+MD7zw80pEma4UFP94Kbh1jDFiK+f/zZrK/tB+VkijZTPLd3gESuTlyx24lvfdwx+Ocqel4aOB2UDNEg6x5YSvm7j2B+5wpsv/jfnJ3HiL5/Fw8UqWFy8+/naU3/ka9atymfeS6Qd4MrJ1lOY7wrj1agHW/Mqvj67WKVwx8zhQzv3psu397dxQJ7h1CaY12Eh3bJt7yAoWnH9+cCTc9KP7Xq/EsvT4opVApi9zrD5B8Xo7EzVmyr93Hnn/yDNKvmMM+koI1GwQ/9erXg+tiV5fInf867pUfQ2MiUjkf/7dNJAZYPwRuhPc4RplEhYDlz14gKgVEtWj159U0vt7S6IyFe3Nm5M8vWut4c1IplQP2GmOeXPfx5xtjvjfWKxN9046LUhpj2lTVJh7FUL52nqkjN2/qtbXiPV5i+Y8ukHnlPPbBZM8jTNYfp2yNsjXpl+wh/RIggqgaQmj4OfdLXPziX3Ph+48TAk79P6NtGjVuNHec2hvei5k/EZ+P5uo342hcS+NHhiCM8CLTdnLOoBwNWimCyGCp9jvZbwX/XIWFf38Ga94l+fzpeCPndZSjsecTeI8Ptp+n6E/bl3il1K8APwY+q5R6VCl1+5pPPzjuCxP9U0qR2nuk683KyPeI2kx/32z+0xXyf/A0weXa0IszNWZaKkthZWysnEPpB3/LxR88vvHYKEBFISoK0dfPkPrk/4Lzt79HJvM1ZpI2U65F1rWYTdokbCveWcjSpN3xDMRSSpGvBqzUApaqAYsVn0ItIBz1K8SgTDzLMrhYbXlD0viGYHEbLrO4Q3V6b/Y+4EXGmBcA7wD+QCl1R/1z26ccEE2cdJbs4WdipSbrbevyZy+MZWD1k3//0+7bb9b/c7/yn0j+H/+SqXe8nI/++FdxrdYrGTpjGHPthRsv0o8MK7WAaMgXtKFoSNyaJfOavaRePIP3dBnjRZg1LyimvuWZ9/hgN5xF/zqVD5Yx5hKAMeabSqlXAZ9XSh1hc+YuiEEZQ9hhYo7SGq+Qx52aQW/COiW9MKWQ6qMrJG+dajtueBB+rY8qUCkqMxY/yN7J+71/QOntLyN/+n0bNllIORZeGPT0R2Cr4VakNUDVD8dW6XeiXMXM3cfQMw7a1UR+RPrn51ff0DXeIUUrPvmHn5b1TTZRp7+QglLqmY1/1EP8lcAbgdvGfF1iCF5h3Xok65gopJa/SvH8E4T+cG9vH0+9lZm3HRvqHA3FP79K9UeFpmpuWDN7+5uWrRQ80/4Tzv3pe7n6jCwn/+iP+cijdzUdo5ViOmmTtDWWimdFpm3F2ukzloJcwhpJlgU9Vtxrx3WPQurn5rBm49AG0I5GO/H9BKXVamtKZ2ysue0xP2C36BTc/yuglVLPaXzAGFMAfhH4jXFfmBhcu+Vbmw8ymCikunCp7/MfOD7Ds+2P873ffx+fOX1+dOEdGopfuMLKn10mGtHEjtte+uy+G3uNpVxsVeOH332Sf3HtDrJvfxm++8jqMVop0o7FdNIh69okHZvplMNcymE2aTOddFCokdzE1D3cqBzHsMDkc3p892MrEgNOjBKDaftTMcb8vTHmceAPlVK/qWIp4HeAf7JpVyj6ZqdzPc+mDCqD9SUPHJ9BW5qT1idWw/vafe8eSYB7PyoQ5j2iNUMFTWTiXmqHnWhamZ7P8QtvehnT87mBrkUpuPiV/8h/OfLb/Pxff3VD9d36MaPtgafs3qbCN8K7MSln2J1z+nrnI83TTaW6/REopTLAh4AXAVPAw8CHjDFtS6KTB2fNx+55zSivU/TBGEPp0k8Je9qSTJE7cetQi01dPpsnCiMeC+/hTXcd4ZbKp1oel3+ojw0QHEX69lkSt+UgNFS+u0xwqYJ9KEX6JXPodBxm60PSmObXrLW/3yv5Gt/52hMUr1+DoPcWUYSiYvaDgdve8lsUPnQvSy163+sZY8hX2/fC1w87bPX5rGvhWP3/bHz3ET7+3dO8+8NPDbz0a+r2GdKvmEevqbpbbSBsvIj8H56XvShH4MAXv/BtY8yLux3XS3C7wAeA1wJZ4F8YY1r/ZdZJcG+90K9RPP9EbwdrCzuVwUlP4WRyA4X42vBupRHofYV3GzNvO8blqVeQDr7HVMbDsQ1+oDBGceHMdY49cwbLVoShIb9QJb9YY+FaFa92Y3x7FAYUzj3W83Mm9xxk4TpgDI8F7+Cf/vKXuTv1NPffdrrj4/wwouC1HlefsjWVNps6NNhaMeVafVfxvvsIuaTN73zjId7zwPnB1uzWkLvjEM7RNErV3/VEoB2FieorLBgof3uJ8lcW+j+/2KDX4O7lVvW3gM8BtwN7gI8qpd5kjHnTkNcoxsmY+C+r/RujG6KQoLRCUC5Qy18ne+im9lujtXHgeDwpQ599sMXpIz5z+h7edNdb4T4GDvCZtx3j8dRb+Q+nz3PS+j0ADh7Lsfdgmmo54NyTy3i1kAtPl+Og6VCTaMvGmZrDLyz29NzVhUvM7zvC4oLhpPokj3wePvCW36Lw9pd1rL4dS5NxDKV1M0Qbk3q8MKLFSMBVQWSobNGoEiJY+cxF7AMJ7INJomKA92QJlbRI3JrFytnUHisSXKpt/rXtcr38Nvy6Mea/1///MvBGpdSvjfGaxAhoJ4FSqusY5ibGEPk1aisLJGf2DvS8jQDfcD1nH+T7n260U97KLW/r+KatpcdTb+Uzp89z0voE2tKrz3X9Wtz2mDtw4wZZp6+7Vliilr+GCXy6JvwalWvn2bP/OE56hotnFvnj/3SRf3b/K7jrOw9z3wtpG94J21qdeQnxOPBGBZ1L2BS9gE4z/6uhIRGZLVuzO7hcI7h8I5yTt+VIv3wPGEPqRbPUHitS+MIVOr4CiZHqGtxrQnvtx/5gPJcjRsEYg1fMt7hB2a2rGvMK+YGDu50Dx2e4fDbPST7BZ07fA7y873OctN7PSYum0O5XbWWJ6uKlG2Hd5+SW8tVzZA6cQFt6dfElq4c2RjzzcuNxSqn6qJHO17FSC5hO2j2NMBmnxK1ZMi/fg3JvvCNSF348AAAWn0lEQVRLPCuLqY8IEptje8y+ECNVy1+jtnx9Qyi5uTm80vKNRZbaMIGHV8jjZKdHOkKiUzulN4MHNsQvaLWlK32H9bqTUMtfA4bftiuor32yfsf5lk9LvMnxVm+2kHrJXFNoQ7xOSfI5UxT/61W2bMHzXUaCe4cxUdgytBufyx44QenyU5gwpFOVV1m4SFArk54/NPJrHCZ8h1Ifuz6s0O+vp2tM3KduBLSjITL9z6gMt3Lqe52VaRMZBnTCIvK3xxo4O50s67rDRL5HuxknYbWM5SaYOvosrGS684mMwS/m6+fbIZRC6eErVstN9XysMYZCLaQaGuK1GcGLBpsG30tLZtzCQtB6HL2CqCShvVkkuHcYZTttWwHaiaclK6XQTi/7PSqCnsaCTwalFImZvb1v9db6JCRne+//B5Hpecp6x6dl8PW6R0knWy+8hVJtN7MQoyetkh1GWzZ2eoqgXKCpFdIIrTo7kcYvLnfp9xpqy9ep5a9iJTMopfBLK5goxE5mSM4dwHITY/taxsHNzQHxfYB+2ybaTZKaP4jlJoEqJ63f49997h28/64jFO66l6XTG0eW9LuPZCu2VmQcq+8bkyvVoGn9koHGcq+zvr+9yhiUqzF9bL8mBrf1L+Fi5NJ7D+Nkp4F40Q1l2aTmD2OvaY84menubQNjiLwqke/hF5bwVhbjdVCMIagUKV48QxT44/1iRkwpRWJ6D1NHn9V35W0nM9iJ+Ht44PgMh27aw0n7QR799L/mL9pMiR92CF+8WJXd93kc79RY1i/xzpZbToWPyiGmLKG9WSS4dyClNem9h8kdv5WpI7cwdfRZuNnpDcdkD92EncndWFVp9ZM9hoSJqC0vrPlnuG02aOim8QLUj6BS2PCxQzfNoW2LW51P8sjnz/IB/c+bFqRytBp48fp4yvtwb4pHvX5J6asL8Xrc9Rmfpr7Te/HPZSjgZpJWyQ6mtO44A1LbDpl9R1f/beoTcKIgoHz16Z5mXYa1ClEYULl2gaBSAgzKsnGzM7i5ObTdSy99cKFXw0Qhlpts+7VGgU916SpBpYjSGje3p/6OpD+qzdrljVEyF88sopVuCmqlFLmETckLV3vdjVZwp1GAaUeTaLORQ78a7Zv7Xggffvdp3vPA+YHPFeV9lj5xltTtMzhHUoSLPuVvLhFeldmTm0mCW6xSSsUBaPVejSrbpXTpKaI1Q+RMGMS98eXrJPccJFHvK49SFPiUrpxtGkWTnDuAm81RXboa9+8BO53FLxVWX4RMCNXFy/1vmKwUien5ga7V0opc0l7dyUYrRcUPO65TEoSGpL09b/ZFhYDSl69v9WXsahLcYoP4BmeWoFyk24y+oNR5l/Tq4mXsVAbLab6JGQU+lYVL9ZuoMeUkSM8faurFt2KMoXT57JoXi/gaqwsXqS5dhehGu6YR4OtOgF/sb3f3xOw+nPRwa043bi4aY+jWsvYiQ8kL0EqhFLiW3vJZk2L7kB73hLt8Ns/FM4tcPNPbYkm9Su89jJ3KEN/g1IMPoTOmKTxNFFFZukrh6Z80hTaA8Ws9LUcbeVWidsuyRj322Pv5erQmkdvT+/EdGGMoeuGGRadaqYWGShBR9iPy1YCajNgQdVJxT5gwmSNKTGEXrnLpiSury4yetD/JxTMLaMsaycxEpS0yB44TBX48y1IpihefHGi6eGPChldapnL1At2q+Gr+Gpn97TdkiKJwBAv3G5zMNH6pRUW+4VBDFPhYzvDbc3mhWV1sql8lP2q7gfG4JX92mvTtc6iExj9XpvSV64RLkzWiaCeR4J4QkZ1k6R/+z9T23gxBAEpx5ctnefWtimMfupfn3f9yfudPXs1J80kunlnk0E2j6Str2wHb6Xua9yqlcDI5/HKxp9AGCL3O/WfbTfV0nk4sN0lq72GU4+Llr3U+2IAewYxLgOqQVbMfmZaLVY1T5jV7ST5/enXvSfeWLM7xNEufOEtUmIxRRDuNBPeYxK2LEa4t8aZ3wvzNYDnxf8Ch1z2Dj1z5Xe588Ot81X2E137k/Uz95sd49NP/motnWi1srwYOdGU5KG112c9y4yp3diqLAkpXz234XDvaslvutNLQtk3SIzuVJb33CEopUrP7SOb2UFm6Up+QtL6FobDTUyhr+OA2xgy98umwkzBDYwhNwJNLP+lpZxyVski9YBq1Ztam0gpsRer2GblJuUUkuEcs3gkm5DH/Hdx591FOlB4a+pxVO8uXTzyHSDcPrQu1y2sPvxOIh3wVHzzFv330Lh5+/XEA1o4gfirzNn7w6ff33U6pFZaoLV3FhEHHCTtOZhorkaS6dK0p/IJKkWJ54/jnTkKvSuHcY6T3H8VOZoB4x5ra8nWCUmGo4NaOS+bA8aaPKcuKF9OaP0Q1H88UbbwIxSE/moW2Bm2RrOUMUW073inuv+0US6cf4aPfeZj3PnC264xKe97FBAa1LimUrXEO975mixgtCe4RunhmEYzhtrf8FqdKD3F39DT3vfTuoc8bhlmiYpvqc10Jdv9tp/nqmt3IGz76nX/Dw68/3lc7Zf3a1SaKe906kcb4HiYKQGksx8Uvr7TuFw9SIhqDMSGly+fIHXsWAMWLZzBBwLDvYtqNxW5IzsyTyM0R+TWU7aC7HN9w+Wx+6GvrJuOMZmRJPCkHPviu7uEdLvuoFi8WJowIFnbQAmQTZlcFd6MaHpfH/Hgvwu9/6F6+evp93O+dghH8btvGUKZ1i8JpMa6s1U4s9992qmU7pVP1Xctf3Ri8xkDgrwaqX1qmfO3C8O/hWzL1tVGieotmBIs19RDESmusRO/VZOMF+7HgHZwqPsjd7zoe/+zXafWz6notQNKOtzkb5Q44jfDuNiEnWgnwzpVxj6VRazYNJoLKN5dGdj2iP7smuNdWw6NoX7RyivoGsg9+HWeExYhSirStKa+bsKGgr4X117dTOlXfxpi2/ey1rYrayuKYQpvV9bODSmlkz2FGuLbKaiFQ3/392Ifu5asvfV/L0Ib6z9HRlHsYCthgYOSh3Ym9L0HqH85i70ngX6pQ/sYSK5+7RPa1+0g+ewpQREWfwhevEl6XinurbOvgjt9+Dq/Rc15bDY9Luz/aYSUdC0srKkFEZAyOVqQGWDEObrRTOlXfqr44Vavw1vaaYXHjXNxfKexkpq81wVN7DhN6Fbw2mwBrZzSrGa6tsv/pL3+Zu6MP9PSCnbQtqn5E79EN8Ure4w9u53ia6TsPgaVQWmHtcUk8O0f+D85R/LMrFP/8KspRmGo/Vy/GYVsGd2NExmPhrw9/MhNxx11HOPahe7k7dXzk1fBmciyNY41mztT66vvf/cmruXVd9Z2Y3Ud14VJzOCtFcm7/jfNkpuPhe20C3EqkSUzvobJ4uWO1qxMporXnqQ8jtBIpXKXiPTS7vEhoN4mbmwFmCP0qYbXChqVtp4ebSNNvlQ3xuxc/MtTq75gcS/W0XVnDZm2gkH3dvqZ2iLIUaMi8ai8rf3QBQoORDYG3hW0V3Ov/KE6NqKVxd/Q0953u/Me1WzWq79e1rL5nAagtXcOEPsp2SM7ux8nkVh/vTs3iFfPx9PM1oZuY2YubmV7dvMFOZ1k5++O24Zs5cIKgXMArLMUb607NYKfj57HcJOl9R6lcu9B+DW2lcLM3evWZ/ccoX7u4OjszXtr2UH0t7cH0WmUbY/BCgxdGKBXfQF6/LIml4u3LDKAVJC1FNTSsH3iSdjZpwo2jsGY2LgimlMI5IqNHtpuxBHfghX23OaIw2vBHMYoRGcDIbhLuVJ2qb21pYF+8OEIE5QXDgeyNxzaWh/VLKwSVAspycKdmN8wyVEqTnDtIdeHihudP7zuG1ho3O71h+dnVa0xPYR87SeTXCH2PyrX6ZB5jQGnsRGp1k4T4uiwy+4/GNzZNhNJWzwHY6ne31yrbGEPBC7tuoBAayCUs7DUrGiZsQzWI8KN4LZOkrZs+Pw6rGy0snGE+fCatFlExFZlqv92olvvHDenQ3DPMva/5V30/7o67jlD40L0snX5fy5ERoj0vjCj7IZGJq7mUY+EO0Fbx3UeYvev/Yuo3P8YfP3x+Q2v1pPVJUINP5AkqRar5a0S+h+WmSO45MNBU8igM8EvLREGAk8qs7tAzjLXv+B4L39H0OWPgn/1y/ebzbafbnqMWRJT83oLOtdTQ620Py3dvjOk+8cbnsvdFR7HcGze8Iy+i9JXrVP9uNPebRGcHvviFbxtjXtztuLEE963ZpPnEc4/09RgD/Nt3He/4RyFaqwWtFy3KuoOFN8BHHr2L9zxwdsMtsWy9nYJiZOuibAfr2yA/+MhfNX3eQE8FRdEL8HrsA2sFM8nxrlfeK999hN/7zmk++vQsc7cdgBCUraj83TKlv+yyJIAYmS0N7lued8L835/77b4fJ1X2YJYqfssRzsMGg99yIs/DPFw5GrdT7Lj61iO6YdqLYV8ourVBGu/41lv/uxkZQ2QMWqmmkT0lL+jrxuNcansEN9yovn/7P1zCnUnjLVQ5lmy/2JcYvS0N7mc992fM737mL0Z+XrGRMYalavv1Q+ZSDsYYgiii7BtCY1DcaGXaWuFaGkM8eqGX8cLd2injMkybplMbBEN8X6VLGwTi73fZD5vC2dHxFmNKKYLIsFLrbeElBcxuo+BuWNs+6WU9EzE6Ety7hDGGfDVoXXEDCVt33GllPVspsonW48MjYwgjg6XjKrPRTtksUwO2ada3Qb6/rg0CvbVBgLY719gacok4hCteQKWHqjtpa9J9TKDaTGvDG5AA3yQS3LtIuzBxFPgD/HgdrZhK3LhpZoyh5Id4oVld/y9hKdJO7yM1RuEjj97Vd5umVRtkmJZcu7YUwJRrYYCi13xz0tFxde2t+RFtxfdvEFJ9by4J7l3EmHgYWTWIVufYJW1FNTADr+4xm7RXQ6VY85tCpyFl676m3I9C322aPtsg1SDCCyNAkbQ1jib+WGQwpvNqKZ1eKLOuha0VkYnbVJO0DVkjvBs3qyXAx0eCexda+7M0QL5D77ubxhjjih9QCVr/jqzv0a5vpYxTP22aRpXd+P60qnKNMRRqN3Zib9i4wnh7uj6pppX172ImkVTf49drcE/2b5Jo0hRIxvQVOusVvZCUbdqGNvVz+2FEJYgIo+bq3rUUmTG2Au6/7TTFB3s7VlUNy15AWA/lVtfmR2ZDaEN/3z9LqQ3L7O4ka5eDfc8DZ3vejEGMnmwWvEMpFb/VH1RkoNJlFTsFq7ME18eVV9/odqtFxlCo3QhtiK8tXw1YqvgsVnxWasNvxKuIp6e3kxjiZ7GdxJsxnGbp9Pv44LuOx7Mu82e2+rJ2Ham4d7CUY6FU3KNdnVFpa1zbouSF1MLOwdqtduz2+VoQ9T1qYm27ZxTVeqPvv+F51vx/t+np7TSuzqnfaNRKkXU33px0LTXQWtzb2drqG+C9D0j1vZkkuHe4pG2RtDeGp6WBLkWmpdmwOFI/+olDY+LV8yprbrCmHN3y2ns9H0A4gu3C2nEtRWbdlHXX0swkFV4YYUy8oqO9w0K7obEVmu8+0nVDBjFaEty7lGt1XtDf1nEfeLnHySTtztGrWhA1bRRhgLIfUfEj7Pra472cb/0EmXFGZrt2tlZq4BccIXqxMxpvom9aKTJterJJWzPlxhs3pDv0ZruFYq9tksiYDbv7NBjiG4crtQC/S2sH4jbF2lmN47xVOMzGvUIMQyruXSxhx4tQNXYfd7Ta0Fd2W2yZ1pC048eu7REr4kBL2XHwN1og1TDus9s63r7L1pogMhS9oO0QuvXKfsR0hwk3YWTa7qQ+zAibducbdAGvnWh1edguu8aL0ZDg3uWUUrgdKkddH51SXRfeVn296JRS9aGABkttDP5KfWJQQ7yWR8iUG49I6UfYZahdp6F4WsXvAPp9TohnRDbWyQZwtSLtbv9Zj5ul0etePN0Y5y03KsdNSgbRVdqxyLoWjlarI1NyiRszKy2tsPXGnVoaMxFb6XXN6vUKtaB+429jSHdaIMvWClurgXreVn3yzFzKYS7lkE3YEzXzcbPEI03ubhomKEMFx0OCW/TEtTRTCZvppFMfZtg9uDq1QAYd7OFHhqIXUvLDDeGtO7x7SNrxNWddq6/wdq3xzwLdSRrjvBdPv48PvPOwjPMeEwluMTad8q7TAJEp1yLR5cafF5qWrZOMY5G09Wo421qRS9ir1bhjaWaSdjxzssv1O/WRNaJ/Un2PlwS3GJtOFXDGsVqGt60VjqXJuDazSZtkhxuAfoulU5WKJ8PM1tsauYS9YRihUoqErUm1GTHjasVM0mZqTTtI9E+q7/GR4BZjlXEskmvCW6t4pTzHivvkCUutbuyQqg9DbFBK0Wmv3GEzNWHrDcsCxJNqWq9HLgazvvp+cuknEuBDkuAWY6WUIl2vnmeTNtMJe3UYnVbxzMPZlMNMm955pyF3ww7HW63Okza5hM1M0l7dyUaM1trqW9Y4GZ4Et9gUqj5UsN9Q1Eox1eKGYnaEVbFS8YgTqbLHT6rv0ZDgFtte44bilGsx5cYVskx+mVxSfQ9PfvvFRFAqvmnpWBvHi4vJJNX34CS4hRBbRqrvwUhwCyG23NrqO5TquysJbiHEtiC76/ROglsIsa1I9d2dBLcQYtuR6rszCW4hxLYl1XdrEtxCiG1tbfX94XefANj14S3BLYSYKP/+n//MVl/ClpPgFkKICSPBLYSYGKExFLyFrb6MLSd7TgohJkJ8ozL+/w+8c3fvbSkVtxBiYjjeKdldBwluIcQE2u2760hwCyEm1m6tviW4hRATbTdW3xLcQogdYTdV3xLcQogdY7dU3xLcQogdZ6fvriPBLYTYkXby7joS3EKIHW0nVt8S3EKIHW+nVd8S3EKIXWOnrO8twS2E2FV2wu46EtxCiF1pkqtvCW4hxK41qdW3BLcQYtebtOpbglsIIZis6ls2UhBCiDUaGzZ88F0PA6xu2LDeVm7gIMEthBDrxNX3KXz3kdUAX+s9D5zlyaWt24FHglsIIdpoBPh6S6cf4aPfeZj3PnCWn+bPbHp4S49bCCH6tNU3MyW4hRBiAFt5M1NaJUIIMYR2NzPH2T6RilsIIYbUqL7X78AzLlJxCyHEiGxW9S0VtxBCjNBmVN9ScQshxBg0qu/F09SHDo6u+paKWwghxsTxTrXcfX5YUnELIcSYjbr6lopbCCE2wSirb6m4hRBiE42i+paKWwghNtmw1bcEtxBCbJG1u89/4J2He36cBLcQQmyxRvXdKwluIYTYBhxv4/Kx7UhwCyHEhJHgFkKICSPBLYQQE0aCWwghJowEtxBCTBgJbiGEmDAS3EIIMWEkuIUQYsIoY8zoT6rUNeDsyE8shBA723FjzN5uB40luIUQQoyPtEqEEGLCSHALIcSEkeAWu45S6otKqbxS6vNbfS1CDEKCW+xGDwC/ttUXIcSgJLjFjqWUul0p9T2lVFIplVFKPaqUeq4x5ktAYauvT4hByZ6TYscyxnxLKfWnwL8BUsD/a4z5wRZflhBDk+AWO937gW8BVeB/3+JrEWIkpFUidro5IAtMAcktvhYhRkKCW+x0HwN+C3gY+NAWX4sQIyGtErFjKaXeBgTGmNNKKQv4a6XUq4F/BdwKZJVS54FfN8b8+VZeqxD9kCnvQggxYaRVIoQQE0aCWwghJowEtxBCTBgJbiGEmDAS3EIIMWEkuIUQYsJIcAshxISR4BZCiAnz/wPLFBPo8uNx1wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize parameters randomly\n",
    "\n",
    "h  = 100 # size of hidden layer\n",
    "W1 = 0.01 * np.random.randn(n,h)\n",
    "b1 = np.zeros((1,h))\n",
    "W2 = 0.01 * np.random.randn(h,num_classes)\n",
    "b2 = np.zeros((1,num_classes))\n",
    "\n",
    "# Select hyperparameters\n",
    "iters      = 10000\n",
    "step_size  = 1e-0\n",
    "lambda_val = 1e-3 # regularization strength\n",
    "\n",
    "# Select nonlinearity\n",
    "g       = relu \n",
    "g_prime = relu_prime\n",
    "\n",
    "#g = logistic\n",
    "#g_prime = logistic_prime\n",
    "\n",
    "# Gradient descent loop\n",
    "for i in range(iters):\n",
    "  \n",
    "    '''\n",
    "    FORWARD PROPAGATION\n",
    "    ''' \n",
    "\n",
    "    # Compute class scores\n",
    "    hidden_layer = g(np.dot(X, W1) + b1)           # shape (m, h)\n",
    "    scores       = np.dot(hidden_layer, W2) + b2   # shape (m, num_classes)\n",
    "\n",
    "    # Compute class probabilities\n",
    "    exp_scores = np.exp(scores)                    # shape (m, num_classes)\n",
    "    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)\n",
    "    \n",
    "    # Compute the loss function\n",
    "    logprob_correct_class = -np.log(probs[range(m),y])\n",
    "    data_loss = np.sum(logprob_correct_class)/m                    # cross-entropy\n",
    "    reg_loss = 0.5 * lambda_val * (np.sum(W1*W1) + np.sum(W2*W2))  # regularization\n",
    "    loss = data_loss + reg_loss                                    # total          \n",
    "    \n",
    "    # Print diagnostic\n",
    "    if i % 1000 == 0:\n",
    "        print (\"iteration %d: loss %f\" % (i, loss))\n",
    "\n",
    "        \n",
    "    ''' \n",
    "    BACKWARD PROPAGATION\n",
    "    '''\n",
    "\n",
    "    # Compute gradient of cross-entropy wrt class scores\n",
    "    dscores = probs\n",
    "    dscores[range(m),y] -= 1\n",
    "    dscores /= m\n",
    "\n",
    "    # Now backpropagate to get gradient of cross-entropy wrt parameters (W2,b2)\n",
    "    # and hidden layer outputs \n",
    "    dW2     = np.dot(hidden_layer.T, dscores)\n",
    "    db2     = np.sum(dscores, axis=0)\n",
    "    dhidden = np.dot(dscores, W2.T)\n",
    "    \n",
    "    # Backprop through the nonlinearity\n",
    "    dhidden = dhidden * g_prime(hidden_layer)\n",
    "    \n",
    "    # Backprop to (W1,b1)\n",
    "    dW1 = np.dot(X.T, dhidden)\n",
    "    db1 = np.sum(dhidden, axis=0)\n",
    "\n",
    "    # Add regularization gradient contribution\n",
    "    dW2 += lambda_val * W2\n",
    "    dW1 += lambda_val * W1\n",
    "    \n",
    "    '''\n",
    "    UPDATE PARAMETERS\n",
    "    '''\n",
    "    \n",
    "    # perform a parameter update\n",
    "    W1 += -step_size * dW1\n",
    "    b1 += -step_size * db1\n",
    "    W2 += -step_size * dW2\n",
    "    b2 += -step_size * db2\n",
    "\n",
    "def predict(X):\n",
    "    hidden_layer = g(np.dot(X, W1) + b1)\n",
    "    scores = np.dot(hidden_layer, W2) + b2\n",
    "    pred = np.argmax(scores, axis=1)\n",
    "    return pred\n",
    "\n",
    "plot_model(predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Multi-Class Logistic Regression Using Backprop\n",
    "\n",
    "This is a simpler model than the one above. You may wish to inspect this first if working through backprop. It is a one-layer neural network, or multi-class logistic regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 0: loss 1.100347\n",
      "iteration 10: loss 0.918425\n",
      "iteration 20: loss 0.851985\n",
      "iteration 30: loss 0.822567\n",
      "iteration 40: loss 0.807708\n",
      "iteration 50: loss 0.799517\n",
      "iteration 60: loss 0.794722\n",
      "iteration 70: loss 0.791789\n",
      "iteration 80: loss 0.789937\n",
      "iteration 90: loss 0.788737\n",
      "iteration 100: loss 0.787944\n",
      "iteration 110: loss 0.787413\n",
      "iteration 120: loss 0.787053\n",
      "iteration 130: loss 0.786805\n",
      "iteration 140: loss 0.786634\n",
      "iteration 150: loss 0.786515\n",
      "iteration 160: loss 0.786432\n",
      "iteration 170: loss 0.786373\n",
      "iteration 180: loss 0.786332\n",
      "iteration 190: loss 0.786303\n",
      "training accuracy: 0.49\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD5CAYAAAAHtt/AAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0pHd54Pvv791q1a7uVi9ueWtvTdsGb2AMGBzCYoPBGGxM2GxiJ7ncm5vJnSwkDGeYMMSTezJzzszk3iEJGHLjE4IJYQmYECAYbAebxe721m277d7VUmutvd73/f3uH69KrZKqSiW1lirp+ZzTB7r0VtVPbel5n/otz6OMMQghhGgf1loPQAghxOJI4BZCiDYjgVsIIdqMBG4hhGgzEriFEKLNSOAWQog2I4FbCCHajARuIYRoMxK4hRCizTgr8aJdyZgZ6EoueF1QDimYfgB6el3ieoxgtLwSQxJCiJa3d2rqlDFm00LXrUjgHuhK8rk7b2jq2qFDE+hQsz+8k1vv2MGuwt8x8aXDKzEsIYRoaQMPfudQM9etSOBejIHBboYOTXAhn2fflw0PhHdx6z0SwIUQop6WmOMeGOxm27m9WLbNhfbneeD+ozyfuJ3uD+1c66EJIUTLWfOMezbJvoUQYmEtkXHPJtm3EEI01lIZ92ySfQshRG0tl3HPJtm3EELM17IZ92zV2Tc8EN4p2bcQYsNq6Yx7ttPZtyXZtxBiQ2uLjHs2yb6FEBtd22Tcs0n2LYTYyNou456tkn1fbN/Hvi9ryb6FEBtCW2bcsw0MdjMw2F0z+5YMXAixHrV94K6oBO+LvfvY9+VP89mvXSfTJ0KIdWndBG6Q7FsIsTGsq8BdIdm3EGI9W5eBG+pn3yP3/J4EcCFEW1u3gbuisnWwkn3L1kEhRLtb94G7QrJvIcR6sWECN0j2LYRYHzZU4K6Q7FsI0c42ZOAGyb6FEO1rwwbuirnZ92f/4VrJvoUQLW3DB26ozr4vdO6T7FsI0dIkcM8i2bcQoh1I4J5jdvaN7aHicQAJ3kKIliGBuwnurl1rPQQhhJghgVsIIdqMBO4GLlSf4yuff4FnT712rYcihBAz2roDzkqa3dty7xfhK9JdRwjRIiTjbkB6WwohWpFk3E2QzvJCiFYiGXeTGmXfkoELIVaTBO5Fku46Qoi1JoF7CaS3pRBiLUngPgOSfQsh1oIE7jMk2bcQYrXJrpJlMjDYDYB16D72fVlHO0/u2MGuD8nOEyHE8pKMe5lJdx0hxEqTwL0CpLuOEGIlSeBeQZJ9CyFWggTuFSbZtxBiuUngXiXSXUcIsVwkcK8i6W0phFgOErjXgGTfQogzIYF7jUj2LYRYKgnca0yybyHEYkngbgGSfQshFkOOvLeQ6oYNhgfCu6RhgxBiHsm4W8zphg22tEsTQtQkGXeLkuxbCFGPZNwtTLJvIUQtknG3AWlWLISYTTLuNtGoWfFa0mgmnQIZu4jBrOlYhNgoJONuM62UfY+6WQ6mTqEMGAW2sbgwu4VUGKt5vcEQYrBRKNSqjlWI9UQy7jbUCtl3wSrzYmoErQyhZdDK4Fshz6aH0Oh51w97U/yi6zC/6D7Ez7sOcTw2IRm6EEskGXcbW6vse8LJcyB9smbYDZXmlJdjc7lj5rERN8Oh5BhamelrDMcSEwBsK3Wv2DiFWK8k425zjbLvlcjAi5bP8+lhjIJ6sx1HEmNV2fSxxMRM0K7QynA8PilZtxBLIIF7najUPKk0bPjs165bkemTk94UulGwVVHWPeHmZx4qW2HNS0PV8JWEEHVI4F5HBga7a7ZLW87su2yFdTPtCgMULH/m74nQrXmda2xZohRiCSRwr0NLyb5LKuBofJyDyRGGvElOuVmydmneVEZXkMAyjcOtZRQJ7c38fWehd95zLKPYme+R3SVCLIEE7nVqMdn3hJNnb9dRjscnGIllOZQc48XUCM90HGdfxzF8dXqqo7+cwtMOanYgnh3bDXjGpttPzDzUFSS4ILuFVOBFQT1wOS+3iX6/AyHE4iljln+W8cKtPeZzd96w7K8rlmboULSDQ4ea/eGd3HrH6Z0nBsPPuw4TWvO38AFgoCuIc1F268xDAZqh+CRjXg5lFMpAzimjUPSWkwwW+nCNvRrfmhDrysCD3/m5MebKha6T7YAbwMBgtOWusnXwgfvv5NY7bod7YPOJv8T86FD9JyuYcooEKsSZDsYOFjuKPewo9qzG8IUQc0jg3kAqAdw6dB/7vhxl3+98+9tR7hMQNvrkpRrvJBFCrCqZ496AZs99f+PbGhXvbXi9bRSOWb8/KipuYfd74LTAQqkCb1eK9Fs3k3xDP3Zv7R05YmOTjHsDMjqkuysgyE9xWenPyWY9bJJ4bogJStFi46wYFirNocQY5xT612zMK8JWdLx1C7GL0pjQoJQi9+gopaczWAmbYLS8wCeRRbxVv4dyLYKTRWpUBIgo6HrfdpxtcSzPxoSG5BXdZL5zktKzmWUZh1gfJHBvMEZrsscPon0fMNiApUqUTYoXSm/nsvNPkH3pUWZHF6NgJJZloNRFQrd5BqjA25XG25XC2RzH6XVRjoWa/k1Iva6f1HV9mCC6e+V+MExx79SS387udel8z3astAPGgIbMPw1RfjE379rYxR042xJYXvTpRtlq5uZSeiELvkxXiYgE7g2mnBlHB1HQrlAYYlaObdbDjB+cxFW1U8Ipp0Ci3MaBeyajjYKjMVGWXXWJFZ3lV9ObYtI3bCYc9/GPFJb2fu8/CytpT79upPM928g/Nkb+R6Mz/xnsXpf4K7tngvZsRhvcsxL4B/PzviY2JgncG4yfz0SZXw1dnWVyk9GX58QzlGFmV0m7il00J6Od+03W4ipSb+gnnPTBUpSenqL8wvxsueZTz06iXFUVtCvvm7yyF8u1yT86Rtet27B7PLCpfTPxLLresx1CQ/GpKbI/GIFAsu+NTAL3BmPZDjUrhxhDeXIMF1PzSLsVs+mZTMz/QptwBxOk37K5ZkbbiFIKZ2scZ1scpRTeuSnKz2fJfGtowedayfo3OmUr4pd24u5MYPe4KLv+uGYCv6WIvaITq8Nh6qvHF/V9iPVFAvcG43X24uen6mTdZs7fFBjwSfFs/r3svOfq1muXZkHsks5ogbGkKT4xOW9aI35FN+k39KPcxkG7VrZrjKnKmC3PIrYrTWF7nOBYEXdnAmdrHD0VUHo+O5MJq6SNitsop/57Gm2wu+cH7dnjmDsmy7XwBpNYXS56MqoH42yJkbimF6fPxT9WJP/TMfRk0PB7Fe1NAvcGY8fiKMfD+KUFr00NDKIsxakTJXY6D/LA/du49Y7b2fWh1gjeztYYXbftQHkWSimMMcTOT5N7dJTCv41HF7lqwaBtQoOe8tHFEKc/NnOtCU3tDbOuwjs/Rfr6fuzNcZStMIEhfcMmJv72CO650ddMCBgzPfU0/2NMvTHNvrbmdI6KgnV50sc7N0nnu7aBHU3J2H0xYpd0MPGlw4Rj/vzninVBAvcGUxofwQTlpq514kmUUgycnZzVsMHwQHjXmjcrtjd5dN9xVlVGq5QCT5F6XT8qYVP8xQR2hxtt9auzpmqMIfvPwxT3TpJ6Yz/O5lgUsBWYQINSKG9O8AwN7uY4zpb4TPBVtsK4io5btuF0Ve9UaaSpefZ537wi/SubGD+UJ/2rW6puAMpWoCxS129i6h9kOmW9Wr+nKkRN5cx43cXJ2exYsiqonG7YYLdEs+LUa/vArh30lKVIXtFN711nY/d58xYHZzO+ofjMFLGLO4hf3o2yLdR09hoF3xrPNWD3ufMyZmUpnF6v7rjmjbNG0K5VO2juY0oprLhN/OruaJvh3Ne1FO5Z7bseIRYmGfc6FRRylLMTgMFNdeMkUtPTCfVOf1QoUIpE/9aaX61ul7a62bd7dpLk1T1YHQ5Wp9swW1W2BTakru8nGC9HUyCzAqoxJprSDzS9dw6CreYtXCpbYUKNLhvQ08HTUmT+aYj0mzbVf+8aN4pa8+c1n1tjjr3mda5FbDDVuDa6Rf3DPqKtSeBeJ4wxBIUs5cwEYblYNR3i56ZwU10k+rfhJFIE+Wzd17ETKZKbtmPZ9X806hWtWsm57/RbNxO/tKtq0a6pYKgg9+NRUtf14fR50aFQW2F8jbIVVtKBZP0AaUKY+NsjWKloL3b5cB58Q3FLjMRVPVizsm6jo7lyK+XUnL+ujNeE09FUzd8qOPf66LIaNwJtUCl77nry6W/bVXTctJXMt4eIX9KBe24KPeVTfGJS5r7XAQnc60Th1DH8XJ3dIsbg5ybxOnqI924lW3wRdO1UTJdLDYP2bKuVfcd2d1QFbWBmMXJBSqEzARNfPBxtu0vaqIRN5zu2zpsfr3kjCDThSIlwuPrh/L+N4Z2bgh4X5VkY30BomPzqcbpu34E9dxpFKUxo8E8VCU6WKPxygp5fazzN1PCmpMDuqv+pQ9kWsfNTOB8dxEo70YGj0JC4vJupb5xoei+6aE0yx70OBKVC/aBdYQz54aP42XGSW+oHDKMXt41sxee+XUX6hk21d2UskG0bYzD5kHA42kETjvvRFr4tMZRb+7mVbNiEGuNrMt85WTur9Q0TXzpM5ptD5B8ZI/f9Ycb+34OEp8roTO1/QxNosg+eJPudk4RDJfTU4jPf2Z80Fvz+AbvTqTpCr1yLjrcPLNh+TrQ2ybjXgaCQbWrB0YQ+pYlRmBoD24FwfoBR1tJ+JKqzb3ggvPOMs2/lKbo/NIiKL3xis960SeHJCXruPhu7wyEYLZP711PoqQDjm3m7RYyvKT+fw0rYhFM+hZ9PEJ5qsAPHQPnF3Ly6I6UDGew+r2oapXJ9MHx6G2bhlxMkr+tf0qGgpq5z6gR3C+xNsZkbmmg/knGvA0pZNJ9CGdC6bq9HE/rTtUwW73T2Pb9d2lLEX9mN3eksbcscQGBIXduH0+OhHAt3S5yuW7YR5gMIDUafvtkZbSCAzIMnmfzKMbLfHW4ctBso/nwyujmU52TvD56sWiws/HwC/+UcxtdoP8ToKJvWZY0JdLQtcRYd6KarFRpt0PmaZ2RRlpoZG4DV4RC7uAN3MCGZeJuQjHsdcNNdFMcWPoI9mwnrBGelCAo5vI7uJY9nKdm3MxBDxW2C4wVMOQpOsQvTC552PD3sGhGnRsapXIvUtX1M3H+Ezndsnal3HYyWyXxzaFlqgJiyZvyLh4jv7sQ7L4XOBBR+USN71zD1tRPY/R7OQDw6BJQPUZYizPh0vXcHdp8H06c3/UN5vMHkwu9vDKXnMpRfyJJ+60BVRm+0IRwvoyei//6pN24i8aqumZuEKWkm/+4o4bgsYLYyCdzrgGU7WLEEurSYCnaK2pO3CmWd+QexZnee2D0uXe/bjko6UbJnQfZHpyj+bAJTqL2A2uzWunrsvhjhqTLjXziElY52Zuhc7ex0yXxD8YlJik9MLnhpeKpcM7uf+NJhnC0xrC6XYLiEnvDpvGUb3tnJ06c7deWwkIkKgxko7psi+71oNdXZNkni8lmBuaCZnD6Y412QJn55V9VhIeNadL5nO+N/9fKZ/xuIFSOBe53wUl0UFxG4nVQnQa2aJQqcRHrZxlXJvi+2o3Zpc7Pvrtt2YHU4Vdvi0m/ahN3lUnxmCvesxPyDLnWCdtV2Oz3dD6LGIqSePB0kdXaZA/YyC06W4OTpueipfzxO8jW90WEhR1E+mCP/b2O42xMoz6J8MFd1E8h9f4TCY+O42+PoXFhVxyX5mt75e9cthd3hYPd5hKNLmyoSK08C9zrhdXRTmhrFBAF1N/fOZgzxvq0UR09QmdhUSpEc2LksGfds9bLvC+768rygXRlH4orumU48dYs/zXrMaEM4UcY/XoSSprh3CvecJKlr+1Czpwp8Te7Ho8v6/a0qDfmHx8g/PFb1cDhSP8jqTEDpuTl79y1wNsdqP0FR9W8mWo8E7nVCWTbpbedSmjhFeWrhwBTkMyQ3bcNLdRIU8yhlYceTZzQFsZC52fcT22/hthtfwq5RaFYpteBCmQmjE41GG0zZMPn3x6qq4gXDJdCG5Gv6UDELnQ3I/XBE9jAD7vZEdIin1gEgS0Ut1kTLksC9jli2Q7xnU1OBGwU6DLFdDzfZsfKDmzY7+3ZHvo7Rr4Al9GcwviH3gxGwFTrjR1vyakyJFx6foPD4hBz/nstS0UJsjQgQjDToiylaggTu9UZZUfuahfZ1K4XlrF0bskr2/ewvTvCKK7csKtPXZY1/tEDxyYUX/k4/aQmDXMf8o4Wan2h0WVN4bAIAlbBIXttLbFcHxtcUfj7R1GKrWHkykbXOKKXwOnrn9x6rvoh4z+KC5UoYGOxmYiLgwL5TaG1m7jX17jlGG4LRErl/HWHqq8dWb6DrUWjIfGsI40f7xY0x0d7xYoi7I4GzPU7PhwdJXN6N3eXi9MdIv3ET6bdvWeuRCyTjXpfivVuiVmTZ8ZnHLMebnhpxiXVvWtXpkYUMnyyRLwyz66JuUp1e3XKn4XiZ8b86tAYjXJ90LpypPa5M1FHe7nSJX95F/LJOMKqqrK3yLOIXd5B/eGym+45YGxK41yE1XZY13rsZE4Yox5k+Xdma+jYn2DGYbhi0g6GSNAZYThZ0vXc7Vo1yApVO97WY0OBujVGSwL2mJHC3MWM0fm4KP5fBsm28zl5sLz7zdWXZKKu1OrM7rsW2s1L0b0kSj9sYA5YFVoNmuZUgorPSR3G5uIPJBSdKax50UoqwThEtsXokcLcpYzS5Ey8Tloszk8Ll7ATxvq3EOnrqPi8o5CiOD6ODMrYXJ96zGTu2Ot1SOro89ryqH8teuLLdXIfTb6F0T0/rNStuU42aGNdT6c0ZHJOtgmutdT8/i4YqDROqVvKMoTh6AlOn1nY5O0nu5CHCUh4TBgSFLNkTLxEs6qj80l18aR+2Yy1hUdSw/9ufW/aSsYWePo6/6lpGLroMbdf/ZFLo6SOzbWfda7JbtrP/Hbez7/13c+LyVzd8rVbhH843bOIAUd2ScNKPFjADjX+swMTfHV2lEYpGJONuU/XrbyvKuSnceBLlnC60b4yJClHNfc704+mt56zoeJNpB6dW/8Ymvfnd53PB8w/ywFfeTLl8Lbfec/uSs28DPH/jbRy/+nVRpURjsMKQy//6z+kYOh2YSh1d7Pu13yQ7sAMVRoeELvz63zLw5GMz15y4/Br2v+uDUbC2bUYveAVHr30Tr/rcf8EOWndKwZQ02R+MRC3YpntsVtq5GT8qGTD590cJhkpYnQ7GN5hCa5cH2EgkcLepusfSjaZ46hhFpbBsl+SWs6J5b6MxNepvA4SlVfjou4Sie7Nbd7mezbkX9XLPlkf47w9cj4rHYYkfFE5dfBnHr3wt2vVmHgu15skP/++c950HwLbpPfA0T3zkt8lt2RZNwk9veX/u3R8keeoknccOEboe+2/+ANo7/To6Fie3eSsnXnUtOx57aGkDXCXFJybxjxWIX9qFFbcpH8pDqDElQ/nl0wea9FTr3oA2KgncbSrW2du4gYIx6KBM9sRLdJ51QcODOarJVmVnIp8LKJc18UXMb8+9zrIt0p0eZ2+bOqOxHLv69ehYvPpBy6Lc2c1zt3wYZTTaijJo5twgtetx5No3sfsrX2Bqx9moGs2XtRdjqA0CN0Q1TnLfH1nrYYhFksDdppxEmlhXP6WJUzRMZ7Vm6vABvM5e3HQPfna8OngrRay7f8XHC/Dsk6PsuXITlmWwrMUvUAJYlmJzX/6MxjE7066iFMbzTv9r1ropKkVm83aeeu+dDO+5Epzav0JTOwZ58VfeyXn/8o0zGmurcM9Jkr5+E3avh8765H4ySunpzFoPa8OSxck2Fu/ZjNdZfwfJDKMpT46iy0XcVHeUeSsFyiLWvQkvvfSmCYuRy/o89tAJnn9mfElTJwBaGxKT/8pXPv/Ckhcqtzz5GFa5ibZdtW4sxpAf2M7w5dfUDdoAWDZHrnszEzvPW/T4Wo17bpKu92zH2RxDOQq726PjV7cQv7xrrYe2YUngbnNBsdns0xCWC3gd3XTuvIiO7efTOXgh8e7ajXhXitaGkaECpVLYXJf2WYwxBIFh+ESOC+3Ps+/Ln+azX7uOkXt+b1EBvP+ZJ3BzzfXprMmyGpcUmKYdlxfe9h6GLr2KcE6WX06lOX7ldRy95g0UunuXNo7VoKDz5q0oe07pXc8i9brV+aQm5pOpkjZn2U7z9ZMMhOUCTjyJsupMF6ySfNYnnljcj58xMDZSZGBnc9115gpicZ57x/sZfuWrlz7wxdzklGJqxznsf9d2Dtx8B6/86/9K+vhhnnvnHZy45g0z35S68X2c8y9f5+yH/nnp41oh3q767eNUzELFLExJKnitNgncbS7W1UdQzDWXPSqF5axtwK4YOVmgpz++qGzfshSbBhIcPBBVr1tMb8uJbTv5xW/8QbTguFqfMJQC2ya0owNOe3/tt+h8+UVGLrvy9BiUwlgWL91wM/0HniY91FrFs7zBBjXatalqOixWj0yVtDknkSbeOzAzZ92IsuxlbUt2JkZHlraXz3ZU1X7wZjrLH37NDfzif/uj1Q3aNZQ6uqqD9izGcRi69Oo1GFVjYcaPWsHNYYwBW9H/f55P+u1bUAkJJatJMu51INbZi5fuJihkyQ8fqXud29GF0eGqbP9bSBgYSsVw0dMllqW44rUDvPjcOD19CVzPYnS4gLLgxEvzs++Ys48XN994ZgHbmPnPr7PjZIHBN36bhvVa1kbp6Qyp1/RVNbuY2V9vKfAU8Us6cLcnogbDS1w2EIvTej8pYkmUZWG5sYbBozw5SubI86tz4KYJB/dPENbI5hpRSuG6Fhft6WPLtiR9mxKcf1E3V79uK1e/6SwuetUAbsyeyb73bboR43o4xXE6jz6Md/I5qBxEyk9Ef2arFZC1Bh1GXzMG5fuVwZz+09zg619rDJuf+kVzr7OKdCZg8qvH0LkAXdLRqUqq99gr28JK2Xjnp9ZqmBtOw3RHKdUJbDLGvDjn8UuNMXtXdGRi0SzXQykLY+ocTTYGMORHjtKx4/xVHVstoyNFnv7lKc7Z1UW6TknXWmodzPFs2HlOJzvPgT1XbKJcCtDhI2TyRzn+g0c4tu95QqIDkC5gLIdKeqh7Bym94w8xfYOoUgk7DDDT9UaMZXHR1/6GLXsfZ3TXbqbOOoehy6+h2Ld57qCW/g9hDP3PPEFy5CSZgR0kxk/htMjNFcA/XGD0fx7E7veIX9oVNXKeQ7kWTn+M8vPSz3M11A3cSqn3Af8NGFZKucBHjDGPT3/5PuBVKz88sRhKKRKbdpAfPtxwsVL7ZXQYYLXAlMnkeJknHhvh8qs3LSp411J5rlIQT0Rn1If2/ZTjTz0//1p9+hi3deogiS/8OtYl7+CS54ZJjp9icvB8tOPS/fLz2H7UQb3/wFP0H3iKk5ct71y0lcvi5nP8+A//DIXBOC4qCOjfv4/zHvwqybEWONloolOWwfEiZo9GxaoLaRnfEIzV7zQvllej39xPAFcYY04opa4G/kYp9QljzD+wYP9tsVbcZJr09vMojJ4gLLRP9vP0k6Ncc93WZf/JevHJlxZuvzn9v+bpb3BwEjrGgbEXAKh1uN7d9wiF190EC/XsNBq0iRZFG17mc+JVrwbHnZkiNq7LyCWXM3ruLrb/p4/hTDbRAHoFKEvRs3sL6cFuShMFju59lt3X9+C61kx1QR1q/HyJAz/bF3XUESuuUeC2jTEnAIwxjyml3gh8Sym1A1mCaG3GEDY4mKMsi3JmAq+juyWybgC/pBkeytO/JYG9jIt0fmkRWaBS5Lvh62/pZrg/RiFW+6BN3P4hb+ONuDqJY3loo1FUsv3T1/umzGjpJbYkLpr1FtWvZ4zGdPTU/qRhWfjxON/9/Q/xxNjfN/99LJMEFn/adS5bbI+EsikZzcDbLgQ1HQCm74jD+PyBeZnx39m26mNcd27d39RljX5rM0qp8yrz29OZ9/XAPwK7z3iAYsWUM+MNp0qMDilNDFOePEVq27nY9Wp3rLLnnx3HGNiyrcHe4UXq3tTF2NDEwhdOU8Cv/nQi+n/xFPbbfhtnz5vnXaeNoRho/FDj2TaOglJgqKwu2Ao6Eyli9sXoBmnOQi3lbMvl0t4buW7ru5r+HpbLeZkfcVbup9jT31VMWTU/EfXbaT5+/p8w7i1PnfSN7FsMNHVdo5+a3wQspdQllQeMMRngrcDHzmh0YkXVK99afZHB6JDi6ImVH1CTjIbnnxnnwFNjBMHyHOzYfe3FS5x+MVDMEn7jv6Bf/Nm8r1pKkXRtuuIuac8h7jp0JVx6Ey49cYeuuItCNQzazbLWaO/5QPGZmaDdiI3PlsLTqzAiUVE3cBtjnjTGPA/8vVLq91UkAfw58FurNkKxaE6ys+ldDkEhu8KjWbyRkwWKhaBqq6AxBq2jP4upcdLV38kbbn0tXf2dSxtM6OM/9MVFPWW5a78knLXpqGNo7n2jvUqys3g1NfOvfQ1wFvAI8DhwHHjtSg5KnBk31VnVNLgxhalRU3otGQNPPj7C0Zcz5HM+uWyZF/dP8OTjw7x0YAK/rNFh7QA+9zFjDOnuNNe9+9Xs+sh1uDe/GTafB9Yi5vZPHVrS92Gpxsn+QuFdAR2ejb1Ai7GVcjRxGeGc2dRat0yDw1BCZk9XUzM/vT5Rr5EEEAdeMq32my6qRNsCt5M9+kITVxumDh/ASaRwkx24qc4F511Xgw4Nhw9mOHywuuZzdsrnxNEcvZvinL2rC8+zsW2F1lHbrZGTBTYNJLAsRRBoXjzcwzm9Q/wWh3nPZe/DLd9I7DIw+QnK//e7aWqdvZgl+NnXca68eVHfg1KKtGeTKdeebog7FoUGU0IGKAQaZ4m1y8/UkdSV9JYP0eMfRhmDURbGgEVIdFsxgOJw8komvR2rPr6NrJnA/TjwdeAqoA/4X0qpW40xt67oyMSZMSaqXdLMPVaHBLkpgnyG0sQp0tvOrd8arQUYA6PDRUaHi3T3xejqieGXQoaHChx9YQwdaiwFF7/3j8n85d0cvP8T3F7+TzBrg4lKdmNdcwv6p19t4g014T//BSrRib37jYs9T7HrAAAZFElEQVQaq2tbpFxDzq/+75ByLTzbohxqGu2gC7Sh4IckvdXf/WOUzRO976PTP0Fn+Tglu4NTsfNwdJGB4jPEwgzD8QuZ8rav+tg2umZ+Gu4yxlRWZ4aAm5VSH1zBMYllYLkxlFKLKzltDNovUZoaJd69acXGtpwmRktMjJ5uiqBDzf7gI/zuzT/kDv0Z7rn/E7jlG6ueEzz5XcIf3QdTI+DGwW/ilGJQIvjaZ1DxNNZ5Vy1qjDHHxrMt/OmVSndWBt0Zc8iWA/wG99diaIhps2ZTJlPuVqbcrTN/31bYx7m5nwCKnfmfMRy/kGe6bkSr1thauhEsmFbNCtqzH/ublRmOWA7GGMrZiRoLlM394pczzW+fa0W3fmAnT//PRwHmB+1ffIvw238OEydAB9NBu8mAqAP8L/8x+ugzix6TUgrPjrLsqjofSjW1a2SqFKCX2vhhGW0pPMN52R/jGB/HlLEJ2Vw8wEVT313roW0orft5WCxZaWKE4uiJedsCvc5eaOLAjQnKlDMTi+5Q0+qMMYQ//Cvw57YtW8T3GZQJH/rSsown0IaiH1Jq4rShAUrLtEXyTJyTexSb6p8rm4CBwtNYWo68rxb5bLPOGB1SmjxV8wCO0SHpgbPJDb2MCUMaBazC6HGCUp5kf/uchjt+cCzan24Moamxlz0oQf7MOsSDQY/W77JT8xkmmqeuBGjXik7CB4u8L4YtcCP1wvoNgl1TokRrHOZa7yTjXme0X6beR/+wmMf2YnScdQF2PNn4hYzBz05Mv15rGzo0wfGDo2AMu2/7JFP3/jrj93+Cj+++v/pCJwbxM2wkoRTWwK6mLzfGkCmFFEMzvd8ZynrxQRvAXsMmEBUlq6Pu7b5kSVnX1SKBe51Rjlv3uLs1fbRdKYXlLlAgKbqSoLS0TjWr5fjBMXQQ8pz/UW68aZA/0p8he9/D8+a2Ifq+7dd/OFqQXConFr1GkwJtCJYhU1ZAzFn7X1fXFGumBQozvU1QrAaZKllnLNvBSXYQ5DNUTYUoRWzWThEnlsTPTi7Qq9JQmjxFaWIYO55CKYWfm8LoECeeIt47gO3FVux7aWTo0AQ6DMHA7ts+yc577+ahaz/Bx2sE7Nnsq28BZRE+9MX5TRQWsu0i3Lf9NtaWc5t+SrAMZ94dS5Fy7TU7+j6bbfyajxsUti6j7WYSAnGm1v4WLpZdctN23HQXEHVcUbZDon87zqzpETfVhbIWONJsDLpcRPtl/Mw45amxaMHTGIJCluzxg+ig9i/ySlpMlj2XUgrn6nfj/faXYTGNky0Ha+cerO0XL2qsZ7qFz1bRlsG12go415g3iK6Rc5etJL61wPSbWDYSuNchZVkkN22nc/AiOnbsouOsC/DSXfOuSW87FyfVOb+lVrOZndGUZtWJNjpEN1Pgaokqc9n7/Y+w+7ZP8qtHP81D175+/lx2M3LjUUuyZukA88Jji34b11JLLjGugPQaHLxp5IWO6wmVRzhdx0SjCHF5tutta9qIeaNprZ8KsayUZTU8AWk5LqnNZ8383UwfwNFBEDUdbuLUZVgqoMOAwsgxgkIOMCjbwUt343X2Yi3UbGCR9od38bvv+j536MN8/L6HsY8fRheeRg2cj3JrT9uYqWGCH34B/eJPUbE01jXvwbrsLVEfycVI9y56vEopOmMOuXI4M9dtT8e3RrsAk65FbM6e71ZQcHp4tP9jDOYeo7t8hLzTy8upV5N1t6z10DYUCdxihlIK24uj7GCBue9Zz3E8cideRs/aG23CIJobnzxFvG8rsc7FB7yFpPIh5f91F2bsaFQwymjst3wce/cbCf71C+i93wNjsC54DfrAI1DMgQkx2THC7/0/mKH57cwacuPYr37fksZqW4rOuDNzgMZSioIfNqxTEoSGuNNaQbuiZHdyoPNX1noYG5oEbjFPtMCZJshnWehwSpBrvMBXHBvCSaSw52TDOvApjJ6YXkSNKDdGsn9b1Vx8LcYY3vqTUUx2ZPpTQXTTCL/93wh/+AXIj89k03rvP8//5OAX0U8+CJbVXNatbOw3fBT7gtcsfG0DlcVFYwwLTVmXtSFXDrCUQinwbKslFidFa5A5blFTctN2nESKaIGzdguvphgT7V6p/FVrCuPDZI4cqAraAMYvkTvxEmGNLYiVBclb3r+NX/7lT+jIh/MDcuhD9lR1MK433WN7NH3UPZbAfvV7mrt2AcYYsuVwXtGpWkqhoRBo8r5mohhQCmS7nYhI4G4zYbwTv2s7xlrZbVfKskkNDNJx1i7SW88hve28JQfvytH5cm6SqUPPUZ5o3LW8OOvrsxckKztIpv74DlzrDE/omRC151dgoZ01AGEAEyfP7P2mlUMzU2xqsXK+XpsyBMawI/czrhv5C64/+V+5dPyrJIOx1R+HmCFTJW1CO3HGX/1hSpvOR+kAsOh46pukX/zJir6v5bjguITz6ns0SSncVCd+Pkth+BjN1AUJy1G1vsoR9so+7Q8kBvn47vsxxQxl/ytLGw+A5aAGduG+8/cIerajf/SFxguxOoTkEjvozFE8w6zZ1wbPXt0pkwsz32Nbfi8O0dbPTaXn6S0f4tH+j1Gyl+ffRSyOZNxtohK0sV2Mm8C4MTJ73kFxYHH7ipdK2e7C+75rTD04iTQKyA8fptliTpbtzGSWe27/FNl77646wm7Gjtd8r+YorPOvxr39syhl4b7+g3j/19exrrgZvMT8yx0P64JrUfGOJb7faVENlTN9jTMexqK4Os/2/BMzQRvAwmAbn8Hc4rdHiuUhGXcbCGMdM0F7NuPEyF7wJuJDz67Ye5cy45TGhzFh0DBwu6ku7Fic4vhIVfYaFLJk8/ULE9USlotkDu/HonvmMZMv4P/r59D7fwwTQyyqot9s/Ttxb//PVQ+pRAfujb8DN/4OwSN/F9XqVhaEAdZ5V+Hc/PtLe685ljpFMpu7ytl2OjiFVg62qf6kYKHpLh9d1bGI0yRwtwEdT6N0iKlxnDhMdNV4xvIoTY1THDsxk+YZHUZFlmJJjF/G6ACUhe16+Pkp/Nzk/BdZSopoDMaExBlDlwtoBeYv/gaTyUcLkGdALbAX27n2duyr3oUZPYJK9y14/WpKuau/s6RgdWKZ+dM7GkXO6VvVsYjTZKqkDTiZ4dpfCANiJ/ev2PuWJobnB15jIPDp3HkBXWdfQrJ/azQnvQKf4TUOsRf/O996Vx9ki2cctIGmDtEoN441sGvZg7a7hGPrCkg4Fl0xh9gadHsvOt2MeTtnTkpWGGwOpa5Z9fGIiATuNqB0SOfer6OCWQuEYYAKSnQ89y8r8p7GmHmNGCp0cLrUa2lqbMUmXm3KHNi+k9fmLo5qaS+HiaHleZ0lUEqRdBf3K2eI9nCvVq2SDn+IPeNf49Wn/oqLJ/+JRDDG3u53czJ+MSE2Gpu83c0TPbeSdTevypjEfDJV0iZSLz2KnRsld8ENBMkuYicP0LH/+9jFGtMTy0BNF6eqFbyt2cWZliloG1Njt6EXx9n5a4Qj38I00/hYWdjv/H3MiQPon/3j/MM1ykL1nVX7uask7tgUfc1ietmY6W7qK6239BKXjz+AIsACUsEpBorP8njvh3i6+x08a96GZXwCFZe6JGtMAncbiQ8fID58YNXeL9azmeLoiergrBTx3tN1KdxUV8OpEjuWJNbVR2FsCFOnkqAxoHFxbIUJp7N5N4518eujpgVXvxu998EaLceqqa0X4Fz2FrjsLZSHD2KOPFU9veK42K9Z2rH1M2FMtHe70nrMtVVT7coqVquBwkVT361qS2ZhUMZnV+YH/LL3drRypCFwi5CpElFXrKOHeN9W1PSiqHJcEv3bcVOn9+56HT1YbmxedcFYz2Y6duwive0c3FQnHTvOr5mlRUHb5o1vvZzvX5FEnXsV1vlX49z8Bzg3/yEA1qazcW75FCS76jdBcONYl7/t9F9v+xOsC66NduLYLnRtwX3fn2Btbr6W9lIZEwXpTCkgWw7IlAOy5RBfTwfw0GCr0zm0pSDpqJrH4JPu6hSasnWZRDi/fIECun3ZPdJq5PYpGop19BDr6Kn79Up5WD83RVDIoGwXr6MH260+2aiURbx3K8XR4zOPVaZHuk8U+cnrrudt5T+r+z72hddi7foHzOgRzKnDBF//bDR14pfATaB2XIL9ytP1uFUshfve/4gpF6BcgFTPqgRAYwyZcrhgA4XQQGfMxplVvTHmGIqBxtdRLZO4Y1V9fSVp5WCwgflTY76qsb9drCkJ3OtEYdulZF7xdoJkL052hM593yR+8rlVeW+lFF66a17N77linT3YrktxYgS/UCS9fQ/xf3uCe39ncMHONRAdw1ebzoZNZ2MNXoZ++gfo7Dj24GWoc15VMzArL1H7YM0KKYem6a43xUCT9k4HZqUUCddmLcKkURbHE5eyrbC3arokwOVQ6uo1GJFoRKZK1oH8Wa9i/OpfI+gcAMcj6N7O+GvuXLVTlYvhJNKkt55DgS0MXPcbOEs8Aa6SXdhXvRv3jXdinXtFy9St9hfRnGE52potp/2dNzAcv4AQG1/FCLE5lrycI8kr13poYg7JuNeBqT3vnNeGyzgeU3vesaKnKpdq6NAEYKaPgK9cx5yVpI1BG4OlVNWhmMXcPlosbmOUw1PdN+OFOeLhJHmnh8CSaZJWJIG7zRlloeucngzS0T5bA5T7zmHq0nfh9+xA+QWcyWi3iDd5jMSRX6JjKezcGG5mearg1TO7cNTUvb/On/77waW1Hlsjxhjyfli1K8S1ohZjSilijk2pyfZtrfEZYb6ynaJsp9Z6GKIBCdztzmiscg4dS8/7kl0YJ7vrejIXvwXjnt57a2Jp/E3RLg9/8/nkdl0fHXBRFu7EUfoe/kssf35N7DDWQZjuw86MYJdzixpmpSv7fv+j/Lt3/YB9lcJRTcxtt5JioOdt5fM1ZMoBnTEXx1IkbEWhie1+MUdmKsXSSOBucwpIP/NdMntuwjizuswEJZyxI0ztfhs4NXoxVj7eq+ngMb3Nzu85i4kr30/vo5+fudQom/Gr7qC4/VJUGGBsh+TLP6Xrl19FNVHsKQramj23fYrBP/1YVJ71vodxyws+teUU67QbCzT4ocbAvKDtWtF/p/Ksp8ZsRUICt1giCdzrQOrFH4NSM5m1Vc6TfvrbZC59Z+2g3YjtUhy4BO3EsKaPmU9e+k6K2y6NSspO7+nOD16FnR+nY//3m3rZ/eZu9kz//3te+QFowaBtTLQdrxxqQBF3LFwrCtZlbTCmcU3Coh/i17jA15D2bJKWQpto37a0IRNnQgL3OqCA9AsPkXrhIYztocIyxksydfktS3tByyJI9eFNHidz3uvJn//6+YdnnBi5XW+oCtxhvIsg3YeTGcEuLa6U62qo1PiutQPFGEOmdLoTOxhyfoii+QKyjTbIlAKNF3MW7DUpRDMkcK8jClCVI+PlQhTAa5SCbeaVxl/9ETr3fp3Mnhvr1qXQboJS3zlkL7yBcv+50VRNUAbbIXHkF3T//MucfHkMHYYQltmZvY99S/7uli7Qhlw5JJwOyp6tSLl2VQD3tZkVtE9bzMYPW6mZTu5CrCSZZFunFIaOp78zv6qeMVHxJR3WLxClFDreydSem+pPtRiNnR1l7HW/SWnrboyXjPo3egmwXQo7XsmJLa9DB9MLkjf/gA8kjpC972HcVVyQ1MaQKQUzQRuiQzITxYDxgs9YwWeqdOaNeBU0rPwnC5FiOUnGvY6lX/wJKiiTveQthPFOnMwwnXu/QXx4P5N7biZ3wRuotynNKIWO1eknaAyEITrZjXHqNO11PLjiV9i9Yzc7772bh65tbgfJ7Ga4y3Gophjomlnz7MeWehCmMjrXViRdG0sp0h5ky9U3Ac9WS6rFLUQ9ErjXudShx0gdmt8b0C6MQxhCnUxQAd7oy5QGLprfCV1FFZKMaRyMVDzJH+nPNLWDpFKYqTAdaBWQcC3iS2weULkBhCt4ysWzFSnPmfOYRXdcUQ41xoBrWzgStMUyk89vG1Ti6BPUncENyrijh+h64qsQBrWnVJQFduP7vnImmz5cUwo0+VnZsQHyvma84JMpBU1nxcYYcuWA8WL0ZyWPldebabKUIu7YJFxbgrZYERK4Nyi7OEXPT7+ECkoovzgToFUpR/rAD+l7+HM4+TG69tZoSABg2Vjl2j0g9XSPwrTTXIcUbQz5OvujDdHC4VQpwA8XrgOSLVefalzJpcLVbtwrRIVMlWxgiRNPEfvmJylv3gWAN3wAa04gjh/fx2SNbYUqKJF+7nsUt72Ccs/ZWLaDNhpsn7iTJOHY2JaamQIphhptwLGi9l2OZRFoQ7YcNF2zI+9ruuz6uUY4Xe+6lsVs62uGIpoWEWItyE/eBmeFZeInniZ+4ul5QRvALmVJP/cv1f0ugzJ2bpTUwUfpf+gv4K//gHNOHuOs330HHV0/Ju05Mz0SC9NTIJV4GmjDVCnEDzVTpeaDNlC1M6SWRlvxLAUd3tLmyzs8u2px0bMUXXGnZSoSio1HMm6xoM5nv4s3fpjcedeh3SSJo0+QfOlRlJ4O9FOnSJZLBKXq+iaVk4i15Pylbb/LlAJijoVrqXmBs1FDXcdSOJZaUuZtW4qOmPyqiNYhP42iKfGhZ+eViJ1dOOrGGodrGmXTS10z9LXBL4c1D9FYSuHZinKNAk9xJ7o27dlky2HTwduzlRxPFy1HpkrEkhw/OIYOQna/75O8+einax6uaRTvGm226PBsYgss/JVDU3PqJOXaxB1rZo+1Yyk6Y6enblzbojvuREG/4TuAa0U3ByFajWTcYlEWU561UQaccm1yfjgv83YshWtbuLZF0hgKfrSwWYsfmnnb0JWKDsMkGwTcqG52tHBaazeLZymSni2ZtmhZErhFUyqlWStNEHbee3dT5VlTro1FSHE6eFsKkq6Na1t0WoqCH1IODUpBzLaIO9U9GC2LutWbzjSuxhwLTXWp1lpTMEK0GgncYkFzs+wP6M9wT5NNEJRSJD2HRI2j7JaKTh426rXi2RZ5v3bGfabb8SrZecKxCKXcqmgjErhFQ5VWY6+47T+w895fj7Ls3fcvup72UjNYSyk6aiwoppdxKkMphSPxWrQRCdxiQXtu/xTZe39jzfpDutP1PyrH150aWwGF2EgkcIu2oJSSI+ZCTJPtgKKuoUMTVI6raNNc53IhxMqTwC1qquzT3u9/lJ3Z+/jsGk2TCCHmk6kSUaWygwTDzLa/ZpsgCCFWh2TcYsbMtr/go9x40yB/pD+z6q3GhBALk8AtquwP7+IPb3mEp/7HT7jnlR9Y6+EIIWqQwC2EEG1GAreYER1p15TC4op2jhFCnBlZnBTzC0f9j58wfv8nZG5biBYlgXuDqxxp333bJ7kx9yU+kDjSVFd2IcTakamSDWro0ATHD46y3/8IN940SObeu3no2tfLXm0h2oAE7g2omSYIQojWJVMlG8himiAIIVqXZNwbiA41e277FL8qWbYQbU0C9wYlh2uEaF8SuIUQos3IHPcGMLtwlDZaDtcI0eYk417nZpdnlcJRQqwPknGvU1KeVYj1SzLudWx/eBc33XS2ZNlCrDMSuIUQos1I4BZCiDYjgXsdqixI3vL+bRjZQyLEuiOBex2RwlFCbAyyq2SdkPKsQmwcknG3Ocmyhdh4JHC3MSnPKsTGJFMlbWimPGvwUf7dzVKeVYiNRjLuNrU/vIs/vOURnpL+kEJsOBK4hRCizUjgbkM61ACUw5Ls0hZiA5I57jYyu3DULe/fxt57fyzTJEJsQJJxtwkpzyqEqJDA3QaGDk0Ahlfc9h9489FPyz5tITY4Cdxt4tIPf5azc1/iT//9oGTZQmxwEriFEKLNSOBucZUFSV0oSKU/IQQgu0pa2uzCUYO5L0aFo2RuW4gNTwJ3C5o50u5/lFs+sCMqHCVH2oUQ0yRwt5h55Vm1lGcVQlSTOe4WIeVZhRDNkoy7BUgTBCHEYkjGvcaGDk2wP7yTm246W7JsIURTJHC3EDlcI4RohgRuIYRoMzLHvYaOHxwDDGgth2uEEE2TwL0GZpdn3X3bJ9l57908dK3s0xZCNEemSlaZlGcVQpwpybhXiWTZQojlIhn3KpAsWwixnCTjXkGSZQshVoJk3Ctsf3gXt9x8Idl775b+kEKIZSGBWwgh2owEbiGEaDPKmOU/+KGUGgEOLfsLCyHE+jZojNm00EUrEriFEEKsHJkqEUKINiOBWwgh2owEbrHhKKUeVEpNKKW+tdZjEWIpJHCLjejPgA+u9SCEWCoJ3GLdUkpdpZTaq5SKK6VSSqmnlVKvMMZ8H8is9fiEWCo58i7WLWPM40qpbwB/AiSA/88Y89QaD0uIMyaBW6x3nwYeB4rA/7HGYxFiWchUiVjveoE00AHE13gsQiwLCdxivfsc8Engb4F713gsQiwLmSoR65ZS6kNAYIy5XyllA48opd4E/EfgIiCtlDoK3GWM+e5ajlWIxZAj70II0WZkqkQIIdqMBG4hhGgzEriFEKLNSOAWQog2I4FbCCHajARuIYRoMxK4hRCizUjgFkKINvP/A8UytnB439W9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize parameters randomly\n",
    "W = 0.01 * np.random.randn(n, num_classes)\n",
    "b = np.zeros((1,num_classes))\n",
    "\n",
    "# Gradient descent loop\n",
    "iters = 200\n",
    "step_size  = 1e-0\n",
    "lambda_val = 1e-3 # regularization strength\n",
    "\n",
    "for i in range(iters):\n",
    "\n",
    "    '''\n",
    "    FORWARD PROPAGATION\n",
    "    '''\n",
    "    \n",
    "    # Compute class probabilities\n",
    "    scores     = np.dot(X, W) + b     # shape (m, num_classes)\n",
    "    exp_scores = np.exp(scores)       # shape (m, num_classes)\n",
    "    probs      = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)  # shape (m, num_classes)\n",
    "\n",
    "    # Compute the loss function\n",
    "    logprob_correct_class = -np.log(probs[range(m),y])\n",
    "    data_loss = np.sum(logprob_correct_class)/m    # cross-entropy\n",
    "    reg_loss = 0.5 * lambda_val * np.sum(W*W)      # regularization\n",
    "    loss = data_loss + reg_loss                    # total          \n",
    "    \n",
    "    # Print diagnostic\n",
    "    if i % 10 == 0:\n",
    "        print (\"iteration %d: loss %f\" % (i, loss))\n",
    "\n",
    "        \n",
    "    '''\n",
    "    BACKWARD PROPAGATION\n",
    "    '''\n",
    "\n",
    "    # Compute gradient of cross-entropy wrt class scores\n",
    "    dscores = probs\n",
    "    dscores[range(m),y] -= 1\n",
    "    dscores /= m                 # shape (m, num_classes)\n",
    "\n",
    "    # Now backpropate get gradient of cross-entropy wrt parameters (W,b)\n",
    "    dW = np.dot(X.T, dscores)\n",
    "    db = np.sum(dscores, axis=0, keepdims=True)\n",
    "\n",
    "    # Add gradient of regularization term\n",
    "    dW += lambda_val * W \n",
    "\n",
    "\n",
    "    '''\n",
    "    UPDATE PARAMETERS\n",
    "    '''\n",
    "\n",
    "    W += -step_size * dW\n",
    "    b += -step_size * db\n",
    "\n",
    "# evaluate training set accuracy\n",
    "scores = np.dot(X, W) + b\n",
    "predicted_class = np.argmax(scores, axis=1)\n",
    "print ('training accuracy: %.2f' % (np.mean(predicted_class == y)))\n",
    "\n",
    "scores_test = np.dot(X_test, W) + b\n",
    "pred = np.argmax(scores_test, axis=1)\n",
    "plot_model(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}